entropymode.c 64.3 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4 5 6 7 8 9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10 11
 */

Yaowu Xu's avatar
Yaowu Xu committed
12
#include "aom_mem/aom_mem.h"
Jingning Han's avatar
Jingning Han committed
13

14
#include "av1/common/reconinter.h"
Angie Chiang's avatar
Angie Chiang committed
15
#include "av1/common/scan.h"
16 17
#include "av1/common/onyxc_int.h"
#include "av1/common/seg_common.h"
18
#include "av1/common/txb_common.h"
Jingning Han's avatar
Jingning Han committed
19

20
static const aom_cdf_prob default_newmv_cdf[NEWMV_MODE_CONTEXTS][CDF_SIZE(2)] =
21 22 23
    { { AOM_CDF2(128 * 155) }, { AOM_CDF2(128 * 116) }, { AOM_CDF2(128 * 94) },
      { AOM_CDF2(128 * 32) },  { AOM_CDF2(128 * 96) },  { AOM_CDF2(128 * 56) },
      { AOM_CDF2(128 * 30) } };
Sarah Parker's avatar
Sarah Parker committed
24
static const aom_cdf_prob default_zeromv_cdf[GLOBALMV_MODE_CONTEXTS][CDF_SIZE(
25
    2)] = { { AOM_CDF2(128 * 45) }, { AOM_CDF2(128 * 13) } };
26
static const aom_cdf_prob default_refmv_cdf[REFMV_MODE_CONTEXTS][CDF_SIZE(2)] =
27 28 29 30 31
    {
      { AOM_CDF2(128 * 178) }, { AOM_CDF2(128 * 212) }, { AOM_CDF2(128 * 135) },
      { AOM_CDF2(128 * 244) }, { AOM_CDF2(128 * 203) }, { AOM_CDF2(128 * 122) },
      { AOM_CDF2(128 * 128) }, { AOM_CDF2(128 * 128) }, { AOM_CDF2(128 * 128) }
    };
32
static const aom_cdf_prob default_drl_cdf[DRL_MODE_CONTEXTS][CDF_SIZE(2)] = {
33 34 35
  { AOM_CDF2(128 * 119) },
  { AOM_CDF2(128 * 189) },
  { AOM_CDF2(128 * 134) },
36
};
37

38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58
#if CONFIG_OPT_REF_MV
static const aom_cdf_prob
    default_inter_compound_mode_cdf[INTER_MODE_CONTEXTS][CDF_SIZE(
        INTER_COMPOUND_MODES)] = {
      { AOM_CDF8(8923, 11946, 15028, 16879, 18399, 19766, 27581) },
      { AOM_CDF8(8032, 15054, 17634, 19079, 20749, 22202, 28726) },
      { AOM_CDF8(4096, 8192, 12288, 16384, 20480, 24576, 28672) },
      { AOM_CDF8(4096, 8192, 12288, 16384, 20480, 24576, 28672) },
      { AOM_CDF8(4096, 8192, 12288, 16384, 20480, 24576, 28672) },
      { AOM_CDF8(4096, 8192, 12288, 16384, 20480, 24576, 28672) },
      { AOM_CDF8(6561, 13379, 15613, 17159, 19041, 20594, 26752) },
      { AOM_CDF8(13968, 15752, 20444, 23598, 24277, 24950, 25748) },
      { AOM_CDF8(13861, 18611, 21783, 23962, 24696, 25424, 30609) },
      { AOM_CDF8(4096, 8192, 12288, 16384, 20480, 24576, 28672) },
      { AOM_CDF8(4096, 8192, 12288, 16384, 20480, 24576, 28672) },
      { AOM_CDF8(4096, 8192, 12288, 16384, 20480, 24576, 28672) },
      { AOM_CDF8(7885, 12311, 15976, 19024, 20515, 21661, 23147) },
      { AOM_CDF8(11407, 16588, 19365, 21657, 22748, 23629, 28912) },
      { AOM_CDF8(10681, 18953, 20791, 22468, 23935, 25024, 28506) }
    };
#else
59 60 61
static const aom_cdf_prob
    default_inter_compound_mode_cdf[INTER_MODE_CONTEXTS][CDF_SIZE(
        INTER_COMPOUND_MODES)] = {
62 63 64 65 66 67 68
      { AOM_CDF8(19712, 28229, 30892, 31437, 31712, 32135, 32360) },
      { AOM_CDF8(9600, 24804, 29268, 30323, 30802, 31726, 32177) },
      { AOM_CDF8(896, 22434, 27015, 29026, 29753, 31114, 31597) },
      { AOM_CDF8(1024, 15904, 22127, 25421, 26864, 28996, 30001) },
      { AOM_CDF8(512, 11222, 17217, 21445, 23473, 26133, 27550) },
      { AOM_CDF8(2944, 13313, 17214, 20751, 23211, 25500, 26992) },
      { AOM_CDF8(3456, 9067, 14069, 16907, 18817, 21214, 23139) }
69
    };
70
#endif
71

72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91
#if CONFIG_JNT_COMP
static const aom_cdf_prob
    default_compound_type_cdf[BLOCK_SIZES_ALL][CDF_SIZE(COMPOUND_TYPES - 1)] = {
      { AOM_CDF2(16384) }, { AOM_CDF2(16384) }, { AOM_CDF2(16384) },
      { AOM_CDF2(16384) }, { AOM_CDF2(16384) }, { AOM_CDF2(16384) },
      { AOM_CDF2(16384) }, { AOM_CDF2(16384) }, { AOM_CDF2(16384) },
      { AOM_CDF2(16384) }, { AOM_CDF2(16384) }, { AOM_CDF2(16384) },
      { AOM_CDF2(16384) },
#if CONFIG_EXT_PARTITION
      { AOM_CDF2(16384) },  // 255, 1
      { AOM_CDF2(16384) }, { AOM_CDF2(16384) },
#endif  // CONFIG_EXT_PARTITION
      { AOM_CDF2(16384) }, { AOM_CDF2(16384) }, { AOM_CDF2(16384) },
      { AOM_CDF2(16384) }, { AOM_CDF2(16384) },  // 208, 1
      { AOM_CDF2(16384) },
#if CONFIG_EXT_PARTITION
      { AOM_CDF2(16384) }, { AOM_CDF2(16384) },
#endif
    };
#else
92
static const aom_cdf_prob
93
    default_compound_type_cdf[BLOCK_SIZES_ALL][CDF_SIZE(COMPOUND_TYPES)] = {
94 95 96 97 98 99 100
      { AOM_CDF3(16384, 24576) }, { AOM_CDF3(32640, 32704) },
      { AOM_CDF3(32640, 32704) }, { AOM_CDF3(8448, 13293) },
      { AOM_CDF3(9216, 12436) },  { AOM_CDF3(10112, 12679) },
      { AOM_CDF3(9088, 10753) },  { AOM_CDF3(10368, 12906) },
      { AOM_CDF3(10368, 12643) }, { AOM_CDF3(8832, 10609) },
      { AOM_CDF3(13312, 13388) }, { AOM_CDF3(12672, 12751) },
      { AOM_CDF3(9600, 9691) },
101
#if CONFIG_EXT_PARTITION
102 103
      { AOM_CDF3(32640, 32641) },  // 255, 1
      { AOM_CDF3(32640, 32641) }, { AOM_CDF3(32640, 32641) },
104
#endif  // CONFIG_EXT_PARTITION
105 106 107 108
      { AOM_CDF3(16384, 24576) }, { AOM_CDF3(16384, 24576) },
      { AOM_CDF3(16384, 24576) }, { AOM_CDF3(16384, 24576) },
      { AOM_CDF3(26624, 26648) },  // 208, 1
      { AOM_CDF3(26624, 26648) },
109
#if CONFIG_EXT_PARTITION
110
      { AOM_CDF3(26624, 26648) }, { AOM_CDF3(26624, 26648) },
111
#endif
112
    };
113
#endif  // CONFIG_JNT_COMP
114

115
static const aom_cdf_prob default_interintra_cdf[BLOCK_SIZE_GROUPS][CDF_SIZE(
116 117 118 119
    2)] = { { AOM_CDF2(128 * 128) },
            { AOM_CDF2(226 * 128) },
            { AOM_CDF2(244 * 128) },
            { AOM_CDF2(254 * 128) } };
120

121
static const aom_cdf_prob
122 123 124 125 126
    default_interintra_mode_cdf[BLOCK_SIZE_GROUPS][CDF_SIZE(INTERINTRA_MODES)] =
        { { AOM_CDF4(16384, 24576, 28672) },
          { AOM_CDF4(3072, 7016, 18987) },
          { AOM_CDF4(4864, 8461, 17481) },
          { AOM_CDF4(6528, 8681, 19031) } };
127

128
static const aom_cdf_prob
129
    default_wedge_interintra_cdf[BLOCK_SIZES_ALL][CDF_SIZE(2)] = {
130 131 132 133 134
      { AOM_CDF2(128 * 128) }, { AOM_CDF2(128 * 128) }, { AOM_CDF2(128 * 128) },
      { AOM_CDF2(194 * 128) }, { AOM_CDF2(213 * 128) }, { AOM_CDF2(217 * 128) },
      { AOM_CDF2(222 * 128) }, { AOM_CDF2(224 * 128) }, { AOM_CDF2(226 * 128) },
      { AOM_CDF2(220 * 128) }, { AOM_CDF2(128 * 128) }, { AOM_CDF2(128 * 128) },
      { AOM_CDF2(128 * 128) },
135
#if CONFIG_EXT_PARTITION
136
      { AOM_CDF2(255 * 128) }, { AOM_CDF2(255 * 128) }, { AOM_CDF2(255 * 128) },
137
#endif  // CONFIG_EXT_PARTITION
138 139
      { AOM_CDF2(208 * 128) }, { AOM_CDF2(208 * 128) }, { AOM_CDF2(208 * 128) },
      { AOM_CDF2(208 * 128) }, { AOM_CDF2(255 * 128) }, { AOM_CDF2(255 * 128) },
140
#if CONFIG_EXT_PARTITION
141
      { AOM_CDF2(255 * 128) }, { AOM_CDF2(255 * 128) },
142
#endif  // CONFIG_EXT_PARTITION
143 144
    };

Thomas Davies's avatar
Thomas Davies committed
145
static const aom_cdf_prob
146
    default_motion_mode_cdf[BLOCK_SIZES_ALL][CDF_SIZE(MOTION_MODES)] = {
147
      { AOM_CDF3(16384, 24576) }, { AOM_CDF3(16384, 24576) },
148 149 150 151 152 153
      { AOM_CDF3(16384, 24576) }, { AOM_CDF3(7936, 19091) },
      { AOM_CDF3(4991, 19205) },  { AOM_CDF3(4992, 19314) },
      { AOM_CDF3(15104, 21590) }, { AOM_CDF3(9855, 21043) },
      { AOM_CDF3(12800, 22238) }, { AOM_CDF3(24320, 26498) },
      { AOM_CDF3(26496, 28995) }, { AOM_CDF3(25216, 28166) },
      { AOM_CDF3(30592, 31238) },
Thomas Davies's avatar
Thomas Davies committed
154
#if CONFIG_EXT_PARTITION
155 156
      { AOM_CDF3(32256, 32656) }, { AOM_CDF3(32256, 32656) },
      { AOM_CDF3(32256, 32656) },
Thomas Davies's avatar
Thomas Davies committed
157
#endif
158 159 160
      { AOM_CDF3(32640, 32740) }, { AOM_CDF3(32640, 32740) },
      { AOM_CDF3(32640, 32740) }, { AOM_CDF3(32640, 32740) },
      { AOM_CDF3(32640, 32740) }, { AOM_CDF3(32640, 32740) },
161
#if CONFIG_EXT_PARTITION
162
      { AOM_CDF3(32256, 32656) }, { AOM_CDF3(32256, 32656) },
163
#endif
Thomas Davies's avatar
Thomas Davies committed
164
    };
Yue Chen's avatar
Yue Chen committed
165

166
static const aom_cdf_prob default_obmc_cdf[BLOCK_SIZES_ALL][CDF_SIZE(2)] = {
167 168 169 170 171
  { AOM_CDF2(128 * 128) }, { AOM_CDF2(128 * 128) }, { AOM_CDF2(128 * 128) },
  { AOM_CDF2(45 * 128) },  { AOM_CDF2(79 * 128) },  { AOM_CDF2(75 * 128) },
  { AOM_CDF2(130 * 128) }, { AOM_CDF2(141 * 128) }, { AOM_CDF2(144 * 128) },
  { AOM_CDF2(208 * 128) }, { AOM_CDF2(201 * 128) }, { AOM_CDF2(186 * 128) },
  { AOM_CDF2(231 * 128) },
172
#if CONFIG_EXT_PARTITION
173
  { AOM_CDF2(252 * 128) }, { AOM_CDF2(252 * 128) }, { AOM_CDF2(252 * 128) },
174
#endif  // CONFIG_EXT_PARTITION
175 176
  { AOM_CDF2(208 * 128) }, { AOM_CDF2(208 * 128) }, { AOM_CDF2(208 * 128) },
  { AOM_CDF2(208 * 128) }, { AOM_CDF2(208 * 128) }, { AOM_CDF2(208 * 128) },
177
#if CONFIG_EXT_PARTITION
178
  { AOM_CDF2(252 * 128) }, { AOM_CDF2(252 * 128) },
179
#endif  // CONFIG_EXT_PARTITION
180
};
181

Thomas Davies's avatar
Thomas Davies committed
182
static const aom_cdf_prob default_delta_q_cdf[CDF_SIZE(DELTA_Q_PROBS + 1)] = {
183
  AOM_CDF4(28160, 32120, 32677)
Thomas Davies's avatar
Thomas Davies committed
184
};
Fangwen Fu's avatar
Fangwen Fu committed
185
#if CONFIG_EXT_DELTA_Q
186
#if CONFIG_LOOPFILTER_LEVEL
187 188 189 190 191
static const aom_cdf_prob default_delta_lf_multi_cdf[FRAME_LF_COUNT][CDF_SIZE(
    DELTA_LF_PROBS + 1)] = { { AOM_CDF4(28160, 32120, 32677) },
                             { AOM_CDF4(28160, 32120, 32677) },
                             { AOM_CDF4(28160, 32120, 32677) },
                             { AOM_CDF4(28160, 32120, 32677) } };
192
#endif  // CONFIG_LOOPFILTER_LEVEL
Fangwen Fu's avatar
Fangwen Fu committed
193
static const aom_cdf_prob default_delta_lf_cdf[CDF_SIZE(DELTA_LF_PROBS + 1)] = {
194
  AOM_CDF4(28160, 32120, 32677)
Fangwen Fu's avatar
Fangwen Fu committed
195 196
};
#endif
197

198 199 200 201 202 203 204
static const aom_cdf_prob default_intra_inter_cdf[INTRA_INTER_CONTEXTS]
                                                 [CDF_SIZE(2)] = {
                                                   { AOM_CDF2(768) },
                                                   { AOM_CDF2(12416) },
                                                   { AOM_CDF2(19328) },
                                                   { AOM_CDF2(26240) }
                                                 };
205

206
static const aom_cdf_prob default_comp_inter_cdf[COMP_INTER_CONTEXTS][CDF_SIZE(
207 208 209 210 211
    2)] = { { AOM_CDF2(24290) },
            { AOM_CDF2(19956) },
            { AOM_CDF2(11641) },
            { AOM_CDF2(9804) },
            { AOM_CDF2(2842) } };
Jingning Han's avatar
Jingning Han committed
212

Zoe Liu's avatar
Zoe Liu committed
213
#if CONFIG_EXT_COMP_REFS
214 215 216 217 218 219 220 221
static const aom_cdf_prob default_comp_ref_type_cdf[COMP_REF_TYPE_CONTEXTS]
                                                   [CDF_SIZE(2)] = {
                                                     { AOM_CDF2(8 * 128) },
                                                     { AOM_CDF2(20 * 128) },
                                                     { AOM_CDF2(78 * 128) },
                                                     { AOM_CDF2(91 * 128) },
                                                     { AOM_CDF2(194 * 128) }
                                                   };
222 223
static const aom_cdf_prob
    default_uni_comp_ref_cdf[UNI_COMP_REF_CONTEXTS][UNIDIR_COMP_REFS - 1]
224 225 226 227 228 229 230 231 232
                            [CDF_SIZE(2)] = { { { AOM_CDF2(88 * 128) },
                                                { AOM_CDF2(30 * 128) },
                                                { AOM_CDF2(28 * 128) } },
                                              { { AOM_CDF2(218 * 128) },
                                                { AOM_CDF2(97 * 128) },
                                                { AOM_CDF2(105 * 128) } },
                                              { { AOM_CDF2(254 * 128) },
                                                { AOM_CDF2(180 * 128) },
                                                { AOM_CDF2(196 * 128) } } };
Zoe Liu's avatar
Zoe Liu committed
233 234
#endif  // CONFIG_EXT_COMP_REFS

235
static const aom_cdf_prob
236
    default_comp_ref_cdf[REF_CONTEXTS][FWD_REFS - 1][CDF_SIZE(2)] = {
237 238 239
      { { AOM_CDF2(4412) }, { AOM_CDF2(11499) }, { AOM_CDF2(478) } },
      { { AOM_CDF2(17926) }, { AOM_CDF2(26419) }, { AOM_CDF2(8615) } },
      { { AOM_CDF2(30449) }, { AOM_CDF2(31477) }, { AOM_CDF2(28035) } }
240
    };
241

242
static const aom_cdf_prob
243
    default_comp_bwdref_cdf[REF_CONTEXTS][BWD_REFS - 1][CDF_SIZE(2)] = {
244 245 246
      { { AOM_CDF2(2762) }, { AOM_CDF2(1614) } },
      { { AOM_CDF2(17976) }, { AOM_CDF2(15912) } },
      { { AOM_CDF2(30894) }, { AOM_CDF2(30639) } },
247
    };
248

249
// TODO(zoelu): To use aom_entropy_optimizer to update the following defaults.
250 251
static const aom_cdf_prob default_single_ref_cdf[REF_CONTEXTS][SINGLE_REFS - 1]
                                                [CDF_SIZE(2)] = {
252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269
                                                  { { AOM_CDF2(6500) },
                                                    { AOM_CDF2(3089) },
                                                    { AOM_CDF2(4026) },
                                                    { AOM_CDF2(8549) },
                                                    { AOM_CDF2(184) },
                                                    { AOM_CDF2(2264) } },
                                                  { { AOM_CDF2(17037) },
                                                    { AOM_CDF2(19408) },
                                                    { AOM_CDF2(15521) },
                                                    { AOM_CDF2(27640) },
                                                    { AOM_CDF2(5047) },
                                                    { AOM_CDF2(16251) } },
                                                  { { AOM_CDF2(28292) },
                                                    { AOM_CDF2(30427) },
                                                    { AOM_CDF2(29003) },
                                                    { AOM_CDF2(31436) },
                                                    { AOM_CDF2(28466) },
                                                    { AOM_CDF2(29371) } }
270
                                                };
Jingning Han's avatar
Jingning Han committed
271

272
// TODO(huisu): tune these cdfs
273
const aom_cdf_prob
Hui Su's avatar
Hui Su committed
274
    default_palette_y_size_cdf[PALATTE_BSIZE_CTXS][CDF_SIZE(PALETTE_SIZES)] = {
275 276
      { AOM_CDF7(12288, 19408, 24627, 26662, 28499, 30667) },
      { AOM_CDF7(12288, 19408, 24627, 26662, 28499, 30667) },
277 278 279 280 281 282 283
      { AOM_CDF7(12288, 19408, 24627, 26662, 28499, 30667) },
      { AOM_CDF7(2815, 4570, 9416, 10875, 13782, 19863) },
      { AOM_CDF7(12032, 14948, 22187, 23138, 24756, 27635) },
      { AOM_CDF7(14847, 20167, 25433, 26751, 28278, 30119) },
      { AOM_CDF7(18816, 25574, 29030, 29877, 30656, 31506) },
      { AOM_CDF7(23039, 27333, 30220, 30708, 31070, 31826) },
      { AOM_CDF7(12543, 20838, 27455, 28762, 29763, 31546) },
284
    };
clang-format's avatar
clang-format committed
285

Hui Su's avatar
Hui Su committed
286 287
const aom_cdf_prob
    default_palette_uv_size_cdf[PALATTE_BSIZE_CTXS][CDF_SIZE(PALETTE_SIZES)] = {
288 289
      { AOM_CDF7(20480, 29888, 32453, 32715, 32751, 32766) },
      { AOM_CDF7(20480, 29888, 32453, 32715, 32751, 32766) },
Hui Su's avatar
Hui Su committed
290 291 292 293 294 295 296 297
      { AOM_CDF7(20480, 29888, 32453, 32715, 32751, 32766) },
      { AOM_CDF7(11135, 23641, 31056, 31998, 32496, 32668) },
      { AOM_CDF7(9984, 21999, 29192, 30645, 31640, 32402) },
      { AOM_CDF7(7552, 16614, 24880, 27283, 29254, 31203) },
      { AOM_CDF7(11391, 18656, 23727, 26058, 27788, 30278) },
      { AOM_CDF7(8576, 13585, 17632, 20884, 23948, 27152) },
      { AOM_CDF7(9216, 14276, 19043, 22689, 25799, 28712) },
    };
298

299 300 301
// When palette mode is enabled, following probability tables indicate the
// probabilities to code the "is_palette" bit (i.e. the bit that indicates
// if this block uses palette mode or DC_PRED mode).
Hui Su's avatar
Hui Su committed
302
const aom_cdf_prob default_palette_y_mode_cdf[PALATTE_BSIZE_CTXS]
303 304 305 306 307 308 309 310 311 312 313 314 315 316 317
                                             [PALETTE_Y_MODE_CONTEXTS]
                                             [CDF_SIZE(2)] = {
                                               { { AOM_CDF2(128 * 240) },
                                                 { AOM_CDF2(128 * 180) },
                                                 { AOM_CDF2(128 * 100) } },
                                               { { AOM_CDF2(128 * 240) },
                                                 { AOM_CDF2(128 * 180) },
                                                 { AOM_CDF2(128 * 100) } },
                                               { { AOM_CDF2(128 * 240) },
                                                 { AOM_CDF2(128 * 180) },
                                                 { AOM_CDF2(128 * 100) } },
                                               { { AOM_CDF2(128 * 240) },
                                                 { AOM_CDF2(128 * 180) },
                                                 { AOM_CDF2(128 * 100) } },
                                               { { AOM_CDF2(128 * 240) },
318 319 320 321 322 323
                                                 { AOM_CDF2(128 * 180) },
                                                 { AOM_CDF2(128 * 100) } },
                                               { { AOM_CDF2(128 * 240) },
                                                 { AOM_CDF2(128 * 180) },
                                                 { AOM_CDF2(128 * 100) } },
                                               { { AOM_CDF2(128 * 240) },
324 325 326 327 328 329 330 331 332
                                                 { AOM_CDF2(128 * 180) },
                                                 { AOM_CDF2(128 * 100) } },
                                               { { AOM_CDF2(128 * 240) },
                                                 { AOM_CDF2(128 * 180) },
                                                 { AOM_CDF2(128 * 100) } },
                                               { { AOM_CDF2(128 * 240) },
                                                 { AOM_CDF2(128 * 180) },
                                                 { AOM_CDF2(128 * 100) } },
                                             };
333 334 335

const aom_cdf_prob
    default_palette_uv_mode_cdf[PALETTE_UV_MODE_CONTEXTS][CDF_SIZE(2)] = {
336
      { AOM_CDF2(128 * 253) }, { AOM_CDF2(128 * 229) }
337 338
    };

339 340 341
const aom_cdf_prob default_palette_y_color_index_cdf
    [PALETTE_SIZES][PALETTE_COLOR_INDEX_CONTEXTS][CDF_SIZE(PALETTE_COLORS)] = {
      {
342 343 344 345 346
          { AOM_CDF2(29568), 0, 0, 0, 0, 0, 0 },
          { AOM_CDF2(16384), 0, 0, 0, 0, 0, 0 },
          { AOM_CDF2(8832), 0, 0, 0, 0, 0, 0 },
          { AOM_CDF2(28672), 0, 0, 0, 0, 0, 0 },
          { AOM_CDF2(31872), 0, 0, 0, 0, 0, 0 },
347 348
      },
      {
349 350 351 352 353
          { AOM_CDF3(28032, 30326), 0, 0, 0, 0, 0 },
          { AOM_CDF3(11647, 27405), 0, 0, 0, 0, 0 },
          { AOM_CDF3(4352, 30659), 0, 0, 0, 0, 0 },
          { AOM_CDF3(23552, 27800), 0, 0, 0, 0, 0 },
          { AOM_CDF3(32256, 32504), 0, 0, 0, 0, 0 },
354 355
      },
      {
356 357 358 359 360
          { AOM_CDF4(26112, 28374, 30039), 0, 0, 0, 0 },
          { AOM_CDF4(9472, 22576, 27712), 0, 0, 0, 0 },
          { AOM_CDF4(6656, 26138, 29608), 0, 0, 0, 0 },
          { AOM_CDF4(19328, 23791, 28946), 0, 0, 0, 0 },
          { AOM_CDF4(31744, 31984, 32336), 0, 0, 0, 0 },
361 362
      },
      {
363 364 365 366 367
          { AOM_CDF5(27904, 29215, 30075, 31190), 0, 0, 0 },
          { AOM_CDF5(9728, 22598, 26134, 29425), 0, 0, 0 },
          { AOM_CDF5(2688, 30066, 31058, 31933), 0, 0, 0 },
          { AOM_CDF5(22015, 25039, 27726, 29932), 0, 0, 0 },
          { AOM_CDF5(32383, 32482, 32554, 32660), 0, 0, 0 },
368 369
      },
      {
370 371 372 373 374
          { AOM_CDF6(24319, 26299, 27486, 28600, 29804), 0, 0 },
          { AOM_CDF6(7935, 18217, 21116, 25440, 28589), 0, 0 },
          { AOM_CDF6(6656, 25016, 27105, 28698, 30399), 0, 0 },
          { AOM_CDF6(19967, 24117, 26550, 28566, 30224), 0, 0 },
          { AOM_CDF6(31359, 31607, 31775, 31977, 32258), 0, 0 },
375 376
      },
      {
377 378 379 380 381
          { AOM_CDF7(26368, 27768, 28588, 29274, 29997, 30917), 0 },
          { AOM_CDF7(8960, 18260, 20810, 23986, 26627, 28882), 0 },
          { AOM_CDF7(7295, 24111, 25836, 27515, 29033, 30769), 0 },
          { AOM_CDF7(22016, 25208, 27305, 28159, 29221, 30274), 0 },
          { AOM_CDF7(31744, 31932, 32050, 32199, 32335, 32521), 0 },
382 383
      },
      {
384 385 386 387 388
          { AOM_CDF8(26624, 27872, 28599, 29153, 29633, 30172, 30841) },
          { AOM_CDF8(6655, 17569, 19587, 23345, 25884, 28088, 29678) },
          { AOM_CDF8(3584, 27296, 28429, 29158, 30032, 30780, 31572) },
          { AOM_CDF8(23551, 25855, 27070, 27893, 28597, 29721, 30970) },
          { AOM_CDF8(32128, 32173, 32245, 32337, 32416, 32500, 32609) },
389 390
      },
    };
391

392 393 394
const aom_cdf_prob default_palette_uv_color_index_cdf
    [PALETTE_SIZES][PALETTE_COLOR_INDEX_CONTEXTS][CDF_SIZE(PALETTE_COLORS)] = {
      {
395 396 397 398 399
          { AOM_CDF2(29824), 0, 0, 0, 0, 0, 0 },
          { AOM_CDF2(16384), 0, 0, 0, 0, 0, 0 },
          { AOM_CDF2(8832), 0, 0, 0, 0, 0, 0 },
          { AOM_CDF2(30720), 0, 0, 0, 0, 0, 0 },
          { AOM_CDF2(31744), 0, 0, 0, 0, 0, 0 },
400 401
      },
      {
402 403 404 405 406
          { AOM_CDF3(27648, 30208), 0, 0, 0, 0, 0 },
          { AOM_CDF3(14080, 26563), 0, 0, 0, 0, 0 },
          { AOM_CDF3(5120, 30932), 0, 0, 0, 0, 0 },
          { AOM_CDF3(24448, 27828), 0, 0, 0, 0, 0 },
          { AOM_CDF3(31616, 32219), 0, 0, 0, 0, 0 },
407 408
      },
      {
409 410 411 412 413
          { AOM_CDF4(25856, 28259, 30584), 0, 0, 0, 0 },
          { AOM_CDF4(11520, 22476, 27944), 0, 0, 0, 0 },
          { AOM_CDF4(8064, 26882, 30308), 0, 0, 0, 0 },
          { AOM_CDF4(19455, 23823, 29134), 0, 0, 0, 0 },
          { AOM_CDF4(30848, 31501, 32174), 0, 0, 0, 0 },
414 415
      },
      {
416 417 418 419 420
          { AOM_CDF5(26751, 28020, 29541, 31230), 0, 0, 0 },
          { AOM_CDF5(12032, 26045, 30772, 31497), 0, 0, 0 },
          { AOM_CDF5(1280, 32153, 32458, 32560), 0, 0, 0 },
          { AOM_CDF5(23424, 24154, 29201, 29856), 0, 0, 0 },
          { AOM_CDF5(32256, 32402, 32561, 32682), 0, 0, 0 },
421 422
      },
      {
423 424 425 426 427
          { AOM_CDF6(24576, 26720, 28114, 28950, 31694), 0, 0 },
          { AOM_CDF6(7551, 16613, 20462, 25269, 29077), 0, 0 },
          { AOM_CDF6(6272, 23039, 25623, 28163, 30861), 0, 0 },
          { AOM_CDF6(17024, 18808, 20771, 27941, 29845), 0, 0 },
          { AOM_CDF6(31616, 31936, 32079, 32321, 32546), 0, 0 },
428 429
      },
      {
430 431 432 433 434
          { AOM_CDF7(23296, 25590, 27833, 29337, 29954, 31229), 0 },
          { AOM_CDF7(7552, 13659, 16570, 21695, 24506, 27701), 0 },
          { AOM_CDF7(6911, 24788, 26284, 27753, 29575, 30872), 0 },
          { AOM_CDF7(17535, 22236, 24457, 26242, 27363, 30191), 0 },
          { AOM_CDF7(30592, 31289, 31745, 31921, 32149, 32321), 0 },
435 436
      },
      {
437 438 439 440 441
          { AOM_CDF8(22016, 24242, 25141, 27137, 27797, 29331, 30848) },
          { AOM_CDF8(8063, 13564, 16940, 21948, 24568, 25689, 26989) },
          { AOM_CDF8(6528, 27028, 27835, 28741, 30031, 31795, 32285) },
          { AOM_CDF8(18047, 23797, 25444, 26274, 27111, 27929, 30367) },
          { AOM_CDF8(30208, 30628, 31046, 31658, 31762, 32367, 32469) },
442 443
      }
    };
hui su's avatar
hui su committed
444

Hui Su's avatar
Hui Su committed
445
#if CONFIG_INTRABC
446 447
static const aom_cdf_prob default_intrabc_cdf[CDF_SIZE(2)] = { AOM_CDF2(192 *
                                                                        128) };
Hui Su's avatar
Hui Su committed
448 449
#endif  // CONFIG_INTRABC

450 451
#define MAX_COLOR_CONTEXT_HASH 8
// Negative values are invalid
452 453 454
static const int palette_color_index_context_lookup[MAX_COLOR_CONTEXT_HASH +
                                                    1] = { -1, -1, 0, -1, -1,
                                                           4,  3,  2, 1 };
455

456
#if CONFIG_LOOP_RESTORATION
457 458
static const aom_cdf_prob default_switchable_restore_cdf[CDF_SIZE(
    RESTORE_SWITCHABLE_TYPES)] = { AOM_CDF3(32 * 128, 144 * 128) };
459

460 461
static const aom_cdf_prob default_wiener_restore_cdf[CDF_SIZE(2)] = { AOM_CDF2(
    64 * 128) };
462

463 464
static const aom_cdf_prob default_sgrproj_restore_cdf[CDF_SIZE(2)] = { AOM_CDF2(
    64 * 128) };
465 466
#endif  // CONFIG_LOOP_RESTORATION

467 468 469 470
#define NUM_PALETTE_NEIGHBORS 3  // left, top-left and top.
int av1_get_palette_color_index_context(const uint8_t *color_map, int stride,
                                        int r, int c, int palette_size,
                                        uint8_t *color_order, int *color_idx) {
471 472 473 474
  int i;
  // The +10 below should not be needed. But we get a warning "array subscript
  // is above array bounds [-Werror=array-bounds]" without it, possibly due to
  // this (or similar) bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=59124
hui su's avatar
hui su committed
475
  int scores[PALETTE_MAX_SIZE + 10];
476 477
  const int weights[NUM_PALETTE_NEIGHBORS] = { 2, 1, 2 };
  const int hash_multipliers[NUM_PALETTE_NEIGHBORS] = { 1, 2, 2 };
478 479
  int color_index_ctx_hash;
  int color_index_ctx;
480
  int color_neighbors[NUM_PALETTE_NEIGHBORS];
481
  int inverse_color_order[PALETTE_MAX_SIZE];
482
  assert(palette_size <= PALETTE_MAX_SIZE);
483
  assert(r > 0 || c > 0);
hui su's avatar
hui su committed
484

485
  // Get color indices of neighbors.
486
  color_neighbors[0] = (c - 1 >= 0) ? color_map[r * stride + c - 1] : -1;
487
  color_neighbors[1] =
488 489
      (c - 1 >= 0 && r - 1 >= 0) ? color_map[(r - 1) * stride + c - 1] : -1;
  color_neighbors[2] = (r - 1 >= 0) ? color_map[(r - 1) * stride + c] : -1;
hui su's avatar
hui su committed
490

491 492 493 494
  for (i = 0; i < PALETTE_MAX_SIZE; ++i) {
    color_order[i] = i;
    inverse_color_order[i] = i;
  }
hui su's avatar
hui su committed
495
  memset(scores, 0, PALETTE_MAX_SIZE * sizeof(scores[0]));
496
  for (i = 0; i < NUM_PALETTE_NEIGHBORS; ++i) {
497 498 499
    if (color_neighbors[i] >= 0) {
      scores[color_neighbors[i]] += weights[i];
    }
hui su's avatar
hui su committed
500 501
  }

502 503
  // Get the top NUM_PALETTE_NEIGHBORS scores (sorted from large to small).
  for (i = 0; i < NUM_PALETTE_NEIGHBORS; ++i) {
504 505 506
    int max = scores[i];
    int max_idx = i;
    int j;
507
    for (j = i + 1; j < palette_size; ++j) {
hui su's avatar
hui su committed
508 509 510 511 512 513
      if (scores[j] > max) {
        max = scores[j];
        max_idx = j;
      }
    }
    if (max_idx != i) {
514 515 516 517 518 519 520 521 522 523 524 525
      // Move the score at index 'max_idx' to index 'i', and shift the scores
      // from 'i' to 'max_idx - 1' by 1.
      const int max_score = scores[max_idx];
      const uint8_t max_color_order = color_order[max_idx];
      int k;
      for (k = max_idx; k > i; --k) {
        scores[k] = scores[k - 1];
        color_order[k] = color_order[k - 1];
        inverse_color_order[color_order[k]] = k;
      }
      scores[i] = max_score;
      color_order[i] = max_color_order;
526
      inverse_color_order[color_order[i]] = i;
hui su's avatar
hui su committed
527 528 529
    }
  }

530
  // Get hash value of context.
531
  color_index_ctx_hash = 0;
532
  for (i = 0; i < NUM_PALETTE_NEIGHBORS; ++i) {
533
    color_index_ctx_hash += scores[i] * hash_multipliers[i];
534
  }
535 536
  assert(color_index_ctx_hash > 0);
  assert(color_index_ctx_hash <= MAX_COLOR_CONTEXT_HASH);
hui su's avatar
hui su committed
537

538
  // Lookup context from hash.
539 540 541
  color_index_ctx = palette_color_index_context_lookup[color_index_ctx_hash];
  assert(color_index_ctx >= 0);
  assert(color_index_ctx < PALETTE_COLOR_INDEX_CONTEXTS);
hui su's avatar
hui su committed
542

543
  if (color_idx != NULL) {
544
    *color_idx = inverse_color_order[color_map[r * stride + c]];
545
  }
546
  return color_index_ctx;
hui su's avatar
hui su committed
547
}
548 549 550
#undef NUM_PALETTE_NEIGHBORS
#undef MAX_COLOR_CONTEXT_HASH

551 552
static const aom_cdf_prob
    default_txfm_partition_cdf[TXFM_PARTITION_CONTEXTS][CDF_SIZE(2)] = {
553 554 555 556 557 558 559 560
      { AOM_CDF2(249 * 128) }, { AOM_CDF2(240 * 128) }, { AOM_CDF2(223 * 128) },
      { AOM_CDF2(249 * 128) }, { AOM_CDF2(229 * 128) }, { AOM_CDF2(177 * 128) },
      { AOM_CDF2(250 * 128) }, { AOM_CDF2(243 * 128) }, { AOM_CDF2(208 * 128) },
      { AOM_CDF2(226 * 128) }, { AOM_CDF2(187 * 128) }, { AOM_CDF2(145 * 128) },
      { AOM_CDF2(236 * 128) }, { AOM_CDF2(204 * 128) }, { AOM_CDF2(150 * 128) },
      { AOM_CDF2(183 * 128) }, { AOM_CDF2(149 * 128) }, { AOM_CDF2(125 * 128) },
      { AOM_CDF2(181 * 128) }, { AOM_CDF2(146 * 128) }, { AOM_CDF2(113 * 128) },
      { AOM_CDF2(128 * 128) }
561
    };
562

Zoe Liu's avatar
Zoe Liu committed
563 564
#if CONFIG_EXT_SKIP
static const aom_cdf_prob default_skip_mode_cdfs[SKIP_MODE_CONTEXTS][CDF_SIZE(
Zoe Liu's avatar
Zoe Liu committed
565 566 567 568 569
    2)] = { { AOM_CDF2(31609) }, { AOM_CDF2(20107) }, { AOM_CDF2(10296) } };
static const aom_cdf_prob default_skip_cdfs[SKIP_CONTEXTS][CDF_SIZE(2)] = {
  { AOM_CDF2(30224) }, { AOM_CDF2(16244) }, { AOM_CDF2(4835) }
};
#else
570
static const aom_cdf_prob default_skip_cdfs[SKIP_CONTEXTS][CDF_SIZE(2)] = {
571
  { AOM_CDF2(24576) }, { AOM_CDF2(16384) }, { AOM_CDF2(8192) }
572
};
Zoe Liu's avatar
Zoe Liu committed
573
#endif  // CONFIG_EXT_SKIP
Jingning Han's avatar
Jingning Han committed
574

575
#if CONFIG_JNT_COMP
Cheng Chen's avatar
Cheng Chen committed
576 577 578 579 580 581 582 583 584
static const aom_cdf_prob
    default_compound_idx_cdfs[COMP_INDEX_CONTEXTS][CDF_SIZE(2)] = {
      { AOM_ICDF(24576), AOM_ICDF(32768), 0 },
      { AOM_ICDF(16384), AOM_ICDF(32768), 0 },
      { AOM_ICDF(8192), AOM_ICDF(32768), 0 },
      { AOM_ICDF(24576), AOM_ICDF(32768), 0 },
      { AOM_ICDF(16384), AOM_ICDF(32768), 0 },
      { AOM_ICDF(8192), AOM_ICDF(32768), 0 },
    };
585 586 587

static const aom_cdf_prob
    default_comp_group_idx_cdfs[COMP_GROUP_IDX_CONTEXTS][CDF_SIZE(2)] = {
588
      { AOM_ICDF(29491), AOM_ICDF(32768), 0 },
589 590
      { AOM_ICDF(24576), AOM_ICDF(32768), 0 },
      { AOM_ICDF(16384), AOM_ICDF(32768), 0 },
591 592 593 594
      { AOM_ICDF(24576), AOM_ICDF(32768), 0 },
      { AOM_ICDF(16384), AOM_ICDF(32768), 0 },
      { AOM_ICDF(13107), AOM_ICDF(32768), 0 },
      { AOM_ICDF(13107), AOM_ICDF(32768), 0 },
595
    };
596 597
#endif  // CONFIG_JNT_COMP

598
#if CONFIG_FILTER_INTRA
599 600
static const aom_cdf_prob default_filter_intra_mode_cdf[CDF_SIZE(
    FILTER_INTRA_MODES)] = { AOM_CDF5(14259, 17304, 20463, 29377) };
601 602

static const aom_cdf_prob default_filter_intra_cdfs[TX_SIZES_ALL][CDF_SIZE(2)] =
603 604
    {
      { AOM_CDF2(10985) }, { AOM_CDF2(10985) }, { AOM_CDF2(16645) },
Yaowu Xu's avatar
Yaowu Xu committed
605 606 607 608 609
      { AOM_CDF2(27378) }, { AOM_CDF2(30378) }, { AOM_CDF2(10985) },
      { AOM_CDF2(10985) }, { AOM_CDF2(15723) }, { AOM_CDF2(12373) },
      { AOM_CDF2(27199) }, { AOM_CDF2(24217) }, { AOM_CDF2(27378) },
      { AOM_CDF2(27378) }, { AOM_CDF2(16767) }, { AOM_CDF2(16767) },
      { AOM_CDF2(27767) }, { AOM_CDF2(27767) }, { AOM_CDF2(27378) },
610 611
      { AOM_CDF2(27378) },
    };
612 613
#endif  // CONFIG_FILTER_INTRA

Debargha Mukherjee's avatar
Debargha Mukherjee committed
614
// FIXME(someone) need real defaults here
615 616
static const aom_cdf_prob default_seg_tree_cdf[CDF_SIZE(MAX_SEGMENTS)] = {
  AOM_CDF8(4096, 8192, 12288, 16384, 20480, 24576, 28672)
Debargha Mukherjee's avatar
Debargha Mukherjee committed
617
};
618

619
static const aom_cdf_prob
620 621 622 623
    default_segment_pred_cdf[SEG_TEMPORAL_PRED_CTXS][CDF_SIZE(2)] = {
      { AOM_CDF2(128 * 128) }, { AOM_CDF2(128 * 128) }, { AOM_CDF2(128 * 128) }
    };

624
#if CONFIG_DUAL_FILTER
625 626 627
static const aom_cdf_prob
    default_switchable_interp_cdf[SWITCHABLE_FILTER_CONTEXTS][CDF_SIZE(
        SWITCHABLE_FILTERS)] = {
628 629 630 631 632 633 634 635
      { AOM_CDF3(32256, 32654) }, { AOM_CDF3(2816, 32651) },
      { AOM_CDF3(512, 764) },     { AOM_CDF3(30464, 31778) },
      { AOM_CDF3(32384, 32483) }, { AOM_CDF3(3072, 32652) },
      { AOM_CDF3(256, 383) },     { AOM_CDF3(25344, 26533) },
      { AOM_CDF3(32000, 32531) }, { AOM_CDF3(2048, 32648) },
      { AOM_CDF3(384, 890) },     { AOM_CDF3(28928, 31358) },
      { AOM_CDF3(31616, 31787) }, { AOM_CDF3(4224, 32433) },
      { AOM_CDF3(128, 256) },     { AOM_CDF3(17408, 18248) }
636 637
    };
#else   // CONFIG_DUAL_FILTER
638
static const aom_cdf_prob
639 640 641 642 643 644 645
    default_switchable_interp_cdf[SWITCHABLE_FILTER_CONTEXTS]
                                 [CDF_SIZE(SWITCHABLE_FILTERS)] = {
                                   { AOM_CDF3(30080, 31781) },
                                   { AOM_CDF3(4608, 32658) },
                                   { AOM_CDF3(4352, 4685) },
                                   { AOM_CDF3(19072, 26776) },
                                 };
646
#endif  // CONFIG_DUAL_FILTER
647

648
#if CONFIG_SPATIAL_SEGMENTATION
649
static const aom_cdf_prob
650 651
    default_spatial_pred_seg_tree_cdf[SPATIAL_PREDICTION_PROBS][CDF_SIZE(
        MAX_SEGMENTS)] = {
652 653 654 655 656 657 658 659 660 661 662 663
      {
          AOM_CDF8(5622, 7893, 16093, 18233, 27809, 28373, 32533),
      },
      {
          AOM_CDF8(14274, 18230, 22557, 24935, 29980, 30851, 32344),
      },
      {
          AOM_CDF8(27527, 28487, 28723, 28890, 32397, 32647, 32679),
      },
    };
#endif

664
static const aom_cdf_prob
665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683
    default_tx_size_cdf[MAX_TX_CATS][TX_SIZE_CONTEXTS][CDF_SIZE(MAX_TX_DEPTH +
                                                                1)] = {
#if MAX_TX_DEPTH == 2
      { { AOM_CDF2(19968) }, { AOM_CDF2(24320) } },
      { { AOM_CDF3(12272, 30172) }, { AOM_CDF3(18677, 30848) } },
      { { AOM_CDF3(12986, 15180) }, { AOM_CDF3(24302, 25602) } },
      { { AOM_CDF3(5782, 11475) }, { AOM_CDF3(16803, 22759) } },
#elif MAX_TX_DEPTH == 3
      { { AOM_CDF2(19968) }, { AOM_CDF2(24320) } },
      { { AOM_CDF3(12272, 30172) }, { AOM_CDF3(18677, 30848) } },
      { { AOM_CDF4(12986, 15180, 32384) }, { AOM_CDF4(24302, 25602, 32128) } },
      { { AOM_CDF4(5782, 11475, 24480) }, { AOM_CDF4(16803, 22759, 28560) } },
#else
      { { AOM_CDF2(19968) }, { AOM_CDF2(24320) } },
      { { AOM_CDF3(12272, 30172) }, { AOM_CDF3(18677, 30848) } },
      { { AOM_CDF4(12986, 15180, 32384) }, { AOM_CDF4(24302, 25602, 32128) } },
      { { AOM_CDF5(5782, 11475, 24480, 32640) },
        { AOM_CDF5(16803, 22759, 28560, 32640) } },
#endif  // MAX_TX_DEPTH == 2
684 685
    };

Urvang Joshi's avatar
Urvang Joshi committed
686 687
static const aom_cdf_prob
    default_if_y_mode_cdf[BLOCK_SIZE_GROUPS][CDF_SIZE(INTRA_MODES)] = {
688 689 690 691 692 693 694 695
      { AOM_CDF13(7168, 10680, 13913, 16928, 20294, 22790, 24706, 26275, 28139,
                  29751, 30563, 31468) },
      { AOM_CDF13(11776, 13823, 15307, 15725, 16638, 17406, 17994, 18814, 19634,
                  21513, 22198, 22928) },
      { AOM_CDF13(14720, 16459, 18091, 18299, 18757, 19125, 19423, 19924, 20504,
                  22922, 24063, 25577) },
      { AOM_CDF13(18944, 19925, 20908, 20998, 21017, 21072, 21084, 21121, 21159,
                  22064, 22820, 24290) },
Urvang Joshi's avatar
Urvang Joshi committed
696 697
    };

698
static const aom_cdf_prob
699 700 701
#if CONFIG_CFL
    default_uv_mode_cdf[CFL_ALLOWED_TYPES][INTRA_MODES][CDF_SIZE(
        UV_INTRA_MODES)] = {
702
#else
703 704 705
    default_uv_mode_cdf[INTRA_MODES][CDF_SIZE(UV_INTRA_MODES)] =
#endif  // CONFIG_CFL
      {
Yue Chen's avatar
Yue Chen committed
706
#if CONFIG_FILTER_INTRA
707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732
          { AOM_CDF13(17902, 18828, 21117, 21487, 21924, 22484, 23588, 24669,
                      25177, 28731, 29903, 31509) },
          { AOM_CDF13(9654, 23559, 23873, 24050, 24203, 24929, 25057, 25286,
                      26027, 28172, 28716, 30913) },
          { AOM_CDF13(10012, 10124, 25394, 25540, 25665, 25752, 26567, 27761,
                      27876, 29497, 30581, 31179) },
          { AOM_CDF13(15143, 15859, 16581, 21567, 21968, 22430, 22867, 24953,
                      26969, 30310, 31125, 32329) },
          { AOM_CDF13(14063, 14416, 14921, 15022, 25164, 26720, 28661, 29083,
                      29277, 31337, 31882, 32565) },
          { AOM_CDF13(12942, 14713, 15178, 15325, 16964, 27421, 27834, 28306,
                      28645, 30804, 31322, 32387) },
          { AOM_CDF13(13687, 13993, 16776, 16912, 18338, 18648, 27557, 28140,
                      28359, 30820, 31669, 32443) },
          { AOM_CDF13(14180, 14439, 16582, 17373, 17675, 17931, 18453, 26308,
                      26761, 30058, 31293, 32156) },
          { AOM_CDF13(12480, 14300, 14838, 16085, 16434, 17023, 17426, 18313,
                      26041, 29653, 30347, 32067) },
          { AOM_CDF13(17202, 18093, 19414, 19910, 20311, 20837, 21554, 22830,
                      23572, 28770, 30259, 32145) },
          { AOM_CDF13(16336, 18149, 19485, 19927, 20365, 20924, 21524, 22561,
                      23421, 28141, 30701, 32020) },
          { AOM_CDF13(16485, 17366, 19874, 20364, 20713, 21057, 21773, 23100,
                      23685, 28079, 29091, 32028) },
          { AOM_CDF13(13638, 16789, 19763, 19903, 19995, 20201, 20405, 20861,
                      21174, 22802, 23566, 24754) },
Yue Chen's avatar
Yue Chen committed
733
#else
734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759
          { AOM_CDF13(23552, 25936, 28623, 29033, 29395, 29892, 30252, 30905,
                      31370, 31980, 32293, 32660) },
          { AOM_CDF13(2944, 26431, 27553, 27746, 28022, 29080, 29204, 29377,
                      30264, 31206, 31613, 32418) },
          { AOM_CDF13(4352, 5120, 27952, 28117, 28473, 28759, 29563, 30864,
                      31051, 31694, 32073, 32435) },
          { AOM_CDF13(17664, 20288, 21839, 26072, 26420, 26972, 27240, 28565,
                      30914, 31694, 32083, 32591) },
          { AOM_CDF13(16640, 18390, 20233, 20557, 25162, 27789, 29397, 29895,
                      30369, 31497, 32025, 32642) },
          { AOM_CDF13(13952, 17947, 18918, 19206, 21131, 30668, 31061, 31317,
                      31838, 32137, 32342, 32547) },
          { AOM_CDF13(15872, 16990, 21479, 21732, 24134, 24854, 30296, 30887,
                      31163, 31902, 32218, 32702) },
          { AOM_CDF13(16256, 17280, 23081, 24039, 24457, 24838, 25346, 30329,
                      30908, 31746, 32206, 32639) },
          { AOM_CDF13(14720, 19249, 20501, 22079, 22439, 23218, 23463, 24107,
                      30308, 31379, 31866, 32556) },
          { AOM_CDF13(16768, 19967, 22374, 22976, 23836, 24050, 24642, 25760,
                      26653, 29585, 30937, 32518) },
          { AOM_CDF13(16768, 20751, 23026, 23591, 24299, 24516, 24981, 25876,
                      26806, 29520, 31286, 32455) },
          { AOM_CDF13(17536, 20055, 22965, 23507, 24210, 24398, 25098, 26366,
                      27033, 29674, 30689, 32530) },
          { AOM_CDF13(17536, 22753, 27126, 27353, 27571, 28139, 28505, 29198,
                      29886, 30801, 31335, 32054) },
Yue Chen's avatar
Yue Chen committed
760
#endif
761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790
#if CONFIG_CFL
      },
      {
          { AOM_CDF14(18377, 18815, 19743, 20178, 20560, 20889, 21359, 22098,
                      22481, 24563, 25781, 26662, 28396) },
          { AOM_CDF14(5350, 16837, 17066, 17360, 17692, 18778, 18969, 19206,
                      20291, 22367, 23212, 24670, 27912) },
          { AOM_CDF14(6671, 6759, 17812, 17998, 18260, 18384, 19408, 20667,
                      20806, 22760, 24142, 24875, 28072) },
          { AOM_CDF14(7461, 8082, 8515, 15013, 15583, 16098, 16522, 18519,
                      20348, 22954, 24130, 25342, 26548) },
          { AOM_CDF14(3694, 4403, 5370, 5854, 17841, 19639, 21625, 22224, 22651,
                      24613, 25399, 26143, 26599) },
          { AOM_CDF14(3700, 5651, 6112, 6541, 8929, 20623, 21213, 21640, 22214,
                      24306, 25412, 26406, 27249) },
          { AOM_CDF14(4649, 4947, 7128, 7432, 9439, 9903, 21163, 21774, 22056,
                      24426, 25403, 26324, 27128) },
          { AOM_CDF14(7208, 7375, 8779, 9683, 10072, 10284, 10796, 19786, 20152,
                      22955, 24246, 25165, 26589) },
          { AOM_CDF14(5897, 7283, 7555, 8910, 9391, 9937, 10276, 11044, 19841,
                      22620, 23784, 25060, 26418) },
          { AOM_CDF14(12171, 12718, 13885, 14348, 14925, 15394, 16108, 17075,
                      17583, 21996, 23614, 25048, 27011) },
          { AOM_CDF14(10192, 11222, 12318, 12877, 13533, 14184, 14866, 15879,
                      16650, 20419, 23265, 24295, 26596) },
          { AOM_CDF14(10776, 11387, 12899, 13471, 14088, 14575, 15366, 16456,
                      17040, 20815, 22009, 24448, 26492) },
          { AOM_CDF14(4015, 6473, 9853, 10285, 10655, 11032, 11431, 12199,
                      12738, 14760, 16121, 17263, 28612) },
      }
791
#endif  // CONFIG_CFL
792
    };
793

794
#if CONFIG_EXT_PARTITION_TYPES
795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816
static const aom_cdf_prob default_partition_cdf[PARTITION_CONTEXTS][CDF_SIZE(
    EXT_PARTITION_TYPES)] = {
  // 8x8 -> 4x4 only supports the four legacy partition types
  { AOM_CDF4(25472, 28949, 31052), 0, 0, 0, 0, 0, 0 },
  { AOM_CDF4(18816, 22250, 28783), 0, 0, 0, 0, 0, 0 },
  { AOM_CDF4(18944, 26126, 29188), 0, 0, 0, 0, 0, 0 },
  { AOM_CDF4(15488, 22508, 27077), 0, 0, 0, 0, 0, 0 },
  // 16x16 -> 8x8
  { AOM_CDF10(22272, 23768, 25043, 29996, 30495, 30994, 31419, 31844, 32343) },
  { AOM_CDF10(11776, 13457, 16315, 28229, 28789, 29349, 30302, 31255, 31816) },
  { AOM_CDF10(10496, 14802, 16136, 27127, 28563, 29999, 30444, 30889, 32324) },
  { AOM_CDF10(6784, 8763, 10440, 29110, 29770, 30430, 30989, 31548, 32208) },
  // 32x32 -> 16x16
  { AOM_CDF10(22656, 23801, 24702, 30721, 31103, 31485, 31785, 32085, 32467) },
  { AOM_CDF10(8704, 9926, 12586, 28885, 29292, 29699, 30586, 31473, 31881) },
  { AOM_CDF10(6656, 10685, 11566, 27857, 29200, 30543, 30837, 31131, 32474) },
  { AOM_CDF10(2176, 3012, 3690, 31253, 31532, 31811, 32037, 32263, 32542) },
  // 64x64 -> 32x32
  { AOM_CDF10(28416, 28705, 28926, 32258, 32354, 32450, 32523, 32596, 32693) },
  { AOM_CDF10(9216, 9952, 11849, 30134, 30379, 30624, 31256, 31888, 32134) },
  { AOM_CDF10(7424, 9008, 9528, 30664, 31192, 31720, 31893, 32066, 32594) },
  { AOM_CDF10(1280, 1710, 2069, 31978, 32121, 32264, 32383, 32502, 32647) },
817
#if CONFIG_EXT_PARTITION
818
#if ALLOW_128X32_BLOCKS
819 820 821 822
  { AOM_CDF10(28416, 28705, 28926, 32258, 32354, 32450, 32523, 32596, 32693) },
  { AOM_CDF10(9216, 9952, 11849, 30134, 30379, 30624, 31256, 31888, 32134) },
  { AOM_CDF10(7424, 9008, 9528, 30664, 31192, 31720, 31893, 32066, 32594) },
  { AOM_CDF10(1280, 1710, 2069, 31978, 32121, 32264, 32383, 32502, 32647) },
823 824 825 826 827 828 829 830
#else
  // 128x128 -> 64x64
  { AOM_CDF8(28416, 28705, 28926, 32258, 32402, 32547, 32548) },
  { AOM_CDF8(9216, 9952, 11849, 30134, 30502, 30870, 30871) },
  { AOM_CDF8(7424, 9008, 9528, 30664, 31456, 32248, 32249) },
  { AOM_CDF8(1280, 1710, 2069, 31978, 32193, 32409, 32410) },
#endif  // ALLOW_128X32_BLOCKS
#endif  // CONFIG_EXT_PARTITION
831
};
832
#else
833 834
static const aom_cdf_prob
    default_partition_cdf[PARTITION_CONTEXTS][CDF_SIZE(PARTITION_TYPES)] = {
835 836 837 838 839 840 841 842
      { AOM_CDF4(25472, 28949, 31052) }, { AOM_CDF4(18816, 22250, 28783) },
      { AOM_CDF4(18944, 26126, 29188) }, { AOM_CDF4(15488, 22508, 27077) },
      { AOM_CDF4(22272, 25265, 27815) }, { AOM_CDF4(11776, 15138, 20854) },
      { AOM_CDF4(10496, 19109, 21777) }, { AOM_CDF4(6784, 10743, 14098) },
      { AOM_CDF4(22656, 24947, 26749) }, { AOM_CDF4(8704, 11148, 16469) },
      { AOM_CDF4(6656, 14714, 16477) },  { AOM_CDF4(2176, 3849, 5205) },
      { AOM_CDF4(28416, 28994, 29436) }, { AOM_CDF4(9216, 10688, 14483) },
      { AOM_CDF4(7424, 10592, 11632) },  { AOM_CDF4(1280, 2141, 2859) },
843
#if CONFIG_EXT_PARTITION
844 845
      { AOM_CDF4(28416, 28994, 29436) }, { AOM_CDF4(9216, 10688, 14483) },
      { AOM_CDF4(7424, 10592, 11632) },  { AOM_CDF4(1280, 2141, 2859) },
846
#endif
847
    };
848
#endif
849

850 851 852
static const aom_cdf_prob default_intra_ext_tx_cdf
    [EXT_TX_SETS_INTRA][EXT_TX_SIZES][INTRA_MODES][CDF_SIZE(TX_TYPES)] = {
      {
853
          // FIXME: unused zero positions, from uncoded trivial transform set
854 855 856 857 858 859 860 861 862 863 864 865 866 867
          {
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
868
          },
869 870 871 872 873 874 875 876 877 878 879 880 881 882
          {
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
883
          },
884 885 886 887 888 889 890 891 892 893 894 895 896 897
          {
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
898
          },
899 900 901 902 903 904 905 906 907 908 909 910 911 912
          {
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
              { 0 },
913 914 915
          },
      },
      {
916
          {
917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974
              { AOM_CDF7(1024, 28800, 29048, 29296, 30164, 31466) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 27118) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
              { AOM_CDF7(1152, 25852, 26284, 26717, 28230, 30499) },
              { AOM_CDF7(1024, 2016, 3938, 5860, 29404, 31086) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 27118) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
              { AOM_CDF7(1280, 4109, 5900, 7691, 15528, 27380) },
              { AOM_CDF7(1280, 4109, 5900, 7691, 15528, 27380) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
          },
          {
              { AOM_CDF7(1024, 28800, 29048, 29296, 30164, 31466) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 27118) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
              { AOM_CDF7(1152, 25852, 26284, 26717, 28230, 30499) },
              { AOM_CDF7(1024, 2016, 3938, 5860, 29404, 31086) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 27118) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
              { AOM_CDF7(1280, 4109, 5900, 7691, 15528, 27380) },
              { AOM_CDF7(1280, 4109, 5900, 7691, 15528, 27380) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
          },
          {
              { AOM_CDF7(1024, 28800, 29048, 29296, 30164, 31466) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 27118) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
              { AOM_CDF7(1152, 25852, 26284, 26717, 28230, 30499) },
              { AOM_CDF7(1024, 2016, 3938, 5860, 29404, 31086) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 27118) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
              { AOM_CDF7(1280, 4109, 5900, 7691, 15528, 27380) },
              { AOM_CDF7(1280, 4109, 5900, 7691, 15528, 27380) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
          },
          {
              { AOM_CDF7(1024, 28800, 29048, 29296, 30164, 31466) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 27118) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
              { AOM_CDF7(1152, 25852, 26284, 26717, 28230, 30499) },
              { AOM_CDF7(1024, 2016, 3938, 5860, 29404, 31086) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 27118) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
              { AOM_CDF7(1280, 4109, 5900, 7691, 15528, 27380) },
              { AOM_CDF7(1280, 4109, 5900, 7691, 15528, 27380) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
              { AOM_CDF7(1280, 5216, 6938, 8660, 10167, 15817) },
              { AOM_CDF7(1280