entropymode.h 26 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4 5 6 7 8 9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10 11
 */

Yaowu Xu's avatar
Yaowu Xu committed
12 13
#ifndef AV1_COMMON_ENTROPYMODE_H_
#define AV1_COMMON_ENTROPYMODE_H_
Jingning Han's avatar
Jingning Han committed
14

15 16 17 18
#include "av1/common/entropy.h"
#include "av1/common/entropymv.h"
#include "av1/common/filter.h"
#include "av1/common/seg_common.h"
Yaowu Xu's avatar
Yaowu Xu committed
19
#include "aom_dsp/aom_filter.h"
Jingning Han's avatar
Jingning Han committed
20

21 22 23 24 25 26
#if CONFIG_PVQ
#include "av1/common/pvq.h"
#include "av1/common/pvq_state.h"
#include "av1/common/generic_code.h"
#endif  // CONFIG_PVQ

Jingning Han's avatar
Jingning Han committed
27 28 29 30 31 32 33 34
#ifdef __cplusplus
extern "C" {
#endif

#define BLOCK_SIZE_GROUPS 4

#define TX_SIZE_CONTEXTS 2

clang-format's avatar
clang-format committed
35
#define INTER_OFFSET(mode) ((mode)-NEARESTMV)
36
#if CONFIG_EXT_INTER
37 38 39
#if CONFIG_COMPOUND_SINGLEREF
#define INTER_SINGLEREF_COMP_OFFSET(mode) ((mode)-SR_NEAREST_NEARMV)
#endif  // CONFIG_COMPOUND_SINGLEREF
clang-format's avatar
clang-format committed
40
#define INTER_COMPOUND_OFFSET(mode) ((mode)-NEAREST_NEARESTMV)
41
#endif  // CONFIG_EXT_INTER
Jingning Han's avatar
Jingning Han committed
42

43 44 45 46 47 48 49
// Number of possible contexts for a color index.
// As can be seen from av1_get_palette_color_index_context(), the possible
// contexts are (2,0,0), (2,2,1), (3,2,0), (4,1,0), (5,0,0). These are mapped to
// a value from 0 to 4 using 'palette_color_index_context_lookup' table.
#define PALETTE_COLOR_INDEX_CONTEXTS 5

// Maximum number of colors in a palette.
hui su's avatar
hui su committed
50
#define PALETTE_MAX_SIZE 8
51 52
// Minimum number of colors in a palette.
#define PALETTE_MIN_SIZE 2
53 54

// Palette mode is available for block sizes >= 8x8.
55
#define PALETTE_BLOCK_SIZES (BLOCK_LARGEST - BLOCK_8X8 + 1)
56 57 58 59 60 61 62

// Palette Y mode context for a block is determined by number of neighboring
// blocks (top and/or left) using a palette for Y plane. So, possible Y mode'
// context values are:
// 0 if neither left nor top block uses palette for Y plane,
// 1 if exactly one of left or top block uses palette for Y plane, and
// 2 if both left and top blocks use palette for Y plane.
hui su's avatar
hui su committed
63
#define PALETTE_Y_MODE_CONTEXTS 3
64 65 66 67 68 69 70

// Palette UV mode context for a block is determined by whether this block uses
// palette for the Y plane. So, possible values are:
// 0 if this block doesn't use palette for Y plane.
// 1 if this block uses palette for Y plane (i.e. Y palette size > 0).
#define PALETTE_UV_MODE_CONTEXTS 2

71
#define PALETTE_MAX_BLOCK_SIZE (64 * 64)
hui su's avatar
hui su committed
72

Yaowu Xu's avatar
Yaowu Xu committed
73
struct AV1Common;
Jingning Han's avatar
Jingning Han committed
74

75 76 77 78 79 80
typedef struct {
  const int16_t *scan;
  const int16_t *iscan;
  const int16_t *neighbors;
} SCAN_ORDER;

81 82 83 84 85 86
struct seg_counts {
  unsigned int tree_total[MAX_SEGMENTS];
  unsigned int tree_mispred[MAX_SEGMENTS];
  unsigned int pred[PREDICTION_PROBS][2];
};

Jingning Han's avatar
Jingning Han committed
87
typedef struct frame_contexts {
Yaowu Xu's avatar
Yaowu Xu committed
88
  aom_prob y_mode_prob[BLOCK_SIZE_GROUPS][INTRA_MODES - 1];
Luc Trudeau's avatar
Luc Trudeau committed
89
  aom_prob uv_mode_prob[INTRA_MODES][UV_INTRA_MODES - 1];
90
#if CONFIG_EXT_PARTITION_TYPES
Yaowu Xu's avatar
Yaowu Xu committed
91
  aom_prob partition_prob[PARTITION_CONTEXTS][EXT_PARTITION_TYPES - 1];
92
#else
Yaowu Xu's avatar
Yaowu Xu committed
93
  aom_prob partition_prob[PARTITION_CONTEXTS][PARTITION_TYPES - 1];
94
#endif
95 96
  coeff_cdf_model coef_tail_cdfs[TX_SIZES][PLANE_TYPES];
  coeff_cdf_model coef_head_cdfs[TX_SIZES][PLANE_TYPES];
clang-format's avatar
clang-format committed
97 98
  aom_prob switchable_interp_prob[SWITCHABLE_FILTER_CONTEXTS]
                                 [SWITCHABLE_FILTERS - 1];
99
#if CONFIG_ADAPT_SCAN
100
// TODO(angiebird): try aom_prob
101
#if CONFIG_CHROMA_2X2
102 103
  uint32_t non_zero_prob_2x2[TX_TYPES][4];
#endif
104 105 106 107 108
  uint32_t non_zero_prob_4X4[TX_TYPES][16];
  uint32_t non_zero_prob_8X8[TX_TYPES][64];
  uint32_t non_zero_prob_16X16[TX_TYPES][256];
  uint32_t non_zero_prob_32X32[TX_TYPES][1024];

109 110 111 112 113 114 115
  uint32_t non_zero_prob_4X8[TX_TYPES][32];
  uint32_t non_zero_prob_8X4[TX_TYPES][32];
  uint32_t non_zero_prob_16X8[TX_TYPES][128];
  uint32_t non_zero_prob_8X16[TX_TYPES][128];
  uint32_t non_zero_prob_32X16[TX_TYPES][512];
  uint32_t non_zero_prob_16X32[TX_TYPES][512];

116
#if CONFIG_CHROMA_2X2
117 118
  DECLARE_ALIGNED(16, int16_t, scan_2x2[TX_TYPES][4]);
#endif
119 120 121 122 123
  DECLARE_ALIGNED(16, int16_t, scan_4X4[TX_TYPES][16]);
  DECLARE_ALIGNED(16, int16_t, scan_8X8[TX_TYPES][64]);
  DECLARE_ALIGNED(16, int16_t, scan_16X16[TX_TYPES][256]);
  DECLARE_ALIGNED(16, int16_t, scan_32X32[TX_TYPES][1024]);

124 125 126 127 128 129 130
  DECLARE_ALIGNED(16, int16_t, scan_4X8[TX_TYPES][32]);
  DECLARE_ALIGNED(16, int16_t, scan_8X4[TX_TYPES][32]);
  DECLARE_ALIGNED(16, int16_t, scan_8X16[TX_TYPES][128]);
  DECLARE_ALIGNED(16, int16_t, scan_16X8[TX_TYPES][128]);
  DECLARE_ALIGNED(16, int16_t, scan_16X32[TX_TYPES][512]);
  DECLARE_ALIGNED(16, int16_t, scan_32X16[TX_TYPES][512]);

131
#if CONFIG_CHROMA_2X2
132 133
  DECLARE_ALIGNED(16, int16_t, iscan_2x2[TX_TYPES][4]);
#endif
134 135 136 137 138
  DECLARE_ALIGNED(16, int16_t, iscan_4X4[TX_TYPES][16]);
  DECLARE_ALIGNED(16, int16_t, iscan_8X8[TX_TYPES][64]);
  DECLARE_ALIGNED(16, int16_t, iscan_16X16[TX_TYPES][256]);
  DECLARE_ALIGNED(16, int16_t, iscan_32X32[TX_TYPES][1024]);

139 140 141 142 143 144 145
  DECLARE_ALIGNED(16, int16_t, iscan_4X8[TX_TYPES][32]);
  DECLARE_ALIGNED(16, int16_t, iscan_8X4[TX_TYPES][32]);
  DECLARE_ALIGNED(16, int16_t, iscan_8X16[TX_TYPES][128]);
  DECLARE_ALIGNED(16, int16_t, iscan_16X8[TX_TYPES][128]);
  DECLARE_ALIGNED(16, int16_t, iscan_16X32[TX_TYPES][512]);
  DECLARE_ALIGNED(16, int16_t, iscan_32X16[TX_TYPES][512]);

146
#if CONFIG_CHROMA_2X2
147 148
  int16_t nb_2x2[TX_TYPES][(4 + 1) * 2];
#endif
149 150 151 152 153
  int16_t nb_4X4[TX_TYPES][(16 + 1) * 2];
  int16_t nb_8X8[TX_TYPES][(64 + 1) * 2];
  int16_t nb_16X16[TX_TYPES][(256 + 1) * 2];
  int16_t nb_32X32[TX_TYPES][(1024 + 1) * 2];

154 155 156 157 158 159 160 161
  int16_t nb_4X8[TX_TYPES][(32 + 1) * 2];
  int16_t nb_8X4[TX_TYPES][(32 + 1) * 2];
  int16_t nb_8X16[TX_TYPES][(128 + 1) * 2];
  int16_t nb_16X8[TX_TYPES][(128 + 1) * 2];
  int16_t nb_16X32[TX_TYPES][(512 + 1) * 2];
  int16_t nb_32X16[TX_TYPES][(512 + 1) * 2];

  SCAN_ORDER sc[TX_SIZES_ALL][TX_TYPES];
162

Yi Luo's avatar
Yi Luo committed
163
  int16_t eob_threshold[TX_SIZES_ALL][TX_TYPES][EOB_THRESHOLD_NUM];
164 165
#endif  // CONFIG_ADAPT_SCAN

166 167 168 169 170 171 172 173
#if CONFIG_LV_MAP
  aom_prob txb_skip[TX_SIZES][TXB_SKIP_CONTEXTS];
  aom_prob nz_map[TX_SIZES][PLANE_TYPES][SIG_COEF_CONTEXTS];
  aom_prob eob_flag[TX_SIZES][PLANE_TYPES][EOB_COEF_CONTEXTS];
  aom_prob dc_sign[PLANE_TYPES][DC_SIGN_CONTEXTS];
  aom_prob coeff_base[TX_SIZES][PLANE_TYPES][NUM_BASE_LEVELS]
                     [COEFF_BASE_CONTEXTS];
  aom_prob coeff_lps[TX_SIZES][PLANE_TYPES][LEVEL_CONTEXTS];
174 175 176
#if BR_NODE
  aom_prob coeff_br[TX_SIZES][PLANE_TYPES][BASE_RANGE_SETS][LEVEL_CONTEXTS];
#endif
177 178 179 180 181 182 183 184 185 186 187 188

#if LV_MAP_PROB
  aom_cdf_prob txb_skip_cdf[TX_SIZES][TXB_SKIP_CONTEXTS][CDF_SIZE(2)];
  aom_cdf_prob nz_map_cdf[TX_SIZES][PLANE_TYPES][SIG_COEF_CONTEXTS]
                         [CDF_SIZE(2)];
  aom_cdf_prob eob_flag_cdf[TX_SIZES][PLANE_TYPES][EOB_COEF_CONTEXTS]
                           [CDF_SIZE(2)];
  aom_cdf_prob dc_sign_cdf[PLANE_TYPES][DC_SIGN_CONTEXTS][CDF_SIZE(2)];
  aom_cdf_prob coeff_base_cdf[TX_SIZES][PLANE_TYPES][NUM_BASE_LEVELS]
                             [COEFF_BASE_CONTEXTS][CDF_SIZE(2)];
  aom_cdf_prob coeff_lps_cdf[TX_SIZES][PLANE_TYPES][LEVEL_CONTEXTS]
                            [CDF_SIZE(2)];
189 190 191 192
#if BR_NODE
  aom_cdf_prob coeff_br_cdf[TX_SIZES][PLANE_TYPES][BASE_RANGE_SETS]
                           [LEVEL_CONTEXTS][CDF_SIZE(2)];
#endif
193
#endif  // LV_MAP_PROB
194 195
#endif

Yaowu Xu's avatar
Yaowu Xu committed
196 197 198 199
  aom_prob newmv_prob[NEWMV_MODE_CONTEXTS];
  aom_prob zeromv_prob[ZEROMV_MODE_CONTEXTS];
  aom_prob refmv_prob[REFMV_MODE_CONTEXTS];
  aom_prob drl_prob[DRL_MODE_CONTEXTS];
200 201 202 203 204 205
#if CONFIG_NEW_MULTISYMBOL
  aom_cdf_prob newmv_cdf[NEWMV_MODE_CONTEXTS][CDF_SIZE(2)];
  aom_cdf_prob zeromv_cdf[ZEROMV_MODE_CONTEXTS][CDF_SIZE(2)];
  aom_cdf_prob refmv_cdf[REFMV_MODE_CONTEXTS][CDF_SIZE(2)];
  aom_cdf_prob drl_cdf[DRL_MODE_CONTEXTS][CDF_SIZE(2)];
#endif
206

207
#if CONFIG_EXT_INTER
clang-format's avatar
clang-format committed
208 209
  aom_prob inter_compound_mode_probs[INTER_MODE_CONTEXTS]
                                    [INTER_COMPOUND_MODES - 1];
210 211
  aom_cdf_prob inter_compound_mode_cdf[INTER_MODE_CONTEXTS]
                                      [CDF_SIZE(INTER_COMPOUND_MODES)];
212 213 214
#if CONFIG_COMPOUND_SINGLEREF
  aom_prob inter_singleref_comp_mode_probs[INTER_MODE_CONTEXTS]
                                          [INTER_SINGLEREF_COMP_MODES - 1];
215 216
  aom_cdf_prob inter_singleref_comp_mode_cdf[INTER_MODE_CONTEXTS][CDF_SIZE(
      INTER_SINGLEREF_COMP_MODES)];
217
#endif  // CONFIG_COMPOUND_SINGLEREF
218
  aom_prob compound_type_prob[BLOCK_SIZES_ALL][COMPOUND_TYPES - 1];
David Barker's avatar
David Barker committed
219
#if CONFIG_WEDGE || CONFIG_COMPOUND_SEGMENT
220
  aom_cdf_prob compound_type_cdf[BLOCK_SIZES_ALL][CDF_SIZE(COMPOUND_TYPES)];
David Barker's avatar
David Barker committed
221
#endif  // CONFIG_WEDGE || CONFIG_COMPOUND_SEGMENT
222
#if CONFIG_INTERINTRA
Yaowu Xu's avatar
Yaowu Xu committed
223
  aom_prob interintra_prob[BLOCK_SIZE_GROUPS];
224
  aom_prob wedge_interintra_prob[BLOCK_SIZES_ALL];
225
  aom_prob interintra_mode_prob[BLOCK_SIZE_GROUPS][INTERINTRA_MODES - 1];
226 227
#if CONFIG_NEW_MULTISYMBOL
  aom_cdf_prob interintra_cdf[BLOCK_SIZE_GROUPS][CDF_SIZE(2)];
228
  aom_cdf_prob wedge_interintra_cdf[BLOCK_SIZES_ALL][CDF_SIZE(2)];
229
#endif
230 231
  aom_cdf_prob interintra_mode_cdf[BLOCK_SIZE_GROUPS]
                                  [CDF_SIZE(INTERINTRA_MODES)];
232
#endif  // CONFIG_INTERINTRA
233
#endif  // CONFIG_EXT_INTER
Yue Chen's avatar
Yue Chen committed
234
#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
235 236
  aom_prob motion_mode_prob[BLOCK_SIZES_ALL][MOTION_MODES - 1];
  aom_cdf_prob motion_mode_cdf[BLOCK_SIZES_ALL][CDF_SIZE(MOTION_MODES)];
237 238
#if CONFIG_NCOBMC_ADAPT_WEIGHT && CONFIG_MOTION_VAR
  aom_prob ncobmc_mode_prob[ADAPT_OVERLAP_BLOCKS][MAX_NCOBMC_MODES - 1];
239 240
  aom_cdf_prob ncobmc_mode_cdf[ADAPT_OVERLAP_BLOCKS]
                              [CDF_SIZE(MAX_NCOBMC_MODES)];
241
#endif
Yue Chen's avatar
Yue Chen committed
242
#if CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION
243 244 245 246
#if CONFIG_NCOBMC_ADAPT_WEIGHT
  aom_prob ncobmc_prob[BLOCK_SIZES_ALL][OBMC_FAMILY_MODES - 1];
  aom_cdf_prob ncobmc_cdf[BLOCK_SIZES_ALL][CDF_SIZE(OBMC_FAMILY_MODES)];
#endif  // CONFIG_NCOBMC_ADAPT_WEIGHT
247
  aom_prob obmc_prob[BLOCK_SIZES_ALL];
248
#if CONFIG_NEW_MULTISYMBOL || CONFIG_NCOBMC_ADAPT_WEIGHT
249
  aom_cdf_prob obmc_cdf[BLOCK_SIZES_ALL][CDF_SIZE(2)];
250
#endif  // CONFIG_NEW_MULTISYMBOL
Yue Chen's avatar
Yue Chen committed
251
#endif  // CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION
Yue Chen's avatar
Yue Chen committed
252
#endif  // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
Yaowu Xu's avatar
Yaowu Xu committed
253 254
  aom_prob intra_inter_prob[INTRA_INTER_CONTEXTS];
  aom_prob comp_inter_prob[COMP_INTER_CONTEXTS];
255 256 257 258 259 260 261 262 263
  aom_cdf_prob palette_y_size_cdf[PALETTE_BLOCK_SIZES][CDF_SIZE(PALETTE_SIZES)];
  aom_cdf_prob palette_uv_size_cdf[PALETTE_BLOCK_SIZES]
                                  [CDF_SIZE(PALETTE_SIZES)];
  aom_cdf_prob palette_y_color_index_cdf[PALETTE_SIZES]
                                        [PALETTE_COLOR_INDEX_CONTEXTS]
                                        [CDF_SIZE(PALETTE_COLORS)];
  aom_cdf_prob palette_uv_color_index_cdf[PALETTE_SIZES]
                                         [PALETTE_COLOR_INDEX_CONTEXTS]
                                         [CDF_SIZE(PALETTE_COLORS)];
264 265 266 267 268 269
#if CONFIG_MRC_TX
  aom_cdf_prob mrc_mask_inter_cdf[PALETTE_SIZES][PALETTE_COLOR_INDEX_CONTEXTS]
                                 [CDF_SIZE(PALETTE_COLORS)];
  aom_cdf_prob mrc_mask_intra_cdf[PALETTE_SIZES][PALETTE_COLOR_INDEX_CONTEXTS]
                                 [CDF_SIZE(PALETTE_COLORS)];
#endif  // CONFIG_MRC_TX
270
#if CONFIG_NEW_MULTISYMBOL
271 272 273
  aom_cdf_prob palette_y_mode_cdf[PALETTE_BLOCK_SIZES][PALETTE_Y_MODE_CONTEXTS]
                                 [CDF_SIZE(2)];
  aom_cdf_prob palette_uv_mode_cdf[PALETTE_UV_MODE_CONTEXTS][CDF_SIZE(2)];
274
  aom_cdf_prob comp_inter_cdf[COMP_INTER_CONTEXTS][CDF_SIZE(2)];
275
  aom_cdf_prob single_ref_cdf[REF_CONTEXTS][SINGLE_REFS - 1][CDF_SIZE(2)];
276
#endif
Zoe Liu's avatar
Zoe Liu committed
277 278 279
#if CONFIG_EXT_COMP_REFS
  aom_prob comp_ref_type_prob[COMP_REF_TYPE_CONTEXTS];
  aom_prob uni_comp_ref_prob[UNI_COMP_REF_CONTEXTS][UNIDIR_COMP_REFS - 1];
280 281 282 283 284
#if CONFIG_NEW_MULTISYMBOL
  aom_cdf_prob comp_ref_type_cdf[COMP_REF_TYPE_CONTEXTS][CDF_SIZE(2)];
  aom_cdf_prob uni_comp_ref_cdf[UNI_COMP_REF_CONTEXTS][UNIDIR_COMP_REFS - 1]
                               [CDF_SIZE(2)];
#endif  // CONFIG_NEW_MULTISYMBOL
Zoe Liu's avatar
Zoe Liu committed
285
#endif  // CONFIG_EXT_COMP_REFS
Yaowu Xu's avatar
Yaowu Xu committed
286
  aom_prob single_ref_prob[REF_CONTEXTS][SINGLE_REFS - 1];
287
#if CONFIG_EXT_REFS
Yaowu Xu's avatar
Yaowu Xu committed
288 289
  aom_prob comp_ref_prob[REF_CONTEXTS][FWD_REFS - 1];
  aom_prob comp_bwdref_prob[REF_CONTEXTS][BWD_REFS - 1];
290
#else
Yaowu Xu's avatar
Yaowu Xu committed
291
  aom_prob comp_ref_prob[REF_CONTEXTS][COMP_REFS - 1];
292
#endif  // CONFIG_EXT_REFS
293 294 295 296 297 298 299 300
#if CONFIG_NEW_MULTISYMBOL
#if CONFIG_EXT_REFS
  aom_cdf_prob comp_ref_cdf[REF_CONTEXTS][FWD_REFS - 1][CDF_SIZE(2)];
  aom_cdf_prob comp_bwdref_cdf[REF_CONTEXTS][BWD_REFS - 1][CDF_SIZE(2)];
#else
  aom_cdf_prob comp_ref_cdf[REF_CONTEXTS][COMP_REFS - 1][CDF_SIZE(2)];
#endif  // CONFIG_EXT_REFS
#endif
301 302 303
#if CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF
  aom_prob comp_inter_mode_prob[COMP_INTER_MODE_CONTEXTS];
#endif  // CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF
Jingning Han's avatar
Jingning Han committed
304
  aom_prob tx_size_probs[MAX_TX_DEPTH][TX_SIZE_CONTEXTS][MAX_TX_DEPTH];
Yue Chen's avatar
Yue Chen committed
305
#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
306
  aom_prob quarter_tx_size_prob;
Yue Chen's avatar
Yue Chen committed
307
#endif
308
#if CONFIG_VAR_TX
Yaowu Xu's avatar
Yaowu Xu committed
309
  aom_prob txfm_partition_prob[TXFM_PARTITION_CONTEXTS];
310 311
#if CONFIG_NEW_MULTISYMBOL
  aom_cdf_prob txfm_partition_cdf[TXFM_PARTITION_CONTEXTS][CDF_SIZE(2)];
312
#endif
313
#endif  // CONFIG_VAR_TX
Yaowu Xu's avatar
Yaowu Xu committed
314
  aom_prob skip_probs[SKIP_CONTEXTS];
315 316
#if CONFIG_NEW_MULTISYMBOL
  aom_cdf_prob skip_cdfs[SKIP_CONTEXTS][CDF_SIZE(2)];
317
  aom_cdf_prob intra_inter_cdf[INTRA_INTER_CONTEXTS][CDF_SIZE(2)];
318
#endif
319
  nmv_context nmvc[NMV_CONTEXTS];
Alex Converse's avatar
Alex Converse committed
320 321
#if CONFIG_INTRABC
  nmv_context ndvc;
Hui Su's avatar
Hui Su committed
322
  aom_cdf_prob intrabc_cdf[CDF_SIZE(2)];
323
#endif
hui su's avatar
hui su committed
324
  int initialized;
325
#if CONFIG_EXT_TX
Yaowu Xu's avatar
Yaowu Xu committed
326
  aom_prob inter_ext_tx_prob[EXT_TX_SETS_INTER][EXT_TX_SIZES][TX_TYPES - 1];
clang-format's avatar
clang-format committed
327 328
  aom_prob intra_ext_tx_prob[EXT_TX_SETS_INTRA][EXT_TX_SIZES][INTRA_MODES]
                            [TX_TYPES - 1];
329
#else
Yaowu Xu's avatar
Yaowu Xu committed
330 331
  aom_prob intra_ext_tx_prob[EXT_TX_SIZES][TX_TYPES][TX_TYPES - 1];
  aom_prob inter_ext_tx_prob[EXT_TX_SIZES][TX_TYPES - 1];
332
#endif  // CONFIG_EXT_TX
333
#if CONFIG_SUPERTX
Yaowu Xu's avatar
Yaowu Xu committed
334
  aom_prob supertx_prob[PARTITION_SUPERTX_CONTEXTS][TX_SIZES];
335
#endif  // CONFIG_SUPERTX
336
  struct segmentation_probs seg;
hui su's avatar
hui su committed
337
#if CONFIG_EXT_INTRA
hui su's avatar
hui su committed
338
#if CONFIG_INTRA_INTERP
Yaowu Xu's avatar
Yaowu Xu committed
339
  aom_prob intra_filter_probs[INTRA_FILTERS + 1][INTRA_FILTERS - 1];
hui su's avatar
hui su committed
340
#endif  // CONFIG_INTRA_INTERP
hui su's avatar
hui su committed
341
#endif  // CONFIG_EXT_INTRA
342 343 344
#if CONFIG_FILTER_INTRA
  aom_prob filter_intra_probs[PLANE_TYPES];
#endif  // CONFIG_FILTER_INTRA
345 346 347
#if CONFIG_LOOP_RESTORATION
  aom_prob switchable_restore_prob[RESTORE_SWITCHABLE_TYPES - 1];
#endif  // CONFIG_LOOP_RESTORATION
348
  aom_cdf_prob y_mode_cdf[BLOCK_SIZE_GROUPS][CDF_SIZE(INTRA_MODES)];
Luc Trudeau's avatar
Luc Trudeau committed
349
  aom_cdf_prob uv_mode_cdf[INTRA_MODES][CDF_SIZE(UV_INTRA_MODES)];
350 351 352
#if CONFIG_EXT_PARTITION_TYPES
  aom_cdf_prob partition_cdf[PARTITION_CONTEXTS][CDF_SIZE(EXT_PARTITION_TYPES)];
#else
353
  aom_cdf_prob partition_cdf[PARTITION_CONTEXTS][CDF_SIZE(PARTITION_TYPES)];
354
#endif
355
  aom_cdf_prob switchable_interp_cdf[SWITCHABLE_FILTER_CONTEXTS]
356
                                    [CDF_SIZE(SWITCHABLE_FILTERS)];
357 358 359 360
  /* kf_y_cdf is discarded after use, so does not require persistent storage.
     However, we keep it with the other CDFs in this struct since it needs to
     be copied to each tile to support parallelism just like the others.
   */
361
  aom_cdf_prob kf_y_cdf[INTRA_MODES][INTRA_MODES][CDF_SIZE(INTRA_MODES)];
362
  aom_cdf_prob tx_size_cdf[MAX_TX_DEPTH][TX_SIZE_CONTEXTS]
363
                          [CDF_SIZE(MAX_TX_DEPTH + 1)];
Thomas Davies's avatar
Thomas Davies committed
364 365
#if CONFIG_DELTA_Q
  aom_cdf_prob delta_q_cdf[CDF_SIZE(DELTA_Q_PROBS + 1)];
Fangwen Fu's avatar
Fangwen Fu committed
366 367 368
#if CONFIG_EXT_DELTA_Q
  aom_cdf_prob delta_lf_cdf[CDF_SIZE(DELTA_LF_PROBS + 1)];
#endif
369
#endif  // CONFIG_DELTA_Q
370 371 372 373 374 375
#if CONFIG_EXT_TX
  aom_cdf_prob intra_ext_tx_cdf[EXT_TX_SETS_INTRA][EXT_TX_SIZES][INTRA_MODES]
                               [CDF_SIZE(TX_TYPES)];
  aom_cdf_prob inter_ext_tx_cdf[EXT_TX_SETS_INTER][EXT_TX_SIZES]
                               [CDF_SIZE(TX_TYPES)];
#else
376 377
  aom_cdf_prob intra_ext_tx_cdf[EXT_TX_SIZES][TX_TYPES][CDF_SIZE(TX_TYPES)];
  aom_cdf_prob inter_ext_tx_cdf[EXT_TX_SIZES][CDF_SIZE(TX_TYPES)];
378
#endif  // CONFIG_EXT_TX
379 380 381
#if CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP
  aom_cdf_prob intra_filter_cdf[INTRA_FILTERS + 1][CDF_SIZE(INTRA_FILTERS)];
#endif  // CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP
382
#if CONFIG_DELTA_Q
383
  aom_prob delta_q_prob[DELTA_Q_PROBS];
Fangwen Fu's avatar
Fangwen Fu committed
384 385 386
#if CONFIG_EXT_DELTA_Q
  aom_prob delta_lf_prob[DELTA_LF_PROBS];
#endif
387
#endif
388 389 390 391 392
#if CONFIG_PVQ
  // TODO(any): If PVQ is enabled, most of coefficient related cdf,
  // such as coef_cdfs[], coef_tail_cdfs[], and coef_heaf_cdfs[] can be removed.
  od_adapt_ctx pvq_context;
#endif  // CONFIG_PVQ
Luc Trudeau's avatar
Luc Trudeau committed
393
#if CONFIG_CFL
394 395
  aom_cdf_prob cfl_sign_cdf[CDF_SIZE(CFL_JOINT_SIGNS)];
  aom_cdf_prob cfl_alpha_cdf[CFL_ALPHA_CONTEXTS][CDF_SIZE(CFL_ALPHABET_SIZE)];
Luc Trudeau's avatar
Luc Trudeau committed
396
#endif
Jingning Han's avatar
Jingning Han committed
397 398 399
} FRAME_CONTEXT;

typedef struct FRAME_COUNTS {
400 401
// Note: This structure should only contain 'unsigned int' fields, or
// aggregates built solely from 'unsigned int' fields/elements
402
#if CONFIG_ENTROPY_STATS
403
  unsigned int kf_y_mode[INTRA_MODES][INTRA_MODES][INTRA_MODES];
Jingning Han's avatar
Jingning Han committed
404
  unsigned int y_mode[BLOCK_SIZE_GROUPS][INTRA_MODES];
Luc Trudeau's avatar
Luc Trudeau committed
405
  unsigned int uv_mode[INTRA_MODES][UV_INTRA_MODES];
406
#endif  // CONFIG_ENTROPY_STATS
407 408 409
#if CONFIG_EXT_PARTITION_TYPES
  unsigned int partition[PARTITION_CONTEXTS][EXT_PARTITION_TYPES];
#else
Jingning Han's avatar
Jingning Han committed
410
  unsigned int partition[PARTITION_CONTEXTS][PARTITION_TYPES];
411
#endif
clang-format's avatar
clang-format committed
412 413
  unsigned int switchable_interp[SWITCHABLE_FILTER_CONTEXTS]
                                [SWITCHABLE_FILTERS];
414
#if CONFIG_ADAPT_SCAN
415
#if CONFIG_CHROMA_2X2
416
  unsigned int non_zero_count_2x2[TX_TYPES][4];
417
#endif  // CONFIG_CHROMA_2X2
418 419 420 421
  unsigned int non_zero_count_4X4[TX_TYPES][16];
  unsigned int non_zero_count_8X8[TX_TYPES][64];
  unsigned int non_zero_count_16X16[TX_TYPES][256];
  unsigned int non_zero_count_32X32[TX_TYPES][1024];
422 423 424 425 426 427 428 429 430

  unsigned int non_zero_count_4x8[TX_TYPES][32];
  unsigned int non_zero_count_8x4[TX_TYPES][32];
  unsigned int non_zero_count_8x16[TX_TYPES][128];
  unsigned int non_zero_count_16x8[TX_TYPES][128];
  unsigned int non_zero_count_16x32[TX_TYPES][512];
  unsigned int non_zero_count_32x16[TX_TYPES][512];

  unsigned int txb_count[TX_SIZES_ALL][TX_TYPES];
431
#endif  // CONFIG_ADAPT_SCAN
432 433 434 435 436 437 438 439 440

#if CONFIG_LV_MAP
  unsigned int txb_skip[TX_SIZES][TXB_SKIP_CONTEXTS][2];
  unsigned int nz_map[TX_SIZES][PLANE_TYPES][SIG_COEF_CONTEXTS][2];
  unsigned int eob_flag[TX_SIZES][PLANE_TYPES][EOB_COEF_CONTEXTS][2];
  unsigned int dc_sign[PLANE_TYPES][DC_SIGN_CONTEXTS][2];
  unsigned int coeff_base[TX_SIZES][PLANE_TYPES][NUM_BASE_LEVELS]
                         [COEFF_BASE_CONTEXTS][2];
  unsigned int coeff_lps[TX_SIZES][PLANE_TYPES][LEVEL_CONTEXTS][2];
441 442
  unsigned int coeff_br[TX_SIZES][PLANE_TYPES][BASE_RANGE_SETS][LEVEL_CONTEXTS]
                       [2];
443 444
#endif  // CONFIG_LV_MAP

445
  av1_blockz_count_model blockz_count[TX_SIZES][PLANE_TYPES];
446

447 448 449
  unsigned int newmv_mode[NEWMV_MODE_CONTEXTS][2];
  unsigned int zeromv_mode[ZEROMV_MODE_CONTEXTS][2];
  unsigned int refmv_mode[REFMV_MODE_CONTEXTS][2];
450
  unsigned int drl_mode[DRL_MODE_CONTEXTS][2];
451

452 453
#if CONFIG_EXT_INTER
  unsigned int inter_compound_mode[INTER_MODE_CONTEXTS][INTER_COMPOUND_MODES];
454 455 456 457
#if CONFIG_COMPOUND_SINGLEREF
  unsigned int inter_singleref_comp_mode[INTER_MODE_CONTEXTS]
                                        [INTER_SINGLEREF_COMP_MODES];
#endif  // CONFIG_COMPOUND_SINGLEREF
458
#if CONFIG_INTERINTRA
459
  unsigned int interintra[BLOCK_SIZE_GROUPS][2];
460
  unsigned int interintra_mode[BLOCK_SIZE_GROUPS][INTERINTRA_MODES];
461
  unsigned int wedge_interintra[BLOCK_SIZES_ALL][2];
462
#endif  // CONFIG_INTERINTRA
463
  unsigned int compound_interinter[BLOCK_SIZES_ALL][COMPOUND_TYPES];
464
#endif  // CONFIG_EXT_INTER
Yue Chen's avatar
Yue Chen committed
465
#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
466
  unsigned int motion_mode[BLOCK_SIZES_ALL][MOTION_MODES];
467 468 469
#if CONFIG_NCOBMC_ADAPT_WEIGHT && CONFIG_MOTION_VAR
  unsigned int ncobmc_mode[ADAPT_OVERLAP_BLOCKS][MAX_NCOBMC_MODES];
#endif
Yue Chen's avatar
Yue Chen committed
470
#if CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION
471 472 473
#if CONFIG_NCOBMC_ADAPT_WEIGHT
  unsigned int ncobmc[BLOCK_SIZES_ALL][OBMC_FAMILY_MODES];
#endif  // CONFIG_NCOBMC_ADAPT_WEIGHT
474
  unsigned int obmc[BLOCK_SIZES_ALL][2];
Yue Chen's avatar
Yue Chen committed
475
#endif  // CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION
Yue Chen's avatar
Yue Chen committed
476
#endif  // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
Jingning Han's avatar
Jingning Han committed
477 478
  unsigned int intra_inter[INTRA_INTER_CONTEXTS][2];
  unsigned int comp_inter[COMP_INTER_CONTEXTS][2];
Zoe Liu's avatar
Zoe Liu committed
479 480 481 482
#if CONFIG_EXT_COMP_REFS
  unsigned int comp_ref_type[COMP_REF_TYPE_CONTEXTS][2];
  unsigned int uni_comp_ref[UNI_COMP_REF_CONTEXTS][UNIDIR_COMP_REFS - 1][2];
#endif  // CONFIG_EXT_COMP_REFS
clang-format's avatar
clang-format committed
483
  unsigned int single_ref[REF_CONTEXTS][SINGLE_REFS - 1][2];
484
#if CONFIG_EXT_REFS
clang-format's avatar
clang-format committed
485 486
  unsigned int comp_ref[REF_CONTEXTS][FWD_REFS - 1][2];
  unsigned int comp_bwdref[REF_CONTEXTS][BWD_REFS - 1][2];
487
#else
clang-format's avatar
clang-format committed
488
  unsigned int comp_ref[REF_CONTEXTS][COMP_REFS - 1][2];
489
#endif  // CONFIG_EXT_REFS
490 491 492
#if CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF
  unsigned int comp_inter_mode[COMP_INTER_MODE_CONTEXTS][2];
#endif  // CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF
493 494 495
  // TODO(any): tx_size_totals is only used by the encoder to decide whether
  // to use forward updates for the coeff probs, and as such it does not really
  // belong into this structure.
496
  unsigned int tx_size_totals[TX_SIZES];
497
  unsigned int tx_size[MAX_TX_DEPTH][TX_SIZE_CONTEXTS][MAX_TX_DEPTH + 1];
Yue Chen's avatar
Yue Chen committed
498
#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
499
  unsigned int quarter_tx_size[2];
Yue Chen's avatar
Yue Chen committed
500
#endif
501 502 503
#if CONFIG_VAR_TX
  unsigned int txfm_partition[TXFM_PARTITION_CONTEXTS][2];
#endif
Jingning Han's avatar
Jingning Han committed
504
  unsigned int skip[SKIP_CONTEXTS][2];
505
  nmv_context_counts mv[NMV_CONTEXTS];
Alex Converse's avatar
Alex Converse committed
506
#if CONFIG_INTRABC
507
  unsigned int intrabc[2];
Alex Converse's avatar
Alex Converse committed
508 509
  nmv_context_counts dv;
#endif
510
#if CONFIG_DELTA_Q
511
  unsigned int delta_q[DELTA_Q_PROBS][2];
Fangwen Fu's avatar
Fangwen Fu committed
512 513 514
#if CONFIG_EXT_DELTA_Q
  unsigned int delta_lf[DELTA_LF_PROBS][2];
#endif
515
#endif
516
#if CONFIG_EXT_TX
Debargha Mukherjee's avatar
Debargha Mukherjee committed
517 518 519
#if CONFIG_RECT_TX
  unsigned int tx_size_implied[TX_SIZES][TX_SIZES];
#endif  // CONFIG_RECT_TX
520
  unsigned int inter_ext_tx[EXT_TX_SETS_INTER][EXT_TX_SIZES][TX_TYPES];
clang-format's avatar
clang-format committed
521 522
  unsigned int intra_ext_tx[EXT_TX_SETS_INTRA][EXT_TX_SIZES][INTRA_MODES]
                           [TX_TYPES];
523
#else
524 525
  unsigned int intra_ext_tx[EXT_TX_SIZES][TX_TYPES][TX_TYPES];
  unsigned int inter_ext_tx[EXT_TX_SIZES][TX_TYPES];
526
#endif  // CONFIG_EXT_TX
527 528 529 530
#if CONFIG_SUPERTX
  unsigned int supertx[PARTITION_SUPERTX_CONTEXTS][TX_SIZES][2];
  unsigned int supertx_size[TX_SIZES];
#endif  // CONFIG_SUPERTX
531
  struct seg_counts seg;
hui su's avatar
hui su committed
532
#if CONFIG_EXT_INTRA
hui su's avatar
hui su committed
533
#if CONFIG_INTRA_INTERP
534
  unsigned int intra_filter[INTRA_FILTERS + 1][INTRA_FILTERS];
hui su's avatar
hui su committed
535
#endif  // CONFIG_INTRA_INTERP
hui su's avatar
hui su committed
536
#endif  // CONFIG_EXT_INTRA
537 538 539
#if CONFIG_FILTER_INTRA
  unsigned int filter_intra[PLANE_TYPES][2];
#endif  // CONFIG_FILTER_INTRA
Jingning Han's avatar
Jingning Han committed
540 541
} FRAME_COUNTS;

542
// CDF version of 'av1_kf_y_mode_prob'.
543 544 545
extern const aom_cdf_prob av1_kf_y_mode_cdf[INTRA_MODES][INTRA_MODES]
                                           [CDF_SIZE(INTRA_MODES)];

Yaowu Xu's avatar
Yaowu Xu committed
546 547
extern const aom_prob av1_default_palette_y_mode_prob[PALETTE_BLOCK_SIZES]
                                                     [PALETTE_Y_MODE_CONTEXTS];
548 549
extern const aom_prob
    av1_default_palette_uv_mode_prob[PALETTE_UV_MODE_CONTEXTS];
hui su's avatar
hui su committed
550

551 552
extern const int av1_intra_mode_ind[INTRA_MODES];
extern const int av1_intra_mode_inv[INTRA_MODES];
553
#if CONFIG_EXT_TX
Hui Su's avatar
Hui Su committed
554 555
extern int av1_ext_tx_ind[EXT_TX_SET_TYPES][TX_TYPES];
extern int av1_ext_tx_inv[EXT_TX_SET_TYPES][TX_TYPES];
556 557
#endif

558
#if CONFIG_EXT_INTER
559
#if CONFIG_INTERINTRA
Yaowu Xu's avatar
Yaowu Xu committed
560 561
extern const aom_tree_index
    av1_interintra_mode_tree[TREE_SIZE(INTERINTRA_MODES)];
562
#endif
Yaowu Xu's avatar
Yaowu Xu committed
563 564
extern const aom_tree_index
    av1_inter_compound_mode_tree[TREE_SIZE(INTER_COMPOUND_MODES)];
565 566 567 568
#if CONFIG_COMPOUND_SINGLEREF
extern const aom_tree_index
    av1_inter_singleref_comp_mode_tree[TREE_SIZE(INTER_SINGLEREF_COMP_MODES)];
#endif  // CONFIG_COMPOUND_SINGLEREF
569
extern const aom_tree_index av1_compound_type_tree[TREE_SIZE(COMPOUND_TYPES)];
570
#endif  // CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
571
extern const aom_tree_index av1_partition_tree[TREE_SIZE(PARTITION_TYPES)];
572
#if CONFIG_EXT_PARTITION_TYPES
Yaowu Xu's avatar
Yaowu Xu committed
573 574
extern const aom_tree_index
    av1_ext_partition_tree[TREE_SIZE(EXT_PARTITION_TYPES)];
575
#endif
Yaowu Xu's avatar
Yaowu Xu committed
576 577
extern const aom_tree_index
    av1_switchable_interp_tree[TREE_SIZE(SWITCHABLE_FILTERS)];
578
extern const aom_tree_index
579
    av1_palette_color_index_tree[PALETTE_SIZES][TREE_SIZE(PALETTE_COLORS)];
580
extern const aom_tree_index av1_tx_size_tree[MAX_TX_DEPTH][TREE_SIZE(TX_SIZES)];
581
#if CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP
Yaowu Xu's avatar
Yaowu Xu committed
582
extern const aom_tree_index av1_intra_filter_tree[TREE_SIZE(INTRA_FILTERS)];
583
#endif  // CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP
584
#if CONFIG_EXT_TX
clang-format's avatar
clang-format committed
585 586 587 588
extern const aom_tree_index av1_ext_tx_inter_tree[EXT_TX_SETS_INTER]
                                                 [TREE_SIZE(TX_TYPES)];
extern const aom_tree_index av1_ext_tx_intra_tree[EXT_TX_SETS_INTRA]
                                                 [TREE_SIZE(TX_TYPES)];
Hui Su's avatar
Hui Su committed
589 590
extern const aom_tree_index av1_ext_tx_tree[EXT_TX_SET_TYPES]
                                           [TREE_SIZE(TX_TYPES)];
591
#else
Yaowu Xu's avatar
Yaowu Xu committed
592
extern const aom_tree_index av1_ext_tx_tree[TREE_SIZE(TX_TYPES)];
593
#endif  // CONFIG_EXT_TX
Yue Chen's avatar
Yue Chen committed
594 595 596
#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
extern const aom_tree_index av1_motion_mode_tree[TREE_SIZE(MOTION_MODES)];
#endif  // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
597
#if CONFIG_NCOBMC_ADAPT_WEIGHT
598
extern const aom_tree_index av1_ncobmc_mode_tree[TREE_SIZE(MAX_NCOBMC_MODES)];
599 600 601 602
#if CONFIG_WARPED_MOTION
extern const aom_tree_index av1_ncobmc_tree[TREE_SIZE(OBMC_FAMILY_MODES)];
#endif  // CONFIG_WARPED_MOTION
#endif  // CONFIG_NCOBMC_ADAPT_WEIGHT
603
#if CONFIG_LOOP_RESTORATION
604
#define RESTORE_NONE_SGRPROJ_PROB 64
605 606
#define RESTORE_NONE_BILATERAL_PROB 16
#define RESTORE_NONE_WIENER_PROB 64
607
#define RESTORE_NONE_DOMAINTXFMRF_PROB 64
608 609 610
extern const aom_tree_index
    av1_switchable_restore_tree[TREE_SIZE(RESTORE_SWITCHABLE_TYPES)];
#endif  // CONFIG_LOOP_RESTORATION
611 612
extern int av1_switchable_interp_ind[SWITCHABLE_FILTERS];
extern int av1_switchable_interp_inv[SWITCHABLE_FILTERS];
613

614 615 616 617
#if CONFIG_EXT_PARTITION_TYPES
extern int av1_num_partition_types[PARTITION_BLOCK_SIZES];
#endif

Yaowu Xu's avatar
Yaowu Xu committed
618
void av1_setup_past_independence(struct AV1Common *cm);
Jingning Han's avatar
Jingning Han committed
619

Yaowu Xu's avatar
Yaowu Xu committed
620 621
void av1_adapt_intra_frame_probs(struct AV1Common *cm);
void av1_adapt_inter_frame_probs(struct AV1Common *cm);
622
#if !CONFIG_EXT_TX
623 624 625
extern int av1_ext_tx_ind[TX_TYPES];
extern int av1_ext_tx_inv[TX_TYPES];
#endif
Jingning Han's avatar
Jingning Han committed
626

Yaowu Xu's avatar
Yaowu Xu committed
627
static INLINE int av1_ceil_log2(int n) {
hui su's avatar
hui su committed
628 629 630 631 632 633 634 635
  int i = 1, p = 2;
  while (p < n) {
    i++;
    p = p << 1;
  }
  return i;
}

636 637
// Returns the context for palette color index at row 'r' and column 'c',
// along with the 'color_order' of neighbors and the 'color_idx'.
638
// The 'color_map' is a 2D array with the given 'stride'.
639 640 641
int av1_get_palette_color_index_context(const uint8_t *color_map, int stride,
                                        int r, int c, int palette_size,
                                        uint8_t *color_order, int *color_idx);
hui su's avatar
hui su committed
642

Jingning Han's avatar
Jingning Han committed
643 644 645 646
#ifdef __cplusplus
}  // extern "C"
#endif

Yaowu Xu's avatar
Yaowu Xu committed
647
#endif  // AV1_COMMON_ENTROPYMODE_H_