block.h 13.1 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4 5 6 7 8 9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10 11
 */

Yaowu Xu's avatar
Yaowu Xu committed
12 13
#ifndef AV1_ENCODER_BLOCK_H_
#define AV1_ENCODER_BLOCK_H_
Jingning Han's avatar
Jingning Han committed
14

15 16 17
#include "av1/common/entropymv.h"
#include "av1/common/entropy.h"
#include "av1/common/mvref_common.h"
18
#include "av1/encoder/hash.h"
19 20 21
#if CONFIG_DIST_8X8
#include "aom/aomcx.h"
#endif
Jingning Han's avatar
Jingning Han committed
22 23 24 25 26 27 28 29 30

#ifdef __cplusplus
extern "C" {
#endif

typedef struct {
  unsigned int sse;
  int sum;
  unsigned int var;
31
} DIFF;
Jingning Han's avatar
Jingning Han committed
32

Angie Chiang's avatar
Angie Chiang committed
33
typedef struct macroblock_plane {
34
  DECLARE_ALIGNED(16, int16_t, src_diff[MAX_SB_SQUARE]);
Jingning Han's avatar
Jingning Han committed
35 36 37
  tran_low_t *qcoeff;
  tran_low_t *coeff;
  uint16_t *eobs;
38 39 40
#if CONFIG_LV_MAP
  uint8_t *txb_entropy_ctx;
#endif
Jingning Han's avatar
Jingning Han committed
41 42 43
  struct buf_2d src;

  // Quantizer setings
44 45 46 47 48 49 50 51 52 53
  // These are used/accessed only in the quantization process
  // RDO does not / must not depend on any of these values
  // All values below share the coefficient scale/shift used in TX
  const int16_t *quant_fp_QTX;
  const int16_t *round_fp_QTX;
  const int16_t *quant_QTX;
  const int16_t *quant_shift_QTX;
  const int16_t *zbin_QTX;
  const int16_t *round_QTX;
  const int16_t *dequant_QTX;
54
#if CONFIG_NEW_QUANT
55
  const cuml_bins_type_nuq *cuml_bins_nuq[X0_PROFILES];
56
  const dequant_val_type_nuq *dequant_val_nuq_QTX[QUANT_PROFILES];
57
#endif  // CONFIG_NEW_QUANT
Angie Chiang's avatar
Angie Chiang committed
58
} MACROBLOCK_PLANE;
Jingning Han's avatar
Jingning Han committed
59

hui su's avatar
hui su committed
60 61
typedef int av1_coeff_cost[PLANE_TYPES][REF_TYPES][COEF_BANDS][COEFF_CONTEXTS]
                          [TAIL_TOKENS];
Jingning Han's avatar
Jingning Han committed
62

63 64 65
#if CONFIG_LV_MAP
typedef struct {
  int txb_skip_cost[TXB_SKIP_CONTEXTS][2];
66
  int base_eob_cost[SIG_COEF_CONTEXTS_EOB][3];
67
  int base_cost[SIG_COEF_CONTEXTS][4];
Angie Chiang's avatar
Angie Chiang committed
68
  int eob_extra_cost[EOB_COEF_CONTEXTS][2];
69
  int dc_sign_cost[DC_SIGN_CONTEXTS][2];
Angie Chiang's avatar
Angie Chiang committed
70
  int lps_cost[LEVEL_CONTEXTS][COEFF_BASE_RANGE + 1];
71 72
} LV_MAP_COEFF_COST;

73 74
typedef struct { int eob_cost[2][11]; } LV_MAP_EOB_COST;

Jingning Han's avatar
Jingning Han committed
75
typedef struct {
76
  tran_low_t tcoeff[MAX_MB_PLANE][MAX_SB_SQUARE];
Angie Chiang's avatar
Angie Chiang committed
77 78 79 80 81
  uint16_t eobs[MAX_MB_PLANE][MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)];
  uint8_t txb_skip_ctx[MAX_MB_PLANE]
                      [MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)];
  int dc_sign_ctx[MAX_MB_PLANE]
                 [MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)];
82 83 84 85 86 87 88 89 90 91 92 93
} CB_COEFF_BUFFER;
#endif

typedef struct {
  int_mv ref_mvs[MODE_CTX_REF_FRAMES][MAX_MV_REF_CANDIDATES];
  int16_t mode_context[MODE_CTX_REF_FRAMES];
#if CONFIG_LV_MAP
  // TODO(angiebird): Reduce the buffer size according to sb_type
  tran_low_t *tcoeff[MAX_MB_PLANE];
  uint16_t *eobs[MAX_MB_PLANE];
  uint8_t *txb_skip_ctx[MAX_MB_PLANE];
  int *dc_sign_ctx[MAX_MB_PLANE];
94
#endif
95 96
  uint8_t ref_mv_count[MODE_CTX_REF_FRAMES];
  CANDIDATE_MV ref_mv_stack[MODE_CTX_REF_FRAMES][MAX_REF_MV_STACK_SIZE];
97
  int16_t compound_mode_context[MODE_CTX_REF_FRAMES];
Jingning Han's avatar
Jingning Han committed
98 99
} MB_MODE_INFO_EXT;

Alex Converse's avatar
Alex Converse committed
100 101 102 103 104 105 106
typedef struct {
  int col_min;
  int col_max;
  int row_min;
  int row_max;
} MvLimits;

hui su's avatar
hui su committed
107
typedef struct {
108
  uint8_t best_palette_color_map[MAX_PALETTE_SQUARE];
109
  int kmeans_data_buf[2 * MAX_PALETTE_SQUARE];
hui su's avatar
hui su committed
110 111
} PALETTE_BUFFER;

112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132
typedef struct {
  TX_TYPE tx_type;
  TX_SIZE tx_size;
  TX_SIZE min_tx_size;
  TX_SIZE inter_tx_size[MAX_MIB_SIZE][MAX_MIB_SIZE];
  uint8_t blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE * 8];
#if CONFIG_TXK_SEL
  TX_TYPE txk_type[MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)];
#endif  // CONFIG_TXK_SEL
  RD_STATS rd_stats;
  uint32_t hash_value;
} TX_RD_INFO;

#define RD_RECORD_BUFFER_LEN 8
typedef struct {
  TX_RD_INFO tx_rd_info[RD_RECORD_BUFFER_LEN];  // Circular buffer.
  int index_start;
  int num;
  CRC_CALCULATOR crc_calculator;  // Hash function.
} TX_RD_RECORD;

133 134 135
typedef struct {
  int64_t dist;
  int rate;
Hui Su's avatar
Hui Su committed
136
  uint16_t eob;
137 138
#if CONFIG_LV_MAP
  uint16_t entropy_context;
139
  uint8_t txb_entropy_ctx;
140
#else
141
  uint8_t entropy_context;
142
#endif
143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159
  uint8_t valid;
  uint8_t fast;
} TX_SIZE_RD_INFO;

#define TX_SIZE_RD_RECORD_BUFFER_LEN 256
typedef struct {
  uint32_t hash_vals[TX_SIZE_RD_RECORD_BUFFER_LEN];
  TX_SIZE_RD_INFO tx_rd_info[TX_SIZE_RD_RECORD_BUFFER_LEN][TX_TYPES];
  int index_start;
  int num;
} TX_SIZE_RD_RECORD;

typedef struct tx_size_rd_info_node {
  TX_SIZE_RD_INFO *rd_info_array;  // Points to array of size TX_TYPES.
  struct tx_size_rd_info_node *children[4];
} TX_SIZE_RD_INFO_NODE;

Jingning Han's avatar
Jingning Han committed
160 161 162 163
typedef struct macroblock MACROBLOCK;
struct macroblock {
  struct macroblock_plane plane[MAX_MB_PLANE];

164 165 166
  // Save the transform RD search info.
  TX_RD_RECORD tx_rd_record;

167 168 169
  // Determine if one would go with reduced complexity transform block
  // search model to select prediction modes, or full complexity model
  // to select transform kernel.
170 171
  int rd_model;

172 173 174 175 176 177 178 179 180 181 182 183
  // Also save RD info on the TX size search level for square TX sizes.
  TX_SIZE_RD_RECORD
  tx_size_rd_record_8X8[(MAX_MIB_SIZE >> 1) * (MAX_MIB_SIZE >> 1)];
  TX_SIZE_RD_RECORD
  tx_size_rd_record_16X16[(MAX_MIB_SIZE >> 2) * (MAX_MIB_SIZE >> 2)];
  TX_SIZE_RD_RECORD
  tx_size_rd_record_32X32[(MAX_MIB_SIZE >> 3) * (MAX_MIB_SIZE >> 3)];
#if CONFIG_TX64X64
  TX_SIZE_RD_RECORD
  tx_size_rd_record_64X64[(MAX_MIB_SIZE >> 4) * (MAX_MIB_SIZE >> 4)];
#endif

Jingning Han's avatar
Jingning Han committed
184 185 186
  MACROBLOCKD e_mbd;
  MB_MODE_INFO_EXT *mbmi_ext;
  int skip_block;
187
  int qindex;
Jingning Han's avatar
Jingning Han committed
188

189 190
  // The equivalent error at the current rdmult of one whole bit (not one
  // bitcost unit).
Jingning Han's avatar
Jingning Han committed
191
  int errorperbit;
192 193
  // The equivalend SAD error of one (whole) bit at the current quantizer
  // for large blocks.
Jingning Han's avatar
Jingning Han committed
194
  int sadperbit16;
195 196
  // The equivalend SAD error of one (whole) bit at the current quantizer
  // for sub-8x8 blocks.
Jingning Han's avatar
Jingning Han committed
197 198 199
  int sadperbit4;
  int rdmult;
  int mb_energy;
200 201
  int *m_search_count_ptr;
  int *ex_search_count_ptr;
Jingning Han's avatar
Jingning Han committed
202

203 204
  unsigned int txb_split_count;

Jingning Han's avatar
Jingning Han committed
205 206 207 208 209
  // These are set to their default values at the beginning, and then adjusted
  // further in the encoding process.
  BLOCK_SIZE min_partition_size;
  BLOCK_SIZE max_partition_size;

210 211
  int mv_best_ref_index[TOTAL_REFS_PER_FRAME];
  unsigned int max_mv_context[TOTAL_REFS_PER_FRAME];
Jingning Han's avatar
Jingning Han committed
212
  unsigned int source_variance;
213 214
  unsigned int pred_sse[TOTAL_REFS_PER_FRAME];
  int pred_mv_sad[TOTAL_REFS_PER_FRAME];
Jingning Han's avatar
Jingning Han committed
215

216 217 218 219 220
  int *nmvjointcost;
  int nmv_vec_cost[NMV_CONTEXTS][MV_JOINTS];
  int *nmvcost[NMV_CONTEXTS][2];
  int *nmvcost_hp[NMV_CONTEXTS][2];
  int **mv_cost_stack[NMV_CONTEXTS];
Jingning Han's avatar
Jingning Han committed
221
  int **mvcost;
222

223 224
  int32_t *wsrc_buf;
  int32_t *mask_buf;
225 226
  uint8_t *above_pred_buf;
  uint8_t *left_pred_buf;
Jingning Han's avatar
Jingning Han committed
227

hui su's avatar
hui su committed
228 229
  PALETTE_BUFFER *palette_buffer;

Jingning Han's avatar
Jingning Han committed
230 231
  // These define limits to motion vector components to prevent them
  // from extending outside the UMV borders
Alex Converse's avatar
Alex Converse committed
232
  MvLimits mv_limits;
Jingning Han's avatar
Jingning Han committed
233

234 235
  uint8_t blk_skip[MAX_MB_PLANE][MAX_MIB_SIZE * MAX_MIB_SIZE * 8];
  uint8_t blk_skip_drl[MAX_MB_PLANE][MAX_MIB_SIZE * MAX_MIB_SIZE * 8];
Jingning Han's avatar
Jingning Han committed
236 237

  int skip;
238
  int skip_chroma_rd;
239 240
  int skip_cost[SKIP_CONTEXTS][2];

Zoe Liu's avatar
Zoe Liu committed
241 242 243 244 245 246 247 248 249 250
#if CONFIG_EXT_SKIP
  int skip_mode;  // 0: off; 1: on
  int skip_mode_cost[SKIP_CONTEXTS][2];

  int64_t skip_mode_rdcost;  // -1: Not set
  int skip_mode_rate;
  int64_t skip_mode_sse;
  int64_t skip_mode_dist;
  MV_REFERENCE_FRAME skip_mode_ref_frame[2];
  int_mv skip_mode_mv[2];
251 252 253
#if CONFIG_JNT_COMP
  int compound_idx;
#endif  // CONFIG_JNT_COMP
Zoe Liu's avatar
Zoe Liu committed
254 255 256 257
  int skip_mode_index_candidate;
  int skip_mode_index;
#endif  // CONFIG_EXT_SKIP

258 259
#if CONFIG_LV_MAP
  LV_MAP_COEFF_COST coeff_costs[TX_SIZES][PLANE_TYPES];
260
  LV_MAP_EOB_COST eob_costs[7][2];
261
  uint16_t cb_offset;
262 263
#endif

hui su's avatar
hui su committed
264 265
  av1_coeff_cost token_head_costs[TX_SIZES];
  av1_coeff_cost token_tail_costs[TX_SIZES];
Jingning Han's avatar
Jingning Han committed
266

267
  // mode costs
268 269
  int intra_inter_cost[INTRA_INTER_CONTEXTS][2];

270 271
  int mbmode_cost[BLOCK_SIZE_GROUPS][INTRA_MODES];
  int newmv_mode_cost[NEWMV_MODE_CONTEXTS][2];
Sarah Parker's avatar
Sarah Parker committed
272
  int zeromv_mode_cost[GLOBALMV_MODE_CONTEXTS][2];
273 274 275
  int refmv_mode_cost[REFMV_MODE_CONTEXTS][2];
  int drl_mode_cost0[DRL_MODE_CONTEXTS][2];

276
  int comp_inter_cost[COMP_INTER_CONTEXTS][2];
277
  int single_ref_cost[REF_CONTEXTS][SINGLE_REFS - 1][2];
278 279 280
#if CONFIG_EXT_COMP_REFS
  int comp_ref_type_cost[COMP_REF_TYPE_CONTEXTS]
                        [CDF_SIZE(COMP_REFERENCE_TYPES)];
281 282
  int uni_comp_ref_cost[UNI_COMP_REF_CONTEXTS][UNIDIR_COMP_REFS - 1]
                       [CDF_SIZE(2)];
283
#endif  // CONFIG_EXT_COMP_REFS
284 285 286 287 288 289
  // Cost for signaling ref_frame[0] (LAST_FRAME, LAST2_FRAME, LAST3_FRAME or
  // GOLDEN_FRAME) in bidir-comp mode.
  int comp_ref_cost[REF_CONTEXTS][FWD_REFS - 1][2];
  // Cost for signaling ref_frame[1] (ALTREF_FRAME, ALTREF2_FRAME, or
  // BWDREF_FRAME) in bidir-comp mode.
  int comp_bwdref_cost[REF_CONTEXTS][BWD_REFS - 1][2];
290
  int inter_compound_mode_cost[INTER_MODE_CONTEXTS][INTER_COMPOUND_MODES];
291 292 293
#if CONFIG_JNT_COMP
  int compound_type_cost[BLOCK_SIZES_ALL][COMPOUND_TYPES - 1];
#else
294
  int compound_type_cost[BLOCK_SIZES_ALL][COMPOUND_TYPES];
295
#endif  // CONFIG_JNT_COMP
296 297
  int interintra_cost[BLOCK_SIZE_GROUPS][2];
  int wedge_interintra_cost[BLOCK_SIZES_ALL][2];
298 299 300
  int interintra_mode_cost[BLOCK_SIZE_GROUPS][INTERINTRA_MODES];
  int motion_mode_cost[BLOCK_SIZES_ALL][MOTION_MODES];
  int motion_mode_cost1[BLOCK_SIZES_ALL][2];
301 302 303
#if CONFIG_CFL
  int intra_uv_mode_cost[CFL_ALLOWED_TYPES][INTRA_MODES][UV_INTRA_MODES];
#else
304
  int intra_uv_mode_cost[INTRA_MODES][UV_INTRA_MODES];
305
#endif
306
  int y_mode_costs[INTRA_MODES][INTRA_MODES][INTRA_MODES];
307
#if CONFIG_FILTER_INTRA
308
  int filter_intra_cost[TX_SIZES_ALL][2];
309
  int filter_intra_mode_cost[FILTER_INTRA_MODES];
310
#endif
311 312
  int switchable_interp_costs[SWITCHABLE_FILTER_CONTEXTS][SWITCHABLE_FILTERS];
#if CONFIG_EXT_PARTITION_TYPES
313
  int partition_cost[PARTITION_CONTEXTS][EXT_PARTITION_TYPES];
314
#else
315
  int partition_cost[PARTITION_CONTEXTS][PARTITION_TYPES];
316
#endif  // CONFIG_EXT_PARTITION_TYPES
Hui Su's avatar
Hui Su committed
317 318
  int palette_y_size_cost[PALATTE_BSIZE_CTXS][PALETTE_SIZES];
  int palette_uv_size_cost[PALATTE_BSIZE_CTXS][PALETTE_SIZES];
319 320 321 322
  int palette_y_color_cost[PALETTE_SIZES][PALETTE_COLOR_INDEX_CONTEXTS]
                          [PALETTE_COLORS];
  int palette_uv_color_cost[PALETTE_SIZES][PALETTE_COLOR_INDEX_CONTEXTS]
                           [PALETTE_COLORS];
Hui Su's avatar
Hui Su committed
323
  int palette_y_mode_cost[PALATTE_BSIZE_CTXS][PALETTE_Y_MODE_CONTEXTS][2];
324
  int palette_uv_mode_cost[PALETTE_UV_MODE_CONTEXTS][2];
325 326 327 328
#if CONFIG_CFL
  // The rate associated with each alpha codeword
  int cfl_cost[CFL_JOINT_SIGNS][CFL_PRED_PLANES][CFL_ALPHABET_SIZE];
#endif  // CONFIG_CFL
329
  int tx_size_cost[TX_SIZES - 1][TX_SIZE_CONTEXTS][TX_SIZES];
330
  int txfm_partition_cost[TXFM_PARTITION_CONTEXTS][2];
331 332 333
  int inter_tx_type_costs[EXT_TX_SETS_INTER][EXT_TX_SIZES][TX_TYPES];
  int intra_tx_type_costs[EXT_TX_SETS_INTRA][EXT_TX_SIZES][INTRA_MODES]
                         [TX_TYPES];
Hui Su's avatar
Hui Su committed
334
#if CONFIG_EXT_INTRA_MOD
Joe Young's avatar
Joe Young committed
335
  int angle_delta_cost[DIRECTIONAL_MODES][2 * MAX_ANGLE_DELTA + 1];
Hui Su's avatar
Hui Su committed
336
#endif  // CONFIG_EXT_INTRA_MOD
337 338
#if CONFIG_LOOP_RESTORATION
  int switchable_restore_cost[RESTORE_SWITCHABLE_TYPES];
339 340
  int wiener_restore_cost[2];
  int sgrproj_restore_cost[2];
341
#endif  // CONFIG_LOOP_RESTORATION
Hui Su's avatar
Hui Su committed
342 343 344
#if CONFIG_INTRABC
  int intrabc_cost[2];
#endif  // CONFIG_INTRABC
345

Jingning Han's avatar
Jingning Han committed
346 347 348
  int optimize;

  // Used to store sub partition's choices.
349
  MV pred_mv[TOTAL_REFS_PER_FRAME];
Jingning Han's avatar
Jingning Han committed
350

hui su's avatar
hui su committed
351 352
  // Store the best motion vector during motion search
  int_mv best_mv;
353 354
  // Store the second best motion vector during full-pixel motion search
  int_mv second_best_mv;
hui su's avatar
hui su committed
355

356 357
  // use default transform and skip transform type search for intra modes
  int use_default_intra_tx_type;
358 359
  // use default transform and skip transform type search for inter modes
  int use_default_inter_tx_type;
Yushin Cho's avatar
Yushin Cho committed
360
#if CONFIG_DIST_8X8
361 362
  int using_dist_8x8;
  aom_tune_metric tune_metric;
Yushin Cho's avatar
Yushin Cho committed
363
#endif  // CONFIG_DIST_8X8
Cheng Chen's avatar
Cheng Chen committed
364 365
#if CONFIG_JNT_COMP
  int comp_idx_cost[COMP_INDEX_CONTEXTS][2];
366
  int comp_group_idx_cost[COMP_GROUP_IDX_CONTEXTS][2];
Cheng Chen's avatar
Cheng Chen committed
367
#endif  // CONFIG_JNT_COMP
368 369
  // Bit flags for pruning tx type search, tx split, etc.
  int tx_search_prune[EXT_TX_SET_TYPES];
Jingning Han's avatar
Jingning Han committed
370 371
};

372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424
static INLINE int is_rect_tx_allowed_bsize(BLOCK_SIZE bsize) {
  static const char LUT[BLOCK_SIZES_ALL] = {
    0,  // BLOCK_4X4
    1,  // BLOCK_4X8
    1,  // BLOCK_8X4
    0,  // BLOCK_8X8
    1,  // BLOCK_8X16
    1,  // BLOCK_16X8
    0,  // BLOCK_16X16
    1,  // BLOCK_16X32
    1,  // BLOCK_32X16
    0,  // BLOCK_32X32
    1,  // BLOCK_32X64
    1,  // BLOCK_64X32
    0,  // BLOCK_64X64
#if CONFIG_EXT_PARTITION
    0,  // BLOCK_64X128
    0,  // BLOCK_128X64
    0,  // BLOCK_128X128
#endif  // CONFIG_EXT_PARTITION
    1,  // BLOCK_4X16
    1,  // BLOCK_16X4
    1,  // BLOCK_8X32
    1,  // BLOCK_32X8
    1,  // BLOCK_16X64
    1,  // BLOCK_64X16
#if CONFIG_EXT_PARTITION
    1,  // BLOCK_32X128
    1,  // BLOCK_128X32
#endif  // CONFIG_EXT_PARTITION
  };

  return LUT[bsize];
}

static INLINE int is_rect_tx_allowed(const MACROBLOCKD *xd,
                                     const MB_MODE_INFO *mbmi) {
  return is_rect_tx_allowed_bsize(mbmi->sb_type) &&
         !xd->lossless[mbmi->segment_id];
}

static INLINE int tx_size_to_depth(TX_SIZE tx_size, BLOCK_SIZE bsize,
                                   int is_inter) {
  TX_SIZE ctx_size = get_max_rect_tx_size(bsize, is_inter);
  int depth = 0;
  while (tx_size != ctx_size) {
    depth++;
    ctx_size = sub_tx_size_map[is_inter][ctx_size];
    assert(depth <= MAX_TX_DEPTH);
  }
  return depth;
}

Jingning Han's avatar
Jingning Han committed
425 426 427 428
#ifdef __cplusplus
}  // extern "C"
#endif

Yaowu Xu's avatar
Yaowu Xu committed
429
#endif  // AV1_ENCODER_BLOCK_H_