blockd.h 44.8 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4 5 6 7 8 9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10 11
 */

Yaowu Xu's avatar
Yaowu Xu committed
12 13
#ifndef AV1_COMMON_BLOCKD_H_
#define AV1_COMMON_BLOCKD_H_
Jingning Han's avatar
Jingning Han committed
14

Yaowu Xu's avatar
Yaowu Xu committed
15
#include "./aom_config.h"
Jingning Han's avatar
Jingning Han committed
16

Yaowu Xu's avatar
Yaowu Xu committed
17
#include "aom_dsp/aom_dsp_common.h"
18 19 20 21 22 23 24 25 26 27 28
#include "aom_ports/mem.h"
#include "aom_scale/yv12config.h"

#include "av1/common/common_data.h"
#include "av1/common/quant_common.h"
#include "av1/common/entropy.h"
#include "av1/common/entropymode.h"
#include "av1/common/mv.h"
#include "av1/common/scale.h"
#include "av1/common/seg_common.h"
#include "av1/common/tile_common.h"
Yushin Cho's avatar
Yushin Cho committed
29

Jingning Han's avatar
Jingning Han committed
30 31 32 33 34 35
#ifdef __cplusplus
extern "C" {
#endif

#define MAX_MB_PLANE 3

36 37 38 39 40
// Set COMPOUND_SEGMENT_TYPE to one of the three
// 0: Uniform
// 1: Difference weighted
#define COMPOUND_SEGMENT_TYPE 1
#define MAX_SEG_MASK_BITS 1
Yaowu Xu's avatar
Yaowu Xu committed
41

42 43
// SEG_MASK_TYPES should not surpass 1 << MAX_SEG_MASK_BITS
typedef enum {
Yaowu Xu's avatar
Yaowu Xu committed
44
#if COMPOUND_SEGMENT_TYPE == 0
45 46
  UNIFORM_45 = 0,
  UNIFORM_45_INV,
47
#elif COMPOUND_SEGMENT_TYPE == 1
48 49
  DIFFWTD_38 = 0,
  DIFFWTD_38_INV,
Yaowu Xu's avatar
Yaowu Xu committed
50
#endif  // COMPOUND_SEGMENT_TYPE
51 52 53
  SEG_MASK_TYPES,
} SEG_MASK_TYPE;

Jingning Han's avatar
Jingning Han committed
54 55 56
typedef enum {
  KEY_FRAME = 0,
  INTER_FRAME = 1,
57 58 59 60
#if CONFIG_OBU
  INTRA_ONLY_FRAME = 2,  // replaces intra-only
  S_FRAME = 3,
#endif
Jingning Han's avatar
Jingning Han committed
61 62 63
  FRAME_TYPES,
} FRAME_TYPE;

64 65 66 67 68
static INLINE int is_comp_ref_allowed(BLOCK_SIZE bsize) {
  (void)bsize;
  return AOMMIN(block_size_wide[bsize], block_size_high[bsize]) >= 8;
}

Jingning Han's avatar
Jingning Han committed
69
static INLINE int is_inter_mode(PREDICTION_MODE mode) {
70
  return mode >= NEARESTMV && mode <= NEW_NEWMV;
Yue Chen's avatar
Yue Chen committed
71 72
}

73 74 75 76 77
typedef struct {
  uint8_t *plane[MAX_MB_PLANE];
  int stride[MAX_MB_PLANE];
} BUFFER_SET;

78
static INLINE int is_inter_singleref_mode(PREDICTION_MODE mode) {
79
  return mode >= NEARESTMV && mode <= NEWMV;
80
}
81 82 83
static INLINE int is_inter_compound_mode(PREDICTION_MODE mode) {
  return mode >= NEAREST_NEARESTMV && mode <= NEW_NEWMV;
}
84

85
static INLINE PREDICTION_MODE compound_ref0_mode(PREDICTION_MODE mode) {
86
  static PREDICTION_MODE lut[] = {
87 88 89 90 91 92 93 94 95 96
    MB_MODE_COUNT,  // DC_PRED
    MB_MODE_COUNT,  // V_PRED
    MB_MODE_COUNT,  // H_PRED
    MB_MODE_COUNT,  // D45_PRED
    MB_MODE_COUNT,  // D135_PRED
    MB_MODE_COUNT,  // D117_PRED
    MB_MODE_COUNT,  // D153_PRED
    MB_MODE_COUNT,  // D207_PRED
    MB_MODE_COUNT,  // D63_PRED
    MB_MODE_COUNT,  // SMOOTH_PRED
97 98
    MB_MODE_COUNT,  // SMOOTH_V_PRED
    MB_MODE_COUNT,  // SMOOTH_H_PRED
Urvang Joshi's avatar
Urvang Joshi committed
99
    MB_MODE_COUNT,  // PAETH_PRED
100 101
    MB_MODE_COUNT,  // NEARESTMV
    MB_MODE_COUNT,  // NEARMV
Sarah Parker's avatar
Sarah Parker committed
102
    MB_MODE_COUNT,  // GLOBALMV
103
    MB_MODE_COUNT,  // NEWMV
104 105 106 107 108 109 110 111
    NEARESTMV,      // NEAREST_NEARESTMV
    NEARMV,         // NEAR_NEARMV
    NEARESTMV,      // NEAREST_NEWMV
    NEWMV,          // NEW_NEARESTMV
    NEARMV,         // NEAR_NEWMV
    NEWMV,          // NEW_NEARMV
    GLOBALMV,       // GLOBAL_GLOBALMV
    NEWMV,          // NEW_NEWMV
112
  };
113
  assert(NELEMENTS(lut) == MB_MODE_COUNT);
114 115 116 117 118
  assert(is_inter_compound_mode(mode));
  return lut[mode];
}

static INLINE PREDICTION_MODE compound_ref1_mode(PREDICTION_MODE mode) {
119
  static PREDICTION_MODE lut[] = {
120 121 122 123 124 125 126 127 128 129
    MB_MODE_COUNT,  // DC_PRED
    MB_MODE_COUNT,  // V_PRED
    MB_MODE_COUNT,  // H_PRED
    MB_MODE_COUNT,  // D45_PRED
    MB_MODE_COUNT,  // D135_PRED
    MB_MODE_COUNT,  // D117_PRED
    MB_MODE_COUNT,  // D153_PRED
    MB_MODE_COUNT,  // D207_PRED
    MB_MODE_COUNT,  // D63_PRED
    MB_MODE_COUNT,  // SMOOTH_PRED
130 131
    MB_MODE_COUNT,  // SMOOTH_V_PRED
    MB_MODE_COUNT,  // SMOOTH_H_PRED
Urvang Joshi's avatar
Urvang Joshi committed
132
    MB_MODE_COUNT,  // PAETH_PRED
133 134
    MB_MODE_COUNT,  // NEARESTMV
    MB_MODE_COUNT,  // NEARMV
Sarah Parker's avatar
Sarah Parker committed
135
    MB_MODE_COUNT,  // GLOBALMV
136
    MB_MODE_COUNT,  // NEWMV
137 138 139 140 141 142 143 144
    NEARESTMV,      // NEAREST_NEARESTMV
    NEARMV,         // NEAR_NEARMV
    NEWMV,          // NEAREST_NEWMV
    NEARESTMV,      // NEW_NEARESTMV
    NEWMV,          // NEAR_NEWMV
    NEARMV,         // NEW_NEARMV
    GLOBALMV,       // GLOBAL_GLOBALMV
    NEWMV,          // NEW_NEWMV
145
  };
146
  assert(NELEMENTS(lut) == MB_MODE_COUNT);
147 148 149 150
  assert(is_inter_compound_mode(mode));
  return lut[mode];
}

David Barker's avatar
David Barker committed
151
static INLINE int have_nearmv_in_inter_mode(PREDICTION_MODE mode) {
152 153
  return (mode == NEARMV || mode == NEAR_NEARMV || mode == NEAR_NEWMV ||
          mode == NEW_NEARMV);
David Barker's avatar
David Barker committed
154 155
}

Yue Chen's avatar
Yue Chen committed
156
static INLINE int have_newmv_in_inter_mode(PREDICTION_MODE mode) {
157 158
  return (mode == NEWMV || mode == NEW_NEWMV || mode == NEAREST_NEWMV ||
          mode == NEW_NEARESTMV || mode == NEAR_NEWMV || mode == NEW_NEARMV);
Jingning Han's avatar
Jingning Han committed
159
}
160

161 162 163 164
static INLINE int use_masked_motion_search(COMPOUND_TYPE type) {
  return (type == COMPOUND_WEDGE);
}

165
static INLINE int is_masked_compound_type(COMPOUND_TYPE type) {
166
  return (type == COMPOUND_WEDGE || type == COMPOUND_SEG);
167
}
168

Jingning Han's avatar
Jingning Han committed
169 170 171 172 173 174
/* For keyframes, intra block modes are predicted by the (already decoded)
   modes for the Y blocks to the left and above us; for interframes, there
   is a single probability table. */

typedef int8_t MV_REFERENCE_FRAME;

hui su's avatar
hui su committed
175 176 177
typedef struct {
  // Number of base colors for Y (0) and UV (1)
  uint8_t palette_size[2];
178
  // Value of base colors for Y, U, and V
hui su's avatar
hui su committed
179 180
  uint16_t palette_colors[3 * PALETTE_MAX_SIZE];
} PALETTE_MODE_INFO;
hui su's avatar
hui su committed
181

182
#if CONFIG_FILTER_INTRA
hui su's avatar
hui su committed
183
typedef struct {
184 185
  uint8_t use_filter_intra;
  FILTER_INTRA_MODE filter_intra_mode;
186
} FILTER_INTRA_MODE_INFO;
Yue Chen's avatar
Yue Chen committed
187 188

static const PREDICTION_MODE fimode_to_intradir[FILTER_INTRA_MODES] = {
189
  DC_PRED, V_PRED, H_PRED, D153_PRED, DC_PRED
Yue Chen's avatar
Yue Chen committed
190
};
191
#endif  // CONFIG_FILTER_INTRA
hui su's avatar
hui su committed
192

193
#if CONFIG_RD_DEBUG
Angie Chiang's avatar
Angie Chiang committed
194
#define TXB_COEFF_COST_MAP_SIZE (2 * MAX_MIB_SIZE)
195
#endif
Angie Chiang's avatar
Angie Chiang committed
196 197 198 199

typedef struct RD_STATS {
  int rate;
  int64_t dist;
Angie Chiang's avatar
Angie Chiang committed
200 201 202 203 204
  // Please be careful of using rdcost, it's not guaranteed to be set all the
  // time.
  // TODO(angiebird): Create a set of functions to manipulate the RD_STATS. In
  // these functions, make sure rdcost is always up-to-date according to
  // rate/dist.
Angie Chiang's avatar
Angie Chiang committed
205
  int64_t rdcost;
Angie Chiang's avatar
Angie Chiang committed
206
  int64_t sse;
Angie Chiang's avatar
Angie Chiang committed
207
  int skip;  // sse should equal to dist when skip == 1
208 209
  int64_t ref_rdcost;
  int zero_rate;
210
  uint8_t invalid_rate;
Angie Chiang's avatar
Angie Chiang committed
211 212 213 214
#if CONFIG_RD_DEBUG
  int txb_coeff_cost[MAX_MB_PLANE];
  int txb_coeff_cost_map[MAX_MB_PLANE][TXB_COEFF_COST_MAP_SIZE]
                        [TXB_COEFF_COST_MAP_SIZE];
215
#endif  // CONFIG_RD_DEBUG
Angie Chiang's avatar
Angie Chiang committed
216 217
} RD_STATS;

218 219
// This struct is used to group function args that are commonly
// sent together in functions related to interinter compound modes
220 221 222
typedef struct {
  int wedge_index;
  int wedge_sign;
223
  SEG_MASK_TYPE mask_type;
224 225
  uint8_t *seg_mask;
  COMPOUND_TYPE interinter_compound_type;
226 227
} INTERINTER_COMPOUND_DATA;

Jingning Han's avatar
Jingning Han committed
228
// This structure now relates to 8x8 block regions.
Luc Trudeau's avatar
Luc Trudeau committed
229
typedef struct MB_MODE_INFO {
Jingning Han's avatar
Jingning Han committed
230 231 232 233
  // Common for both INTER and INTRA blocks
  BLOCK_SIZE sb_type;
  PREDICTION_MODE mode;
  TX_SIZE tx_size;
234 235
  // TODO(jingning): This effectively assigned a separate entry for each
  // 8x8 block. Apparently it takes much more space than needed.
236
  TX_SIZE inter_tx_size[MAX_MIB_SIZE][MAX_MIB_SIZE];
237
  TX_SIZE min_tx_size;
Jingning Han's avatar
Jingning Han committed
238
  int8_t skip;
Zoe Liu's avatar
Zoe Liu committed
239 240 241
#if CONFIG_EXT_SKIP
  int8_t skip_mode;
#endif  // CONFIG_EXT_SKIP
Jingning Han's avatar
Jingning Han committed
242 243 244 245
  int8_t segment_id;
  int8_t seg_id_predicted;  // valid only when temporal_update is enabled

  // Only for INTRA blocks
Luc Trudeau's avatar
Luc Trudeau committed
246
  UV_PREDICTION_MODE uv_mode;
Urvang Joshi's avatar
Urvang Joshi committed
247

hui su's avatar
hui su committed
248
  PALETTE_MODE_INFO palette_mode_info;
Alex Converse's avatar
Alex Converse committed
249 250 251
#if CONFIG_INTRABC
  uint8_t use_intrabc;
#endif  // CONFIG_INTRABC
Jingning Han's avatar
Jingning Han committed
252

253 254
  // Only for INTER blocks
  InterpFilters interp_filters;
Jingning Han's avatar
Jingning Han committed
255
  MV_REFERENCE_FRAME ref_frame[2];
hui su's avatar
hui su committed
256
  TX_TYPE tx_type;
Angie Chiang's avatar
Angie Chiang committed
257
#if CONFIG_TXK_SEL
Angie Chiang's avatar
Angie Chiang committed
258 259
  TX_TYPE txk_type[MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)];
#endif
Jingning Han's avatar
Jingning Han committed
260

261 262 263
#if CONFIG_FILTER_INTRA
  FILTER_INTRA_MODE_INFO filter_intra_mode_info;
#endif  // CONFIG_FILTER_INTRA
hui su's avatar
hui su committed
264
#if CONFIG_EXT_INTRA
hui su's avatar
hui su committed
265
  // The actual prediction angle is the base angle + (angle_delta * step).
hui su's avatar
hui su committed
266
  int8_t angle_delta[2];
hui su's avatar
hui su committed
267 268
#endif  // CONFIG_EXT_INTRA

269
  // interintra members
270
  INTERINTRA_MODE interintra_mode;
271 272 273
  // TODO(debargha): Consolidate these flags
  int use_wedge_interintra;
  int interintra_wedge_index;
Debargha Mukherjee's avatar
Debargha Mukherjee committed
274
  int interintra_wedge_sign;
275 276 277 278 279
  // interinter members
  COMPOUND_TYPE interinter_compound_type;
  int wedge_index;
  int wedge_sign;
  SEG_MASK_TYPE mask_type;
Yue Chen's avatar
Yue Chen committed
280
  MOTION_MODE motion_mode;
281
  int overlappable_neighbors[2];
Jingning Han's avatar
Jingning Han committed
282
  int_mv mv[2];
283
  int_mv pred_mv[2];
284
  uint8_t ref_mv_idx;
285 286 287
#if CONFIG_EXT_PARTITION_TYPES
  PARTITION_TYPE partition;
#endif
288 289 290 291
#if CONFIG_NEW_QUANT
  int dq_off_index;
  int send_dq_bit;
#endif  // CONFIG_NEW_QUANT
Yaowu Xu's avatar
Yaowu Xu committed
292
  /* deringing gain *per-superblock* */
293
  int8_t cdef_strength;
294
  int current_q_index;
Fangwen Fu's avatar
Fangwen Fu committed
295 296
#if CONFIG_EXT_DELTA_Q
  int current_delta_lf_from_base;
297 298 299
#if CONFIG_LOOPFILTER_LEVEL
  int curr_delta_lf[FRAME_LF_COUNT];
#endif  // CONFIG_LOOPFILTER_LEVEL
Fangwen Fu's avatar
Fangwen Fu committed
300
#endif
301
#if CONFIG_RD_DEBUG
Angie Chiang's avatar
Angie Chiang committed
302
  RD_STATS rd_stats;
303 304 305
  int mi_row;
  int mi_col;
#endif
Yue Chen's avatar
Yue Chen committed
306 307
  int num_proj_ref[2];
  WarpedMotionParams wm_params[2];
308

Luc Trudeau's avatar
Luc Trudeau committed
309 310
#if CONFIG_CFL
  // Index of the alpha Cb and alpha Cr combination
311
  int cfl_alpha_idx;
312 313
  // Joint sign of alpha Cb and alpha Cr
  int cfl_alpha_signs;
Luc Trudeau's avatar
Luc Trudeau committed
314 315
#endif

316
  BOUNDARY_TYPE boundary_info;
317 318 319

#if CONFIG_JNT_COMP
  int compound_idx;
320
  int comp_group_idx;
321
#endif
Jingning Han's avatar
Jingning Han committed
322 323
} MB_MODE_INFO;

324
typedef struct MODE_INFO { MB_MODE_INFO mbmi; } MODE_INFO;
Jingning Han's avatar
Jingning Han committed
325

Alex Converse's avatar
Alex Converse committed
326
#if CONFIG_INTRABC
327 328
#define NO_FILTER_FOR_IBC 1  // Disable in-loop filters for frame with intrabc

Alex Converse's avatar
Alex Converse committed
329 330 331 332 333
static INLINE int is_intrabc_block(const MB_MODE_INFO *mbmi) {
  return mbmi->use_intrabc;
}
#endif

Luc Trudeau's avatar
Luc Trudeau committed
334 335
#if CONFIG_CFL
static INLINE PREDICTION_MODE get_uv_mode(UV_PREDICTION_MODE mode) {
Luc Trudeau's avatar
Luc Trudeau committed
336
  assert(mode < UV_INTRA_MODES);
Luc Trudeau's avatar
Luc Trudeau committed
337
  static const PREDICTION_MODE uv2y[UV_INTRA_MODES] = {
338 339 340 341 342 343 344 345 346 347
    DC_PRED,        // UV_DC_PRED
    V_PRED,         // UV_V_PRED
    H_PRED,         // UV_H_PRED
    D45_PRED,       // UV_D45_PRED
    D135_PRED,      // UV_D135_PRED
    D117_PRED,      // UV_D117_PRED
    D153_PRED,      // UV_D153_PRED
    D207_PRED,      // UV_D207_PRED
    D63_PRED,       // UV_D63_PRED
    SMOOTH_PRED,    // UV_SMOOTH_PRED
Luc Trudeau's avatar
Luc Trudeau committed
348 349
    SMOOTH_V_PRED,  // UV_SMOOTH_V_PRED
    SMOOTH_H_PRED,  // UV_SMOOTH_H_PRED
Urvang Joshi's avatar
Urvang Joshi committed
350
    PAETH_PRED,     // UV_PAETH_PRED
351
    DC_PRED,        // CFL_PRED
Luc Trudeau's avatar
Luc Trudeau committed
352 353 354 355 356 357 358
  };
  return uv2y[mode];
}
#else
static INLINE PREDICTION_MODE get_uv_mode(PREDICTION_MODE mode) { return mode; }
#endif  // CONFIG_CFL

Jingning Han's avatar
Jingning Han committed
359
static INLINE int is_inter_block(const MB_MODE_INFO *mbmi) {
Alex Converse's avatar
Alex Converse committed
360 361 362
#if CONFIG_INTRABC
  if (is_intrabc_block(mbmi)) return 1;
#endif
Jingning Han's avatar
Jingning Han committed
363 364 365 366 367 368 369
  return mbmi->ref_frame[0] > INTRA_FRAME;
}

static INLINE int has_second_ref(const MB_MODE_INFO *mbmi) {
  return mbmi->ref_frame[1] > INTRA_FRAME;
}

Zoe Liu's avatar
Zoe Liu committed
370 371 372 373 374 375 376 377 378
#if CONFIG_EXT_COMP_REFS
static INLINE int has_uni_comp_refs(const MB_MODE_INFO *mbmi) {
  return has_second_ref(mbmi) && (!((mbmi->ref_frame[0] >= BWDREF_FRAME) ^
                                    (mbmi->ref_frame[1] >= BWDREF_FRAME)));
}

static INLINE MV_REFERENCE_FRAME comp_ref0(int ref_idx) {
  static const MV_REFERENCE_FRAME lut[] = {
    LAST_FRAME,    // LAST_LAST2_FRAMES,
379
    LAST_FRAME,    // LAST_LAST3_FRAMES,
Zoe Liu's avatar
Zoe Liu committed
380 381 382 383 384 385 386 387 388 389
    LAST_FRAME,    // LAST_GOLDEN_FRAMES,
    BWDREF_FRAME,  // BWDREF_ALTREF_FRAMES,
  };
  assert(NELEMENTS(lut) == UNIDIR_COMP_REFS);
  return lut[ref_idx];
}

static INLINE MV_REFERENCE_FRAME comp_ref1(int ref_idx) {
  static const MV_REFERENCE_FRAME lut[] = {
    LAST2_FRAME,   // LAST_LAST2_FRAMES,
390
    LAST3_FRAME,   // LAST_LAST3_FRAMES,
Zoe Liu's avatar
Zoe Liu committed
391 392 393 394 395 396 397 398
    GOLDEN_FRAME,  // LAST_GOLDEN_FRAMES,
    ALTREF_FRAME,  // BWDREF_ALTREF_FRAMES,
  };
  assert(NELEMENTS(lut) == UNIDIR_COMP_REFS);
  return lut[ref_idx];
}
#endif  // CONFIG_EXT_COMP_REFS

399
PREDICTION_MODE av1_left_block_mode(const MODE_INFO *left_mi);
Jingning Han's avatar
Jingning Han committed
400

401
PREDICTION_MODE av1_above_block_mode(const MODE_INFO *above_mi);
Jingning Han's avatar
Jingning Han committed
402

Luc Trudeau's avatar
Luc Trudeau committed
403
static INLINE int is_global_mv_block(const MODE_INFO *mi,
404
                                     TransformationType type) {
Luc Trudeau's avatar
Luc Trudeau committed
405 406
  const MB_MODE_INFO *const mbmi = &mi->mbmi;
  const PREDICTION_MODE mode = mbmi->mode;
407 408 409
#if GLOBAL_SUB8X8_USED
  const int block_size_allowed = 1;
#else
Luc Trudeau's avatar
Luc Trudeau committed
410
  const BLOCK_SIZE bsize = mbmi->sb_type;
411 412
  const int block_size_allowed =
      AOMMIN(block_size_wide[bsize], block_size_high[bsize]) >= 8;
413
#endif  // GLOBAL_SUB8X8_USED
Sarah Parker's avatar
Sarah Parker committed
414
  return (mode == GLOBALMV || mode == GLOBAL_GLOBALMV) && type > TRANSLATION &&
415 416 417
         block_size_allowed;
}

Angie Chiang's avatar
Angie Chiang committed
418 419 420 421 422 423 424 425 426 427 428
#if CONFIG_MISMATCH_DEBUG
static INLINE void mi_to_pixel_loc(int *pixel_c, int *pixel_r, int mi_col,
                                   int mi_row, int tx_blk_col, int tx_blk_row,
                                   int subsampling_x, int subsampling_y) {
  *pixel_c = ((mi_col >> subsampling_x) << MI_SIZE_LOG2) +
             (tx_blk_col << tx_size_wide_log2[0]);
  *pixel_r = ((mi_row >> subsampling_y) << MI_SIZE_LOG2) +
             (tx_blk_row << tx_size_high_log2[0]);
}
#endif

clang-format's avatar
clang-format committed
429
enum mv_precision { MV_PRECISION_Q3, MV_PRECISION_Q4 };
Jingning Han's avatar
Jingning Han committed
430 431 432

struct buf_2d {
  uint8_t *buf;
433 434 435
  uint8_t *buf0;
  int width;
  int height;
Jingning Han's avatar
Jingning Han committed
436 437 438
  int stride;
};

Angie Chiang's avatar
Angie Chiang committed
439
typedef struct macroblockd_plane {
Jingning Han's avatar
Jingning Han committed
440 441 442 443 444 445 446 447
  tran_low_t *dqcoeff;
  PLANE_TYPE plane_type;
  int subsampling_x;
  int subsampling_y;
  struct buf_2d dst;
  struct buf_2d pre[2];
  ENTROPY_CONTEXT *above_context;
  ENTROPY_CONTEXT *left_context;
448 449 450 451 452

  // The dequantizers below are true dequntizers used only in the
  // dequantization process.  They have the same coefficient
  // shift/scale as TX.
  int16_t seg_dequant_QTX[MAX_SEGMENTS][2];
453
#if CONFIG_NEW_QUANT
454
  dequant_val_type_nuq seg_dequant_nuq_QTX[MAX_SEGMENTS][QUANT_PROFILES][2];
455
#endif
hui su's avatar
hui su committed
456
  uint8_t *color_index_map;
Jingning Han's avatar
Jingning Han committed
457

458 459
  // block size in pixels
  uint8_t width, height;
Jingning Han's avatar
Jingning Han committed
460

461
#if CONFIG_AOM_QM
462 463
  qm_val_t *seg_iqmatrix[MAX_SEGMENTS][TX_SIZES_ALL];
  qm_val_t *seg_qmatrix[MAX_SEGMENTS][TX_SIZES_ALL];
464
#endif
465 466 467 468 469 470

  // the 'dequantizers' below are not literal dequantizer values.
  // They're used by encoder RDO to generate ad-hoc lambda values.
  // They use a hardwired Q3 coeff shift and do not necessarily match
  // the TX scale in use.
  const int16_t *dequant_Q3;
471

Yushin Cho's avatar
Yushin Cho committed
472
#if CONFIG_DIST_8X8
473
  DECLARE_ALIGNED(16, int16_t, pred[MAX_SB_SQUARE]);
Yushin Cho's avatar
Yushin Cho committed
474
#endif
Angie Chiang's avatar
Angie Chiang committed
475
} MACROBLOCKD_PLANE;
Jingning Han's avatar
Jingning Han committed
476

477 478
#define BLOCK_OFFSET(x, i) \
  ((x) + (i) * (1 << (tx_size_wide_log2[0] + tx_size_high_log2[0])))
Jingning Han's avatar
Jingning Han committed
479 480 481 482 483 484 485

typedef struct RefBuffer {
  int idx;
  YV12_BUFFER_CONFIG *buf;
  struct scale_factors sf;
} RefBuffer;

486 487 488 489 490 491 492 493 494 495 496 497
#if CONFIG_LOOP_RESTORATION
typedef struct {
  DECLARE_ALIGNED(16, InterpKernel, vfilter);
  DECLARE_ALIGNED(16, InterpKernel, hfilter);
} WienerInfo;

typedef struct {
  int ep;
  int xqd[2];
} SgrprojInfo;
#endif  // CONFIG_LOOP_RESTORATION

498
#if CONFIG_CFL
499
#if CONFIG_DEBUG
500 501 502
#define CFL_SUB8X8_VAL_MI_SIZE (4)
#define CFL_SUB8X8_VAL_MI_SQUARE \
  (CFL_SUB8X8_VAL_MI_SIZE * CFL_SUB8X8_VAL_MI_SIZE)
503
#endif  // CONFIG_DEBUG
504
#define CFL_MAX_BLOCK_SIZE (BLOCK_32X32)
505 506
#define CFL_BUF_LINE (32)
#define CFL_BUF_SQUARE (CFL_BUF_LINE * CFL_BUF_LINE)
507
typedef struct cfl_ctx {
Luc Trudeau's avatar
Luc Trudeau committed
508 509 510 511
  // The CfL prediction buffer is used in two steps:
  //   1. Stores Q3 reconstructed luma pixels
  //      (only Q2 is required, but Q3 is used to avoid shifts)
  //   2. Stores Q3 AC contributions (step1 - tx block avg)
512 513 514 515 516 517 518 519 520
  int16_t pred_buf_q3[CFL_BUF_SQUARE];

  // Cache the DC_PRED when performing RDO, so it does not have to be recomputed
  // for every scaling parameter
  int dc_pred_is_cached[CFL_PRED_PLANES];
  // The DC_PRED cache is disable when decoding
  int use_dc_pred_cache;
  // Only cache the first row of the DC_PRED
  int16_t dc_pred_cache[CFL_PRED_PLANES][CFL_BUF_LINE];
Luc Trudeau's avatar
Luc Trudeau committed
521 522 523

  // Height and width currently used in the CfL prediction buffer.
  int buf_height, buf_width;
524 525 526 527 528 529 530 531 532 533 534

  int are_parameters_computed;

  // Chroma subsampling
  int subsampling_x, subsampling_y;

  int mi_row, mi_col;

  // Whether the reconstructed luma pixels need to be stored
  int store_y;

535 536 537 538
#if CONFIG_DEBUG
  int rate;
#endif  // CONFIG_DEBUG

539
  int is_chroma_reference;
540
#if CONFIG_DEBUG
541 542 543 544 545 546
  // Validation buffer is usually 2x2, except for 16x4 and 4x16 in that case it
  // is 4x2 and 2x4 respectively. To simplify accessing the buffer we use a
  // stride of CFL_SUB8X8_VAL_MI_SIZE resulting in a square of 16.
  uint16_t sub8x8_val[CFL_SUB8X8_VAL_MI_SQUARE];
  uint16_t store_counter;
  uint16_t last_compute_counter;
547
#endif  // CONFIG_DEBUG
548 549 550
} CFL_CTX;
#endif  // CONFIG_CFL

Cheng Chen's avatar
Cheng Chen committed
551 552
#if CONFIG_JNT_COMP
typedef struct jnt_comp_params {
Cheng Chen's avatar
Cheng Chen committed
553
  int use_jnt_comp_avg;
Cheng Chen's avatar
Cheng Chen committed
554 555 556 557 558
  int fwd_offset;
  int bck_offset;
} JNT_COMP_PARAMS;
#endif  // CONFIG_JNT_COMP

Jingning Han's avatar
Jingning Han committed
559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576
typedef struct macroblockd {
  struct macroblockd_plane plane[MAX_MB_PLANE];
  uint8_t bmode_blocks_wl;
  uint8_t bmode_blocks_hl;

  FRAME_COUNTS *counts;
  TileInfo tile;

  int mi_stride;

  MODE_INFO **mi;
  MODE_INFO *left_mi;
  MODE_INFO *above_mi;
  MB_MODE_INFO *left_mbmi;
  MB_MODE_INFO *above_mbmi;

  int up_available;
  int left_available;
577 578
  int chroma_up_available;
  int chroma_left_available;
Jingning Han's avatar
Jingning Han committed
579

580
  /* Distance of MB away from frame edges in subpixels (1/8th pixel)  */
Jingning Han's avatar
Jingning Han committed
581 582 583 584 585 586 587 588
  int mb_to_left_edge;
  int mb_to_right_edge;
  int mb_to_top_edge;
  int mb_to_bottom_edge;

  FRAME_CONTEXT *fc;

  /* pointers to reference frames */
589
  const RefBuffer *block_refs[2];
Jingning Han's avatar
Jingning Han committed
590 591 592 593

  /* pointer to current frame */
  const YV12_BUFFER_CONFIG *cur_buf;

594 595 596 597 598
#if CONFIG_INTRABC
  /* Scale of the current frame with respect to itself */
  struct scale_factors sf_identity;
#endif

Jingning Han's avatar
Jingning Han committed
599
  ENTROPY_CONTEXT *above_context[MAX_MB_PLANE];
600
  ENTROPY_CONTEXT left_context[MAX_MB_PLANE][2 * MAX_MIB_SIZE];
Jingning Han's avatar
Jingning Han committed
601 602

  PARTITION_CONTEXT *above_seg_context;
603
  PARTITION_CONTEXT left_seg_context[MAX_MIB_SIZE];
Jingning Han's avatar
Jingning Han committed
604

605 606
  TXFM_CONTEXT *above_txfm_context;
  TXFM_CONTEXT *left_txfm_context;
607
  TXFM_CONTEXT left_txfm_context_buffer[2 * MAX_MIB_SIZE];
608

609 610 611 612 613
#if CONFIG_LOOP_RESTORATION
  WienerInfo wiener_info[MAX_MB_PLANE];
  SgrprojInfo sgrproj_info[MAX_MB_PLANE];
#endif  // CONFIG_LOOP_RESTORATION

614
  // block dimension in the unit of mode_info.
615 616
  uint8_t n8_w, n8_h;

617 618
  uint8_t ref_mv_count[MODE_CTX_REF_FRAMES];
  CANDIDATE_MV ref_mv_stack[MODE_CTX_REF_FRAMES][MAX_REF_MV_STACK_SIZE];
619
  uint8_t is_sec_rect;
620

621
  FRAME_CONTEXT *tile_ctx;
Jingning Han's avatar
Jingning Han committed
622 623 624
  /* Bit depth: 8, 10, 12 */
  int bd;

625
  int qindex[MAX_SEGMENTS];
626
  int lossless[MAX_SEGMENTS];
Jingning Han's avatar
Jingning Han committed
627
  int corrupted;
RogerZhou's avatar
RogerZhou committed
628
#if CONFIG_AMVR
629
  int cur_frame_force_integer_mv;
RogerZhou's avatar
RogerZhou committed
630 631
// same with that in AV1_COMMON
#endif
Yaowu Xu's avatar
Yaowu Xu committed
632
  struct aom_internal_error_info *error_info;
633
  const WarpedMotionParams *global_motion;
634 635 636
  int prev_qindex;
  int delta_qindex;
  int current_qindex;
Fangwen Fu's avatar
Fangwen Fu committed
637 638 639 640 641 642 643 644 645
#if CONFIG_EXT_DELTA_Q
  // Since actual frame level loop filtering level value is not available
  // at the beginning of the tile (only available during actual filtering)
  // at encoder side.we record the delta_lf (against the frame level loop
  // filtering level) and code the delta between previous superblock's delta
  // lf and current delta lf. It is equivalent to the delta between previous
  // superblock's actual lf and current lf.
  int prev_delta_lf_from_base;
  int current_delta_lf_from_base;
646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662
#if CONFIG_LOOPFILTER_LEVEL
  // For this experiment, we have four frame filter levels for different plane
  // and direction. So, to support the per superblock update, we need to add
  // a few more params as below.
  // 0: delta loop filter level for y plane vertical
  // 1: delta loop filter level for y plane horizontal
  // 2: delta loop filter level for u plane
  // 3: delta loop filter level for v plane
  // To make it consistent with the reference to each filter level in segment,
  // we need to -1, since
  // SEG_LVL_ALT_LF_Y_V = 1;
  // SEG_LVL_ALT_LF_Y_H = 2;
  // SEG_LVL_ALT_LF_U   = 3;
  // SEG_LVL_ALT_LF_V   = 4;
  int prev_delta_lf[FRAME_LF_COUNT];
  int curr_delta_lf[FRAME_LF_COUNT];
#endif  // CONFIG_LOOPFILTER_LEVEL
663
#endif
Luc Trudeau's avatar
Luc Trudeau committed
664

665 666
  DECLARE_ALIGNED(16, uint8_t, seg_mask[2 * MAX_SB_SQUARE]);

Luc Trudeau's avatar
Luc Trudeau committed
667
#if CONFIG_CFL
668
  CFL_CTX cfl;
Luc Trudeau's avatar
Luc Trudeau committed
669
#endif
Cheng Chen's avatar
Cheng Chen committed
670 671 672 673

#if CONFIG_JNT_COMP
  JNT_COMP_PARAMS jcp_param;
#endif
674 675 676 677

#if CONFIG_EXT_SKIP
  int all_one_sided_refs;
#endif  // CONFIG_EXT_SKIP
Jingning Han's avatar
Jingning Han committed
678 679
} MACROBLOCKD;

680 681 682 683
static INLINE int get_bitdepth_data_path_index(const MACROBLOCKD *xd) {
  return xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH ? 1 : 0;
}

Jingning Han's avatar
Jingning Han committed
684 685
static INLINE BLOCK_SIZE get_subsize(BLOCK_SIZE bsize,
                                     PARTITION_TYPE partition) {
686
  if (partition == PARTITION_INVALID)
687
    return BLOCK_INVALID;
688 689
  else
    return subsize_lookup[partition][bsize];
690 691
}

692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712
static TX_TYPE intra_mode_to_tx_type_context(const MB_MODE_INFO *mbmi,
                                             PLANE_TYPE plane_type) {
  static const TX_TYPE _intra_mode_to_tx_type_context[INTRA_MODES] = {
    DCT_DCT,    // DC
    ADST_DCT,   // V
    DCT_ADST,   // H
    DCT_DCT,    // D45
    ADST_ADST,  // D135
    ADST_DCT,   // D117
    DCT_ADST,   // D153
    DCT_ADST,   // D207
    ADST_DCT,   // D63
    ADST_ADST,  // SMOOTH
    ADST_DCT,   // SMOOTH_V
    DCT_ADST,   // SMOOTH_H
    ADST_ADST,  // PAETH
  };
  return plane_type == PLANE_TYPE_Y
             ? _intra_mode_to_tx_type_context[mbmi->mode]
             : _intra_mode_to_tx_type_context[get_uv_mode(mbmi->uv_mode)];
}
Jingning Han's avatar
Jingning Han committed
713

714
#define USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4 1
715

716 717
static INLINE int is_rect_tx(TX_SIZE tx_size) { return tx_size >= TX_SIZES; }

718 719 720 721
static INLINE int block_signals_txsize(BLOCK_SIZE bsize) {
  return bsize > BLOCK_4X4;
}

clang-format's avatar
clang-format committed
722
#define ALLOW_INTRA_EXT_TX 1
723

Hui Su's avatar
Hui Su committed
724 725
// Number of transform types in each set type
static const int av1_num_ext_tx_set[EXT_TX_SET_TYPES] = {
726
  1, 2, 5, 7, 7, 10, 12, 16, 16,
Sarah Parker's avatar
Sarah Parker committed
727 728
};

729 730 731 732
static const int av1_ext_tx_set_idx_to_type[2][AOMMAX(EXT_TX_SETS_INTRA,
                                                      EXT_TX_SETS_INTER)] = {
  {
      // Intra
733 734 735 736 737 738
      EXT_TX_SET_DCTONLY, EXT_TX_SET_DTT4_IDTX_1DDCT,
#if EXT_TX_16X16_SET == 0
      EXT_TX_SET_DTT4_IDTX_1DDCT_16X16,
#else
      EXT_TX_SET_DTT4_IDTX,
#endif  // EXT_TX_16X16_SET
739 740 741
  },
  {
      // Inter
742
      EXT_TX_SET_DCTONLY, EXT_TX_SET_ALL16,
743 744 745
#if EXT_TX_16X16_SET == 0
      EXT_TX_SET_ALL16_16X16,
#elif EXT_TX_16X16_SET == 1
746 747 748
      EXT_TX_SET_DTT9_IDTX_1DDCT,
#else
      EXT_TX_SET_DTT9_IDTX,
749
#endif  // EXT_TX_16X16_SET
750
      EXT_TX_SET_DCT_IDTX,
751
  },
752
};
753

Hui Su's avatar
Hui Su committed
754 755 756 757 758 759 760 761 762 763 764 765 766
static const int av1_ext_tx_used[EXT_TX_SET_TYPES][TX_TYPES] = {
  {
      1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
  },
  {
      1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0,
  },
  {
      1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0,
  },
  {
      1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0,
  },
767 768 769
  {
      1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0,
  },
770 771 772
  {
      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,
  },
Hui Su's avatar
Hui Su committed
773 774 775 776 777 778
  {
      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0,
  },
  {
      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  },
779 780 781
  {
      1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
  },
782 783 784
};

static INLINE TxSetType get_ext_tx_set_type(TX_SIZE tx_size, BLOCK_SIZE bs,
785
                                            int is_inter, int use_reduced_set) {
786 787
  const TX_SIZE tx_size_sqr_up = txsize_sqr_up_map[tx_size];
  const TX_SIZE tx_size_sqr = txsize_sqr_map[tx_size];
788
#if USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4
789
  (void)bs;
790
  if (tx_size_sqr_up > TX_32X32) return EXT_TX_SET_DCTONLY;
791
#else
792
  if (tx_size_sqr_up > TX_32X32 || bs < BLOCK_8X8) return EXT_TX_SET_DCTONLY;
793
#endif  // USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4
794 795
  if (use_reduced_set)
    return is_inter ? EXT_TX_SET_DCT_IDTX : EXT_TX_SET_DTT4_IDTX;
796
#if CONFIG_DAALA_TX_DST32
797 798 799 800 801 802 803 804 805
  if (tx_size_sqr_up > TX_32X32)
    return is_inter ? EXT_TX_SET_DCT_IDTX : EXT_TX_SET_DCTONLY;
  if (is_inter)
    return (tx_size_sqr >= TX_16X16 ? EXT_TX_SET_DTT9_IDTX_1DDCT
                                    : EXT_TX_SET_ALL16);
  else
    return (tx_size_sqr >= TX_16X16 ? EXT_TX_SET_DTT4_IDTX
                                    : EXT_TX_SET_DTT4_IDTX_1DDCT);
#endif
806
  if (tx_size_sqr_up == TX_32X32)
807 808
    return is_inter ? EXT_TX_SET_DCT_IDTX : EXT_TX_SET_DCTONLY;
  if (is_inter)
809 810 811 812 813
    return (tx_size_sqr == TX_16X16 ?
#if EXT_TX_16X16_SET == 0
                                    EXT_TX_SET_ALL16_16X16
#elif EXT_TX_16X16_SET == 1
                                    EXT_TX_SET_DTT9_IDTX_1DDCT
814
#else
815 816 817
                                    EXT_TX_SET_DTT9_IDTX
#endif  // EXT_TX_16X16_SET
                                    : EXT_TX_SET_ALL16);
818
  else
819 820 821 822 823 824
    return (tx_size_sqr == TX_16X16 ?
#if EXT_TX_16X16_SET == 0
                                    EXT_TX_SET_DTT4_IDTX_1DDCT_16X16
#else
                                    EXT_TX_SET_DTT4_IDTX
#endif  // EXT_TX_16X16_SET
825
                                    : EXT_TX_SET_DTT4_IDTX_1DDCT);
826 827
}

828 829 830 831
// Maps tx set types to the indices.
static const int ext_tx_set_index[2][EXT_TX_SET_TYPES] = {
  {
      // Intra
832 833 834 835 836 837 838
      0, -1,
#if EXT_TX_16X16_SET == 0
      -1, 2,
#else
      2, -1,
#endif  // EXT_TX_16X16_SET
      1, -1, -1, -1, -1,
839 840 841
  },
  {
      // Inter
842 843 844 845 846
      0, 3, -1, -1, -1,
#if EXT_TX_16X16_SET == 0
      -1, -1, 2,
#elif EXT_TX_16X16_SET == 1
      -1, 2, -1,
847
#else
848 849
      2, -1, -1,
#endif  // EXT_TX_16X16_SET
850
      1,
851 852 853
  },
};

854
static INLINE int get_ext_tx_set(TX_SIZE tx_size, BLOCK_SIZE bs, int is_inter,
855
                                 int use_reduced_set) {
856
  const TxSetType set_type =
857
      get_ext_tx_set_type(tx_size, bs, is_inter, use_reduced_set);
858
  return ext_tx_set_index[is_inter][set_type];
859 860
}

861
static INLINE int get_ext_tx_types(TX_SIZE tx_size, BLOCK_SIZE bs, int is_inter,
862 863 864
                                   int use_reduced_set) {
  const int set_type =
      get_ext_tx_set_type(tx_size, bs, is_inter, use_reduced_set);
Hui Su's avatar
Hui Su committed
865
  return av1_num_ext_tx_set[set_type];
866
}
867 868

static INLINE int is_rect_tx_allowed_bsize(BLOCK_SIZE bsize) {
869
  static const char LUT[BLOCK_SIZES_ALL] = {
870 871 872 873 874 875 876 877 878 879
    0,  // BLOCK_4X4
    1,  // BLOCK_4X8
    1,  // BLOCK_8X4
    0,  // BLOCK_8X8
    1,  // BLOCK_8X16
    1,  // BLOCK_16X8
    0,  // BLOCK_16X16
    1,  // BLOCK_16X32
    1,  // BLOCK_32X16
    0,  // BLOCK_32X32
880 881
    1,  // BLOCK_32X64
    1,  // BLOCK_64X32
882 883 884 885 886 887
    0,  // BLOCK_64X64
#if CONFIG_EXT_PARTITION
    0,  // BLOCK_64X128
    0,  // BLOCK_128X64
    0,  // BLOCK_128X128
#endif  // CONFIG_EXT_PARTITION
888 889 890 891 892 893
    1,  // BLOCK_4X16
    1,  // BLOCK_16X4
    1,  // BLOCK_8X32
    1,  // BLOCK_32X8
    1,  // BLOCK_16X64
    1,  // BLOCK_64X16
894
#if CONFIG_EXT_PARTITION
895 896
    1,  // BLOCK_32X128
    1,  // BLOCK_128X32
897
#endif  // CONFIG_EXT_PARTITION
898 899 900 901 902
  };

  return LUT[bsize];
}

903 904 905
#define TXSIZEMAX(t1, t2) (tx_size_2d[(t1)] >= tx_size_2d[(t2)] ? (t1) : (t2))
#define TXSIZEMIN(t1, t2) (tx_size_2d[(t1)] <= tx_size_2d[(t2)] ? (t1) : (t2))

906 907
static INLINE int is_rect_tx_allowed(const MACROBLOCKD *xd,
                                     const MB_MODE_INFO *mbmi) {
908
  return is_rect_tx_allowed_bsize(mbmi->sb_type) &&
909
         !xd->lossless[mbmi->segment_id];
910
}
911

912
static INLINE TX_SIZE get_max_rect_tx_size(BLOCK_SIZE bsize, int is_inter) {
913
  return max_txsize_rect_lookup[is_inter][bsize];
914 915
}

916 917 918
static INLINE TX_SIZE tx_size_from_tx_mode(BLOCK_SIZE bsize, TX_MODE tx_mode,
                                           int is_inter) {
  const TX_SIZE largest_tx_size = tx_mode_to_biggest_tx_size[tx_mode];
919
  const TX_SIZE max_rect_tx_size = get_max_rect_tx_size(bsize, is_inter);
920 921
  (void)is_inter;
  if (bsize == BLOCK_4X4)
922
    return AOMMIN(max_txsize_lookup[bsize], largest_tx_size);
923 924
  if (txsize_sqr_map[max_rect_tx_size] <= largest_tx_size)
    return max_rect_tx_size;
925 926
  else
    return largest_tx_size;
927 928
}

929 930
#if CONFIG_EXT_INTRA
extern const int16_t dr_intra_derivative[90];
931
static const uint8_t mode_to_angle_map[] = {
932
  0, 90, 180, 45, 135, 113, 157, 203, 67, 0, 0, 0, 0,
933