blockd.h 41.1 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4
5
6
7
8
9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10
11
 */

Yaowu Xu's avatar
Yaowu Xu committed
12
13
#ifndef AV1_COMMON_BLOCKD_H_
#define AV1_COMMON_BLOCKD_H_
Jingning Han's avatar
Jingning Han committed
14

Yaowu Xu's avatar
Yaowu Xu committed
15
#include "./aom_config.h"
Jingning Han's avatar
Jingning Han committed
16

Yaowu Xu's avatar
Yaowu Xu committed
17
#include "aom_dsp/aom_dsp_common.h"
18
19
20
21
22
23
24
25
26
27
28
#include "aom_ports/mem.h"
#include "aom_scale/yv12config.h"

#include "av1/common/common_data.h"
#include "av1/common/quant_common.h"
#include "av1/common/entropy.h"
#include "av1/common/entropymode.h"
#include "av1/common/mv.h"
#include "av1/common/scale.h"
#include "av1/common/seg_common.h"
#include "av1/common/tile_common.h"
29
30
31
32
33
#if CONFIG_PVQ
#include "av1/common/pvq.h"
#include "av1/common/pvq_state.h"
#include "av1/decoder/decint.h"
#endif
Jingning Han's avatar
Jingning Han committed
34
35
36
37
38

#ifdef __cplusplus
extern "C" {
#endif

39
#define SUB8X8_COMP_REF (!(CONFIG_CB4X4 && CONFIG_CHROMA_2X2))
40

Jingning Han's avatar
Jingning Han committed
41
42
#define MAX_MB_PLANE 3

43
#if CONFIG_EXT_INTER
44

45
#if CONFIG_COMPOUND_SEGMENT
46
47
48
49
50
51
52
// Set COMPOUND_SEGMENT_TYPE to one of the three
// 0: Uniform
// 1: Difference weighted
#define COMPOUND_SEGMENT_TYPE 1

#if COMPOUND_SEGMENT_TYPE == 0
#define MAX_SEG_MASK_BITS 1
53
54
55
56
57
58
// SEG_MASK_TYPES should not surpass 1 << MAX_SEG_MASK_BITS
typedef enum {
  UNIFORM_45 = 0,
  UNIFORM_45_INV,
  SEG_MASK_TYPES,
} SEG_MASK_TYPE;
59
60
61
62
63

#elif COMPOUND_SEGMENT_TYPE == 1
#define MAX_SEG_MASK_BITS 1
// SEG_MASK_TYPES should not surpass 1 << MAX_SEG_MASK_BITS
typedef enum {
Yaowu Xu's avatar
Yaowu Xu committed
64
65
  DIFFWTD_42 = 0,
  DIFFWTD_42_INV,
66
67
68
69
  SEG_MASK_TYPES,
} SEG_MASK_TYPE;

#endif  // COMPOUND_SEGMENT_TYPE
70
#endif  // CONFIG_COMPOUND_SEGMENT
71
#endif  // CONFIG_EXT_INTER
72

Jingning Han's avatar
Jingning Han committed
73
74
75
76
77
78
79
typedef enum {
  KEY_FRAME = 0,
  INTER_FRAME = 1,
  FRAME_TYPES,
} FRAME_TYPE;

static INLINE int is_inter_mode(PREDICTION_MODE mode) {
Yue Chen's avatar
Yue Chen committed
80
#if CONFIG_EXT_INTER
81
  return mode >= NEARESTMV && mode <= NEW_NEWMV;
Yue Chen's avatar
Yue Chen committed
82
#else
Jingning Han's avatar
Jingning Han committed
83
  return mode >= NEARESTMV && mode <= NEWMV;
Yue Chen's avatar
Yue Chen committed
84
85
86
#endif  // CONFIG_EXT_INTER
}

87
88
89
90
91
#if CONFIG_PVQ
typedef struct PVQ_INFO {
  int theta[PVQ_MAX_PARTITIONS];
  int qg[PVQ_MAX_PARTITIONS];
  int k[PVQ_MAX_PARTITIONS];
92
  od_coeff y[OD_TXSIZE_MAX * OD_TXSIZE_MAX];
93
94
95
96
97
  int nb_bands;
  int off[PVQ_MAX_PARTITIONS];
  int size[PVQ_MAX_PARTITIONS];
  int skip_rest;
  int skip_dir;
ltrudeau's avatar
ltrudeau committed
98
99
100
101
  int bs;  // log of the block size minus two,
           // i.e. equivalent to aom's TX_SIZE
  // Block skip info, indicating whether DC/AC, is coded.
  PVQ_SKIP_TYPE ac_dc_coded;  // bit0: DC coded, bit1 : AC coded (1 means coded)
102
103
104
105
106
107
108
109
110
111
112
  tran_low_t dq_dc_residue;
} PVQ_INFO;

typedef struct PVQ_QUEUE {
  PVQ_INFO *buf;  // buffer for pvq info, stored in encoding order
  int curr_pos;   // curr position to write PVQ_INFO
  int buf_len;    // allocated buffer length
  int last_pos;   // last written position of PVQ_INFO in a tile
} PVQ_QUEUE;
#endif

113
114
115
116
117
typedef struct {
  uint8_t *plane[MAX_MB_PLANE];
  int stride[MAX_MB_PLANE];
} BUFFER_SET;

Yue Chen's avatar
Yue Chen committed
118
#if CONFIG_EXT_INTER
119
static INLINE int is_inter_singleref_mode(PREDICTION_MODE mode) {
120
  return mode >= NEARESTMV && mode <= NEWMV;
121
}
122
123
124
125
126
#if CONFIG_COMPOUND_SINGLEREF
static INLINE int is_inter_singleref_comp_mode(PREDICTION_MODE mode) {
  return mode >= SR_NEAREST_NEARMV && mode <= SR_NEW_NEWMV;
}
#endif  // CONFIG_COMPOUND_SINGLEREF
127
128
129
130
static INLINE int is_inter_compound_mode(PREDICTION_MODE mode) {
  return mode >= NEAREST_NEARESTMV && mode <= NEW_NEWMV;
}

131
132
static INLINE PREDICTION_MODE compound_ref0_mode(PREDICTION_MODE mode) {
  static PREDICTION_MODE lut[MB_MODE_COUNT] = {
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
    MB_MODE_COUNT,  // DC_PRED
    MB_MODE_COUNT,  // V_PRED
    MB_MODE_COUNT,  // H_PRED
    MB_MODE_COUNT,  // D45_PRED
    MB_MODE_COUNT,  // D135_PRED
    MB_MODE_COUNT,  // D117_PRED
    MB_MODE_COUNT,  // D153_PRED
    MB_MODE_COUNT,  // D207_PRED
    MB_MODE_COUNT,  // D63_PRED
#if CONFIG_ALT_INTRA
    MB_MODE_COUNT,  // SMOOTH_PRED
#endif              // CONFIG_ALT_INTRA
    MB_MODE_COUNT,  // TM_PRED
    MB_MODE_COUNT,  // NEARESTMV
    MB_MODE_COUNT,  // NEARMV
    MB_MODE_COUNT,  // ZEROMV
    MB_MODE_COUNT,  // NEWMV
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
#if CONFIG_COMPOUND_SINGLEREF
    NEARESTMV,  // SR_NEAREST_NEARMV
    NEARESTMV,  // SR_NEAREST_NEWMV
    NEARMV,     // SR_NEAR_NEWMV
    ZEROMV,     // SR_ZERO_NEWMV
    NEWMV,      // SR_NEW_NEWMV
#endif          // CONFIG_COMPOUND_SINGLEREF
    NEARESTMV,  // NEAREST_NEARESTMV
    NEARESTMV,  // NEAREST_NEARMV
    NEARMV,     // NEAR_NEARESTMV
    NEARMV,     // NEAR_NEARMV
    NEARESTMV,  // NEAREST_NEWMV
    NEWMV,      // NEW_NEARESTMV
    NEARMV,     // NEAR_NEWMV
    NEWMV,      // NEW_NEARMV
    ZEROMV,     // ZERO_ZEROMV
    NEWMV,      // NEW_NEWMV
167
168
169
170
171
172
173
  };
  assert(is_inter_compound_mode(mode));
  return lut[mode];
}

static INLINE PREDICTION_MODE compound_ref1_mode(PREDICTION_MODE mode) {
  static PREDICTION_MODE lut[MB_MODE_COUNT] = {
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
    MB_MODE_COUNT,  // DC_PRED
    MB_MODE_COUNT,  // V_PRED
    MB_MODE_COUNT,  // H_PRED
    MB_MODE_COUNT,  // D45_PRED
    MB_MODE_COUNT,  // D135_PRED
    MB_MODE_COUNT,  // D117_PRED
    MB_MODE_COUNT,  // D153_PRED
    MB_MODE_COUNT,  // D207_PRED
    MB_MODE_COUNT,  // D63_PRED
#if CONFIG_ALT_INTRA
    MB_MODE_COUNT,  // SMOOTH_PRED
#endif              // CONFIG_ALT_INTRA
    MB_MODE_COUNT,  // TM_PRED
    MB_MODE_COUNT,  // NEARESTMV
    MB_MODE_COUNT,  // NEARMV
    MB_MODE_COUNT,  // ZEROMV
    MB_MODE_COUNT,  // NEWMV
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
#if CONFIG_COMPOUND_SINGLEREF
    NEARMV,     // SR_NEAREST_NEARMV
    NEWMV,      // SR_NEAREST_NEWMV
    NEWMV,      // SR_NEAR_NEWMV
    NEWMV,      // SR_ZERO_NEWMV
    NEWMV,      // SR_NEW_NEWMV
#endif          // CONFIG_COMPOUND_SINGLEREF
    NEARESTMV,  // NEAREST_NEARESTMV
    NEARMV,     // NEAREST_NEARMV
    NEARESTMV,  // NEAR_NEARESTMV
    NEARMV,     // NEAR_NEARMV
    NEWMV,      // NEAREST_NEWMV
    NEARESTMV,  // NEW_NEARESTMV
    NEWMV,      // NEAR_NEWMV
    NEARMV,     // NEW_NEARMV
    ZEROMV,     // ZERO_ZEROMV
    NEWMV,      // NEW_NEWMV
208
209
210
211
212
  };
  assert(is_inter_compound_mode(mode));
  return lut[mode];
}

David Barker's avatar
David Barker committed
213
214
215
216
217
static INLINE int have_nearmv_in_inter_mode(PREDICTION_MODE mode) {
  return (mode == NEARMV || mode == NEAR_NEARMV || mode == NEAREST_NEARMV ||
          mode == NEAR_NEARESTMV || mode == NEAR_NEWMV || mode == NEW_NEARMV);
}

Yue Chen's avatar
Yue Chen committed
218
static INLINE int have_newmv_in_inter_mode(PREDICTION_MODE mode) {
219
220
  return (mode == NEWMV || mode == NEW_NEWMV || mode == NEAREST_NEWMV ||
          mode == NEW_NEARESTMV || mode == NEAR_NEWMV || mode == NEW_NEARMV);
Jingning Han's avatar
Jingning Han committed
221
}
222

223
static INLINE int use_masked_motion_search(COMPOUND_TYPE type) {
224
#if CONFIG_WEDGE
225
  return (type == COMPOUND_WEDGE);
226
227
228
229
#else
  (void)type;
  return 0;
#endif
230
231
}

232
static INLINE int is_masked_compound_type(COMPOUND_TYPE type) {
233
#if CONFIG_COMPOUND_SEGMENT && CONFIG_WEDGE
234
  return (type == COMPOUND_WEDGE || type == COMPOUND_SEG);
235
#elif !CONFIG_COMPOUND_SEGMENT && CONFIG_WEDGE
236
  return (type == COMPOUND_WEDGE);
237
238
#elif CONFIG_COMPOUND_SEGMENT && !CONFIG_WEDGE
  return (type == COMPOUND_SEG);
239
#endif  // CONFIG_COMPOUND_SEGMENT
240
241
  (void)type;
  return 0;
242
}
243
244
#else

David Barker's avatar
David Barker committed
245
246
247
248
static INLINE int have_nearmv_in_inter_mode(PREDICTION_MODE mode) {
  return (mode == NEARMV);
}

249
250
251
static INLINE int have_newmv_in_inter_mode(PREDICTION_MODE mode) {
  return (mode == NEWMV);
}
Yue Chen's avatar
Yue Chen committed
252
#endif  // CONFIG_EXT_INTER
Jingning Han's avatar
Jingning Han committed
253
254
255
256
257
258
259
260

/* For keyframes, intra block modes are predicted by the (already decoded)
   modes for the Y blocks to the left and above us; for interframes, there
   is a single probability table. */

typedef struct {
  PREDICTION_MODE as_mode;
  int_mv as_mv[2];  // first, second inter predictor motion vectors
Jingning Han's avatar
Jingning Han committed
261
#if CONFIG_REF_MV
Yaowu Xu's avatar
Yaowu Xu committed
262
  int_mv pred_mv[2];
Jingning Han's avatar
Jingning Han committed
263
#endif
Yue Chen's avatar
Yue Chen committed
264
265
266
#if CONFIG_EXT_INTER
  int_mv ref_mv[2];
#endif  // CONFIG_EXT_INTER
Jingning Han's avatar
Jingning Han committed
267
268
269
270
} b_mode_info;

typedef int8_t MV_REFERENCE_FRAME;

271
#if CONFIG_PALETTE
hui su's avatar
hui su committed
272
273
274
typedef struct {
  // Number of base colors for Y (0) and UV (1)
  uint8_t palette_size[2];
clang-format's avatar
clang-format committed
275
// Value of base colors for Y, U, and V
276
#if CONFIG_HIGHBITDEPTH
hui su's avatar
hui su committed
277
278
279
  uint16_t palette_colors[3 * PALETTE_MAX_SIZE];
#else
  uint8_t palette_colors[3 * PALETTE_MAX_SIZE];
280
#endif  // CONFIG_HIGHBITDEPTH
hui su's avatar
hui su committed
281
282
283
284
  // Only used by encoder to store the color index of the top left pixel.
  // TODO(huisu): move this to encoder
  uint8_t palette_first_color_idx[2];
} PALETTE_MODE_INFO;
285
#endif  // CONFIG_PALETTE
hui su's avatar
hui su committed
286

287
#if CONFIG_FILTER_INTRA
288
#define USE_3TAP_INTRA_FILTER 1  // 0: 4-tap; 1: 3-tap
hui su's avatar
hui su committed
289
290
typedef struct {
  // 1: an ext intra mode is used; 0: otherwise.
291
292
293
294
  uint8_t use_filter_intra_mode[PLANE_TYPES];
  FILTER_INTRA_MODE filter_intra_mode[PLANE_TYPES];
} FILTER_INTRA_MODE_INFO;
#endif  // CONFIG_FILTER_INTRA
hui su's avatar
hui su committed
295

Angie Chiang's avatar
Angie Chiang committed
296
#if CONFIG_VAR_TX
297
#if CONFIG_RD_DEBUG
Angie Chiang's avatar
Angie Chiang committed
298
#define TXB_COEFF_COST_MAP_SIZE (2 * MAX_MIB_SIZE)
299
300
#endif
#endif
Angie Chiang's avatar
Angie Chiang committed
301
302
303
304

typedef struct RD_STATS {
  int rate;
  int64_t dist;
Angie Chiang's avatar
Angie Chiang committed
305
  int64_t rdcost;
Angie Chiang's avatar
Angie Chiang committed
306
307
308
309
  int64_t sse;
  int skip;
#if CONFIG_RD_DEBUG
  int txb_coeff_cost[MAX_MB_PLANE];
310
#if CONFIG_VAR_TX
Angie Chiang's avatar
Angie Chiang committed
311
312
  int txb_coeff_cost_map[MAX_MB_PLANE][TXB_COEFF_COST_MAP_SIZE]
                        [TXB_COEFF_COST_MAP_SIZE];
313
314
#endif  // CONFIG_VAR_TX
#endif  // CONFIG_RD_DEBUG
Angie Chiang's avatar
Angie Chiang committed
315
316
} RD_STATS;

317
318
319
320
321
#if CONFIG_EXT_INTER
typedef struct {
  COMPOUND_TYPE type;
  int wedge_index;
  int wedge_sign;
322
#if CONFIG_COMPOUND_SEGMENT
323
324
  SEG_MASK_TYPE mask_type;
  DECLARE_ALIGNED(16, uint8_t, seg_mask[2 * MAX_SB_SQUARE]);
325
#endif  // CONFIG_COMPOUND_SEGMENT
326
327
328
} INTERINTER_COMPOUND_DATA;
#endif  // CONFIG_EXT_INTER

Jingning Han's avatar
Jingning Han committed
329
330
331
332
333
334
// This structure now relates to 8x8 block regions.
typedef struct {
  // Common for both INTER and INTRA blocks
  BLOCK_SIZE sb_type;
  PREDICTION_MODE mode;
  TX_SIZE tx_size;
335
#if CONFIG_VAR_TX
336
337
  // TODO(jingning): This effectively assigned a separate entry for each
  // 8x8 block. Apparently it takes much more space than needed.
338
  TX_SIZE inter_tx_size[MAX_MIB_SIZE][MAX_MIB_SIZE];
339
  TX_SIZE min_tx_size;
340
#endif
Jingning Han's avatar
Jingning Han committed
341
342
  int8_t skip;
  int8_t segment_id;
343
344
345
#if CONFIG_SUPERTX
  // Minimum of all segment IDs under the current supertx block.
  int8_t segment_id_supertx;
clang-format's avatar
clang-format committed
346
#endif                      // CONFIG_SUPERTX
Jingning Han's avatar
Jingning Han committed
347
348
349
350
  int8_t seg_id_predicted;  // valid only when temporal_update is enabled

  // Only for INTRA blocks
  PREDICTION_MODE uv_mode;
351
#if CONFIG_PALETTE
hui su's avatar
hui su committed
352
  PALETTE_MODE_INFO palette_mode_info;
353
#endif  // CONFIG_PALETTE
Alex Converse's avatar
Alex Converse committed
354
355
356
#if CONFIG_INTRABC
  uint8_t use_intrabc;
#endif  // CONFIG_INTRABC
Jingning Han's avatar
Jingning Han committed
357

clang-format's avatar
clang-format committed
358
// Only for INTER blocks
359
#if CONFIG_DUAL_FILTER
James Zern's avatar
James Zern committed
360
  InterpFilter interp_filter[4];
361
#else
James Zern's avatar
James Zern committed
362
  InterpFilter interp_filter;
363
#endif
Jingning Han's avatar
Jingning Han committed
364
  MV_REFERENCE_FRAME ref_frame[2];
hui su's avatar
hui su committed
365
  TX_TYPE tx_type;
Angie Chiang's avatar
Angie Chiang committed
366
#if CONFIG_TXK_SEL
Angie Chiang's avatar
Angie Chiang committed
367
368
  TX_TYPE txk_type[MAX_SB_SQUARE / (TX_SIZE_W_MIN * TX_SIZE_H_MIN)];
#endif
Jingning Han's avatar
Jingning Han committed
369

370
371
372
#if CONFIG_FILTER_INTRA
  FILTER_INTRA_MODE_INFO filter_intra_mode_info;
#endif  // CONFIG_FILTER_INTRA
hui su's avatar
hui su committed
373
#if CONFIG_EXT_INTRA
hui su's avatar
hui su committed
374
  // The actual prediction angle is the base angle + (angle_delta * step).
hui su's avatar
hui su committed
375
  int8_t angle_delta[2];
hui su's avatar
hui su committed
376
#if CONFIG_INTRA_INTERP
377
378
  // To-Do (huisu): this may be replaced by interp_filter
  INTRA_FILTER intra_filter;
hui su's avatar
hui su committed
379
#endif  // CONFIG_INTRA_INTERP
hui su's avatar
hui su committed
380
381
#endif  // CONFIG_EXT_INTRA

382
#if CONFIG_EXT_INTER
383
  INTERINTRA_MODE interintra_mode;
384
385
386
  // TODO(debargha): Consolidate these flags
  int use_wedge_interintra;
  int interintra_wedge_index;
Debargha Mukherjee's avatar
Debargha Mukherjee committed
387
  int interintra_wedge_sign;
388
  INTERINTER_COMPOUND_DATA interinter_compound_data;
389
#endif  // CONFIG_EXT_INTER
Yue Chen's avatar
Yue Chen committed
390
  MOTION_MODE motion_mode;
391
392
393
#if CONFIG_MOTION_VAR
  int overlappable_neighbors[2];
#endif  // CONFIG_MOTION_VAR
Jingning Han's avatar
Jingning Han committed
394
  int_mv mv[2];
395
  int_mv pred_mv[2];
396
397
398
#if CONFIG_REF_MV
  uint8_t ref_mv_idx;
#endif
399
400
401
#if CONFIG_EXT_PARTITION_TYPES
  PARTITION_TYPE partition;
#endif
402
403
404
405
#if CONFIG_NEW_QUANT
  int dq_off_index;
  int send_dq_bit;
#endif  // CONFIG_NEW_QUANT
Yaowu Xu's avatar
Yaowu Xu committed
406
  /* deringing gain *per-superblock* */
407
  int8_t cdef_strength;
408
409
#if CONFIG_DELTA_Q
  int current_q_index;
Fangwen Fu's avatar
Fangwen Fu committed
410
411
412
#if CONFIG_EXT_DELTA_Q
  int current_delta_lf_from_base;
#endif
413
#endif
414
#if CONFIG_RD_DEBUG
Angie Chiang's avatar
Angie Chiang committed
415
  RD_STATS rd_stats;
416
417
418
  int mi_row;
  int mi_col;
#endif
Yue Chen's avatar
Yue Chen committed
419
420
421
422
#if CONFIG_WARPED_MOTION
  int num_proj_ref[2];
  WarpedMotionParams wm_params[2];
#endif  // CONFIG_WARPED_MOTION
423
424

  BOUNDARY_TYPE boundary_info;
Jingning Han's avatar
Jingning Han committed
425
426
427
428
429
430
431
} MB_MODE_INFO;

typedef struct MODE_INFO {
  MB_MODE_INFO mbmi;
  b_mode_info bmi[4];
} MODE_INFO;

Alex Converse's avatar
Alex Converse committed
432
433
434
435
436
437
#if CONFIG_INTRABC
static INLINE int is_intrabc_block(const MB_MODE_INFO *mbmi) {
  return mbmi->use_intrabc;
}
#endif

Jingning Han's avatar
Jingning Han committed
438
static INLINE PREDICTION_MODE get_y_mode(const MODE_INFO *mi, int block) {
439
440
441
442
#if CONFIG_CB4X4
  (void)block;
  return mi->mbmi.mode;
#else
clang-format's avatar
clang-format committed
443
  return mi->mbmi.sb_type < BLOCK_8X8 ? mi->bmi[block].as_mode : mi->mbmi.mode;
444
#endif
Jingning Han's avatar
Jingning Han committed
445
446
447
}

static INLINE int is_inter_block(const MB_MODE_INFO *mbmi) {
Alex Converse's avatar
Alex Converse committed
448
449
450
#if CONFIG_INTRABC
  if (is_intrabc_block(mbmi)) return 1;
#endif
Jingning Han's avatar
Jingning Han committed
451
452
453
454
455
456
457
  return mbmi->ref_frame[0] > INTRA_FRAME;
}

static INLINE int has_second_ref(const MB_MODE_INFO *mbmi) {
  return mbmi->ref_frame[1] > INTRA_FRAME;
}

Yaowu Xu's avatar
Yaowu Xu committed
458
459
PREDICTION_MODE av1_left_block_mode(const MODE_INFO *cur_mi,
                                    const MODE_INFO *left_mi, int b);
Jingning Han's avatar
Jingning Han committed
460

Yaowu Xu's avatar
Yaowu Xu committed
461
462
PREDICTION_MODE av1_above_block_mode(const MODE_INFO *cur_mi,
                                     const MODE_INFO *above_mi, int b);
Jingning Han's avatar
Jingning Han committed
463

464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
#if CONFIG_GLOBAL_MOTION
static INLINE int is_global_mv_block(const MODE_INFO *mi, int block,
                                     TransformationType type) {
  PREDICTION_MODE mode = get_y_mode(mi, block);
#if GLOBAL_SUB8X8_USED
  const int block_size_allowed = 1;
#else
  const BLOCK_SIZE bsize = mi->mbmi.sb_type;
  const int block_size_allowed = (bsize >= BLOCK_8X8);
#endif  // GLOBAL_SUB8X8_USED
#if CONFIG_EXT_INTER
  return (mode == ZEROMV || mode == ZERO_ZEROMV) && type > TRANSLATION &&
         block_size_allowed;
#else
  return mode == ZEROMV && type > TRANSLATION && block_size_allowed;
#endif  // CONFIG_EXT_INTER
}
#endif  // CONFIG_GLOBAL_MOTION

clang-format's avatar
clang-format committed
483
enum mv_precision { MV_PRECISION_Q3, MV_PRECISION_Q4 };
Jingning Han's avatar
Jingning Han committed
484
485
486

struct buf_2d {
  uint8_t *buf;
487
488
489
  uint8_t *buf0;
  int width;
  int height;
Jingning Han's avatar
Jingning Han committed
490
491
492
  int stride;
};

Angie Chiang's avatar
Angie Chiang committed
493
typedef struct macroblockd_plane {
Jingning Han's avatar
Jingning Han committed
494
495
496
497
498
499
500
501
502
  tran_low_t *dqcoeff;
  PLANE_TYPE plane_type;
  int subsampling_x;
  int subsampling_y;
  struct buf_2d dst;
  struct buf_2d pre[2];
  ENTROPY_CONTEXT *above_context;
  ENTROPY_CONTEXT *left_context;
  int16_t seg_dequant[MAX_SEGMENTS][2];
503
#if CONFIG_NEW_QUANT
clang-format's avatar
clang-format committed
504
505
  dequant_val_type_nuq seg_dequant_nuq[MAX_SEGMENTS][QUANT_PROFILES]
                                      [COEF_BANDS];
506
#endif
507
#if CONFIG_PALETTE
hui su's avatar
hui su committed
508
  uint8_t *color_index_map;
509
#endif  // CONFIG_PALETTE
Jingning Han's avatar
Jingning Han committed
510
511
512
513
514

  // number of 4x4s in current block
  uint16_t n4_w, n4_h;
  // log2 of n4_w, n4_h
  uint8_t n4_wl, n4_hl;
515
516
  // block size in pixels
  uint8_t width, height;
Jingning Han's avatar
Jingning Han committed
517

518
519
520
#if CONFIG_AOM_QM
  const qm_val_t *seg_iqmatrix[MAX_SEGMENTS][2][TX_SIZES];
#endif
Jingning Han's avatar
Jingning Han committed
521
522
  // encoder
  const int16_t *dequant;
523
#if CONFIG_NEW_QUANT
clang-format's avatar
clang-format committed
524
  const dequant_val_type_nuq *dequant_val_nuq[QUANT_PROFILES];
525
#endif  // CONFIG_NEW_QUANT
526
527
528
#if CONFIG_AOM_QM
  const qm_val_t *seg_qmatrix[MAX_SEGMENTS][2][TX_SIZES];
#endif
529

Yushin Cho's avatar
Yushin Cho committed
530
#if CONFIG_PVQ || CONFIG_DAALA_DIST
531
532
533
534
  DECLARE_ALIGNED(16, int16_t, pred[MAX_SB_SQUARE]);
  // PVQ: forward transformed predicted image, a reference for PVQ.
  tran_low_t *pvq_ref_coeff;
#endif
Angie Chiang's avatar
Angie Chiang committed
535
} MACROBLOCKD_PLANE;
Jingning Han's avatar
Jingning Han committed
536

537
538
#define BLOCK_OFFSET(x, i) \
  ((x) + (i) * (1 << (tx_size_wide_log2[0] + tx_size_high_log2[0])))
Jingning Han's avatar
Jingning Han committed
539
540
541

typedef struct RefBuffer {
  // TODO(dkovalev): idx is not really required and should be removed, now it
Yaowu Xu's avatar
Yaowu Xu committed
542
  // is used in av1_onyxd_if.c
Jingning Han's avatar
Jingning Han committed
543
544
545
546
547
  int idx;
  YV12_BUFFER_CONFIG *buf;
  struct scale_factors sf;
} RefBuffer;

548
typedef int16_t EobThresholdMD[TX_SIZES_ALL][TX_TYPES];
Luc Trudeau's avatar
Luc Trudeau committed
549
550
551

#if CONFIG_CFL
typedef struct {
552
553
554
555
556
557
  // Pixel buffer containing the luma pixels used as prediction for chroma
  uint8_t y_pix[MAX_SB_SQUARE];

  // Height and width of the luma prediction block currently in the pixel buffer
  int y_height, y_width;

Luc Trudeau's avatar
Luc Trudeau committed
558
  // CfL Performs its own block level DC_PRED for each chromatic plane
559
  int dc_pred[CFL_PRED_PLANES];
Luc Trudeau's avatar
Luc Trudeau committed
560
561
562
} CFL_CTX;
#endif

Jingning Han's avatar
Jingning Han committed
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
typedef struct macroblockd {
  struct macroblockd_plane plane[MAX_MB_PLANE];
  uint8_t bmode_blocks_wl;
  uint8_t bmode_blocks_hl;

  FRAME_COUNTS *counts;
  TileInfo tile;

  int mi_stride;

  MODE_INFO **mi;
  MODE_INFO *left_mi;
  MODE_INFO *above_mi;
  MB_MODE_INFO *left_mbmi;
  MB_MODE_INFO *above_mbmi;

  int up_available;
  int left_available;

Yaowu Xu's avatar
Yaowu Xu committed
582
  const aom_prob (*partition_probs)[PARTITION_TYPES - 1];
Jingning Han's avatar
Jingning Han committed
583
584
585
586
587
588
589
590
591
592

  /* Distance of MB away from frame edges */
  int mb_to_left_edge;
  int mb_to_right_edge;
  int mb_to_top_edge;
  int mb_to_bottom_edge;

  FRAME_CONTEXT *fc;

  /* pointers to reference frames */
593
  const RefBuffer *block_refs[2];
Jingning Han's avatar
Jingning Han committed
594
595
596
597
598

  /* pointer to current frame */
  const YV12_BUFFER_CONFIG *cur_buf;

  ENTROPY_CONTEXT *above_context[MAX_MB_PLANE];
599
  ENTROPY_CONTEXT left_context[MAX_MB_PLANE][2 * MAX_MIB_SIZE];
Jingning Han's avatar
Jingning Han committed
600
601

  PARTITION_CONTEXT *above_seg_context;
602
  PARTITION_CONTEXT left_seg_context[MAX_MIB_SIZE];
Jingning Han's avatar
Jingning Han committed
603

604
605
606
#if CONFIG_VAR_TX
  TXFM_CONTEXT *above_txfm_context;
  TXFM_CONTEXT *left_txfm_context;
607
  TXFM_CONTEXT left_txfm_context_buffer[MAX_MIB_SIZE];
608

609
  TX_SIZE max_tx_size;
610
611
612
#if CONFIG_SUPERTX
  TX_SIZE supertx_size;
#endif
613
614
#endif

615
  // block dimension in the unit of mode_info.
616
617
  uint8_t n8_w, n8_h;

618
#if CONFIG_REF_MV
619
620
  uint8_t ref_mv_count[MODE_CTX_REF_FRAMES];
  CANDIDATE_MV ref_mv_stack[MODE_CTX_REF_FRAMES][MAX_REF_MV_STACK_SIZE];
621
  uint8_t is_sec_rect;
622
623
#endif

624
625
626
#if CONFIG_PVQ
  daala_dec_ctx daala_dec;
#endif
627
628
629
#if CONFIG_EC_ADAPT
  FRAME_CONTEXT *tile_ctx;
#endif
630
#if CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
631
632
633
634
  /* Bit depth: 8, 10, 12 */
  int bd;
#endif

635
  int qindex[MAX_SEGMENTS];
636
  int lossless[MAX_SEGMENTS];
Jingning Han's avatar
Jingning Han committed
637
638
  int corrupted;

Yaowu Xu's avatar
Yaowu Xu committed
639
  struct aom_internal_error_info *error_info;
640
#if CONFIG_GLOBAL_MOTION
641
  WarpedMotionParams *global_motion;
642
#endif  // CONFIG_GLOBAL_MOTION
643
644
645
646
#if CONFIG_DELTA_Q
  int prev_qindex;
  int delta_qindex;
  int current_qindex;
Fangwen Fu's avatar
Fangwen Fu committed
647
648
649
650
651
652
653
654
655
656
#if CONFIG_EXT_DELTA_Q
  // Since actual frame level loop filtering level value is not available
  // at the beginning of the tile (only available during actual filtering)
  // at encoder side.we record the delta_lf (against the frame level loop
  // filtering level) and code the delta between previous superblock's delta
  // lf and current delta lf. It is equivalent to the delta between previous
  // superblock's actual lf and current lf.
  int prev_delta_lf_from_base;
  int current_delta_lf_from_base;
#endif
657
#endif
658
659
660
#if CONFIG_ADAPT_SCAN
  const EobThresholdMD *eob_threshold_md;
#endif
Luc Trudeau's avatar
Luc Trudeau committed
661
662
663
664

#if CONFIG_CFL
  CFL_CTX *cfl;
#endif
Jingning Han's avatar
Jingning Han committed
665
666
667
668
} MACROBLOCKD;

static INLINE BLOCK_SIZE get_subsize(BLOCK_SIZE bsize,
                                     PARTITION_TYPE partition) {
669
  if (partition == PARTITION_INVALID)
670
    return BLOCK_INVALID;
671
672
  else
    return subsize_lookup[partition][bsize];
673
674
}

675
static const TX_TYPE intra_mode_to_tx_type_context[INTRA_MODES] = {
Jingning Han's avatar
Jingning Han committed
676
677
678
679
680
681
682
683
684
  DCT_DCT,    // DC
  ADST_DCT,   // V
  DCT_ADST,   // H
  DCT_DCT,    // D45
  ADST_ADST,  // D135
  ADST_DCT,   // D117
  DCT_ADST,   // D153
  DCT_ADST,   // D207
  ADST_DCT,   // D63
685
686
687
#if CONFIG_ALT_INTRA
  ADST_ADST,  // SMOOTH
#endif        // CONFIG_ALT_INTRA
Jingning Han's avatar
Jingning Han committed
688
689
690
  ADST_ADST,  // TM
};

691
692
#if CONFIG_SUPERTX
static INLINE int supertx_enabled(const MB_MODE_INFO *mbmi) {
Jingning Han's avatar
Jingning Han committed
693
694
695
  TX_SIZE max_tx_size = txsize_sqr_map[mbmi->tx_size];
  return tx_size_wide[max_tx_size] >
         AOMMIN(block_size_wide[mbmi->sb_type], block_size_high[mbmi->sb_type]);
696
697
698
}
#endif  // CONFIG_SUPERTX

699
#define USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4 1
700

701
702
703
704
#if CONFIG_RECT_TX
static INLINE int is_rect_tx(TX_SIZE tx_size) { return tx_size >= TX_SIZES; }
#endif  // CONFIG_RECT_TX

705
#if CONFIG_EXT_TX
clang-format's avatar
clang-format committed
706
#define ALLOW_INTRA_EXT_TX 1
707

708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
typedef enum {
  // DCT only
  EXT_TX_SET_DCTONLY = 0,
  // DCT + Identity only
  EXT_TX_SET_DCT_IDTX = 1,
  // Discrete Trig transforms w/o flip (4) + Identity (1)
  EXT_TX_SET_DTT4_IDTX = 2,
  // Discrete Trig transforms w/o flip (4) + Identity (1) + 1D Hor/vert DCT (2)
  EXT_TX_SET_DTT4_IDTX_1DDCT = 3,
  // Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver DCT (2)
  EXT_TX_SET_DTT9_IDTX_1DDCT = 4,
  // Discrete Trig transforms w/ flip (9) + Identity (1) + 1D Hor/Ver (6)
  EXT_TX_SET_ALL16 = 5,
  EXT_TX_SET_TYPES
} TxSetType;

// Number of transform types in each set type
static const int num_ext_tx_set[EXT_TX_SET_TYPES] = { 1, 2, 5, 7, 12, 16 };

// Maps intra set index to the set type
static const int ext_tx_set_type_intra[EXT_TX_SETS_INTRA] = {
  EXT_TX_SET_DCTONLY, EXT_TX_SET_DTT4_IDTX_1DDCT, EXT_TX_SET_DTT4_IDTX
};
731

732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
// Maps inter set index to the set type
static const int ext_tx_set_type_inter[EXT_TX_SETS_INTER] = {
  EXT_TX_SET_DCTONLY, EXT_TX_SET_ALL16, EXT_TX_SET_DTT9_IDTX_1DDCT,
  EXT_TX_SET_DCT_IDTX
};

// Maps set types above to the indices used for intra
static const int ext_tx_set_index_intra[EXT_TX_SET_TYPES] = { 0, -1, 2,
                                                              1, -1, -1 };

// Maps set types above to the indices used for inter
static const int ext_tx_set_index_inter[EXT_TX_SET_TYPES] = {
  0, 3, -1, -1, 2, 1
};

static INLINE TxSetType get_ext_tx_set_type(TX_SIZE tx_size, BLOCK_SIZE bs,
748
                                            int is_inter, int use_reduced_set) {
749
  const TX_SIZE tx_size2 = txsize_sqr_up_map[tx_size];
750
  tx_size = txsize_sqr_map[tx_size];
751
#if CONFIG_CB4X4 && USE_TXTYPE_SEARCH_FOR_SUB8X8_IN_CB4X4
752
  (void)bs;
753
  if (tx_size > TX_32X32) return EXT_TX_SET_DCTONLY;
754
#else
755
  if (tx_size > TX_32X32 || bs < BLOCK_8X8) return EXT_TX_SET_DCTONLY;
756
#endif
757
758
  if (use_reduced_set)
    return is_inter ? EXT_TX_SET_DCT_IDTX : EXT_TX_SET_DTT4_IDTX;
759
760
761
762
763
764
765
766
767
768
769
  if (tx_size2 == TX_32X32)
    return is_inter ? EXT_TX_SET_DCT_IDTX : EXT_TX_SET_DCTONLY;
  if (is_inter)
    return (tx_size == TX_16X16 ? EXT_TX_SET_DTT9_IDTX_1DDCT
                                : EXT_TX_SET_ALL16);
  else
    return (tx_size == TX_16X16 ? EXT_TX_SET_DTT4_IDTX
                                : EXT_TX_SET_DTT4_IDTX_1DDCT);
}

static INLINE int get_ext_tx_set(TX_SIZE tx_size, BLOCK_SIZE bs, int is_inter,
770
                                 int use_reduced_set) {
771
  const TxSetType set_type =
772
      get_ext_tx_set_type(tx_size, bs, is_inter, use_reduced_set);
773
774
  return is_inter ? ext_tx_set_index_inter[set_type]
                  : ext_tx_set_index_intra[set_type];
775
776
}

clang-format's avatar
clang-format committed
777
778
static const int use_intra_ext_tx_for_txsize[EXT_TX_SETS_INTRA][EXT_TX_SIZES] =
    {
779
#if CONFIG_CB4X4
clang-format's avatar
clang-format committed
780
781
782
      { 1, 1, 1, 1, 1 },  // unused
      { 0, 1, 1, 0, 0 },
      { 0, 0, 0, 1, 0 },
783
#else
clang-format's avatar
clang-format committed
784
785
786
      { 1, 1, 1, 1 },  // unused
      { 1, 1, 0, 0 },
      { 0, 0, 1, 0 },
787
#endif  // CONFIG_CB4X4
clang-format's avatar
clang-format committed
788
    };
789

clang-format's avatar
clang-format committed
790
791
static const int use_inter_ext_tx_for_txsize[EXT_TX_SETS_INTER][EXT_TX_SIZES] =
    {
792
#if CONFIG_CB4X4
clang-format's avatar
clang-format committed
793
794
795
796
      { 1, 1, 1, 1, 1 },  // unused
      { 0, 1, 1, 0, 0 },
      { 0, 0, 0, 1, 0 },
      { 0, 0, 0, 0, 1 },
797
#else
clang-format's avatar
clang-format committed
798
799
800
801
      { 1, 1, 1, 1 },  // unused
      { 1, 1, 0, 0 },
      { 0, 0, 1, 0 },
      { 0, 0, 0, 1 },
802
#endif  // CONFIG_CB4X4
clang-format's avatar
clang-format committed
803
    };
804

805
// Transform types used in each intra set
806
static const int ext_tx_used_intra[EXT_TX_SETS_INTRA][TX_TYPES] = {
clang-format's avatar
clang-format committed
807
808
809
  { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
  { 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0 },
  { 1, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0 },
810
811
};

812
813
814
// Numbers of transform types used in each intra set
static const int ext_tx_cnt_intra[EXT_TX_SETS_INTRA] = { 1, 7, 5 };

815
// Transform types used in each inter set
816
static const int ext_tx_used_inter[EXT_TX_SETS_INTER][TX_TYPES] = {
clang-format's avatar
clang-format committed
817
818
819
820
  { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
  { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 },
  { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0 },
  { 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0 },
821
};
822

823
824
825
// Numbers of transform types used in each inter set
static const int ext_tx_cnt_inter[EXT_TX_SETS_INTER] = { 1, 16, 12, 2 };

826
827
828
// 1D Transforms used in inter set, this needs to be changed if
// ext_tx_used_inter is changed
static const int ext_tx_used_inter_1D[EXT_TX_SETS_INTER][TX_TYPES_1D] = {
clang-format's avatar
clang-format committed
829
  { 1, 0, 0, 0 }, { 1, 1, 1, 1 }, { 1, 1, 1, 1 }, { 1, 0, 0, 1 },
830
831
};

832
static INLINE int get_ext_tx_types(TX_SIZE tx_size, BLOCK_SIZE bs, int is_inter,
833
834
835
                                   int use_reduced_set) {
  const int set_type =
      get_ext_tx_set_type(tx_size, bs, is_inter, use_reduced_set);
836
  return num_ext_tx_set[set_type];
837
}
838
839
840
841

#if CONFIG_RECT_TX
static INLINE int is_rect_tx_allowed_bsize(BLOCK_SIZE bsize) {
  static const char LUT[BLOCK_SIZES] = {
842
843
844
845
846
#if CONFIG_CB4X4
    0,  // BLOCK_2X2
    0,  // BLOCK_2X4
    0,  // BLOCK_4X2
#endif
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
    0,  // BLOCK_4X4
    1,  // BLOCK_4X8
    1,  // BLOCK_8X4
    0,  // BLOCK_8X8
    1,  // BLOCK_8X16
    1,  // BLOCK_16X8
    0,  // BLOCK_16X16
    1,  // BLOCK_16X32
    1,  // BLOCK_32X16
    0,  // BLOCK_32X32
    0,  // BLOCK_32X64
    0,  // BLOCK_64X32
    0,  // BLOCK_64X64
#if CONFIG_EXT_PARTITION
    0,  // BLOCK_64X128
    0,  // BLOCK_128X64
    0,  // BLOCK_128X128
#endif  // CONFIG_EXT_PARTITION
  };

  return LUT[bsize];
}

870
871
static INLINE int is_rect_tx_allowed(const MACROBLOCKD *xd,
                                     const MB_MODE_INFO *mbmi) {
872
  return is_rect_tx_allowed_bsize(mbmi->sb_type) &&
873
         !xd->lossless[mbmi->segment_id];
874
875
}
#endif  // CONFIG_RECT_TX
876
877
#endif  // CONFIG_EXT_TX

878
879
880
static INLINE TX_SIZE tx_size_from_tx_mode(BLOCK_SIZE bsize, TX_MODE tx_mode,
                                           int is_inter) {
  const TX_SIZE largest_tx_size = tx_mode_to_biggest_tx_size[tx_mode];
881
#if (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX
882
883
884
  const TX_SIZE max_rect_tx_size = max_txsize_rect_lookup[bsize];
#else
  const TX_SIZE max_tx_size = max_txsize_lookup[bsize];
885
#endif  // (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX
886
  (void)is_inter;
887
#if CONFIG_VAR_TX && CONFIG_RECT_TX
888
#if CONFIG_CB4X4
889
  if (bsize == BLOCK_4X4)
890
891
    return AOMMIN(max_txsize_lookup[bsize], largest_tx_size);
#else
892
  if (bsize < BLOCK_8X8)
893
    return AOMMIN(max_txsize_lookup[bsize], largest_tx_size);
894
#endif
895
896
  if (txsize_sqr_map[max_rect_tx_size] <= largest_tx_size)
    return max_rect_tx_size;
897
898
  else
    return largest_tx_size;
899
900
901
#elif CONFIG_EXT_TX && CONFIG_RECT_TX
  if (txsize_sqr_up_map[max_rect_tx_size] <= largest_tx_size) {
    return max_rect_tx_size;
902
  } else {
903
    return largest_tx_size;
904
905
  }
#else
Yaowu Xu's avatar
Yaowu Xu committed
906
  return AOMMIN(max_tx_size, largest_tx_size);
907
#endif  // CONFIG_VAR_TX && CONFIG_RECT_TX
908
909
}

910
#if CONFIG_EXT_INTRA
hui su's avatar
hui su committed
911
912
#define MAX_ANGLE_DELTA 3
#define ANGLE_STEP 3
913
914
915
916
extern const int16_t dr_intra_derivative[90];
static const uint8_t mode_to_angle_map[INTRA_MODES] = {
  0, 90, 180, 45, 135, 111, 157, 203, 67, 0,
};
hui su's avatar
hui su committed
917
#if CONFIG_INTRA_INTERP
918
919
// Returns whether filter selection is needed for a given
// intra prediction angle.
Yaowu Xu's avatar
Yaowu Xu committed
920
int av1_is_intra_filter_switchable(int angle);
hui su's avatar
hui su committed
921
#endif  // CONFIG_INTRA_INTERP
hui su's avatar
hui su committed
922
923
#endif  // CONFIG_EXT_INTRA

924
925
#define FIXED_TX_TYPE 0

926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
// Converts block_index for given transform size to index of the block in raster
// order.
static INLINE int av1_block_index_to_raster_order(TX_SIZE tx_size,
                                                  int block_idx) {
  // For transform size 4x8, the possible block_idx values are 0 & 2, because
  // block_idx values are incremented in steps of size 'tx_width_unit x
  // tx_height_unit'. But, for this transform size, block_idx = 2 corresponds to
  // block number 1 in raster order, inside an 8x8 MI block.
  // For any other transform size, the two indices are equivalent.
  return (tx_size == TX_4X8 && block_idx == 2) ? 1 : block_idx;
}

// Inverse of above function.
// Note: only implemented for transform sizes 4x4, 4x8 and 8x4 right now.
static INLINE int av1_raster_order_to_block_index(TX_SIZE tx_size,
                                                  int raster_order) {
  assert(tx_size == TX_4X4 || tx_size == TX_4X8 || tx_size == TX_8X4);
  // We ensure that block indices are 0 & 2 if tx size is 4x8 or 8x4.
  return (tx_size == TX_4X4) ? raster_order : (raster_order > 0) ? 2 : 0;
}

947
static INLINE TX_TYPE get_default_tx_type(PLANE_TYPE plane_type,
clang-format's avatar
clang-format committed
948
949
                                          const MACROBLOCKD *xd, int block_idx,
                                          TX_SIZE tx_size) {
950
951
952
953
954
955
  const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;

  if (is_inter_block(mbmi) || plane_type != PLANE_TYPE_Y ||
      xd->lossless[mbmi->segment_id] || tx_size >= TX_32X32)
    return DCT_DCT;

clang-format's avatar
clang-format committed
956
957
958
  return intra_mode_to_tx_type_context[plane_type == PLANE_TYPE_Y
                                           ? get_y_mode(xd->mi[0], block_idx)
                                           : mbmi->uv_mode];
959
960
}

clang-format's avatar
clang-format committed
961
static INLINE TX_TYPE get_tx_type(PLANE_TYPE plane_type, const MACROBLOCKD *xd,
962
                                  int block, TX_SIZE tx_size) {
Jingning Han's avatar
Jingning Han committed
963
  const MODE_INFO *const mi = xd->mi[0];
hui su's avatar
hui su committed
964
  const MB_MODE_INFO *const mbmi = &mi->mbmi;
Alex Converse's avatar
Alex Converse committed
965
966
967
968
#if CONFIG_INTRABC
  // TODO(aconverse@google.com): Revisit this decision
  if (is_intrabc_block(mbmi)) return DCT_DCT;
#endif  // CONFIG_INTRABC
Angie Chiang's avatar
Angie Chiang committed
969
#if !CONFIG_TXK_SEL
James Zern's avatar
James Zern committed
970
971
972
973
#if FIXED_TX_TYPE
  const int block_raster_idx = av1_block_index_to_raster_order(tx_size, block);
  return get_default_tx_type(plane_type, xd, block_raster_idx, tx_size);
#elif CONFIG_EXT_TX
974
#if !CONFIG_CB4X4
975
  const int block_raster_idx = av1_block_index_to_raster_order(tx_size, block);
976
#endif  // !CONFIG_CB4X4
clang-format's avatar
clang-format committed
977
  if (xd->lossless[mbmi->segment_id] || txsize_sqr_map[tx_size] > TX_32X32 ||
978
      (txsize_sqr_map[tx_size] >= TX_32X32 && !is_inter_block(mbmi)))
hui su's avatar
hui su committed
979
    return DCT_DCT;
980
  if (mbmi->sb_type >= BLOCK_8X8 || CONFIG_CB4X4) {
981
    if (plane_type == PLANE_TYPE_Y) {
982
983
984
985
#if !ALLOW_INTRA_EXT_TX
      if (is_inter_block(mbmi))
#endif  // ALLOW_INTRA_EXT_TX
        return mbmi->tx_type;
986
    }
987
988
989
990
991
992

    if (is_inter_block(mbmi)) {
// UV Inter only
#if CONFIG_CB4X4
      if (tx_size < TX_4X4) return DCT_DCT;
#endif
993
      return (mbmi->tx_type == IDTX && txsize_sqr_map[tx_size] >= TX_32X32)
clang-format's avatar
clang-format committed
994
995
                 ? DCT_DCT
                 : mbmi->tx_type;
996
    }
997
  }
998

999
#if CONFIG_CB4X4
1000
  (void)block;
1001
1002
1003
1004
  if (tx_size < TX_4X4)
    return DCT_DCT;
  else
    return intra_mode_to_tx_type_context[mbmi->uv_mode];
1005
#else
1006

1007
1008
  // Sub8x8-Inter/Intra OR UV-Intra
  if (is_inter_block(mbmi))  // Sub8x8-Inter
1009
    return DCT_DCT;
1010
  else  // Sub8x8 Intra OR UV-Intra
clang-format's avatar
clang-format committed
1011
    return intra_mode_to_tx_type_context[plane_type == PLANE_TYPE_Y
1012
                                             ? get_y_mode(mi, block_raster_idx)
clang-format's avatar
clang-format committed
1013
                                             : mbmi->uv_mode];
1014
#endif  // CONFIG_CB4X4
1015
#else   // CONFIG_EXT_TX
1016
  (void)block;
1017
  if (plane_type != PLANE_TYPE_Y || xd->lossless[mbmi->segment_id] ||
1018
      txsize_sqr_map[tx_size] >= TX_32X32)
Jingning Han's avatar
Jingning Han committed
1019
    return DCT_DCT;
1020
  return mbmi->tx_type;
hui su's avatar
hui su committed
1021
#endif  // CONFIG_EXT_TX
Angie Chiang's avatar
Angie Chiang committed
1022
#else   // !CONFIG_TXK_SEL
1023
  (void)tx_size;
1024
  TX_TYPE tx_type;
1025
  if (plane_type != PLANE_TYPE_Y || xd->lossless[mbmi->segment_id] ||
1026
1027
1028
1029
      mbmi->tx_size >= TX_32X32) {
    tx_type = DCT_DCT;
  } else {
    tx_type = mbmi->txk_type[block];
1030
  }
1031
1032
  assert(tx_type >= DCT_DCT && tx_type < TX_TYPES);
  return tx_type;
Angie Chiang's avatar
Angie Chiang committed
1033
#endif  // !CONFIG_TXK_SEL
Jingning Han's avatar
Jingning Han committed
1034
1035
}

Yaowu Xu's avatar
Yaowu Xu committed
1036
void av1_setup_block_planes(MACROBLOCKD *xd, int ss_x, int ss_y);
Jingning Han's avatar
Jingning Han committed
1037

1038
static INLINE int tx_size_to_depth(TX_SIZE tx_size) {
1039
1040
1041
  return (int)(tx_size - TX_4X4);
}

1042
static INLINE TX_SIZE depth_to_tx_size(int depth) {
1043
1044
1045
  return (TX_SIZE)(depth + TX_4X4);
}

Jingning Han's avatar
Jingning Han committed
1046
1047
static INLINE TX_SIZE get_uv_tx_size(const MB_MODE_INFO *mbmi,
                                     const struct macroblockd_plane *pd) {
1048
  TX_SIZE uv_txsize;
Jingning Han's avatar
Jingning Han committed
1049
1050
1051
1052
#if CONFIG_CB4X4
  assert(mbmi->tx_size > TX_2X2);
#endif

1053
#if CONFIG_SUPERTX
hui su's avatar
hui su committed
1054
  if (supertx_enabled(mbmi))
1055
1056
    return uvsupertx_size_lookup[txsize_sqr_map[mbmi->tx_size]]
                                [pd->subsampling_x][pd->subsampling_y];
hui su's avatar
hui su committed
1057
#endif  // CONFIG_SUPERTX
1058

1059
1060
  uv_txsize = uv_txsize_lookup[mbmi->sb_type][mbmi->tx_size][pd->subsampling_x]
                              [pd->subsampling_y];
Jingning Han's avatar
Jingning Han committed
1061
#if CONFIG_CB4X4 && !CONFIG_CHROMA_2X2
1062
1063
  uv_txsize = AOMMAX(uv_txsize, TX_4X4);
#endif
1064
1065
  assert(uv_txsize != TX_INVALID);
  return uv_txsize;
Jingning Han's avatar
Jingning Han committed
1066
1067
}

Angie Chiang's avatar
Angie Chiang committed
1068
static INLINE TX_SIZE get_tx_size(int plane, const MACROBLOCKD *xd) {
1069
1070
1071
1072
1073
1074
  const MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
  const MACROBLOCKD_PLANE *pd = &xd->plane[plane];
  const TX_SIZE tx_size = plane ? get_uv_tx_size(mbmi, pd) : mbmi->tx_size;
  return tx_size;
}

clang-format's avatar
clang-format committed
1075
1076
static INLINE BLOCK_SIZE
get_plane_block_size(BLOCK_SIZE bsize, const struct macroblockd_plane *pd) {
Jingning Han's avatar
Jingning Han committed
1077
1078
1079
1080
1081
1082
1083
1084
  return ss_size_lookup[bsize][pd->subsampling_x][pd->subsampling_y];
}

static INLINE void reset_skip_context(MACROBLOCKD *xd, BLOCK_SIZE bsize) {
  int i;
  for (i = 0; i < MAX_MB_PLANE; i++) {
    struct macroblockd_plane *const pd = &xd->plane[i];
    const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd);
1085
1086
1087
1088
    const int txs_wide = block_size_wide[plane_bsize] >> tx_size_wide_log2[0];
    const int txs_high = block_size_high[plane_bsize] >> tx_size_high_log2[0];
    memset(pd->above_context, 0, sizeof(ENTROPY_CONTEXT) * txs_wide);
    memset(pd->left_context, 0, sizeof(ENTROPY_CONTEXT) * txs_high);
Jingning Han's avatar
Jingning Han committed
1089
1090
1091
1092
  }
}

typedef void (*foreach_transformed_block_visitor)(int plane, int block,
Jingning Han's avatar