encodeframe.c 260 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4
5
6
7
8
9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10
11
12
13
14
15
 */

#include <limits.h>
#include <math.h>
#include <stdio.h>

Yaowu Xu's avatar
Yaowu Xu committed
16
17
18
#include "./av1_rtcd.h"
#include "./aom_dsp_rtcd.h"
#include "./aom_config.h"
Jingning Han's avatar
Jingning Han committed
19

Yaowu Xu's avatar
Yaowu Xu committed
20
#include "aom_dsp/aom_dsp_common.h"
21
#include "aom_ports/mem.h"
Yaowu Xu's avatar
Yaowu Xu committed
22
#include "aom_ports/aom_timer.h"
23
#include "aom_ports/system_state.h"
Jingning Han's avatar
Jingning Han committed
24

25
26
27
28
#include "av1/common/common.h"
#include "av1/common/entropy.h"
#include "av1/common/entropymode.h"
#include "av1/common/idct.h"
29
#include "av1/common/mv.h"
30
31
32
33
34
35
36
#include "av1/common/mvref_common.h"
#include "av1/common/pred_common.h"
#include "av1/common/quant_common.h"
#include "av1/common/reconintra.h"
#include "av1/common/reconinter.h"
#include "av1/common/seg_common.h"
#include "av1/common/tile_common.h"
Jingning Han's avatar
Jingning Han committed
37

38
39
40
#include "av1/encoder/aq_complexity.h"
#include "av1/encoder/aq_cyclicrefresh.h"
#include "av1/encoder/aq_variance.h"
41
#if CONFIG_SUPERTX
42
#include "av1/encoder/cost.h"
43
#endif
Yue Chen's avatar
Yue Chen committed
44
#if CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
45
#include "av1/common/warped_motion.h"
Yue Chen's avatar
Yue Chen committed
46
47
#endif  // CONFIG_GLOBAL_MOTION || CONFIG_WARPED_MOTION
#if CONFIG_GLOBAL_MOTION
48
#include "av1/encoder/global_motion.h"
Yue Chen's avatar
Yue Chen committed
49
#endif  // CONFIG_GLOBAL_MOTION
50
51
52
#include "av1/encoder/encodeframe.h"
#include "av1/encoder/encodemb.h"
#include "av1/encoder/encodemv.h"
Angie Chiang's avatar
Angie Chiang committed
53
54
55
#if CONFIG_LV_MAP
#include "av1/encoder/encodetxb.h"
#endif
56
57
58
59
60
61
#include "av1/encoder/ethread.h"
#include "av1/encoder/extend.h"
#include "av1/encoder/rd.h"
#include "av1/encoder/rdopt.h"
#include "av1/encoder/segmentation.h"
#include "av1/encoder/tokenize.h"
62
#if CONFIG_PVQ
63
#include "av1/common/pvq.h"
64
65
#include "av1/encoder/pvq_encoder.h"
#endif
Yaowu Xu's avatar
Yaowu Xu committed
66
#if CONFIG_AOM_HIGHBITDEPTH
67
#define IF_HBD(...) __VA_ARGS__
68
#else
69
#define IF_HBD(...)
Yaowu Xu's avatar
Yaowu Xu committed
70
#endif  // CONFIG_AOM_HIGHBITDEPTH
71

72
73
74
75
static void encode_superblock(const AV1_COMP *const cpi, ThreadData *td,
                              TOKENEXTRA **t, RUN_TYPE dry_run, int mi_row,
                              int mi_col, BLOCK_SIZE bsize,
                              PICK_MODE_CONTEXT *ctx, int *rate);
Jingning Han's avatar
Jingning Han committed
76

77
78
79
#if CONFIG_SUPERTX
static int check_intra_b(PICK_MODE_CONTEXT *ctx);

80
81
82
83
static int check_intra_sb(const AV1_COMP *cpi, const TileInfo *const tile,
                          int mi_row, int mi_col, BLOCK_SIZE bsize,
                          PC_TREE *pc_tree);
static void predict_superblock(const AV1_COMP *const cpi, ThreadData *td,
84
85
86
#if CONFIG_EXT_INTER
                               int mi_row_ori, int mi_col_ori,
#endif  // CONFIG_EXT_INTER
87
88
89
90
                               int mi_row_pred, int mi_col_pred,
                               BLOCK_SIZE bsize_pred, int b_sub8x8, int block);
static int check_supertx_sb(BLOCK_SIZE bsize, TX_SIZE supertx_size,
                            PC_TREE *pc_tree);
91
static void predict_sb_complex(const AV1_COMP *const cpi, ThreadData *td,
92
93
                               const TileInfo *const tile, int mi_row,
                               int mi_col, int mi_row_ori, int mi_col_ori,
94
                               RUN_TYPE dry_run, BLOCK_SIZE bsize,
95
96
                               BLOCK_SIZE top_bsize, uint8_t *dst_buf[3],
                               int dst_stride[3], PC_TREE *pc_tree);
97
static void update_state_sb_supertx(const AV1_COMP *const cpi, ThreadData *td,
98
99
                                    const TileInfo *const tile, int mi_row,
                                    int mi_col, BLOCK_SIZE bsize,
100
                                    RUN_TYPE dry_run, PC_TREE *pc_tree);
101
static void rd_supertx_sb(const AV1_COMP *const cpi, ThreadData *td,
102
103
104
                          const TileInfo *const tile, int mi_row, int mi_col,
                          BLOCK_SIZE bsize, int *tmp_rate, int64_t *tmp_dist,
                          TX_TYPE *best_tx, PC_TREE *pc_tree);
105
106
#endif  // CONFIG_SUPERTX

Jingning Han's avatar
Jingning Han committed
107
108
109
110
// This is used as a reference when computing the source variance for the
//  purposes of activity masking.
// Eventually this should be replaced by custom no-reference routines,
//  which will be faster.
Yaowu Xu's avatar
Yaowu Xu committed
111
static const uint8_t AV1_VAR_OFFS[MAX_SB_SIZE] = {
112
113
114
115
116
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
117
#if CONFIG_EXT_PARTITION
118
119
120
121
122
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128
123
#endif  // CONFIG_EXT_PARTITION
Jingning Han's avatar
Jingning Han committed
124
125
};

Yaowu Xu's avatar
Yaowu Xu committed
126
127
#if CONFIG_AOM_HIGHBITDEPTH
static const uint16_t AV1_HIGH_VAR_OFFS_8[MAX_SB_SIZE] = {
128
129
130
131
132
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
133
#if CONFIG_EXT_PARTITION
134
135
136
137
138
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128
139
#endif  // CONFIG_EXT_PARTITION
Jingning Han's avatar
Jingning Han committed
140
141
};

Yaowu Xu's avatar
Yaowu Xu committed
142
static const uint16_t AV1_HIGH_VAR_OFFS_10[MAX_SB_SIZE] = {
143
144
145
146
147
148
149
150
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4,
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4,
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4,
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4,
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4,
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4,
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4,
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4,
151
#if CONFIG_EXT_PARTITION
152
153
154
155
156
157
158
159
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4,
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4,
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4,
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4,
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4,
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4,
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4,
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4
160
#endif  // CONFIG_EXT_PARTITION
Jingning Han's avatar
Jingning Han committed
161
162
};

Yaowu Xu's avatar
Yaowu Xu committed
163
static const uint16_t AV1_HIGH_VAR_OFFS_12[MAX_SB_SIZE] = {
164
165
166
167
168
169
170
171
172
173
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16,
174
#if CONFIG_EXT_PARTITION
175
176
177
178
179
180
181
182
183
184
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16
185
#endif  // CONFIG_EXT_PARTITION
Jingning Han's avatar
Jingning Han committed
186
};
Yaowu Xu's avatar
Yaowu Xu committed
187
#endif  // CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
188

189
unsigned int av1_get_sby_perpixel_variance(const AV1_COMP *cpi,
Yaowu Xu's avatar
Yaowu Xu committed
190
191
                                           const struct buf_2d *ref,
                                           BLOCK_SIZE bs) {
Jingning Han's avatar
Jingning Han committed
192
  unsigned int sse;
193
  const unsigned int var =
Yaowu Xu's avatar
Yaowu Xu committed
194
      cpi->fn_ptr[bs].vf(ref->buf, ref->stride, AV1_VAR_OFFS, 0, &sse);
Jingning Han's avatar
Jingning Han committed
195
196
197
  return ROUND_POWER_OF_TWO(var, num_pels_log2_lookup[bs]);
}

Yaowu Xu's avatar
Yaowu Xu committed
198
#if CONFIG_AOM_HIGHBITDEPTH
199
unsigned int av1_high_get_sby_perpixel_variance(const AV1_COMP *cpi,
Yaowu Xu's avatar
Yaowu Xu committed
200
201
                                                const struct buf_2d *ref,
                                                BLOCK_SIZE bs, int bd) {
Jingning Han's avatar
Jingning Han committed
202
203
204
  unsigned int var, sse;
  switch (bd) {
    case 10:
Yaowu Xu's avatar
Yaowu Xu committed
205
206
207
      var =
          cpi->fn_ptr[bs].vf(ref->buf, ref->stride,
                             CONVERT_TO_BYTEPTR(AV1_HIGH_VAR_OFFS_10), 0, &sse);
Jingning Han's avatar
Jingning Han committed
208
209
      break;
    case 12:
Yaowu Xu's avatar
Yaowu Xu committed
210
211
212
      var =
          cpi->fn_ptr[bs].vf(ref->buf, ref->stride,
                             CONVERT_TO_BYTEPTR(AV1_HIGH_VAR_OFFS_12), 0, &sse);
Jingning Han's avatar
Jingning Han committed
213
214
215
      break;
    case 8:
    default:
216
217
      var =
          cpi->fn_ptr[bs].vf(ref->buf, ref->stride,
Yaowu Xu's avatar
Yaowu Xu committed
218
                             CONVERT_TO_BYTEPTR(AV1_HIGH_VAR_OFFS_8), 0, &sse);
Jingning Han's avatar
Jingning Han committed
219
220
221
222
      break;
  }
  return ROUND_POWER_OF_TWO(var, num_pels_log2_lookup[bs]);
}
Yaowu Xu's avatar
Yaowu Xu committed
223
#endif  // CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
224

225
static unsigned int get_sby_perpixel_diff_variance(const AV1_COMP *const cpi,
Jingning Han's avatar
Jingning Han committed
226
227
228
229
230
231
232
233
234
235
236
237
238
239
                                                   const struct buf_2d *ref,
                                                   int mi_row, int mi_col,
                                                   BLOCK_SIZE bs) {
  unsigned int sse, var;
  uint8_t *last_y;
  const YV12_BUFFER_CONFIG *last = get_ref_frame_buffer(cpi, LAST_FRAME);

  assert(last != NULL);
  last_y =
      &last->y_buffer[mi_row * MI_SIZE * last->y_stride + mi_col * MI_SIZE];
  var = cpi->fn_ptr[bs].vf(ref->buf, ref->stride, last_y, last->y_stride, &sse);
  return ROUND_POWER_OF_TWO(var, num_pels_log2_lookup[bs]);
}

Yaowu Xu's avatar
Yaowu Xu committed
240
241
static BLOCK_SIZE get_rd_var_based_fixed_partition(AV1_COMP *cpi, MACROBLOCK *x,
                                                   int mi_row, int mi_col) {
242
243
  unsigned int var = get_sby_perpixel_diff_variance(
      cpi, &x->plane[0].src, mi_row, mi_col, BLOCK_64X64);
Jingning Han's avatar
Jingning Han committed
244
245
246
247
248
249
250
251
252
253
254
255
  if (var < 8)
    return BLOCK_64X64;
  else if (var < 128)
    return BLOCK_32X32;
  else if (var < 2048)
    return BLOCK_16X16;
  else
    return BLOCK_8X8;
}

// Lighter version of set_offsets that only sets the mode info
// pointers.
256
257
258
259
static void set_mode_info_offsets(const AV1_COMP *const cpi,
                                  MACROBLOCK *const x, MACROBLOCKD *const xd,
                                  int mi_row, int mi_col) {
  const AV1_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
260
261
262
  const int idx_str = xd->mi_stride * mi_row + mi_col;
  xd->mi = cm->mi_grid_visible + idx_str;
  xd->mi[0] = cm->mi + idx_str;
263
  x->mbmi_ext = cpi->mbmi_ext_base + (mi_row * cm->mi_cols + mi_col);
Jingning Han's avatar
Jingning Han committed
264
265
}

266
static void set_offsets_without_segment_id(const AV1_COMP *const cpi,
267
                                           const TileInfo *const tile,
268
269
                                           MACROBLOCK *const x, int mi_row,
                                           int mi_col, BLOCK_SIZE bsize) {
270
  const AV1_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
271
  MACROBLOCKD *const xd = &x->e_mbd;
272
273
  const int mi_width = mi_size_wide[bsize];
  const int mi_height = mi_size_high[bsize];
Jingning Han's avatar
Jingning Han committed
274
275
276

  set_skip_context(xd, mi_row, mi_col);

277
  set_mode_info_offsets(cpi, x, xd, mi_row, mi_col);
Jingning Han's avatar
Jingning Han committed
278

279
280
#if CONFIG_VAR_TX
  xd->above_txfm_context = cm->above_txfm_context + mi_col;
281
  xd->left_txfm_context =
282
      xd->left_txfm_context_buffer + (mi_row & MAX_MIB_MASK);
283
  xd->max_tx_size = max_txsize_lookup[bsize];
284
285
#endif

Jingning Han's avatar
Jingning Han committed
286
  // Set up destination pointers.
Yaowu Xu's avatar
Yaowu Xu committed
287
  av1_setup_dst_planes(xd->plane, get_frame_new_buffer(cm), mi_row, mi_col);
Jingning Han's avatar
Jingning Han committed
288
289
290

  // Set up limit values for MV components.
  // Mv beyond the range do not produce new/different prediction block.
Yaowu Xu's avatar
Yaowu Xu committed
291
292
293
294
  x->mv_row_min = -(((mi_row + mi_height) * MI_SIZE) + AOM_INTERP_EXTEND);
  x->mv_col_min = -(((mi_col + mi_width) * MI_SIZE) + AOM_INTERP_EXTEND);
  x->mv_row_max = (cm->mi_rows - mi_row) * MI_SIZE + AOM_INTERP_EXTEND;
  x->mv_col_max = (cm->mi_cols - mi_col) * MI_SIZE + AOM_INTERP_EXTEND;
Jingning Han's avatar
Jingning Han committed
295

Jingning Han's avatar
Jingning Han committed
296
  set_plane_n4(xd, mi_width, mi_height);
297

Jingning Han's avatar
Jingning Han committed
298
299
  // Set up distance of MB to edge of frame in 1/8th pel units.
  assert(!(mi_col & (mi_width - 1)) && !(mi_row & (mi_height - 1)));
300
301
302
303
#if CONFIG_DEPENDENT_HORZTILES
  set_mi_row_col(xd, tile, mi_row, mi_height, mi_col, mi_width, cm->mi_rows,
                 cm->mi_cols, cm->dependent_horz_tiles);
#else
304
305
  set_mi_row_col(xd, tile, mi_row, mi_height, mi_col, mi_width, cm->mi_rows,
                 cm->mi_cols);
306
#endif
Jingning Han's avatar
Jingning Han committed
307
308

  // Set up source buffers.
Yaowu Xu's avatar
Yaowu Xu committed
309
  av1_setup_src_planes(x, cpi->Source, mi_row, mi_col);
Jingning Han's avatar
Jingning Han committed
310
311
312
313
314

  // R/D setup.
  x->rddiv = cpi->rd.RDDIV;
  x->rdmult = cpi->rd.RDMULT;

Yaowu Xu's avatar
Yaowu Xu committed
315
  // required by av1_append_sub8x8_mvs_for_idx() and av1_find_best_ref_mvs()
316
317
318
  xd->tile = *tile;
}

319
static void set_offsets(const AV1_COMP *const cpi, const TileInfo *const tile,
320
321
                        MACROBLOCK *const x, int mi_row, int mi_col,
                        BLOCK_SIZE bsize) {
322
  const AV1_COMMON *const cm = &cpi->common;
323
324
325
326
327
328
329
330
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *mbmi;
  const struct segmentation *const seg = &cm->seg;

  set_offsets_without_segment_id(cpi, tile, x, mi_row, mi_col, bsize);

  mbmi = &xd->mi[0]->mbmi;

Jingning Han's avatar
Jingning Han committed
331
332
  // Setup segment ID.
  if (seg->enabled) {
Geza Lore's avatar
Geza Lore committed
333
    if (!cpi->vaq_refresh) {
334
335
      const uint8_t *const map =
          seg->update_map ? cpi->segmentation_map : cm->last_frame_seg_map;
Jingning Han's avatar
Jingning Han committed
336
337
      mbmi->segment_id = get_segment_id(cm, map, bsize, mi_row, mi_col);
    }
Yaowu Xu's avatar
Yaowu Xu committed
338
    av1_init_plane_quantizers(cpi, x, mbmi->segment_id);
Jingning Han's avatar
Jingning Han committed
339
340
341
  } else {
    mbmi->segment_id = 0;
  }
342
343
344
345

#if CONFIG_SUPERTX
  mbmi->segment_id_supertx = MAX_SEGMENTS;
#endif  // CONFIG_SUPERTX
Jingning Han's avatar
Jingning Han committed
346
347
}

348
#if CONFIG_SUPERTX
349
static void set_offsets_supertx(const AV1_COMP *const cpi, ThreadData *td,
350
351
                                const TileInfo *const tile, int mi_row,
                                int mi_col, BLOCK_SIZE bsize) {
352
  MACROBLOCK *const x = &td->mb;
353
  const AV1_COMMON *const cm = &cpi->common;
354
  MACROBLOCKD *const xd = &x->e_mbd;
Jingning Han's avatar
Jingning Han committed
355
356
  const int mi_width = mi_size_wide[bsize];
  const int mi_height = mi_size_high[bsize];
357
358
359
#if CONFIG_DEPENDENT_HORZTILES
  set_mode_info_offsets(cpi, x, xd, mi_row, mi_col, cm->dependent_horz_tiles);
#else
360
  set_mode_info_offsets(cpi, x, xd, mi_row, mi_col);
361
#endif
362
363
364

  // Set up distance of MB to edge of frame in 1/8th pel units.
  assert(!(mi_col & (mi_width - 1)) && !(mi_row & (mi_height - 1)));
365
366
  set_mi_row_col(xd, tile, mi_row, mi_height, mi_col, mi_width, cm->mi_rows,
                 cm->mi_cols);
367
368
}

369
static void set_offsets_extend(const AV1_COMP *const cpi, ThreadData *td,
370
371
                               const TileInfo *const tile, int mi_row_pred,
                               int mi_col_pred, int mi_row_ori, int mi_col_ori,
372
                               BLOCK_SIZE bsize_pred) {
373
374
375
376
  // Used in supertx
  // (mi_row_ori, mi_col_ori, bsize_ori): region for mv
  // (mi_row_pred, mi_col_pred, bsize_pred): region to predict
  MACROBLOCK *const x = &td->mb;
377
  const AV1_COMMON *const cm = &cpi->common;
378
  MACROBLOCKD *const xd = &x->e_mbd;
Jingning Han's avatar
Jingning Han committed
379
380
  const int mi_width = mi_size_wide[bsize_pred];
  const int mi_height = mi_size_high[bsize_pred];
381

382
383
384
385
#if CONFIG_DEPENDENT_HORZTILES
  set_mode_info_offsets(cpi, x, xd, mi_row_ori, mi_col_ori,
                        cm->dependent_horz_tiles);
#else
386
  set_mode_info_offsets(cpi, x, xd, mi_row_ori, mi_col_ori);
387
#endif
388
389
390

  // Set up limit values for MV components.
  // Mv beyond the range do not produce new/different prediction block.
Yaowu Xu's avatar
Yaowu Xu committed
391
392
393
394
  x->mv_row_min = -(((mi_row_pred + mi_height) * MI_SIZE) + AOM_INTERP_EXTEND);
  x->mv_col_min = -(((mi_col_pred + mi_width) * MI_SIZE) + AOM_INTERP_EXTEND);
  x->mv_row_max = (cm->mi_rows - mi_row_pred) * MI_SIZE + AOM_INTERP_EXTEND;
  x->mv_col_max = (cm->mi_cols - mi_col_pred) * MI_SIZE + AOM_INTERP_EXTEND;
395

Jingning Han's avatar
Jingning Han committed
396
397
// Set up distance of MB to edge of frame in 1/8th pel units.
#if !CONFIG_CB4X4
Jingning Han's avatar
Jingning Han committed
398
399
  assert(!(mi_col_pred & (mi_width - mi_size_wide[BLOCK_8X8])) &&
         !(mi_row_pred & (mi_height - mi_size_high[BLOCK_8X8])));
Jingning Han's avatar
Jingning Han committed
400
#endif
401
402
  set_mi_row_col(xd, tile, mi_row_pred, mi_height, mi_col_pred, mi_width,
                 cm->mi_rows, cm->mi_cols);
403
404
  xd->up_available = (mi_row_ori > tile->mi_row_start);
  xd->left_available = (mi_col_ori > tile->mi_col_start);
405
406
407
408

  // R/D setup.
  x->rddiv = cpi->rd.RDDIV;
  x->rdmult = cpi->rd.RDMULT;
409
}
410

Yaowu Xu's avatar
Yaowu Xu committed
411
static void set_segment_id_supertx(const AV1_COMP *const cpi,
412
413
                                   MACROBLOCK *const x, const int mi_row,
                                   const int mi_col, const BLOCK_SIZE bsize) {
Yaowu Xu's avatar
Yaowu Xu committed
414
  const AV1_COMMON *cm = &cpi->common;
415
  const struct segmentation *seg = &cm->seg;
Jingning Han's avatar
Jingning Han committed
416
417
  const int miw = AOMMIN(mi_size_wide[bsize], cm->mi_cols - mi_col);
  const int mih = AOMMIN(mi_size_high[bsize], cm->mi_rows - mi_row);
418
419
420
421
  const int mi_offset = mi_row * cm->mi_stride + mi_col;
  MODE_INFO **const mip = cm->mi_grid_visible + mi_offset;
  int r, c;
  int seg_id_supertx = MAX_SEGMENTS;
422

423
424
425
426
  if (!seg->enabled) {
    seg_id_supertx = 0;
  } else {
    // Find the minimum segment_id
427
428
429
    for (r = 0; r < mih; r++)
      for (c = 0; c < miw; c++)
        seg_id_supertx =
Yaowu Xu's avatar
Yaowu Xu committed
430
            AOMMIN(mip[r * cm->mi_stride + c]->mbmi.segment_id, seg_id_supertx);
431
432
433
    assert(0 <= seg_id_supertx && seg_id_supertx < MAX_SEGMENTS);

    // Initialize plane quantisers
Yaowu Xu's avatar
Yaowu Xu committed
434
    av1_init_plane_quantizers(cpi, x, seg_id_supertx);
435
  }
436
437

  // Assign the the segment_id back to segment_id_supertx
438
439
  for (r = 0; r < mih; r++)
    for (c = 0; c < miw; c++)
440
      mip[r * cm->mi_stride + c]->mbmi.segment_id_supertx = seg_id_supertx;
441
442
443
}
#endif  // CONFIG_SUPERTX

Yaowu Xu's avatar
Yaowu Xu committed
444
static void set_block_size(AV1_COMP *const cpi, MACROBLOCK *const x,
445
                           MACROBLOCKD *const xd, int mi_row, int mi_col,
Jingning Han's avatar
Jingning Han committed
446
447
                           BLOCK_SIZE bsize) {
  if (cpi->common.mi_cols > mi_col && cpi->common.mi_rows > mi_row) {
448
    set_mode_info_offsets(cpi, x, xd, mi_row, mi_col);
Jingning Han's avatar
Jingning Han committed
449
450
451
452
    xd->mi[0]->mbmi.sb_type = bsize;
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
453
static void set_vt_partitioning(AV1_COMP *cpi, MACROBLOCK *const x,
454
455
456
                                MACROBLOCKD *const xd, VAR_TREE *vt, int mi_row,
                                int mi_col, const int64_t *const threshold,
                                const BLOCK_SIZE *const bsize_min) {
Yaowu Xu's avatar
Yaowu Xu committed
457
  AV1_COMMON *const cm = &cpi->common;
458
459
  const int hbw = mi_size_wide[vt->bsize] / 2;
  const int hbh = mi_size_high[vt->bsize] / 2;
460
461
  const int has_cols = mi_col + hbw < cm->mi_cols;
  const int has_rows = mi_row + hbh < cm->mi_rows;
Jingning Han's avatar
Jingning Han committed
462

463
  if (mi_row >= cm->mi_rows || mi_col >= cm->mi_cols) return;
Jingning Han's avatar
Jingning Han committed
464

465
466
467
468
  assert(vt->bsize >= BLOCK_8X8);

  assert(hbh == hbw);

469
470
471
472
473
  if (vt->bsize == BLOCK_8X8 && cm->frame_type != KEY_FRAME) {
    set_block_size(cpi, x, xd, mi_row, mi_col, BLOCK_8X8);
    return;
  }

474
  if (vt->force_split || (!has_cols && !has_rows)) goto split;
Jingning Han's avatar
Jingning Han committed
475
476
477
478

  // For bsize=bsize_min (16x16/8x8 for 8x8/4x4 downsampling), select if
  // variance is below threshold, otherwise split will be selected.
  // No check for vert/horiz split as too few samples for variance.
479
  if (vt->bsize == bsize_min[0]) {
480
    if (has_cols && has_rows && vt->variances.none.variance < threshold[0]) {
481
482
483
484
485
486
487
488
489
490
491
      set_block_size(cpi, x, xd, mi_row, mi_col, vt->bsize);
      return;
    } else {
      BLOCK_SIZE subsize = get_subsize(vt->bsize, PARTITION_SPLIT);
      set_block_size(cpi, x, xd, mi_row, mi_col, subsize);
      if (vt->bsize > BLOCK_8X8) {
        set_block_size(cpi, x, xd, mi_row, mi_col + hbw, subsize);
        set_block_size(cpi, x, xd, mi_row + hbh, mi_col, subsize);
        set_block_size(cpi, x, xd, mi_row + hbh, mi_col + hbw, subsize);
      }
      return;
Jingning Han's avatar
Jingning Han committed
492
    }
493
  } else if (vt->bsize > bsize_min[0]) {
Jingning Han's avatar
Jingning Han committed
494
495
    // For key frame: take split for bsize above 32X32 or very high variance.
    if (cm->frame_type == KEY_FRAME &&
496
        (vt->bsize > BLOCK_32X32 ||
497
         vt->variances.none.variance > (threshold[0] << 4))) {
498
      goto split;
Jingning Han's avatar
Jingning Han committed
499
500
    }
    // If variance is low, take the bsize (no split).
501
    if (has_cols && has_rows && vt->variances.none.variance < threshold[0]) {
502
503
      set_block_size(cpi, x, xd, mi_row, mi_col, vt->bsize);
      return;
Jingning Han's avatar
Jingning Han committed
504
505
506
    }

    // Check vertical split.
507
508
509
510
    if (has_rows) {
      BLOCK_SIZE subsize = get_subsize(vt->bsize, PARTITION_VERT);
      if (vt->variances.vert[0].variance < threshold[0] &&
          vt->variances.vert[1].variance < threshold[0] &&
Jingning Han's avatar
Jingning Han committed
511
512
          get_plane_block_size(subsize, &xd->plane[1]) < BLOCK_INVALID) {
        set_block_size(cpi, x, xd, mi_row, mi_col, subsize);
513
514
        set_block_size(cpi, x, xd, mi_row, mi_col + hbw, subsize);
        return;
Jingning Han's avatar
Jingning Han committed
515
516
517
      }
    }
    // Check horizontal split.
518
519
520
521
    if (has_cols) {
      BLOCK_SIZE subsize = get_subsize(vt->bsize, PARTITION_HORZ);
      if (vt->variances.horz[0].variance < threshold[0] &&
          vt->variances.horz[1].variance < threshold[0] &&
Jingning Han's avatar
Jingning Han committed
522
523
          get_plane_block_size(subsize, &xd->plane[1]) < BLOCK_INVALID) {
        set_block_size(cpi, x, xd, mi_row, mi_col, subsize);
524
525
        set_block_size(cpi, x, xd, mi_row + hbh, mi_col, subsize);
        return;
Jingning Han's avatar
Jingning Han committed
526
527
      }
    }
528
  }
Jingning Han's avatar
Jingning Han committed
529

530
531
532
533
534
535
536
537
538
539
540
split : {
  set_vt_partitioning(cpi, x, xd, vt->split[0], mi_row, mi_col, threshold + 1,
                      bsize_min + 1);
  set_vt_partitioning(cpi, x, xd, vt->split[1], mi_row, mi_col + hbw,
                      threshold + 1, bsize_min + 1);
  set_vt_partitioning(cpi, x, xd, vt->split[2], mi_row + hbh, mi_col,
                      threshold + 1, bsize_min + 1);
  set_vt_partitioning(cpi, x, xd, vt->split[3], mi_row + hbh, mi_col + hbw,
                      threshold + 1, bsize_min + 1);
  return;
}
Jingning Han's avatar
Jingning Han committed
541
542
543
544
545
546
}

// Set the variance split thresholds for following the block sizes:
// 0 - threshold_64x64, 1 - threshold_32x32, 2 - threshold_16x16,
// 3 - vbp_threshold_8x8. vbp_threshold_8x8 (to split to 4x4 partition) is
// currently only used on key frame.
Yaowu Xu's avatar
Yaowu Xu committed
547
548
static void set_vbp_thresholds(AV1_COMP *cpi, int64_t thresholds[], int q) {
  AV1_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
549
550
  const int is_key_frame = (cm->frame_type == KEY_FRAME);
  const int threshold_multiplier = is_key_frame ? 20 : 1;
551
552
  const int64_t threshold_base =
      (int64_t)(threshold_multiplier * cpi->y_dequant[q][1]);
Jingning Han's avatar
Jingning Han committed
553
  if (is_key_frame) {
554
    thresholds[1] = threshold_base;
Jingning Han's avatar
Jingning Han committed
555
    thresholds[2] = threshold_base >> 2;
556
557
    thresholds[3] = threshold_base >> 2;
    thresholds[4] = threshold_base << 2;
Jingning Han's avatar
Jingning Han committed
558
  } else {
559
    thresholds[2] = threshold_base;
Jingning Han's avatar
Jingning Han committed
560
    if (cm->width <= 352 && cm->height <= 288) {
561
562
      thresholds[1] = threshold_base >> 2;
      thresholds[3] = threshold_base << 3;
Jingning Han's avatar
Jingning Han committed
563
    } else {
564
565
      thresholds[1] = threshold_base;
      thresholds[2] = (5 * threshold_base) >> 2;
Jingning Han's avatar
Jingning Han committed
566
      if (cm->width >= 1920 && cm->height >= 1080)
567
568
        thresholds[2] = (7 * threshold_base) >> 2;
      thresholds[3] = threshold_base << cpi->oxcf.speed;
Jingning Han's avatar
Jingning Han committed
569
570
    }
  }
571
  thresholds[0] = INT64_MIN;
Jingning Han's avatar
Jingning Han committed
572
573
}

Yaowu Xu's avatar
Yaowu Xu committed
574
575
void av1_set_variance_partition_thresholds(AV1_COMP *cpi, int q) {
  AV1_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
  SPEED_FEATURES *const sf = &cpi->sf;
  const int is_key_frame = (cm->frame_type == KEY_FRAME);
  if (sf->partition_search_type != VAR_BASED_PARTITION &&
      sf->partition_search_type != REFERENCE_PARTITION) {
    return;
  } else {
    set_vbp_thresholds(cpi, cpi->vbp_thresholds, q);
    // The thresholds below are not changed locally.
    if (is_key_frame) {
      cpi->vbp_threshold_sad = 0;
      cpi->vbp_bsize_min = BLOCK_8X8;
    } else {
      if (cm->width <= 352 && cm->height <= 288)
        cpi->vbp_threshold_sad = 100;
      else
591
592
593
        cpi->vbp_threshold_sad = (cpi->y_dequant[q][1] << 1) > 1000
                                     ? (cpi->y_dequant[q][1] << 1)
                                     : 1000;
Jingning Han's avatar
Jingning Han committed
594
595
596
597
598
599
600
      cpi->vbp_bsize_min = BLOCK_16X16;
    }
    cpi->vbp_threshold_minmax = 15 + (q >> 3);
  }
}

// Compute the minmax over the 8x8 subblocks.
601
602
static int compute_minmax_8x8(const uint8_t *src, int src_stride,
                              const uint8_t *ref, int ref_stride,
Yaowu Xu's avatar
Yaowu Xu committed
603
#if CONFIG_AOM_HIGHBITDEPTH
604
                              int highbd,
Jingning Han's avatar
Jingning Han committed
605
#endif
606
                              int pixels_wide, int pixels_high) {
Jingning Han's avatar
Jingning Han committed
607
608
609
610
611
  int k;
  int minmax_max = 0;
  int minmax_min = 255;
  // Loop over the 4 8x8 subblocks.
  for (k = 0; k < 4; k++) {
612
613
    const int x8_idx = ((k & 1) << 3);
    const int y8_idx = ((k >> 1) << 3);
Jingning Han's avatar
Jingning Han committed
614
615
616
    int min = 0;
    int max = 0;
    if (x8_idx < pixels_wide && y8_idx < pixels_high) {
617
618
      const int src_offset = y8_idx * src_stride + x8_idx;
      const int ref_offset = y8_idx * ref_stride + x8_idx;
Yaowu Xu's avatar
Yaowu Xu committed
619
#if CONFIG_AOM_HIGHBITDEPTH
620
      if (highbd) {
Yaowu Xu's avatar
Yaowu Xu committed
621
        aom_highbd_minmax_8x8(src + src_offset, src_stride, ref + ref_offset,
622
                              ref_stride, &min, &max);
Jingning Han's avatar
Jingning Han committed
623
      } else {
Yaowu Xu's avatar
Yaowu Xu committed
624
        aom_minmax_8x8(src + src_offset, src_stride, ref + ref_offset,
625
                       ref_stride, &min, &max);
Jingning Han's avatar
Jingning Han committed
626
627
      }
#else
Yaowu Xu's avatar
Yaowu Xu committed
628
      aom_minmax_8x8(src + src_offset, src_stride, ref + ref_offset, ref_stride,
Jingning Han's avatar
Jingning Han committed
629
630
                     &min, &max);
#endif
631
632
      if ((max - min) > minmax_max) minmax_max = (max - min);
      if ((max - min) < minmax_min) minmax_min = (max - min);
Jingning Han's avatar
Jingning Han committed
633
634
635
636
637
    }
  }
  return (minmax_max - minmax_min);
}

Yaowu Xu's avatar
Yaowu Xu committed
638
#if CONFIG_AOM_HIGHBITDEPTH
639
640
641
static INLINE int avg_4x4(const uint8_t *const src, const int stride,
                          const int highbd) {
  if (highbd) {
Yaowu Xu's avatar
Yaowu Xu committed
642
    return aom_highbd_avg_4x4(src, stride);
643
  } else {
Yaowu Xu's avatar
Yaowu Xu committed
644
    return aom_avg_4x4(src, stride);
645
646
  }
}
Jingning Han's avatar
Jingning Han committed
647
#else
648
static INLINE int avg_4x4(const uint8_t *const src, const int stride) {
Yaowu Xu's avatar
Yaowu Xu committed
649
  return aom_avg_4x4(src, stride);
650
}
Jingning Han's avatar
Jingning Han committed
651
#endif
652

Yaowu Xu's avatar
Yaowu Xu committed
653
#if CONFIG_AOM_HIGHBITDEPTH
654
655
656
static INLINE int avg_8x8(const uint8_t *const src, const int stride,
                          const int highbd) {
  if (highbd) {
Yaowu Xu's avatar
Yaowu Xu committed
657
    return aom_highbd_avg_8x8(src, stride);
658
  } else {
Yaowu Xu's avatar
Yaowu Xu committed
659
    return aom_avg_8x8(src, stride);
Jingning Han's avatar
Jingning Han committed
660
661
  }
}
662
663
#else
static INLINE int avg_8x8(const uint8_t *const src, const int stride) {
Yaowu Xu's avatar
Yaowu Xu committed
664
  return aom_avg_8x8(src, stride);
665
666
}
#endif
Jingning Han's avatar
Jingning Han committed
667

668
static void init_variance_tree(VAR_TREE *const vt,
Yaowu Xu's avatar
Yaowu Xu committed
669
#if CONFIG_AOM_HIGHBITDEPTH
670
                               const int highbd,
Jingning Han's avatar
Jingning Han committed
671
#endif
672
                               BLOCK_SIZE bsize, BLOCK_SIZE leaf_size,
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
                               const int width, const int height,
                               const uint8_t *const src, const int src_stride,
                               const uint8_t *const ref, const int ref_stride) {
  assert(bsize >= leaf_size);

  vt->bsize = bsize;

  vt->force_split = 0;

  vt->src = src;
  vt->src_stride = src_stride;
  vt->ref = ref;
  vt->ref_stride = ref_stride;

  vt->width = width;
  vt->height = height;

Yaowu Xu's avatar
Yaowu Xu committed
690
#if CONFIG_AOM_HIGHBITDEPTH
691
  vt->highbd = highbd;
Yaowu Xu's avatar
Yaowu Xu committed
692
#endif  // CONFIG_AOM_HIGHBITDEPTH
693
694
695

  if (bsize > leaf_size) {
    const BLOCK_SIZE subsize = get_subsize(bsize, PARTITION_SPLIT);
696
    const int px = block_size_wide[subsize];
697
698

    init_variance_tree(vt->split[0],
Yaowu Xu's avatar
Yaowu Xu committed
699
#if CONFIG_AOM_HIGHBITDEPTH
700
                       highbd,
Yaowu Xu's avatar
Yaowu Xu committed
701
702
703
#endif  // CONFIG_AOM_HIGHBITDEPTH
                       subsize, leaf_size, AOMMIN(px, width),
                       AOMMIN(px, height), src, src_stride, ref, ref_stride);
704
    init_variance_tree(vt->split[1],
Yaowu Xu's avatar
Yaowu Xu committed
705
#if CONFIG_AOM_HIGHBITDEPTH
706
                       highbd,
Yaowu Xu's avatar
Yaowu Xu committed
707
708
#endif  // CONFIG_AOM_HIGHBITDEPTH
                       subsize, leaf_size, width - px, AOMMIN(px, height),
709
                       src + px, src_stride, ref + px, ref_stride);
710
    init_variance_tree(vt->split[2],
Yaowu Xu's avatar
Yaowu Xu committed
711
#if CONFIG_AOM_HIGHBITDEPTH
712
                       highbd,
Yaowu Xu's avatar
Yaowu Xu committed
713
714
#endif  // CONFIG_AOM_HIGHBITDEPTH
                       subsize, leaf_size, AOMMIN(px, width), height - px,
715
716
                       src + px * src_stride, src_stride, ref + px * ref_stride,
                       ref_stride);
717
    init_variance_tree(vt->split[3],
Yaowu Xu's avatar
Yaowu Xu committed
718
#if CONFIG_AOM_HIGHBITDEPTH
719
                       highbd,
Yaowu Xu's avatar
Yaowu Xu committed
720
#endif  // CONFIG_AOM_HIGHBITDEPTH
721
                       subsize, leaf_size, width - px, height - px,
722
723
724
725
726
727
728
                       src + px * src_stride + px, src_stride,
                       ref + px * ref_stride + px, ref_stride);
  }
}

// Fill the variance tree based on averaging pixel values (sub-sampling), at
// the leaf node size.
729
static void fill_variance_tree(VAR_TREE *const vt, const BLOCK_SIZE leaf_size) {
730
731
732
733
734
735
736
737
738
  if (vt->bsize > leaf_size) {
    fill_variance_tree(vt->split[0], leaf_size);
    fill_variance_tree(vt->split[1], leaf_size);
    fill_variance_tree(vt->split[2], leaf_size);
    fill_variance_tree(vt->split[3], leaf_size);
    fill_variance_node(vt);
  } else if (vt->width <= 0 || vt->height <= 0) {
    fill_variance(0, 0, 0, &vt->variances.none);
  } else {
Jingning Han's avatar
Jingning Han committed
739
740
    unsigned int sse = 0;
    int sum = 0;
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
    int src_avg;
    int ref_avg;
    assert(leaf_size == BLOCK_4X4 || leaf_size == BLOCK_8X8);
    if (leaf_size == BLOCK_4X4) {
      src_avg = avg_4x4(vt->src, vt->src_stride IF_HBD(, vt->highbd));
      ref_avg = avg_4x4(vt->ref, vt->ref_stride IF_HBD(, vt->highbd));
    } else {
      src_avg = avg_8x8(vt->src, vt->src_stride IF_HBD(, vt->highbd));
      ref_avg = avg_8x8(vt->ref, vt->ref_stride IF_HBD(, vt->highbd));
    }
    sum = src_avg - ref_avg;
    sse = sum * sum;
    fill_variance(sse, sum, 0, &vt->variances.none);
  }
}

static void refine_variance_tree(VAR_TREE *const vt, const int64_t threshold) {
  if (vt->bsize >= BLOCK_8X8) {
    if (vt->bsize == BLOCK_16X16) {
      if (vt->variances.none.variance <= threshold)
        return;
      else
        vt->force_split = 0;
    }

    refine_variance_tree(vt->split[0], threshold);
    refine_variance_tree(vt->split[1], threshold);
    refine_variance_tree(vt->split[2], threshold);
    refine_variance_tree(vt->split[3], threshold);

771
    if (vt->bsize <= BLOCK_16X16) fill_variance_node(vt);
772
773
774
775
776
777
  } else if (vt->width <= 0 || vt->height <= 0) {
    fill_variance(0, 0, 0, &vt->variances.none);
  } else {
    const int src_avg = avg_4x4(vt->src, vt->src_stride IF_HBD(, vt->highbd));
    const int ref_avg = avg_4x4(vt->ref, vt->ref_stride IF_HBD(, vt->highbd));
    const int sum = src_avg - ref_avg;
778
    const unsigned int sse = sum * sum;
779
780
781
782
783
    assert(vt->bsize == BLOCK_4X4);
    fill_variance(sse, sum, 0, &vt->variances.none);
  }
}

784
static int check_split_key_frame(VAR_TREE *const vt, const int64_t threshold) {
785
786
787
788
789
790
791
792
793
794
795
  if (vt->bsize == BLOCK_32X32) {
    vt->force_split = vt->variances.none.variance > threshold;
  } else {
    vt->force_split |= check_split_key_frame(vt->split[0], threshold);
    vt->force_split |= check_split_key_frame(vt->split[1], threshold);
    vt->force_split |= check_split_key_frame(vt->split[2], threshold);
    vt->force_split |= check_split_key_frame(vt->split[3], threshold);
  }
  return vt->force_split;
}

Yaowu Xu's avatar
Yaowu Xu committed
796
static int check_split(AV1_COMP *const cpi, VAR_TREE *const vt,
797
                       const int segment_id, const int64_t *const thresholds) {
798
799
  if (vt->bsize == BLOCK_16X16) {
    vt->force_split = vt->variances.none.variance > thresholds[0];
800
801
    if (!vt->force_split && vt->variances.none.variance > thresholds[-1] &&
        !cyclic_refresh_segment_id_boosted(segment_id)) {
802
803
804
      // We have some nominal amount of 16x16 variance (based on average),
      // compute the minmax over the 8x8 sub-blocks, and if above threshold,
      // force split to 8x8 block for this 16x16 block.
805
806
      int minmax =
          compute_minmax_8x8(vt->src, vt->src_stride, vt->ref, vt->ref_stride,
Yaowu Xu's avatar
Yaowu Xu committed
807
#if CONFIG_AOM_HIGHBITDEPTH
808
                             vt->highbd,
Jingning Han's avatar
Jingning Han committed
809
#endif
810
                             vt->width, vt->height);
811
812
813
      vt->force_split = minmax > cpi->vbp_threshold_minmax;
    }
  } else {
814
815
816
817
818
819
820
821
    vt->force_split |=
        check_split(cpi, vt->split[0], segment_id, thresholds + 1);
    vt->force_split |=
        check_split(cpi, vt->split[1], segment_id, thresholds + 1);
    vt->force_split |=
        check_split(cpi, vt->split[2], segment_id, thresholds + 1);
    vt->force_split |=
        check_split(cpi, vt->split[3], segment_id, thresholds + 1);
822
823
824

    if (vt->bsize == BLOCK_32X32 && !vt->force_split) {
      vt->force_split = vt->variances.none.variance > thresholds[0];
Jingning Han's avatar
Jingning Han committed
825
826
    }
  }
827
828

  return vt->force_split;
Jingning Han's avatar
Jingning Han committed
829
830
831
}

// This function chooses partitioning based on the variance between source and
832
833
// reconstructed last (or golden), where variance is computed for down-sampled
// inputs.
Yaowu Xu's avatar
Yaowu Xu committed
834
static void choose_partitioning(AV1_COMP *const cpi, ThreadData *const td,
835
                                const TileInfo *const tile, MACROBLOCK *const x,
836
                                const int mi_row, const int mi_col) {
Yaowu Xu's avatar
Yaowu Xu committed
837
  AV1_COMMON *const cm = &cpi->common;
838
839
  MACROBLOCKD *const xd = &x->e_mbd;
  VAR_TREE *const vt = td->var_root[cm->mib_size_log2 - MIN_MIB_SIZE_LOG2];
840
#if CONFIG_DUAL_FILTER
841
  int i;
842
#endif
843
844
845
846
  const uint8_t *src;
  const uint8_t *ref;
  int src_stride;
  int ref_stride;
847
848
  int pixels_wide = MI_SIZE * mi_size_wide[cm->sb_size];
  int pixels_high = MI_SIZE * mi_size_high[cm->sb_size];
849
  int64_t thresholds[5] = {
850
851
    cpi->vbp_thresholds[0], cpi->vbp_thresholds[1], cpi->vbp_thresholds[2],
    cpi->vbp_thresholds[3], cpi->vbp_thresholds[4],
852
  };
853
854
  BLOCK_SIZE bsize_min[5] = { BLOCK_16X16, BLOCK_16X16, BLOCK_16X16,
                              cpi->vbp_bsize_min, BLOCK_8X8 };
855
856
857
  const int start_level = cm->sb_size == BLOCK_64X64 ? 1 : 0;
  const int64_t *const thre = thresholds + start_level;
  const BLOCK_SIZE *const bmin = bsize_min + start_level;
Jingning Han's avatar
Jingning Han committed
858
859
860
861
862

  const int is_key_frame = (cm->frame_type == KEY_FRAME);
  const int low_res = (cm->width <= 352 && cm->height <= 288);

  int segment_id = CR_SEGMENT_ID_BASE;
863

Jingning Han's avatar
Jingning Han committed
864
  if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ && cm->seg.enabled) {
865
866
    const uint8_t *const map =
        cm->seg.update_map ? cpi->segmentation_map : cm->last_frame_seg_map;
867
    segment_id = get_segment_id(cm, map, cm->sb_size, mi_row, mi_col);
Jingning Han's avatar
Jingning Han committed
868
869

    if (cyclic_refresh_segment_id_boosted(segment_id)) {
Yaowu Xu's avatar
Yaowu Xu committed
870
      int q = av1_get_qindex(&cm->seg, segment_id, cm->base_qindex);
Jingning Han's avatar
Jingning Han committed
871
872
873
874
      set_vbp_thresholds(cpi, thresholds, q);
    }
  }

875
  set_offsets(cpi, tile, x, mi_row, mi_col, cm->sb_size);
Jingning Han's avatar
Jingning Han committed
876

877
878
  if (xd->mb_to_right_edge < 0) pixels_wide += (xd->mb_to_right_edge >> 3);
  if (xd->mb_to_bottom_edge < 0) pixels_high += (xd->mb_to_bottom_edge >> 3);
Jingning Han's avatar
Jingning Han committed
879

880
881
  src = x->plane[0].src.buf;
  src_stride = x->plane[0].src.stride;
Jingning Han's avatar
Jingning Han committed
882

Yunqing Wang's avatar
Yunqing Wang committed
883
  if (!is_key_frame) {
Jingning Han's avatar
Jingning Han committed
884
885
    MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
    const YV12_BUFFER_CONFIG *yv12 = get_ref_frame_buffer(cpi, LAST_FRAME);
886
    const YV12_BUFFER_CONFIG *yv12_g = get_ref_frame_buffer(cpi, GOLDEN_FRAME);
Jingning Han's avatar
Jingning Han committed
887
    unsigned int y_sad, y_sad_g;
888

889
890
891
    const int hbs = cm->mib_size / 2;
    const int split_vert = mi_col + hbs >= cm->mi_cols;
    const int split_horz = mi_row + hbs >= cm->mi_rows;
892
893
    BLOCK_SIZE bsize;

894
    if (split_vert && split_horz)
895
      bsize = get_subsize(cm->sb_size, PARTITION_SPLIT);
896
    else if (split_vert)
897
      bsize = get_subsize(cm->sb_size, PARTITION_VERT);
898
    else if (split_horz)
899
      bsize = get_subsize(cm->sb_size, PARTITION_HORZ);
900
    else
901
      bsize = cm->sb_size;
Jingning Han's avatar
Jingning Han committed
902
903
904
905

    assert(yv12 != NULL);

    if (yv12_g && yv12_g != yv12) {
Yaowu Xu's avatar
Yaowu Xu committed
906
907
      av1_setup_pre_planes(xd, 0, yv12_g, mi_row, mi_col,
                           &cm->frame_refs[GOLDEN_FRAME - 1].sf);
908
909
910
      y_sad_g = cpi->fn_ptr[bsize].sdf(
          x->plane[0].src.buf, x->plane[0].src.stride, xd->plane[0].pre[0].buf,
          xd->plane[0].pre[0].stride);
Jingning Han's avatar
Jingning Han committed
911
912
913
914
    } else {
      y_sad_g = UINT_MAX;
    }

Yaowu Xu's avatar
Yaowu Xu committed
915
916
    av1_setup_pre_planes(xd, 0, yv12, mi_row, mi_col,
                         &cm->frame_refs[LAST_FRAME - 1].sf);
Jingning Han's avatar
Jingning Han committed
917
    mbmi->ref_frame[0] = LAST_FRAME;
Emil Keyder's avatar
Emil Keyder committed
918
    mbmi->ref_frame[1] = NONE_FRAME;