encodeframe.c 121 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
4
5
6
7
8
9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10
11
12
13
14
15
 */

#include <limits.h>
#include <math.h>
#include <stdio.h>

Yaowu Xu's avatar
Yaowu Xu committed
16
#include "./av1_rtcd.h"
Adrian Grange's avatar
Adrian Grange committed
17
18
#include "./aom_dsp_rtcd.h"
#include "./aom_config.h"
Jingning Han's avatar
Jingning Han committed
19

Adrian Grange's avatar
Adrian Grange committed
20
#include "aom_dsp/aom_dsp_common.h"
Yaowu Xu's avatar
Yaowu Xu committed
21
#include "aom_ports/mem.h"
Adrian Grange's avatar
Adrian Grange committed
22
#include "aom_ports/aom_timer.h"
Yaowu Xu's avatar
Yaowu Xu committed
23
#include "aom_ports/system_state.h"
Jingning Han's avatar
Jingning Han committed
24

Yaowu Xu's avatar
Yaowu Xu committed
25
26
27
28
29
30
31
32
33
34
35
#include "av1/common/common.h"
#include "av1/common/entropy.h"
#include "av1/common/entropymode.h"
#include "av1/common/idct.h"
#include "av1/common/mvref_common.h"
#include "av1/common/pred_common.h"
#include "av1/common/quant_common.h"
#include "av1/common/reconintra.h"
#include "av1/common/reconinter.h"
#include "av1/common/seg_common.h"
#include "av1/common/tile_common.h"
Jingning Han's avatar
Jingning Han committed
36

Yaowu Xu's avatar
Yaowu Xu committed
37
38
39
40
41
42
43
44
45
46
47
48
#include "av1/encoder/aq_complexity.h"
#include "av1/encoder/aq_cyclicrefresh.h"
#include "av1/encoder/aq_variance.h"
#include "av1/encoder/encodeframe.h"
#include "av1/encoder/encodemb.h"
#include "av1/encoder/encodemv.h"
#include "av1/encoder/ethread.h"
#include "av1/encoder/extend.h"
#include "av1/encoder/rd.h"
#include "av1/encoder/rdopt.h"
#include "av1/encoder/segmentation.h"
#include "av1/encoder/tokenize.h"
Jingning Han's avatar
Jingning Han committed
49

50
51
52
53
#if CONFIG_PVQ
#include "av1/encoder/pvq_encoder.h"
#endif

54
55
56
57
static void encode_superblock(const AV1_COMP *const cpi, ThreadData *td,
                              TOKENEXTRA **t, int output_enabled, int mi_row,
                              int mi_col, BLOCK_SIZE bsize,
                              PICK_MODE_CONTEXT *ctx);
Jingning Han's avatar
Jingning Han committed
58
59
60
61
62

// This is used as a reference when computing the source variance for the
//  purposes of activity masking.
// Eventually this should be replaced by custom no-reference routines,
//  which will be faster.
Adrian Grange's avatar
Adrian Grange committed
63
static const uint8_t AV1_VAR_OFFS[64] = {
clang-format's avatar
clang-format committed
64
65
66
67
68
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128
Jingning Han's avatar
Jingning Han committed
69
70
};

71
#if CONFIG_AOM_HIGHBITDEPTH
Adrian Grange's avatar
Adrian Grange committed
72
static const uint16_t AV1_HIGH_VAR_OFFS_8[64] = {
clang-format's avatar
clang-format committed
73
74
75
76
77
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128
Jingning Han's avatar
Jingning Han committed
78
79
};

Adrian Grange's avatar
Adrian Grange committed
80
static const uint16_t AV1_HIGH_VAR_OFFS_10[64] = {
clang-format's avatar
clang-format committed
81
82
83
84
85
86
87
88
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4,
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4,
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4,
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4,
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4,
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4,
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4,
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4
Jingning Han's avatar
Jingning Han committed
89
90
};

Adrian Grange's avatar
Adrian Grange committed
91
static const uint16_t AV1_HIGH_VAR_OFFS_12[64] = {
clang-format's avatar
clang-format committed
92
93
94
95
96
97
98
99
100
101
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16
Jingning Han's avatar
Jingning Han committed
102
};
103
#endif  // CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
104

105
unsigned int av1_get_sby_perpixel_variance(const AV1_COMP *cpi,
clang-format's avatar
clang-format committed
106
107
                                           const struct buf_2d *ref,
                                           BLOCK_SIZE bs) {
Jingning Han's avatar
Jingning Han committed
108
  unsigned int sse;
clang-format's avatar
clang-format committed
109
  const unsigned int var =
Adrian Grange's avatar
Adrian Grange committed
110
      cpi->fn_ptr[bs].vf(ref->buf, ref->stride, AV1_VAR_OFFS, 0, &sse);
Jingning Han's avatar
Jingning Han committed
111
112
113
  return ROUND_POWER_OF_TWO(var, num_pels_log2_lookup[bs]);
}

114
#if CONFIG_AOM_HIGHBITDEPTH
115
unsigned int av1_high_get_sby_perpixel_variance(const AV1_COMP *cpi,
clang-format's avatar
clang-format committed
116
117
                                                const struct buf_2d *ref,
                                                BLOCK_SIZE bs, int bd) {
Jingning Han's avatar
Jingning Han committed
118
119
120
  unsigned int var, sse;
  switch (bd) {
    case 10:
clang-format's avatar
clang-format committed
121
122
      var =
          cpi->fn_ptr[bs].vf(ref->buf, ref->stride,
Adrian Grange's avatar
Adrian Grange committed
123
                             CONVERT_TO_BYTEPTR(AV1_HIGH_VAR_OFFS_10), 0, &sse);
Jingning Han's avatar
Jingning Han committed
124
125
      break;
    case 12:
clang-format's avatar
clang-format committed
126
127
      var =
          cpi->fn_ptr[bs].vf(ref->buf, ref->stride,
Adrian Grange's avatar
Adrian Grange committed
128
                             CONVERT_TO_BYTEPTR(AV1_HIGH_VAR_OFFS_12), 0, &sse);
Jingning Han's avatar
Jingning Han committed
129
130
131
      break;
    case 8:
    default:
clang-format's avatar
clang-format committed
132
133
      var =
          cpi->fn_ptr[bs].vf(ref->buf, ref->stride,
Adrian Grange's avatar
Adrian Grange committed
134
                             CONVERT_TO_BYTEPTR(AV1_HIGH_VAR_OFFS_8), 0, &sse);
Jingning Han's avatar
Jingning Han committed
135
136
137
138
      break;
  }
  return ROUND_POWER_OF_TWO(var, num_pels_log2_lookup[bs]);
}
139
#endif  // CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
140

141
static unsigned int get_sby_perpixel_diff_variance(const AV1_COMP *const cpi,
Jingning Han's avatar
Jingning Han committed
142
143
144
145
146
147
148
149
150
151
152
153
154
155
                                                   const struct buf_2d *ref,
                                                   int mi_row, int mi_col,
                                                   BLOCK_SIZE bs) {
  unsigned int sse, var;
  uint8_t *last_y;
  const YV12_BUFFER_CONFIG *last = get_ref_frame_buffer(cpi, LAST_FRAME);

  assert(last != NULL);
  last_y =
      &last->y_buffer[mi_row * MI_SIZE * last->y_stride + mi_col * MI_SIZE];
  var = cpi->fn_ptr[bs].vf(ref->buf, ref->stride, last_y, last->y_stride, &sse);
  return ROUND_POWER_OF_TWO(var, num_pels_log2_lookup[bs]);
}

clang-format's avatar
clang-format committed
156
157
static BLOCK_SIZE get_rd_var_based_fixed_partition(AV1_COMP *cpi, MACROBLOCK *x,
                                                   int mi_row, int mi_col) {
clang-format's avatar
clang-format committed
158
159
  unsigned int var = get_sby_perpixel_diff_variance(
      cpi, &x->plane[0].src, mi_row, mi_col, BLOCK_64X64);
Jingning Han's avatar
Jingning Han committed
160
161
162
163
164
165
166
167
168
169
170
171
  if (var < 8)
    return BLOCK_64X64;
  else if (var < 128)
    return BLOCK_32X32;
  else if (var < 2048)
    return BLOCK_16X16;
  else
    return BLOCK_8X8;
}

// Lighter version of set_offsets that only sets the mode info
// pointers.
172
static INLINE void set_mode_info_offsets(const AV1_COMP *const cpi,
Jingning Han's avatar
Jingning Han committed
173
                                         MACROBLOCK *const x,
clang-format's avatar
clang-format committed
174
                                         MACROBLOCKD *const xd, int mi_row,
Jingning Han's avatar
Jingning Han committed
175
                                         int mi_col) {
176
  const AV1_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
177
178
179
  const int idx_str = xd->mi_stride * mi_row + mi_col;
  xd->mi = cm->mi_grid_visible + idx_str;
  xd->mi[0] = cm->mi + idx_str;
Jingning Han's avatar
Jingning Han committed
180
  x->mbmi_ext = cpi->mbmi_ext_base + (mi_row * cm->mi_cols + mi_col);
Jingning Han's avatar
Jingning Han committed
181
182
}

183
static void set_offsets(const AV1_COMP *const cpi, const TileInfo *const tile,
Jingning Han's avatar
Jingning Han committed
184
185
                        MACROBLOCK *const x, int mi_row, int mi_col,
                        BLOCK_SIZE bsize) {
186
  const AV1_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
187
188
189
190
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *mbmi;
  const int mi_width = num_8x8_blocks_wide_lookup[bsize];
  const int mi_height = num_8x8_blocks_high_lookup[bsize];
191
192
  const int bwl = b_width_log2_lookup[AOMMAX(bsize, BLOCK_8X8)];
  const int bhl = b_height_log2_lookup[AOMMAX(bsize, BLOCK_8X8)];
Jingning Han's avatar
Jingning Han committed
193
194
195
196
  const struct segmentation *const seg = &cm->seg;

  set_skip_context(xd, mi_row, mi_col);

Jingning Han's avatar
Jingning Han committed
197
  set_mode_info_offsets(cpi, x, xd, mi_row, mi_col);
Jingning Han's avatar
Jingning Han committed
198
199
200
201

  mbmi = &xd->mi[0]->mbmi;

  // Set up destination pointers.
202
  av1_setup_dst_planes(xd->plane, get_frame_new_buffer(cm), mi_row, mi_col);
Jingning Han's avatar
Jingning Han committed
203
204
205

  // Set up limit values for MV components.
  // Mv beyond the range do not produce new/different prediction block.
Adrian Grange's avatar
Adrian Grange committed
206
207
208
209
  x->mv_row_min = -(((mi_row + mi_height) * MI_SIZE) + AOM_INTERP_EXTEND);
  x->mv_col_min = -(((mi_col + mi_width) * MI_SIZE) + AOM_INTERP_EXTEND);
  x->mv_row_max = (cm->mi_rows - mi_row) * MI_SIZE + AOM_INTERP_EXTEND;
  x->mv_col_max = (cm->mi_cols - mi_col) * MI_SIZE + AOM_INTERP_EXTEND;
Jingning Han's avatar
Jingning Han committed
210

211
212
  set_plane_n4(xd, mi_width, mi_height, bwl, bhl);

Jingning Han's avatar
Jingning Han committed
213
214
  // Set up distance of MB to edge of frame in 1/8th pel units.
  assert(!(mi_col & (mi_width - 1)) && !(mi_row & (mi_height - 1)));
clang-format's avatar
clang-format committed
215
216
  set_mi_row_col(xd, tile, mi_row, mi_height, mi_col, mi_width, cm->mi_rows,
                 cm->mi_cols);
Jingning Han's avatar
Jingning Han committed
217
218

  // Set up source buffers.
219
  av1_setup_src_planes(x, cpi->Source, mi_row, mi_col);
Jingning Han's avatar
Jingning Han committed
220
221
222
223
224
225
226
227

  // R/D setup.
  x->rddiv = cpi->rd.RDDIV;
  x->rdmult = cpi->rd.RDMULT;

  // Setup segment ID.
  if (seg->enabled) {
    if (cpi->oxcf.aq_mode != VARIANCE_AQ) {
clang-format's avatar
clang-format committed
228
229
      const uint8_t *const map =
          seg->update_map ? cpi->segmentation_map : cm->last_frame_seg_map;
Jingning Han's avatar
Jingning Han committed
230
231
      mbmi->segment_id = get_segment_id(cm, map, bsize, mi_row, mi_col);
    }
232
    av1_init_plane_quantizers(cpi, x);
Jingning Han's avatar
Jingning Han committed
233
234
235
236
  } else {
    mbmi->segment_id = 0;
  }

237
  // required by av1_append_sub8x8_mvs_for_idx() and av1_find_best_ref_mvs()
Jingning Han's avatar
Jingning Han committed
238
239
240
  xd->tile = *tile;
}

241
static void set_block_size(AV1_COMP *const cpi, MACROBLOCK *const x,
clang-format's avatar
clang-format committed
242
                           MACROBLOCKD *const xd, int mi_row, int mi_col,
Jingning Han's avatar
Jingning Han committed
243
244
                           BLOCK_SIZE bsize) {
  if (cpi->common.mi_cols > mi_col && cpi->common.mi_rows > mi_row) {
Jingning Han's avatar
Jingning Han committed
245
    set_mode_info_offsets(cpi, x, xd, mi_row, mi_col);
Jingning Han's avatar
Jingning Han committed
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
    xd->mi[0]->mbmi.sb_type = bsize;
  }
}

typedef struct {
  int64_t sum_square_error;
  int64_t sum_error;
  int log2_count;
  int variance;
} var;

typedef struct {
  var none;
  var horz[2];
  var vert[2];
} partition_variance;

typedef struct {
  partition_variance part_variances;
  var split[4];
} v4x4;

typedef struct {
  partition_variance part_variances;
  v4x4 split[4];
} v8x8;

typedef struct {
  partition_variance part_variances;
  v8x8 split[4];
} v16x16;

typedef struct {
  partition_variance part_variances;
  v16x16 split[4];
} v32x32;

typedef struct {
  partition_variance part_variances;
  v32x32 split[4];
} v64x64;

typedef struct {
  partition_variance *part_variances;
  var *split[4];
} variance_node;

typedef enum {
  V16X16,
  V32X32,
  V64X64,
} TREE_LEVEL;

static void tree_to_node(void *data, BLOCK_SIZE bsize, variance_node *node) {
  int i;
  node->part_variances = NULL;
  switch (bsize) {
    case BLOCK_64X64: {
clang-format's avatar
clang-format committed
304
      v64x64 *vt = (v64x64 *)data;
Jingning Han's avatar
Jingning Han committed
305
306
307
308
309
310
      node->part_variances = &vt->part_variances;
      for (i = 0; i < 4; i++)
        node->split[i] = &vt->split[i].part_variances.none;
      break;
    }
    case BLOCK_32X32: {
clang-format's avatar
clang-format committed
311
      v32x32 *vt = (v32x32 *)data;
Jingning Han's avatar
Jingning Han committed
312
313
314
315
316
317
      node->part_variances = &vt->part_variances;
      for (i = 0; i < 4; i++)
        node->split[i] = &vt->split[i].part_variances.none;
      break;
    }
    case BLOCK_16X16: {
clang-format's avatar
clang-format committed
318
      v16x16 *vt = (v16x16 *)data;
Jingning Han's avatar
Jingning Han committed
319
320
321
322
323
324
      node->part_variances = &vt->part_variances;
      for (i = 0; i < 4; i++)
        node->split[i] = &vt->split[i].part_variances.none;
      break;
    }
    case BLOCK_8X8: {
clang-format's avatar
clang-format committed
325
      v8x8 *vt = (v8x8 *)data;
Jingning Han's avatar
Jingning Han committed
326
327
328
329
330
331
      node->part_variances = &vt->part_variances;
      for (i = 0; i < 4; i++)
        node->split[i] = &vt->split[i].part_variances.none;
      break;
    }
    case BLOCK_4X4: {
clang-format's avatar
clang-format committed
332
      v4x4 *vt = (v4x4 *)data;
Jingning Han's avatar
Jingning Han committed
333
      node->part_variances = &vt->part_variances;
clang-format's avatar
clang-format committed
334
      for (i = 0; i < 4; i++) node->split[i] = &vt->split[i];
Jingning Han's avatar
Jingning Han committed
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
      break;
    }
    default: {
      assert(0);
      break;
    }
  }
}

// Set variance values given sum square error, sum error, count.
static void fill_variance(int64_t s2, int64_t s, int c, var *v) {
  v->sum_square_error = s2;
  v->sum_error = s;
  v->log2_count = c;
}

static void get_variance(var *v) {
clang-format's avatar
clang-format committed
352
353
354
355
  v->variance =
      (int)(256 * (v->sum_square_error -
                   ((v->sum_error * v->sum_error) >> v->log2_count)) >>
            v->log2_count);
Jingning Han's avatar
Jingning Han committed
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
}

static void sum_2_variances(const var *a, const var *b, var *r) {
  assert(a->log2_count == b->log2_count);
  fill_variance(a->sum_square_error + b->sum_square_error,
                a->sum_error + b->sum_error, a->log2_count + 1, r);
}

static void fill_variance_tree(void *data, BLOCK_SIZE bsize) {
  variance_node node;
  memset(&node, 0, sizeof(node));
  tree_to_node(data, bsize, &node);
  sum_2_variances(node.split[0], node.split[1], &node.part_variances->horz[0]);
  sum_2_variances(node.split[2], node.split[3], &node.part_variances->horz[1]);
  sum_2_variances(node.split[0], node.split[2], &node.part_variances->vert[0]);
  sum_2_variances(node.split[1], node.split[3], &node.part_variances->vert[1]);
  sum_2_variances(&node.part_variances->vert[0], &node.part_variances->vert[1],
                  &node.part_variances->none);
}

376
static int set_vt_partitioning(AV1_COMP *cpi, MACROBLOCK *const x,
clang-format's avatar
clang-format committed
377
378
379
                               MACROBLOCKD *const xd, void *data,
                               BLOCK_SIZE bsize, int mi_row, int mi_col,
                               int64_t threshold, BLOCK_SIZE bsize_min,
Jingning Han's avatar
Jingning Han committed
380
                               int force_split) {
381
  AV1_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
382
383
384
385
386
387
388
389
  variance_node vt;
  const int block_width = num_8x8_blocks_wide_lookup[bsize];
  const int block_height = num_8x8_blocks_high_lookup[bsize];
  const int low_res = (cm->width <= 352 && cm->height <= 288);

  assert(block_height == block_width);
  tree_to_node(data, bsize, &vt);

clang-format's avatar
clang-format committed
390
  if (force_split == 1) return 0;
Jingning Han's avatar
Jingning Han committed
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412

  // For bsize=bsize_min (16x16/8x8 for 8x8/4x4 downsampling), select if
  // variance is below threshold, otherwise split will be selected.
  // No check for vert/horiz split as too few samples for variance.
  if (bsize == bsize_min) {
    // Variance already computed to set the force_split.
    if (low_res || cm->frame_type == KEY_FRAME)
      get_variance(&vt.part_variances->none);
    if (mi_col + block_width / 2 < cm->mi_cols &&
        mi_row + block_height / 2 < cm->mi_rows &&
        vt.part_variances->none.variance < threshold) {
      set_block_size(cpi, x, xd, mi_row, mi_col, bsize);
      return 1;
    }
    return 0;
  } else if (bsize > bsize_min) {
    // Variance already computed to set the force_split.
    if (low_res || cm->frame_type == KEY_FRAME)
      get_variance(&vt.part_variances->none);
    // For key frame: take split for bsize above 32X32 or very high variance.
    if (cm->frame_type == KEY_FRAME &&
        (bsize > BLOCK_32X32 ||
clang-format's avatar
clang-format committed
413
         vt.part_variances->none.variance > (threshold << 4))) {
Jingning Han's avatar
Jingning Han committed
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
      return 0;
    }
    // If variance is low, take the bsize (no split).
    if (mi_col + block_width / 2 < cm->mi_cols &&
        mi_row + block_height / 2 < cm->mi_rows &&
        vt.part_variances->none.variance < threshold) {
      set_block_size(cpi, x, xd, mi_row, mi_col, bsize);
      return 1;
    }

    // Check vertical split.
    if (mi_row + block_height / 2 < cm->mi_rows) {
      BLOCK_SIZE subsize = get_subsize(bsize, PARTITION_VERT);
      get_variance(&vt.part_variances->vert[0]);
      get_variance(&vt.part_variances->vert[1]);
      if (vt.part_variances->vert[0].variance < threshold &&
          vt.part_variances->vert[1].variance < threshold &&
          get_plane_block_size(subsize, &xd->plane[1]) < BLOCK_INVALID) {
        set_block_size(cpi, x, xd, mi_row, mi_col, subsize);
        set_block_size(cpi, x, xd, mi_row, mi_col + block_width / 2, subsize);
        return 1;
      }
    }
    // Check horizontal split.
    if (mi_col + block_width / 2 < cm->mi_cols) {
      BLOCK_SIZE subsize = get_subsize(bsize, PARTITION_HORZ);
      get_variance(&vt.part_variances->horz[0]);
      get_variance(&vt.part_variances->horz[1]);
      if (vt.part_variances->horz[0].variance < threshold &&
          vt.part_variances->horz[1].variance < threshold &&
          get_plane_block_size(subsize, &xd->plane[1]) < BLOCK_INVALID) {
        set_block_size(cpi, x, xd, mi_row, mi_col, subsize);
        set_block_size(cpi, x, xd, mi_row + block_height / 2, mi_col, subsize);
        return 1;
      }
    }

    return 0;
  }
  return 0;
}

// Set the variance split thresholds for following the block sizes:
// 0 - threshold_64x64, 1 - threshold_32x32, 2 - threshold_16x16,
// 3 - vbp_threshold_8x8. vbp_threshold_8x8 (to split to 4x4 partition) is
// currently only used on key frame.
460
461
static void set_vbp_thresholds(AV1_COMP *cpi, int64_t thresholds[], int q) {
  AV1_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
462
463
  const int is_key_frame = (cm->frame_type == KEY_FRAME);
  const int threshold_multiplier = is_key_frame ? 20 : 1;
clang-format's avatar
clang-format committed
464
465
  const int64_t threshold_base =
      (int64_t)(threshold_multiplier * cpi->y_dequant[q][1]);
Jingning Han's avatar
Jingning Han committed
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
  if (is_key_frame) {
    thresholds[0] = threshold_base;
    thresholds[1] = threshold_base >> 2;
    thresholds[2] = threshold_base >> 2;
    thresholds[3] = threshold_base << 2;
  } else {
    thresholds[1] = threshold_base;
    if (cm->width <= 352 && cm->height <= 288) {
      thresholds[0] = threshold_base >> 2;
      thresholds[2] = threshold_base << 3;
    } else {
      thresholds[0] = threshold_base;
      thresholds[1] = (5 * threshold_base) >> 2;
      if (cm->width >= 1920 && cm->height >= 1080)
        thresholds[1] = (7 * threshold_base) >> 2;
      thresholds[2] = threshold_base << cpi->oxcf.speed;
    }
  }
}

486
487
void av1_set_variance_partition_thresholds(AV1_COMP *cpi, int q) {
  AV1_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
  SPEED_FEATURES *const sf = &cpi->sf;
  const int is_key_frame = (cm->frame_type == KEY_FRAME);
  if (sf->partition_search_type != VAR_BASED_PARTITION &&
      sf->partition_search_type != REFERENCE_PARTITION) {
    return;
  } else {
    set_vbp_thresholds(cpi, cpi->vbp_thresholds, q);
    // The thresholds below are not changed locally.
    if (is_key_frame) {
      cpi->vbp_threshold_sad = 0;
      cpi->vbp_bsize_min = BLOCK_8X8;
    } else {
      if (cm->width <= 352 && cm->height <= 288)
        cpi->vbp_threshold_sad = 100;
      else
clang-format's avatar
clang-format committed
503
504
505
        cpi->vbp_threshold_sad = (cpi->y_dequant[q][1] << 1) > 1000
                                     ? (cpi->y_dequant[q][1] << 1)
                                     : 1000;
Jingning Han's avatar
Jingning Han committed
506
507
508
509
510
511
512
513
514
      cpi->vbp_bsize_min = BLOCK_16X16;
    }
    cpi->vbp_threshold_minmax = 15 + (q >> 3);
  }
}

// Compute the minmax over the 8x8 subblocks.
static int compute_minmax_8x8(const uint8_t *s, int sp, const uint8_t *d,
                              int dp, int x16_idx, int y16_idx,
515
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
516
517
                              int highbd_flag,
#endif
clang-format's avatar
clang-format committed
518
                              int pixels_wide, int pixels_high) {
Jingning Han's avatar
Jingning Han committed
519
520
521
522
523
524
525
526
527
528
  int k;
  int minmax_max = 0;
  int minmax_min = 255;
  // Loop over the 4 8x8 subblocks.
  for (k = 0; k < 4; k++) {
    int x8_idx = x16_idx + ((k & 1) << 3);
    int y8_idx = y16_idx + ((k >> 1) << 3);
    int min = 0;
    int max = 0;
    if (x8_idx < pixels_wide && y8_idx < pixels_high) {
529
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
530
      if (highbd_flag & YV12_FLAG_HIGHBITDEPTH) {
Adrian Grange's avatar
Adrian Grange committed
531
        aom_highbd_minmax_8x8(s + y8_idx * sp + x8_idx, sp,
clang-format's avatar
clang-format committed
532
                              d + y8_idx * dp + x8_idx, dp, &min, &max);
Jingning Han's avatar
Jingning Han committed
533
      } else {
Adrian Grange's avatar
Adrian Grange committed
534
        aom_minmax_8x8(s + y8_idx * sp + x8_idx, sp, d + y8_idx * dp + x8_idx,
clang-format's avatar
clang-format committed
535
                       dp, &min, &max);
Jingning Han's avatar
Jingning Han committed
536
537
      }
#else
Adrian Grange's avatar
Adrian Grange committed
538
      aom_minmax_8x8(s + y8_idx * sp + x8_idx, sp, d + y8_idx * dp + x8_idx, dp,
Jingning Han's avatar
Jingning Han committed
539
540
                     &min, &max);
#endif
clang-format's avatar
clang-format committed
541
542
      if ((max - min) > minmax_max) minmax_max = (max - min);
      if ((max - min) < minmax_min) minmax_min = (max - min);
Jingning Han's avatar
Jingning Han committed
543
544
545
546
547
548
549
    }
  }
  return (minmax_max - minmax_min);
}

static void fill_variance_4x4avg(const uint8_t *s, int sp, const uint8_t *d,
                                 int dp, int x8_idx, int y8_idx, v8x8 *vst,
550
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
551
552
                                 int highbd_flag,
#endif
clang-format's avatar
clang-format committed
553
                                 int pixels_wide, int pixels_high,
Jingning Han's avatar
Jingning Han committed
554
555
556
557
558
559
560
561
562
563
                                 int is_key_frame) {
  int k;
  for (k = 0; k < 4; k++) {
    int x4_idx = x8_idx + ((k & 1) << 2);
    int y4_idx = y8_idx + ((k >> 1) << 2);
    unsigned int sse = 0;
    int sum = 0;
    if (x4_idx < pixels_wide && y4_idx < pixels_high) {
      int s_avg;
      int d_avg = 128;
564
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
565
      if (highbd_flag & YV12_FLAG_HIGHBITDEPTH) {
Adrian Grange's avatar
Adrian Grange committed
566
        s_avg = aom_highbd_avg_4x4(s + y4_idx * sp + x4_idx, sp);
Jingning Han's avatar
Jingning Han committed
567
        if (!is_key_frame)
Adrian Grange's avatar
Adrian Grange committed
568
          d_avg = aom_highbd_avg_4x4(d + y4_idx * dp + x4_idx, dp);
Jingning Han's avatar
Jingning Han committed
569
      } else {
Adrian Grange's avatar
Adrian Grange committed
570
571
        s_avg = aom_avg_4x4(s + y4_idx * sp + x4_idx, sp);
        if (!is_key_frame) d_avg = aom_avg_4x4(d + y4_idx * dp + x4_idx, dp);
Jingning Han's avatar
Jingning Han committed
572
573
      }
#else
Adrian Grange's avatar
Adrian Grange committed
574
575
      s_avg = aom_avg_4x4(s + y4_idx * sp + x4_idx, sp);
      if (!is_key_frame) d_avg = aom_avg_4x4(d + y4_idx * dp + x4_idx, dp);
Jingning Han's avatar
Jingning Han committed
576
577
578
579
580
581
582
583
584
585
#endif
      sum = s_avg - d_avg;
      sse = sum * sum;
    }
    fill_variance(sse, sum, 0, &vst->split[k].part_variances.none);
  }
}

static void fill_variance_8x8avg(const uint8_t *s, int sp, const uint8_t *d,
                                 int dp, int x16_idx, int y16_idx, v16x16 *vst,
586
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
587
588
                                 int highbd_flag,
#endif
clang-format's avatar
clang-format committed
589
                                 int pixels_wide, int pixels_high,
Jingning Han's avatar
Jingning Han committed
590
591
592
593
594
595
596
597
598
599
                                 int is_key_frame) {
  int k;
  for (k = 0; k < 4; k++) {
    int x8_idx = x16_idx + ((k & 1) << 3);
    int y8_idx = y16_idx + ((k >> 1) << 3);
    unsigned int sse = 0;
    int sum = 0;
    if (x8_idx < pixels_wide && y8_idx < pixels_high) {
      int s_avg;
      int d_avg = 128;
600
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
601
      if (highbd_flag & YV12_FLAG_HIGHBITDEPTH) {
Adrian Grange's avatar
Adrian Grange committed
602
        s_avg = aom_highbd_avg_8x8(s + y8_idx * sp + x8_idx, sp);
Jingning Han's avatar
Jingning Han committed
603
        if (!is_key_frame)
Adrian Grange's avatar
Adrian Grange committed
604
          d_avg = aom_highbd_avg_8x8(d + y8_idx * dp + x8_idx, dp);
Jingning Han's avatar
Jingning Han committed
605
      } else {
Adrian Grange's avatar
Adrian Grange committed
606
607
        s_avg = aom_avg_8x8(s + y8_idx * sp + x8_idx, sp);
        if (!is_key_frame) d_avg = aom_avg_8x8(d + y8_idx * dp + x8_idx, dp);
Jingning Han's avatar
Jingning Han committed
608
609
      }
#else
Adrian Grange's avatar
Adrian Grange committed
610
611
      s_avg = aom_avg_8x8(s + y8_idx * sp + x8_idx, sp);
      if (!is_key_frame) d_avg = aom_avg_8x8(d + y8_idx * dp + x8_idx, dp);
Jingning Han's avatar
Jingning Han committed
612
613
614
615
616
617
618
619
620
621
#endif
      sum = s_avg - d_avg;
      sse = sum * sum;
    }
    fill_variance(sse, sum, 0, &vst->split[k].part_variances.none);
  }
}

// This function chooses partitioning based on the variance between source and
// reconstructed last, where variance is computed for down-sampled inputs.
622
static int choose_partitioning(AV1_COMP *cpi, const TileInfo *const tile,
clang-format's avatar
clang-format committed
623
                               MACROBLOCK *x, int mi_row, int mi_col) {
624
  AV1_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
625
626
627
628
629
630
631
632
633
634
  MACROBLOCKD *xd = &x->e_mbd;
  int i, j, k, m;
  v64x64 vt;
  v16x16 vt2[16];
  int force_split[21];
  uint8_t *s;
  const uint8_t *d;
  int sp;
  int dp;
  int pixels_wide = 64, pixels_high = 64;
clang-format's avatar
clang-format committed
635
636
  int64_t thresholds[4] = { cpi->vbp_thresholds[0], cpi->vbp_thresholds[1],
                            cpi->vbp_thresholds[2], cpi->vbp_thresholds[3] };
Jingning Han's avatar
Jingning Han committed
637
638
639
640
641
642
643
644
645

  // Always use 4x4 partition for key frame.
  const int is_key_frame = (cm->frame_type == KEY_FRAME);
  const int use_4x4_partition = is_key_frame;
  const int low_res = (cm->width <= 352 && cm->height <= 288);
  int variance4x4downsample[16];

  int segment_id = CR_SEGMENT_ID_BASE;
  if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ && cm->seg.enabled) {
clang-format's avatar
clang-format committed
646
647
    const uint8_t *const map =
        cm->seg.update_map ? cpi->segmentation_map : cm->last_frame_seg_map;
Jingning Han's avatar
Jingning Han committed
648
649
650
    segment_id = get_segment_id(cm, map, BLOCK_64X64, mi_row, mi_col);

    if (cyclic_refresh_segment_id_boosted(segment_id)) {
651
      int q = av1_get_qindex(&cm->seg, segment_id, cm->base_qindex);
Jingning Han's avatar
Jingning Han committed
652
653
654
655
656
657
      set_vbp_thresholds(cpi, thresholds, q);
    }
  }

  set_offsets(cpi, tile, x, mi_row, mi_col, BLOCK_64X64);

clang-format's avatar
clang-format committed
658
659
  if (xd->mb_to_right_edge < 0) pixels_wide += (xd->mb_to_right_edge >> 3);
  if (xd->mb_to_bottom_edge < 0) pixels_high += (xd->mb_to_bottom_edge >> 3);
Jingning Han's avatar
Jingning Han committed
660
661
662
663

  s = x->plane[0].src.buf;
  sp = x->plane[0].src.stride;

Yunqing Wang's avatar
Yunqing Wang committed
664
  if (!is_key_frame) {
Jingning Han's avatar
Jingning Han committed
665
666
667
668
669
    MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
    const YV12_BUFFER_CONFIG *yv12 = get_ref_frame_buffer(cpi, LAST_FRAME);

    const YV12_BUFFER_CONFIG *yv12_g = NULL;
    unsigned int y_sad, y_sad_g;
clang-format's avatar
clang-format committed
670
671
    const BLOCK_SIZE bsize = BLOCK_32X32 + (mi_col + 4 < cm->mi_cols) * 2 +
                             (mi_row + 4 < cm->mi_rows);
Jingning Han's avatar
Jingning Han committed
672
673

    assert(yv12 != NULL);
Yunqing Wang's avatar
Yunqing Wang committed
674
    yv12_g = get_ref_frame_buffer(cpi, GOLDEN_FRAME);
Jingning Han's avatar
Jingning Han committed
675
676

    if (yv12_g && yv12_g != yv12) {
677
      av1_setup_pre_planes(xd, 0, yv12_g, mi_row, mi_col,
clang-format's avatar
clang-format committed
678
                           &cm->frame_refs[GOLDEN_FRAME - 1].sf);
clang-format's avatar
clang-format committed
679
680
681
      y_sad_g = cpi->fn_ptr[bsize].sdf(
          x->plane[0].src.buf, x->plane[0].src.stride, xd->plane[0].pre[0].buf,
          xd->plane[0].pre[0].stride);
Jingning Han's avatar
Jingning Han committed
682
683
684
685
    } else {
      y_sad_g = UINT_MAX;
    }

686
    av1_setup_pre_planes(xd, 0, yv12, mi_row, mi_col,
clang-format's avatar
clang-format committed
687
                         &cm->frame_refs[LAST_FRAME - 1].sf);
Jingning Han's avatar
Jingning Han committed
688
689
690
691
692
693
    mbmi->ref_frame[0] = LAST_FRAME;
    mbmi->ref_frame[1] = NONE;
    mbmi->sb_type = BLOCK_64X64;
    mbmi->mv[0].as_int = 0;
    mbmi->interp_filter = BILINEAR;

694
    y_sad = av1_int_pro_motion_estimation(cpi, x, bsize, mi_row, mi_col);
Jingning Han's avatar
Jingning Han committed
695
    if (y_sad_g < y_sad) {
696
      av1_setup_pre_planes(xd, 0, yv12_g, mi_row, mi_col,
clang-format's avatar
clang-format committed
697
                           &cm->frame_refs[GOLDEN_FRAME - 1].sf);
Jingning Han's avatar
Jingning Han committed
698
699
700
701
702
703
704
      mbmi->ref_frame[0] = GOLDEN_FRAME;
      mbmi->mv[0].as_int = 0;
      y_sad = y_sad_g;
    } else {
      x->pred_mv[LAST_FRAME] = mbmi->mv[0].as_mv;
    }

705
    av1_build_inter_predictors_sb(xd, mi_row, mi_col, BLOCK_64X64);
Jingning Han's avatar
Jingning Han committed
706
707
708
709
710
711

    d = xd->plane[0].dst.buf;
    dp = xd->plane[0].dst.stride;

    // If the y_sad is very small, take 64x64 as partition and exit.
    // Don't check on boosted segment for now, as 64x64 is suppressed there.
clang-format's avatar
clang-format committed
712
    if (segment_id == CR_SEGMENT_ID_BASE && y_sad < cpi->vbp_threshold_sad) {
Jingning Han's avatar
Jingning Han committed
713
714
715
716
717
718
719
720
721
      const int block_width = num_8x8_blocks_wide_lookup[BLOCK_64X64];
      const int block_height = num_8x8_blocks_high_lookup[BLOCK_64X64];
      if (mi_col + block_width / 2 < cm->mi_cols &&
          mi_row + block_height / 2 < cm->mi_rows) {
        set_block_size(cpi, x, xd, mi_row, mi_col, BLOCK_64X64);
        return 0;
      }
    }
  } else {
Adrian Grange's avatar
Adrian Grange committed
722
    d = AV1_VAR_OFFS;
Jingning Han's avatar
Jingning Han committed
723
    dp = 0;
724
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
725
726
    if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
      switch (xd->bd) {
Adrian Grange's avatar
Adrian Grange committed
727
728
        case 10: d = CONVERT_TO_BYTEPTR(AV1_HIGH_VAR_OFFS_10); break;
        case 12: d = CONVERT_TO_BYTEPTR(AV1_HIGH_VAR_OFFS_12); break;
Jingning Han's avatar
Jingning Han committed
729
        case 8:
Adrian Grange's avatar
Adrian Grange committed
730
        default: d = CONVERT_TO_BYTEPTR(AV1_HIGH_VAR_OFFS_8); break;
Jingning Han's avatar
Jingning Han committed
731
732
      }
    }
733
#endif  // CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
  }

  // Index for force_split: 0 for 64x64, 1-4 for 32x32 blocks,
  // 5-20 for the 16x16 blocks.
  force_split[0] = 0;
  // Fill in the entire tree of 8x8 (or 4x4 under some conditions) variances
  // for splits.
  for (i = 0; i < 4; i++) {
    const int x32_idx = ((i & 1) << 5);
    const int y32_idx = ((i >> 1) << 5);
    const int i2 = i << 2;
    force_split[i + 1] = 0;
    for (j = 0; j < 4; j++) {
      const int x16_idx = x32_idx + ((j & 1) << 4);
      const int y16_idx = y32_idx + ((j >> 1) << 4);
      const int split_index = 5 + i2 + j;
      v16x16 *vst = &vt.split[i].split[j];
      force_split[split_index] = 0;
      variance4x4downsample[i2 + j] = 0;
      if (!is_key_frame) {
        fill_variance_8x8avg(s, sp, d, dp, x16_idx, y16_idx, vst,
755
#if CONFIG_AOM_HIGHBITDEPTH
clang-format's avatar
clang-format committed
756
                             xd->cur_buf->flags,
Jingning Han's avatar
Jingning Han committed
757
#endif
clang-format's avatar
clang-format committed
758
                             pixels_wide, pixels_high, is_key_frame);
Jingning Han's avatar
Jingning Han committed
759
760
        fill_variance_tree(&vt.split[i].split[j], BLOCK_16X16);
        get_variance(&vt.split[i].split[j].part_variances.none);
clang-format's avatar
clang-format committed
761
        if (vt.split[i].split[j].part_variances.none.variance > thresholds[2]) {
Jingning Han's avatar
Jingning Han committed
762
763
764
765
766
767
          // 16X16 variance is above threshold for split, so force split to 8x8
          // for this 16x16 block (this also forces splits for upper levels).
          force_split[split_index] = 1;
          force_split[i + 1] = 1;
          force_split[0] = 1;
        } else if (vt.split[i].split[j].part_variances.none.variance >
clang-format's avatar
clang-format committed
768
                       thresholds[1] &&
Jingning Han's avatar
Jingning Han committed
769
770
771
772
773
                   !cyclic_refresh_segment_id_boosted(segment_id)) {
          // We have some nominal amount of 16x16 variance (based on average),
          // compute the minmax over the 8x8 sub-blocks, and if above threshold,
          // force split to 8x8 block for this 16x16 block.
          int minmax = compute_minmax_8x8(s, sp, d, dp, x16_idx, y16_idx,
774
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
775
776
777
778
779
780
781
782
783
784
                                          xd->cur_buf->flags,
#endif
                                          pixels_wide, pixels_high);
          if (minmax > cpi->vbp_threshold_minmax) {
            force_split[split_index] = 1;
            force_split[i + 1] = 1;
            force_split[0] = 1;
          }
        }
      }
Yunqing Wang's avatar
Yunqing Wang committed
785
      if (is_key_frame || (low_res &&
clang-format's avatar
clang-format committed
786
787
                           vt.split[i].split[j].part_variances.none.variance >
                               (thresholds[1] << 1))) {
Jingning Han's avatar
Jingning Han committed
788
789
790
791
792
793
        force_split[split_index] = 0;
        // Go down to 4x4 down-sampling for variance.
        variance4x4downsample[i2 + j] = 1;
        for (k = 0; k < 4; k++) {
          int x8_idx = x16_idx + ((k & 1) << 3);
          int y8_idx = y16_idx + ((k >> 1) << 3);
clang-format's avatar
clang-format committed
794
          v8x8 *vst2 = is_key_frame ? &vst->split[k] : &vt2[i2 + j].split[k];
Jingning Han's avatar
Jingning Han committed
795
          fill_variance_4x4avg(s, sp, d, dp, x8_idx, y8_idx, vst2,
796
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
797
798
                               xd->cur_buf->flags,
#endif
clang-format's avatar
clang-format committed
799
                               pixels_wide, pixels_high, is_key_frame);
Jingning Han's avatar
Jingning Han committed
800
801
802
803
804
805
806
807
808
809
        }
      }
    }
  }

  // Fill the rest of the variance tree by summing split partition values.
  for (i = 0; i < 4; i++) {
    const int i2 = i << 2;
    for (j = 0; j < 4; j++) {
      if (variance4x4downsample[i2 + j] == 1) {
clang-format's avatar
clang-format committed
810
811
        v16x16 *vtemp = (!is_key_frame) ? &vt2[i2 + j] : &vt.split[i].split[j];
        for (m = 0; m < 4; m++) fill_variance_tree(&vtemp->split[m], BLOCK_8X8);
Jingning Han's avatar
Jingning Han committed
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
        fill_variance_tree(vtemp, BLOCK_16X16);
      }
    }
    fill_variance_tree(&vt.split[i], BLOCK_32X32);
    // If variance of this 32x32 block is above the threshold, force the block
    // to split. This also forces a split on the upper (64x64) level.
    if (!force_split[i + 1]) {
      get_variance(&vt.split[i].part_variances.none);
      if (vt.split[i].part_variances.none.variance > thresholds[1]) {
        force_split[i + 1] = 1;
        force_split[0] = 1;
      }
    }
  }
  if (!force_split[0]) {
    fill_variance_tree(&vt, BLOCK_64X64);
    get_variance(&vt.part_variances.none);
  }

  // Now go through the entire structure, splitting every block size until
  // we get to one that's got a variance lower than our threshold.
clang-format's avatar
clang-format committed
833
  if (mi_col + 8 > cm->mi_cols || mi_row + 8 > cm->mi_rows ||
Jingning Han's avatar
Jingning Han committed
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
      !set_vt_partitioning(cpi, x, xd, &vt, BLOCK_64X64, mi_row, mi_col,
                           thresholds[0], BLOCK_16X16, force_split[0])) {
    for (i = 0; i < 4; ++i) {
      const int x32_idx = ((i & 1) << 2);
      const int y32_idx = ((i >> 1) << 2);
      const int i2 = i << 2;
      if (!set_vt_partitioning(cpi, x, xd, &vt.split[i], BLOCK_32X32,
                               (mi_row + y32_idx), (mi_col + x32_idx),
                               thresholds[1], BLOCK_16X16,
                               force_split[i + 1])) {
        for (j = 0; j < 4; ++j) {
          const int x16_idx = ((j & 1) << 1);
          const int y16_idx = ((j >> 1) << 1);
          // For inter frames: if variance4x4downsample[] == 1 for this 16x16
          // block, then the variance is based on 4x4 down-sampling, so use vt2
          // in set_vt_partioning(), otherwise use vt.
clang-format's avatar
clang-format committed
850
851
852
853
854
855
856
          v16x16 *vtemp = (!is_key_frame && variance4x4downsample[i2 + j] == 1)
                              ? &vt2[i2 + j]
                              : &vt.split[i].split[j];
          if (!set_vt_partitioning(
                  cpi, x, xd, vtemp, BLOCK_16X16, mi_row + y32_idx + y16_idx,
                  mi_col + x32_idx + x16_idx, thresholds[2], cpi->vbp_bsize_min,
                  force_split[5 + i2 + j])) {
Jingning Han's avatar
Jingning Han committed
857
858
859
860
861
862
863
864
865
            for (k = 0; k < 4; ++k) {
              const int x8_idx = (k & 1);
              const int y8_idx = (k >> 1);
              if (use_4x4_partition) {
                if (!set_vt_partitioning(cpi, x, xd, &vtemp->split[k],
                                         BLOCK_8X8,
                                         mi_row + y32_idx + y16_idx + y8_idx,
                                         mi_col + x32_idx + x16_idx + x8_idx,
                                         thresholds[3], BLOCK_8X8, 0)) {
clang-format's avatar
clang-format committed
866
867
868
                  set_block_size(
                      cpi, x, xd, (mi_row + y32_idx + y16_idx + y8_idx),
                      (mi_col + x32_idx + x16_idx + x8_idx), BLOCK_4X4);
Jingning Han's avatar
Jingning Han committed
869
870
                }
              } else {
clang-format's avatar
clang-format committed
871
872
873
                set_block_size(
                    cpi, x, xd, (mi_row + y32_idx + y16_idx + y8_idx),
                    (mi_col + x32_idx + x16_idx + x8_idx), BLOCK_8X8);
Jingning Han's avatar
Jingning Han committed
874
875
876
877
878
879
880
881
882
883
              }
            }
          }
        }
      }
    }
  }
  return 0;
}

884
885
static void update_state(const AV1_COMP *const cpi, ThreadData *td,
                         PICK_MODE_CONTEXT *ctx, int mi_row, int mi_col,
886
                         BLOCK_SIZE bsize) {
Jingning Han's avatar
Jingning Han committed
887
  int i, x_idx, y;
888
  const AV1_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
889
890
891
892
893
894
895
896
897
898
899
900
  MACROBLOCK *const x = &td->mb;
  MACROBLOCKD *const xd = &x->e_mbd;
  struct macroblock_plane *const p = x->plane;
  struct macroblockd_plane *const pd = xd->plane;
  MODE_INFO *mi = &ctx->mic;
  MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
  MODE_INFO *mi_addr = xd->mi[0];
  const struct segmentation *const seg = &cm->seg;

  const int mis = cm->mi_stride;
  const int mi_width = num_8x8_blocks_wide_lookup[bsize];
  const int mi_height = num_8x8_blocks_high_lookup[bsize];
901
902
903
#if CONFIG_REF_MV
  int8_t rf_type;
#endif
Jingning Han's avatar
Jingning Han committed
904
905
906
907
908
909

  assert(mi->mbmi.sb_type == bsize);

  *mi_addr = *mi;
  *x->mbmi_ext = ctx->mbmi_ext;

910
911
#if CONFIG_REF_MV
  rf_type = av1_ref_frame_type(mbmi->ref_frame);
clang-format's avatar
clang-format committed
912
  if (x->mbmi_ext->ref_mv_count[rf_type] > 1 && mbmi->sb_type >= BLOCK_8X8 &&
913
914
      mbmi->mode == NEWMV) {
    for (i = 0; i < 1 + has_second_ref(mbmi); ++i) {
clang-format's avatar
clang-format committed
915
916
917
918
      int_mv this_mv =
          (i == 0)
              ? x->mbmi_ext->ref_mv_stack[rf_type][mbmi->ref_mv_idx].this_mv
              : x->mbmi_ext->ref_mv_stack[rf_type][mbmi->ref_mv_idx].comp_mv;
919
920
921
922
      clamp_mv_ref(&this_mv.as_mv, xd->n8_w << 3, xd->n8_h << 3, xd);
      lower_mv_precision(&this_mv.as_mv, cm->allow_high_precision_mv);
      x->mbmi_ext->ref_mvs[mbmi->ref_frame[i]][0] = this_mv;
      mbmi->pred_mv[i] = this_mv;
923
      mi->mbmi.pred_mv[i] = this_mv;
924
925
926
927
    }
  }
#endif

Jingning Han's avatar
Jingning Han committed
928
929
930
931
  // If segmentation in use
  if (seg->enabled) {
    // For in frame complexity AQ copy the segment id from the segment map.
    if (cpi->oxcf.aq_mode == COMPLEXITY_AQ) {
clang-format's avatar
clang-format committed
932
933
934
      const uint8_t *const map =
          seg->update_map ? cpi->segmentation_map : cm->last_frame_seg_map;
      mi_addr->mbmi.segment_id = get_segment_id(cm, map, bsize, mi_row, mi_col);
Jingning Han's avatar
Jingning Han committed
935
936
937
938
    }
    // Else for cyclic refresh mode update the segment map, set the segment id
    // and then update the quantizer.
    if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ) {
939
      av1_cyclic_refresh_update_segment(cpi, &xd->mi[0]->mbmi, mi_row, mi_col,
clang-format's avatar
clang-format committed
940
                                        bsize, ctx->rate, ctx->dist, x->skip);
Jingning Han's avatar
Jingning Han committed
941
942
943
    }
  }

944
945
946
947
  for (i = 0; i < MAX_MB_PLANE; ++i) {
    p[i].coeff = ctx->coeff[i];
    p[i].qcoeff = ctx->qcoeff[i];
    pd[i].dqcoeff = ctx->dqcoeff[i];
948
949
950
#if CONFIG_PVQ
    pd[i].pvq_ref_coeff = ctx->pvq_ref_coeff[i];
#endif
951
    p[i].eobs = ctx->eobs[i];
Jingning Han's avatar
Jingning Han committed
952
953
  }

954
955
956
957
#if CONFIG_PALETTE
  for (i = 0; i < 2; ++i) pd[i].color_index_map = ctx->color_index_map[i];
#endif  // CONFIG_PALETTE

Jingning Han's avatar
Jingning Han committed
958
959
960
961
  // Restore the coding context of the MB to that that was in place
  // when the mode was picked for it
  for (y = 0; y < mi_height; y++)
    for (x_idx = 0; x_idx < mi_width; x_idx++)
clang-format's avatar
clang-format committed
962
963
      if ((xd->mb_to_right_edge >> (3 + MI_SIZE_LOG2)) + mi_width > x_idx &&
          (xd->mb_to_bottom_edge >> (3 + MI_SIZE_LOG2)) + mi_height > y) {
Jingning Han's avatar
Jingning Han committed
964
965
966
        xd->mi[x_idx + y * mis] = mi_addr;
      }

967
968
969
970
#if CONFIG_DELTA_Q
  if (cpi->oxcf.aq_mode > NO_AQ && cpi->oxcf.aq_mode < DELTA_AQ)
    av1_init_plane_quantizers(cpi, x);
#else
971
  if (cpi->oxcf.aq_mode) av1_init_plane_quantizers(cpi, x);
972
#endif
Jingning Han's avatar
Jingning Han committed
973
974
975
976
977
978
979
980
981

  if (is_inter_block(mbmi) && mbmi->sb_type < BLOCK_8X8) {
    mbmi->mv[0].as_int = mi->bmi[3].as_mv[0].as_int;
    mbmi->mv[1].as_int = mi->bmi[3].as_mv[1].as_int;
  }

  x->skip = ctx->skip;
}

982
void av1_setup_src_planes(MACROBLOCK *x, const YV12_BUFFER_CONFIG *src,
clang-format's avatar
clang-format committed
983
                          int mi_row, int mi_col) {
clang-format's avatar
clang-format committed
984
985
  uint8_t *const buffers[3] = { src->y_buffer, src->u_buffer, src->v_buffer };
  const int strides[3] = { src->y_stride, src->uv_stride, src->uv_stride };
Jingning Han's avatar
Jingning Han committed
986
987
988
989
990
991
992
993
994
995
996
  int i;

  // Set current frame pointer.
  x->e_mbd.cur_buf = src;

  for (i = 0; i < MAX_MB_PLANE; i++)
    setup_pred_plane(&x->plane[i].src, buffers[i], strides[i], mi_row, mi_col,
                     NULL, x->e_mbd.plane[i].subsampling_x,
                     x->e_mbd.plane[i].subsampling_y);
}

997
static int set_segment_rdmult(const AV1_COMP *const cpi, MACROBLOCK *const x,
clang-format's avatar
clang-format committed
998
                              int8_t segment_id) {
Jingning Han's avatar
Jingning Han committed
999
  int segment_qindex;
1000
  const AV1_COMMON *const cm = &cpi->common;
1001
  av1_init_plane_quantizers(cpi, x);
Adrian Grange's avatar
Adrian Grange committed
1002
  aom_clear_system_state();
1003
1004
  segment_qindex = av1_get_qindex(&cm->seg, segment_id, cm->base_qindex);
  return av1_compute_rd_mult(cpi, segment_qindex + cm->y_dc_delta_q);
Jingning Han's avatar
Jingning Han committed
1005
1006
}

1007
static void rd_pick_sb_modes(const AV1_COMP *const cpi, TileDataEnc *tile_data,
clang-format's avatar
clang-format committed
1008
1009
1010
                             MACROBLOCK *const x, int mi_row, int mi_col,
                             RD_COST *rd_cost, BLOCK_SIZE bsize,
                             PICK_MODE_CONTEXT *ctx, int64_t best_rd) {
1011
  const AV1_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
1012
1013
1014
1015
1016
1017
1018
1019
  TileInfo *const tile_info = &tile_data->tile_info;
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *mbmi;
  struct macroblock_plane *const p = x->plane;
  struct macroblockd_plane *const pd = xd->plane;
  const AQ_MODE aq_mode = cpi->oxcf.aq_mode;
  int i, orig_rdmult;

Adrian Grange's avatar
Adrian Grange committed
1020
  aom_clear_system_state();
Jingning Han's avatar
Jingning Han committed
1021
1022
1023
1024

  // Use the lower precision, but faster, 32x32 fdct for mode selection.
  x->use_lp32x32fdct = 1;

1025
1026
1027
1028
1029
#if CONFIG_PVQ
  x->pvq_speed = 1;
  x->pvq_coded = 0;
#endif

Jingning Han's avatar
Jingning Han committed
1030
1031
1032
1033
1034
  set_offsets(cpi, tile_info, x, mi_row, mi_col, bsize);
  mbmi = &xd->mi[0]->mbmi;
  mbmi->sb_type = bsize;

  for (i = 0; i < MAX_MB_PLANE; ++i) {
1035
1036
1037
    p[i].coeff = ctx->coeff[i];
    p[i].qcoeff = ctx->qcoeff[i];
    pd[i].dqcoeff = ctx->dqcoeff[i];
1038
1039
1040
#if CONFIG_PVQ
    pd[i].pvq_ref_coeff = ctx->pvq_ref_coeff[i];
#endif
1041
    p[i].eobs = ctx->eobs[i];
Jingning Han's avatar
Jingning Han committed
1042
  }
hui su's avatar
hui su committed
1043

1044
1045
1046
1047
#if CONFIG_PALETTE
  for (i = 0; i < 2; ++i) pd[i].color_index_map = ctx->color_index_map[i];
#endif  // CONFIG_PALETTE

Jingning Han's avatar
Jingning Han committed
1048
1049
1050
1051
1052
1053
  ctx->skippable = 0;
  ctx->pred_pixel_ready = 0;

  // Set to zero to make sure we do not use the previous encoded frame stats
  mbmi->skip = 0;

1054
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
1055
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
1056
    x->source_variance = av1_high_get_sby_perpixel_variance(
clang-format's avatar
clang-format committed
1057
        cpi, &x->plane[0].src, bsize, xd->bd);
Jingning Han's avatar
Jingning Han committed
1058
1059
  } else {
    x->source_variance =
1060
        av1_get_sby_perpixel_variance(cpi, &x->plane[0].src, bsize);
Jingning Han's avatar
Jingning Han committed
1061
1062
1063
  }
#else
  x->source_variance =
1064
1065
      av1_get_sby_perpixel_variance(cpi, &x->plane[0].src, bsize);
#endif  // CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
1066
1067
1068
1069
1070

  // Save rdmult before it might be changed, so it can be restored later.
  orig_rdmult = x->rdmult;

  if (aq_mode == VARIANCE_AQ) {
clang-format's avatar
clang-format committed
1071
    const int energy =
1072
        bsize <= BLOCK_16X16 ? x->mb_energy : av1_block_energy(cpi, x, bsize);
clang-format's avatar
clang-format committed
1073
    if (cm->frame_type == KEY_FRAME || cpi->refresh_alt_ref_frame ||
Jingning Han's avatar
Jingning Han committed
1074
        (cpi->refresh_golden_frame && !cpi->rc.is_src_frame_alt_ref)) {
1075
      mbmi->segment_id = av1_vaq_segment_id(energy);
Jingning Han's avatar
Jingning Han committed
1076
    } else {
clang-format's avatar
clang-format committed
1077
1078
      const uint8_t *const map =
          cm->seg.update_map ? cpi->segmentation_map : cm->last_frame_seg_map;
Jingning Han's avatar
Jingning Han committed
1079
1080
1081
1082
1083
1084
      mbmi->segment_id = get_segment_id(cm, map, bsize, mi_row, mi_col);
    }
    x->rdmult = set_segment_rdmult(cpi, x, mbmi->segment_id);
  } else if (aq_mode == COMPLEXITY_AQ) {
    x->rdmult = set_segment_rdmult(cpi, x, mbmi->segment_id);
  } else if (aq_mode == CYCLIC_REFRESH_AQ) {
clang-format's avatar
clang-format committed
1085
1086
    const uint8_t *const map =
        cm->seg.update_map ? cpi->segmentation_map : cm->last_frame_seg_map;
Jingning Han's avatar
Jingning Han committed
1087
1088
1089
    // If segment is boosted, use rdmult for that segment.
    if (cyclic_refresh_segment_id_boosted(
            get_segment_id(cm, map, bsize, mi_row, mi_col)))
1090
      x->rdmult = av1_cyclic_refresh_get_rdmult(cpi->cyclic_refresh);
Jingning Han's avatar
Jingning Han committed
1091
1092
1093
1094
1095
  }

  // Find best coding mode & reconstruct the MB so it is available
  // as a predictor for MBs that follow in the SB
  if (frame_is_intra_only(cm)) {
1096
    av1_rd_pick_intra_mode_sb(cpi, x, rd_cost, bsize, ctx, best_rd);
Jingning Han's avatar
Jingning Han committed
1097
1098
1099
  } else {
    if (bsize >= BLOCK_8X8) {
      if (segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP))
1100
        av1_rd_pick_inter_mode_sb_seg_skip(cpi, tile_data, x, rd_cost, bsize,
clang-format's avatar
clang-format committed
1101
                                           ctx, best_rd);
Jingning Han's avatar
Jingning Han committed
1102
      else
1103
        av1_rd_pick_inter_mode_sb(cpi, tile_data, x, mi_row, mi_col, rd_cost,
clang-format's avatar
clang-format committed
1104
                                  bsize, ctx, best_rd);
Jingning Han's avatar
Jingning Han committed
1105
    } else {
1106
      av1_rd_pick_inter_mode_sub8x8(cpi, tile_data, x, mi_row, mi_col, rd_cost,
clang-format's avatar
clang-format committed
1107
                                    bsize, ctx, best_rd);
Jingning Han's avatar
Jingning Han committed
1108
1109
1110
1111
    }
  }

  // Examine the resulting rate and for AQ mode 2 make a segment choice.
clang-format's avatar
clang-format committed
1112
1113
1114
  if ((rd_cost->rate != INT_MAX) && (aq_mode == COMPLEXITY_AQ) &&
      (bsize >= BLOCK_16X16) &&
      (cm->frame_type == KEY_FRAME || cpi->refresh_alt_ref_frame ||
Jingning Han's avatar
Jingning Han committed
1115
       (cpi->refresh_golden_frame && !cpi->rc.is_src_frame_alt_ref))) {
1116
    av1_caq_select_segment(cpi, x, bsize, mi_row, mi_col, rd_cost->rate);
Jingning Han's avatar
Jingning Han committed
1117
1118
1119
1120
1121
1122
  }

  x->rdmult = orig_rdmult;

  // TODO(jingning) The rate-distortion optimization flow needs to be
  // refactored to provide proper exit/return handle.
clang-format's avatar
clang-format committed
1123
  if (rd_cost->rate == INT_MAX) rd_cost->rdcost = INT64_MAX;
Jingning Han's avatar
Jingning Han committed
1124
1125
1126
1127
1128

  ctx->rate = rd_cost->rate;
  ctx->dist = rd_cost->dist;
}

1129
#if CONFIG_REF_MV
clang-format's avatar
clang-format committed
1130
static void update_inter_mode_stats(FRAME_COUNTS *counts, PREDICTION_MODE mode,
1131
1132
                                    const int16_t mode_context) {
  int16_t mode_ctx = mode_context & NEWMV_CTX_MASK;
1133
1134
1135
1136
1137
  if (mode == NEWMV) {
    ++counts->newmv_mode[mode_ctx][0];
    return;
  } else {
    ++counts->newmv_mode[mode_ctx][1];
1138