encodeframe.c 108 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
4 5 6 7 8 9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10 11 12 13 14 15
 */

#include <limits.h>
#include <math.h>
#include <stdio.h>

Yaowu Xu's avatar
Yaowu Xu committed
16
#include "./av1_rtcd.h"
Jingning Han's avatar
Jingning Han committed
17 18 19
#include "./vpx_dsp_rtcd.h"
#include "./vpx_config.h"

Yaowu Xu's avatar
Yaowu Xu committed
20 21 22 23
#include "aom_dsp/vpx_dsp_common.h"
#include "aom_ports/mem.h"
#include "aom_ports/vpx_timer.h"
#include "aom_ports/system_state.h"
Jingning Han's avatar
Jingning Han committed
24

Yaowu Xu's avatar
Yaowu Xu committed
25 26 27 28 29 30 31 32 33 34 35
#include "av1/common/common.h"
#include "av1/common/entropy.h"
#include "av1/common/entropymode.h"
#include "av1/common/idct.h"
#include "av1/common/mvref_common.h"
#include "av1/common/pred_common.h"
#include "av1/common/quant_common.h"
#include "av1/common/reconintra.h"
#include "av1/common/reconinter.h"
#include "av1/common/seg_common.h"
#include "av1/common/tile_common.h"
Jingning Han's avatar
Jingning Han committed
36

Yaowu Xu's avatar
Yaowu Xu committed
37 38 39 40 41 42 43 44 45 46 47 48
#include "av1/encoder/aq_complexity.h"
#include "av1/encoder/aq_cyclicrefresh.h"
#include "av1/encoder/aq_variance.h"
#include "av1/encoder/encodeframe.h"
#include "av1/encoder/encodemb.h"
#include "av1/encoder/encodemv.h"
#include "av1/encoder/ethread.h"
#include "av1/encoder/extend.h"
#include "av1/encoder/rd.h"
#include "av1/encoder/rdopt.h"
#include "av1/encoder/segmentation.h"
#include "av1/encoder/tokenize.h"
Jingning Han's avatar
Jingning Han committed
49

clang-format's avatar
clang-format committed
50 51 52
static void encode_superblock(VP10_COMP *cpi, ThreadData *td, TOKENEXTRA **t,
                              int output_enabled, int mi_row, int mi_col,
                              BLOCK_SIZE bsize, PICK_MODE_CONTEXT *ctx);
Jingning Han's avatar
Jingning Han committed
53 54 55 56 57 58

// This is used as a reference when computing the source variance for the
//  purposes of activity masking.
// Eventually this should be replaced by custom no-reference routines,
//  which will be faster.
static const uint8_t VP9_VAR_OFFS[64] = {
clang-format's avatar
clang-format committed
59 60 61 62 63
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128
Jingning Han's avatar
Jingning Han committed
64 65
};

66
#if CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
67
static const uint16_t VP9_HIGH_VAR_OFFS_8[64] = {
clang-format's avatar
clang-format committed
68 69 70 71 72
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
  128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128
Jingning Han's avatar
Jingning Han committed
73 74 75
};

static const uint16_t VP9_HIGH_VAR_OFFS_10[64] = {
clang-format's avatar
clang-format committed
76 77 78 79 80 81 82 83
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4,
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4,
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4,
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4,
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4,
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4,
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4,
  128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4, 128 * 4
Jingning Han's avatar
Jingning Han committed
84 85 86
};

static const uint16_t VP9_HIGH_VAR_OFFS_12[64] = {
clang-format's avatar
clang-format committed
87 88 89 90 91 92 93 94 95 96
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16, 128 * 16,
  128 * 16
Jingning Han's avatar
Jingning Han committed
97
};
98
#endif  // CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
99

Yaowu Xu's avatar
Yaowu Xu committed
100
unsigned int vp10_get_sby_perpixel_variance(VP10_COMP *cpi,
clang-format's avatar
clang-format committed
101 102
                                            const struct buf_2d *ref,
                                            BLOCK_SIZE bs) {
Jingning Han's avatar
Jingning Han committed
103
  unsigned int sse;
clang-format's avatar
clang-format committed
104 105
  const unsigned int var =
      cpi->fn_ptr[bs].vf(ref->buf, ref->stride, VP9_VAR_OFFS, 0, &sse);
Jingning Han's avatar
Jingning Han committed
106 107 108
  return ROUND_POWER_OF_TWO(var, num_pels_log2_lookup[bs]);
}

109
#if CONFIG_VPX_HIGHBITDEPTH
clang-format's avatar
clang-format committed
110 111 112
unsigned int vp10_high_get_sby_perpixel_variance(VP10_COMP *cpi,
                                                 const struct buf_2d *ref,
                                                 BLOCK_SIZE bs, int bd) {
Jingning Han's avatar
Jingning Han committed
113 114 115
  unsigned int var, sse;
  switch (bd) {
    case 10:
clang-format's avatar
clang-format committed
116 117 118
      var =
          cpi->fn_ptr[bs].vf(ref->buf, ref->stride,
                             CONVERT_TO_BYTEPTR(VP9_HIGH_VAR_OFFS_10), 0, &sse);
Jingning Han's avatar
Jingning Han committed
119 120
      break;
    case 12:
clang-format's avatar
clang-format committed
121 122 123
      var =
          cpi->fn_ptr[bs].vf(ref->buf, ref->stride,
                             CONVERT_TO_BYTEPTR(VP9_HIGH_VAR_OFFS_12), 0, &sse);
Jingning Han's avatar
Jingning Han committed
124 125 126
      break;
    case 8:
    default:
clang-format's avatar
clang-format committed
127 128 129
      var =
          cpi->fn_ptr[bs].vf(ref->buf, ref->stride,
                             CONVERT_TO_BYTEPTR(VP9_HIGH_VAR_OFFS_8), 0, &sse);
Jingning Han's avatar
Jingning Han committed
130 131 132 133
      break;
  }
  return ROUND_POWER_OF_TWO(var, num_pels_log2_lookup[bs]);
}
134
#endif  // CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
135

Yaowu Xu's avatar
Yaowu Xu committed
136
static unsigned int get_sby_perpixel_diff_variance(VP10_COMP *cpi,
Jingning Han's avatar
Jingning Han committed
137 138 139 140 141 142 143 144 145 146 147 148 149 150
                                                   const struct buf_2d *ref,
                                                   int mi_row, int mi_col,
                                                   BLOCK_SIZE bs) {
  unsigned int sse, var;
  uint8_t *last_y;
  const YV12_BUFFER_CONFIG *last = get_ref_frame_buffer(cpi, LAST_FRAME);

  assert(last != NULL);
  last_y =
      &last->y_buffer[mi_row * MI_SIZE * last->y_stride + mi_col * MI_SIZE];
  var = cpi->fn_ptr[bs].vf(ref->buf, ref->stride, last_y, last->y_stride, &sse);
  return ROUND_POWER_OF_TWO(var, num_pels_log2_lookup[bs]);
}

Yaowu Xu's avatar
Yaowu Xu committed
151
static BLOCK_SIZE get_rd_var_based_fixed_partition(VP10_COMP *cpi,
clang-format's avatar
clang-format committed
152
                                                   MACROBLOCK *x, int mi_row,
Jingning Han's avatar
Jingning Han committed
153
                                                   int mi_col) {
clang-format's avatar
clang-format committed
154 155
  unsigned int var = get_sby_perpixel_diff_variance(
      cpi, &x->plane[0].src, mi_row, mi_col, BLOCK_64X64);
Jingning Han's avatar
Jingning Han committed
156 157 158 159 160 161 162 163 164 165 166 167
  if (var < 8)
    return BLOCK_64X64;
  else if (var < 128)
    return BLOCK_32X32;
  else if (var < 2048)
    return BLOCK_16X16;
  else
    return BLOCK_8X8;
}

// Lighter version of set_offsets that only sets the mode info
// pointers.
Jingning Han's avatar
Jingning Han committed
168
static INLINE void set_mode_info_offsets(VP10_COMP *const cpi,
Jingning Han's avatar
Jingning Han committed
169
                                         MACROBLOCK *const x,
clang-format's avatar
clang-format committed
170
                                         MACROBLOCKD *const xd, int mi_row,
Jingning Han's avatar
Jingning Han committed
171
                                         int mi_col) {
Jingning Han's avatar
Jingning Han committed
172
  VP10_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
173 174 175
  const int idx_str = xd->mi_stride * mi_row + mi_col;
  xd->mi = cm->mi_grid_visible + idx_str;
  xd->mi[0] = cm->mi + idx_str;
Jingning Han's avatar
Jingning Han committed
176
  x->mbmi_ext = cpi->mbmi_ext_base + (mi_row * cm->mi_cols + mi_col);
Jingning Han's avatar
Jingning Han committed
177 178
}

Yaowu Xu's avatar
Yaowu Xu committed
179
static void set_offsets(VP10_COMP *cpi, const TileInfo *const tile,
Jingning Han's avatar
Jingning Han committed
180 181
                        MACROBLOCK *const x, int mi_row, int mi_col,
                        BLOCK_SIZE bsize) {
Yaowu Xu's avatar
Yaowu Xu committed
182
  VP10_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
183 184 185 186 187 188 189 190
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *mbmi;
  const int mi_width = num_8x8_blocks_wide_lookup[bsize];
  const int mi_height = num_8x8_blocks_high_lookup[bsize];
  const struct segmentation *const seg = &cm->seg;

  set_skip_context(xd, mi_row, mi_col);

Jingning Han's avatar
Jingning Han committed
191
  set_mode_info_offsets(cpi, x, xd, mi_row, mi_col);
Jingning Han's avatar
Jingning Han committed
192 193 194 195 196 197 198 199

  mbmi = &xd->mi[0]->mbmi;

  // Set up destination pointers.
  vp10_setup_dst_planes(xd->plane, get_frame_new_buffer(cm), mi_row, mi_col);

  // Set up limit values for MV components.
  // Mv beyond the range do not produce new/different prediction block.
200 201 202 203
  x->mv_row_min = -(((mi_row + mi_height) * MI_SIZE) + VPX_INTERP_EXTEND);
  x->mv_col_min = -(((mi_col + mi_width) * MI_SIZE) + VPX_INTERP_EXTEND);
  x->mv_row_max = (cm->mi_rows - mi_row) * MI_SIZE + VPX_INTERP_EXTEND;
  x->mv_col_max = (cm->mi_cols - mi_col) * MI_SIZE + VPX_INTERP_EXTEND;
Jingning Han's avatar
Jingning Han committed
204 205 206

  // Set up distance of MB to edge of frame in 1/8th pel units.
  assert(!(mi_col & (mi_width - 1)) && !(mi_row & (mi_height - 1)));
clang-format's avatar
clang-format committed
207 208
  set_mi_row_col(xd, tile, mi_row, mi_height, mi_col, mi_width, cm->mi_rows,
                 cm->mi_cols);
Jingning Han's avatar
Jingning Han committed
209 210 211 212 213 214 215 216 217 218 219

  // Set up source buffers.
  vp10_setup_src_planes(x, cpi->Source, mi_row, mi_col);

  // R/D setup.
  x->rddiv = cpi->rd.RDDIV;
  x->rdmult = cpi->rd.RDMULT;

  // Setup segment ID.
  if (seg->enabled) {
    if (cpi->oxcf.aq_mode != VARIANCE_AQ) {
clang-format's avatar
clang-format committed
220 221
      const uint8_t *const map =
          seg->update_map ? cpi->segmentation_map : cm->last_frame_seg_map;
Jingning Han's avatar
Jingning Han committed
222 223 224 225 226 227 228 229 230 231 232 233 234 235
      mbmi->segment_id = get_segment_id(cm, map, bsize, mi_row, mi_col);
    }
    vp10_init_plane_quantizers(cpi, x);

    x->encode_breakout = cpi->segment_encode_breakout[mbmi->segment_id];
  } else {
    mbmi->segment_id = 0;
    x->encode_breakout = cpi->encode_breakout;
  }

  // required by vp10_append_sub8x8_mvs_for_idx() and vp10_find_best_ref_mvs()
  xd->tile = *tile;
}

clang-format's avatar
clang-format committed
236 237
static void set_block_size(VP10_COMP *const cpi, MACROBLOCK *const x,
                           MACROBLOCKD *const xd, int mi_row, int mi_col,
Jingning Han's avatar
Jingning Han committed
238 239
                           BLOCK_SIZE bsize) {
  if (cpi->common.mi_cols > mi_col && cpi->common.mi_rows > mi_row) {
Jingning Han's avatar
Jingning Han committed
240
    set_mode_info_offsets(cpi, x, xd, mi_row, mi_col);
Jingning Han's avatar
Jingning Han committed
241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298
    xd->mi[0]->mbmi.sb_type = bsize;
  }
}

typedef struct {
  int64_t sum_square_error;
  int64_t sum_error;
  int log2_count;
  int variance;
} var;

typedef struct {
  var none;
  var horz[2];
  var vert[2];
} partition_variance;

typedef struct {
  partition_variance part_variances;
  var split[4];
} v4x4;

typedef struct {
  partition_variance part_variances;
  v4x4 split[4];
} v8x8;

typedef struct {
  partition_variance part_variances;
  v8x8 split[4];
} v16x16;

typedef struct {
  partition_variance part_variances;
  v16x16 split[4];
} v32x32;

typedef struct {
  partition_variance part_variances;
  v32x32 split[4];
} v64x64;

typedef struct {
  partition_variance *part_variances;
  var *split[4];
} variance_node;

typedef enum {
  V16X16,
  V32X32,
  V64X64,
} TREE_LEVEL;

static void tree_to_node(void *data, BLOCK_SIZE bsize, variance_node *node) {
  int i;
  node->part_variances = NULL;
  switch (bsize) {
    case BLOCK_64X64: {
clang-format's avatar
clang-format committed
299
      v64x64 *vt = (v64x64 *)data;
Jingning Han's avatar
Jingning Han committed
300 301 302 303 304 305
      node->part_variances = &vt->part_variances;
      for (i = 0; i < 4; i++)
        node->split[i] = &vt->split[i].part_variances.none;
      break;
    }
    case BLOCK_32X32: {
clang-format's avatar
clang-format committed
306
      v32x32 *vt = (v32x32 *)data;
Jingning Han's avatar
Jingning Han committed
307 308 309 310 311 312
      node->part_variances = &vt->part_variances;
      for (i = 0; i < 4; i++)
        node->split[i] = &vt->split[i].part_variances.none;
      break;
    }
    case BLOCK_16X16: {
clang-format's avatar
clang-format committed
313
      v16x16 *vt = (v16x16 *)data;
Jingning Han's avatar
Jingning Han committed
314 315 316 317 318 319
      node->part_variances = &vt->part_variances;
      for (i = 0; i < 4; i++)
        node->split[i] = &vt->split[i].part_variances.none;
      break;
    }
    case BLOCK_8X8: {
clang-format's avatar
clang-format committed
320
      v8x8 *vt = (v8x8 *)data;
Jingning Han's avatar
Jingning Han committed
321 322 323 324 325 326
      node->part_variances = &vt->part_variances;
      for (i = 0; i < 4; i++)
        node->split[i] = &vt->split[i].part_variances.none;
      break;
    }
    case BLOCK_4X4: {
clang-format's avatar
clang-format committed
327
      v4x4 *vt = (v4x4 *)data;
Jingning Han's avatar
Jingning Han committed
328
      node->part_variances = &vt->part_variances;
clang-format's avatar
clang-format committed
329
      for (i = 0; i < 4; i++) node->split[i] = &vt->split[i];
Jingning Han's avatar
Jingning Han committed
330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346
      break;
    }
    default: {
      assert(0);
      break;
    }
  }
}

// Set variance values given sum square error, sum error, count.
static void fill_variance(int64_t s2, int64_t s, int c, var *v) {
  v->sum_square_error = s2;
  v->sum_error = s;
  v->log2_count = c;
}

static void get_variance(var *v) {
clang-format's avatar
clang-format committed
347 348 349 350
  v->variance =
      (int)(256 * (v->sum_square_error -
                   ((v->sum_error * v->sum_error) >> v->log2_count)) >>
            v->log2_count);
Jingning Han's avatar
Jingning Han committed
351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370
}

static void sum_2_variances(const var *a, const var *b, var *r) {
  assert(a->log2_count == b->log2_count);
  fill_variance(a->sum_square_error + b->sum_square_error,
                a->sum_error + b->sum_error, a->log2_count + 1, r);
}

static void fill_variance_tree(void *data, BLOCK_SIZE bsize) {
  variance_node node;
  memset(&node, 0, sizeof(node));
  tree_to_node(data, bsize, &node);
  sum_2_variances(node.split[0], node.split[1], &node.part_variances->horz[0]);
  sum_2_variances(node.split[2], node.split[3], &node.part_variances->horz[1]);
  sum_2_variances(node.split[0], node.split[2], &node.part_variances->vert[0]);
  sum_2_variances(node.split[1], node.split[3], &node.part_variances->vert[1]);
  sum_2_variances(&node.part_variances->vert[0], &node.part_variances->vert[1],
                  &node.part_variances->none);
}

clang-format's avatar
clang-format committed
371 372 373 374
static int set_vt_partitioning(VP10_COMP *cpi, MACROBLOCK *const x,
                               MACROBLOCKD *const xd, void *data,
                               BLOCK_SIZE bsize, int mi_row, int mi_col,
                               int64_t threshold, BLOCK_SIZE bsize_min,
Jingning Han's avatar
Jingning Han committed
375
                               int force_split) {
clang-format's avatar
clang-format committed
376
  VP10_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
377 378 379 380 381 382 383 384
  variance_node vt;
  const int block_width = num_8x8_blocks_wide_lookup[bsize];
  const int block_height = num_8x8_blocks_high_lookup[bsize];
  const int low_res = (cm->width <= 352 && cm->height <= 288);

  assert(block_height == block_width);
  tree_to_node(data, bsize, &vt);

clang-format's avatar
clang-format committed
385
  if (force_split == 1) return 0;
Jingning Han's avatar
Jingning Han committed
386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407

  // For bsize=bsize_min (16x16/8x8 for 8x8/4x4 downsampling), select if
  // variance is below threshold, otherwise split will be selected.
  // No check for vert/horiz split as too few samples for variance.
  if (bsize == bsize_min) {
    // Variance already computed to set the force_split.
    if (low_res || cm->frame_type == KEY_FRAME)
      get_variance(&vt.part_variances->none);
    if (mi_col + block_width / 2 < cm->mi_cols &&
        mi_row + block_height / 2 < cm->mi_rows &&
        vt.part_variances->none.variance < threshold) {
      set_block_size(cpi, x, xd, mi_row, mi_col, bsize);
      return 1;
    }
    return 0;
  } else if (bsize > bsize_min) {
    // Variance already computed to set the force_split.
    if (low_res || cm->frame_type == KEY_FRAME)
      get_variance(&vt.part_variances->none);
    // For key frame: take split for bsize above 32X32 or very high variance.
    if (cm->frame_type == KEY_FRAME &&
        (bsize > BLOCK_32X32 ||
clang-format's avatar
clang-format committed
408
         vt.part_variances->none.variance > (threshold << 4))) {
Jingning Han's avatar
Jingning Han committed
409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454
      return 0;
    }
    // If variance is low, take the bsize (no split).
    if (mi_col + block_width / 2 < cm->mi_cols &&
        mi_row + block_height / 2 < cm->mi_rows &&
        vt.part_variances->none.variance < threshold) {
      set_block_size(cpi, x, xd, mi_row, mi_col, bsize);
      return 1;
    }

    // Check vertical split.
    if (mi_row + block_height / 2 < cm->mi_rows) {
      BLOCK_SIZE subsize = get_subsize(bsize, PARTITION_VERT);
      get_variance(&vt.part_variances->vert[0]);
      get_variance(&vt.part_variances->vert[1]);
      if (vt.part_variances->vert[0].variance < threshold &&
          vt.part_variances->vert[1].variance < threshold &&
          get_plane_block_size(subsize, &xd->plane[1]) < BLOCK_INVALID) {
        set_block_size(cpi, x, xd, mi_row, mi_col, subsize);
        set_block_size(cpi, x, xd, mi_row, mi_col + block_width / 2, subsize);
        return 1;
      }
    }
    // Check horizontal split.
    if (mi_col + block_width / 2 < cm->mi_cols) {
      BLOCK_SIZE subsize = get_subsize(bsize, PARTITION_HORZ);
      get_variance(&vt.part_variances->horz[0]);
      get_variance(&vt.part_variances->horz[1]);
      if (vt.part_variances->horz[0].variance < threshold &&
          vt.part_variances->horz[1].variance < threshold &&
          get_plane_block_size(subsize, &xd->plane[1]) < BLOCK_INVALID) {
        set_block_size(cpi, x, xd, mi_row, mi_col, subsize);
        set_block_size(cpi, x, xd, mi_row + block_height / 2, mi_col, subsize);
        return 1;
      }
    }

    return 0;
  }
  return 0;
}

// Set the variance split thresholds for following the block sizes:
// 0 - threshold_64x64, 1 - threshold_32x32, 2 - threshold_16x16,
// 3 - vbp_threshold_8x8. vbp_threshold_8x8 (to split to 4x4 partition) is
// currently only used on key frame.
Yaowu Xu's avatar
Yaowu Xu committed
455
static void set_vbp_thresholds(VP10_COMP *cpi, int64_t thresholds[], int q) {
Yaowu Xu's avatar
Yaowu Xu committed
456
  VP10_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
457 458
  const int is_key_frame = (cm->frame_type == KEY_FRAME);
  const int threshold_multiplier = is_key_frame ? 20 : 1;
clang-format's avatar
clang-format committed
459 460
  const int64_t threshold_base =
      (int64_t)(threshold_multiplier * cpi->y_dequant[q][1]);
Jingning Han's avatar
Jingning Han committed
461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480
  if (is_key_frame) {
    thresholds[0] = threshold_base;
    thresholds[1] = threshold_base >> 2;
    thresholds[2] = threshold_base >> 2;
    thresholds[3] = threshold_base << 2;
  } else {
    thresholds[1] = threshold_base;
    if (cm->width <= 352 && cm->height <= 288) {
      thresholds[0] = threshold_base >> 2;
      thresholds[2] = threshold_base << 3;
    } else {
      thresholds[0] = threshold_base;
      thresholds[1] = (5 * threshold_base) >> 2;
      if (cm->width >= 1920 && cm->height >= 1080)
        thresholds[1] = (7 * threshold_base) >> 2;
      thresholds[2] = threshold_base << cpi->oxcf.speed;
    }
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
481
void vp10_set_variance_partition_thresholds(VP10_COMP *cpi, int q) {
Yaowu Xu's avatar
Yaowu Xu committed
482
  VP10_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
483 484 485 486 487 488 489 490 491 492 493 494 495 496 497
  SPEED_FEATURES *const sf = &cpi->sf;
  const int is_key_frame = (cm->frame_type == KEY_FRAME);
  if (sf->partition_search_type != VAR_BASED_PARTITION &&
      sf->partition_search_type != REFERENCE_PARTITION) {
    return;
  } else {
    set_vbp_thresholds(cpi, cpi->vbp_thresholds, q);
    // The thresholds below are not changed locally.
    if (is_key_frame) {
      cpi->vbp_threshold_sad = 0;
      cpi->vbp_bsize_min = BLOCK_8X8;
    } else {
      if (cm->width <= 352 && cm->height <= 288)
        cpi->vbp_threshold_sad = 100;
      else
clang-format's avatar
clang-format committed
498 499 500
        cpi->vbp_threshold_sad = (cpi->y_dequant[q][1] << 1) > 1000
                                     ? (cpi->y_dequant[q][1] << 1)
                                     : 1000;
Jingning Han's avatar
Jingning Han committed
501 502 503 504 505 506 507 508 509
      cpi->vbp_bsize_min = BLOCK_16X16;
    }
    cpi->vbp_threshold_minmax = 15 + (q >> 3);
  }
}

// Compute the minmax over the 8x8 subblocks.
static int compute_minmax_8x8(const uint8_t *s, int sp, const uint8_t *d,
                              int dp, int x16_idx, int y16_idx,
510
#if CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
511 512
                              int highbd_flag,
#endif
clang-format's avatar
clang-format committed
513
                              int pixels_wide, int pixels_high) {
Jingning Han's avatar
Jingning Han committed
514 515 516 517 518 519 520 521 522 523
  int k;
  int minmax_max = 0;
  int minmax_min = 255;
  // Loop over the 4 8x8 subblocks.
  for (k = 0; k < 4; k++) {
    int x8_idx = x16_idx + ((k & 1) << 3);
    int y8_idx = y16_idx + ((k >> 1) << 3);
    int min = 0;
    int max = 0;
    if (x8_idx < pixels_wide && y8_idx < pixels_high) {
524
#if CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
525
      if (highbd_flag & YV12_FLAG_HIGHBITDEPTH) {
James Zern's avatar
James Zern committed
526
        vpx_highbd_minmax_8x8(s + y8_idx * sp + x8_idx, sp,
clang-format's avatar
clang-format committed
527
                              d + y8_idx * dp + x8_idx, dp, &min, &max);
Jingning Han's avatar
Jingning Han committed
528
      } else {
clang-format's avatar
clang-format committed
529 530
        vpx_minmax_8x8(s + y8_idx * sp + x8_idx, sp, d + y8_idx * dp + x8_idx,
                       dp, &min, &max);
Jingning Han's avatar
Jingning Han committed
531 532
      }
#else
clang-format's avatar
clang-format committed
533
      vpx_minmax_8x8(s + y8_idx * sp + x8_idx, sp, d + y8_idx * dp + x8_idx, dp,
Jingning Han's avatar
Jingning Han committed
534 535
                     &min, &max);
#endif
clang-format's avatar
clang-format committed
536 537
      if ((max - min) > minmax_max) minmax_max = (max - min);
      if ((max - min) < minmax_min) minmax_min = (max - min);
Jingning Han's avatar
Jingning Han committed
538 539 540 541 542 543 544
    }
  }
  return (minmax_max - minmax_min);
}

static void fill_variance_4x4avg(const uint8_t *s, int sp, const uint8_t *d,
                                 int dp, int x8_idx, int y8_idx, v8x8 *vst,
545
#if CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
546 547
                                 int highbd_flag,
#endif
clang-format's avatar
clang-format committed
548
                                 int pixels_wide, int pixels_high,
Jingning Han's avatar
Jingning Han committed
549 550 551 552 553 554 555 556 557 558
                                 int is_key_frame) {
  int k;
  for (k = 0; k < 4; k++) {
    int x4_idx = x8_idx + ((k & 1) << 2);
    int y4_idx = y8_idx + ((k >> 1) << 2);
    unsigned int sse = 0;
    int sum = 0;
    if (x4_idx < pixels_wide && y4_idx < pixels_high) {
      int s_avg;
      int d_avg = 128;
559
#if CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
560
      if (highbd_flag & YV12_FLAG_HIGHBITDEPTH) {
James Zern's avatar
James Zern committed
561
        s_avg = vpx_highbd_avg_4x4(s + y4_idx * sp + x4_idx, sp);
Jingning Han's avatar
Jingning Han committed
562
        if (!is_key_frame)
James Zern's avatar
James Zern committed
563
          d_avg = vpx_highbd_avg_4x4(d + y4_idx * dp + x4_idx, dp);
Jingning Han's avatar
Jingning Han committed
564
      } else {
James Zern's avatar
James Zern committed
565
        s_avg = vpx_avg_4x4(s + y4_idx * sp + x4_idx, sp);
clang-format's avatar
clang-format committed
566
        if (!is_key_frame) d_avg = vpx_avg_4x4(d + y4_idx * dp + x4_idx, dp);
Jingning Han's avatar
Jingning Han committed
567 568
      }
#else
James Zern's avatar
James Zern committed
569
      s_avg = vpx_avg_4x4(s + y4_idx * sp + x4_idx, sp);
clang-format's avatar
clang-format committed
570
      if (!is_key_frame) d_avg = vpx_avg_4x4(d + y4_idx * dp + x4_idx, dp);
Jingning Han's avatar
Jingning Han committed
571 572 573 574 575 576 577 578 579 580
#endif
      sum = s_avg - d_avg;
      sse = sum * sum;
    }
    fill_variance(sse, sum, 0, &vst->split[k].part_variances.none);
  }
}

static void fill_variance_8x8avg(const uint8_t *s, int sp, const uint8_t *d,
                                 int dp, int x16_idx, int y16_idx, v16x16 *vst,
581
#if CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
582 583
                                 int highbd_flag,
#endif
clang-format's avatar
clang-format committed
584
                                 int pixels_wide, int pixels_high,
Jingning Han's avatar
Jingning Han committed
585 586 587 588 589 590 591 592 593 594
                                 int is_key_frame) {
  int k;
  for (k = 0; k < 4; k++) {
    int x8_idx = x16_idx + ((k & 1) << 3);
    int y8_idx = y16_idx + ((k >> 1) << 3);
    unsigned int sse = 0;
    int sum = 0;
    if (x8_idx < pixels_wide && y8_idx < pixels_high) {
      int s_avg;
      int d_avg = 128;
595
#if CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
596
      if (highbd_flag & YV12_FLAG_HIGHBITDEPTH) {
James Zern's avatar
James Zern committed
597
        s_avg = vpx_highbd_avg_8x8(s + y8_idx * sp + x8_idx, sp);
Jingning Han's avatar
Jingning Han committed
598
        if (!is_key_frame)
James Zern's avatar
James Zern committed
599
          d_avg = vpx_highbd_avg_8x8(d + y8_idx * dp + x8_idx, dp);
Jingning Han's avatar
Jingning Han committed
600
      } else {
James Zern's avatar
James Zern committed
601
        s_avg = vpx_avg_8x8(s + y8_idx * sp + x8_idx, sp);
clang-format's avatar
clang-format committed
602
        if (!is_key_frame) d_avg = vpx_avg_8x8(d + y8_idx * dp + x8_idx, dp);
Jingning Han's avatar
Jingning Han committed
603 604
      }
#else
James Zern's avatar
James Zern committed
605
      s_avg = vpx_avg_8x8(s + y8_idx * sp + x8_idx, sp);
clang-format's avatar
clang-format committed
606
      if (!is_key_frame) d_avg = vpx_avg_8x8(d + y8_idx * dp + x8_idx, dp);
Jingning Han's avatar
Jingning Han committed
607 608 609 610 611 612 613 614 615 616
#endif
      sum = s_avg - d_avg;
      sse = sum * sum;
    }
    fill_variance(sse, sum, 0, &vst->split[k].part_variances.none);
  }
}

// This function chooses partitioning based on the variance between source and
// reconstructed last, where variance is computed for down-sampled inputs.
clang-format's avatar
clang-format committed
617 618 619
static int choose_partitioning(VP10_COMP *cpi, const TileInfo *const tile,
                               MACROBLOCK *x, int mi_row, int mi_col) {
  VP10_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
620 621 622 623 624 625 626 627 628 629
  MACROBLOCKD *xd = &x->e_mbd;
  int i, j, k, m;
  v64x64 vt;
  v16x16 vt2[16];
  int force_split[21];
  uint8_t *s;
  const uint8_t *d;
  int sp;
  int dp;
  int pixels_wide = 64, pixels_high = 64;
clang-format's avatar
clang-format committed
630 631
  int64_t thresholds[4] = { cpi->vbp_thresholds[0], cpi->vbp_thresholds[1],
                            cpi->vbp_thresholds[2], cpi->vbp_thresholds[3] };
Jingning Han's avatar
Jingning Han committed
632 633 634 635 636 637 638 639 640

  // Always use 4x4 partition for key frame.
  const int is_key_frame = (cm->frame_type == KEY_FRAME);
  const int use_4x4_partition = is_key_frame;
  const int low_res = (cm->width <= 352 && cm->height <= 288);
  int variance4x4downsample[16];

  int segment_id = CR_SEGMENT_ID_BASE;
  if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ && cm->seg.enabled) {
clang-format's avatar
clang-format committed
641 642
    const uint8_t *const map =
        cm->seg.update_map ? cpi->segmentation_map : cm->last_frame_seg_map;
Jingning Han's avatar
Jingning Han committed
643 644 645 646 647 648 649 650 651 652
    segment_id = get_segment_id(cm, map, BLOCK_64X64, mi_row, mi_col);

    if (cyclic_refresh_segment_id_boosted(segment_id)) {
      int q = vp10_get_qindex(&cm->seg, segment_id, cm->base_qindex);
      set_vbp_thresholds(cpi, thresholds, q);
    }
  }

  set_offsets(cpi, tile, x, mi_row, mi_col, BLOCK_64X64);

clang-format's avatar
clang-format committed
653 654
  if (xd->mb_to_right_edge < 0) pixels_wide += (xd->mb_to_right_edge >> 3);
  if (xd->mb_to_bottom_edge < 0) pixels_high += (xd->mb_to_bottom_edge >> 3);
Jingning Han's avatar
Jingning Han committed
655 656 657 658

  s = x->plane[0].src.buf;
  sp = x->plane[0].src.stride;

Yunqing Wang's avatar
Yunqing Wang committed
659
  if (!is_key_frame) {
Jingning Han's avatar
Jingning Han committed
660 661 662 663 664 665
    MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
    unsigned int uv_sad;
    const YV12_BUFFER_CONFIG *yv12 = get_ref_frame_buffer(cpi, LAST_FRAME);

    const YV12_BUFFER_CONFIG *yv12_g = NULL;
    unsigned int y_sad, y_sad_g;
clang-format's avatar
clang-format committed
666 667
    const BLOCK_SIZE bsize = BLOCK_32X32 + (mi_col + 4 < cm->mi_cols) * 2 +
                             (mi_row + 4 < cm->mi_rows);
Jingning Han's avatar
Jingning Han committed
668 669

    assert(yv12 != NULL);
Yunqing Wang's avatar
Yunqing Wang committed
670
    yv12_g = get_ref_frame_buffer(cpi, GOLDEN_FRAME);
Jingning Han's avatar
Jingning Han committed
671 672 673

    if (yv12_g && yv12_g != yv12) {
      vp10_setup_pre_planes(xd, 0, yv12_g, mi_row, mi_col,
clang-format's avatar
clang-format committed
674 675 676 677
                            &cm->frame_refs[GOLDEN_FRAME - 1].sf);
      y_sad_g = cpi->fn_ptr[bsize].sdf(
          x->plane[0].src.buf, x->plane[0].src.stride, xd->plane[0].pre[0].buf,
          xd->plane[0].pre[0].stride);
Jingning Han's avatar
Jingning Han committed
678 679 680 681 682
    } else {
      y_sad_g = UINT_MAX;
    }

    vp10_setup_pre_planes(xd, 0, yv12, mi_row, mi_col,
clang-format's avatar
clang-format committed
683
                          &cm->frame_refs[LAST_FRAME - 1].sf);
Jingning Han's avatar
Jingning Han committed
684 685 686 687 688 689 690 691 692
    mbmi->ref_frame[0] = LAST_FRAME;
    mbmi->ref_frame[1] = NONE;
    mbmi->sb_type = BLOCK_64X64;
    mbmi->mv[0].as_int = 0;
    mbmi->interp_filter = BILINEAR;

    y_sad = vp10_int_pro_motion_estimation(cpi, x, bsize, mi_row, mi_col);
    if (y_sad_g < y_sad) {
      vp10_setup_pre_planes(xd, 0, yv12_g, mi_row, mi_col,
clang-format's avatar
clang-format committed
693
                            &cm->frame_refs[GOLDEN_FRAME - 1].sf);
Jingning Han's avatar
Jingning Han committed
694 695 696 697 698 699 700 701 702 703
      mbmi->ref_frame[0] = GOLDEN_FRAME;
      mbmi->mv[0].as_int = 0;
      y_sad = y_sad_g;
    } else {
      x->pred_mv[LAST_FRAME] = mbmi->mv[0].as_mv;
    }

    vp10_build_inter_predictors_sb(xd, mi_row, mi_col, BLOCK_64X64);

    for (i = 1; i <= 2; ++i) {
clang-format's avatar
clang-format committed
704
      struct macroblock_plane *p = &x->plane[i];
Jingning Han's avatar
Jingning Han committed
705 706 707 708 709 710
      struct macroblockd_plane *pd = &xd->plane[i];
      const BLOCK_SIZE bs = get_plane_block_size(bsize, pd);

      if (bs == BLOCK_INVALID)
        uv_sad = UINT_MAX;
      else
clang-format's avatar
clang-format committed
711 712
        uv_sad = cpi->fn_ptr[bs].sdf(p->src.buf, p->src.stride, pd->dst.buf,
                                     pd->dst.stride);
Jingning Han's avatar
Jingning Han committed
713 714 715 716 717 718 719 720 721

      x->color_sensitivity[i - 1] = uv_sad > (y_sad >> 2);
    }

    d = xd->plane[0].dst.buf;
    dp = xd->plane[0].dst.stride;

    // If the y_sad is very small, take 64x64 as partition and exit.
    // Don't check on boosted segment for now, as 64x64 is suppressed there.
clang-format's avatar
clang-format committed
722
    if (segment_id == CR_SEGMENT_ID_BASE && y_sad < cpi->vbp_threshold_sad) {
Jingning Han's avatar
Jingning Han committed
723 724 725 726 727 728 729 730 731 732 733
      const int block_width = num_8x8_blocks_wide_lookup[BLOCK_64X64];
      const int block_height = num_8x8_blocks_high_lookup[BLOCK_64X64];
      if (mi_col + block_width / 2 < cm->mi_cols &&
          mi_row + block_height / 2 < cm->mi_rows) {
        set_block_size(cpi, x, xd, mi_row, mi_col, BLOCK_64X64);
        return 0;
      }
    }
  } else {
    d = VP9_VAR_OFFS;
    dp = 0;
734
#if CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
735 736
    if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
      switch (xd->bd) {
clang-format's avatar
clang-format committed
737 738
        case 10: d = CONVERT_TO_BYTEPTR(VP9_HIGH_VAR_OFFS_10); break;
        case 12: d = CONVERT_TO_BYTEPTR(VP9_HIGH_VAR_OFFS_12); break;
Jingning Han's avatar
Jingning Han committed
739
        case 8:
clang-format's avatar
clang-format committed
740
        default: d = CONVERT_TO_BYTEPTR(VP9_HIGH_VAR_OFFS_8); break;
Jingning Han's avatar
Jingning Han committed
741 742
      }
    }
743
#endif  // CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764
  }

  // Index for force_split: 0 for 64x64, 1-4 for 32x32 blocks,
  // 5-20 for the 16x16 blocks.
  force_split[0] = 0;
  // Fill in the entire tree of 8x8 (or 4x4 under some conditions) variances
  // for splits.
  for (i = 0; i < 4; i++) {
    const int x32_idx = ((i & 1) << 5);
    const int y32_idx = ((i >> 1) << 5);
    const int i2 = i << 2;
    force_split[i + 1] = 0;
    for (j = 0; j < 4; j++) {
      const int x16_idx = x32_idx + ((j & 1) << 4);
      const int y16_idx = y32_idx + ((j >> 1) << 4);
      const int split_index = 5 + i2 + j;
      v16x16 *vst = &vt.split[i].split[j];
      force_split[split_index] = 0;
      variance4x4downsample[i2 + j] = 0;
      if (!is_key_frame) {
        fill_variance_8x8avg(s, sp, d, dp, x16_idx, y16_idx, vst,
765
#if CONFIG_VPX_HIGHBITDEPTH
clang-format's avatar
clang-format committed
766
                             xd->cur_buf->flags,
Jingning Han's avatar
Jingning Han committed
767
#endif
clang-format's avatar
clang-format committed
768
                             pixels_wide, pixels_high, is_key_frame);
Jingning Han's avatar
Jingning Han committed
769 770
        fill_variance_tree(&vt.split[i].split[j], BLOCK_16X16);
        get_variance(&vt.split[i].split[j].part_variances.none);
clang-format's avatar
clang-format committed
771
        if (vt.split[i].split[j].part_variances.none.variance > thresholds[2]) {
Jingning Han's avatar
Jingning Han committed
772 773 774 775 776 777
          // 16X16 variance is above threshold for split, so force split to 8x8
          // for this 16x16 block (this also forces splits for upper levels).
          force_split[split_index] = 1;
          force_split[i + 1] = 1;
          force_split[0] = 1;
        } else if (vt.split[i].split[j].part_variances.none.variance >
clang-format's avatar
clang-format committed
778
                       thresholds[1] &&
Jingning Han's avatar
Jingning Han committed
779 780 781 782 783
                   !cyclic_refresh_segment_id_boosted(segment_id)) {
          // We have some nominal amount of 16x16 variance (based on average),
          // compute the minmax over the 8x8 sub-blocks, and if above threshold,
          // force split to 8x8 block for this 16x16 block.
          int minmax = compute_minmax_8x8(s, sp, d, dp, x16_idx, y16_idx,
784
#if CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
785 786 787 788 789 790 791 792 793 794
                                          xd->cur_buf->flags,
#endif
                                          pixels_wide, pixels_high);
          if (minmax > cpi->vbp_threshold_minmax) {
            force_split[split_index] = 1;
            force_split[i + 1] = 1;
            force_split[0] = 1;
          }
        }
      }
Yunqing Wang's avatar
Yunqing Wang committed
795
      if (is_key_frame || (low_res &&
clang-format's avatar
clang-format committed
796 797
                           vt.split[i].split[j].part_variances.none.variance >
                               (thresholds[1] << 1))) {
Jingning Han's avatar
Jingning Han committed
798 799 800 801 802 803
        force_split[split_index] = 0;
        // Go down to 4x4 down-sampling for variance.
        variance4x4downsample[i2 + j] = 1;
        for (k = 0; k < 4; k++) {
          int x8_idx = x16_idx + ((k & 1) << 3);
          int y8_idx = y16_idx + ((k >> 1) << 3);
clang-format's avatar
clang-format committed
804
          v8x8 *vst2 = is_key_frame ? &vst->split[k] : &vt2[i2 + j].split[k];
Jingning Han's avatar
Jingning Han committed
805
          fill_variance_4x4avg(s, sp, d, dp, x8_idx, y8_idx, vst2,
806
#if CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
807 808
                               xd->cur_buf->flags,
#endif
clang-format's avatar
clang-format committed
809
                               pixels_wide, pixels_high, is_key_frame);
Jingning Han's avatar
Jingning Han committed
810 811 812 813 814 815 816 817 818 819
        }
      }
    }
  }

  // Fill the rest of the variance tree by summing split partition values.
  for (i = 0; i < 4; i++) {
    const int i2 = i << 2;
    for (j = 0; j < 4; j++) {
      if (variance4x4downsample[i2 + j] == 1) {
clang-format's avatar
clang-format committed
820 821
        v16x16 *vtemp = (!is_key_frame) ? &vt2[i2 + j] : &vt.split[i].split[j];
        for (m = 0; m < 4; m++) fill_variance_tree(&vtemp->split[m], BLOCK_8X8);
Jingning Han's avatar
Jingning Han committed
822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842
        fill_variance_tree(vtemp, BLOCK_16X16);
      }
    }
    fill_variance_tree(&vt.split[i], BLOCK_32X32);
    // If variance of this 32x32 block is above the threshold, force the block
    // to split. This also forces a split on the upper (64x64) level.
    if (!force_split[i + 1]) {
      get_variance(&vt.split[i].part_variances.none);
      if (vt.split[i].part_variances.none.variance > thresholds[1]) {
        force_split[i + 1] = 1;
        force_split[0] = 1;
      }
    }
  }
  if (!force_split[0]) {
    fill_variance_tree(&vt, BLOCK_64X64);
    get_variance(&vt.part_variances.none);
  }

  // Now go through the entire structure, splitting every block size until
  // we get to one that's got a variance lower than our threshold.
clang-format's avatar
clang-format committed
843
  if (mi_col + 8 > cm->mi_cols || mi_row + 8 > cm->mi_rows ||
Jingning Han's avatar
Jingning Han committed
844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859
      !set_vt_partitioning(cpi, x, xd, &vt, BLOCK_64X64, mi_row, mi_col,
                           thresholds[0], BLOCK_16X16, force_split[0])) {
    for (i = 0; i < 4; ++i) {
      const int x32_idx = ((i & 1) << 2);
      const int y32_idx = ((i >> 1) << 2);
      const int i2 = i << 2;
      if (!set_vt_partitioning(cpi, x, xd, &vt.split[i], BLOCK_32X32,
                               (mi_row + y32_idx), (mi_col + x32_idx),
                               thresholds[1], BLOCK_16X16,
                               force_split[i + 1])) {
        for (j = 0; j < 4; ++j) {
          const int x16_idx = ((j & 1) << 1);
          const int y16_idx = ((j >> 1) << 1);
          // For inter frames: if variance4x4downsample[] == 1 for this 16x16
          // block, then the variance is based on 4x4 down-sampling, so use vt2
          // in set_vt_partioning(), otherwise use vt.
clang-format's avatar
clang-format committed
860 861 862 863 864 865 866
          v16x16 *vtemp = (!is_key_frame && variance4x4downsample[i2 + j] == 1)
                              ? &vt2[i2 + j]
                              : &vt.split[i].split[j];
          if (!set_vt_partitioning(
                  cpi, x, xd, vtemp, BLOCK_16X16, mi_row + y32_idx + y16_idx,
                  mi_col + x32_idx + x16_idx, thresholds[2], cpi->vbp_bsize_min,
                  force_split[5 + i2 + j])) {
Jingning Han's avatar
Jingning Han committed
867 868 869 870 871 872 873 874 875
            for (k = 0; k < 4; ++k) {
              const int x8_idx = (k & 1);
              const int y8_idx = (k >> 1);
              if (use_4x4_partition) {
                if (!set_vt_partitioning(cpi, x, xd, &vtemp->split[k],
                                         BLOCK_8X8,
                                         mi_row + y32_idx + y16_idx + y8_idx,
                                         mi_col + x32_idx + x16_idx + x8_idx,
                                         thresholds[3], BLOCK_8X8, 0)) {
clang-format's avatar
clang-format committed
876 877 878
                  set_block_size(
                      cpi, x, xd, (mi_row + y32_idx + y16_idx + y8_idx),
                      (mi_col + x32_idx + x16_idx + x8_idx), BLOCK_4X4);
Jingning Han's avatar
Jingning Han committed
879 880
                }
              } else {
clang-format's avatar
clang-format committed
881 882 883
                set_block_size(
                    cpi, x, xd, (mi_row + y32_idx + y16_idx + y8_idx),
                    (mi_col + x32_idx + x16_idx + x8_idx), BLOCK_8X8);
Jingning Han's avatar
Jingning Han committed
884 885 886 887 888 889 890 891 892 893
              }
            }
          }
        }
      }
    }
  }
  return 0;
}

clang-format's avatar
clang-format committed
894
static void update_state(VP10_COMP *cpi, ThreadData *td, PICK_MODE_CONTEXT *ctx,
Jingning Han's avatar
Jingning Han committed
895 896 897
                         int mi_row, int mi_col, BLOCK_SIZE bsize,
                         int output_enabled) {
  int i, x_idx, y;
Yaowu Xu's avatar
Yaowu Xu committed
898
  VP10_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
899 900 901 902 903 904 905 906 907 908 909
  RD_COUNTS *const rdc = &td->rd_counts;
  MACROBLOCK *const x = &td->mb;
  MACROBLOCKD *const xd = &x->e_mbd;
  struct macroblock_plane *const p = x->plane;
  struct macroblockd_plane *const pd = xd->plane;
  MODE_INFO *mi = &ctx->mic;
  MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
  MODE_INFO *mi_addr = xd->mi[0];
  const struct segmentation *const seg = &cm->seg;
  const int bw = num_8x8_blocks_wide_lookup[mi->mbmi.sb_type];
  const int bh = num_8x8_blocks_high_lookup[mi->mbmi.sb_type];
910 911
  const int x_mis = VPXMIN(bw, cm->mi_cols - mi_col);
  const int y_mis = VPXMIN(bh, cm->mi_rows - mi_row);
clang-format's avatar
clang-format committed
912
  MV_REF *const frame_mvs = cm->cur_frame->mvs + mi_row * cm->mi_cols + mi_col;
Jingning Han's avatar
Jingning Han committed
913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928
  int w, h;

  const int mis = cm->mi_stride;
  const int mi_width = num_8x8_blocks_wide_lookup[bsize];
  const int mi_height = num_8x8_blocks_high_lookup[bsize];
  int max_plane;

  assert(mi->mbmi.sb_type == bsize);

  *mi_addr = *mi;
  *x->mbmi_ext = ctx->mbmi_ext;

  // If segmentation in use
  if (seg->enabled) {
    // For in frame complexity AQ copy the segment id from the segment map.
    if (cpi->oxcf.aq_mode == COMPLEXITY_AQ) {
clang-format's avatar
clang-format committed
929 930 931
      const uint8_t *const map =
          seg->update_map ? cpi->segmentation_map : cm->last_frame_seg_map;
      mi_addr->mbmi.segment_id = get_segment_id(cm, map, bsize, mi_row, mi_col);
Jingning Han's avatar
Jingning Han committed
932 933 934 935
    }
    // Else for cyclic refresh mode update the segment map, set the segment id
    // and then update the quantizer.
    if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ) {
clang-format's avatar
clang-format committed
936 937
      vp10_cyclic_refresh_update_segment(cpi, &xd->mi[0]->mbmi, mi_row, mi_col,
                                         bsize, ctx->rate, ctx->dist, x->skip);
Jingning Han's avatar
Jingning Han committed
938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955
    }
  }

  max_plane = is_inter_block(mbmi) ? MAX_MB_PLANE : 1;
  for (i = 0; i < max_plane; ++i) {
    p[i].coeff = ctx->coeff_pbuf[i][1];
    p[i].qcoeff = ctx->qcoeff_pbuf[i][1];
    pd[i].dqcoeff = ctx->dqcoeff_pbuf[i][1];
    p[i].eobs = ctx->eobs_pbuf[i][1];
  }

  for (i = max_plane; i < MAX_MB_PLANE; ++i) {
    p[i].coeff = ctx->coeff_pbuf[i][2];
    p[i].qcoeff = ctx->qcoeff_pbuf[i][2];
    pd[i].dqcoeff = ctx->dqcoeff_pbuf[i][2];
    p[i].eobs = ctx->eobs_pbuf[i][2];
  }

clang-format's avatar
clang-format committed
956
  for (i = 0; i < 2; ++i) pd[i].color_index_map = ctx->color_index_map[i];
hui su's avatar
hui su committed
957

Jingning Han's avatar
Jingning Han committed
958 959 960 961
  // Restore the coding context of the MB to that that was in place
  // when the mode was picked for it
  for (y = 0; y < mi_height; y++)
    for (x_idx = 0; x_idx < mi_width; x_idx++)
clang-format's avatar
clang-format committed
962 963
      if ((xd->mb_to_right_edge >> (3 + MI_SIZE_LOG2)) + mi_width > x_idx &&
          (xd->mb_to_bottom_edge >> (3 + MI_SIZE_LOG2)) + mi_height > y) {
Jingning Han's avatar
Jingning Han committed
964 965 966
        xd->mi[x_idx + y * mis] = mi_addr;
      }

clang-format's avatar
clang-format committed
967
  if (cpi->oxcf.aq_mode) vp10_init_plane_quantizers(cpi, x);
Jingning Han's avatar
Jingning Han committed
968 969 970 971 972 973 974 975

  if (is_inter_block(mbmi) && mbmi->sb_type < BLOCK_8X8) {
    mbmi->mv[0].as_int = mi->bmi[3].as_mv[0].as_int;
    mbmi->mv[1].as_int = mi->bmi[3].as_mv[1].as_int;
  }

  x->skip = ctx->skip;
  memcpy(x->zcoeff_blk[mbmi->tx_size], ctx->zcoeff_blk,
976
         sizeof(ctx->zcoeff_blk[0]) * ctx->num_4x4_blk);
Jingning Han's avatar
Jingning Han committed
977

clang-format's avatar
clang-format committed
978
  if (!output_enabled) return;
Jingning Han's avatar
Jingning Han committed
979 980 981 982

#if CONFIG_INTERNAL_STATS
  if (frame_is_intra_only(cm)) {
    static const int kf_mode_index[] = {
clang-format's avatar
clang-format committed
983 984 985 986 987
      THR_DC /*DC_PRED*/,          THR_V_PRED /*V_PRED*/,
      THR_H_PRED /*H_PRED*/,       THR_D45_PRED /*D45_PRED*/,
      THR_D135_PRED /*D135_PRED*/, THR_D117_PRED /*D117_PRED*/,
      THR_D153_PRED /*D153_PRED*/, THR_D207_PRED /*D207_PRED*/,
      THR_D63_PRED /*D63_PRED*/,   THR_TM /*TM_PRED*/,
Jingning Han's avatar
Jingning Han committed
988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025
    };
    ++cpi->mode_chosen_counts[kf_mode_index[mbmi->mode]];
  } else {
    // Note how often each mode chosen as best
    ++cpi->mode_chosen_counts[ctx->best_mode_index];
  }
#endif
  if (!frame_is_intra_only(cm)) {
    if (is_inter_block(mbmi)) {
      vp10_update_mv_count(td);

      if (cm->interp_filter == SWITCHABLE) {
        const int ctx = vp10_get_pred_context_switchable_interp(xd);
        ++td->counts->switchable_interp[ctx][mbmi->interp_filter];
      }
    }

    rdc->comp_pred_diff[SINGLE_REFERENCE] += ctx->single_pred_diff;
    rdc->comp_pred_diff[COMPOUND_REFERENCE] += ctx->comp_pred_diff;
    rdc->comp_pred_diff[REFERENCE_MODE_SELECT] += ctx->hybrid_pred_diff;

    for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; ++i)
      rdc->filter_diff[i] += ctx->best_filter_diff[i];
  }

  for (h = 0; h < y_mis; ++h) {
    MV_REF *const frame_mv = frame_mvs + h * cm->mi_cols;
    for (w = 0; w < x_mis; ++w) {
      MV_REF *const mv = frame_mv + w;
      mv->ref_frame[0] = mi->mbmi.ref_frame[0];
      mv->ref_frame[1] = mi->mbmi.ref_frame[1];
      mv->mv[0].as_int = mi->mbmi.mv[0].as_int;
      mv->mv[1].as_int = mi->mbmi.mv[1].as_int;
    }
  }
}

void vp10_setup_src_planes(MACROBLOCK *x, const YV12_BUFFER_CONFIG *src,
clang-format's avatar
clang-format committed
1026 1027 1028
                           int mi_row, int mi_col) {
  uint8_t *const buffers[3] = { src->y_buffer, src->u_buffer, src->v_buffer };
  const int strides[3] = { src->y_stride, src->uv_stride, src->uv_stride };
Jingning Han's avatar
Jingning Han committed
1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039
  int i;

  // Set current frame pointer.
  x->e_mbd.cur_buf = src;

  for (i = 0; i < MAX_MB_PLANE; i++)
    setup_pred_plane(&x->plane[i].src, buffers[i], strides[i], mi_row, mi_col,
                     NULL, x->e_mbd.plane[i].subsampling_x,
                     x->e_mbd.plane[i].subsampling_y);
}

clang-format's avatar
clang-format committed
1040 1041
static int set_segment_rdmult(VP10_COMP *const cpi, MACROBLOCK *const x,
                              int8_t segment_id) {
Jingning Han's avatar
Jingning Han committed
1042
  int segment_qindex;
Yaowu Xu's avatar
Yaowu Xu committed
1043
  VP10_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
1044 1045
  vp10_init_plane_quantizers(cpi, x);
  vpx_clear_system_state();
clang-format's avatar
clang-format committed
1046
  segment_qindex = vp10_get_qindex(&cm->seg, segment_id, cm->base_qindex);
Jingning Han's avatar
Jingning Han committed
1047 1048 1049
  return vp10_compute_rd_mult(cpi, segment_qindex + cm->y_dc_delta_q);
}

clang-format's avatar
clang-format committed
1050 1051 1052 1053
static void rd_pick_sb_modes(VP10_COMP *cpi, TileDataEnc *tile_data,
                             MACROBLOCK *const x, int mi_row, int mi_col,
                             RD_COST *rd_cost, BLOCK_SIZE bsize,
                             PICK_MODE_CONTEXT *ctx, int64_t best_rd) {
Yaowu Xu's avatar
Yaowu Xu committed
1054
  VP10_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077
  TileInfo *const tile_info = &tile_data->tile_info;
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *mbmi;
  struct macroblock_plane *const p = x->plane;
  struct macroblockd_plane *const pd = xd->plane;
  const AQ_MODE aq_mode = cpi->oxcf.aq_mode;
  int i, orig_rdmult;

  vpx_clear_system_state();

  // Use the lower precision, but faster, 32x32 fdct for mode selection.
  x->use_lp32x32fdct = 1;

  set_offsets(cpi, tile_info, x, mi_row, mi_col, bsize);
  mbmi = &xd->mi[0]->mbmi;
  mbmi->sb_type = bsize;

  for (i = 0; i < MAX_MB_PLANE; ++i) {
    p[i].coeff = ctx->coeff_pbuf[i][0];
    p[i].qcoeff = ctx->qcoeff_pbuf[i][0];
    pd[i].dqcoeff = ctx->dqcoeff_pbuf[i][0];
    p[i].eobs = ctx->eobs_pbuf[i][0];
  }
hui su's avatar
hui su committed
1078

clang-format's avatar
clang-format committed
1079
  for (i = 0; i < 2; ++i) pd[i].color_index_map = ctx->color_index_map[i];
hui su's avatar
hui su committed
1080

Jingning Han's avatar
Jingning Han committed
1081 1082 1083 1084 1085 1086 1087 1088
  ctx->is_coded = 0;
  ctx->skippable = 0;
  ctx->pred_pixel_ready = 0;
  x->skip_recode = 0;

  // Set to zero to make sure we do not use the previous encoded frame stats
  mbmi->skip = 0;

1089
#if CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
1090
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
clang-format's avatar
clang-format committed
1091 1092
    x->source_variance = vp10_high_get_sby_perpixel_variance(
        cpi, &x->plane[0].src, bsize, xd->bd);
Jingning Han's avatar
Jingning Han committed
1093 1094
  } else {
    x->source_variance =
clang-format's avatar
clang-format committed
1095
        vp10_get_sby_perpixel_variance(cpi, &x->plane[0].src, bsize);
Jingning Han's avatar
Jingning Han committed
1096 1097 1098
  }
#else
  x->source_variance =
clang-format's avatar
clang-format committed
1099
      vp10_get_sby_perpixel_variance(cpi, &x->plane[0].src, bsize);
1100
#endif  // CONFIG_VPX_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
1101 1102 1103 1104 1105

  // Save rdmult before it might be changed, so it can be restored later.