vp9_encodeframe.c 142 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5 6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9 10
 */

Dmitry Kovalev's avatar
Dmitry Kovalev committed
11 12 13 14
#include <limits.h>
#include <math.h>
#include <stdio.h>

Jim Bankoski's avatar
Jim Bankoski committed
15
#include "./vp9_rtcd.h"
Dmitry Kovalev's avatar
Dmitry Kovalev committed
16 17 18 19
#include "./vpx_config.h"

#include "vpx_ports/vpx_timer.h"

20
#include "vp9/common/vp9_common.h"
Yaowu Xu's avatar
Yaowu Xu committed
21
#include "vp9/common/vp9_entropy.h"
22
#include "vp9/common/vp9_entropymode.h"
23
#include "vp9/common/vp9_idct.h"
Dmitry Kovalev's avatar
Dmitry Kovalev committed
24 25 26
#include "vp9/common/vp9_mvref_common.h"
#include "vp9/common/vp9_pred_common.h"
#include "vp9/common/vp9_quant_common.h"
27
#include "vp9/common/vp9_reconintra.h"
Dmitry Kovalev's avatar
Dmitry Kovalev committed
28
#include "vp9/common/vp9_reconinter.h"
29
#include "vp9/common/vp9_seg_common.h"
30
#include "vp9/common/vp9_systemdependent.h"
31
#include "vp9/common/vp9_tile_common.h"
32

33
#include "vp9/encoder/vp9_aq_complexity.h"
Marco Paniconi's avatar
Marco Paniconi committed
34 35
#include "vp9/encoder/vp9_aq_cyclicrefresh.h"
#include "vp9/encoder/vp9_aq_variance.h"
Dmitry Kovalev's avatar
Dmitry Kovalev committed
36 37 38
#include "vp9/encoder/vp9_encodeframe.h"
#include "vp9/encoder/vp9_encodemb.h"
#include "vp9/encoder/vp9_encodemv.h"
39
#include "vp9/encoder/vp9_extend.h"
40
#include "vp9/encoder/vp9_pickmode.h"
41
#include "vp9/encoder/vp9_rd.h"
Dmitry Kovalev's avatar
Dmitry Kovalev committed
42 43
#include "vp9/encoder/vp9_rdopt.h"
#include "vp9/encoder/vp9_segmentation.h"
44
#include "vp9/encoder/vp9_tokenize.h"
45

46 47 48 49 50 51
#define GF_ZEROMV_ZBIN_BOOST 0
#define LF_ZEROMV_ZBIN_BOOST 0
#define MV_ZBIN_BOOST        0
#define SPLIT_MV_ZBIN_BOOST  0
#define INTRA_ZBIN_BOOST     0

52 53
static void encode_superblock(VP9_COMP *cpi, ThreadData * td,
                              TOKENEXTRA **t, int output_enabled,
54 55
                              int mi_row, int mi_col, BLOCK_SIZE bsize,
                              PICK_MODE_CONTEXT *ctx);
56

57
// Motion vector component magnitude threshold for defining fast motion.
58
#define FAST_MOTION_MV_THRESH 24
59

60 61 62 63
// This is used as a reference when computing the source variance for the
//  purposes of activity masking.
// Eventually this should be replaced by custom no-reference routines,
//  which will be faster.
64
static const uint8_t VP9_VAR_OFFS[64] = {
65 66 67 68 69 70 71 72
    128, 128, 128, 128, 128, 128, 128, 128,
    128, 128, 128, 128, 128, 128, 128, 128,
    128, 128, 128, 128, 128, 128, 128, 128,
    128, 128, 128, 128, 128, 128, 128, 128,
    128, 128, 128, 128, 128, 128, 128, 128,
    128, 128, 128, 128, 128, 128, 128, 128,
    128, 128, 128, 128, 128, 128, 128, 128,
    128, 128, 128, 128, 128, 128, 128, 128
73 74
};

75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109
#if CONFIG_VP9_HIGHBITDEPTH
static const uint16_t VP9_HIGH_VAR_OFFS_8[64] = {
    128, 128, 128, 128, 128, 128, 128, 128,
    128, 128, 128, 128, 128, 128, 128, 128,
    128, 128, 128, 128, 128, 128, 128, 128,
    128, 128, 128, 128, 128, 128, 128, 128,
    128, 128, 128, 128, 128, 128, 128, 128,
    128, 128, 128, 128, 128, 128, 128, 128,
    128, 128, 128, 128, 128, 128, 128, 128,
    128, 128, 128, 128, 128, 128, 128, 128
};

static const uint16_t VP9_HIGH_VAR_OFFS_10[64] = {
    128*4, 128*4, 128*4, 128*4, 128*4, 128*4, 128*4, 128*4,
    128*4, 128*4, 128*4, 128*4, 128*4, 128*4, 128*4, 128*4,
    128*4, 128*4, 128*4, 128*4, 128*4, 128*4, 128*4, 128*4,
    128*4, 128*4, 128*4, 128*4, 128*4, 128*4, 128*4, 128*4,
    128*4, 128*4, 128*4, 128*4, 128*4, 128*4, 128*4, 128*4,
    128*4, 128*4, 128*4, 128*4, 128*4, 128*4, 128*4, 128*4,
    128*4, 128*4, 128*4, 128*4, 128*4, 128*4, 128*4, 128*4,
    128*4, 128*4, 128*4, 128*4, 128*4, 128*4, 128*4, 128*4
};

static const uint16_t VP9_HIGH_VAR_OFFS_12[64] = {
    128*16, 128*16, 128*16, 128*16, 128*16, 128*16, 128*16, 128*16,
    128*16, 128*16, 128*16, 128*16, 128*16, 128*16, 128*16, 128*16,
    128*16, 128*16, 128*16, 128*16, 128*16, 128*16, 128*16, 128*16,
    128*16, 128*16, 128*16, 128*16, 128*16, 128*16, 128*16, 128*16,
    128*16, 128*16, 128*16, 128*16, 128*16, 128*16, 128*16, 128*16,
    128*16, 128*16, 128*16, 128*16, 128*16, 128*16, 128*16, 128*16,
    128*16, 128*16, 128*16, 128*16, 128*16, 128*16, 128*16, 128*16,
    128*16, 128*16, 128*16, 128*16, 128*16, 128*16, 128*16, 128*16
};
#endif  // CONFIG_VP9_HIGHBITDEPTH

110
static unsigned int get_sby_perpixel_variance(VP9_COMP *cpi,
111
                                              const struct buf_2d *ref,
112
                                              BLOCK_SIZE bs) {
113 114 115
  unsigned int sse;
  const unsigned int var = cpi->fn_ptr[bs].vf(ref->buf, ref->stride,
                                              VP9_VAR_OFFS, 0, &sse);
116
  return ROUND_POWER_OF_TWO(var, num_pels_log2_lookup[bs]);
117 118
}

119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144
#if CONFIG_VP9_HIGHBITDEPTH
static unsigned int high_get_sby_perpixel_variance(
    VP9_COMP *cpi, const struct buf_2d *ref, BLOCK_SIZE bs, int bd) {
  unsigned int var, sse;
  switch (bd) {
    case 10:
      var = cpi->fn_ptr[bs].vf(ref->buf, ref->stride,
                               CONVERT_TO_BYTEPTR(VP9_HIGH_VAR_OFFS_10),
                               0, &sse);
      break;
    case 12:
      var = cpi->fn_ptr[bs].vf(ref->buf, ref->stride,
                               CONVERT_TO_BYTEPTR(VP9_HIGH_VAR_OFFS_12),
                               0, &sse);
      break;
    case 8:
    default:
      var = cpi->fn_ptr[bs].vf(ref->buf, ref->stride,
                               CONVERT_TO_BYTEPTR(VP9_HIGH_VAR_OFFS_8),
                               0, &sse);
      break;
  }
  return ROUND_POWER_OF_TWO(var, num_pels_log2_lookup[bs]);
}
#endif  // CONFIG_VP9_HIGHBITDEPTH

145
static unsigned int get_sby_perpixel_diff_variance(VP9_COMP *cpi,
146 147
                                                   const struct buf_2d *ref,
                                                   int mi_row, int mi_col,
148
                                                   BLOCK_SIZE bs) {
149 150 151 152 153 154
  const YV12_BUFFER_CONFIG *last = get_ref_frame_buffer(cpi, LAST_FRAME);
  const uint8_t* last_y = &last->y_buffer[mi_row * MI_SIZE * last->y_stride +
                                              mi_col * MI_SIZE];
  unsigned int sse;
  const unsigned int var = cpi->fn_ptr[bs].vf(ref->buf, ref->stride,
                                              last_y, last->y_stride, &sse);
155 156 157
  return ROUND_POWER_OF_TWO(var, num_pels_log2_lookup[bs]);
}

158
static BLOCK_SIZE get_rd_var_based_fixed_partition(VP9_COMP *cpi, MACROBLOCK *x,
159 160
                                                   int mi_row,
                                                   int mi_col) {
161
  unsigned int var = get_sby_perpixel_diff_variance(cpi, &x->plane[0].src,
162 163 164
                                                    mi_row, mi_col,
                                                    BLOCK_64X64);
  if (var < 8)
165
    return BLOCK_64X64;
166
  else if (var < 128)
167
    return BLOCK_32X32;
168 169 170 171
  else if (var < 2048)
    return BLOCK_16X16;
  else
    return BLOCK_8X8;
172 173
}

174
static BLOCK_SIZE get_nonrd_var_based_fixed_partition(VP9_COMP *cpi,
175
                                                      MACROBLOCK *x,
176 177
                                                      int mi_row,
                                                      int mi_col) {
178
  unsigned int var = get_sby_perpixel_diff_variance(cpi, &x->plane[0].src,
179 180
                                                    mi_row, mi_col,
                                                    BLOCK_64X64);
181
  if (var < 4)
182
    return BLOCK_64X64;
183
  else if (var < 10)
184 185
    return BLOCK_32X32;
  else
186
    return BLOCK_16X16;
187 188
}

189 190
// Lighter version of set_offsets that only sets the mode info
// pointers.
191
static INLINE void set_modeinfo_offsets(VP9_COMMON *const cm,
192 193 194
                                        MACROBLOCKD *const xd,
                                        int mi_row,
                                        int mi_col) {
195
  const int idx_str = xd->mi_stride * mi_row + mi_col;
hkuang's avatar
hkuang committed
196 197
  xd->mi = cm->mi + idx_str;
  xd->mi[0].src_mi = &xd->mi[0];
198 199 200
}

static void set_offsets(VP9_COMP *cpi, const TileInfo *const tile,
201 202
                        MACROBLOCK *const x, int mi_row, int mi_col,
                        BLOCK_SIZE bsize) {
203 204 205 206 207 208 209
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *mbmi;
  const int mi_width = num_8x8_blocks_wide_lookup[bsize];
  const int mi_height = num_8x8_blocks_high_lookup[bsize];
  const struct segmentation *const seg = &cm->seg;

210
  set_skip_context(xd, mi_row, mi_col);
211 212 213

  set_modeinfo_offsets(cm, xd, mi_row, mi_col);

hkuang's avatar
hkuang committed
214
  mbmi = &xd->mi[0].src_mi->mbmi;
215 216

  // Set up destination pointers.
217
  vp9_setup_dst_planes(xd->plane, get_frame_new_buffer(cm), mi_row, mi_col);
218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234

  // Set up limit values for MV components.
  // Mv beyond the range do not produce new/different prediction block.
  x->mv_row_min = -(((mi_row + mi_height) * MI_SIZE) + VP9_INTERP_EXTEND);
  x->mv_col_min = -(((mi_col + mi_width) * MI_SIZE) + VP9_INTERP_EXTEND);
  x->mv_row_max = (cm->mi_rows - mi_row) * MI_SIZE + VP9_INTERP_EXTEND;
  x->mv_col_max = (cm->mi_cols - mi_col) * MI_SIZE + VP9_INTERP_EXTEND;

  // Set up distance of MB to edge of frame in 1/8th pel units.
  assert(!(mi_col & (mi_width - 1)) && !(mi_row & (mi_height - 1)));
  set_mi_row_col(xd, tile, mi_row, mi_height, mi_col, mi_width,
                 cm->mi_rows, cm->mi_cols);

  // Set up source buffers.
  vp9_setup_src_planes(x, cpi->Source, mi_row, mi_col);

  // R/D setup.
Dmitry Kovalev's avatar
Dmitry Kovalev committed
235 236
  x->rddiv = cpi->rd.RDDIV;
  x->rdmult = cpi->rd.RDMULT;
237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253

  // Setup segment ID.
  if (seg->enabled) {
    if (cpi->oxcf.aq_mode != VARIANCE_AQ) {
      const uint8_t *const map = seg->update_map ? cpi->segmentation_map
                                                 : cm->last_frame_seg_map;
      mbmi->segment_id = vp9_get_segment_id(cm, map, bsize, mi_row, mi_col);
    }
    vp9_init_plane_quantizers(cpi, x);

    x->encode_breakout = cpi->segment_encode_breakout[mbmi->segment_id];
  } else {
    mbmi->segment_id = 0;
    x->encode_breakout = cpi->encode_breakout;
  }
}

254 255 256
static void duplicate_mode_info_in_sb(VP9_COMMON *cm, MACROBLOCKD *xd,
                                      int mi_row, int mi_col,
                                      BLOCK_SIZE bsize) {
257 258 259 260 261 262
  const int block_width = num_8x8_blocks_wide_lookup[bsize];
  const int block_height = num_8x8_blocks_high_lookup[bsize];
  int i, j;
  for (j = 0; j < block_height; ++j)
    for (i = 0; i < block_width; ++i) {
      if (mi_row + j < cm->mi_rows && mi_col + i < cm->mi_cols)
hkuang's avatar
hkuang committed
263
        xd->mi[j * xd->mi_stride + i].src_mi = &xd->mi[0];
264 265 266 267
    }
}

static void set_block_size(VP9_COMP * const cpi,
268
                           MACROBLOCKD *const xd,
269 270 271 272
                           int mi_row, int mi_col,
                           BLOCK_SIZE bsize) {
  if (cpi->common.mi_cols > mi_col && cpi->common.mi_rows > mi_row) {
    set_modeinfo_offsets(&cpi->common, xd, mi_row, mi_col);
hkuang's avatar
hkuang committed
273
    xd->mi[0].src_mi->mbmi.sb_type = bsize;
274
    duplicate_mode_info_in_sb(&cpi->common, xd, mi_row, mi_col, bsize);
275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323
  }
}

typedef struct {
  int64_t sum_square_error;
  int64_t sum_error;
  int count;
  int variance;
} var;

typedef struct {
  var none;
  var horz[2];
  var vert[2];
} partition_variance;

typedef struct {
  partition_variance part_variances;
  var split[4];
} v8x8;

typedef struct {
  partition_variance part_variances;
  v8x8 split[4];
} v16x16;

typedef struct {
  partition_variance part_variances;
  v16x16 split[4];
} v32x32;

typedef struct {
  partition_variance part_variances;
  v32x32 split[4];
} v64x64;

typedef struct {
  partition_variance *part_variances;
  var *split[4];
} variance_node;

typedef enum {
  V16X16,
  V32X32,
  V64X64,
} TREE_LEVEL;

static void tree_to_node(void *data, BLOCK_SIZE bsize, variance_node *node) {
  int i;
324 325
  node->part_variances = NULL;
  vpx_memset(node->split, 0, sizeof(node->split));
326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356
  switch (bsize) {
    case BLOCK_64X64: {
      v64x64 *vt = (v64x64 *) data;
      node->part_variances = &vt->part_variances;
      for (i = 0; i < 4; i++)
        node->split[i] = &vt->split[i].part_variances.none;
      break;
    }
    case BLOCK_32X32: {
      v32x32 *vt = (v32x32 *) data;
      node->part_variances = &vt->part_variances;
      for (i = 0; i < 4; i++)
        node->split[i] = &vt->split[i].part_variances.none;
      break;
    }
    case BLOCK_16X16: {
      v16x16 *vt = (v16x16 *) data;
      node->part_variances = &vt->part_variances;
      for (i = 0; i < 4; i++)
        node->split[i] = &vt->split[i].part_variances.none;
      break;
    }
    case BLOCK_8X8: {
      v8x8 *vt = (v8x8 *) data;
      node->part_variances = &vt->part_variances;
      for (i = 0; i < 4; i++)
        node->split[i] = &vt->split[i];
      break;
    }
    default: {
      assert(0);
357
      break;
358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391
    }
  }
}

// Set variance values given sum square error, sum error, count.
static void fill_variance(int64_t s2, int64_t s, int c, var *v) {
  v->sum_square_error = s2;
  v->sum_error = s;
  v->count = c;
  if (c > 0)
    v->variance = (int)(256 *
                        (v->sum_square_error - v->sum_error * v->sum_error /
                         v->count) / v->count);
  else
    v->variance = 0;
}

void sum_2_variances(const var *a, const var *b, var *r) {
  fill_variance(a->sum_square_error + b->sum_square_error,
                a->sum_error + b->sum_error, a->count + b->count, r);
}

static void fill_variance_tree(void *data, BLOCK_SIZE bsize) {
  variance_node node;
  tree_to_node(data, bsize, &node);
  sum_2_variances(node.split[0], node.split[1], &node.part_variances->horz[0]);
  sum_2_variances(node.split[2], node.split[3], &node.part_variances->horz[1]);
  sum_2_variances(node.split[0], node.split[2], &node.part_variances->vert[0]);
  sum_2_variances(node.split[1], node.split[3], &node.part_variances->vert[1]);
  sum_2_variances(&node.part_variances->vert[0], &node.part_variances->vert[1],
                  &node.part_variances->none);
}

static int set_vt_partitioning(VP9_COMP *cpi,
392
                               MACROBLOCKD *const xd,
393 394 395
                               void *data,
                               BLOCK_SIZE bsize,
                               int mi_row,
Yaowu Xu's avatar
Yaowu Xu committed
396
                               int mi_col) {
397 398 399 400 401
  VP9_COMMON * const cm = &cpi->common;
  variance_node vt;
  const int block_width = num_8x8_blocks_wide_lookup[bsize];
  const int block_height = num_8x8_blocks_high_lookup[bsize];
  // TODO(debargha): Choose this more intelligently.
402 403 404 405
  const int threshold_multiplier = cm->frame_type == KEY_FRAME ? 64 : 4;
  int64_t threshold =
      (int64_t)(threshold_multiplier *
                vp9_convert_qindex_to_q(cm->base_qindex, cm->bit_depth));
406 407 408 409 410 411 412 413
  assert(block_height == block_width);
  tree_to_node(data, bsize, &vt);

  // Split none is available only if we have more than half a block size
  // in width and height inside the visible image.
  if (mi_col + block_width / 2 < cm->mi_cols &&
      mi_row + block_height / 2 < cm->mi_rows &&
      vt.part_variances->none.variance < threshold) {
414
    set_block_size(cpi, xd, mi_row, mi_col, bsize);
415 416 417
    return 1;
  }

418 419 420 421 422 423 424
  // Only allow split for blocks above 16x16.
  if (bsize > BLOCK_16X16) {
    // Vertical split is available on all but the bottom border.
    if (mi_row + block_height / 2 < cm->mi_rows &&
        vt.part_variances->vert[0].variance < threshold &&
        vt.part_variances->vert[1].variance < threshold) {
      BLOCK_SIZE subsize = get_subsize(bsize, PARTITION_VERT);
425 426
      set_block_size(cpi, xd, mi_row, mi_col, subsize);
      set_block_size(cpi, xd, mi_row, mi_col + block_width / 2, subsize);
427 428 429 430 431 432 433 434
      return 1;
    }

    // Horizontal split is available on all but the right border.
    if (mi_col + block_width / 2 < cm->mi_cols &&
        vt.part_variances->horz[0].variance < threshold &&
        vt.part_variances->horz[1].variance < threshold) {
      BLOCK_SIZE subsize = get_subsize(bsize, PARTITION_HORZ);
435 436
      set_block_size(cpi, xd, mi_row, mi_col, subsize);
      set_block_size(cpi, xd, mi_row + block_height / 2, mi_col, subsize);
437 438
      return 1;
    }
439 440
  }

441 442 443 444 445
  // This will only allow 8x8 if the 16x16 variance is very large.
  if (bsize == BLOCK_16X16) {
    if (mi_col + block_width / 2 < cm->mi_cols &&
        mi_row + block_height / 2 < cm->mi_rows &&
        vt.part_variances->none.variance < (threshold << 6)) {
446
      set_block_size(cpi, xd, mi_row, mi_col, bsize);
447 448
      return 1;
    }
449 450 451 452
  }
  return 0;
}

453 454 455 456 457 458
// This function chooses partitioning based on the variance
// between source and reconstructed last, where variance is
// computed for 8x8 downsampled inputs. Some things to check:
// using the last source rather than reconstructed last, and
// allowing for small downsampling (4x4 or 2x2) for selection
// of smaller block sizes (i.e., < 16x16).
459 460
static void choose_partitioning(VP9_COMP *cpi,
                                const TileInfo *const tile,
461
                                MACROBLOCK *x,
462 463
                                int mi_row, int mi_col) {
  VP9_COMMON * const cm = &cpi->common;
464
  MACROBLOCKD *xd = &x->e_mbd;
465 466 467 468 469 470 471 472 473 474 475

  int i, j, k;
  v64x64 vt;
  uint8_t *s;
  const uint8_t *d;
  int sp;
  int dp;
  int pixels_wide = 64, pixels_high = 64;
  const YV12_BUFFER_CONFIG *yv12 = get_ref_frame_buffer(cpi, LAST_FRAME);
  const struct scale_factors *const sf = &cm->frame_refs[LAST_FRAME - 1].sf;

476
  vp9_clear_system_state();
477
  vp9_zero(vt);
478
  set_offsets(cpi, tile, x, mi_row, mi_col, BLOCK_64X64);
479 480 481 482 483 484 485 486 487 488 489 490

  if (xd->mb_to_right_edge < 0)
    pixels_wide += (xd->mb_to_right_edge >> 3);
  if (xd->mb_to_bottom_edge < 0)
    pixels_high += (xd->mb_to_bottom_edge >> 3);

  s = x->plane[0].src.buf;
  sp = x->plane[0].src.stride;

  if (cm->frame_type != KEY_FRAME) {
    vp9_setup_pre_planes(xd, 0, yv12, mi_row, mi_col, sf);

hkuang's avatar
hkuang committed
491 492
    xd->mi[0].src_mi->mbmi.ref_frame[0] = LAST_FRAME;
    xd->mi[0].src_mi->mbmi.sb_type = BLOCK_64X64;
493
    xd->mi[0].src_mi->mbmi.mv[0].as_int = 0;
494 495 496 497 498 499 500
    vp9_build_inter_predictors_sby(xd, mi_row, mi_col, BLOCK_64X64);

    d = xd->plane[0].dst.buf;
    dp = xd->plane[0].dst.stride;
  } else {
    d = VP9_VAR_OFFS;
    dp = 0;
501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516
#if CONFIG_VP9_HIGHBITDEPTH
    if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
      switch (xd->bd) {
        case 10:
          d = CONVERT_TO_BYTEPTR(VP9_HIGH_VAR_OFFS_10);
          break;
        case 12:
          d = CONVERT_TO_BYTEPTR(VP9_HIGH_VAR_OFFS_12);
          break;
        case 8:
        default:
          d = CONVERT_TO_BYTEPTR(VP9_HIGH_VAR_OFFS_8);
          break;
      }
    }
#endif  // CONFIG_VP9_HIGHBITDEPTH
517 518 519 520 521 522 523 524 525 526 527 528 529 530 531
  }

  // Fill in the entire tree of 8x8 variances for splits.
  for (i = 0; i < 4; i++) {
    const int x32_idx = ((i & 1) << 5);
    const int y32_idx = ((i >> 1) << 5);
    for (j = 0; j < 4; j++) {
      const int x16_idx = x32_idx + ((j & 1) << 4);
      const int y16_idx = y32_idx + ((j >> 1) << 4);
      v16x16 *vst = &vt.split[i].split[j];
      for (k = 0; k < 4; k++) {
        int x_idx = x16_idx + ((k & 1) << 3);
        int y_idx = y16_idx + ((k >> 1) << 3);
        unsigned int sse = 0;
        int sum = 0;
532 533

        if (x_idx < pixels_wide && y_idx < pixels_high) {
534 535 536 537 538 539 540 541 542 543 544 545 546
          int s_avg, d_avg;
#if CONFIG_VP9_HIGHBITDEPTH
          if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
            s_avg = vp9_highbd_avg_8x8(s + y_idx * sp + x_idx, sp);
            d_avg = vp9_highbd_avg_8x8(d + y_idx * dp + x_idx, dp);
          } else {
            s_avg = vp9_avg_8x8(s + y_idx * sp + x_idx, sp);
            d_avg = vp9_avg_8x8(d + y_idx * dp + x_idx, dp);
          }
#else
          s_avg = vp9_avg_8x8(s + y_idx * sp + x_idx, sp);
          d_avg = vp9_avg_8x8(d + y_idx * dp + x_idx, dp);
#endif
547 548 549 550 551 552 553
          sum = s_avg - d_avg;
          sse = sum * sum;
        }
        // For an 8x8 block we have just one value the average of all 64
        // pixels,  so use 1.   This means of course that there is no variance
        // in an 8x8 block.
        fill_variance(sse, sum, 1, &vst->split[k].part_variances.none);
554 555 556 557 558 559 560 561 562 563 564 565 566 567 568
      }
    }
  }
  // Fill the rest of the variance tree by summing split partition values.
  for (i = 0; i < 4; i++) {
    for (j = 0; j < 4; j++) {
      fill_variance_tree(&vt.split[i].split[j], BLOCK_16X16);
    }
    fill_variance_tree(&vt.split[i], BLOCK_32X32);
  }
  fill_variance_tree(&vt, BLOCK_64X64);

  // Now go through the entire structure,  splitting every block size until
  // we get to one that's got a variance lower than our threshold,  or we
  // hit 8x8.
569
  if ( mi_col + 8 > cm->mi_cols || mi_row + 8 > cm->mi_rows ||
570
      !set_vt_partitioning(cpi, xd, &vt, BLOCK_64X64, mi_row, mi_col)) {
571 572 573
    for (i = 0; i < 4; ++i) {
      const int x32_idx = ((i & 1) << 2);
      const int y32_idx = ((i >> 1) << 2);
574
      if (!set_vt_partitioning(cpi, xd, &vt.split[i], BLOCK_32X32,
Yaowu Xu's avatar
Yaowu Xu committed
575
                               (mi_row + y32_idx), (mi_col + x32_idx))) {
576 577 578
        for (j = 0; j < 4; ++j) {
          const int x16_idx = ((j & 1) << 1);
          const int y16_idx = ((j >> 1) << 1);
579 580 581 582 583
          // NOTE: Since this uses 8x8 downsampling for variance calculation
          // we cannot really select block size 8x8 (or even 8x16/16x8),
          // since we do not sufficient samples for variance.
          // For now, 8x8 partition is only set if the variance of the 16x16
          // block is very high. This is controlled in set_vt_partitioning.
584
          if (!set_vt_partitioning(cpi, xd, &vt.split[i].split[j],
585
                                   BLOCK_16X16,
586 587
                                   mi_row + y32_idx + y16_idx,
                                   mi_col + x32_idx + x16_idx)) {
588 589 590
            for (k = 0; k < 4; ++k) {
              const int x8_idx = (k & 1);
              const int y8_idx = (k >> 1);
591
              set_block_size(cpi, xd,
592 593 594 595 596 597 598 599 600 601 602
                             (mi_row + y32_idx + y16_idx + y8_idx),
                             (mi_col + x32_idx + x16_idx + x8_idx),
                             BLOCK_8X8);
            }
          }
        }
      }
    }
  }
}

603 604
static void update_state(VP9_COMP *cpi, ThreadData *td,
                         PICK_MODE_CONTEXT *ctx,
605 606
                         int mi_row, int mi_col, BLOCK_SIZE bsize,
                         int output_enabled) {
Ronald S. Bultje's avatar
Ronald S. Bultje committed
607
  int i, x_idx, y;
608
  VP9_COMMON *const cm = &cpi->common;
609 610
  RD_COUNTS *const rdc = &td->rd_counts;
  MACROBLOCK *const x = &td->mb;
611
  MACROBLOCKD *const xd = &x->e_mbd;
612 613
  struct macroblock_plane *const p = x->plane;
  struct macroblockd_plane *const pd = xd->plane;
John Koleszar's avatar
John Koleszar committed
614
  MODE_INFO *mi = &ctx->mic;
hkuang's avatar
hkuang committed
615 616
  MB_MODE_INFO *const mbmi = &xd->mi[0].src_mi->mbmi;
  MODE_INFO *mi_addr = &xd->mi[0];
617
  const struct segmentation *const seg = &cm->seg;
618 619 620 621 622 623 624
  const int bw = num_8x8_blocks_wide_lookup[mi->mbmi.sb_type];
  const int bh = num_8x8_blocks_high_lookup[mi->mbmi.sb_type];
  const int x_mis = MIN(bw, cm->mi_cols - mi_col);
  const int y_mis = MIN(bh, cm->mi_rows - mi_row);
  MV_REF *const frame_mvs =
      cm->cur_frame->mvs + mi_row * cm->mi_cols + mi_col;
  int w, h;
625

626
  const int mis = cm->mi_stride;
Jim Bankoski's avatar
Jim Bankoski committed
627
  const int mi_width = num_8x8_blocks_wide_lookup[bsize];
Jim Bankoski's avatar
Jim Bankoski committed
628
  const int mi_height = num_8x8_blocks_high_lookup[bsize];
629
  int max_plane;
Adrian Grange's avatar
Adrian Grange committed
630

631
  assert(mi->mbmi.sb_type == bsize);
632

633
  *mi_addr = *mi;
hkuang's avatar
hkuang committed
634
  mi_addr->src_mi = mi_addr;
635

Paul Wilkins's avatar
Paul Wilkins committed
636 637 638 639 640 641 642 643 644 645 646
  // If segmentation in use
  if (seg->enabled && output_enabled) {
    // For in frame complexity AQ copy the segment id from the segment map.
    if (cpi->oxcf.aq_mode == COMPLEXITY_AQ) {
      const uint8_t *const map = seg->update_map ? cpi->segmentation_map
                                                 : cm->last_frame_seg_map;
      mi_addr->mbmi.segment_id =
        vp9_get_segment_id(cm, map, bsize, mi_row, mi_col);
    }
    // Else for cyclic refresh mode update the segment map, set the segment id
    // and then update the quantizer.
647
    if (cpi->oxcf.aq_mode == CYCLIC_REFRESH_AQ) {
hkuang's avatar
hkuang committed
648
      vp9_cyclic_refresh_update_segment(cpi, &xd->mi[0].src_mi->mbmi,
649
                                        mi_row, mi_col, bsize, 1, ctx->rate);
Paul Wilkins's avatar
Paul Wilkins committed
650
    }
651
  }
652

653 654
  max_plane = is_inter_block(mbmi) ? MAX_MB_PLANE : 1;
  for (i = 0; i < max_plane; ++i) {
655
    p[i].coeff = ctx->coeff_pbuf[i][1];
656
    p[i].qcoeff = ctx->qcoeff_pbuf[i][1];
657
    pd[i].dqcoeff = ctx->dqcoeff_pbuf[i][1];
658
    p[i].eobs = ctx->eobs_pbuf[i][1];
659 660
  }

661 662
  for (i = max_plane; i < MAX_MB_PLANE; ++i) {
    p[i].coeff = ctx->coeff_pbuf[i][2];
663
    p[i].qcoeff = ctx->qcoeff_pbuf[i][2];
664
    pd[i].dqcoeff = ctx->dqcoeff_pbuf[i][2];
665
    p[i].eobs = ctx->eobs_pbuf[i][2];
666 667
  }

John Koleszar's avatar
John Koleszar committed
668 669
  // Restore the coding context of the MB to that that was in place
  // when the mode was picked for it
670 671
  for (y = 0; y < mi_height; y++)
    for (x_idx = 0; x_idx < mi_width; x_idx++)
James Zern's avatar
James Zern committed
672
      if ((xd->mb_to_right_edge >> (3 + MI_SIZE_LOG2)) + mi_width > x_idx
673
        && (xd->mb_to_bottom_edge >> (3 + MI_SIZE_LOG2)) + mi_height > y) {
hkuang's avatar
hkuang committed
674
        xd->mi[x_idx + y * mis].src_mi = mi_addr;
675
      }
676

677
  if (cpi->oxcf.aq_mode)
Dmitry Kovalev's avatar
Dmitry Kovalev committed
678
    vp9_init_plane_quantizers(cpi, x);
679

680 681
  // FIXME(rbultje) I'm pretty sure this should go to the end of this block
  // (i.e. after the output_enabled)
682 683
  if (bsize < BLOCK_32X32) {
    if (bsize < BLOCK_16X16)
684 685
      ctx->tx_rd_diff[ALLOW_16X16] = ctx->tx_rd_diff[ALLOW_8X8];
    ctx->tx_rd_diff[ALLOW_32X32] = ctx->tx_rd_diff[ALLOW_16X16];
Ronald S. Bultje's avatar
Ronald S. Bultje committed
686
  }
Adrian Grange's avatar
Adrian Grange committed
687

688
  if (is_inter_block(mbmi) && mbmi->sb_type < BLOCK_8X8) {
689 690
    mbmi->mv[0].as_int = mi->bmi[3].as_mv[0].as_int;
    mbmi->mv[1].as_int = mi->bmi[3].as_mv[1].as_int;
John Koleszar's avatar
John Koleszar committed
691 692
  }

693
  x->skip = ctx->skip;
694
  vpx_memcpy(x->zcoeff_blk[mbmi->tx_size], ctx->zcoeff_blk,
695
             sizeof(uint8_t) * ctx->num_4x4_blk);
696

Ronald S. Bultje's avatar
Ronald S. Bultje committed
697 698 699
  if (!output_enabled)
    return;

700
  if (!vp9_segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
701
    for (i = 0; i < TX_MODES; i++)
702
      rdc->tx_select_diff[i] += ctx->tx_rd_diff[i];
703 704
  }

705
#if CONFIG_INTERNAL_STATS
706
  if (frame_is_intra_only(cm)) {
John Koleszar's avatar
John Koleszar committed
707
    static const int kf_mode_index[] = {
708 709 710 711
      THR_DC        /*DC_PRED*/,
      THR_V_PRED    /*V_PRED*/,
      THR_H_PRED    /*H_PRED*/,
      THR_D45_PRED  /*D45_PRED*/,
John Koleszar's avatar
John Koleszar committed
712 713 714
      THR_D135_PRED /*D135_PRED*/,
      THR_D117_PRED /*D117_PRED*/,
      THR_D153_PRED /*D153_PRED*/,
Dmitry Kovalev's avatar
Dmitry Kovalev committed
715
      THR_D207_PRED /*D207_PRED*/,
716 717
      THR_D63_PRED  /*D63_PRED*/,
      THR_TM        /*TM_PRED*/,
John Koleszar's avatar
John Koleszar committed
718
    };
719
    ++cpi->mode_chosen_counts[kf_mode_index[mbmi->mode]];
John Koleszar's avatar
John Koleszar committed
720 721
  } else {
    // Note how often each mode chosen as best
722 723 724 725
    ++cpi->mode_chosen_counts[ctx->best_mode_index];
  }
#endif
  if (!frame_is_intra_only(cm)) {
726
    if (is_inter_block(mbmi)) {
727
      vp9_update_mv_count(td);
728 729 730

      if (cm->interp_filter == SWITCHABLE) {
        const int ctx = vp9_get_pred_context_switchable_interp(xd);
731
        ++td->counts->switchable_interp[ctx][mbmi->interp_filter];
732
      }
733
    }
Adrian Grange's avatar
Adrian Grange committed
734

735 736 737
    rdc->comp_pred_diff[SINGLE_REFERENCE] += ctx->single_pred_diff;
    rdc->comp_pred_diff[COMPOUND_REFERENCE] += ctx->comp_pred_diff;
    rdc->comp_pred_diff[REFERENCE_MODE_SELECT] += ctx->hybrid_pred_diff;
738

739
    for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; ++i)
740
      rdc->filter_diff[i] += ctx->best_filter_diff[i];
John Koleszar's avatar
John Koleszar committed
741
  }
742 743 744 745 746 747 748 749 750 751 752

  for (h = 0; h < y_mis; ++h) {
    MV_REF *const frame_mv = frame_mvs + h * cm->mi_cols;
    for (w = 0; w < x_mis; ++w) {
      MV_REF *const mv = frame_mv + w;
      mv->ref_frame[0] = mi->src_mi->mbmi.ref_frame[0];
      mv->ref_frame[1] = mi->src_mi->mbmi.ref_frame[1];
      mv->mv[0].as_int = mi->src_mi->mbmi.mv[0].as_int;
      mv->mv[1].as_int = mi->src_mi->mbmi.mv[1].as_int;
    }
  }
Adrian Grange's avatar
Adrian Grange committed
753 754
}

Jim Bankoski's avatar
Jim Bankoski committed
755
void vp9_setup_src_planes(MACROBLOCK *x, const YV12_BUFFER_CONFIG *src,
756
                          int mi_row, int mi_col) {
757 758
  uint8_t *const buffers[3] = {src->y_buffer, src->u_buffer, src->v_buffer };
  const int strides[3] = {src->y_stride, src->uv_stride, src->uv_stride };
759 760
  int i;

761 762 763
  // Set current frame pointer.
  x->e_mbd.cur_buf = src;

764 765
  for (i = 0; i < MAX_MB_PLANE; i++)
    setup_pred_plane(&x->plane[i].src, buffers[i], strides[i], mi_row, mi_col,
Jim Bankoski's avatar
Jim Bankoski committed
766
                     NULL, x->e_mbd.plane[i].subsampling_x,
767
                     x->e_mbd.plane[i].subsampling_y);
John Koleszar's avatar
John Koleszar committed
768 769
}

770 771
static void set_mode_info_seg_skip(MACROBLOCK *x, TX_MODE tx_mode,
                                   RD_COST *rd_cost, BLOCK_SIZE bsize) {
772
  MACROBLOCKD *const xd = &x->e_mbd;
hkuang's avatar
hkuang committed
773
  MB_MODE_INFO *const mbmi = &xd->mi[0].src_mi->mbmi;
774 775 776
  INTERP_FILTER filter_ref;

  if (xd->up_available)
hkuang's avatar
hkuang committed
777
    filter_ref = xd->mi[-xd->mi_stride].src_mi->mbmi.interp_filter;
778
  else if (xd->left_available)
hkuang's avatar
hkuang committed
779
    filter_ref = xd->mi[-1].src_mi->mbmi.interp_filter;
780 781 782 783 784 785 786 787 788 789 790 791 792 793
  else
    filter_ref = EIGHTTAP;

  mbmi->sb_type = bsize;
  mbmi->mode = ZEROMV;
  mbmi->tx_size = MIN(max_txsize_lookup[bsize],
                      tx_mode_to_biggest_tx_size[tx_mode]);
  mbmi->skip = 1;
  mbmi->uv_mode = DC_PRED;
  mbmi->ref_frame[0] = LAST_FRAME;
  mbmi->ref_frame[1] = NONE;
  mbmi->mv[0].as_int = 0;
  mbmi->interp_filter = filter_ref;

hkuang's avatar
hkuang committed
794
  xd->mi[0].src_mi->bmi[0].as_mv[0].as_int = 0;
795 796
  x->skip = 1;

797
  vp9_rd_cost_init(rd_cost);
798 799
}

800
static void rd_pick_sb_modes(VP9_COMP *cpi,
801
                             TileDataEnc *tile_data,
802
                             MACROBLOCK *const x,
803
                             int mi_row, int mi_col, RD_COST *rd_cost,
804
                             BLOCK_SIZE bsize, PICK_MODE_CONTEXT *ctx,
805
                             int64_t best_rd) {
806
  VP9_COMMON *const cm = &cpi->common;
807
  TileInfo *const tile_info = &tile_data->tile_info;
808
  MACROBLOCKD *const xd = &x->e_mbd;
809
  MB_MODE_INFO *mbmi;
810 811
  struct macroblock_plane *const p = x->plane;
  struct macroblockd_plane *const pd = xd->plane;
812
  const AQ_MODE aq_mode = cpi->oxcf.aq_mode;
813
  int i, orig_rdmult;
814

815
  vp9_clear_system_state();
Ronald S. Bultje's avatar
Ronald S. Bultje committed
816

817
  // Use the lower precision, but faster, 32x32 fdct for mode selection.
818
  x->use_lp32x32fdct = 1;
819

820
  set_offsets(cpi, tile_info, x, mi_row, mi_col, bsize);
hkuang's avatar
hkuang committed
821
  mbmi = &xd->mi[0].src_mi->mbmi;
822
  mbmi->sb_type = bsize;
823

824
  for (i = 0; i < MAX_MB_PLANE; ++i) {
825
    p[i].coeff = ctx->coeff_pbuf[i][0];
826
    p[i].qcoeff = ctx->qcoeff_pbuf[i][0];
827
    pd[i].dqcoeff = ctx->dqcoeff_pbuf[i][0];
828
    p[i].eobs = ctx->eobs_pbuf[i][0];
829
  }
830
  ctx->is_coded = 0;
831
  ctx->skippable = 0;
832
  ctx->pred_pixel_ready = 0;
833
  x->skip_recode = 0;
834

835
  // Set to zero to make sure we do not use the previous encoded frame stats
836
  mbmi->skip = 0;
837

838 839 840 841 842 843 844 845 846
#if CONFIG_VP9_HIGHBITDEPTH
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
    x->source_variance =
        high_get_sby_perpixel_variance(cpi, &x->plane[0].src, bsize, xd->bd);
  } else {
    x->source_variance =
        get_sby_perpixel_variance(cpi, &x->plane[0].src, bsize);
  }
#else
847
  x->source_variance = get_sby_perpixel_variance(cpi, &x->plane[0].src, bsize);
848
#endif  // CONFIG_VP9_HIGHBITDEPTH
849

850 851 852
  // Save rdmult before it might be changed, so it can be restored later.
  orig_rdmult = x->rdmult;

853
  if (aq_mode == VARIANCE_AQ) {
854 855
    const int energy = bsize <= BLOCK_16X16 ? x->mb_energy
                                            : vp9_block_energy(cpi, x, bsize);
Paul Wilkins's avatar
Paul Wilkins committed
856
    int segment_qindex;
Paul Wilkins's avatar
Paul Wilkins committed
857 858 859
    if (cm->frame_type == KEY_FRAME ||
        cpi->refresh_alt_ref_frame ||
        (cpi->refresh_golden_frame && !cpi->rc.is_src_frame_alt_ref)) {
860
      mbmi->segment_id = vp9_vaq_segment_id(energy);
Paul Wilkins's avatar
Paul Wilkins committed
861 862 863
    } else {
      const uint8_t *const map = cm->seg.update_map ? cpi->segmentation_map
                                                    : cm->last_frame_seg_map;
864
      mbmi->segment_id = vp9_get_segment_id(cm, map, bsize, mi_row, mi_col);
Paul Wilkins's avatar
Paul Wilkins committed
865
    }
Dmitry Kovalev's avatar
Dmitry Kovalev committed
866
    vp9_init_plane_quantizers(cpi, x);
867
    vp9_clear_system_state();
Paul Wilkins's avatar
Paul Wilkins committed
868 869 870
    segment_qindex = vp9_get_qindex(&cm->seg, mbmi->segment_id,
                                    cm->base_qindex);
    x->rdmult = vp9_compute_rd_mult(cpi, segment_qindex + cm->y_dc_delta_q);
871
  } else if (aq_mode == COMPLEXITY_AQ) {
872 873
    const int mi_offset = mi_row * cm->mi_cols + mi_col;
    unsigned char complexity = cpi->complexity_map[mi_offset];
Paul Wilkins's avatar
Paul Wilkins committed
874 875
    const int is_edge = (mi_row <= 1) || (mi_row >= (cm->mi_rows - 2)) ||
                        (mi_col <= 1) || (mi_col >= (cm->mi_cols - 2));
876 877
    if (!is_edge && (complexity > 128))
      x->rdmult += ((x->rdmult * (complexity - 128)) / 256);
878
  } else if (aq_mode == CYCLIC_REFRESH_AQ) {
879
    const uint8_t *const map = cm->seg.update_map ? cpi->segmentation_map
880
                                                  : cm->last_frame_seg_map;
881 882
    // If segment 1, use rdmult for that segment.
    if (vp9_get_segment_id(cm, map, bsize, mi_row, mi_col))
883
      x->rdmult = vp9_cyclic_refresh_get_rdmult(cpi->cyclic_refresh);
884
  }
885

886 887
  // Find best coding mode & reconstruct the MB so it is available
  // as a predictor for MBs that follow in the SB
888
  if (frame_is_intra_only(cm)) {
889
    vp9_rd_pick_intra_mode_sb(cpi, x, rd_cost, bsize, ctx, best_rd);
890
  } else {
891 892
    if (bsize >= BLOCK_8X8) {
      if (vp9_segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP))
893
        vp9_rd_pick_inter_mode_sb_seg_skip(cpi, tile_data, x, rd_cost, bsize,
894
                                           ctx, best_rd);
895
      else
896
        vp9_rd_pick_inter_mode_sb(cpi, tile_data, x, mi_row, mi_col,