vp9_encoder.c 137 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 * Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5 6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9 10
 */

Jim Bankoski's avatar
Jim Bankoski committed
11 12 13 14 15 16
#include <math.h>
#include <stdio.h>
#include <limits.h>

#include "./vpx_config.h"
#include "./vpx_scale_rtcd.h"
17 18
#include "vpx/internal/vpx_psnr.h"
#include "vpx_ports/vpx_timer.h"
John Koleszar's avatar
John Koleszar committed
19

Jim Bankoski's avatar
Jim Bankoski committed
20
#include "vp9/common/vp9_alloccommon.h"
21
#include "vp9/common/vp9_filter.h"
22
#include "vp9/common/vp9_idct.h"
Jim Bankoski's avatar
Jim Bankoski committed
23 24 25
#if CONFIG_VP9_POSTPROC
#include "vp9/common/vp9_postproc.h"
#endif
26
#include "vp9/common/vp9_reconinter.h"
27
#include "vp9/common/vp9_reconintra.h"
28
#include "vp9/common/vp9_systemdependent.h"
Jim Bankoski's avatar
Jim Bankoski committed
29
#include "vp9/common/vp9_tile_common.h"
30

31
#include "vp9/encoder/vp9_aq_complexity.h"
Marco Paniconi's avatar
Marco Paniconi committed
32 33
#include "vp9/encoder/vp9_aq_cyclicrefresh.h"
#include "vp9/encoder/vp9_aq_variance.h"
34
#include "vp9/encoder/vp9_bitstream.h"
35
#include "vp9/encoder/vp9_context_tree.h"
36
#include "vp9/encoder/vp9_encodeframe.h"
37
#include "vp9/encoder/vp9_encodemv.h"
38
#include "vp9/encoder/vp9_firstpass.h"
Jim Bankoski's avatar
Jim Bankoski committed
39
#include "vp9/encoder/vp9_mbgraph.h"
Dmitry Kovalev's avatar
Dmitry Kovalev committed
40
#include "vp9/encoder/vp9_encoder.h"
Jim Bankoski's avatar
Jim Bankoski committed
41
#include "vp9/encoder/vp9_picklpf.h"
42
#include "vp9/encoder/vp9_ratectrl.h"
43
#include "vp9/encoder/vp9_rd.h"
Jim Bankoski's avatar
Jim Bankoski committed
44
#include "vp9/encoder/vp9_segmentation.h"
Yaowu Xu's avatar
Yaowu Xu committed
45
#include "vp9/encoder/vp9_speed_features.h"
Dmitry Kovalev's avatar
Dmitry Kovalev committed
46 47 48
#if CONFIG_INTERNAL_STATS
#include "vp9/encoder/vp9_ssim.h"
#endif
49
#include "vp9/encoder/vp9_temporal_filter.h"
50
#include "vp9/encoder/vp9_resize.h"
51
#include "vp9/encoder/vp9_svc_layercontext.h"
Paul Wilkins's avatar
Paul Wilkins committed
52

53
void vp9_coef_tree_initialize();
54

55 56
#define SHARP_FILTER_QTHRESH 0          /* Q threshold for 8-tap sharp filter */

Jim Bankoski's avatar
Jim Bankoski committed
57 58 59 60 61 62
#define ALTREF_HIGH_PRECISION_MV 1      // Whether to use high precision mv
                                         //  for altref computation.
#define HIGH_PRECISION_MV_QTHRESH 200   // Q threshold for high precision
                                         // mv. Choose a very high value for
                                         // now so that HIGH_PRECISION is always
                                         // chosen.
63

John Koleszar's avatar
John Koleszar committed
64
// #define OUTPUT_YUV_REC
John Koleszar's avatar
John Koleszar committed
65

66
#ifdef OUTPUT_YUV_DENOISED
67
FILE *yuv_denoised_file = NULL;
68
#endif
69 70 71
#ifdef OUTPUT_YUV_REC
FILE *yuv_rec_file;
#endif
John Koleszar's avatar
John Koleszar committed
72 73 74

#if 0
FILE *framepsnr;
Yaowu Xu's avatar
Yaowu Xu committed
75
FILE *kf_list;
John Koleszar's avatar
John Koleszar committed
76 77 78
FILE *keyfile;
#endif

79
static INLINE void Scale2Ratio(VPX_SCALING mode, int *hr, int *hs) {
80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104
  switch (mode) {
    case NORMAL:
      *hr = 1;
      *hs = 1;
      break;
    case FOURFIVE:
      *hr = 4;
      *hs = 5;
      break;
    case THREEFIVE:
      *hr = 3;
      *hs = 5;
    break;
    case ONETWO:
      *hr = 1;
      *hs = 2;
    break;
    default:
      *hr = 1;
      *hs = 1;
       assert(0);
      break;
  }
}

105
void vp9_set_high_precision_mv(VP9_COMP *cpi, int allow_high_precision_mv) {
106
  MACROBLOCK *const mb = &cpi->td.mb;
Deb Mukherjee's avatar
Deb Mukherjee committed
107
  cpi->common.allow_high_precision_mv = allow_high_precision_mv;
108
  if (cpi->common.allow_high_precision_mv) {
109 110 111 112 113 114 115
    mb->mvcost = mb->nmvcost_hp;
    mb->mvsadcost = mb->nmvsadcost_hp;
  } else {
    mb->mvcost = mb->nmvcost;
    mb->mvsadcost = mb->nmvsadcost;
  }
}
Paul Wilkins's avatar
Paul Wilkins committed
116

117 118 119 120 121 122 123
static void setup_frame(VP9_COMP *cpi) {
  VP9_COMMON *const cm = &cpi->common;
  // Set up entropy context depending on frame type. The decoder mandates
  // the use of the default context, index 0, for keyframes and inter
  // frames where the error_resilient_mode or intra_only flag is set. For
  // other inter-frames the encoder currently uses only two contexts;
  // context 1 for ALTREF frames and context 0 for the others.
124 125 126 127 128 129 130
  if (frame_is_intra_only(cm) || cm->error_resilient_mode) {
    vp9_setup_past_independence(cm);
  } else {
    if (!cpi->use_svc)
      cm->frame_context_idx = cpi->refresh_alt_ref_frame;
  }

131
  if (cm->frame_type == KEY_FRAME) {
132
    if (!is_two_pass_svc(cpi))
133
      cpi->refresh_golden_frame = 1;
134
    cpi->refresh_alt_ref_frame = 1;
135
    vp9_zero(cpi->interp_filter_selected);
136
  } else {
137
    *cm->fc = cm->frame_contexts[cm->frame_context_idx];
138
    vp9_zero(cpi->interp_filter_selected[0]);
139 140 141
  }
}

142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182
static void vp9_enc_setup_mi(VP9_COMMON *cm) {
  int i;
  cm->mi = cm->mip + cm->mi_stride + 1;
  vpx_memset(cm->mip, 0, cm->mi_stride * (cm->mi_rows + 1) * sizeof(*cm->mip));
  cm->prev_mi = cm->prev_mip + cm->mi_stride + 1;
  // Clear top border row
  vpx_memset(cm->prev_mip, 0, sizeof(*cm->prev_mip) * cm->mi_stride);
  // Clear left border column
  for (i = 1; i < cm->mi_rows + 1; ++i)
    vpx_memset(&cm->prev_mip[i * cm->mi_stride], 0, sizeof(*cm->prev_mip));
}

static int vp9_enc_alloc_mi(VP9_COMMON *cm, int mi_size) {
  cm->mip = vpx_calloc(mi_size, sizeof(*cm->mip));
  if (!cm->mip)
    return 1;
  cm->prev_mip = vpx_calloc(mi_size, sizeof(*cm->prev_mip));
  if (!cm->prev_mip)
    return 1;
  cm->mi_alloc_size = mi_size;
  return 0;
}

static void vp9_enc_free_mi(VP9_COMMON *cm) {
  vpx_free(cm->mip);
  cm->mip = NULL;
  vpx_free(cm->prev_mip);
  cm->prev_mip = NULL;
}

static void vp9_swap_mi_and_prev_mi(VP9_COMMON *cm) {
  // Current mip will be the prev_mip for the next frame.
  MODE_INFO *temp = cm->prev_mip;
  cm->prev_mip = cm->mip;
  cm->mip = temp;

  // Update the upper left visible macroblock ptrs.
  cm->mi = cm->mip + cm->mi_stride + 1;
  cm->prev_mi = cm->prev_mip + cm->mi_stride + 1;
}

183
void vp9_initialize_enc() {
John Koleszar's avatar
John Koleszar committed
184 185 186
  static int init_done = 0;

  if (!init_done) {
187 188
    vp9_rtcd();
    vp9_init_intra_predictors();
189
    vp9_coef_tree_initialize();
190 191
    vp9_tokenize_initialize();
    vp9_init_me_luts();
192
    vp9_rc_init_minq_luts();
193
    vp9_entropy_mv_init();
194
    vp9_entropy_mode_init();
195
    vp9_temporal_filter_init();
John Koleszar's avatar
John Koleszar committed
196 197
    init_done = 1;
  }
John Koleszar's avatar
John Koleszar committed
198 199
}

200
static void dealloc_compressor_data(VP9_COMP *cpi) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
201
  VP9_COMMON *const cm = &cpi->common;
202
  int i;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
203

204 205
  vpx_free(cpi->tile_data);
  cpi->tile_data = NULL;
206

207
  // Delete sementation map
John Koleszar's avatar
John Koleszar committed
208
  vpx_free(cpi->segmentation_map);
Dmitry Kovalev's avatar
Dmitry Kovalev committed
209 210 211
  cpi->segmentation_map = NULL;
  vpx_free(cm->last_frame_seg_map);
  cm->last_frame_seg_map = NULL;
John Koleszar's avatar
John Koleszar committed
212
  vpx_free(cpi->coding_context.last_frame_seg_map_copy);
Dmitry Kovalev's avatar
Dmitry Kovalev committed
213
  cpi->coding_context.last_frame_seg_map_copy = NULL;
John Koleszar's avatar
John Koleszar committed
214

215
  vpx_free(cpi->complexity_map);
216 217
  cpi->complexity_map = NULL;

Jingning Han's avatar
Jingning Han committed
218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237
  vpx_free(cpi->nmvcosts[0]);
  vpx_free(cpi->nmvcosts[1]);
  cpi->nmvcosts[0] = NULL;
  cpi->nmvcosts[1] = NULL;

  vpx_free(cpi->nmvcosts_hp[0]);
  vpx_free(cpi->nmvcosts_hp[1]);
  cpi->nmvcosts_hp[0] = NULL;
  cpi->nmvcosts_hp[1] = NULL;

  vpx_free(cpi->nmvsadcosts[0]);
  vpx_free(cpi->nmvsadcosts[1]);
  cpi->nmvsadcosts[0] = NULL;
  cpi->nmvsadcosts[1] = NULL;

  vpx_free(cpi->nmvsadcosts_hp[0]);
  vpx_free(cpi->nmvsadcosts_hp[1]);
  cpi->nmvsadcosts_hp[0] = NULL;
  cpi->nmvsadcosts_hp[1] = NULL;

238 239 240
  vp9_cyclic_refresh_free(cpi->cyclic_refresh);
  cpi->cyclic_refresh = NULL;

Adrian Grange's avatar
Adrian Grange committed
241
  vp9_free_ref_frame_buffers(cm);
242
  vp9_free_context_buffers(cm);
John Koleszar's avatar
John Koleszar committed
243

244 245
  vp9_free_frame_buffer(&cpi->last_frame_uf);
  vp9_free_frame_buffer(&cpi->scaled_source);
246
  vp9_free_frame_buffer(&cpi->scaled_last_source);
247
  vp9_free_frame_buffer(&cpi->alt_ref_buffer);
248
  vp9_lookahead_destroy(cpi->lookahead);
John Koleszar's avatar
John Koleszar committed
249

250 251
  vpx_free(cpi->tile_tok[0][0]);
  cpi->tile_tok[0][0] = 0;
John Koleszar's avatar
John Koleszar committed
252

253
  vp9_free_pc_tree(&cpi->td);
254

255 256 257 258 259 260
  for (i = 0; i < cpi->svc.number_spatial_layers; ++i) {
    LAYER_CONTEXT *const lc = &cpi->svc.layer_context[i];
    vpx_free(lc->rc_twopass_stats_in.buf);
    lc->rc_twopass_stats_in.buf = NULL;
    lc->rc_twopass_stats_in.sz = 0;
  }
261 262 263 264 265

  if (cpi->source_diff_var != NULL) {
    vpx_free(cpi->source_diff_var);
    cpi->source_diff_var = NULL;
  }
266

267 268 269 270 271
  for (i = 0; i < MAX_LAG_BUFFERS; ++i) {
    vp9_free_frame_buffer(&cpi->svc.scaled_frames[i]);
  }
  vpx_memset(&cpi->svc.scaled_frames[0], 0,
             MAX_LAG_BUFFERS * sizeof(cpi->svc.scaled_frames[0]));
272 273 274

  vp9_free_frame_buffer(&cpi->svc.empty_frame.img);
  vpx_memset(&cpi->svc.empty_frame, 0, sizeof(cpi->svc.empty_frame));
John Koleszar's avatar
John Koleszar committed
275 276
}

277 278 279 280 281 282 283 284
static void save_coding_context(VP9_COMP *cpi) {
  CODING_CONTEXT *const cc = &cpi->coding_context;
  VP9_COMMON *cm = &cpi->common;

  // Stores a snapshot of key state variables which can subsequently be
  // restored with a call to vp9_restore_coding_context. These functions are
  // intended for use in a re-code loop in vp9_compress_frame where the
  // quantizer value is adjusted between loop iterations.
285
  vp9_copy(cc->nmvjointcost,  cpi->td.mb.nmvjointcost);
Jingning Han's avatar
Jingning Han committed
286 287 288 289 290 291 292 293 294

  vpx_memcpy(cc->nmvcosts[0], cpi->nmvcosts[0],
             MV_VALS * sizeof(*cpi->nmvcosts[0]));
  vpx_memcpy(cc->nmvcosts[1], cpi->nmvcosts[1],
             MV_VALS * sizeof(*cpi->nmvcosts[1]));
  vpx_memcpy(cc->nmvcosts_hp[0], cpi->nmvcosts_hp[0],
             MV_VALS * sizeof(*cpi->nmvcosts_hp[0]));
  vpx_memcpy(cc->nmvcosts_hp[1], cpi->nmvcosts_hp[1],
             MV_VALS * sizeof(*cpi->nmvcosts_hp[1]));
295 296 297 298 299 300 301 302 303

  vp9_copy(cc->segment_pred_probs, cm->seg.pred_probs);

  vpx_memcpy(cpi->coding_context.last_frame_seg_map_copy,
             cm->last_frame_seg_map, (cm->mi_rows * cm->mi_cols));

  vp9_copy(cc->last_ref_lf_deltas, cm->lf.last_ref_deltas);
  vp9_copy(cc->last_mode_lf_deltas, cm->lf.last_mode_deltas);

304
  cc->fc = *cm->fc;
305 306 307 308 309 310 311 312
}

static void restore_coding_context(VP9_COMP *cpi) {
  CODING_CONTEXT *const cc = &cpi->coding_context;
  VP9_COMMON *cm = &cpi->common;

  // Restore key state variables to the snapshot state stored in the
  // previous call to vp9_save_coding_context.
313
  vp9_copy(cpi->td.mb.nmvjointcost, cc->nmvjointcost);
Jingning Han's avatar
Jingning Han committed
314 315 316 317 318 319 320 321 322

  vpx_memcpy(cpi->nmvcosts[0], cc->nmvcosts[0],
             MV_VALS * sizeof(*cc->nmvcosts[0]));
  vpx_memcpy(cpi->nmvcosts[1], cc->nmvcosts[1],
             MV_VALS * sizeof(*cc->nmvcosts[1]));
  vpx_memcpy(cpi->nmvcosts_hp[0], cc->nmvcosts_hp[0],
             MV_VALS * sizeof(*cc->nmvcosts_hp[0]));
  vpx_memcpy(cpi->nmvcosts_hp[1], cc->nmvcosts_hp[1],
             MV_VALS * sizeof(*cc->nmvcosts_hp[1]));
323 324 325 326 327 328 329 330 331 332

  vp9_copy(cm->seg.pred_probs, cc->segment_pred_probs);

  vpx_memcpy(cm->last_frame_seg_map,
             cpi->coding_context.last_frame_seg_map_copy,
             (cm->mi_rows * cm->mi_cols));

  vp9_copy(cm->lf.last_ref_deltas, cc->last_ref_lf_deltas);
  vp9_copy(cm->lf.last_mode_deltas, cc->last_mode_lf_deltas);

333
  *cm->fc = cc->fc;
334 335
}

336
static void configure_static_seg_features(VP9_COMP *cpi) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
337
  VP9_COMMON *const cm = &cpi->common;
338
  const RATE_CONTROL *const rc = &cpi->rc;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
339
  struct segmentation *const seg = &cm->seg;
340

341
  int high_q = (int)(rc->avg_q > 48.0);
John Koleszar's avatar
John Koleszar committed
342
  int qi_delta;
343

John Koleszar's avatar
John Koleszar committed
344 345 346
  // Disable and clear down for KF
  if (cm->frame_type == KEY_FRAME) {
    // Clear down the global segmentation map
347
    vpx_memset(cpi->segmentation_map, 0, cm->mi_rows * cm->mi_cols);
348 349
    seg->update_map = 0;
    seg->update_data = 0;
John Koleszar's avatar
John Koleszar committed
350
    cpi->static_mb_pct = 0;
351

John Koleszar's avatar
John Koleszar committed
352
    // Disable segmentation
353
    vp9_disable_segmentation(seg);
Paul Wilkins's avatar
Paul Wilkins committed
354

John Koleszar's avatar
John Koleszar committed
355
    // Clear down the segment features.
356
    vp9_clearall_segfeatures(seg);
357 358
  } else if (cpi->refresh_alt_ref_frame) {
    // If this is an alt ref frame
John Koleszar's avatar
John Koleszar committed
359
    // Clear down the global segmentation map
360
    vpx_memset(cpi->segmentation_map, 0, cm->mi_rows * cm->mi_cols);
361 362
    seg->update_map = 0;
    seg->update_data = 0;
John Koleszar's avatar
John Koleszar committed
363
    cpi->static_mb_pct = 0;
Paul Wilkins's avatar
Paul Wilkins committed
364

John Koleszar's avatar
John Koleszar committed
365
    // Disable segmentation and individual segment features by default
366
    vp9_disable_segmentation(seg);
367
    vp9_clearall_segfeatures(seg);
Paul Wilkins's avatar
Paul Wilkins committed
368

John Koleszar's avatar
John Koleszar committed
369 370
    // Scan frames from current to arf frame.
    // This function re-enables segmentation if appropriate.
371
    vp9_update_mbgraph_stats(cpi);
Paul Wilkins's avatar
Paul Wilkins committed
372

John Koleszar's avatar
John Koleszar committed
373 374
    // If segmentation was enabled set those features needed for the
    // arf itself.
375 376 377
    if (seg->enabled) {
      seg->update_map = 1;
      seg->update_data = 1;
Paul Wilkins's avatar
Paul Wilkins committed
378

379 380
      qi_delta = vp9_compute_qdelta(rc, rc->avg_q, rc->avg_q * 0.875,
                                    cm->bit_depth);
381
      vp9_set_segdata(seg, 1, SEG_LVL_ALT_Q, qi_delta - 2);
382
      vp9_set_segdata(seg, 1, SEG_LVL_ALT_LF, -2);
Paul Wilkins's avatar
Paul Wilkins committed
383

384 385
      vp9_enable_segfeature(seg, 1, SEG_LVL_ALT_Q);
      vp9_enable_segfeature(seg, 1, SEG_LVL_ALT_LF);
386

John Koleszar's avatar
John Koleszar committed
387
      // Where relevant assume segment data is delta data
388
      seg->abs_delta = SEGMENT_DELTADATA;
389
    }
390
  } else if (seg->enabled) {
391 392
    // All other frames if segmentation has been enabled

John Koleszar's avatar
John Koleszar committed
393
    // First normal frame in a valid gf or alt ref group
394
    if (rc->frames_since_golden == 0) {
Paul Wilkins's avatar
Paul Wilkins committed
395
      // Set up segment features for normal frames in an arf group
396
      if (rc->source_alt_ref_active) {
397 398 399
        seg->update_map = 0;
        seg->update_data = 1;
        seg->abs_delta = SEGMENT_DELTADATA;
Paul Wilkins's avatar
Paul Wilkins committed
400

401 402
        qi_delta = vp9_compute_qdelta(rc, rc->avg_q, rc->avg_q * 1.125,
                                      cm->bit_depth);
403
        vp9_set_segdata(seg, 1, SEG_LVL_ALT_Q, qi_delta + 2);
404
        vp9_enable_segfeature(seg, 1, SEG_LVL_ALT_Q);
Paul Wilkins's avatar
Paul Wilkins committed
405

406 407
        vp9_set_segdata(seg, 1, SEG_LVL_ALT_LF, -2);
        vp9_enable_segfeature(seg, 1, SEG_LVL_ALT_LF);
Paul Wilkins's avatar
Paul Wilkins committed
408

John Koleszar's avatar
John Koleszar committed
409 410
        // Segment coding disabled for compred testing
        if (high_q || (cpi->static_mb_pct == 100)) {
411 412 413
          vp9_set_segdata(seg, 1, SEG_LVL_REF_FRAME, ALTREF_FRAME);
          vp9_enable_segfeature(seg, 1, SEG_LVL_REF_FRAME);
          vp9_enable_segfeature(seg, 1, SEG_LVL_SKIP);
John Koleszar's avatar
John Koleszar committed
414
        }
415 416 417 418
      } else {
        // Disable segmentation and clear down features if alt ref
        // is not active for this group

419
        vp9_disable_segmentation(seg);
Paul Wilkins's avatar
Paul Wilkins committed
420

421
        vpx_memset(cpi->segmentation_map, 0, cm->mi_rows * cm->mi_cols);
Paul Wilkins's avatar
Paul Wilkins committed
422

423 424
        seg->update_map = 0;
        seg->update_data = 0;
Paul Wilkins's avatar
Paul Wilkins committed
425

426
        vp9_clearall_segfeatures(seg);
John Koleszar's avatar
John Koleszar committed
427
      }
428
    } else if (rc->is_src_frame_alt_ref) {
429 430 431
      // Special case where we are coding over the top of a previous
      // alt ref frame.
      // Segment coding disabled for compred testing
John Koleszar's avatar
John Koleszar committed
432

Paul Wilkins's avatar
Paul Wilkins committed
433
      // Enable ref frame features for segment 0 as well
434 435
      vp9_enable_segfeature(seg, 0, SEG_LVL_REF_FRAME);
      vp9_enable_segfeature(seg, 1, SEG_LVL_REF_FRAME);
John Koleszar's avatar
John Koleszar committed
436

Paul Wilkins's avatar
Paul Wilkins committed
437
      // All mbs should use ALTREF_FRAME
438 439 440 441
      vp9_clear_segdata(seg, 0, SEG_LVL_REF_FRAME);
      vp9_set_segdata(seg, 0, SEG_LVL_REF_FRAME, ALTREF_FRAME);
      vp9_clear_segdata(seg, 1, SEG_LVL_REF_FRAME);
      vp9_set_segdata(seg, 1, SEG_LVL_REF_FRAME, ALTREF_FRAME);
John Koleszar's avatar
John Koleszar committed
442

Paul Wilkins's avatar
Paul Wilkins committed
443
      // Skip all MBs if high Q (0,0 mv and skip coeffs)
John Koleszar's avatar
John Koleszar committed
444
      if (high_q) {
445 446
        vp9_enable_segfeature(seg, 0, SEG_LVL_SKIP);
        vp9_enable_segfeature(seg, 1, SEG_LVL_SKIP);
John Koleszar's avatar
John Koleszar committed
447
      }
Adrian Grange's avatar
Adrian Grange committed
448
      // Enable data update
449
      seg->update_data = 1;
450 451 452
    } else {
      // All other frames.

John Koleszar's avatar
John Koleszar committed
453
      // No updates.. leave things as they are.
454 455
      seg->update_map = 0;
      seg->update_data = 0;
John Koleszar's avatar
John Koleszar committed
456 457
    }
  }
458 459
}

460
static void update_reference_segmentation_map(VP9_COMP *cpi) {
Ronald S. Bultje's avatar
Ronald S. Bultje committed
461
  VP9_COMMON *const cm = &cpi->common;
hkuang's avatar
hkuang committed
462
  MODE_INFO *mi_8x8_ptr = cm->mi;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
463
  uint8_t *cache_ptr = cm->last_frame_seg_map;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
464 465
  int row, col;

466
  for (row = 0; row < cm->mi_rows; row++) {
hkuang's avatar
hkuang committed
467
    MODE_INFO *mi_8x8 = mi_8x8_ptr;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
468
    uint8_t *cache = cache_ptr;
469
    for (col = 0; col < cm->mi_cols; col++, mi_8x8++, cache++)
hkuang's avatar
hkuang committed
470
      cache[0] = mi_8x8[0].src_mi->mbmi.segment_id;
471
    mi_8x8_ptr += cm->mi_stride;
472
    cache_ptr += cm->mi_cols;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
473 474 475
  }
}

476
static void alloc_raw_frame_buffers(VP9_COMP *cpi) {
477
  VP9_COMMON *cm = &cpi->common;
478
  const VP9EncoderConfig *oxcf = &cpi->oxcf;
479

Dmitry Kovalev's avatar
Dmitry Kovalev committed
480
  cpi->lookahead = vp9_lookahead_init(oxcf->width, oxcf->height,
481
                                      cm->subsampling_x, cm->subsampling_y,
482 483 484
#if CONFIG_VP9_HIGHBITDEPTH
                                      cm->use_highbitdepth,
#endif
Dmitry Kovalev's avatar
Dmitry Kovalev committed
485
                                      oxcf->lag_in_frames);
John Koleszar's avatar
John Koleszar committed
486
  if (!cpi->lookahead)
487
    vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
John Koleszar's avatar
John Koleszar committed
488
                       "Failed to allocate lag buffers");
John Koleszar's avatar
John Koleszar committed
489

490
  if (vp9_realloc_frame_buffer(&cpi->alt_ref_buffer,
Dmitry Kovalev's avatar
Dmitry Kovalev committed
491
                               oxcf->width, oxcf->height,
492
                               cm->subsampling_x, cm->subsampling_y,
493 494 495
#if CONFIG_VP9_HIGHBITDEPTH
                               cm->use_highbitdepth,
#endif
496
                               VP9_ENC_BORDER_IN_PIXELS, NULL, NULL, NULL))
497
    vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
John Koleszar's avatar
John Koleszar committed
498
                       "Failed to allocate altref buffer");
John Koleszar's avatar
John Koleszar committed
499
}
500

501 502
static void alloc_ref_frame_buffers(VP9_COMP *cpi) {
  VP9_COMMON *const cm = &cpi->common;
Adrian Grange's avatar
Adrian Grange committed
503
  if (vp9_alloc_ref_frame_buffers(cm, cm->width, cm->height))
504
    vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
John Koleszar's avatar
John Koleszar committed
505
                       "Failed to allocate frame buffers");
John Koleszar's avatar
John Koleszar committed
506 507
}

508
static void alloc_util_frame_buffers(VP9_COMP *cpi) {
509
  VP9_COMMON *const cm = &cpi->common;
510 511 512
  if (vp9_realloc_frame_buffer(&cpi->last_frame_uf,
                               cm->width, cm->height,
                               cm->subsampling_x, cm->subsampling_y,
513 514 515
#if CONFIG_VP9_HIGHBITDEPTH
                               cm->use_highbitdepth,
#endif
516
                               VP9_ENC_BORDER_IN_PIXELS, NULL, NULL, NULL))
517
    vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
518
                       "Failed to allocate last frame buffer");
519

520 521 522
  if (vp9_realloc_frame_buffer(&cpi->scaled_source,
                               cm->width, cm->height,
                               cm->subsampling_x, cm->subsampling_y,
523 524 525
#if CONFIG_VP9_HIGHBITDEPTH
                               cm->use_highbitdepth,
#endif
526
                               VP9_ENC_BORDER_IN_PIXELS, NULL, NULL, NULL))
527
    vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
528
                       "Failed to allocate scaled source buffer");
529

530 531 532
  if (vp9_realloc_frame_buffer(&cpi->scaled_last_source,
                               cm->width, cm->height,
                               cm->subsampling_x, cm->subsampling_y,
533 534 535
#if CONFIG_VP9_HIGHBITDEPTH
                               cm->use_highbitdepth,
#endif
536 537
                               VP9_ENC_BORDER_IN_PIXELS, NULL, NULL, NULL))
    vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
538 539
                       "Failed to allocate scaled last source buffer");
}
540

541 542
void vp9_alloc_compressor_data(VP9_COMP *cpi) {
  VP9_COMMON *cm = &cpi->common;
543

544 545
  vp9_alloc_context_buffers(cm, cm->width, cm->height);

546
  vpx_free(cpi->tile_tok[0][0]);
547 548 549

  {
    unsigned int tokens = get_token_alloc(cm->mb_rows, cm->mb_cols);
550 551
    CHECK_MEM_ERROR(cm, cpi->tile_tok[0][0],
        vpx_calloc(tokens, sizeof(*cpi->tile_tok[0][0])));
552
  }
553

554
  vp9_setup_pc_tree(&cpi->common, &cpi->td);
555 556 557 558
}

static void update_frame_size(VP9_COMP *cpi) {
  VP9_COMMON *const cm = &cpi->common;
559
  MACROBLOCKD *const xd = &cpi->td.mb.e_mbd;
Adrian Grange's avatar
Adrian Grange committed
560 561 562

  vp9_set_mb_mi(cm, cm->width, cm->height);
  vp9_init_context_buffers(cm);
563
  init_macroblockd(cm, xd);
564

565
  if (is_two_pass_svc(cpi)) {
566 567 568
    if (vp9_realloc_frame_buffer(&cpi->alt_ref_buffer,
                                 cm->width, cm->height,
                                 cm->subsampling_x, cm->subsampling_y,
569 570 571
#if CONFIG_VP9_HIGHBITDEPTH
                                 cm->use_highbitdepth,
#endif
572 573 574 575
                                 VP9_ENC_BORDER_IN_PIXELS, NULL, NULL, NULL))
      vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
                         "Failed to reallocate alt_ref_buffer");
  }
576 577
}

James Zern's avatar
James Zern committed
578
void vp9_new_framerate(VP9_COMP *cpi, double framerate) {
579
  cpi->framerate = framerate < 0.1 ? 30 : framerate;
580
  vp9_rc_update_framerate(cpi);
John Koleszar's avatar
John Koleszar committed
581 582
}

Ronald S. Bultje's avatar
Ronald S. Bultje committed
583 584
static void set_tile_limits(VP9_COMP *cpi) {
  VP9_COMMON *const cm = &cpi->common;
585

Dmitry Kovalev's avatar
Dmitry Kovalev committed
586 587
  int min_log2_tile_cols, max_log2_tile_cols;
  vp9_get_tile_n_bits(cm->mi_cols, &min_log2_tile_cols, &max_log2_tile_cols);
588

Dmitry Kovalev's avatar
Dmitry Kovalev committed
589 590 591
  cm->log2_tile_cols = clamp(cpi->oxcf.tile_columns,
                             min_log2_tile_cols, max_log2_tile_cols);
  cm->log2_tile_rows = cpi->oxcf.tile_rows;
592
}
593

594 595 596 597 598 599
static void init_buffer_indices(VP9_COMP *cpi) {
  cpi->lst_fb_idx = 0;
  cpi->gld_fb_idx = 1;
  cpi->alt_fb_idx = 2;
}

600
static void init_config(struct VP9_COMP *cpi, VP9EncoderConfig *oxcf) {
601
  VP9_COMMON *const cm = &cpi->common;
John Koleszar's avatar
John Koleszar committed
602

John Koleszar's avatar
John Koleszar committed
603
  cpi->oxcf = *oxcf;
604
  cpi->framerate = oxcf->init_framerate;
John Koleszar's avatar
John Koleszar committed
605

606 607
  cm->profile = oxcf->profile;
  cm->bit_depth = oxcf->bit_depth;
608 609 610
#if CONFIG_VP9_HIGHBITDEPTH
  cm->use_highbitdepth = oxcf->use_highbitdepth;
#endif
611
  cm->color_space = UNKNOWN;
John Koleszar's avatar
John Koleszar committed
612

613 614
  cm->width = oxcf->width;
  cm->height = oxcf->height;
615
  vp9_alloc_compressor_data(cpi);
616

617 618 619
  // Single thread case: use counts in common.
  cpi->td.counts = &cm->counts;

620 621 622 623 624
  // Spatial scalability.
  cpi->svc.number_spatial_layers = oxcf->ss_number_layers;
  // Temporal scalability.
  cpi->svc.number_temporal_layers = oxcf->ts_number_layers;

625
  if ((cpi->svc.number_temporal_layers > 1 && cpi->oxcf.rc_mode == VPX_CBR) ||
626 627
      ((cpi->svc.number_temporal_layers > 1 ||
        cpi->svc.number_spatial_layers > 1) &&
628
       cpi->oxcf.pass != 1)) {
629
    vp9_init_layer_context(cpi);
630 631
  }

John Koleszar's avatar
John Koleszar committed
632
  // change includes all joint functionality
Dmitry Kovalev's avatar
Dmitry Kovalev committed
633
  vp9_change_config(cpi, oxcf);
John Koleszar's avatar
John Koleszar committed
634

John Koleszar's avatar
John Koleszar committed
635
  cpi->static_mb_pct = 0;
636
  cpi->ref_frame_flags = 0;
637

638
  init_buffer_indices(cpi);
John Koleszar's avatar
John Koleszar committed
639 640
}

Dmitry Kovalev's avatar
Dmitry Kovalev committed
641 642 643 644 645 646 647 648 649 650 651 652 653 654
static void set_rc_buffer_sizes(RATE_CONTROL *rc,
                                const VP9EncoderConfig *oxcf) {
  const int64_t bandwidth = oxcf->target_bandwidth;
  const int64_t starting = oxcf->starting_buffer_level_ms;
  const int64_t optimal = oxcf->optimal_buffer_level_ms;
  const int64_t maximum = oxcf->maximum_buffer_size_ms;

  rc->starting_buffer_level = starting * bandwidth / 1000;
  rc->optimal_buffer_level = (optimal == 0) ? bandwidth / 8
                                            : optimal * bandwidth / 1000;
  rc->maximum_buffer_size = (maximum == 0) ? bandwidth / 8
                                           : maximum * bandwidth / 1000;
}

655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796
#if CONFIG_VP9_HIGHBITDEPTH
#define HIGHBD_BFP(BT, SDF, SDAF, VF, SVF, SVAF, SDX3F, SDX8F, SDX4DF) \
    cpi->fn_ptr[BT].sdf = SDF; \
    cpi->fn_ptr[BT].sdaf = SDAF; \
    cpi->fn_ptr[BT].vf = VF; \
    cpi->fn_ptr[BT].svf = SVF; \
    cpi->fn_ptr[BT].svaf = SVAF; \
    cpi->fn_ptr[BT].sdx3f = SDX3F; \
    cpi->fn_ptr[BT].sdx8f = SDX8F; \
    cpi->fn_ptr[BT].sdx4df = SDX4DF;

#define MAKE_BFP_SAD_WRAPPER(fnname) \
static unsigned int fnname##_bits8(const uint8_t *src_ptr, \
                                   int source_stride, \
                                   const uint8_t *ref_ptr, \
                                   int ref_stride) {  \
  return fnname(src_ptr, source_stride, ref_ptr, ref_stride); \
} \
static unsigned int fnname##_bits10(const uint8_t *src_ptr, \
                                    int source_stride, \
                                    const uint8_t *ref_ptr, \
                                    int ref_stride) {  \
  return fnname(src_ptr, source_stride, ref_ptr, ref_stride) >> 2; \
} \
static unsigned int fnname##_bits12(const uint8_t *src_ptr, \
                                    int source_stride, \
                                    const uint8_t *ref_ptr, \
                                    int ref_stride) {  \
  return fnname(src_ptr, source_stride, ref_ptr, ref_stride) >> 4; \
}

#define MAKE_BFP_SADAVG_WRAPPER(fnname) static unsigned int \
fnname##_bits8(const uint8_t *src_ptr, \
               int source_stride, \
               const uint8_t *ref_ptr, \
               int ref_stride, \
               const uint8_t *second_pred) {  \
  return fnname(src_ptr, source_stride, ref_ptr, ref_stride, second_pred); \
} \
static unsigned int fnname##_bits10(const uint8_t *src_ptr, \
                                    int source_stride, \
                                    const uint8_t *ref_ptr, \
                                    int ref_stride, \
                                    const uint8_t *second_pred) {  \
  return fnname(src_ptr, source_stride, ref_ptr, ref_stride, \
                second_pred) >> 2; \
} \
static unsigned int fnname##_bits12(const uint8_t *src_ptr, \
                                    int source_stride, \
                                    const uint8_t *ref_ptr, \
                                    int ref_stride, \
                                    const uint8_t *second_pred) {  \
  return fnname(src_ptr, source_stride, ref_ptr, ref_stride, \
                second_pred) >> 4; \
}

#define MAKE_BFP_SAD3_WRAPPER(fnname) \
static void fnname##_bits8(const uint8_t *src_ptr, \
                           int source_stride, \
                           const uint8_t *ref_ptr, \
                           int  ref_stride, \
                           unsigned int *sad_array) {  \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
} \
static void fnname##_bits10(const uint8_t *src_ptr, \
                            int source_stride, \
                            const uint8_t *ref_ptr, \
                            int  ref_stride, \
                            unsigned int *sad_array) {  \
  int i; \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
  for (i = 0; i < 3; i++) \
    sad_array[i] >>= 2; \
} \
static void fnname##_bits12(const uint8_t *src_ptr, \
                            int source_stride, \
                            const uint8_t *ref_ptr, \
                            int  ref_stride, \
                            unsigned int *sad_array) {  \
  int i; \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
  for (i = 0; i < 3; i++) \
    sad_array[i] >>= 4; \
}

#define MAKE_BFP_SAD8_WRAPPER(fnname) \
static void fnname##_bits8(const uint8_t *src_ptr, \
                           int source_stride, \
                           const uint8_t *ref_ptr, \
                           int  ref_stride, \
                           unsigned int *sad_array) {  \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
} \
static void fnname##_bits10(const uint8_t *src_ptr, \
                            int source_stride, \
                            const uint8_t *ref_ptr, \
                            int  ref_stride, \
                            unsigned int *sad_array) {  \
  int i; \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
  for (i = 0; i < 8; i++) \
    sad_array[i] >>= 2; \
} \
static void fnname##_bits12(const uint8_t *src_ptr, \
                            int source_stride, \
                            const uint8_t *ref_ptr, \
                            int  ref_stride, \
                            unsigned int *sad_array) {  \
  int i; \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
  for (i = 0; i < 8; i++) \
    sad_array[i] >>= 4; \
}
#define MAKE_BFP_SAD4D_WRAPPER(fnname) \
static void fnname##_bits8(const uint8_t *src_ptr, \
                           int source_stride, \
                           const uint8_t* const ref_ptr[], \
                           int  ref_stride, \
                           unsigned int *sad_array) {  \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
} \
static void fnname##_bits10(const uint8_t *src_ptr, \
                            int source_stride, \
                            const uint8_t* const ref_ptr[], \
                            int  ref_stride, \
                            unsigned int *sad_array) {  \
  int i; \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
  for (i = 0; i < 4; i++) \
  sad_array[i] >>= 2; \
} \
static void fnname##_bits12(const uint8_t *src_ptr, \
                            int source_stride, \
                            const uint8_t* const ref_ptr[], \
                            int  ref_stride, \
                            unsigned int *sad_array) {  \
  int i; \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
  for (i = 0; i < 4; i++) \
  sad_array[i] >>= 4; \
}

797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad32x16)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad32x16_avg)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad32x16x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad16x32)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad16x32_avg)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad16x32x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad64x32)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad64x32_avg)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad64x32x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad32x64)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad32x64_avg)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad32x64x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad32x32)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad32x32_avg)
MAKE_BFP_SAD3_WRAPPER(vp9_highbd_sad32x32x3)
MAKE_BFP_SAD8_WRAPPER(vp9_highbd_sad32x32x8)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad32x32x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad64x64)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad64x64_avg)
MAKE_BFP_SAD3_WRAPPER(vp9_highbd_sad64x64x3)
MAKE_BFP_SAD8_WRAPPER(vp9_highbd_sad64x64x8)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad64x64x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad16x16)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad16x16_avg)
MAKE_BFP_SAD3_WRAPPER(vp9_highbd_sad16x16x3)
MAKE_BFP_SAD8_WRAPPER(vp9_highbd_sad16x16x8)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad16x16x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad16x8)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad16x8_avg)
MAKE_BFP_SAD3_WRAPPER(vp9_highbd_sad16x8x3)
MAKE_BFP_SAD8_WRAPPER(vp9_highbd_sad16x8x8)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad16x8x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad8x16)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad8x16_avg)
MAKE_BFP_SAD3_WRAPPER(vp9_highbd_sad8x16x3)
MAKE_BFP_SAD8_WRAPPER(vp9_highbd_sad8x16x8)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad8x16x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad8x8)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad8x8_avg)
MAKE_BFP_SAD3_WRAPPER(vp9_highbd_sad8x8x3)
MAKE_BFP_SAD8_WRAPPER(vp9_highbd_sad8x8x8)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad8x8x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad8x4)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad8x4_avg)
MAKE_BFP_SAD8_WRAPPER(vp9_highbd_sad8x4x8)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad8x4x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad4x8)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad4x8_avg)
MAKE_BFP_SAD8_WRAPPER(vp9_highbd_sad4x8x8)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad4x8x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad4x4)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad4x4_avg)
MAKE_BFP_SAD3_WRAPPER(vp9_highbd_sad4x4x3)
MAKE_BFP_SAD8_WRAPPER(vp9_highbd_sad4x4x8)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad4x4x4d)

static void  highbd_set_var_fns(VP9_COMP *const cpi) {
854 855 856 857 858
  VP9_COMMON *const cm = &cpi->common;
  if (cm->use_highbitdepth) {
    switch (cm->bit_depth) {
      case VPX_BITS_8:
        HIGHBD_BFP(BLOCK_32X16,
859 860 861 862 863
                   vp9_highbd_sad32x16_bits8,
                   vp9_highbd_sad32x16_avg_bits8,
                   vp9_highbd_variance32x16,
                   vp9_highbd_sub_pixel_variance32x16,
                   vp9_highbd_sub_pixel_avg_variance32x16,
864 865
                   NULL,
                   NULL,
866
                   vp9_highbd_sad32x16x4d_bits8)
867 868

        HIGHBD_BFP(BLOCK_16X32,
869 870 871 872 873
                   vp9_highbd_sad16x32_bits8,
                   vp9_highbd_sad16x32_avg_bits8,
                   vp9_highbd_variance16x32,
                   vp9_highbd_sub_pixel_variance16x32,
                   vp9_highbd_sub_pixel_avg_variance16x32,
874 875
                   NULL,
                   NULL,
876
                   vp9_highbd_sad16x32x4d_bits8)
877 878

        HIGHBD_BFP(BLOCK_64X32,
879 880 881 882 883
                   vp9_highbd_sad64x32_bits8,
                   vp9_highbd_sad64x32_avg_bits8,
                   vp9_highbd_variance64x32,
                   vp9_highbd_sub_pixel_variance64x32,
                   vp9_highbd_sub_pixel_avg_variance64x32,
884 885
                   NULL,
                   NULL,
886
                   vp9_highbd_sad64x32x4d_bits8)
887 888

        HIGHBD_BFP(BLOCK_32X64,
889 890 891 892 893
                   vp9_highbd_sad32x64_bits8,
                   vp9_highbd_sad32x64_avg_bits8,
                   vp9_highbd_variance32x64,
                   vp9_highbd_sub_pixel_variance32x64,
                   vp9_highbd_sub_pixel_avg_variance32x64,
894 895
                   NULL,
                   NULL,
896
                   vp9_highbd_sad32x64x4d_bits8)
897 898

        HIGHBD_BFP(BLOCK_32X32,
899 900 901 902 903 904 905 906
                   vp9_highbd_sad32x32_bits8,
                   vp9_highbd_sad32x32_avg_bits8,
                   vp9_highbd_variance32x32,
                   vp9_highbd_sub_pixel_variance32x32,
                   vp9_highbd_sub_pixel_avg_variance32x32,
                   vp9_highbd_sad32x32x3_bits8,
                   vp9_highbd_sad32x32x8_bits8,
                   vp9_highbd_sad32x32x4d_bits8)
907 908

        HIGHBD_BFP(BLOCK_64X64,
909 910 911 912 913 914 915 916
                   vp9_highbd_sad64x64_bits8,
                   vp9_highbd_sad64x64_avg_bits8,
                   vp9_highbd_variance64x64,
                   vp9_highbd_sub_pixel_variance64x64,
                   vp9_highbd_sub_pixel_avg_variance64x64,
                   vp9_highbd_sad64x64x3_bits8,
                   vp9_highbd_sad64x64x8_bits8,
                   vp9_highbd_sad64x64x4d_bits8)
917 918

        HIGHBD_BFP(BLOCK_16X16,
919 920 921 922 923 924 925 926
                   vp9_highbd_sad16x16_bits8,
                   vp9_highbd_sad16x16_avg_bits8,
                   vp9_highbd_variance16x16,
                   vp9_highbd_sub_pixel_variance16x16,
                   vp9_highbd_sub_pixel_avg_variance16x16,
                   vp9_highbd_sad16x16x3_bits8,
                   vp9_highbd_sad16x16x8_bits8,
                   vp9_highbd_sad16x16x4d_bits8)
927 928

        HIGHBD_BFP(BLOCK_16X8,
929 930 931 932 933 934 935 936
                   vp9_highbd_sad16x8_bits8,
                   vp9_highbd_sad16x8_avg_bits8,
                   vp9_highbd_variance16x8,
                   vp9_highbd_sub_pixel_variance16x8,
                   vp9_highbd_sub_pixel_avg_variance16x8,
                   vp9_highbd_sad16x8x3_bits8,
                   vp9_highbd_sad16x8x8_bits8,
                   vp9_highbd_sad16x8x4d_bits8)
937 938

        HIGHBD_BFP(BLOCK_8X16,
939 940 941 942 943 944 945 946
                   vp9_highbd_sad8x16_bits8,
                   vp9_highbd_sad8x16_avg_bits8,
                   vp9_highbd_variance8x16,
                   vp9_highbd_sub_pixel_variance8x16,
                   vp9_highbd_sub_pixel_avg_variance8x16,
                   vp9_highbd_sad8x16x3_bits8,
                   vp9_highbd_sad8x16x8_bits8,
                   vp9_highbd_sad8x16x4d_bits8)
947 948

        HIGHBD_BFP(BLOCK_8X8,
949 950 951 952 953 954 955 956
                   vp9_highbd_sad8x8_bits8,
                   vp9_highbd_sad8x8_avg_bits8,
                   vp9_highbd_variance8x8,
                   vp9_highbd_sub_pixel_variance8x8,
                   vp9_highbd_sub_pixel_avg_variance8x8,
                   vp9_highbd_sad8x8x3_bits8,
                   vp9_highbd_sad8x8x8_bits8,
                   vp9_highbd_sad8x8x4d_bits8)
957 958

        HIGHBD_BFP(BLOCK_8X4,
959 960 961 962 963
                   vp9_highbd_sad8x4_bits8,
                   vp9_highbd_sad8x4_avg_bits8,
                   vp9_highbd_variance8x4,
                   vp9_highbd_sub_pixel_variance8x4