vp9_encoder.c 135 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 * Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5
6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9
10
 */

Jim Bankoski's avatar
Jim Bankoski committed
11
12
13
14
15
16
#include <math.h>
#include <stdio.h>
#include <limits.h>

#include "./vpx_config.h"
#include "./vpx_scale_rtcd.h"
17
18
#include "vpx/internal/vpx_psnr.h"
#include "vpx_ports/vpx_timer.h"
John Koleszar's avatar
John Koleszar committed
19

Jim Bankoski's avatar
Jim Bankoski committed
20
#include "vp9/common/vp9_alloccommon.h"
21
#include "vp9/common/vp9_filter.h"
22
#include "vp9/common/vp9_idct.h"
Jim Bankoski's avatar
Jim Bankoski committed
23
24
25
#if CONFIG_VP9_POSTPROC
#include "vp9/common/vp9_postproc.h"
#endif
26
#include "vp9/common/vp9_reconinter.h"
27
#include "vp9/common/vp9_reconintra.h"
28
#include "vp9/common/vp9_systemdependent.h"
Jim Bankoski's avatar
Jim Bankoski committed
29
#include "vp9/common/vp9_tile_common.h"
30

31
#include "vp9/encoder/vp9_aq_complexity.h"
Marco Paniconi's avatar
Marco Paniconi committed
32
33
#include "vp9/encoder/vp9_aq_cyclicrefresh.h"
#include "vp9/encoder/vp9_aq_variance.h"
34
#include "vp9/encoder/vp9_bitstream.h"
35
#include "vp9/encoder/vp9_context_tree.h"
36
#include "vp9/encoder/vp9_encodeframe.h"
37
#include "vp9/encoder/vp9_encodemv.h"
38
#include "vp9/encoder/vp9_firstpass.h"
Jim Bankoski's avatar
Jim Bankoski committed
39
#include "vp9/encoder/vp9_mbgraph.h"
Dmitry Kovalev's avatar
Dmitry Kovalev committed
40
#include "vp9/encoder/vp9_encoder.h"
Jim Bankoski's avatar
Jim Bankoski committed
41
#include "vp9/encoder/vp9_picklpf.h"
42
#include "vp9/encoder/vp9_ratectrl.h"
43
#include "vp9/encoder/vp9_rd.h"
Jim Bankoski's avatar
Jim Bankoski committed
44
#include "vp9/encoder/vp9_segmentation.h"
Yaowu Xu's avatar
Yaowu Xu committed
45
#include "vp9/encoder/vp9_speed_features.h"
Dmitry Kovalev's avatar
Dmitry Kovalev committed
46
47
48
#if CONFIG_INTERNAL_STATS
#include "vp9/encoder/vp9_ssim.h"
#endif
49
#include "vp9/encoder/vp9_temporal_filter.h"
50
#include "vp9/encoder/vp9_resize.h"
51
#include "vp9/encoder/vp9_svc_layercontext.h"
Paul Wilkins's avatar
Paul Wilkins committed
52

53
void vp9_coef_tree_initialize();
54

55
56
#define SHARP_FILTER_QTHRESH 0          /* Q threshold for 8-tap sharp filter */

Jim Bankoski's avatar
Jim Bankoski committed
57
58
59
60
61
62
#define ALTREF_HIGH_PRECISION_MV 1      // Whether to use high precision mv
                                         //  for altref computation.
#define HIGH_PRECISION_MV_QTHRESH 200   // Q threshold for high precision
                                         // mv. Choose a very high value for
                                         // now so that HIGH_PRECISION is always
                                         // chosen.
63

John Koleszar's avatar
John Koleszar committed
64
// #define OUTPUT_YUV_REC
John Koleszar's avatar
John Koleszar committed
65

66
#ifdef OUTPUT_YUV_DENOISED
67
FILE *yuv_denoised_file = NULL;
68
#endif
69
70
71
#ifdef OUTPUT_YUV_REC
FILE *yuv_rec_file;
#endif
John Koleszar's avatar
John Koleszar committed
72
73
74

#if 0
FILE *framepsnr;
Yaowu Xu's avatar
Yaowu Xu committed
75
FILE *kf_list;
John Koleszar's avatar
John Koleszar committed
76
77
78
FILE *keyfile;
#endif

79
static INLINE void Scale2Ratio(VPX_SCALING mode, int *hr, int *hs) {
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
  switch (mode) {
    case NORMAL:
      *hr = 1;
      *hs = 1;
      break;
    case FOURFIVE:
      *hr = 4;
      *hs = 5;
      break;
    case THREEFIVE:
      *hr = 3;
      *hs = 5;
    break;
    case ONETWO:
      *hr = 1;
      *hs = 2;
    break;
    default:
      *hr = 1;
      *hs = 1;
       assert(0);
      break;
  }
}

105
void vp9_set_high_precision_mv(VP9_COMP *cpi, int allow_high_precision_mv) {
106
  MACROBLOCK *const mb = &cpi->mb;
Deb Mukherjee's avatar
Deb Mukherjee committed
107
  cpi->common.allow_high_precision_mv = allow_high_precision_mv;
108
  if (cpi->common.allow_high_precision_mv) {
109
110
111
112
113
114
115
    mb->mvcost = mb->nmvcost_hp;
    mb->mvsadcost = mb->nmvsadcost_hp;
  } else {
    mb->mvcost = mb->nmvcost;
    mb->mvsadcost = mb->nmvsadcost;
  }
}
Paul Wilkins's avatar
Paul Wilkins committed
116

117
118
119
120
121
122
123
static void setup_frame(VP9_COMP *cpi) {
  VP9_COMMON *const cm = &cpi->common;
  // Set up entropy context depending on frame type. The decoder mandates
  // the use of the default context, index 0, for keyframes and inter
  // frames where the error_resilient_mode or intra_only flag is set. For
  // other inter-frames the encoder currently uses only two contexts;
  // context 1 for ALTREF frames and context 0 for the others.
124
125
126
127
128
129
130
  if (frame_is_intra_only(cm) || cm->error_resilient_mode) {
    vp9_setup_past_independence(cm);
  } else {
    if (!cpi->use_svc)
      cm->frame_context_idx = cpi->refresh_alt_ref_frame;
  }

131
  if (cm->frame_type == KEY_FRAME) {
132
    if (!is_two_pass_svc(cpi))
133
      cpi->refresh_golden_frame = 1;
134
    cpi->refresh_alt_ref_frame = 1;
135
    vp9_zero(cpi->interp_filter_selected);
136
  } else {
137
    *cm->fc = cm->frame_contexts[cm->frame_context_idx];
138
    vp9_zero(cpi->interp_filter_selected[0]);
139
140
141
  }
}

142
void vp9_initialize_enc() {
John Koleszar's avatar
John Koleszar committed
143
144
145
  static int init_done = 0;

  if (!init_done) {
146
147
    vp9_rtcd();
    vp9_init_intra_predictors();
148
    vp9_coef_tree_initialize();
149
150
    vp9_tokenize_initialize();
    vp9_init_me_luts();
151
    vp9_rc_init_minq_luts();
152
    vp9_entropy_mv_init();
153
    vp9_entropy_mode_init();
154
    vp9_temporal_filter_init();
John Koleszar's avatar
John Koleszar committed
155
156
    init_done = 1;
  }
John Koleszar's avatar
John Koleszar committed
157
158
}

159
static void dealloc_compressor_data(VP9_COMP *cpi) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
160
  VP9_COMMON *const cm = &cpi->common;
161
  int i;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
162

163
164
165
166
  vpx_free(cm->fc);
  cm->fc = NULL;
  vpx_free(cm->frame_contexts);
  cm->frame_contexts = NULL;
167
168
  vpx_free(cpi->tile_data);
  cpi->tile_data = NULL;
169

170
  // Delete sementation map
John Koleszar's avatar
John Koleszar committed
171
  vpx_free(cpi->segmentation_map);
Dmitry Kovalev's avatar
Dmitry Kovalev committed
172
173
174
  cpi->segmentation_map = NULL;
  vpx_free(cm->last_frame_seg_map);
  cm->last_frame_seg_map = NULL;
John Koleszar's avatar
John Koleszar committed
175
  vpx_free(cpi->coding_context.last_frame_seg_map_copy);
Dmitry Kovalev's avatar
Dmitry Kovalev committed
176
  cpi->coding_context.last_frame_seg_map_copy = NULL;
John Koleszar's avatar
John Koleszar committed
177

178
  vpx_free(cpi->complexity_map);
179
180
  cpi->complexity_map = NULL;

Jingning Han's avatar
Jingning Han committed
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
  vpx_free(cpi->nmvcosts[0]);
  vpx_free(cpi->nmvcosts[1]);
  cpi->nmvcosts[0] = NULL;
  cpi->nmvcosts[1] = NULL;

  vpx_free(cpi->nmvcosts_hp[0]);
  vpx_free(cpi->nmvcosts_hp[1]);
  cpi->nmvcosts_hp[0] = NULL;
  cpi->nmvcosts_hp[1] = NULL;

  vpx_free(cpi->nmvsadcosts[0]);
  vpx_free(cpi->nmvsadcosts[1]);
  cpi->nmvsadcosts[0] = NULL;
  cpi->nmvsadcosts[1] = NULL;

  vpx_free(cpi->nmvsadcosts_hp[0]);
  vpx_free(cpi->nmvsadcosts_hp[1]);
  cpi->nmvsadcosts_hp[0] = NULL;
  cpi->nmvsadcosts_hp[1] = NULL;

201
202
203
  vp9_cyclic_refresh_free(cpi->cyclic_refresh);
  cpi->cyclic_refresh = NULL;

Adrian Grange's avatar
Adrian Grange committed
204
  vp9_free_ref_frame_buffers(cm);
205
  vp9_free_context_buffers(cm);
John Koleszar's avatar
John Koleszar committed
206

207
208
  vp9_free_frame_buffer(&cpi->last_frame_uf);
  vp9_free_frame_buffer(&cpi->scaled_source);
209
  vp9_free_frame_buffer(&cpi->scaled_last_source);
210
  vp9_free_frame_buffer(&cpi->alt_ref_buffer);
211
  vp9_lookahead_destroy(cpi->lookahead);
John Koleszar's avatar
John Koleszar committed
212

John Koleszar's avatar
John Koleszar committed
213
214
  vpx_free(cpi->tok);
  cpi->tok = 0;
John Koleszar's avatar
John Koleszar committed
215

216
  vp9_free_pc_tree(cpi);
217

218
219
220
221
222
223
  for (i = 0; i < cpi->svc.number_spatial_layers; ++i) {
    LAYER_CONTEXT *const lc = &cpi->svc.layer_context[i];
    vpx_free(lc->rc_twopass_stats_in.buf);
    lc->rc_twopass_stats_in.buf = NULL;
    lc->rc_twopass_stats_in.sz = 0;
  }
224
225
226
227
228

  if (cpi->source_diff_var != NULL) {
    vpx_free(cpi->source_diff_var);
    cpi->source_diff_var = NULL;
  }
229

230
231
232
233
234
  for (i = 0; i < MAX_LAG_BUFFERS; ++i) {
    vp9_free_frame_buffer(&cpi->svc.scaled_frames[i]);
  }
  vpx_memset(&cpi->svc.scaled_frames[0], 0,
             MAX_LAG_BUFFERS * sizeof(cpi->svc.scaled_frames[0]));
235
236
237

  vp9_free_frame_buffer(&cpi->svc.empty_frame.img);
  vpx_memset(&cpi->svc.empty_frame, 0, sizeof(cpi->svc.empty_frame));
John Koleszar's avatar
John Koleszar committed
238
239
}

240
241
242
243
244
245
246
247
248
static void save_coding_context(VP9_COMP *cpi) {
  CODING_CONTEXT *const cc = &cpi->coding_context;
  VP9_COMMON *cm = &cpi->common;

  // Stores a snapshot of key state variables which can subsequently be
  // restored with a call to vp9_restore_coding_context. These functions are
  // intended for use in a re-code loop in vp9_compress_frame where the
  // quantizer value is adjusted between loop iterations.
  vp9_copy(cc->nmvjointcost,  cpi->mb.nmvjointcost);
Jingning Han's avatar
Jingning Han committed
249
250
251
252
253
254
255
256
257

  vpx_memcpy(cc->nmvcosts[0], cpi->nmvcosts[0],
             MV_VALS * sizeof(*cpi->nmvcosts[0]));
  vpx_memcpy(cc->nmvcosts[1], cpi->nmvcosts[1],
             MV_VALS * sizeof(*cpi->nmvcosts[1]));
  vpx_memcpy(cc->nmvcosts_hp[0], cpi->nmvcosts_hp[0],
             MV_VALS * sizeof(*cpi->nmvcosts_hp[0]));
  vpx_memcpy(cc->nmvcosts_hp[1], cpi->nmvcosts_hp[1],
             MV_VALS * sizeof(*cpi->nmvcosts_hp[1]));
258
259
260
261
262
263
264
265
266

  vp9_copy(cc->segment_pred_probs, cm->seg.pred_probs);

  vpx_memcpy(cpi->coding_context.last_frame_seg_map_copy,
             cm->last_frame_seg_map, (cm->mi_rows * cm->mi_cols));

  vp9_copy(cc->last_ref_lf_deltas, cm->lf.last_ref_deltas);
  vp9_copy(cc->last_mode_lf_deltas, cm->lf.last_mode_deltas);

267
  cc->fc = *cm->fc;
268
269
270
271
272
273
274
275
276
}

static void restore_coding_context(VP9_COMP *cpi) {
  CODING_CONTEXT *const cc = &cpi->coding_context;
  VP9_COMMON *cm = &cpi->common;

  // Restore key state variables to the snapshot state stored in the
  // previous call to vp9_save_coding_context.
  vp9_copy(cpi->mb.nmvjointcost, cc->nmvjointcost);
Jingning Han's avatar
Jingning Han committed
277
278
279
280
281
282
283
284
285

  vpx_memcpy(cpi->nmvcosts[0], cc->nmvcosts[0],
             MV_VALS * sizeof(*cc->nmvcosts[0]));
  vpx_memcpy(cpi->nmvcosts[1], cc->nmvcosts[1],
             MV_VALS * sizeof(*cc->nmvcosts[1]));
  vpx_memcpy(cpi->nmvcosts_hp[0], cc->nmvcosts_hp[0],
             MV_VALS * sizeof(*cc->nmvcosts_hp[0]));
  vpx_memcpy(cpi->nmvcosts_hp[1], cc->nmvcosts_hp[1],
             MV_VALS * sizeof(*cc->nmvcosts_hp[1]));
286
287
288
289
290
291
292
293
294
295

  vp9_copy(cm->seg.pred_probs, cc->segment_pred_probs);

  vpx_memcpy(cm->last_frame_seg_map,
             cpi->coding_context.last_frame_seg_map_copy,
             (cm->mi_rows * cm->mi_cols));

  vp9_copy(cm->lf.last_ref_deltas, cc->last_ref_lf_deltas);
  vp9_copy(cm->lf.last_mode_deltas, cc->last_mode_lf_deltas);

296
  *cm->fc = cc->fc;
297
298
}

299
static void configure_static_seg_features(VP9_COMP *cpi) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
300
  VP9_COMMON *const cm = &cpi->common;
301
  const RATE_CONTROL *const rc = &cpi->rc;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
302
  struct segmentation *const seg = &cm->seg;
303

304
  int high_q = (int)(rc->avg_q > 48.0);
John Koleszar's avatar
John Koleszar committed
305
  int qi_delta;
306

John Koleszar's avatar
John Koleszar committed
307
308
309
  // Disable and clear down for KF
  if (cm->frame_type == KEY_FRAME) {
    // Clear down the global segmentation map
310
    vpx_memset(cpi->segmentation_map, 0, cm->mi_rows * cm->mi_cols);
311
312
    seg->update_map = 0;
    seg->update_data = 0;
John Koleszar's avatar
John Koleszar committed
313
    cpi->static_mb_pct = 0;
314

John Koleszar's avatar
John Koleszar committed
315
    // Disable segmentation
316
    vp9_disable_segmentation(seg);
Paul Wilkins's avatar
Paul Wilkins committed
317

John Koleszar's avatar
John Koleszar committed
318
    // Clear down the segment features.
319
    vp9_clearall_segfeatures(seg);
320
321
  } else if (cpi->refresh_alt_ref_frame) {
    // If this is an alt ref frame
John Koleszar's avatar
John Koleszar committed
322
    // Clear down the global segmentation map
323
    vpx_memset(cpi->segmentation_map, 0, cm->mi_rows * cm->mi_cols);
324
325
    seg->update_map = 0;
    seg->update_data = 0;
John Koleszar's avatar
John Koleszar committed
326
    cpi->static_mb_pct = 0;
Paul Wilkins's avatar
Paul Wilkins committed
327

John Koleszar's avatar
John Koleszar committed
328
    // Disable segmentation and individual segment features by default
329
    vp9_disable_segmentation(seg);
330
    vp9_clearall_segfeatures(seg);
Paul Wilkins's avatar
Paul Wilkins committed
331

John Koleszar's avatar
John Koleszar committed
332
333
    // Scan frames from current to arf frame.
    // This function re-enables segmentation if appropriate.
334
    vp9_update_mbgraph_stats(cpi);
Paul Wilkins's avatar
Paul Wilkins committed
335

John Koleszar's avatar
John Koleszar committed
336
337
    // If segmentation was enabled set those features needed for the
    // arf itself.
338
339
340
    if (seg->enabled) {
      seg->update_map = 1;
      seg->update_data = 1;
Paul Wilkins's avatar
Paul Wilkins committed
341

342
343
      qi_delta = vp9_compute_qdelta(rc, rc->avg_q, rc->avg_q * 0.875,
                                    cm->bit_depth);
344
      vp9_set_segdata(seg, 1, SEG_LVL_ALT_Q, qi_delta - 2);
345
      vp9_set_segdata(seg, 1, SEG_LVL_ALT_LF, -2);
Paul Wilkins's avatar
Paul Wilkins committed
346

347
348
      vp9_enable_segfeature(seg, 1, SEG_LVL_ALT_Q);
      vp9_enable_segfeature(seg, 1, SEG_LVL_ALT_LF);
349

John Koleszar's avatar
John Koleszar committed
350
      // Where relevant assume segment data is delta data
351
      seg->abs_delta = SEGMENT_DELTADATA;
352
    }
353
  } else if (seg->enabled) {
354
355
    // All other frames if segmentation has been enabled

John Koleszar's avatar
John Koleszar committed
356
    // First normal frame in a valid gf or alt ref group
357
    if (rc->frames_since_golden == 0) {
Paul Wilkins's avatar
Paul Wilkins committed
358
      // Set up segment features for normal frames in an arf group
359
      if (rc->source_alt_ref_active) {
360
361
362
        seg->update_map = 0;
        seg->update_data = 1;
        seg->abs_delta = SEGMENT_DELTADATA;
Paul Wilkins's avatar
Paul Wilkins committed
363

364
365
        qi_delta = vp9_compute_qdelta(rc, rc->avg_q, rc->avg_q * 1.125,
                                      cm->bit_depth);
366
        vp9_set_segdata(seg, 1, SEG_LVL_ALT_Q, qi_delta + 2);
367
        vp9_enable_segfeature(seg, 1, SEG_LVL_ALT_Q);
Paul Wilkins's avatar
Paul Wilkins committed
368

369
370
        vp9_set_segdata(seg, 1, SEG_LVL_ALT_LF, -2);
        vp9_enable_segfeature(seg, 1, SEG_LVL_ALT_LF);
Paul Wilkins's avatar
Paul Wilkins committed
371

John Koleszar's avatar
John Koleszar committed
372
373
        // Segment coding disabled for compred testing
        if (high_q || (cpi->static_mb_pct == 100)) {
374
375
376
          vp9_set_segdata(seg, 1, SEG_LVL_REF_FRAME, ALTREF_FRAME);
          vp9_enable_segfeature(seg, 1, SEG_LVL_REF_FRAME);
          vp9_enable_segfeature(seg, 1, SEG_LVL_SKIP);
John Koleszar's avatar
John Koleszar committed
377
        }
378
379
380
381
      } else {
        // Disable segmentation and clear down features if alt ref
        // is not active for this group

382
        vp9_disable_segmentation(seg);
Paul Wilkins's avatar
Paul Wilkins committed
383

384
        vpx_memset(cpi->segmentation_map, 0, cm->mi_rows * cm->mi_cols);
Paul Wilkins's avatar
Paul Wilkins committed
385

386
387
        seg->update_map = 0;
        seg->update_data = 0;
Paul Wilkins's avatar
Paul Wilkins committed
388

389
        vp9_clearall_segfeatures(seg);
John Koleszar's avatar
John Koleszar committed
390
      }
391
    } else if (rc->is_src_frame_alt_ref) {
392
393
394
      // Special case where we are coding over the top of a previous
      // alt ref frame.
      // Segment coding disabled for compred testing
John Koleszar's avatar
John Koleszar committed
395

Paul Wilkins's avatar
Paul Wilkins committed
396
      // Enable ref frame features for segment 0 as well
397
398
      vp9_enable_segfeature(seg, 0, SEG_LVL_REF_FRAME);
      vp9_enable_segfeature(seg, 1, SEG_LVL_REF_FRAME);
John Koleszar's avatar
John Koleszar committed
399

Paul Wilkins's avatar
Paul Wilkins committed
400
      // All mbs should use ALTREF_FRAME
401
402
403
404
      vp9_clear_segdata(seg, 0, SEG_LVL_REF_FRAME);
      vp9_set_segdata(seg, 0, SEG_LVL_REF_FRAME, ALTREF_FRAME);
      vp9_clear_segdata(seg, 1, SEG_LVL_REF_FRAME);
      vp9_set_segdata(seg, 1, SEG_LVL_REF_FRAME, ALTREF_FRAME);
John Koleszar's avatar
John Koleszar committed
405

Paul Wilkins's avatar
Paul Wilkins committed
406
      // Skip all MBs if high Q (0,0 mv and skip coeffs)
John Koleszar's avatar
John Koleszar committed
407
      if (high_q) {
408
409
        vp9_enable_segfeature(seg, 0, SEG_LVL_SKIP);
        vp9_enable_segfeature(seg, 1, SEG_LVL_SKIP);
John Koleszar's avatar
John Koleszar committed
410
      }
Adrian Grange's avatar
Adrian Grange committed
411
      // Enable data update
412
      seg->update_data = 1;
413
414
415
    } else {
      // All other frames.

John Koleszar's avatar
John Koleszar committed
416
      // No updates.. leave things as they are.
417
418
      seg->update_map = 0;
      seg->update_data = 0;
John Koleszar's avatar
John Koleszar committed
419
420
    }
  }
421
422
}

423
static void update_reference_segmentation_map(VP9_COMP *cpi) {
Ronald S. Bultje's avatar
Ronald S. Bultje committed
424
  VP9_COMMON *const cm = &cpi->common;
hkuang's avatar
hkuang committed
425
  MODE_INFO *mi_8x8_ptr = cm->mi;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
426
  uint8_t *cache_ptr = cm->last_frame_seg_map;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
427
428
  int row, col;

429
  for (row = 0; row < cm->mi_rows; row++) {
hkuang's avatar
hkuang committed
430
    MODE_INFO *mi_8x8 = mi_8x8_ptr;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
431
    uint8_t *cache = cache_ptr;
432
    for (col = 0; col < cm->mi_cols; col++, mi_8x8++, cache++)
hkuang's avatar
hkuang committed
433
      cache[0] = mi_8x8[0].src_mi->mbmi.segment_id;
434
    mi_8x8_ptr += cm->mi_stride;
435
    cache_ptr += cm->mi_cols;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
436
437
438
  }
}

439
static void alloc_raw_frame_buffers(VP9_COMP *cpi) {
440
  VP9_COMMON *cm = &cpi->common;
441
  const VP9EncoderConfig *oxcf = &cpi->oxcf;
442

Dmitry Kovalev's avatar
Dmitry Kovalev committed
443
  cpi->lookahead = vp9_lookahead_init(oxcf->width, oxcf->height,
444
                                      cm->subsampling_x, cm->subsampling_y,
445
446
447
#if CONFIG_VP9_HIGHBITDEPTH
                                      cm->use_highbitdepth,
#endif
Dmitry Kovalev's avatar
Dmitry Kovalev committed
448
                                      oxcf->lag_in_frames);
John Koleszar's avatar
John Koleszar committed
449
  if (!cpi->lookahead)
450
    vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
John Koleszar's avatar
John Koleszar committed
451
                       "Failed to allocate lag buffers");
John Koleszar's avatar
John Koleszar committed
452

453
  if (vp9_realloc_frame_buffer(&cpi->alt_ref_buffer,
Dmitry Kovalev's avatar
Dmitry Kovalev committed
454
                               oxcf->width, oxcf->height,
455
                               cm->subsampling_x, cm->subsampling_y,
456
457
458
#if CONFIG_VP9_HIGHBITDEPTH
                               cm->use_highbitdepth,
#endif
459
                               VP9_ENC_BORDER_IN_PIXELS, NULL, NULL, NULL))
460
    vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
John Koleszar's avatar
John Koleszar committed
461
                       "Failed to allocate altref buffer");
John Koleszar's avatar
John Koleszar committed
462
}
463

464
465
static void alloc_ref_frame_buffers(VP9_COMP *cpi) {
  VP9_COMMON *const cm = &cpi->common;
Adrian Grange's avatar
Adrian Grange committed
466
  if (vp9_alloc_ref_frame_buffers(cm, cm->width, cm->height))
467
    vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
John Koleszar's avatar
John Koleszar committed
468
                       "Failed to allocate frame buffers");
John Koleszar's avatar
John Koleszar committed
469
470
}

471
static void alloc_util_frame_buffers(VP9_COMP *cpi) {
472
  VP9_COMMON *const cm = &cpi->common;
473
474
475
  if (vp9_realloc_frame_buffer(&cpi->last_frame_uf,
                               cm->width, cm->height,
                               cm->subsampling_x, cm->subsampling_y,
476
477
478
#if CONFIG_VP9_HIGHBITDEPTH
                               cm->use_highbitdepth,
#endif
479
                               VP9_ENC_BORDER_IN_PIXELS, NULL, NULL, NULL))
480
    vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
481
                       "Failed to allocate last frame buffer");
482

483
484
485
  if (vp9_realloc_frame_buffer(&cpi->scaled_source,
                               cm->width, cm->height,
                               cm->subsampling_x, cm->subsampling_y,
486
487
488
#if CONFIG_VP9_HIGHBITDEPTH
                               cm->use_highbitdepth,
#endif
489
                               VP9_ENC_BORDER_IN_PIXELS, NULL, NULL, NULL))
490
    vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
491
                       "Failed to allocate scaled source buffer");
492

493
494
495
  if (vp9_realloc_frame_buffer(&cpi->scaled_last_source,
                               cm->width, cm->height,
                               cm->subsampling_x, cm->subsampling_y,
496
497
498
#if CONFIG_VP9_HIGHBITDEPTH
                               cm->use_highbitdepth,
#endif
499
500
                               VP9_ENC_BORDER_IN_PIXELS, NULL, NULL, NULL))
    vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
501
502
                       "Failed to allocate scaled last source buffer");
}
503

504
505
void vp9_alloc_compressor_data(VP9_COMP *cpi) {
  VP9_COMMON *cm = &cpi->common;
506

507
508
509
510
511
512
513
  vp9_alloc_context_buffers(cm, cm->width, cm->height);

  vpx_free(cpi->tok);

  {
    unsigned int tokens = get_token_alloc(cm->mb_rows, cm->mb_cols);
    CHECK_MEM_ERROR(cm, cpi->tok, vpx_calloc(tokens, sizeof(*cpi->tok)));
514
  }
515

516
517
518
519
520
521
  vp9_setup_pc_tree(&cpi->common, cpi);
}

static void update_frame_size(VP9_COMP *cpi) {
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &cpi->mb.e_mbd;
Adrian Grange's avatar
Adrian Grange committed
522
523
524

  vp9_set_mb_mi(cm, cm->width, cm->height);
  vp9_init_context_buffers(cm);
525
  init_macroblockd(cm, xd);
526

527
  if (is_two_pass_svc(cpi)) {
528
529
530
    if (vp9_realloc_frame_buffer(&cpi->alt_ref_buffer,
                                 cm->width, cm->height,
                                 cm->subsampling_x, cm->subsampling_y,
531
532
533
#if CONFIG_VP9_HIGHBITDEPTH
                                 cm->use_highbitdepth,
#endif
534
535
536
537
                                 VP9_ENC_BORDER_IN_PIXELS, NULL, NULL, NULL))
      vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
                         "Failed to reallocate alt_ref_buffer");
  }
538
539
}

James Zern's avatar
James Zern committed
540
void vp9_new_framerate(VP9_COMP *cpi, double framerate) {
541
  cpi->framerate = framerate < 0.1 ? 30 : framerate;
542
  vp9_rc_update_framerate(cpi);
John Koleszar's avatar
John Koleszar committed
543
544
}

Ronald S. Bultje's avatar
Ronald S. Bultje committed
545
546
static void set_tile_limits(VP9_COMP *cpi) {
  VP9_COMMON *const cm = &cpi->common;
547

Dmitry Kovalev's avatar
Dmitry Kovalev committed
548
549
  int min_log2_tile_cols, max_log2_tile_cols;
  vp9_get_tile_n_bits(cm->mi_cols, &min_log2_tile_cols, &max_log2_tile_cols);
550

Dmitry Kovalev's avatar
Dmitry Kovalev committed
551
552
553
  cm->log2_tile_cols = clamp(cpi->oxcf.tile_columns,
                             min_log2_tile_cols, max_log2_tile_cols);
  cm->log2_tile_rows = cpi->oxcf.tile_rows;
554
}
555

556
557
558
559
560
561
static void init_buffer_indices(VP9_COMP *cpi) {
  cpi->lst_fb_idx = 0;
  cpi->gld_fb_idx = 1;
  cpi->alt_fb_idx = 2;
}

562
static void init_config(struct VP9_COMP *cpi, VP9EncoderConfig *oxcf) {
563
  VP9_COMMON *const cm = &cpi->common;
John Koleszar's avatar
John Koleszar committed
564

John Koleszar's avatar
John Koleszar committed
565
  cpi->oxcf = *oxcf;
566
  cpi->framerate = oxcf->init_framerate;
John Koleszar's avatar
John Koleszar committed
567

568
569
  cm->profile = oxcf->profile;
  cm->bit_depth = oxcf->bit_depth;
570
571
572
#if CONFIG_VP9_HIGHBITDEPTH
  cm->use_highbitdepth = oxcf->use_highbitdepth;
#endif
573
  cm->color_space = UNKNOWN;
John Koleszar's avatar
John Koleszar committed
574

575
576
  cm->width = oxcf->width;
  cm->height = oxcf->height;
577
  vp9_alloc_compressor_data(cpi);
578

579
580
581
582
583
  // Spatial scalability.
  cpi->svc.number_spatial_layers = oxcf->ss_number_layers;
  // Temporal scalability.
  cpi->svc.number_temporal_layers = oxcf->ts_number_layers;

584
  if ((cpi->svc.number_temporal_layers > 1 && cpi->oxcf.rc_mode == VPX_CBR) ||
585
586
      ((cpi->svc.number_temporal_layers > 1 ||
        cpi->svc.number_spatial_layers > 1) &&
587
       cpi->oxcf.pass != 1)) {
588
    vp9_init_layer_context(cpi);
589
590
  }

John Koleszar's avatar
John Koleszar committed
591
  // change includes all joint functionality
Dmitry Kovalev's avatar
Dmitry Kovalev committed
592
  vp9_change_config(cpi, oxcf);
John Koleszar's avatar
John Koleszar committed
593

John Koleszar's avatar
John Koleszar committed
594
  cpi->static_mb_pct = 0;
595
  cpi->ref_frame_flags = 0;
596

597
  init_buffer_indices(cpi);
John Koleszar's avatar
John Koleszar committed
598
599
}

Dmitry Kovalev's avatar
Dmitry Kovalev committed
600
601
602
603
604
605
606
607
608
609
610
611
612
613
static void set_rc_buffer_sizes(RATE_CONTROL *rc,
                                const VP9EncoderConfig *oxcf) {
  const int64_t bandwidth = oxcf->target_bandwidth;
  const int64_t starting = oxcf->starting_buffer_level_ms;
  const int64_t optimal = oxcf->optimal_buffer_level_ms;
  const int64_t maximum = oxcf->maximum_buffer_size_ms;

  rc->starting_buffer_level = starting * bandwidth / 1000;
  rc->optimal_buffer_level = (optimal == 0) ? bandwidth / 8
                                            : optimal * bandwidth / 1000;
  rc->maximum_buffer_size = (maximum == 0) ? bandwidth / 8
                                           : maximum * bandwidth / 1000;
}

614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
#if CONFIG_VP9_HIGHBITDEPTH
#define HIGHBD_BFP(BT, SDF, SDAF, VF, SVF, SVAF, SDX3F, SDX8F, SDX4DF) \
    cpi->fn_ptr[BT].sdf = SDF; \
    cpi->fn_ptr[BT].sdaf = SDAF; \
    cpi->fn_ptr[BT].vf = VF; \
    cpi->fn_ptr[BT].svf = SVF; \
    cpi->fn_ptr[BT].svaf = SVAF; \
    cpi->fn_ptr[BT].sdx3f = SDX3F; \
    cpi->fn_ptr[BT].sdx8f = SDX8F; \
    cpi->fn_ptr[BT].sdx4df = SDX4DF;

#define MAKE_BFP_SAD_WRAPPER(fnname) \
static unsigned int fnname##_bits8(const uint8_t *src_ptr, \
                                   int source_stride, \
                                   const uint8_t *ref_ptr, \
                                   int ref_stride) {  \
  return fnname(src_ptr, source_stride, ref_ptr, ref_stride); \
} \
static unsigned int fnname##_bits10(const uint8_t *src_ptr, \
                                    int source_stride, \
                                    const uint8_t *ref_ptr, \
                                    int ref_stride) {  \
  return fnname(src_ptr, source_stride, ref_ptr, ref_stride) >> 2; \
} \
static unsigned int fnname##_bits12(const uint8_t *src_ptr, \
                                    int source_stride, \
                                    const uint8_t *ref_ptr, \
                                    int ref_stride) {  \
  return fnname(src_ptr, source_stride, ref_ptr, ref_stride) >> 4; \
}

#define MAKE_BFP_SADAVG_WRAPPER(fnname) static unsigned int \
fnname##_bits8(const uint8_t *src_ptr, \
               int source_stride, \
               const uint8_t *ref_ptr, \
               int ref_stride, \
               const uint8_t *second_pred) {  \
  return fnname(src_ptr, source_stride, ref_ptr, ref_stride, second_pred); \
} \
static unsigned int fnname##_bits10(const uint8_t *src_ptr, \
                                    int source_stride, \
                                    const uint8_t *ref_ptr, \
                                    int ref_stride, \
                                    const uint8_t *second_pred) {  \
  return fnname(src_ptr, source_stride, ref_ptr, ref_stride, \
                second_pred) >> 2; \
} \
static unsigned int fnname##_bits12(const uint8_t *src_ptr, \
                                    int source_stride, \
                                    const uint8_t *ref_ptr, \
                                    int ref_stride, \
                                    const uint8_t *second_pred) {  \
  return fnname(src_ptr, source_stride, ref_ptr, ref_stride, \
                second_pred) >> 4; \
}

#define MAKE_BFP_SAD3_WRAPPER(fnname) \
static void fnname##_bits8(const uint8_t *src_ptr, \
                           int source_stride, \
                           const uint8_t *ref_ptr, \
                           int  ref_stride, \
                           unsigned int *sad_array) {  \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
} \
static void fnname##_bits10(const uint8_t *src_ptr, \
                            int source_stride, \
                            const uint8_t *ref_ptr, \
                            int  ref_stride, \
                            unsigned int *sad_array) {  \
  int i; \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
  for (i = 0; i < 3; i++) \
    sad_array[i] >>= 2; \
} \
static void fnname##_bits12(const uint8_t *src_ptr, \
                            int source_stride, \
                            const uint8_t *ref_ptr, \
                            int  ref_stride, \
                            unsigned int *sad_array) {  \
  int i; \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
  for (i = 0; i < 3; i++) \
    sad_array[i] >>= 4; \
}

#define MAKE_BFP_SAD8_WRAPPER(fnname) \
static void fnname##_bits8(const uint8_t *src_ptr, \
                           int source_stride, \
                           const uint8_t *ref_ptr, \
                           int  ref_stride, \
                           unsigned int *sad_array) {  \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
} \
static void fnname##_bits10(const uint8_t *src_ptr, \
                            int source_stride, \
                            const uint8_t *ref_ptr, \
                            int  ref_stride, \
                            unsigned int *sad_array) {  \
  int i; \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
  for (i = 0; i < 8; i++) \
    sad_array[i] >>= 2; \
} \
static void fnname##_bits12(const uint8_t *src_ptr, \
                            int source_stride, \
                            const uint8_t *ref_ptr, \
                            int  ref_stride, \
                            unsigned int *sad_array) {  \
  int i; \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
  for (i = 0; i < 8; i++) \
    sad_array[i] >>= 4; \
}
#define MAKE_BFP_SAD4D_WRAPPER(fnname) \
static void fnname##_bits8(const uint8_t *src_ptr, \
                           int source_stride, \
                           const uint8_t* const ref_ptr[], \
                           int  ref_stride, \
                           unsigned int *sad_array) {  \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
} \
static void fnname##_bits10(const uint8_t *src_ptr, \
                            int source_stride, \
                            const uint8_t* const ref_ptr[], \
                            int  ref_stride, \
                            unsigned int *sad_array) {  \
  int i; \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
  for (i = 0; i < 4; i++) \
  sad_array[i] >>= 2; \
} \
static void fnname##_bits12(const uint8_t *src_ptr, \
                            int source_stride, \
                            const uint8_t* const ref_ptr[], \
                            int  ref_stride, \
                            unsigned int *sad_array) {  \
  int i; \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
  for (i = 0; i < 4; i++) \
  sad_array[i] >>= 4; \
}

756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad32x16)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad32x16_avg)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad32x16x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad16x32)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad16x32_avg)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad16x32x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad64x32)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad64x32_avg)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad64x32x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad32x64)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad32x64_avg)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad32x64x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad32x32)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad32x32_avg)
MAKE_BFP_SAD3_WRAPPER(vp9_highbd_sad32x32x3)
MAKE_BFP_SAD8_WRAPPER(vp9_highbd_sad32x32x8)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad32x32x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad64x64)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad64x64_avg)
MAKE_BFP_SAD3_WRAPPER(vp9_highbd_sad64x64x3)
MAKE_BFP_SAD8_WRAPPER(vp9_highbd_sad64x64x8)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad64x64x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad16x16)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad16x16_avg)
MAKE_BFP_SAD3_WRAPPER(vp9_highbd_sad16x16x3)
MAKE_BFP_SAD8_WRAPPER(vp9_highbd_sad16x16x8)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad16x16x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad16x8)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad16x8_avg)
MAKE_BFP_SAD3_WRAPPER(vp9_highbd_sad16x8x3)
MAKE_BFP_SAD8_WRAPPER(vp9_highbd_sad16x8x8)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad16x8x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad8x16)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad8x16_avg)
MAKE_BFP_SAD3_WRAPPER(vp9_highbd_sad8x16x3)
MAKE_BFP_SAD8_WRAPPER(vp9_highbd_sad8x16x8)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad8x16x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad8x8)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad8x8_avg)
MAKE_BFP_SAD3_WRAPPER(vp9_highbd_sad8x8x3)
MAKE_BFP_SAD8_WRAPPER(vp9_highbd_sad8x8x8)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad8x8x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad8x4)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad8x4_avg)
MAKE_BFP_SAD8_WRAPPER(vp9_highbd_sad8x4x8)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad8x4x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad4x8)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad4x8_avg)
MAKE_BFP_SAD8_WRAPPER(vp9_highbd_sad4x8x8)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad4x8x4d)
MAKE_BFP_SAD_WRAPPER(vp9_highbd_sad4x4)
MAKE_BFP_SADAVG_WRAPPER(vp9_highbd_sad4x4_avg)
MAKE_BFP_SAD3_WRAPPER(vp9_highbd_sad4x4x3)
MAKE_BFP_SAD8_WRAPPER(vp9_highbd_sad4x4x8)
MAKE_BFP_SAD4D_WRAPPER(vp9_highbd_sad4x4x4d)

static void  highbd_set_var_fns(VP9_COMP *const cpi) {
813
814
815
816
817
  VP9_COMMON *const cm = &cpi->common;
  if (cm->use_highbitdepth) {
    switch (cm->bit_depth) {
      case VPX_BITS_8:
        HIGHBD_BFP(BLOCK_32X16,
818
819
820
821
822
                   vp9_highbd_sad32x16_bits8,
                   vp9_highbd_sad32x16_avg_bits8,
                   vp9_highbd_variance32x16,
                   vp9_highbd_sub_pixel_variance32x16,
                   vp9_highbd_sub_pixel_avg_variance32x16,
823
824
                   NULL,
                   NULL,
825
                   vp9_highbd_sad32x16x4d_bits8)
826
827

        HIGHBD_BFP(BLOCK_16X32,
828
829
830
831
832
                   vp9_highbd_sad16x32_bits8,
                   vp9_highbd_sad16x32_avg_bits8,
                   vp9_highbd_variance16x32,
                   vp9_highbd_sub_pixel_variance16x32,
                   vp9_highbd_sub_pixel_avg_variance16x32,
833
834
                   NULL,
                   NULL,
835
                   vp9_highbd_sad16x32x4d_bits8)
836
837

        HIGHBD_BFP(BLOCK_64X32,
838
839
840
841
842
                   vp9_highbd_sad64x32_bits8,
                   vp9_highbd_sad64x32_avg_bits8,
                   vp9_highbd_variance64x32,
                   vp9_highbd_sub_pixel_variance64x32,
                   vp9_highbd_sub_pixel_avg_variance64x32,
843
844
                   NULL,
                   NULL,
845
                   vp9_highbd_sad64x32x4d_bits8)
846
847

        HIGHBD_BFP(BLOCK_32X64,
848
849
850
851
852
                   vp9_highbd_sad32x64_bits8,
                   vp9_highbd_sad32x64_avg_bits8,
                   vp9_highbd_variance32x64,
                   vp9_highbd_sub_pixel_variance32x64,
                   vp9_highbd_sub_pixel_avg_variance32x64,
853
854
                   NULL,
                   NULL,
855
                   vp9_highbd_sad32x64x4d_bits8)
856
857

        HIGHBD_BFP(BLOCK_32X32,
858
859
860
861
862
863
864
865
                   vp9_highbd_sad32x32_bits8,
                   vp9_highbd_sad32x32_avg_bits8,
                   vp9_highbd_variance32x32,
                   vp9_highbd_sub_pixel_variance32x32,
                   vp9_highbd_sub_pixel_avg_variance32x32,
                   vp9_highbd_sad32x32x3_bits8,
                   vp9_highbd_sad32x32x8_bits8,
                   vp9_highbd_sad32x32x4d_bits8)
866
867

        HIGHBD_BFP(BLOCK_64X64,
868
869
870
871
872
873
874
875
                   vp9_highbd_sad64x64_bits8,
                   vp9_highbd_sad64x64_avg_bits8,
                   vp9_highbd_variance64x64,
                   vp9_highbd_sub_pixel_variance64x64,
                   vp9_highbd_sub_pixel_avg_variance64x64,
                   vp9_highbd_sad64x64x3_bits8,
                   vp9_highbd_sad64x64x8_bits8,
                   vp9_highbd_sad64x64x4d_bits8)
876
877

        HIGHBD_BFP(BLOCK_16X16,
878
879
880
881
882
883
884
885
                   vp9_highbd_sad16x16_bits8,
                   vp9_highbd_sad16x16_avg_bits8,
                   vp9_highbd_variance16x16,
                   vp9_highbd_sub_pixel_variance16x16,
                   vp9_highbd_sub_pixel_avg_variance16x16,
                   vp9_highbd_sad16x16x3_bits8,
                   vp9_highbd_sad16x16x8_bits8,
                   vp9_highbd_sad16x16x4d_bits8)
886
887

        HIGHBD_BFP(BLOCK_16X8,
888
889
890
891
892
893
894
895
                   vp9_highbd_sad16x8_bits8,
                   vp9_highbd_sad16x8_avg_bits8,
                   vp9_highbd_variance16x8,
                   vp9_highbd_sub_pixel_variance16x8,
                   vp9_highbd_sub_pixel_avg_variance16x8,
                   vp9_highbd_sad16x8x3_bits8,
                   vp9_highbd_sad16x8x8_bits8,
                   vp9_highbd_sad16x8x4d_bits8)
896
897

        HIGHBD_BFP(BLOCK_8X16,
898
899
900
901
902
903
904
905
                   vp9_highbd_sad8x16_bits8,
                   vp9_highbd_sad8x16_avg_bits8,
                   vp9_highbd_variance8x16,
                   vp9_highbd_sub_pixel_variance8x16,
                   vp9_highbd_sub_pixel_avg_variance8x16,
                   vp9_highbd_sad8x16x3_bits8,
                   vp9_highbd_sad8x16x8_bits8,
                   vp9_highbd_sad8x16x4d_bits8)
906
907

        HIGHBD_BFP(BLOCK_8X8,
908
909
910
911
912
913
914
915
                   vp9_highbd_sad8x8_bits8,
                   vp9_highbd_sad8x8_avg_bits8,
                   vp9_highbd_variance8x8,
                   vp9_highbd_sub_pixel_variance8x8,
                   vp9_highbd_sub_pixel_avg_variance8x8,
                   vp9_highbd_sad8x8x3_bits8,
                   vp9_highbd_sad8x8x8_bits8,
                   vp9_highbd_sad8x8x4d_bits8)
916
917

        HIGHBD_BFP(BLOCK_8X4,
918
919
920
921
922
                   vp9_highbd_sad8x4_bits8,
                   vp9_highbd_sad8x4_avg_bits8,
                   vp9_highbd_variance8x4,
                   vp9_highbd_sub_pixel_variance8x4,
                   vp9_highbd_sub_pixel_avg_variance8x4,
923
                   NULL,
924
925
                   vp9_highbd_sad8x4x8_bits8,
                   vp9_highbd_sad8x4x4d_bits8)
926
927

        HIGHBD_BFP(BLOCK_4X8,
928
929
930
931
932
                   vp9_highbd_sad4x8_bits8,
                   vp9_highbd_sad4x8_avg_bits8,
                   vp9_highbd_variance4x8,
                   vp9_highbd_sub_pixel_variance4x8,
                   vp9_highbd_sub_pixel_avg_variance4x8,
933
                   NULL,
934
935
                   vp9_highbd_sad4x8x8_bits8,
                   vp9_highbd_sad4x8x4d_bits8)
936
937

        HIGHBD_BFP(BLOCK_4X4,
938
939
940
941
942
943
944
945
                   vp9_highbd_sad4x4_bits8,
                   vp9_highbd_sad4x4_avg_bits8,
                   vp9_highbd_variance4x4,
                   vp9_highbd_sub_pixel_variance4x4,
                   vp9_highbd_sub_pixel_avg_variance4x4,
                   vp9_highbd_sad4x4x3_bits8,
                   vp9_highbd_sad4x4x8_bits8,
                   vp9_highbd_sad4x4x4d_bits8)
946
947
948
949
        break;

      case VPX_BITS_10:
        HIGHBD_BFP(BLOCK_32X16,
950
951
952
953
954
                   vp9_highbd_sad32x16_bits10,
                   vp9_highbd_sad32x16_avg_bits10,
                   vp9_highbd_10_variance32x16,
                   vp9_highbd_10_sub_pixel_variance32x16,
                   vp9_highbd_10_sub_pixel_avg_variance32x16,
955
956
                   NULL,
                   NULL,
957
                   vp9_highbd_sad32x16x4d_bits10)
958
959

        HIGHBD_BFP(BLOCK_16X32,
960
961
962
963
964
                   vp9_highbd_sad16x32_bits10,
                   vp9_highbd_sad16x32_avg_bits10,
                   vp9_highbd_10_variance16x32,
                   vp9_highbd_10_sub_pixel_variance16x32,
                   vp9_highbd_10_sub_pixel_avg_variance16x32,
965
966
                   NULL,
                   NULL,
967
                   vp9_highbd_sad16x32x4d_bits10)
968
969

        HIGHBD_BFP(BLOCK_64X32,
970
971
972
973
974
                   vp9_highbd_sad64x32_bits10,
                   vp9_highbd_sad64x32_avg_bits10,
                   vp9_highbd_10_variance64x32,
                   vp9_highbd_10_sub_pixel_variance64x32,
                   vp9_highbd_10_sub_pixel_avg_variance64x32,
975
976
                   NULL,
                   NULL,
977
                   vp9_highbd_sad64x32x4d_bits10)
978
979

        HIGHBD_BFP(BLOCK_32X64,
980
981
982
983
984
                   vp9_highbd_sad32x64_bits10,
                   vp9_highbd_sad32x64_avg_bits10,
                   vp9_highbd_10_variance32x64,
                   vp9_highbd_10_sub_pixel_variance32x64,
                   vp9_highbd_10_sub_pixel_avg_variance32x64,
985
986
                   NULL,
                   NULL,
987
                   vp9_highbd_sad32x64x4d_bits10)
988
989

        HIGHBD_BFP(BLOCK_32X32,
990
991
992
993
994
995
996
997
                   vp9_highbd_sad32x32_bits10,
                   vp9_highbd_sad32x32_avg_bits10,
                   vp9_highbd_10_variance32x32,
                   vp9_highbd_10_sub_pixel_variance32x32,
                   vp9_highbd_10_sub_pixel_avg_variance32x32,
                   vp9_highbd_sad32x32x3_bits10,
                   vp9_highbd_sad32x32x8_bits10,
                   vp9_highbd_sad32x32x4d_bits10)
998
999

        HIGHBD_BFP(BLOCK_64X64,
1000
1001
1002
1003
1004
1005
1006
1007
                   vp9_highbd_sad64x64_bits10,
                   vp9_highbd_sad64x64_avg_bits10,
                   vp9_highbd_10_variance64x64,
                   vp9_highbd_10_sub_pixel_variance64x64,
                   vp9_highbd_10_sub_pixel_avg_variance64x64,
                   vp9_highbd_sad64x64x3_bits10,
                   vp9_highbd_sad64x64x8_bits10,
                   vp9_highbd_sad64x64x4d_bits10)
1008
1009

        HIGHBD_BFP(BLOCK_16X16,
1010
1011
1012
1013
1014
1015
1016
1017
                   vp9_highbd_sad16x16_bits10,
                   vp9_highbd_sad16x16_avg_bits10,
                   vp9_highbd_10_variance16x16,
                   vp9_highbd_10_sub_pixel_variance16x16,
                   vp9_highbd_10_sub_pixel_avg_variance16x16,
                   vp9_highbd_sad16x16x3_bits10,
                   vp9_highbd_sad16x16x8_bits10,
                   vp9_highbd_sad16x16x4d_bits10)
1018
1019

        HIGHBD_BFP(BLOCK_16X8,
1020
1021
1022
1023
1024
1025
1026
1027
                   vp9_highbd_sad16x8_bits10,
                   vp9_highbd_sad16x8_avg_bits10,
                   vp9_highbd_10_variance16x8,
                   vp9_highbd_10_sub_pixel_variance16x8,
                   vp9_highbd_10_sub_pixel_avg_variance16x8,
                   vp9_highbd_sad16x8x3_bits10,
                   vp9_highbd_sad16x8x8_bits10,
                   vp9_highbd_sad16x8x4d_bits10)
1028
1029

        HIGHBD_BFP(BLOCK_8X16,
1030
1031
1032
1033
1034
1035
1036
1037
                   vp9_highbd_sad8x16_bits10,
                   vp9_highbd_sad8x16_avg_bits10,
                   vp9_highbd_10_variance8x16,
                   vp9_highbd_10_sub_pixel_variance8x16,
                   vp9_highbd_10_sub_pixel_avg_variance8x16,
                   vp9_highbd_sad8x16x3_bits10,
                   vp9_highbd_sad8x16x8_bits10,
                   vp9_highbd_sad8x16x4d_bits10)
1038
1039

        HIGHBD_BFP(BLOCK_8X8,
1040
1041
1042
1043
1044
1045
1046
1047
                   vp9_highbd_sad8x8_bits10,
                   vp9_highbd_sad8x8_avg_bits10,
                   vp9_highbd_10_variance8x8,
                   vp9_highbd_10_sub_pixel_variance8x8,
                   vp9_highbd_10_sub_pixel_avg_variance8x8,
                   vp9_highbd_sad8x8x3_bits10,
                   vp9_highbd_sad8x8x8_bits10,
                   vp9_highbd_sad8x8x4d_bits10)
1048
1049

        HIGHBD_BFP(BLOCK_8X4,
1050
1051
1052
1053
1054
                   vp9_highbd_sad8x4_bits10,
                   vp9_highbd_sad8x4_avg_bits10,
                   vp9_highbd_10_variance8x4,
                   vp9_highbd_10_sub_pixel_variance8x4,
                   vp9_highbd_10_sub_pixel_avg_variance8x4,
1055
                   NULL,
1056
1057
                   vp9_highbd_sad8x4x8_bits10,
                   vp9_highbd_sad8x4x4d_bits10)
1058
1059

        HIGHBD_BFP(BLOCK_4X8,
1060
1061
1062
1063
1064
                   vp9_highbd_sad4x8_bits10,
                   vp9_highbd_sad4x8_avg_bits10,
                   vp9_highbd_10_variance4x8,
                   vp9_highbd_10_sub_pixel_variance4x8,
                   vp9_highbd_10_sub_pixel_avg_variance4x8,
1065
                   NULL,
1066
1067
                   vp9_highbd_sad4x8x8_bits10,
                   vp9_highbd_sad4x8x4d_bits10)
1068
1069

        HIGHBD_BFP(BLOCK_4X4,
1070
1071
1072
1073
1074
1075
1076
1077
                   vp9_highbd_sad4x4_bits10,
                   vp9_highbd_sad4x4_avg_bits10,
                   vp9_highbd_10_variance4x4,
                   vp9_highbd_10_sub_pixel_variance4x4,
                   vp9_highbd_10_sub_pixel_avg_variance4x4,
                   vp9_highbd_sad4x4x3_bits10,
                   vp9_highbd_sad4x4x8_bits10,
                   vp9_highbd_sad4x4x4d_bits10)
1078
1079
1080
1081
        break;

      case VPX_BITS_12:
        HIGHBD_BFP(BLOCK_32X16,
1082
1083
1084
1085
1086
                   vp9_highbd_sad32x16_bits12,
                   vp9_highbd_sad32x16_avg_bits12,
                   vp9_highbd_12_variance32x16,
                   vp9_highbd_12_sub_pixel_variance32x16,
                   vp9_highbd_12_sub_pixel_avg_variance32x16,
1087
1088
                   NULL,
                   NULL,
1089
                   vp9_highbd_sad32x16x4d_bits12)
1090
1091

        HIGHBD_BFP(BLOCK_16X32,
1092
1093
1094
1095
1096
                   vp9_highbd_sad16x32_bits12,
                   vp9_highbd_sad16x32_avg_bits12,
                   vp9_highbd_12_variance16x32,
                   vp9_highbd_12_sub_pixel_variance16x32,
                   vp9_highbd_12_sub_pixel_avg_variance16x32,
1097
1098
                   NULL,
                   NULL,
1099
                   vp9_highbd_sad16x32x4d_bits12)
1100
1101

        HIGHBD_BFP(BLOCK_64X32,
1102
1103
1104
1105
1106
                   vp9_highbd_sad64x32_bits12,
                   vp9_highbd_sad64x32_avg_bits12,
                   vp9_highbd_12_variance64x32,
                   vp9_highbd_12_sub_pixel_variance64x32,
                   vp9_highbd_12_sub_pixel_avg_variance64x32,
1107
1108
                   NULL,
                   NULL,
1109
                   vp9_highbd_sad64x32x4d_bits12)
1110
1111

        HIGHBD_BFP(BLOCK_32X64,
1112
1113
1114
1115
1116
                   vp9_highbd_sad32x64_bits12,
                   vp9_highbd_sad32x64_avg_bits12,
                   vp9_highbd_12_variance32x64,
                   vp9_highbd_12_sub_pixel_variance32x64,
                   vp9_highbd_12_sub_pixel_avg_variance32x64,
1117
1118
                   NULL,
                   NULL,
1119
                   vp9_highbd_sad32x64x4d_bits12)
1120
1121

        HIGHBD_BFP(BLOCK_32X32,
1122
1123
1124
1125
1126
1127
1128
1129
                   vp9_highbd_sad32x32_bits12,
                   vp9_highbd_sad32x32_avg_bits12,
                   vp9_highbd_12_variance32x32,
                   vp9_highbd_12_sub_pixel_variance32x32,
                   vp9_highbd_12_sub_pixel_avg_variance32x32,
                   vp9_highbd_sad32x32x3_bits12,
                   vp9_highbd_sad32x32x8_bits12,
                   vp9_highbd_sad32x32x4d_bits12)
1130
1131

        HIGHBD_BFP(BLOCK_64X64,
1132
1133
1134
1135
1136
1137
1138
1139
                   vp9_highbd_sad64x64_bits12,
                   vp9_highbd_sad64x64_avg_bits12,
                   vp9_highbd_12_variance64x64,
                   vp9_highbd_12_sub_pixel_variance64x64,
                   vp9_highbd_12_sub_pixel_avg_variance64x64,
                   vp9_highbd_sad64x64x3_bits12,
                   vp9_highbd_sad64x64x8_bits12,
                   vp9_highbd_sad64x64x4d_bits12)
1140
1141

        HIGHBD_BFP(BLOCK_16X16,
1142
1143
1144
1145
1146
1147
1148
1149
                   vp9_highbd_sad16x16_bits12,
                   vp9_highbd_sad16x16_avg_bits12,
                   vp9_highbd_12_variance16x16,
                   vp9_highbd_12_sub_pixel_variance16x16,
                   vp9_highbd_12_sub_pixel_avg_variance16x16,
                   vp9_highbd_sad16x16x3_bits12,
                   vp9_highbd_sad16x16x8_bits12,
                   vp9_highbd_sad16x16x4d_bits12)
1150
1151

        HIGHBD_BFP(BLOCK_16X8,
1152
1153
1154
1155