vp9_encoder.c 163 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 * Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5
6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9
10
 */

Jim Bankoski's avatar
Jim Bankoski committed
11
12
13
14
#include <math.h>
#include <stdio.h>
#include <limits.h>

15
#include "./vp9_rtcd.h"
Zoe Liu's avatar
Zoe Liu committed
16
#include "./vpx_config.h"
Johann's avatar
Johann committed
17
#include "./vpx_dsp_rtcd.h"
Jim Bankoski's avatar
Jim Bankoski committed
18
#include "./vpx_scale_rtcd.h"
19
#include "vpx/internal/vpx_psnr.h"
20
#include "vpx_dsp/vpx_dsp_common.h"
Zoe Liu's avatar
Zoe Liu committed
21
#include "vpx_dsp/vpx_filter.h"
22
23
24
#if CONFIG_INTERNAL_STATS
#include "vpx_dsp/ssim.h"
#endif
25
#include "vpx_ports/mem.h"
26
#include "vpx_ports/system_state.h"
27
#include "vpx_ports/vpx_timer.h"
John Koleszar's avatar
John Koleszar committed
28

Jim Bankoski's avatar
Jim Bankoski committed
29
#include "vp9/common/vp9_alloccommon.h"
30
#include "vp9/common/vp9_filter.h"
31
#include "vp9/common/vp9_idct.h"
Jim Bankoski's avatar
Jim Bankoski committed
32
33
34
#if CONFIG_VP9_POSTPROC
#include "vp9/common/vp9_postproc.h"
#endif
35
#include "vp9/common/vp9_reconinter.h"
36
#include "vp9/common/vp9_reconintra.h"
Jim Bankoski's avatar
Jim Bankoski committed
37
#include "vp9/common/vp9_tile_common.h"
38

39
#include "vp9/encoder/vp9_aq_complexity.h"
Marco Paniconi's avatar
Marco Paniconi committed
40
41
#include "vp9/encoder/vp9_aq_cyclicrefresh.h"
#include "vp9/encoder/vp9_aq_variance.h"
42
#include "vp9/encoder/vp9_bitstream.h"
43
#include "vp9/encoder/vp9_context_tree.h"
44
#include "vp9/encoder/vp9_encodeframe.h"
45
#include "vp9/encoder/vp9_encodemv.h"
46
#include "vp9/encoder/vp9_encoder.h"
47
#include "vp9/encoder/vp9_ethread.h"
48
#include "vp9/encoder/vp9_firstpass.h"
Jim Bankoski's avatar
Jim Bankoski committed
49
#include "vp9/encoder/vp9_mbgraph.h"
50
#include "vp9/encoder/vp9_noise_estimate.h"
Jim Bankoski's avatar
Jim Bankoski committed
51
#include "vp9/encoder/vp9_picklpf.h"
52
#include "vp9/encoder/vp9_ratectrl.h"
53
#include "vp9/encoder/vp9_rd.h"
54
#include "vp9/encoder/vp9_resize.h"
Jim Bankoski's avatar
Jim Bankoski committed
55
#include "vp9/encoder/vp9_segmentation.h"
56
#include "vp9/encoder/vp9_skin_detection.h"
Yaowu Xu's avatar
Yaowu Xu committed
57
#include "vp9/encoder/vp9_speed_features.h"
58
#include "vp9/encoder/vp9_svc_layercontext.h"
59
#include "vp9/encoder/vp9_temporal_filter.h"
Paul Wilkins's avatar
Paul Wilkins committed
60

61
62
#define AM_SEGMENT_ID_INACTIVE 7
#define AM_SEGMENT_ID_ACTIVE 0
63

64
65
#define SHARP_FILTER_QTHRESH 0          /* Q threshold for 8-tap sharp filter */

Jim Bankoski's avatar
Jim Bankoski committed
66
67
68
69
70
71
#define ALTREF_HIGH_PRECISION_MV 1      // Whether to use high precision mv
                                         //  for altref computation.
#define HIGH_PRECISION_MV_QTHRESH 200   // Q threshold for high precision
                                         // mv. Choose a very high value for
                                         // now so that HIGH_PRECISION is always
                                         // chosen.
John Koleszar's avatar
John Koleszar committed
72
// #define OUTPUT_YUV_REC
John Koleszar's avatar
John Koleszar committed
73

74
#ifdef OUTPUT_YUV_DENOISED
75
FILE *yuv_denoised_file = NULL;
76
#endif
Marco's avatar
Marco committed
77
78
79
#ifdef OUTPUT_YUV_SKINMAP
FILE *yuv_skinmap_file = NULL;
#endif
80
81
82
#ifdef OUTPUT_YUV_REC
FILE *yuv_rec_file;
#endif
John Koleszar's avatar
John Koleszar committed
83
84
85

#if 0
FILE *framepsnr;
Yaowu Xu's avatar
Yaowu Xu committed
86
FILE *kf_list;
John Koleszar's avatar
John Koleszar committed
87
88
89
FILE *keyfile;
#endif

90
static INLINE void Scale2Ratio(VPX_SCALING mode, int *hr, int *hs) {
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
  switch (mode) {
    case NORMAL:
      *hr = 1;
      *hs = 1;
      break;
    case FOURFIVE:
      *hr = 4;
      *hs = 5;
      break;
    case THREEFIVE:
      *hr = 3;
      *hs = 5;
    break;
    case ONETWO:
      *hr = 1;
      *hs = 2;
    break;
    default:
      *hr = 1;
      *hs = 1;
       assert(0);
      break;
  }
}

116
117
// Mark all inactive blocks as active. Other segmentation features may be set
// so memset cannot be used, instead only inactive blocks should be reset.
118
static void suppress_active_map(VP9_COMP *cpi) {
119
120
121
122
123
124
125
126
  unsigned char *const seg_map = cpi->segmentation_map;
  int i;
  if (cpi->active_map.enabled || cpi->active_map.update)
    for (i = 0; i < cpi->common.mi_rows * cpi->common.mi_cols; ++i)
      if (seg_map[i] == AM_SEGMENT_ID_INACTIVE)
        seg_map[i] = AM_SEGMENT_ID_ACTIVE;
}

127
static void apply_active_map(VP9_COMP *cpi) {
128
129
130
131
132
133
134
  struct segmentation *const seg = &cpi->common.seg;
  unsigned char *const seg_map = cpi->segmentation_map;
  const unsigned char *const active_map = cpi->active_map.map;
  int i;

  assert(AM_SEGMENT_ID_ACTIVE == CR_SEGMENT_ID_BASE);

135
136
137
138
139
  if (frame_is_intra_only(&cpi->common)) {
    cpi->active_map.enabled = 0;
    cpi->active_map.update = 1;
  }

140
141
142
143
144
145
  if (cpi->active_map.update) {
    if (cpi->active_map.enabled) {
      for (i = 0; i < cpi->common.mi_rows * cpi->common.mi_cols; ++i)
        if (seg_map[i] == AM_SEGMENT_ID_ACTIVE) seg_map[i] = active_map[i];
      vp9_enable_segmentation(seg);
      vp9_enable_segfeature(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_SKIP);
146
147
148
149
150
      vp9_enable_segfeature(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_ALT_LF);
      // Setting the data to -MAX_LOOP_FILTER will result in the computed loop
      // filter level being zero regardless of the value of seg->abs_delta.
      vp9_set_segdata(seg, AM_SEGMENT_ID_INACTIVE,
                      SEG_LVL_ALT_LF, -MAX_LOOP_FILTER);
151
152
    } else {
      vp9_disable_segfeature(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_SKIP);
153
      vp9_disable_segfeature(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_ALT_LF);
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
      if (seg->enabled) {
        seg->update_data = 1;
        seg->update_map = 1;
      }
    }
    cpi->active_map.update = 0;
  }
}

int vp9_set_active_map(VP9_COMP* cpi,
                       unsigned char* new_map_16x16,
                       int rows,
                       int cols) {
  if (rows == cpi->common.mb_rows && cols == cpi->common.mb_cols) {
    unsigned char *const active_map_8x8 = cpi->active_map.map;
    const int mi_rows = cpi->common.mi_rows;
    const int mi_cols = cpi->common.mi_cols;
    cpi->active_map.update = 1;
    if (new_map_16x16) {
      int r, c;
      for (r = 0; r < mi_rows; ++r) {
        for (c = 0; c < mi_cols; ++c) {
          active_map_8x8[r * mi_cols + c] =
              new_map_16x16[(r >> 1) * cols + (c >> 1)]
                  ? AM_SEGMENT_ID_ACTIVE
                  : AM_SEGMENT_ID_INACTIVE;
        }
      }
      cpi->active_map.enabled = 1;
    } else {
      cpi->active_map.enabled = 0;
    }
    return 0;
  } else {
    return -1;
  }
}

192
193
194
195
196
197
198
199
200
int vp9_get_active_map(VP9_COMP* cpi,
                       unsigned char* new_map_16x16,
                       int rows,
                       int cols) {
  if (rows == cpi->common.mb_rows && cols == cpi->common.mb_cols &&
      new_map_16x16) {
    unsigned char* const seg_map_8x8 = cpi->segmentation_map;
    const int mi_rows = cpi->common.mi_rows;
    const int mi_cols = cpi->common.mi_cols;
James Zern's avatar
James Zern committed
201
    memset(new_map_16x16, !cpi->active_map.enabled, rows * cols);
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
    if (cpi->active_map.enabled) {
      int r, c;
      for (r = 0; r < mi_rows; ++r) {
        for (c = 0; c < mi_cols; ++c) {
          // Cyclic refresh segments are considered active despite not having
          // AM_SEGMENT_ID_ACTIVE
          new_map_16x16[(r >> 1) * cols + (c >> 1)] |=
              seg_map_8x8[r * mi_cols + c] != AM_SEGMENT_ID_INACTIVE;
        }
      }
    }
    return 0;
  } else {
    return -1;
  }
}

219
void vp9_set_high_precision_mv(VP9_COMP *cpi, int allow_high_precision_mv) {
220
  MACROBLOCK *const mb = &cpi->td.mb;
Deb Mukherjee's avatar
Deb Mukherjee committed
221
  cpi->common.allow_high_precision_mv = allow_high_precision_mv;
222
  if (cpi->common.allow_high_precision_mv) {
223
224
225
226
227
228
229
    mb->mvcost = mb->nmvcost_hp;
    mb->mvsadcost = mb->nmvsadcost_hp;
  } else {
    mb->mvcost = mb->nmvcost;
    mb->mvsadcost = mb->nmvsadcost;
  }
}
Paul Wilkins's avatar
Paul Wilkins committed
230

231
232
233
234
235
236
237
static void setup_frame(VP9_COMP *cpi) {
  VP9_COMMON *const cm = &cpi->common;
  // Set up entropy context depending on frame type. The decoder mandates
  // the use of the default context, index 0, for keyframes and inter
  // frames where the error_resilient_mode or intra_only flag is set. For
  // other inter-frames the encoder currently uses only two contexts;
  // context 1 for ALTREF frames and context 0 for the others.
238
239
240
241
242
243
244
  if (frame_is_intra_only(cm) || cm->error_resilient_mode) {
    vp9_setup_past_independence(cm);
  } else {
    if (!cpi->use_svc)
      cm->frame_context_idx = cpi->refresh_alt_ref_frame;
  }

245
  if (cm->frame_type == KEY_FRAME) {
246
    if (!is_two_pass_svc(cpi))
247
      cpi->refresh_golden_frame = 1;
248
    cpi->refresh_alt_ref_frame = 1;
249
    vp9_zero(cpi->interp_filter_selected);
250
  } else {
251
    *cm->fc = cm->frame_contexts[cm->frame_context_idx];
252
    vp9_zero(cpi->interp_filter_selected[0]);
253
254
255
  }
}

256
257
258
static void vp9_enc_setup_mi(VP9_COMMON *cm) {
  int i;
  cm->mi = cm->mip + cm->mi_stride + 1;
James Zern's avatar
James Zern committed
259
  memset(cm->mip, 0, cm->mi_stride * (cm->mi_rows + 1) * sizeof(*cm->mip));
260
261
  cm->prev_mi = cm->prev_mip + cm->mi_stride + 1;
  // Clear top border row
James Zern's avatar
James Zern committed
262
  memset(cm->prev_mip, 0, sizeof(*cm->prev_mip) * cm->mi_stride);
263
264
  // Clear left border column
  for (i = 1; i < cm->mi_rows + 1; ++i)
James Zern's avatar
James Zern committed
265
    memset(&cm->prev_mip[i * cm->mi_stride], 0, sizeof(*cm->prev_mip));
266
267
268
269

  cm->mi_grid_visible = cm->mi_grid_base + cm->mi_stride + 1;
  cm->prev_mi_grid_visible = cm->prev_mi_grid_base + cm->mi_stride + 1;

James Zern's avatar
James Zern committed
270
271
  memset(cm->mi_grid_base, 0,
         cm->mi_stride * (cm->mi_rows + 1) * sizeof(*cm->mi_grid_base));
272
273
274
275
276
277
278
279
280
281
}

static int vp9_enc_alloc_mi(VP9_COMMON *cm, int mi_size) {
  cm->mip = vpx_calloc(mi_size, sizeof(*cm->mip));
  if (!cm->mip)
    return 1;
  cm->prev_mip = vpx_calloc(mi_size, sizeof(*cm->prev_mip));
  if (!cm->prev_mip)
    return 1;
  cm->mi_alloc_size = mi_size;
282
283
284
285
286
287
288
289

  cm->mi_grid_base = (MODE_INFO **)vpx_calloc(mi_size, sizeof(MODE_INFO*));
  if (!cm->mi_grid_base)
    return 1;
  cm->prev_mi_grid_base = (MODE_INFO **)vpx_calloc(mi_size, sizeof(MODE_INFO*));
  if (!cm->prev_mi_grid_base)
    return 1;

290
291
292
293
294
295
296
297
  return 0;
}

static void vp9_enc_free_mi(VP9_COMMON *cm) {
  vpx_free(cm->mip);
  cm->mip = NULL;
  vpx_free(cm->prev_mip);
  cm->prev_mip = NULL;
298
299
300
301
  vpx_free(cm->mi_grid_base);
  cm->mi_grid_base = NULL;
  vpx_free(cm->prev_mi_grid_base);
  cm->prev_mi_grid_base = NULL;
302
303
304
305
}

static void vp9_swap_mi_and_prev_mi(VP9_COMMON *cm) {
  // Current mip will be the prev_mip for the next frame.
306
  MODE_INFO **temp_base = cm->prev_mi_grid_base;
307
308
309
310
311
312
313
  MODE_INFO *temp = cm->prev_mip;
  cm->prev_mip = cm->mip;
  cm->mip = temp;

  // Update the upper left visible macroblock ptrs.
  cm->mi = cm->mip + cm->mi_stride + 1;
  cm->prev_mi = cm->prev_mip + cm->mi_stride + 1;
314
315
316
317
318

  cm->prev_mi_grid_base = cm->mi_grid_base;
  cm->mi_grid_base = temp_base;
  cm->mi_grid_visible = cm->mi_grid_base + cm->mi_stride + 1;
  cm->prev_mi_grid_visible = cm->prev_mi_grid_base + cm->mi_stride + 1;
319
320
}

321
void vp9_initialize_enc(void) {
322
  static volatile int init_done = 0;
John Koleszar's avatar
John Koleszar committed
323
324

  if (!init_done) {
325
    vp9_rtcd();
Johann's avatar
Johann committed
326
    vpx_dsp_rtcd();
327
    vpx_scale_rtcd();
328
    vp9_init_intra_predictors();
329
    vp9_init_me_luts();
330
    vp9_rc_init_minq_luts();
331
    vp9_entropy_mv_init();
332
    vp9_temporal_filter_init();
John Koleszar's avatar
John Koleszar committed
333
334
    init_done = 1;
  }
John Koleszar's avatar
John Koleszar committed
335
336
}

337
static void dealloc_compressor_data(VP9_COMP *cpi) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
338
  VP9_COMMON *const cm = &cpi->common;
339
  int i;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
340

341
342
343
  vpx_free(cpi->mbmi_ext_base);
  cpi->mbmi_ext_base = NULL;

344
345
  vpx_free(cpi->tile_data);
  cpi->tile_data = NULL;
346

347
  // Delete sementation map
John Koleszar's avatar
John Koleszar committed
348
  vpx_free(cpi->segmentation_map);
Dmitry Kovalev's avatar
Dmitry Kovalev committed
349
  cpi->segmentation_map = NULL;
John Koleszar's avatar
John Koleszar committed
350
  vpx_free(cpi->coding_context.last_frame_seg_map_copy);
Dmitry Kovalev's avatar
Dmitry Kovalev committed
351
  cpi->coding_context.last_frame_seg_map_copy = NULL;
John Koleszar's avatar
John Koleszar committed
352

Jingning Han's avatar
Jingning Han committed
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
  vpx_free(cpi->nmvcosts[0]);
  vpx_free(cpi->nmvcosts[1]);
  cpi->nmvcosts[0] = NULL;
  cpi->nmvcosts[1] = NULL;

  vpx_free(cpi->nmvcosts_hp[0]);
  vpx_free(cpi->nmvcosts_hp[1]);
  cpi->nmvcosts_hp[0] = NULL;
  cpi->nmvcosts_hp[1] = NULL;

  vpx_free(cpi->nmvsadcosts[0]);
  vpx_free(cpi->nmvsadcosts[1]);
  cpi->nmvsadcosts[0] = NULL;
  cpi->nmvsadcosts[1] = NULL;

  vpx_free(cpi->nmvsadcosts_hp[0]);
  vpx_free(cpi->nmvsadcosts_hp[1]);
  cpi->nmvsadcosts_hp[0] = NULL;
  cpi->nmvsadcosts_hp[1] = NULL;

373
374
375
  vp9_cyclic_refresh_free(cpi->cyclic_refresh);
  cpi->cyclic_refresh = NULL;

376
377
378
  vpx_free(cpi->active_map.map);
  cpi->active_map.map = NULL;

379
380
381
382
  vp9_free_ref_frame_buffers(cm->buffer_pool);
#if CONFIG_VP9_POSTPROC
  vp9_free_postproc_buffers(cm);
#endif
383
  vp9_free_context_buffers(cm);
John Koleszar's avatar
John Koleszar committed
384

385
386
387
388
  vpx_free_frame_buffer(&cpi->last_frame_uf);
  vpx_free_frame_buffer(&cpi->scaled_source);
  vpx_free_frame_buffer(&cpi->scaled_last_source);
  vpx_free_frame_buffer(&cpi->alt_ref_buffer);
389
  vp9_lookahead_destroy(cpi->lookahead);
John Koleszar's avatar
John Koleszar committed
390

391
392
  vpx_free(cpi->tile_tok[0][0]);
  cpi->tile_tok[0][0] = 0;
John Koleszar's avatar
John Koleszar committed
393

394
  vp9_free_pc_tree(&cpi->td);
395

396
397
398
399
400
401
  for (i = 0; i < cpi->svc.number_spatial_layers; ++i) {
    LAYER_CONTEXT *const lc = &cpi->svc.layer_context[i];
    vpx_free(lc->rc_twopass_stats_in.buf);
    lc->rc_twopass_stats_in.buf = NULL;
    lc->rc_twopass_stats_in.sz = 0;
  }
402
403
404
405
406

  if (cpi->source_diff_var != NULL) {
    vpx_free(cpi->source_diff_var);
    cpi->source_diff_var = NULL;
  }
407

408
  for (i = 0; i < MAX_LAG_BUFFERS; ++i) {
409
    vpx_free_frame_buffer(&cpi->svc.scaled_frames[i]);
410
  }
James Zern's avatar
James Zern committed
411
412
  memset(&cpi->svc.scaled_frames[0], 0,
         MAX_LAG_BUFFERS * sizeof(cpi->svc.scaled_frames[0]));
413

414
  vpx_free_frame_buffer(&cpi->svc.empty_frame.img);
James Zern's avatar
James Zern committed
415
  memset(&cpi->svc.empty_frame, 0, sizeof(cpi->svc.empty_frame));
416
417

  vp9_free_svc_cyclic_refresh(cpi);
John Koleszar's avatar
John Koleszar committed
418
419
}

420
421
422
423
424
425
426
427
static void save_coding_context(VP9_COMP *cpi) {
  CODING_CONTEXT *const cc = &cpi->coding_context;
  VP9_COMMON *cm = &cpi->common;

  // Stores a snapshot of key state variables which can subsequently be
  // restored with a call to vp9_restore_coding_context. These functions are
  // intended for use in a re-code loop in vp9_compress_frame where the
  // quantizer value is adjusted between loop iterations.
428
  vp9_copy(cc->nmvjointcost,  cpi->td.mb.nmvjointcost);
Jingning Han's avatar
Jingning Han committed
429

James Zern's avatar
James Zern committed
430
431
432
433
434
435
436
437
  memcpy(cc->nmvcosts[0], cpi->nmvcosts[0],
         MV_VALS * sizeof(*cpi->nmvcosts[0]));
  memcpy(cc->nmvcosts[1], cpi->nmvcosts[1],
         MV_VALS * sizeof(*cpi->nmvcosts[1]));
  memcpy(cc->nmvcosts_hp[0], cpi->nmvcosts_hp[0],
         MV_VALS * sizeof(*cpi->nmvcosts_hp[0]));
  memcpy(cc->nmvcosts_hp[1], cpi->nmvcosts_hp[1],
         MV_VALS * sizeof(*cpi->nmvcosts_hp[1]));
438
439
440

  vp9_copy(cc->segment_pred_probs, cm->seg.pred_probs);

James Zern's avatar
James Zern committed
441
442
  memcpy(cpi->coding_context.last_frame_seg_map_copy,
         cm->last_frame_seg_map, (cm->mi_rows * cm->mi_cols));
443
444
445
446

  vp9_copy(cc->last_ref_lf_deltas, cm->lf.last_ref_deltas);
  vp9_copy(cc->last_mode_lf_deltas, cm->lf.last_mode_deltas);

447
  cc->fc = *cm->fc;
448
449
450
451
452
453
454
455
}

static void restore_coding_context(VP9_COMP *cpi) {
  CODING_CONTEXT *const cc = &cpi->coding_context;
  VP9_COMMON *cm = &cpi->common;

  // Restore key state variables to the snapshot state stored in the
  // previous call to vp9_save_coding_context.
456
  vp9_copy(cpi->td.mb.nmvjointcost, cc->nmvjointcost);
Jingning Han's avatar
Jingning Han committed
457

James Zern's avatar
James Zern committed
458
459
460
461
462
463
  memcpy(cpi->nmvcosts[0], cc->nmvcosts[0], MV_VALS * sizeof(*cc->nmvcosts[0]));
  memcpy(cpi->nmvcosts[1], cc->nmvcosts[1], MV_VALS * sizeof(*cc->nmvcosts[1]));
  memcpy(cpi->nmvcosts_hp[0], cc->nmvcosts_hp[0],
         MV_VALS * sizeof(*cc->nmvcosts_hp[0]));
  memcpy(cpi->nmvcosts_hp[1], cc->nmvcosts_hp[1],
         MV_VALS * sizeof(*cc->nmvcosts_hp[1]));
464
465
466

  vp9_copy(cm->seg.pred_probs, cc->segment_pred_probs);

James Zern's avatar
James Zern committed
467
468
469
  memcpy(cm->last_frame_seg_map,
         cpi->coding_context.last_frame_seg_map_copy,
         (cm->mi_rows * cm->mi_cols));
470
471
472
473

  vp9_copy(cm->lf.last_ref_deltas, cc->last_ref_lf_deltas);
  vp9_copy(cm->lf.last_mode_deltas, cc->last_mode_lf_deltas);

474
  *cm->fc = cc->fc;
475
476
}

477
static void configure_static_seg_features(VP9_COMP *cpi) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
478
  VP9_COMMON *const cm = &cpi->common;
479
  const RATE_CONTROL *const rc = &cpi->rc;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
480
  struct segmentation *const seg = &cm->seg;
481

482
  int high_q = (int)(rc->avg_q > 48.0);
John Koleszar's avatar
John Koleszar committed
483
  int qi_delta;
484

John Koleszar's avatar
John Koleszar committed
485
486
487
  // Disable and clear down for KF
  if (cm->frame_type == KEY_FRAME) {
    // Clear down the global segmentation map
James Zern's avatar
James Zern committed
488
    memset(cpi->segmentation_map, 0, cm->mi_rows * cm->mi_cols);
489
490
    seg->update_map = 0;
    seg->update_data = 0;
John Koleszar's avatar
John Koleszar committed
491
    cpi->static_mb_pct = 0;
492

John Koleszar's avatar
John Koleszar committed
493
    // Disable segmentation
494
    vp9_disable_segmentation(seg);
Paul Wilkins's avatar
Paul Wilkins committed
495

John Koleszar's avatar
John Koleszar committed
496
    // Clear down the segment features.
497
    vp9_clearall_segfeatures(seg);
498
499
  } else if (cpi->refresh_alt_ref_frame) {
    // If this is an alt ref frame
John Koleszar's avatar
John Koleszar committed
500
    // Clear down the global segmentation map
James Zern's avatar
James Zern committed
501
    memset(cpi->segmentation_map, 0, cm->mi_rows * cm->mi_cols);
502
503
    seg->update_map = 0;
    seg->update_data = 0;
John Koleszar's avatar
John Koleszar committed
504
    cpi->static_mb_pct = 0;
Paul Wilkins's avatar
Paul Wilkins committed
505

John Koleszar's avatar
John Koleszar committed
506
    // Disable segmentation and individual segment features by default
507
    vp9_disable_segmentation(seg);
508
    vp9_clearall_segfeatures(seg);
Paul Wilkins's avatar
Paul Wilkins committed
509

John Koleszar's avatar
John Koleszar committed
510
511
    // Scan frames from current to arf frame.
    // This function re-enables segmentation if appropriate.
512
    vp9_update_mbgraph_stats(cpi);
Paul Wilkins's avatar
Paul Wilkins committed
513

John Koleszar's avatar
John Koleszar committed
514
515
    // If segmentation was enabled set those features needed for the
    // arf itself.
516
517
518
    if (seg->enabled) {
      seg->update_map = 1;
      seg->update_data = 1;
Paul Wilkins's avatar
Paul Wilkins committed
519

520
521
      qi_delta = vp9_compute_qdelta(rc, rc->avg_q, rc->avg_q * 0.875,
                                    cm->bit_depth);
522
      vp9_set_segdata(seg, 1, SEG_LVL_ALT_Q, qi_delta - 2);
523
      vp9_set_segdata(seg, 1, SEG_LVL_ALT_LF, -2);
Paul Wilkins's avatar
Paul Wilkins committed
524

525
526
      vp9_enable_segfeature(seg, 1, SEG_LVL_ALT_Q);
      vp9_enable_segfeature(seg, 1, SEG_LVL_ALT_LF);
527

John Koleszar's avatar
John Koleszar committed
528
      // Where relevant assume segment data is delta data
529
      seg->abs_delta = SEGMENT_DELTADATA;
530
    }
531
  } else if (seg->enabled) {
532
533
    // All other frames if segmentation has been enabled

John Koleszar's avatar
John Koleszar committed
534
    // First normal frame in a valid gf or alt ref group
535
    if (rc->frames_since_golden == 0) {
Paul Wilkins's avatar
Paul Wilkins committed
536
      // Set up segment features for normal frames in an arf group
537
      if (rc->source_alt_ref_active) {
538
539
540
        seg->update_map = 0;
        seg->update_data = 1;
        seg->abs_delta = SEGMENT_DELTADATA;
Paul Wilkins's avatar
Paul Wilkins committed
541

542
543
        qi_delta = vp9_compute_qdelta(rc, rc->avg_q, rc->avg_q * 1.125,
                                      cm->bit_depth);
544
        vp9_set_segdata(seg, 1, SEG_LVL_ALT_Q, qi_delta + 2);
545
        vp9_enable_segfeature(seg, 1, SEG_LVL_ALT_Q);
Paul Wilkins's avatar
Paul Wilkins committed
546

547
548
        vp9_set_segdata(seg, 1, SEG_LVL_ALT_LF, -2);
        vp9_enable_segfeature(seg, 1, SEG_LVL_ALT_LF);
Paul Wilkins's avatar
Paul Wilkins committed
549

John Koleszar's avatar
John Koleszar committed
550
551
        // Segment coding disabled for compred testing
        if (high_q || (cpi->static_mb_pct == 100)) {
552
553
554
          vp9_set_segdata(seg, 1, SEG_LVL_REF_FRAME, ALTREF_FRAME);
          vp9_enable_segfeature(seg, 1, SEG_LVL_REF_FRAME);
          vp9_enable_segfeature(seg, 1, SEG_LVL_SKIP);
John Koleszar's avatar
John Koleszar committed
555
        }
556
557
558
559
      } else {
        // Disable segmentation and clear down features if alt ref
        // is not active for this group

560
        vp9_disable_segmentation(seg);
Paul Wilkins's avatar
Paul Wilkins committed
561

James Zern's avatar
James Zern committed
562
        memset(cpi->segmentation_map, 0, cm->mi_rows * cm->mi_cols);
Paul Wilkins's avatar
Paul Wilkins committed
563

564
565
        seg->update_map = 0;
        seg->update_data = 0;
Paul Wilkins's avatar
Paul Wilkins committed
566

567
        vp9_clearall_segfeatures(seg);
John Koleszar's avatar
John Koleszar committed
568
      }
569
    } else if (rc->is_src_frame_alt_ref) {
570
571
572
      // Special case where we are coding over the top of a previous
      // alt ref frame.
      // Segment coding disabled for compred testing
John Koleszar's avatar
John Koleszar committed
573

Paul Wilkins's avatar
Paul Wilkins committed
574
      // Enable ref frame features for segment 0 as well
575
576
      vp9_enable_segfeature(seg, 0, SEG_LVL_REF_FRAME);
      vp9_enable_segfeature(seg, 1, SEG_LVL_REF_FRAME);
John Koleszar's avatar
John Koleszar committed
577

Paul Wilkins's avatar
Paul Wilkins committed
578
      // All mbs should use ALTREF_FRAME
579
580
581
582
      vp9_clear_segdata(seg, 0, SEG_LVL_REF_FRAME);
      vp9_set_segdata(seg, 0, SEG_LVL_REF_FRAME, ALTREF_FRAME);
      vp9_clear_segdata(seg, 1, SEG_LVL_REF_FRAME);
      vp9_set_segdata(seg, 1, SEG_LVL_REF_FRAME, ALTREF_FRAME);
John Koleszar's avatar
John Koleszar committed
583

Paul Wilkins's avatar
Paul Wilkins committed
584
      // Skip all MBs if high Q (0,0 mv and skip coeffs)
John Koleszar's avatar
John Koleszar committed
585
      if (high_q) {
586
587
        vp9_enable_segfeature(seg, 0, SEG_LVL_SKIP);
        vp9_enable_segfeature(seg, 1, SEG_LVL_SKIP);
John Koleszar's avatar
John Koleszar committed
588
      }
Adrian Grange's avatar
Adrian Grange committed
589
      // Enable data update
590
      seg->update_data = 1;
591
592
593
    } else {
      // All other frames.

John Koleszar's avatar
John Koleszar committed
594
      // No updates.. leave things as they are.
595
596
      seg->update_map = 0;
      seg->update_data = 0;
John Koleszar's avatar
John Koleszar committed
597
598
    }
  }
599
600
}

601
static void update_reference_segmentation_map(VP9_COMP *cpi) {
Ronald S. Bultje's avatar
Ronald S. Bultje committed
602
  VP9_COMMON *const cm = &cpi->common;
603
  MODE_INFO **mi_8x8_ptr = cm->mi_grid_visible;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
604
  uint8_t *cache_ptr = cm->last_frame_seg_map;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
605
606
  int row, col;

607
  for (row = 0; row < cm->mi_rows; row++) {
608
    MODE_INFO **mi_8x8 = mi_8x8_ptr;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
609
    uint8_t *cache = cache_ptr;
610
    for (col = 0; col < cm->mi_cols; col++, mi_8x8++, cache++)
611
      cache[0] = mi_8x8[0]->mbmi.segment_id;
612
    mi_8x8_ptr += cm->mi_stride;
613
    cache_ptr += cm->mi_cols;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
614
615
616
  }
}

617
static void alloc_raw_frame_buffers(VP9_COMP *cpi) {
618
  VP9_COMMON *cm = &cpi->common;
619
  const VP9EncoderConfig *oxcf = &cpi->oxcf;
620

621
622
623
  if (!cpi->lookahead)
    cpi->lookahead = vp9_lookahead_init(oxcf->width, oxcf->height,
                                        cm->subsampling_x, cm->subsampling_y,
624
625
626
#if CONFIG_VP9_HIGHBITDEPTH
                                      cm->use_highbitdepth,
#endif
Dmitry Kovalev's avatar
Dmitry Kovalev committed
627
                                      oxcf->lag_in_frames);
John Koleszar's avatar
John Koleszar committed
628
  if (!cpi->lookahead)
629
    vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
John Koleszar's avatar
John Koleszar committed
630
                       "Failed to allocate lag buffers");
John Koleszar's avatar
John Koleszar committed
631

632
  // TODO(agrange) Check if ARF is enabled and skip allocation if not.
633
  if (vpx_realloc_frame_buffer(&cpi->alt_ref_buffer,
Dmitry Kovalev's avatar
Dmitry Kovalev committed
634
                               oxcf->width, oxcf->height,
635
                               cm->subsampling_x, cm->subsampling_y,
636
637
638
#if CONFIG_VP9_HIGHBITDEPTH
                               cm->use_highbitdepth,
#endif
639
640
                               VP9_ENC_BORDER_IN_PIXELS, cm->byte_alignment,
                               NULL, NULL, NULL))
641
    vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
John Koleszar's avatar
John Koleszar committed
642
                       "Failed to allocate altref buffer");
John Koleszar's avatar
John Koleszar committed
643
}
644

645
static void alloc_util_frame_buffers(VP9_COMP *cpi) {
646
  VP9_COMMON *const cm = &cpi->common;
647
  if (vpx_realloc_frame_buffer(&cpi->last_frame_uf,
648
649
                               cm->width, cm->height,
                               cm->subsampling_x, cm->subsampling_y,
650
651
652
#if CONFIG_VP9_HIGHBITDEPTH
                               cm->use_highbitdepth,
#endif
653
654
                               VP9_ENC_BORDER_IN_PIXELS, cm->byte_alignment,
                               NULL, NULL, NULL))
655
    vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
656
                       "Failed to allocate last frame buffer");
657

658
  if (vpx_realloc_frame_buffer(&cpi->scaled_source,
659
660
                               cm->width, cm->height,
                               cm->subsampling_x, cm->subsampling_y,
661
662
663
#if CONFIG_VP9_HIGHBITDEPTH
                               cm->use_highbitdepth,
#endif
664
665
                               VP9_ENC_BORDER_IN_PIXELS, cm->byte_alignment,
                               NULL, NULL, NULL))
666
    vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
667
                       "Failed to allocate scaled source buffer");
668

669
  if (vpx_realloc_frame_buffer(&cpi->scaled_last_source,
670
671
                               cm->width, cm->height,
                               cm->subsampling_x, cm->subsampling_y,
672
673
674
#if CONFIG_VP9_HIGHBITDEPTH
                               cm->use_highbitdepth,
#endif
675
676
                               VP9_ENC_BORDER_IN_PIXELS, cm->byte_alignment,
                               NULL, NULL, NULL))
677
    vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
678
679
                       "Failed to allocate scaled last source buffer");
}
680

681
682
683
684
685
686
687
688
689
690
691
692

static int alloc_context_buffers_ext(VP9_COMP *cpi) {
  VP9_COMMON *cm = &cpi->common;
  int mi_size = cm->mi_cols * cm->mi_rows;

  cpi->mbmi_ext_base = vpx_calloc(mi_size, sizeof(*cpi->mbmi_ext_base));
  if (!cpi->mbmi_ext_base)
    return 1;

  return 0;
}

693
static void alloc_compressor_data(VP9_COMP *cpi) {
694
  VP9_COMMON *cm = &cpi->common;
695

696
697
  vp9_alloc_context_buffers(cm, cm->width, cm->height);

698
699
  alloc_context_buffers_ext(cpi);

700
  vpx_free(cpi->tile_tok[0][0]);
701
702
703

  {
    unsigned int tokens = get_token_alloc(cm->mb_rows, cm->mb_cols);
704
705
    CHECK_MEM_ERROR(cm, cpi->tile_tok[0][0],
        vpx_calloc(tokens, sizeof(*cpi->tile_tok[0][0])));
706
  }
707

708
  vp9_setup_pc_tree(&cpi->common, &cpi->td);
709
710
}

711
712
713
714
715
716
717
718
719
720
721
void vp9_new_framerate(VP9_COMP *cpi, double framerate) {
  cpi->framerate = framerate < 0.1 ? 30 : framerate;
  vp9_rc_update_framerate(cpi);
}

static void set_tile_limits(VP9_COMP *cpi) {
  VP9_COMMON *const cm = &cpi->common;

  int min_log2_tile_cols, max_log2_tile_cols;
  vp9_get_tile_n_bits(cm->mi_cols, &min_log2_tile_cols, &max_log2_tile_cols);

722
723
724
  if (is_two_pass_svc(cpi) &&
      (cpi->svc.encode_empty_frame_state == ENCODING ||
      cpi->svc.number_spatial_layers > 1)) {
725
726
727
728
729
730
731
732
733
    cm->log2_tile_cols = 0;
    cm->log2_tile_rows = 0;
  } else {
    cm->log2_tile_cols = clamp(cpi->oxcf.tile_columns,
                               min_log2_tile_cols, max_log2_tile_cols);
    cm->log2_tile_rows = cpi->oxcf.tile_rows;
  }
}

734
735
static void update_frame_size(VP9_COMP *cpi) {
  VP9_COMMON *const cm = &cpi->common;
736
  MACROBLOCKD *const xd = &cpi->td.mb.e_mbd;
Adrian Grange's avatar
Adrian Grange committed
737
738
739

  vp9_set_mb_mi(cm, cm->width, cm->height);
  vp9_init_context_buffers(cm);
740
  vp9_init_macroblockd(cm, xd, NULL);
741
742
743
  cpi->td.mb.mbmi_ext_base = cpi->mbmi_ext_base;
  memset(cpi->mbmi_ext_base, 0,
         cm->mi_rows * cm->mi_cols * sizeof(*cpi->mbmi_ext_base));
744

745
746
  set_tile_limits(cpi);

747
  if (is_two_pass_svc(cpi)) {
748
    if (vpx_realloc_frame_buffer(&cpi->alt_ref_buffer,
749
750
                                 cm->width, cm->height,
                                 cm->subsampling_x, cm->subsampling_y,
751
752
753
#if CONFIG_VP9_HIGHBITDEPTH
                                 cm->use_highbitdepth,
#endif
754
755
                                 VP9_ENC_BORDER_IN_PIXELS, cm->byte_alignment,
                                 NULL, NULL, NULL))
756
757
758
      vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
                         "Failed to reallocate alt_ref_buffer");
  }
759
760
}

761
762
763
764
765
766
static void init_buffer_indices(VP9_COMP *cpi) {
  cpi->lst_fb_idx = 0;
  cpi->gld_fb_idx = 1;
  cpi->alt_fb_idx = 2;
}

767
static void init_config(struct VP9_COMP *cpi, VP9EncoderConfig *oxcf) {
768
  VP9_COMMON *const cm = &cpi->common;
John Koleszar's avatar
John Koleszar committed
769

John Koleszar's avatar
John Koleszar committed
770
  cpi->oxcf = *oxcf;
771
  cpi->framerate = oxcf->init_framerate;
John Koleszar's avatar
John Koleszar committed
772

773
774
  cm->profile = oxcf->profile;
  cm->bit_depth = oxcf->bit_depth;
775
776
777
#if CONFIG_VP9_HIGHBITDEPTH
  cm->use_highbitdepth = oxcf->use_highbitdepth;
#endif
778
  cm->color_space = oxcf->color_space;
779
  cm->color_range = oxcf->color_range;
John Koleszar's avatar
John Koleszar committed
780

781
782
  cm->width = oxcf->width;
  cm->height = oxcf->height;
783
  alloc_compressor_data(cpi);
784

785
786
  cpi->svc.temporal_layering_mode = oxcf->temporal_layering_mode;

787
788
789
  // Single thread case: use counts in common.
  cpi->td.counts = &cm->counts;

790
791
792
793
794
  // Spatial scalability.
  cpi->svc.number_spatial_layers = oxcf->ss_number_layers;
  // Temporal scalability.
  cpi->svc.number_temporal_layers = oxcf->ts_number_layers;

795
  if ((cpi->svc.number_temporal_layers > 1 && cpi->oxcf.rc_mode == VPX_CBR) ||
796
797
      ((cpi->svc.number_temporal_layers > 1 ||
        cpi->svc.number_spatial_layers > 1) &&
798
       cpi->oxcf.pass != 1)) {
799
    vp9_init_layer_context(cpi);
800
801
  }

John Koleszar's avatar
John Koleszar committed
802
  // change includes all joint functionality
Dmitry Kovalev's avatar
Dmitry Kovalev committed
803
  vp9_change_config(cpi, oxcf);
John Koleszar's avatar
John Koleszar committed
804

John Koleszar's avatar
John Koleszar committed
805
  cpi->static_mb_pct = 0;
806
  cpi->ref_frame_flags = 0;
807

808
  init_buffer_indices(cpi);
809
810

  vp9_noise_estimate_init(&cpi->noise_estimate, cm->width, cm->height);
John Koleszar's avatar
John Koleszar committed
811
812
}

Dmitry Kovalev's avatar
Dmitry Kovalev committed
813
814
815
816
817
818
819
820
821
822
823
824
825
826
static void set_rc_buffer_sizes(RATE_CONTROL *rc,
                                const VP9EncoderConfig *oxcf) {
  const int64_t bandwidth = oxcf->target_bandwidth;
  const int64_t starting = oxcf->starting_buffer_level_ms;
  const int64_t optimal = oxcf->optimal_buffer_level_ms;
  const int64_t maximum = oxcf->maximum_buffer_size_ms;

  rc->starting_buffer_level = starting * bandwidth / 1000;
  rc->optimal_buffer_level = (optimal == 0) ? bandwidth / 8
                                            : optimal * bandwidth / 1000;
  rc->maximum_buffer_size = (maximum == 0) ? bandwidth / 8
                                           : maximum * bandwidth / 1000;
}

827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
#if CONFIG_VP9_HIGHBITDEPTH
#define HIGHBD_BFP(BT, SDF, SDAF, VF, SVF, SVAF, SDX3F, SDX8F, SDX4DF) \
    cpi->fn_ptr[BT].sdf = SDF; \
    cpi->fn_ptr[BT].sdaf = SDAF; \
    cpi->fn_ptr[BT].vf = VF; \
    cpi->fn_ptr[BT].svf = SVF; \
    cpi->fn_ptr[BT].svaf = SVAF; \
    cpi->fn_ptr[BT].sdx3f = SDX3F; \
    cpi->fn_ptr[BT].sdx8f = SDX8F; \
    cpi->fn_ptr[BT].sdx4df = SDX4DF;

#define MAKE_BFP_SAD_WRAPPER(fnname) \
static unsigned int fnname##_bits8(const uint8_t *src_ptr, \
                                   int source_stride, \
                                   const uint8_t *ref_ptr, \
                                   int ref_stride) {  \
  return fnname(src_ptr, source_stride, ref_ptr, ref_stride); \
} \
static unsigned int fnname##_bits10(const uint8_t *src_ptr, \
                                    int source_stride, \
                                    const uint8_t *ref_ptr, \
                                    int ref_stride) {  \
  return fnname(src_ptr, source_stride, ref_ptr, ref_stride) >> 2; \
} \
static unsigned int fnname##_bits12(const uint8_t *src_ptr, \
                                    int source_stride, \
                                    const uint8_t *ref_ptr, \
                                    int ref_stride) {  \
  return fnname(src_ptr, source_stride, ref_ptr, ref_stride) >> 4; \
}

#define MAKE_BFP_SADAVG_WRAPPER(fnname) static unsigned int \
fnname##_bits8(const uint8_t *src_ptr, \
               int source_stride, \
               const uint8_t *ref_ptr, \
               int ref_stride, \
               const uint8_t *second_pred) {  \
  return fnname(src_ptr, source_stride, ref_ptr, ref_stride, second_pred); \
} \
static unsigned int fnname##_bits10(const uint8_t *src_ptr, \
                                    int source_stride, \
                                    const uint8_t *ref_ptr, \
                                    int ref_stride, \
                                    const uint8_t *second_pred) {  \
  return fnname(src_ptr, source_stride, ref_ptr, ref_stride, \
                second_pred) >> 2; \
} \
static unsigned int fnname##_bits12(const uint8_t *src_ptr, \
                                    int source_stride, \
                                    const uint8_t *ref_ptr, \
                                    int ref_stride, \
                                    const uint8_t *second_pred) {  \
  return fnname(src_ptr, source_stride, ref_ptr, ref_stride, \
                second_pred) >> 4; \
}

#define MAKE_BFP_SAD3_WRAPPER(fnname) \
static void fnname##_bits8(const uint8_t *src_ptr, \
                           int source_stride, \
                           const uint8_t *ref_ptr, \
                           int  ref_stride, \
                           unsigned int *sad_array) {  \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
} \
static void fnname##_bits10(const uint8_t *src_ptr, \
                            int source_stride, \
                            const uint8_t *ref_ptr, \
                            int  ref_stride, \
                            unsigned int *sad_array) {  \
  int i; \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
  for (i = 0; i < 3; i++) \
    sad_array[i] >>= 2; \
} \
static void fnname##_bits12(const uint8_t *src_ptr, \
                            int source_stride, \
                            const uint8_t *ref_ptr, \
                            int  ref_stride, \
                            unsigned int *sad_array) {  \
  int i; \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
  for (i = 0; i < 3; i++) \
    sad_array[i] >>= 4; \
}

#define MAKE_BFP_SAD8_WRAPPER(fnname) \
static void fnname##_bits8(const uint8_t *src_ptr, \
                           int source_stride, \
                           const uint8_t *ref_ptr, \
                           int  ref_stride, \
                           unsigned int *sad_array) {  \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
} \
static void fnname##_bits10(const uint8_t *src_ptr, \
                            int source_stride, \
                            const uint8_t *ref_ptr, \
                            int  ref_stride, \
                            unsigned int *sad_array) {  \
  int i; \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
  for (i = 0; i < 8; i++) \
    sad_array[i] >>= 2; \
} \
static void fnname##_bits12(const uint8_t *src_ptr, \
                            int source_stride, \
                            const uint8_t *ref_ptr, \
                            int  ref_stride, \
                            unsigned int *sad_array) {  \
  int i; \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
  for (i = 0; i < 8; i++) \
    sad_array[i] >>= 4; \
}
#define MAKE_BFP_SAD4D_WRAPPER(fnname) \
static void fnname##_bits8(const uint8_t *src_ptr, \
                           int source_stride, \
                           const uint8_t* const ref_ptr[], \
                           int  ref_stride, \
                           unsigned int *sad_array) {  \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
} \
static void fnname##_bits10(const uint8_t *src_ptr, \
                            int source_stride, \
                            const uint8_t* const ref_ptr[], \
                            int  ref_stride, \
                            unsigned int *sad_array) {  \
  int i; \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
  for (i = 0; i < 4; i++) \
  sad_array[i] >>= 2; \
} \
static void fnname##_bits12(const uint8_t *src_ptr, \
                            int source_stride, \
                            const uint8_t* const ref_ptr[], \
                            int  ref_stride, \
                            unsigned int *sad_array) {  \
  int i; \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
  for (i = 0; i < 4; i++) \
  sad_array[i] >>= 4; \
}

Johann's avatar
Johann committed
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
MAKE_BFP_SAD_WRAPPER(vpx_highbd_sad32x16)
MAKE_BFP_SADAVG_WRAPPER(vpx_highbd_sad32x16_avg)
MAKE_BFP_SAD4D_WRAPPER(vpx_highbd_sad32x16x4d)
MAKE_BFP_SAD_WRAPPER(vpx_highbd_sad16x32)
MAKE_BFP_SADAVG_WRAPPER(vpx_highbd_sad16x32_avg)
MAKE_BFP_SAD4D_WRAPPER(vpx_highbd_sad16x32x4d)
MAKE_BFP_SAD_WRAPPER(vpx_highbd_sad64x32)
MAKE_BFP_SADAVG_WRAPPER(vpx_highbd_sad64x32_avg)
MAKE_BFP_SAD4D_WRAPPER(vpx_highbd_sad64x32x4d)
MAKE_BFP_SAD_WRAPPER(vpx_highbd_sad32x64)
MAKE_BFP_SADAVG_WRAPPER(vpx_highbd_sad32x64_avg)
MAKE_BFP_SAD4D_WRAPPER(vpx_highbd_sad32x64x4d)
MAKE_BFP_SAD_WRAPPER(vpx_highbd_sad32x32)
MAKE_BFP_SADAVG_WRAPPER(vpx_highbd_sad32x32_avg)
MAKE_BFP_SAD3_WRAPPER(vpx_highbd_sad32x32x3)
MAKE_BFP_SAD8_WRAPPER(vpx_highbd_sad32x32x8)
MAKE_BFP_SAD4D_WRAPPER(vpx_highbd_sad32x32x4d)
MAKE_BFP_SAD_WRAPPER(vpx_highbd_sad64x64)
MAKE_BFP_SADAVG_WRAPPER(vpx_highbd_sad64x64_avg)
MAKE_BFP_SAD3_WRAPPER(vpx_highbd_sad64x64x3)
MAKE_BFP_SAD8_WRAPPER(vpx_highbd_sad64x64x8)
MAKE_BFP_SAD4D_WRAPPER(vpx_highbd_sad64x64x4d)
MAKE_BFP_SAD_WRAPPER(vpx_highbd_sad16x16)
MAKE_BFP_SADAVG_WRAPPER(vpx_highbd_sad16x16_avg)
MAKE_BFP_SAD3_WRAPPER(vpx_highbd_sad16x16x3)
MAKE_BFP_SAD8_WRAPPER(vpx_highbd_sad16x16x8)
MAKE_BFP_SAD4D_WRAPPER(vpx_highbd_sad16x16x4d)
MAKE_BFP_SAD_WRAPPER(vpx_highbd_sad16x8)
MAKE_BFP_SADAVG_WRAPPER(vpx_highbd_sad16x8_avg)
MAKE_BFP_SAD3_WRAPPER(vpx_highbd_sad16x8x3)
MAKE_BFP_SAD8_WRAPPER(vpx_highbd_sad16x8x8)
MAKE_BFP_SAD4D_WRAPPER(vpx_highbd_sad16x8x4d)
MAKE_BFP_SAD_WRAPPER(vpx_highbd_sad8x16)
MAKE_BFP_SADAVG_WRAPPER(vpx_highbd_sad8x16_avg)
MAKE_BFP_SAD3_WRAPPER(vpx_highbd_sad8x16x3)
MAKE_BFP_SAD8_WRAPPER(vpx_highbd_sad8x16x8)
MAKE_BFP_SAD4D_WRAPPER(vpx_highbd_sad8x16x4d)
MAKE_BFP_SAD_WRAPPER(vpx_highbd_sad8x8)
MAKE_BFP_SADAVG_WRAPPER(vpx_highbd_sad8x8_avg)
MAKE_BFP_SAD3_WRAPPER(vpx_highbd_sad8x8x3)
MAKE_BFP_SAD8_WRAPPER(vpx_highbd_sad8x8x8)
MAKE_BFP_SAD4D_WRAPPER(vpx_highbd_sad8x8x4d)
MAKE_BFP_SAD_WRAPPER(vpx_highbd_sad8x4)
MAKE_BFP_SADAVG_WRAPPER(vpx_highbd_sad8x4_avg)
MAKE_BFP_SAD8_WRAPPER(vpx_highbd_sad8x4x8)
MAKE_BFP_SAD4D_WRAPPER(vpx_highbd_sad8x4x4d)
MAKE_BFP_SAD_WRAPPER(vpx_highbd_sad4x8)
MAKE_BFP_SADAVG_WRAPPER(vpx_highbd_sad4x8_avg)
MAKE_BFP_SAD8_WRAPPER(vpx_highbd_sad4x8x8)
MAKE_BFP_SAD4D_WRAPPER(vpx_highbd_sad4x8x4d)
MAKE_BFP_SAD_WRAPPER(vpx_highbd_sad4x4)
MAKE_BFP_SADAVG_WRAPPER(vpx_highbd_sad4x4_avg)
MAKE_BFP_SAD3_WRAPPER(vpx_highbd_sad4x4x3)
MAKE_BFP_SAD8_WRAPPER(vpx_highbd_sad4x4x8)
MAKE_BFP_SAD4D_WRAPPER(vpx_highbd_sad4x4x4d)
1024
1025

static void  highbd_set_var_fns(VP9_COMP *const cpi) {
1026
1027
1028
1029
1030
  VP9_COMMON *const cm = &cpi->common;
  if (cm->use_highbitdepth) {
    switch (cm->bit_depth) {
      case VPX_BITS_8:
        HIGHBD_BFP(BLOCK_32X16,
Johann's avatar
Johann committed
1031
1032
                   vpx_highbd_sad32x16_bits8,
                   vpx_highbd_sad32x16_avg_bits8,
Johann's avatar
Johann committed
1033
                   vpx_highbd_8_variance32x16,
Johann's avatar
Johann committed
1034
1035
                   vpx_highbd_8_sub_pixel_variance32x16,
                   vpx_highbd_8_sub_pixel_avg_variance32x16,
1036
1037
                   NULL,
                   NULL,
Johann's avatar
Johann committed
1038
                   vpx_highbd_sad32x16x4d_bits8)
1039
1040

        HIGHBD_BFP(BLOCK_16X32,
Johann's avatar
Johann committed
1041
1042
                   vpx_highbd_sad16x32_bits8,
                   vpx_highbd_sad16x32_avg_bits8,
Johann's avatar
Johann committed
1043
                   vpx_highbd_8_variance16x32,
Johann's avatar
Johann committed
1044
1045
                   vpx_highbd_8_sub_pixel_variance16x32,
                   vpx_highbd_8_sub_pixel_avg_variance16x32,
1046
1047
                   NULL,
                   NULL,
Johann's avatar
Johann committed
1048
                   vpx_highbd_sad16x32x4d_bits8)
1049
1050

        HIGHBD_BFP(BLOCK_64X32,
Johann's avatar
Johann committed
1051
1052
                   vpx_highbd_sad64x32_bits8,
                   vpx_highbd_sad64x32_avg_bits8,
Johann's avatar
Johann committed
1053
                   vpx_highbd_8_variance64x32,
Johann's avatar
Johann committed
1054
1055
                   vpx_highbd_8_sub_pixel_variance64x32,
                   vpx_highbd_8_sub_pixel_avg_variance64x32,
1056
1057
                   NULL,
                   NULL,
Johann's avatar
Johann committed
1058
                   vpx_highbd_sad64x32x4d_bits8)
1059
1060

        HIGHBD_BFP(BLOCK_32X64,
Johann's avatar
Johann committed
1061
1062
                   vpx_highbd_sad32x64_bits8,
                   vpx_highbd_sad32x64_avg_bits8,
Johann's avatar
Johann committed
1063
                   vpx_highbd_8_variance32x64,
Johann's avatar
Johann committed
1064
1065
                   vpx_highbd_8_sub_pixel_variance32x64,
                   vpx_highbd_8_sub_pixel_avg_variance32x64,
1066
1067
                   NULL,
                   NULL,
Johann's avatar
Johann committed
1068
                   vpx_highbd_sad32x64x4d_bits8)
1069
1070

        HIGHBD_BFP(BLOCK_32X32,
Johann's avatar
Johann committed
1071
1072
                   vpx_highbd_sad32x32_bits8,
                   vpx_highbd_sad32x32_avg_bits8,
Johann's avatar
Johann committed
1073
                   vpx_highbd_8_variance32x32,
Johann's avatar
Johann committed
1074
1075
                   vpx_highbd_8_sub_pixel_variance32x32,
                   vpx_highbd_8_sub_pixel_avg_variance32x32,
Johann's avatar
Johann committed
1076
1077
1078
                   vpx_highbd_sad32x32x3_bits8,
                   vpx_highbd_sad32x32x8_bits8,
                   vpx_highbd_sad32x32x4d_bits8)
1079
1080

        HIGHBD_BFP(BLOCK_64X64,
Johann's avatar
Johann committed
1081
1082
                   vpx_highbd_sad64x64_bits8,
                   vpx_highbd_sad64x64_avg_bits8,
Johann's avatar
Johann committed
1083
                   vpx_highbd_8_variance64x64,
Johann's avatar
Johann committed
1084
1085
                   vpx_highbd_8_sub_pixel_variance64x64,
                   vpx_highbd_8_sub_pixel_avg_variance64x64,
Johann's avatar
Johann committed
1086
1087
1088
                   vpx_highbd_sad64x64x3_bits8,
                   vpx_highbd_sad64x64x8_bits8,
                   vpx_highbd_sad64x64x4d_bits8)
1089
1090

        HIGHBD_BFP(BLOCK_16X16,