vp9_encoder.c 154 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 * Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5 6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9 10
 */

Jim Bankoski's avatar
Jim Bankoski committed
11 12 13 14 15
#include <math.h>
#include <stdio.h>
#include <limits.h>

#include "./vpx_config.h"
16
#include "./vp9_rtcd.h"
Johann's avatar
Johann committed
17
#include "./vpx_dsp_rtcd.h"
Jim Bankoski's avatar
Jim Bankoski committed
18
#include "./vpx_scale_rtcd.h"
19
#include "vpx/internal/vpx_psnr.h"
20
#include "vpx_ports/mem.h"
21
#include "vpx_ports/vpx_timer.h"
John Koleszar's avatar
John Koleszar committed
22

Jim Bankoski's avatar
Jim Bankoski committed
23
#include "vp9/common/vp9_alloccommon.h"
24
#include "vp9/common/vp9_filter.h"
25
#include "vp9/common/vp9_idct.h"
Jim Bankoski's avatar
Jim Bankoski committed
26 27 28
#if CONFIG_VP9_POSTPROC
#include "vp9/common/vp9_postproc.h"
#endif
29
#include "vp9/common/vp9_reconinter.h"
30
#include "vp9/common/vp9_reconintra.h"
31
#include "vp9/common/vp9_systemdependent.h"
Jim Bankoski's avatar
Jim Bankoski committed
32
#include "vp9/common/vp9_tile_common.h"
33

34
#include "vp9/encoder/vp9_aq_complexity.h"
Marco Paniconi's avatar
Marco Paniconi committed
35 36
#include "vp9/encoder/vp9_aq_cyclicrefresh.h"
#include "vp9/encoder/vp9_aq_variance.h"
37
#include "vp9/encoder/vp9_bitstream.h"
38
#include "vp9/encoder/vp9_context_tree.h"
39
#include "vp9/encoder/vp9_encodeframe.h"
40
#include "vp9/encoder/vp9_encodemv.h"
41
#include "vp9/encoder/vp9_encoder.h"
42
#include "vp9/encoder/vp9_ethread.h"
43
#include "vp9/encoder/vp9_firstpass.h"
Jim Bankoski's avatar
Jim Bankoski committed
44 45
#include "vp9/encoder/vp9_mbgraph.h"
#include "vp9/encoder/vp9_picklpf.h"
46
#include "vp9/encoder/vp9_ratectrl.h"
47
#include "vp9/encoder/vp9_rd.h"
48
#include "vp9/encoder/vp9_resize.h"
Jim Bankoski's avatar
Jim Bankoski committed
49
#include "vp9/encoder/vp9_segmentation.h"
50
#include "vp9/encoder/vp9_skin_detection.h"
Yaowu Xu's avatar
Yaowu Xu committed
51
#include "vp9/encoder/vp9_speed_features.h"
Dmitry Kovalev's avatar
Dmitry Kovalev committed
52 53 54
#if CONFIG_INTERNAL_STATS
#include "vp9/encoder/vp9_ssim.h"
#endif
55
#include "vp9/encoder/vp9_svc_layercontext.h"
56
#include "vp9/encoder/vp9_temporal_filter.h"
Paul Wilkins's avatar
Paul Wilkins committed
57

58 59
#define AM_SEGMENT_ID_INACTIVE 7
#define AM_SEGMENT_ID_ACTIVE 0
60

61 62
#define SHARP_FILTER_QTHRESH 0          /* Q threshold for 8-tap sharp filter */

Jim Bankoski's avatar
Jim Bankoski committed
63 64 65 66 67 68
#define ALTREF_HIGH_PRECISION_MV 1      // Whether to use high precision mv
                                         //  for altref computation.
#define HIGH_PRECISION_MV_QTHRESH 200   // Q threshold for high precision
                                         // mv. Choose a very high value for
                                         // now so that HIGH_PRECISION is always
                                         // chosen.
John Koleszar's avatar
John Koleszar committed
69
// #define OUTPUT_YUV_REC
John Koleszar's avatar
John Koleszar committed
70

71
#ifdef OUTPUT_YUV_DENOISED
72
FILE *yuv_denoised_file = NULL;
73
#endif
Marco's avatar
Marco committed
74 75 76
#ifdef OUTPUT_YUV_SKINMAP
FILE *yuv_skinmap_file = NULL;
#endif
77 78 79
#ifdef OUTPUT_YUV_REC
FILE *yuv_rec_file;
#endif
John Koleszar's avatar
John Koleszar committed
80 81 82

#if 0
FILE *framepsnr;
Yaowu Xu's avatar
Yaowu Xu committed
83
FILE *kf_list;
John Koleszar's avatar
John Koleszar committed
84 85 86
FILE *keyfile;
#endif

87
static INLINE void Scale2Ratio(VPX_SCALING mode, int *hr, int *hs) {
88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112
  switch (mode) {
    case NORMAL:
      *hr = 1;
      *hs = 1;
      break;
    case FOURFIVE:
      *hr = 4;
      *hs = 5;
      break;
    case THREEFIVE:
      *hr = 3;
      *hs = 5;
    break;
    case ONETWO:
      *hr = 1;
      *hs = 2;
    break;
    default:
      *hr = 1;
      *hs = 1;
       assert(0);
      break;
  }
}

113 114
// Mark all inactive blocks as active. Other segmentation features may be set
// so memset cannot be used, instead only inactive blocks should be reset.
115
static void suppress_active_map(VP9_COMP *cpi) {
116 117 118 119 120 121 122 123
  unsigned char *const seg_map = cpi->segmentation_map;
  int i;
  if (cpi->active_map.enabled || cpi->active_map.update)
    for (i = 0; i < cpi->common.mi_rows * cpi->common.mi_cols; ++i)
      if (seg_map[i] == AM_SEGMENT_ID_INACTIVE)
        seg_map[i] = AM_SEGMENT_ID_ACTIVE;
}

124
static void apply_active_map(VP9_COMP *cpi) {
125 126 127 128 129 130 131
  struct segmentation *const seg = &cpi->common.seg;
  unsigned char *const seg_map = cpi->segmentation_map;
  const unsigned char *const active_map = cpi->active_map.map;
  int i;

  assert(AM_SEGMENT_ID_ACTIVE == CR_SEGMENT_ID_BASE);

132 133 134 135 136
  if (frame_is_intra_only(&cpi->common)) {
    cpi->active_map.enabled = 0;
    cpi->active_map.update = 1;
  }

137 138 139 140 141 142
  if (cpi->active_map.update) {
    if (cpi->active_map.enabled) {
      for (i = 0; i < cpi->common.mi_rows * cpi->common.mi_cols; ++i)
        if (seg_map[i] == AM_SEGMENT_ID_ACTIVE) seg_map[i] = active_map[i];
      vp9_enable_segmentation(seg);
      vp9_enable_segfeature(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_SKIP);
143 144 145 146 147
      vp9_enable_segfeature(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_ALT_LF);
      // Setting the data to -MAX_LOOP_FILTER will result in the computed loop
      // filter level being zero regardless of the value of seg->abs_delta.
      vp9_set_segdata(seg, AM_SEGMENT_ID_INACTIVE,
                      SEG_LVL_ALT_LF, -MAX_LOOP_FILTER);
148 149
    } else {
      vp9_disable_segfeature(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_SKIP);
150
      vp9_disable_segfeature(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_ALT_LF);
151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188
      if (seg->enabled) {
        seg->update_data = 1;
        seg->update_map = 1;
      }
    }
    cpi->active_map.update = 0;
  }
}

int vp9_set_active_map(VP9_COMP* cpi,
                       unsigned char* new_map_16x16,
                       int rows,
                       int cols) {
  if (rows == cpi->common.mb_rows && cols == cpi->common.mb_cols) {
    unsigned char *const active_map_8x8 = cpi->active_map.map;
    const int mi_rows = cpi->common.mi_rows;
    const int mi_cols = cpi->common.mi_cols;
    cpi->active_map.update = 1;
    if (new_map_16x16) {
      int r, c;
      for (r = 0; r < mi_rows; ++r) {
        for (c = 0; c < mi_cols; ++c) {
          active_map_8x8[r * mi_cols + c] =
              new_map_16x16[(r >> 1) * cols + (c >> 1)]
                  ? AM_SEGMENT_ID_ACTIVE
                  : AM_SEGMENT_ID_INACTIVE;
        }
      }
      cpi->active_map.enabled = 1;
    } else {
      cpi->active_map.enabled = 0;
    }
    return 0;
  } else {
    return -1;
  }
}

189 190 191 192 193 194 195 196 197
int vp9_get_active_map(VP9_COMP* cpi,
                       unsigned char* new_map_16x16,
                       int rows,
                       int cols) {
  if (rows == cpi->common.mb_rows && cols == cpi->common.mb_cols &&
      new_map_16x16) {
    unsigned char* const seg_map_8x8 = cpi->segmentation_map;
    const int mi_rows = cpi->common.mi_rows;
    const int mi_cols = cpi->common.mi_cols;
James Zern's avatar
James Zern committed
198
    memset(new_map_16x16, !cpi->active_map.enabled, rows * cols);
199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215
    if (cpi->active_map.enabled) {
      int r, c;
      for (r = 0; r < mi_rows; ++r) {
        for (c = 0; c < mi_cols; ++c) {
          // Cyclic refresh segments are considered active despite not having
          // AM_SEGMENT_ID_ACTIVE
          new_map_16x16[(r >> 1) * cols + (c >> 1)] |=
              seg_map_8x8[r * mi_cols + c] != AM_SEGMENT_ID_INACTIVE;
        }
      }
    }
    return 0;
  } else {
    return -1;
  }
}

216
void vp9_set_high_precision_mv(VP9_COMP *cpi, int allow_high_precision_mv) {
217
  MACROBLOCK *const mb = &cpi->td.mb;
Deb Mukherjee's avatar
Deb Mukherjee committed
218
  cpi->common.allow_high_precision_mv = allow_high_precision_mv;
219
  if (cpi->common.allow_high_precision_mv) {
220 221 222 223 224 225 226
    mb->mvcost = mb->nmvcost_hp;
    mb->mvsadcost = mb->nmvsadcost_hp;
  } else {
    mb->mvcost = mb->nmvcost;
    mb->mvsadcost = mb->nmvsadcost;
  }
}
Paul Wilkins's avatar
Paul Wilkins committed
227

228 229 230 231 232 233 234
static void setup_frame(VP9_COMP *cpi) {
  VP9_COMMON *const cm = &cpi->common;
  // Set up entropy context depending on frame type. The decoder mandates
  // the use of the default context, index 0, for keyframes and inter
  // frames where the error_resilient_mode or intra_only flag is set. For
  // other inter-frames the encoder currently uses only two contexts;
  // context 1 for ALTREF frames and context 0 for the others.
235 236 237 238 239 240 241
  if (frame_is_intra_only(cm) || cm->error_resilient_mode) {
    vp9_setup_past_independence(cm);
  } else {
    if (!cpi->use_svc)
      cm->frame_context_idx = cpi->refresh_alt_ref_frame;
  }

242
  if (cm->frame_type == KEY_FRAME) {
243
    if (!is_two_pass_svc(cpi))
244
      cpi->refresh_golden_frame = 1;
245
    cpi->refresh_alt_ref_frame = 1;
246
    vp9_zero(cpi->interp_filter_selected);
247
  } else {
248
    *cm->fc = cm->frame_contexts[cm->frame_context_idx];
249
    vp9_zero(cpi->interp_filter_selected[0]);
250 251 252
  }
}

253 254 255
static void vp9_enc_setup_mi(VP9_COMMON *cm) {
  int i;
  cm->mi = cm->mip + cm->mi_stride + 1;
James Zern's avatar
James Zern committed
256
  memset(cm->mip, 0, cm->mi_stride * (cm->mi_rows + 1) * sizeof(*cm->mip));
257 258
  cm->prev_mi = cm->prev_mip + cm->mi_stride + 1;
  // Clear top border row
James Zern's avatar
James Zern committed
259
  memset(cm->prev_mip, 0, sizeof(*cm->prev_mip) * cm->mi_stride);
260 261
  // Clear left border column
  for (i = 1; i < cm->mi_rows + 1; ++i)
James Zern's avatar
James Zern committed
262
    memset(&cm->prev_mip[i * cm->mi_stride], 0, sizeof(*cm->prev_mip));
263 264 265 266

  cm->mi_grid_visible = cm->mi_grid_base + cm->mi_stride + 1;
  cm->prev_mi_grid_visible = cm->prev_mi_grid_base + cm->mi_stride + 1;

James Zern's avatar
James Zern committed
267 268
  memset(cm->mi_grid_base, 0,
         cm->mi_stride * (cm->mi_rows + 1) * sizeof(*cm->mi_grid_base));
269 270 271 272 273 274 275 276 277 278
}

static int vp9_enc_alloc_mi(VP9_COMMON *cm, int mi_size) {
  cm->mip = vpx_calloc(mi_size, sizeof(*cm->mip));
  if (!cm->mip)
    return 1;
  cm->prev_mip = vpx_calloc(mi_size, sizeof(*cm->prev_mip));
  if (!cm->prev_mip)
    return 1;
  cm->mi_alloc_size = mi_size;
279 280 281 282 283 284 285 286

  cm->mi_grid_base = (MODE_INFO **)vpx_calloc(mi_size, sizeof(MODE_INFO*));
  if (!cm->mi_grid_base)
    return 1;
  cm->prev_mi_grid_base = (MODE_INFO **)vpx_calloc(mi_size, sizeof(MODE_INFO*));
  if (!cm->prev_mi_grid_base)
    return 1;

287 288 289 290 291 292 293 294
  return 0;
}

static void vp9_enc_free_mi(VP9_COMMON *cm) {
  vpx_free(cm->mip);
  cm->mip = NULL;
  vpx_free(cm->prev_mip);
  cm->prev_mip = NULL;
295 296 297 298
  vpx_free(cm->mi_grid_base);
  cm->mi_grid_base = NULL;
  vpx_free(cm->prev_mi_grid_base);
  cm->prev_mi_grid_base = NULL;
299 300 301 302
}

static void vp9_swap_mi_and_prev_mi(VP9_COMMON *cm) {
  // Current mip will be the prev_mip for the next frame.
303
  MODE_INFO **temp_base = cm->prev_mi_grid_base;
304 305 306 307 308 309 310
  MODE_INFO *temp = cm->prev_mip;
  cm->prev_mip = cm->mip;
  cm->mip = temp;

  // Update the upper left visible macroblock ptrs.
  cm->mi = cm->mip + cm->mi_stride + 1;
  cm->prev_mi = cm->prev_mip + cm->mi_stride + 1;
311 312 313 314 315

  cm->prev_mi_grid_base = cm->mi_grid_base;
  cm->mi_grid_base = temp_base;
  cm->mi_grid_visible = cm->mi_grid_base + cm->mi_stride + 1;
  cm->prev_mi_grid_visible = cm->prev_mi_grid_base + cm->mi_stride + 1;
316 317
}

318
void vp9_initialize_enc(void) {
319
  static volatile int init_done = 0;
John Koleszar's avatar
John Koleszar committed
320 321

  if (!init_done) {
322
    vp9_rtcd();
Johann's avatar
Johann committed
323
    vpx_dsp_rtcd();
324
    vpx_scale_rtcd();
325
    vp9_init_intra_predictors();
326
    vp9_init_me_luts();
327
    vp9_rc_init_minq_luts();
328
    vp9_entropy_mv_init();
329
    vp9_temporal_filter_init();
John Koleszar's avatar
John Koleszar committed
330 331
    init_done = 1;
  }
John Koleszar's avatar
John Koleszar committed
332 333
}

334
static void dealloc_compressor_data(VP9_COMP *cpi) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
335
  VP9_COMMON *const cm = &cpi->common;
336
  int i;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
337

338 339 340
  vpx_free(cpi->mbmi_ext_base);
  cpi->mbmi_ext_base = NULL;

341 342
  vpx_free(cpi->tile_data);
  cpi->tile_data = NULL;
343

344
  // Delete sementation map
John Koleszar's avatar
John Koleszar committed
345
  vpx_free(cpi->segmentation_map);
Dmitry Kovalev's avatar
Dmitry Kovalev committed
346
  cpi->segmentation_map = NULL;
John Koleszar's avatar
John Koleszar committed
347
  vpx_free(cpi->coding_context.last_frame_seg_map_copy);
Dmitry Kovalev's avatar
Dmitry Kovalev committed
348
  cpi->coding_context.last_frame_seg_map_copy = NULL;
John Koleszar's avatar
John Koleszar committed
349

Jingning Han's avatar
Jingning Han committed
350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369
  vpx_free(cpi->nmvcosts[0]);
  vpx_free(cpi->nmvcosts[1]);
  cpi->nmvcosts[0] = NULL;
  cpi->nmvcosts[1] = NULL;

  vpx_free(cpi->nmvcosts_hp[0]);
  vpx_free(cpi->nmvcosts_hp[1]);
  cpi->nmvcosts_hp[0] = NULL;
  cpi->nmvcosts_hp[1] = NULL;

  vpx_free(cpi->nmvsadcosts[0]);
  vpx_free(cpi->nmvsadcosts[1]);
  cpi->nmvsadcosts[0] = NULL;
  cpi->nmvsadcosts[1] = NULL;

  vpx_free(cpi->nmvsadcosts_hp[0]);
  vpx_free(cpi->nmvsadcosts_hp[1]);
  cpi->nmvsadcosts_hp[0] = NULL;
  cpi->nmvsadcosts_hp[1] = NULL;

370 371 372
  vp9_cyclic_refresh_free(cpi->cyclic_refresh);
  cpi->cyclic_refresh = NULL;

373 374 375
  vpx_free(cpi->active_map.map);
  cpi->active_map.map = NULL;

376 377 378 379
  vp9_free_ref_frame_buffers(cm->buffer_pool);
#if CONFIG_VP9_POSTPROC
  vp9_free_postproc_buffers(cm);
#endif
380
  vp9_free_context_buffers(cm);
John Koleszar's avatar
John Koleszar committed
381

382 383
  vp9_free_frame_buffer(&cpi->last_frame_uf);
  vp9_free_frame_buffer(&cpi->scaled_source);
384
  vp9_free_frame_buffer(&cpi->scaled_last_source);
385
  vp9_free_frame_buffer(&cpi->alt_ref_buffer);
386
  vp9_lookahead_destroy(cpi->lookahead);
John Koleszar's avatar
John Koleszar committed
387

388 389
  vpx_free(cpi->tile_tok[0][0]);
  cpi->tile_tok[0][0] = 0;
John Koleszar's avatar
John Koleszar committed
390

391
  vp9_free_pc_tree(&cpi->td);
392

393 394 395 396 397 398
  for (i = 0; i < cpi->svc.number_spatial_layers; ++i) {
    LAYER_CONTEXT *const lc = &cpi->svc.layer_context[i];
    vpx_free(lc->rc_twopass_stats_in.buf);
    lc->rc_twopass_stats_in.buf = NULL;
    lc->rc_twopass_stats_in.sz = 0;
  }
399 400 401 402 403

  if (cpi->source_diff_var != NULL) {
    vpx_free(cpi->source_diff_var);
    cpi->source_diff_var = NULL;
  }
404

405 406 407
  for (i = 0; i < MAX_LAG_BUFFERS; ++i) {
    vp9_free_frame_buffer(&cpi->svc.scaled_frames[i]);
  }
James Zern's avatar
James Zern committed
408 409
  memset(&cpi->svc.scaled_frames[0], 0,
         MAX_LAG_BUFFERS * sizeof(cpi->svc.scaled_frames[0]));
410 411

  vp9_free_frame_buffer(&cpi->svc.empty_frame.img);
James Zern's avatar
James Zern committed
412
  memset(&cpi->svc.empty_frame, 0, sizeof(cpi->svc.empty_frame));
John Koleszar's avatar
John Koleszar committed
413 414
}

415 416 417 418 419 420 421 422
static void save_coding_context(VP9_COMP *cpi) {
  CODING_CONTEXT *const cc = &cpi->coding_context;
  VP9_COMMON *cm = &cpi->common;

  // Stores a snapshot of key state variables which can subsequently be
  // restored with a call to vp9_restore_coding_context. These functions are
  // intended for use in a re-code loop in vp9_compress_frame where the
  // quantizer value is adjusted between loop iterations.
423
  vp9_copy(cc->nmvjointcost,  cpi->td.mb.nmvjointcost);
Jingning Han's avatar
Jingning Han committed
424

James Zern's avatar
James Zern committed
425 426 427 428 429 430 431 432
  memcpy(cc->nmvcosts[0], cpi->nmvcosts[0],
         MV_VALS * sizeof(*cpi->nmvcosts[0]));
  memcpy(cc->nmvcosts[1], cpi->nmvcosts[1],
         MV_VALS * sizeof(*cpi->nmvcosts[1]));
  memcpy(cc->nmvcosts_hp[0], cpi->nmvcosts_hp[0],
         MV_VALS * sizeof(*cpi->nmvcosts_hp[0]));
  memcpy(cc->nmvcosts_hp[1], cpi->nmvcosts_hp[1],
         MV_VALS * sizeof(*cpi->nmvcosts_hp[1]));
433 434 435

  vp9_copy(cc->segment_pred_probs, cm->seg.pred_probs);

James Zern's avatar
James Zern committed
436 437
  memcpy(cpi->coding_context.last_frame_seg_map_copy,
         cm->last_frame_seg_map, (cm->mi_rows * cm->mi_cols));
438 439 440 441

  vp9_copy(cc->last_ref_lf_deltas, cm->lf.last_ref_deltas);
  vp9_copy(cc->last_mode_lf_deltas, cm->lf.last_mode_deltas);

442
  cc->fc = *cm->fc;
443 444 445 446 447 448 449 450
}

static void restore_coding_context(VP9_COMP *cpi) {
  CODING_CONTEXT *const cc = &cpi->coding_context;
  VP9_COMMON *cm = &cpi->common;

  // Restore key state variables to the snapshot state stored in the
  // previous call to vp9_save_coding_context.
451
  vp9_copy(cpi->td.mb.nmvjointcost, cc->nmvjointcost);
Jingning Han's avatar
Jingning Han committed
452

James Zern's avatar
James Zern committed
453 454 455 456 457 458
  memcpy(cpi->nmvcosts[0], cc->nmvcosts[0], MV_VALS * sizeof(*cc->nmvcosts[0]));
  memcpy(cpi->nmvcosts[1], cc->nmvcosts[1], MV_VALS * sizeof(*cc->nmvcosts[1]));
  memcpy(cpi->nmvcosts_hp[0], cc->nmvcosts_hp[0],
         MV_VALS * sizeof(*cc->nmvcosts_hp[0]));
  memcpy(cpi->nmvcosts_hp[1], cc->nmvcosts_hp[1],
         MV_VALS * sizeof(*cc->nmvcosts_hp[1]));
459 460 461

  vp9_copy(cm->seg.pred_probs, cc->segment_pred_probs);

James Zern's avatar
James Zern committed
462 463 464
  memcpy(cm->last_frame_seg_map,
         cpi->coding_context.last_frame_seg_map_copy,
         (cm->mi_rows * cm->mi_cols));
465 466 467 468

  vp9_copy(cm->lf.last_ref_deltas, cc->last_ref_lf_deltas);
  vp9_copy(cm->lf.last_mode_deltas, cc->last_mode_lf_deltas);

469
  *cm->fc = cc->fc;
470 471
}

472
static void configure_static_seg_features(VP9_COMP *cpi) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
473
  VP9_COMMON *const cm = &cpi->common;
474
  const RATE_CONTROL *const rc = &cpi->rc;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
475
  struct segmentation *const seg = &cm->seg;
476

477
  int high_q = (int)(rc->avg_q > 48.0);
John Koleszar's avatar
John Koleszar committed
478
  int qi_delta;
479

John Koleszar's avatar
John Koleszar committed
480 481 482
  // Disable and clear down for KF
  if (cm->frame_type == KEY_FRAME) {
    // Clear down the global segmentation map
James Zern's avatar
James Zern committed
483
    memset(cpi->segmentation_map, 0, cm->mi_rows * cm->mi_cols);
484 485
    seg->update_map = 0;
    seg->update_data = 0;
John Koleszar's avatar
John Koleszar committed
486
    cpi->static_mb_pct = 0;
487

John Koleszar's avatar
John Koleszar committed
488
    // Disable segmentation
489
    vp9_disable_segmentation(seg);
Paul Wilkins's avatar
Paul Wilkins committed
490

John Koleszar's avatar
John Koleszar committed
491
    // Clear down the segment features.
492
    vp9_clearall_segfeatures(seg);
493 494
  } else if (cpi->refresh_alt_ref_frame) {
    // If this is an alt ref frame
John Koleszar's avatar
John Koleszar committed
495
    // Clear down the global segmentation map
James Zern's avatar
James Zern committed
496
    memset(cpi->segmentation_map, 0, cm->mi_rows * cm->mi_cols);
497 498
    seg->update_map = 0;
    seg->update_data = 0;
John Koleszar's avatar
John Koleszar committed
499
    cpi->static_mb_pct = 0;
Paul Wilkins's avatar
Paul Wilkins committed
500

John Koleszar's avatar
John Koleszar committed
501
    // Disable segmentation and individual segment features by default
502
    vp9_disable_segmentation(seg);
503
    vp9_clearall_segfeatures(seg);
Paul Wilkins's avatar
Paul Wilkins committed
504

John Koleszar's avatar
John Koleszar committed
505 506
    // Scan frames from current to arf frame.
    // This function re-enables segmentation if appropriate.
507
    vp9_update_mbgraph_stats(cpi);
Paul Wilkins's avatar
Paul Wilkins committed
508

John Koleszar's avatar
John Koleszar committed
509 510
    // If segmentation was enabled set those features needed for the
    // arf itself.
511 512 513
    if (seg->enabled) {
      seg->update_map = 1;
      seg->update_data = 1;
Paul Wilkins's avatar
Paul Wilkins committed
514

515 516
      qi_delta = vp9_compute_qdelta(rc, rc->avg_q, rc->avg_q * 0.875,
                                    cm->bit_depth);
517
      vp9_set_segdata(seg, 1, SEG_LVL_ALT_Q, qi_delta - 2);
518
      vp9_set_segdata(seg, 1, SEG_LVL_ALT_LF, -2);
Paul Wilkins's avatar
Paul Wilkins committed
519

520 521
      vp9_enable_segfeature(seg, 1, SEG_LVL_ALT_Q);
      vp9_enable_segfeature(seg, 1, SEG_LVL_ALT_LF);
522

John Koleszar's avatar
John Koleszar committed
523
      // Where relevant assume segment data is delta data
524
      seg->abs_delta = SEGMENT_DELTADATA;
525
    }
526
  } else if (seg->enabled) {
527 528
    // All other frames if segmentation has been enabled

John Koleszar's avatar
John Koleszar committed
529
    // First normal frame in a valid gf or alt ref group
530
    if (rc->frames_since_golden == 0) {
Paul Wilkins's avatar
Paul Wilkins committed
531
      // Set up segment features for normal frames in an arf group
532
      if (rc->source_alt_ref_active) {
533 534 535
        seg->update_map = 0;
        seg->update_data = 1;
        seg->abs_delta = SEGMENT_DELTADATA;
Paul Wilkins's avatar
Paul Wilkins committed
536

537 538
        qi_delta = vp9_compute_qdelta(rc, rc->avg_q, rc->avg_q * 1.125,
                                      cm->bit_depth);
539
        vp9_set_segdata(seg, 1, SEG_LVL_ALT_Q, qi_delta + 2);
540
        vp9_enable_segfeature(seg, 1, SEG_LVL_ALT_Q);
Paul Wilkins's avatar
Paul Wilkins committed
541

542 543
        vp9_set_segdata(seg, 1, SEG_LVL_ALT_LF, -2);
        vp9_enable_segfeature(seg, 1, SEG_LVL_ALT_LF);
Paul Wilkins's avatar
Paul Wilkins committed
544

John Koleszar's avatar
John Koleszar committed
545 546
        // Segment coding disabled for compred testing
        if (high_q || (cpi->static_mb_pct == 100)) {
547 548 549
          vp9_set_segdata(seg, 1, SEG_LVL_REF_FRAME, ALTREF_FRAME);
          vp9_enable_segfeature(seg, 1, SEG_LVL_REF_FRAME);
          vp9_enable_segfeature(seg, 1, SEG_LVL_SKIP);
John Koleszar's avatar
John Koleszar committed
550
        }
551 552 553 554
      } else {
        // Disable segmentation and clear down features if alt ref
        // is not active for this group

555
        vp9_disable_segmentation(seg);
Paul Wilkins's avatar
Paul Wilkins committed
556

James Zern's avatar
James Zern committed
557
        memset(cpi->segmentation_map, 0, cm->mi_rows * cm->mi_cols);
Paul Wilkins's avatar
Paul Wilkins committed
558

559 560
        seg->update_map = 0;
        seg->update_data = 0;
Paul Wilkins's avatar
Paul Wilkins committed
561

562
        vp9_clearall_segfeatures(seg);
John Koleszar's avatar
John Koleszar committed
563
      }
564
    } else if (rc->is_src_frame_alt_ref) {
565 566 567
      // Special case where we are coding over the top of a previous
      // alt ref frame.
      // Segment coding disabled for compred testing
John Koleszar's avatar
John Koleszar committed
568

Paul Wilkins's avatar
Paul Wilkins committed
569
      // Enable ref frame features for segment 0 as well
570 571
      vp9_enable_segfeature(seg, 0, SEG_LVL_REF_FRAME);
      vp9_enable_segfeature(seg, 1, SEG_LVL_REF_FRAME);
John Koleszar's avatar
John Koleszar committed
572

Paul Wilkins's avatar
Paul Wilkins committed
573
      // All mbs should use ALTREF_FRAME
574 575 576 577
      vp9_clear_segdata(seg, 0, SEG_LVL_REF_FRAME);
      vp9_set_segdata(seg, 0, SEG_LVL_REF_FRAME, ALTREF_FRAME);
      vp9_clear_segdata(seg, 1, SEG_LVL_REF_FRAME);
      vp9_set_segdata(seg, 1, SEG_LVL_REF_FRAME, ALTREF_FRAME);
John Koleszar's avatar
John Koleszar committed
578

Paul Wilkins's avatar
Paul Wilkins committed
579
      // Skip all MBs if high Q (0,0 mv and skip coeffs)
John Koleszar's avatar
John Koleszar committed
580
      if (high_q) {
581 582
        vp9_enable_segfeature(seg, 0, SEG_LVL_SKIP);
        vp9_enable_segfeature(seg, 1, SEG_LVL_SKIP);
John Koleszar's avatar
John Koleszar committed
583
      }
Adrian Grange's avatar
Adrian Grange committed
584
      // Enable data update
585
      seg->update_data = 1;
586 587 588
    } else {
      // All other frames.

John Koleszar's avatar
John Koleszar committed
589
      // No updates.. leave things as they are.
590 591
      seg->update_map = 0;
      seg->update_data = 0;
John Koleszar's avatar
John Koleszar committed
592 593
    }
  }
594 595
}

596
static void update_reference_segmentation_map(VP9_COMP *cpi) {
Ronald S. Bultje's avatar
Ronald S. Bultje committed
597
  VP9_COMMON *const cm = &cpi->common;
598
  MODE_INFO **mi_8x8_ptr = cm->mi_grid_visible;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
599
  uint8_t *cache_ptr = cm->last_frame_seg_map;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
600 601
  int row, col;

602
  for (row = 0; row < cm->mi_rows; row++) {
603
    MODE_INFO **mi_8x8 = mi_8x8_ptr;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
604
    uint8_t *cache = cache_ptr;
605
    for (col = 0; col < cm->mi_cols; col++, mi_8x8++, cache++)
606
      cache[0] = mi_8x8[0]->mbmi.segment_id;
607
    mi_8x8_ptr += cm->mi_stride;
608
    cache_ptr += cm->mi_cols;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
609 610 611
  }
}

612
static void alloc_raw_frame_buffers(VP9_COMP *cpi) {
613
  VP9_COMMON *cm = &cpi->common;
614
  const VP9EncoderConfig *oxcf = &cpi->oxcf;
615

616 617 618
  if (!cpi->lookahead)
    cpi->lookahead = vp9_lookahead_init(oxcf->width, oxcf->height,
                                        cm->subsampling_x, cm->subsampling_y,
619 620 621
#if CONFIG_VP9_HIGHBITDEPTH
                                      cm->use_highbitdepth,
#endif
Dmitry Kovalev's avatar
Dmitry Kovalev committed
622
                                      oxcf->lag_in_frames);
John Koleszar's avatar
John Koleszar committed
623
  if (!cpi->lookahead)
624
    vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
John Koleszar's avatar
John Koleszar committed
625
                       "Failed to allocate lag buffers");
John Koleszar's avatar
John Koleszar committed
626

627
  // TODO(agrange) Check if ARF is enabled and skip allocation if not.
628
  if (vp9_realloc_frame_buffer(&cpi->alt_ref_buffer,
Dmitry Kovalev's avatar
Dmitry Kovalev committed
629
                               oxcf->width, oxcf->height,
630
                               cm->subsampling_x, cm->subsampling_y,
631 632 633
#if CONFIG_VP9_HIGHBITDEPTH
                               cm->use_highbitdepth,
#endif
634 635
                               VP9_ENC_BORDER_IN_PIXELS, cm->byte_alignment,
                               NULL, NULL, NULL))
636
    vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
John Koleszar's avatar
John Koleszar committed
637
                       "Failed to allocate altref buffer");
John Koleszar's avatar
John Koleszar committed
638
}
639

640
static void alloc_util_frame_buffers(VP9_COMP *cpi) {
641
  VP9_COMMON *const cm = &cpi->common;
642 643 644
  if (vp9_realloc_frame_buffer(&cpi->last_frame_uf,
                               cm->width, cm->height,
                               cm->subsampling_x, cm->subsampling_y,
645 646 647
#if CONFIG_VP9_HIGHBITDEPTH
                               cm->use_highbitdepth,
#endif
648 649
                               VP9_ENC_BORDER_IN_PIXELS, cm->byte_alignment,
                               NULL, NULL, NULL))
650
    vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
651
                       "Failed to allocate last frame buffer");
652

653 654 655
  if (vp9_realloc_frame_buffer(&cpi->scaled_source,
                               cm->width, cm->height,
                               cm->subsampling_x, cm->subsampling_y,
656 657 658
#if CONFIG_VP9_HIGHBITDEPTH
                               cm->use_highbitdepth,
#endif
659 660
                               VP9_ENC_BORDER_IN_PIXELS, cm->byte_alignment,
                               NULL, NULL, NULL))
661
    vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
662
                       "Failed to allocate scaled source buffer");
663

664 665 666
  if (vp9_realloc_frame_buffer(&cpi->scaled_last_source,
                               cm->width, cm->height,
                               cm->subsampling_x, cm->subsampling_y,
667 668 669
#if CONFIG_VP9_HIGHBITDEPTH
                               cm->use_highbitdepth,
#endif
670 671
                               VP9_ENC_BORDER_IN_PIXELS, cm->byte_alignment,
                               NULL, NULL, NULL))
672
    vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
673 674
                       "Failed to allocate scaled last source buffer");
}
675

676 677 678 679 680 681 682 683 684 685 686 687

static int alloc_context_buffers_ext(VP9_COMP *cpi) {
  VP9_COMMON *cm = &cpi->common;
  int mi_size = cm->mi_cols * cm->mi_rows;

  cpi->mbmi_ext_base = vpx_calloc(mi_size, sizeof(*cpi->mbmi_ext_base));
  if (!cpi->mbmi_ext_base)
    return 1;

  return 0;
}

688 689
void vp9_alloc_compressor_data(VP9_COMP *cpi) {
  VP9_COMMON *cm = &cpi->common;
690

691 692
  vp9_alloc_context_buffers(cm, cm->width, cm->height);

693 694
  alloc_context_buffers_ext(cpi);

695
  vpx_free(cpi->tile_tok[0][0]);
696 697 698

  {
    unsigned int tokens = get_token_alloc(cm->mb_rows, cm->mb_cols);
699 700
    CHECK_MEM_ERROR(cm, cpi->tile_tok[0][0],
        vpx_calloc(tokens, sizeof(*cpi->tile_tok[0][0])));
701
  }
702

703
  vp9_setup_pc_tree(&cpi->common, &cpi->td);
704 705
}

706 707 708 709 710 711 712 713 714 715 716
void vp9_new_framerate(VP9_COMP *cpi, double framerate) {
  cpi->framerate = framerate < 0.1 ? 30 : framerate;
  vp9_rc_update_framerate(cpi);
}

static void set_tile_limits(VP9_COMP *cpi) {
  VP9_COMMON *const cm = &cpi->common;

  int min_log2_tile_cols, max_log2_tile_cols;
  vp9_get_tile_n_bits(cm->mi_cols, &min_log2_tile_cols, &max_log2_tile_cols);

717 718 719
  if (is_two_pass_svc(cpi) &&
      (cpi->svc.encode_empty_frame_state == ENCODING ||
      cpi->svc.number_spatial_layers > 1)) {
720 721 722 723 724 725 726 727 728
    cm->log2_tile_cols = 0;
    cm->log2_tile_rows = 0;
  } else {
    cm->log2_tile_cols = clamp(cpi->oxcf.tile_columns,
                               min_log2_tile_cols, max_log2_tile_cols);
    cm->log2_tile_rows = cpi->oxcf.tile_rows;
  }
}

729 730
static void update_frame_size(VP9_COMP *cpi) {
  VP9_COMMON *const cm = &cpi->common;
731
  MACROBLOCKD *const xd = &cpi->td.mb.e_mbd;
Adrian Grange's avatar
Adrian Grange committed
732 733 734

  vp9_set_mb_mi(cm, cm->width, cm->height);
  vp9_init_context_buffers(cm);
735
  vp9_init_macroblockd(cm, xd);
736 737 738
  cpi->td.mb.mbmi_ext_base = cpi->mbmi_ext_base;
  memset(cpi->mbmi_ext_base, 0,
         cm->mi_rows * cm->mi_cols * sizeof(*cpi->mbmi_ext_base));
739

740 741
  set_tile_limits(cpi);

742
  if (is_two_pass_svc(cpi)) {
743 744 745
    if (vp9_realloc_frame_buffer(&cpi->alt_ref_buffer,
                                 cm->width, cm->height,
                                 cm->subsampling_x, cm->subsampling_y,
746 747 748
#if CONFIG_VP9_HIGHBITDEPTH
                                 cm->use_highbitdepth,
#endif
749 750
                                 VP9_ENC_BORDER_IN_PIXELS, cm->byte_alignment,
                                 NULL, NULL, NULL))
751 752 753
      vpx_internal_error(&cm->error, VPX_CODEC_MEM_ERROR,
                         "Failed to reallocate alt_ref_buffer");
  }
754 755
}

756 757 758 759 760 761
static void init_buffer_indices(VP9_COMP *cpi) {
  cpi->lst_fb_idx = 0;
  cpi->gld_fb_idx = 1;
  cpi->alt_fb_idx = 2;
}

762
static void init_config(struct VP9_COMP *cpi, VP9EncoderConfig *oxcf) {
763
  VP9_COMMON *const cm = &cpi->common;
John Koleszar's avatar
John Koleszar committed
764

John Koleszar's avatar
John Koleszar committed
765
  cpi->oxcf = *oxcf;
766
  cpi->framerate = oxcf->init_framerate;
John Koleszar's avatar
John Koleszar committed
767

768 769
  cm->profile = oxcf->profile;
  cm->bit_depth = oxcf->bit_depth;
770 771 772
#if CONFIG_VP9_HIGHBITDEPTH
  cm->use_highbitdepth = oxcf->use_highbitdepth;
#endif
773
  cm->color_space = oxcf->color_space;
John Koleszar's avatar
John Koleszar committed
774

775 776
  cm->width = oxcf->width;
  cm->height = oxcf->height;
777
  vp9_alloc_compressor_data(cpi);
778

779 780
  cpi->svc.temporal_layering_mode = oxcf->temporal_layering_mode;

781 782 783
  // Single thread case: use counts in common.
  cpi->td.counts = &cm->counts;

784 785 786 787 788
  // Spatial scalability.
  cpi->svc.number_spatial_layers = oxcf->ss_number_layers;
  // Temporal scalability.
  cpi->svc.number_temporal_layers = oxcf->ts_number_layers;

789
  if ((cpi->svc.number_temporal_layers > 1 && cpi->oxcf.rc_mode == VPX_CBR) ||
790 791
      ((cpi->svc.number_temporal_layers > 1 ||
        cpi->svc.number_spatial_layers > 1) &&
792
       cpi->oxcf.pass != 1)) {
793
    vp9_init_layer_context(cpi);
794 795
  }

John Koleszar's avatar
John Koleszar committed
796
  // change includes all joint functionality
Dmitry Kovalev's avatar
Dmitry Kovalev committed
797
  vp9_change_config(cpi, oxcf);
John Koleszar's avatar
John Koleszar committed
798

John Koleszar's avatar
John Koleszar committed
799
  cpi->static_mb_pct = 0;
800
  cpi->ref_frame_flags = 0;
801

802
  init_buffer_indices(cpi);
John Koleszar's avatar
John Koleszar committed
803 804
}

Dmitry Kovalev's avatar
Dmitry Kovalev committed
805 806 807 808 809 810 811 812 813 814 815 816 817 818
static void set_rc_buffer_sizes(RATE_CONTROL *rc,
                                const VP9EncoderConfig *oxcf) {
  const int64_t bandwidth = oxcf->target_bandwidth;
  const int64_t starting = oxcf->starting_buffer_level_ms;
  const int64_t optimal = oxcf->optimal_buffer_level_ms;
  const int64_t maximum = oxcf->maximum_buffer_size_ms;

  rc->starting_buffer_level = starting * bandwidth / 1000;
  rc->optimal_buffer_level = (optimal == 0) ? bandwidth / 8
                                            : optimal * bandwidth / 1000;
  rc->maximum_buffer_size = (maximum == 0) ? bandwidth / 8
                                           : maximum * bandwidth / 1000;
}

819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960
#if CONFIG_VP9_HIGHBITDEPTH
#define HIGHBD_BFP(BT, SDF, SDAF, VF, SVF, SVAF, SDX3F, SDX8F, SDX4DF) \
    cpi->fn_ptr[BT].sdf = SDF; \
    cpi->fn_ptr[BT].sdaf = SDAF; \
    cpi->fn_ptr[BT].vf = VF; \
    cpi->fn_ptr[BT].svf = SVF; \
    cpi->fn_ptr[BT].svaf = SVAF; \
    cpi->fn_ptr[BT].sdx3f = SDX3F; \
    cpi->fn_ptr[BT].sdx8f = SDX8F; \
    cpi->fn_ptr[BT].sdx4df = SDX4DF;

#define MAKE_BFP_SAD_WRAPPER(fnname) \
static unsigned int fnname##_bits8(const uint8_t *src_ptr, \
                                   int source_stride, \
                                   const uint8_t *ref_ptr, \
                                   int ref_stride) {  \
  return fnname(src_ptr, source_stride, ref_ptr, ref_stride); \
} \
static unsigned int fnname##_bits10(const uint8_t *src_ptr, \
                                    int source_stride, \
                                    const uint8_t *ref_ptr, \
                                    int ref_stride) {  \
  return fnname(src_ptr, source_stride, ref_ptr, ref_stride) >> 2; \
} \
static unsigned int fnname##_bits12(const uint8_t *src_ptr, \
                                    int source_stride, \
                                    const uint8_t *ref_ptr, \
                                    int ref_stride) {  \
  return fnname(src_ptr, source_stride, ref_ptr, ref_stride) >> 4; \
}

#define MAKE_BFP_SADAVG_WRAPPER(fnname) static unsigned int \
fnname##_bits8(const uint8_t *src_ptr, \
               int source_stride, \
               const uint8_t *ref_ptr, \
               int ref_stride, \
               const uint8_t *second_pred) {  \
  return fnname(src_ptr, source_stride, ref_ptr, ref_stride, second_pred); \
} \
static unsigned int fnname##_bits10(const uint8_t *src_ptr, \
                                    int source_stride, \
                                    const uint8_t *ref_ptr, \
                                    int ref_stride, \
                                    const uint8_t *second_pred) {  \
  return fnname(src_ptr, source_stride, ref_ptr, ref_stride, \
                second_pred) >> 2; \
} \
static unsigned int fnname##_bits12(const uint8_t *src_ptr, \
                                    int source_stride, \
                                    const uint8_t *ref_ptr, \
                                    int ref_stride, \
                                    const uint8_t *second_pred) {  \
  return fnname(src_ptr, source_stride, ref_ptr, ref_stride, \
                second_pred) >> 4; \
}

#define MAKE_BFP_SAD3_WRAPPER(fnname) \
static void fnname##_bits8(const uint8_t *src_ptr, \
                           int source_stride, \
                           const uint8_t *ref_ptr, \
                           int  ref_stride, \
                           unsigned int *sad_array) {  \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
} \
static void fnname##_bits10(const uint8_t *src_ptr, \
                            int source_stride, \
                            const uint8_t *ref_ptr, \
                            int  ref_stride, \
                            unsigned int *sad_array) {  \
  int i; \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
  for (i = 0; i < 3; i++) \
    sad_array[i] >>= 2; \
} \
static void fnname##_bits12(const uint8_t *src_ptr, \
                            int source_stride, \
                            const uint8_t *ref_ptr, \
                            int  ref_stride, \
                            unsigned int *sad_array) {  \
  int i; \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
  for (i = 0; i < 3; i++) \
    sad_array[i] >>= 4; \
}

#define MAKE_BFP_SAD8_WRAPPER(fnname) \
static void fnname##_bits8(const uint8_t *src_ptr, \
                           int source_stride, \
                           const uint8_t *ref_ptr, \
                           int  ref_stride, \
                           unsigned int *sad_array) {  \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
} \
static void fnname##_bits10(const uint8_t *src_ptr, \
                            int source_stride, \
                            const uint8_t *ref_ptr, \
                            int  ref_stride, \
                            unsigned int *sad_array) {  \
  int i; \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
  for (i = 0; i < 8; i++) \
    sad_array[i] >>= 2; \
} \
static void fnname##_bits12(const uint8_t *src_ptr, \
                            int source_stride, \
                            const uint8_t *ref_ptr, \
                            int  ref_stride, \
                            unsigned int *sad_array) {  \
  int i; \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
  for (i = 0; i < 8; i++) \
    sad_array[i] >>= 4; \
}
#define MAKE_BFP_SAD4D_WRAPPER(fnname) \
static void fnname##_bits8(const uint8_t *src_ptr, \
                           int source_stride, \
                           const uint8_t* const ref_ptr[], \
                           int  ref_stride, \
                           unsigned int *sad_array) {  \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
} \
static void fnname##_bits10(const uint8_t *src_ptr, \
                            int source_stride, \
                            const uint8_t* const ref_ptr[], \
                            int  ref_stride, \
                            unsigned int *sad_array) {  \
  int i; \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
  for (i = 0; i < 4; i++) \
  sad_array[i] >>= 2; \
} \
static void fnname##_bits12(const uint8_t *src_ptr, \
                            int source_stride, \
                            const uint8_t* const ref_ptr[], \
                            int  ref_stride, \
                            unsigned int *sad_array) {  \
  int i; \
  fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array); \
  for (i = 0; i < 4; i++) \
  sad_array[i] >>= 4; \
}

Johann's avatar
Johann committed
961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015
MAKE_BFP_SAD_WRAPPER(vpx_highbd_sad32x16)
MAKE_BFP_SADAVG_WRAPPER(vpx_highbd_sad32x16_avg)
MAKE_BFP_SAD4D_WRAPPER(vpx_highbd_sad32x16x4d)
MAKE_BFP_SAD_WRAPPER(vpx_highbd_sad16x32)
MAKE_BFP_SADAVG_WRAPPER(vpx_highbd_sad16x32_avg)
MAKE_BFP_SAD4D_WRAPPER(vpx_highbd_sad16x32x4d)
MAKE_BFP_SAD_WRAPPER(vpx_highbd_sad64x32)
MAKE_BFP_SADAVG_WRAPPER(vpx_highbd_sad64x32_avg)
MAKE_BFP_SAD4D_WRAPPER(vpx_highbd_sad64x32x4d)
MAKE_BFP_SAD_WRAPPER(vpx_highbd_sad32x64)
MAKE_BFP_SADAVG_WRAPPER(vpx_highbd_sad32x64_avg)
MAKE_BFP_SAD4D_WRAPPER(vpx_highbd_sad32x64x4d)
MAKE_BFP_SAD_WRAPPER(vpx_highbd_sad32x32)
MAKE_BFP_SADAVG_WRAPPER(vpx_highbd_sad32x32_avg)
MAKE_BFP_SAD3_WRAPPER(vpx_highbd_sad32x32x3)
MAKE_BFP_SAD8_WRAPPER(vpx_highbd_sad32x32x8)
MAKE_BFP_SAD4D_WRAPPER(vpx_highbd_sad32x32x4d)
MAKE_BFP_SAD_WRAPPER(vpx_highbd_sad64x64)
MAKE_BFP_SADAVG_WRAPPER(vpx_highbd_sad64x64_avg)
MAKE_BFP_SAD3_WRAPPER(vpx_highbd_sad64x64x3)
MAKE_BFP_SAD8_WRAPPER(vpx_highbd_sad64x64x8)
MAKE_BFP_SAD4D_WRAPPER(vpx_highbd_sad64x64x4d)
MAKE_BFP_SAD_WRAPPER(vpx_highbd_sad16x16)
MAKE_BFP_SADAVG_WRAPPER(vpx_highbd_sad16x16_avg)
MAKE_BFP_SAD3_WRAPPER(vpx_highbd_sad16x16x3)
MAKE_BFP_SAD8_WRAPPER(vpx_highbd_sad16x16x8)
MAKE_BFP_SAD4D_WRAPPER(vpx_highbd_sad16x16x4d)
MAKE_BFP_SAD_WRAPPER(vpx_highbd_sad16x8)
MAKE_BFP_SADAVG_WRAPPER(vpx_highbd_sad16x8_avg)
MAKE_BFP_SAD3_WRAPPER(vpx_highbd_sad16x8x3)
MAKE_BFP_SAD8_WRAPPER(vpx_highbd_sad16x8x8)
MAKE_BFP_SAD4D_WRAPPER(vpx_highbd_sad16x8x4d)
MAKE_BFP_SAD_WRAPPER(vpx_highbd_sad8x16)
MAKE_BFP_SADAVG_WRAPPER(vpx_highbd_sad8x16_avg)
MAKE_BFP_SAD3_WRAPPER(vpx_highbd_sad8x16x3)
MAKE_BFP_SAD8_WRAPPER(vpx_highbd_sad8x16x8)
MAKE_BFP_SAD4D_WRAPPER(vpx_highbd_sad8x16x4d)
MAKE_BFP_SAD_WRAPPER(vpx_highbd_sad8x8)
MAKE_BFP_SADAVG_WRAPPER(vpx_highbd_sad8x8_avg)
MAKE_BFP_SAD3_WRAPPER(vpx_highbd_sad8x8x3)
MAKE_BFP_SAD8_WRAPPER(vpx_highbd_sad8x8x8)
MAKE_BFP_SAD4D_WRAPPER(vpx_highbd_sad8x8x4d)
MAKE_BFP_SAD_WRAPPER(vpx_highbd_sad8x4)
MAKE_BFP_SADAVG_WRAPPER(vpx_highbd_sad8x4_avg)
MAKE_BFP_SAD8_WRAPPER(vpx_highbd_sad8x4x8)
MAKE_BFP_SAD4D_WRAPPER(vpx_highbd_sad8x4x4d)
MAKE_BFP_SAD_WRAPPER(vpx_highbd_sad4x8)
MAKE_BFP_SADAVG_WRAPPER(vpx_highbd_sad4x8_avg)
MAKE_BFP_SAD8_WRAPPER(vpx_highbd_sad4x8x8)
MAKE_BFP_SAD4D_WRAPPER(vpx_highbd_sad4x8x4d)
MAKE_BFP_SAD_WRAPPER(vpx_highbd_sad4x4)
MAKE_BFP_SADAVG_WRAPPER(vpx_highbd_sad4x4_avg)
MAKE_BFP_SAD3_WRAPPER(vpx_highbd_sad4x4x3)
MAKE_BFP_SAD8_WRAPPER(vpx_highbd_sad4x4x8)
MAKE_BFP_SAD4D_WRAPPER(vpx_highbd_sad4x4x4d)
1016 1017

static void  highbd_set_var_fns(VP9_COMP *const cpi) {
1018 1019 1020 1021 1022
  VP9_COMMON *const cm = &cpi->common;
  if (cm->use_highbitdepth) {
    switch (cm->bit_depth) {
      case VPX_BITS_8:
        HIGHBD_BFP(BLOCK_32X16,
Johann's avatar
Johann committed
1023 1024
                   vpx_highbd_sad32x16_bits8,
                   vpx_highbd_sad32x16_avg_bits8,
Johann's avatar
Johann committed
1025
                   vpx_highbd_8_variance32x16,
1026 1027
                   vp9_highbd_sub_pixel_variance32x16,
                   vp9_highbd_sub_pixel_avg_variance32x16,
1028 1029
                   NULL,
                   NULL,
Johann's avatar
Johann committed
1030
                   vpx_highbd_sad32x16x4d_bits8)
1031 1032

        HIGHBD_BFP(BLOCK_16X32,
Johann's avatar
Johann committed
1033 1034
                   vpx_highbd_sad16x32_bits8,
                   vpx_highbd_sad16x32_avg_bits8,
Johann's avatar
Johann committed
1035
                   vpx_highbd_8_variance16x32,