encoder.c 138 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
4 5 6 7 8 9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10 11
 */

Jingning Han's avatar
Jingning Han committed
12
#include <limits.h>
Jingning Han's avatar
Jingning Han committed
13 14 15
#include <math.h>
#include <stdio.h>

Adrian Grange's avatar
Adrian Grange committed
16
#include "./aom_config.h"
Jingning Han's avatar
Jingning Han committed
17

Yaowu Xu's avatar
Yaowu Xu committed
18
#include "av1/common/alloccommon.h"
Steinar Midtskogen's avatar
Steinar Midtskogen committed
19
#if CONFIG_CLPF
Yaowu Xu's avatar
Yaowu Xu committed
20
#include "av1/common/clpf.h"
Steinar Midtskogen's avatar
Steinar Midtskogen committed
21
#endif
22
#if CONFIG_DERING
Yaowu Xu's avatar
Yaowu Xu committed
23
#include "av1/common/dering.h"
24
#endif  // CONFIG_DERING
Yaowu Xu's avatar
Yaowu Xu committed
25 26 27 28 29
#include "av1/common/filter.h"
#include "av1/common/idct.h"
#include "av1/common/reconinter.h"
#include "av1/common/reconintra.h"
#include "av1/common/tile_common.h"
Jingning Han's avatar
Jingning Han committed
30

Yaowu Xu's avatar
Yaowu Xu committed
31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49
#include "av1/encoder/aq_complexity.h"
#include "av1/encoder/aq_cyclicrefresh.h"
#include "av1/encoder/aq_variance.h"
#include "av1/encoder/bitstream.h"
#include "av1/encoder/context_tree.h"
#include "av1/encoder/encodeframe.h"
#include "av1/encoder/encodemv.h"
#include "av1/encoder/encoder.h"
#include "av1/encoder/ethread.h"
#include "av1/encoder/firstpass.h"
#include "av1/encoder/mbgraph.h"
#include "av1/encoder/picklpf.h"
#include "av1/encoder/ratectrl.h"
#include "av1/encoder/rd.h"
#include "av1/encoder/resize.h"
#include "av1/encoder/segmentation.h"
#include "av1/encoder/skin_detection.h"
#include "av1/encoder/speed_features.h"
#include "av1/encoder/temporal_filter.h"
Jingning Han's avatar
Jingning Han committed
50

Adrian Grange's avatar
Adrian Grange committed
51 52
#include "./aom_dsp_rtcd.h"
#include "./aom_scale_rtcd.h"
53 54
#include "./av1_rtcd.h"
#include "aom_dsp/psnr.h"
55
#if CONFIG_INTERNAL_STATS
Yaowu Xu's avatar
Yaowu Xu committed
56
#include "aom_dsp/ssim.h"
57
#endif
Adrian Grange's avatar
Adrian Grange committed
58 59
#include "aom_dsp/aom_dsp_common.h"
#include "aom_dsp/aom_filter.h"
Jingning Han's avatar
Jingning Han committed
60
#include "aom_ports/aom_timer.h"
Yaowu Xu's avatar
Yaowu Xu committed
61 62
#include "aom_ports/mem.h"
#include "aom_ports/system_state.h"
Adrian Grange's avatar
Adrian Grange committed
63
#include "aom_scale/aom_scale.h"
64

Jingning Han's avatar
Jingning Han committed
65 66 67
#define AM_SEGMENT_ID_INACTIVE 7
#define AM_SEGMENT_ID_ACTIVE 0

clang-format's avatar
clang-format committed
68 69 70 71 72 73 74 75
#define SHARP_FILTER_QTHRESH 0 /* Q threshold for 8-tap sharp filter */

#define ALTREF_HIGH_PRECISION_MV 1     // Whether to use high precision mv
                                       //  for altref computation.
#define HIGH_PRECISION_MV_QTHRESH 200  // Q threshold for high precision
                                       // mv. Choose a very high value for
                                       // now so that HIGH_PRECISION is always
                                       // chosen.
Jingning Han's avatar
Jingning Han committed
76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93
// #define OUTPUT_YUV_REC

#ifdef OUTPUT_YUV_DENOISED
FILE *yuv_denoised_file = NULL;
#endif
#ifdef OUTPUT_YUV_SKINMAP
FILE *yuv_skinmap_file = NULL;
#endif
#ifdef OUTPUT_YUV_REC
FILE *yuv_rec_file;
#endif

#if 0
FILE *framepsnr;
FILE *kf_list;
FILE *keyfile;
#endif

Adrian Grange's avatar
Adrian Grange committed
94
static INLINE void Scale2Ratio(AOM_SCALING mode, int *hr, int *hs) {
Jingning Han's avatar
Jingning Han committed
95 96 97 98 99 100 101 102 103 104 105 106
  switch (mode) {
    case NORMAL:
      *hr = 1;
      *hs = 1;
      break;
    case FOURFIVE:
      *hr = 4;
      *hs = 5;
      break;
    case THREEFIVE:
      *hr = 3;
      *hs = 5;
clang-format's avatar
clang-format committed
107
      break;
Jingning Han's avatar
Jingning Han committed
108 109 110
    case ONETWO:
      *hr = 1;
      *hs = 2;
clang-format's avatar
clang-format committed
111
      break;
Jingning Han's avatar
Jingning Han committed
112 113 114
    default:
      *hr = 1;
      *hs = 1;
clang-format's avatar
clang-format committed
115
      assert(0);
Jingning Han's avatar
Jingning Han committed
116 117 118 119 120 121
      break;
  }
}

// Mark all inactive blocks as active. Other segmentation features may be set
// so memset cannot be used, instead only inactive blocks should be reset.
122
static void suppress_active_map(AV1_COMP *cpi) {
Jingning Han's avatar
Jingning Han committed
123 124 125 126 127 128 129 130
  unsigned char *const seg_map = cpi->segmentation_map;
  int i;
  if (cpi->active_map.enabled || cpi->active_map.update)
    for (i = 0; i < cpi->common.mi_rows * cpi->common.mi_cols; ++i)
      if (seg_map[i] == AM_SEGMENT_ID_INACTIVE)
        seg_map[i] = AM_SEGMENT_ID_ACTIVE;
}

131
static void apply_active_map(AV1_COMP *cpi) {
Jingning Han's avatar
Jingning Han committed
132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147
  struct segmentation *const seg = &cpi->common.seg;
  unsigned char *const seg_map = cpi->segmentation_map;
  const unsigned char *const active_map = cpi->active_map.map;
  int i;

  assert(AM_SEGMENT_ID_ACTIVE == CR_SEGMENT_ID_BASE);

  if (frame_is_intra_only(&cpi->common)) {
    cpi->active_map.enabled = 0;
    cpi->active_map.update = 1;
  }

  if (cpi->active_map.update) {
    if (cpi->active_map.enabled) {
      for (i = 0; i < cpi->common.mi_rows * cpi->common.mi_cols; ++i)
        if (seg_map[i] == AM_SEGMENT_ID_ACTIVE) seg_map[i] = active_map[i];
148 149 150
      av1_enable_segmentation(seg);
      av1_enable_segfeature(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_SKIP);
      av1_enable_segfeature(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_ALT_LF);
Jingning Han's avatar
Jingning Han committed
151 152
      // Setting the data to -MAX_LOOP_FILTER will result in the computed loop
      // filter level being zero regardless of the value of seg->abs_delta.
153
      av1_set_segdata(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_ALT_LF,
clang-format's avatar
clang-format committed
154
                      -MAX_LOOP_FILTER);
Jingning Han's avatar
Jingning Han committed
155
    } else {
156 157
      av1_disable_segfeature(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_SKIP);
      av1_disable_segfeature(seg, AM_SEGMENT_ID_INACTIVE, SEG_LVL_ALT_LF);
Jingning Han's avatar
Jingning Han committed
158 159 160 161 162 163 164 165 166
      if (seg->enabled) {
        seg->update_data = 1;
        seg->update_map = 1;
      }
    }
    cpi->active_map.update = 0;
  }
}

167
int av1_set_active_map(AV1_COMP *cpi, unsigned char *new_map_16x16, int rows,
clang-format's avatar
clang-format committed
168
                       int cols) {
Jingning Han's avatar
Jingning Han committed
169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193
  if (rows == cpi->common.mb_rows && cols == cpi->common.mb_cols) {
    unsigned char *const active_map_8x8 = cpi->active_map.map;
    const int mi_rows = cpi->common.mi_rows;
    const int mi_cols = cpi->common.mi_cols;
    cpi->active_map.update = 1;
    if (new_map_16x16) {
      int r, c;
      for (r = 0; r < mi_rows; ++r) {
        for (c = 0; c < mi_cols; ++c) {
          active_map_8x8[r * mi_cols + c] =
              new_map_16x16[(r >> 1) * cols + (c >> 1)]
                  ? AM_SEGMENT_ID_ACTIVE
                  : AM_SEGMENT_ID_INACTIVE;
        }
      }
      cpi->active_map.enabled = 1;
    } else {
      cpi->active_map.enabled = 0;
    }
    return 0;
  } else {
    return -1;
  }
}

194
int av1_get_active_map(AV1_COMP *cpi, unsigned char *new_map_16x16, int rows,
clang-format's avatar
clang-format committed
195
                       int cols) {
Jingning Han's avatar
Jingning Han committed
196 197
  if (rows == cpi->common.mb_rows && cols == cpi->common.mb_cols &&
      new_map_16x16) {
clang-format's avatar
clang-format committed
198
    unsigned char *const seg_map_8x8 = cpi->segmentation_map;
Jingning Han's avatar
Jingning Han committed
199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218
    const int mi_rows = cpi->common.mi_rows;
    const int mi_cols = cpi->common.mi_cols;
    memset(new_map_16x16, !cpi->active_map.enabled, rows * cols);
    if (cpi->active_map.enabled) {
      int r, c;
      for (r = 0; r < mi_rows; ++r) {
        for (c = 0; c < mi_cols; ++c) {
          // Cyclic refresh segments are considered active despite not having
          // AM_SEGMENT_ID_ACTIVE
          new_map_16x16[(r >> 1) * cols + (c >> 1)] |=
              seg_map_8x8[r * mi_cols + c] != AM_SEGMENT_ID_INACTIVE;
        }
      }
    }
    return 0;
  } else {
    return -1;
  }
}

219
void av1_set_high_precision_mv(AV1_COMP *cpi, int allow_high_precision_mv) {
Jingning Han's avatar
Jingning Han committed
220 221
  MACROBLOCK *const mb = &cpi->td.mb;
  cpi->common.allow_high_precision_mv = allow_high_precision_mv;
222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237

#if CONFIG_REF_MV
  if (cpi->common.allow_high_precision_mv) {
    int i;
    for (i = 0; i < NMV_CONTEXTS; ++i) {
      mb->mv_cost_stack[i] = mb->nmvcost_hp[i];
      mb->mvsadcost = mb->nmvsadcost_hp;
    }
  } else {
    int i;
    for (i = 0; i < NMV_CONTEXTS; ++i) {
      mb->mv_cost_stack[i] = mb->nmvcost[i];
      mb->mvsadcost = mb->nmvsadcost;
    }
  }
#else
Jingning Han's avatar
Jingning Han committed
238 239 240 241 242 243 244
  if (cpi->common.allow_high_precision_mv) {
    mb->mvcost = mb->nmvcost_hp;
    mb->mvsadcost = mb->nmvsadcost_hp;
  } else {
    mb->mvcost = mb->nmvcost;
    mb->mvsadcost = mb->nmvsadcost;
  }
245
#endif
Jingning Han's avatar
Jingning Han committed
246 247
}

248 249
static void setup_frame(AV1_COMP *cpi) {
  AV1_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
250 251 252 253 254 255
  // Set up entropy context depending on frame type. The decoder mandates
  // the use of the default context, index 0, for keyframes and inter
  // frames where the error_resilient_mode or intra_only flag is set. For
  // other inter-frames the encoder currently uses only two contexts;
  // context 1 for ALTREF frames and context 0 for the others.
  if (frame_is_intra_only(cm) || cm->error_resilient_mode) {
256
    av1_setup_past_independence(cm);
Jingning Han's avatar
Jingning Han committed
257
  } else {
Yunqing Wang's avatar
Yunqing Wang committed
258
    cm->frame_context_idx = cpi->refresh_alt_ref_frame;
Jingning Han's avatar
Jingning Han committed
259 260 261
  }

  if (cm->frame_type == KEY_FRAME) {
Yunqing Wang's avatar
Yunqing Wang committed
262
    cpi->refresh_golden_frame = 1;
Jingning Han's avatar
Jingning Han committed
263
    cpi->refresh_alt_ref_frame = 1;
264
    av1_zero(cpi->interp_filter_selected);
Jingning Han's avatar
Jingning Han committed
265 266
  } else {
    *cm->fc = cm->frame_contexts[cm->frame_context_idx];
267
    av1_zero(cpi->interp_filter_selected[0]);
Jingning Han's avatar
Jingning Han committed
268 269 270
  }
}

271
static void av1_enc_setup_mi(AV1_COMMON *cm) {
Jingning Han's avatar
Jingning Han committed
272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288
  int i;
  cm->mi = cm->mip + cm->mi_stride + 1;
  memset(cm->mip, 0, cm->mi_stride * (cm->mi_rows + 1) * sizeof(*cm->mip));
  cm->prev_mi = cm->prev_mip + cm->mi_stride + 1;
  // Clear top border row
  memset(cm->prev_mip, 0, sizeof(*cm->prev_mip) * cm->mi_stride);
  // Clear left border column
  for (i = 1; i < cm->mi_rows + 1; ++i)
    memset(&cm->prev_mip[i * cm->mi_stride], 0, sizeof(*cm->prev_mip));

  cm->mi_grid_visible = cm->mi_grid_base + cm->mi_stride + 1;
  cm->prev_mi_grid_visible = cm->prev_mi_grid_base + cm->mi_stride + 1;

  memset(cm->mi_grid_base, 0,
         cm->mi_stride * (cm->mi_rows + 1) * sizeof(*cm->mi_grid_base));
}

289
static int av1_enc_alloc_mi(AV1_COMMON *cm, int mi_size) {
Adrian Grange's avatar
Adrian Grange committed
290
  cm->mip = aom_calloc(mi_size, sizeof(*cm->mip));
clang-format's avatar
clang-format committed
291
  if (!cm->mip) return 1;
Adrian Grange's avatar
Adrian Grange committed
292
  cm->prev_mip = aom_calloc(mi_size, sizeof(*cm->prev_mip));
clang-format's avatar
clang-format committed
293
  if (!cm->prev_mip) return 1;
Jingning Han's avatar
Jingning Han committed
294 295
  cm->mi_alloc_size = mi_size;

Adrian Grange's avatar
Adrian Grange committed
296
  cm->mi_grid_base = (MODE_INFO **)aom_calloc(mi_size, sizeof(MODE_INFO *));
clang-format's avatar
clang-format committed
297 298
  if (!cm->mi_grid_base) return 1;
  cm->prev_mi_grid_base =
Adrian Grange's avatar
Adrian Grange committed
299
      (MODE_INFO **)aom_calloc(mi_size, sizeof(MODE_INFO *));
clang-format's avatar
clang-format committed
300
  if (!cm->prev_mi_grid_base) return 1;
Jingning Han's avatar
Jingning Han committed
301 302 303 304

  return 0;
}

305
static void av1_enc_free_mi(AV1_COMMON *cm) {
Adrian Grange's avatar
Adrian Grange committed
306
  aom_free(cm->mip);
Jingning Han's avatar
Jingning Han committed
307
  cm->mip = NULL;
Adrian Grange's avatar
Adrian Grange committed
308
  aom_free(cm->prev_mip);
Jingning Han's avatar
Jingning Han committed
309
  cm->prev_mip = NULL;
Adrian Grange's avatar
Adrian Grange committed
310
  aom_free(cm->mi_grid_base);
Jingning Han's avatar
Jingning Han committed
311
  cm->mi_grid_base = NULL;
Adrian Grange's avatar
Adrian Grange committed
312
  aom_free(cm->prev_mi_grid_base);
Jingning Han's avatar
Jingning Han committed
313 314 315
  cm->prev_mi_grid_base = NULL;
}

316
static void av1_swap_mi_and_prev_mi(AV1_COMMON *cm) {
Jingning Han's avatar
Jingning Han committed
317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332
  // Current mip will be the prev_mip for the next frame.
  MODE_INFO **temp_base = cm->prev_mi_grid_base;
  MODE_INFO *temp = cm->prev_mip;
  cm->prev_mip = cm->mip;
  cm->mip = temp;

  // Update the upper left visible macroblock ptrs.
  cm->mi = cm->mip + cm->mi_stride + 1;
  cm->prev_mi = cm->prev_mip + cm->mi_stride + 1;

  cm->prev_mi_grid_base = cm->mi_grid_base;
  cm->mi_grid_base = temp_base;
  cm->mi_grid_visible = cm->mi_grid_base + cm->mi_stride + 1;
  cm->prev_mi_grid_visible = cm->prev_mi_grid_base + cm->mi_stride + 1;
}

333
void av1_initialize_enc(void) {
Jingning Han's avatar
Jingning Han committed
334 335 336
  static volatile int init_done = 0;

  if (!init_done) {
Yaowu Xu's avatar
Yaowu Xu committed
337
    av1_rtcd();
Adrian Grange's avatar
Adrian Grange committed
338 339
    aom_dsp_rtcd();
    aom_scale_rtcd();
340 341 342 343 344
    av1_init_intra_predictors();
    av1_init_me_luts();
    av1_rc_init_minq_luts();
    av1_entropy_mv_init();
    av1_encode_token_init();
Jingning Han's avatar
Jingning Han committed
345 346 347 348
    init_done = 1;
  }
}

349 350
static void dealloc_compressor_data(AV1_COMP *cpi) {
  AV1_COMMON *const cm = &cpi->common;
351 352 353
#if CONFIG_REF_MV
  int i;
#endif
Jingning Han's avatar
Jingning Han committed
354

Adrian Grange's avatar
Adrian Grange committed
355
  aom_free(cpi->mbmi_ext_base);
Jingning Han's avatar
Jingning Han committed
356 357
  cpi->mbmi_ext_base = NULL;

Adrian Grange's avatar
Adrian Grange committed
358
  aom_free(cpi->tile_data);
Jingning Han's avatar
Jingning Han committed
359 360 361
  cpi->tile_data = NULL;

  // Delete sementation map
Adrian Grange's avatar
Adrian Grange committed
362
  aom_free(cpi->segmentation_map);
Jingning Han's avatar
Jingning Han committed
363
  cpi->segmentation_map = NULL;
Adrian Grange's avatar
Adrian Grange committed
364
  aom_free(cpi->coding_context.last_frame_seg_map_copy);
Jingning Han's avatar
Jingning Han committed
365 366
  cpi->coding_context.last_frame_seg_map_copy = NULL;

367 368 369 370 371 372 373 374 375 376 377 378 379
#if CONFIG_REF_MV
  for (i = 0; i < NMV_CONTEXTS; ++i) {
    aom_free(cpi->nmv_costs[i][0]);
    aom_free(cpi->nmv_costs[i][1]);
    aom_free(cpi->nmv_costs_hp[i][0]);
    aom_free(cpi->nmv_costs_hp[i][1]);
    cpi->nmv_costs[i][0] = NULL;
    cpi->nmv_costs[i][1] = NULL;
    cpi->nmv_costs_hp[i][0] = NULL;
    cpi->nmv_costs_hp[i][1] = NULL;
  }
#endif

Adrian Grange's avatar
Adrian Grange committed
380 381
  aom_free(cpi->nmvcosts[0]);
  aom_free(cpi->nmvcosts[1]);
Jingning Han's avatar
Jingning Han committed
382 383 384
  cpi->nmvcosts[0] = NULL;
  cpi->nmvcosts[1] = NULL;

Adrian Grange's avatar
Adrian Grange committed
385 386
  aom_free(cpi->nmvcosts_hp[0]);
  aom_free(cpi->nmvcosts_hp[1]);
Jingning Han's avatar
Jingning Han committed
387 388 389
  cpi->nmvcosts_hp[0] = NULL;
  cpi->nmvcosts_hp[1] = NULL;

Adrian Grange's avatar
Adrian Grange committed
390 391
  aom_free(cpi->nmvsadcosts[0]);
  aom_free(cpi->nmvsadcosts[1]);
Jingning Han's avatar
Jingning Han committed
392 393 394
  cpi->nmvsadcosts[0] = NULL;
  cpi->nmvsadcosts[1] = NULL;

Adrian Grange's avatar
Adrian Grange committed
395 396
  aom_free(cpi->nmvsadcosts_hp[0]);
  aom_free(cpi->nmvsadcosts_hp[1]);
Jingning Han's avatar
Jingning Han committed
397 398 399
  cpi->nmvsadcosts_hp[0] = NULL;
  cpi->nmvsadcosts_hp[1] = NULL;

400
  av1_cyclic_refresh_free(cpi->cyclic_refresh);
Jingning Han's avatar
Jingning Han committed
401 402
  cpi->cyclic_refresh = NULL;

Adrian Grange's avatar
Adrian Grange committed
403
  aom_free(cpi->active_map.map);
Jingning Han's avatar
Jingning Han committed
404 405
  cpi->active_map.map = NULL;

406 407
  av1_free_ref_frame_buffers(cm->buffer_pool);
  av1_free_context_buffers(cm);
Jingning Han's avatar
Jingning Han committed
408

Adrian Grange's avatar
Adrian Grange committed
409 410 411 412
  aom_free_frame_buffer(&cpi->last_frame_uf);
  aom_free_frame_buffer(&cpi->scaled_source);
  aom_free_frame_buffer(&cpi->scaled_last_source);
  aom_free_frame_buffer(&cpi->alt_ref_buffer);
413
  av1_lookahead_destroy(cpi->lookahead);
Jingning Han's avatar
Jingning Han committed
414

Adrian Grange's avatar
Adrian Grange committed
415
  aom_free(cpi->tile_tok[0][0]);
Jingning Han's avatar
Jingning Han committed
416 417
  cpi->tile_tok[0][0] = 0;

418
  av1_free_pc_tree(&cpi->td);
Jingning Han's avatar
Jingning Han committed
419 420

  if (cpi->source_diff_var != NULL) {
Adrian Grange's avatar
Adrian Grange committed
421
    aom_free(cpi->source_diff_var);
Jingning Han's avatar
Jingning Han committed
422 423 424 425
    cpi->source_diff_var = NULL;
  }
}

426
static void save_coding_context(AV1_COMP *cpi) {
Jingning Han's avatar
Jingning Han committed
427
  CODING_CONTEXT *const cc = &cpi->coding_context;
428
  AV1_COMMON *cm = &cpi->common;
429 430 431
#if CONFIG_REF_MV
  int i;
#endif
Jingning Han's avatar
Jingning Han committed
432

clang-format's avatar
clang-format committed
433 434 435 436
// Stores a snapshot of key state variables which can subsequently be
// restored with a call to av1_restore_coding_context. These functions are
// intended for use in a re-code loop in av1_compress_frame where the
// quantizer value is adjusted between loop iterations.
437 438 439 440 441 442 443 444 445 446 447 448 449
#if CONFIG_REF_MV
  for (i = 0; i < NMV_CONTEXTS; ++i) {
    av1_copy(cc->nmv_vec_cost[i], cpi->td.mb.nmv_vec_cost[i]);
    memcpy(cc->nmv_costs[i][0], cpi->nmv_costs[i][0],
           MV_VALS * sizeof(*cpi->nmv_costs[i][0]));
    memcpy(cc->nmv_costs[i][1], cpi->nmv_costs[i][1],
           MV_VALS * sizeof(*cpi->nmv_costs[i][1]));
    memcpy(cc->nmv_costs_hp[i][0], cpi->nmv_costs_hp[i][0],
           MV_VALS * sizeof(*cpi->nmv_costs_hp[i][0]));
    memcpy(cc->nmv_costs_hp[i][1], cpi->nmv_costs_hp[i][1],
           MV_VALS * sizeof(*cpi->nmv_costs_hp[i][1]));
  }
#else
450
  av1_copy(cc->nmvjointcost, cpi->td.mb.nmvjointcost);
451
#endif
Jingning Han's avatar
Jingning Han committed
452 453 454 455 456 457 458 459 460 461

  memcpy(cc->nmvcosts[0], cpi->nmvcosts[0],
         MV_VALS * sizeof(*cpi->nmvcosts[0]));
  memcpy(cc->nmvcosts[1], cpi->nmvcosts[1],
         MV_VALS * sizeof(*cpi->nmvcosts[1]));
  memcpy(cc->nmvcosts_hp[0], cpi->nmvcosts_hp[0],
         MV_VALS * sizeof(*cpi->nmvcosts_hp[0]));
  memcpy(cc->nmvcosts_hp[1], cpi->nmvcosts_hp[1],
         MV_VALS * sizeof(*cpi->nmvcosts_hp[1]));

462
#if !CONFIG_MISC_FIXES
463
  av1_copy(cc->segment_pred_probs, cm->segp.pred_probs);
464
#endif
Jingning Han's avatar
Jingning Han committed
465

clang-format's avatar
clang-format committed
466 467
  memcpy(cpi->coding_context.last_frame_seg_map_copy, cm->last_frame_seg_map,
         (cm->mi_rows * cm->mi_cols));
Jingning Han's avatar
Jingning Han committed
468

469 470
  av1_copy(cc->last_ref_lf_deltas, cm->lf.last_ref_deltas);
  av1_copy(cc->last_mode_lf_deltas, cm->lf.last_mode_deltas);
Jingning Han's avatar
Jingning Han committed
471 472 473 474

  cc->fc = *cm->fc;
}

475
static void restore_coding_context(AV1_COMP *cpi) {
Jingning Han's avatar
Jingning Han committed
476
  CODING_CONTEXT *const cc = &cpi->coding_context;
477
  AV1_COMMON *cm = &cpi->common;
478 479 480
#if CONFIG_REF_MV
  int i;
#endif
Jingning Han's avatar
Jingning Han committed
481

clang-format's avatar
clang-format committed
482 483
// Restore key state variables to the snapshot state stored in the
// previous call to av1_save_coding_context.
484 485 486 487 488 489 490 491 492 493 494 495 496
#if CONFIG_REF_MV
  for (i = 0; i < NMV_CONTEXTS; ++i) {
    av1_copy(cpi->td.mb.nmv_vec_cost[i], cc->nmv_vec_cost[i]);
    memcpy(cpi->nmv_costs[i][0], cc->nmv_costs[i][0],
           MV_VALS * sizeof(*cc->nmv_costs[i][0]));
    memcpy(cpi->nmv_costs[i][1], cc->nmv_costs[i][1],
           MV_VALS * sizeof(*cc->nmv_costs[i][1]));
    memcpy(cpi->nmv_costs_hp[i][0], cc->nmv_costs_hp[i][0],
           MV_VALS * sizeof(*cc->nmv_costs_hp[i][0]));
    memcpy(cpi->nmv_costs_hp[i][1], cc->nmv_costs_hp[i][1],
           MV_VALS * sizeof(*cc->nmv_costs_hp[i][1]));
  }
#else
497
  av1_copy(cpi->td.mb.nmvjointcost, cc->nmvjointcost);
498
#endif
Jingning Han's avatar
Jingning Han committed
499 500 501 502 503 504 505 506

  memcpy(cpi->nmvcosts[0], cc->nmvcosts[0], MV_VALS * sizeof(*cc->nmvcosts[0]));
  memcpy(cpi->nmvcosts[1], cc->nmvcosts[1], MV_VALS * sizeof(*cc->nmvcosts[1]));
  memcpy(cpi->nmvcosts_hp[0], cc->nmvcosts_hp[0],
         MV_VALS * sizeof(*cc->nmvcosts_hp[0]));
  memcpy(cpi->nmvcosts_hp[1], cc->nmvcosts_hp[1],
         MV_VALS * sizeof(*cc->nmvcosts_hp[1]));

507
#if !CONFIG_MISC_FIXES
508
  av1_copy(cm->segp.pred_probs, cc->segment_pred_probs);
509
#endif
Jingning Han's avatar
Jingning Han committed
510

clang-format's avatar
clang-format committed
511
  memcpy(cm->last_frame_seg_map, cpi->coding_context.last_frame_seg_map_copy,
Jingning Han's avatar
Jingning Han committed
512 513
         (cm->mi_rows * cm->mi_cols));

514 515
  av1_copy(cm->lf.last_ref_deltas, cc->last_ref_lf_deltas);
  av1_copy(cm->lf.last_mode_deltas, cc->last_mode_lf_deltas);
Jingning Han's avatar
Jingning Han committed
516 517 518 519

  *cm->fc = cc->fc;
}

520 521
static void configure_static_seg_features(AV1_COMP *cpi) {
  AV1_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
522 523 524 525 526 527 528 529 530 531 532 533 534 535 536
  const RATE_CONTROL *const rc = &cpi->rc;
  struct segmentation *const seg = &cm->seg;

  int high_q = (int)(rc->avg_q > 48.0);
  int qi_delta;

  // Disable and clear down for KF
  if (cm->frame_type == KEY_FRAME) {
    // Clear down the global segmentation map
    memset(cpi->segmentation_map, 0, cm->mi_rows * cm->mi_cols);
    seg->update_map = 0;
    seg->update_data = 0;
    cpi->static_mb_pct = 0;

    // Disable segmentation
537
    av1_disable_segmentation(seg);
Jingning Han's avatar
Jingning Han committed
538 539

    // Clear down the segment features.
540
    av1_clearall_segfeatures(seg);
Jingning Han's avatar
Jingning Han committed
541 542 543 544 545 546 547 548 549
  } else if (cpi->refresh_alt_ref_frame) {
    // If this is an alt ref frame
    // Clear down the global segmentation map
    memset(cpi->segmentation_map, 0, cm->mi_rows * cm->mi_cols);
    seg->update_map = 0;
    seg->update_data = 0;
    cpi->static_mb_pct = 0;

    // Disable segmentation and individual segment features by default
550 551
    av1_disable_segmentation(seg);
    av1_clearall_segfeatures(seg);
Jingning Han's avatar
Jingning Han committed
552 553 554

    // Scan frames from current to arf frame.
    // This function re-enables segmentation if appropriate.
555
    av1_update_mbgraph_stats(cpi);
Jingning Han's avatar
Jingning Han committed
556 557 558 559 560 561 562

    // If segmentation was enabled set those features needed for the
    // arf itself.
    if (seg->enabled) {
      seg->update_map = 1;
      seg->update_data = 1;

clang-format's avatar
clang-format committed
563
      qi_delta =
564 565 566
          av1_compute_qdelta(rc, rc->avg_q, rc->avg_q * 0.875, cm->bit_depth);
      av1_set_segdata(seg, 1, SEG_LVL_ALT_Q, qi_delta - 2);
      av1_set_segdata(seg, 1, SEG_LVL_ALT_LF, -2);
Jingning Han's avatar
Jingning Han committed
567

568 569
      av1_enable_segfeature(seg, 1, SEG_LVL_ALT_Q);
      av1_enable_segfeature(seg, 1, SEG_LVL_ALT_LF);
Jingning Han's avatar
Jingning Han committed
570 571 572 573 574 575 576 577 578 579 580 581 582 583 584

      // Where relevant assume segment data is delta data
      seg->abs_delta = SEGMENT_DELTADATA;
    }
  } else if (seg->enabled) {
    // All other frames if segmentation has been enabled

    // First normal frame in a valid gf or alt ref group
    if (rc->frames_since_golden == 0) {
      // Set up segment features for normal frames in an arf group
      if (rc->source_alt_ref_active) {
        seg->update_map = 0;
        seg->update_data = 1;
        seg->abs_delta = SEGMENT_DELTADATA;

clang-format's avatar
clang-format committed
585 586
        qi_delta =
            av1_compute_qdelta(rc, rc->avg_q, rc->avg_q * 1.125, cm->bit_depth);
587 588
        av1_set_segdata(seg, 1, SEG_LVL_ALT_Q, qi_delta + 2);
        av1_enable_segfeature(seg, 1, SEG_LVL_ALT_Q);
Jingning Han's avatar
Jingning Han committed
589

590 591
        av1_set_segdata(seg, 1, SEG_LVL_ALT_LF, -2);
        av1_enable_segfeature(seg, 1, SEG_LVL_ALT_LF);
Jingning Han's avatar
Jingning Han committed
592 593 594

        // Segment coding disabled for compred testing
        if (high_q || (cpi->static_mb_pct == 100)) {
595 596 597
          av1_set_segdata(seg, 1, SEG_LVL_REF_FRAME, ALTREF_FRAME);
          av1_enable_segfeature(seg, 1, SEG_LVL_REF_FRAME);
          av1_enable_segfeature(seg, 1, SEG_LVL_SKIP);
Jingning Han's avatar
Jingning Han committed
598 599 600 601 602
        }
      } else {
        // Disable segmentation and clear down features if alt ref
        // is not active for this group

603
        av1_disable_segmentation(seg);
Jingning Han's avatar
Jingning Han committed
604 605 606 607 608 609

        memset(cpi->segmentation_map, 0, cm->mi_rows * cm->mi_cols);

        seg->update_map = 0;
        seg->update_data = 0;

610
        av1_clearall_segfeatures(seg);
Jingning Han's avatar
Jingning Han committed
611 612 613 614 615 616 617
      }
    } else if (rc->is_src_frame_alt_ref) {
      // Special case where we are coding over the top of a previous
      // alt ref frame.
      // Segment coding disabled for compred testing

      // Enable ref frame features for segment 0 as well
618 619
      av1_enable_segfeature(seg, 0, SEG_LVL_REF_FRAME);
      av1_enable_segfeature(seg, 1, SEG_LVL_REF_FRAME);
Jingning Han's avatar
Jingning Han committed
620 621

      // All mbs should use ALTREF_FRAME
622 623 624 625
      av1_clear_segdata(seg, 0, SEG_LVL_REF_FRAME);
      av1_set_segdata(seg, 0, SEG_LVL_REF_FRAME, ALTREF_FRAME);
      av1_clear_segdata(seg, 1, SEG_LVL_REF_FRAME);
      av1_set_segdata(seg, 1, SEG_LVL_REF_FRAME, ALTREF_FRAME);
Jingning Han's avatar
Jingning Han committed
626 627 628

      // Skip all MBs if high Q (0,0 mv and skip coeffs)
      if (high_q) {
629 630
        av1_enable_segfeature(seg, 0, SEG_LVL_SKIP);
        av1_enable_segfeature(seg, 1, SEG_LVL_SKIP);
Jingning Han's avatar
Jingning Han committed
631 632 633 634 635 636 637 638 639 640 641 642 643
      }
      // Enable data update
      seg->update_data = 1;
    } else {
      // All other frames.

      // No updates.. leave things as they are.
      seg->update_map = 0;
      seg->update_data = 0;
    }
  }
}

644 645
static void update_reference_segmentation_map(AV1_COMP *cpi) {
  AV1_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
646 647 648 649 650 651 652 653 654 655 656 657 658 659
  MODE_INFO **mi_8x8_ptr = cm->mi_grid_visible;
  uint8_t *cache_ptr = cm->last_frame_seg_map;
  int row, col;

  for (row = 0; row < cm->mi_rows; row++) {
    MODE_INFO **mi_8x8 = mi_8x8_ptr;
    uint8_t *cache = cache_ptr;
    for (col = 0; col < cm->mi_cols; col++, mi_8x8++, cache++)
      cache[0] = mi_8x8[0]->mbmi.segment_id;
    mi_8x8_ptr += cm->mi_stride;
    cache_ptr += cm->mi_cols;
  }
}

660 661 662
static void alloc_raw_frame_buffers(AV1_COMP *cpi) {
  AV1_COMMON *cm = &cpi->common;
  const AV1EncoderConfig *oxcf = &cpi->oxcf;
Jingning Han's avatar
Jingning Han committed
663 664

  if (!cpi->lookahead)
665
    cpi->lookahead = av1_lookahead_init(oxcf->width, oxcf->height,
clang-format's avatar
clang-format committed
666
                                        cm->subsampling_x, cm->subsampling_y,
667
#if CONFIG_AOM_HIGHBITDEPTH
clang-format's avatar
clang-format committed
668
                                        cm->use_highbitdepth,
Jingning Han's avatar
Jingning Han committed
669
#endif
clang-format's avatar
clang-format committed
670
                                        oxcf->lag_in_frames);
Jingning Han's avatar
Jingning Han committed
671
  if (!cpi->lookahead)
Adrian Grange's avatar
Adrian Grange committed
672
    aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR,
Jingning Han's avatar
Jingning Han committed
673 674 675
                       "Failed to allocate lag buffers");

  // TODO(agrange) Check if ARF is enabled and skip allocation if not.
Adrian Grange's avatar
Adrian Grange committed
676
  if (aom_realloc_frame_buffer(&cpi->alt_ref_buffer, oxcf->width, oxcf->height,
Jingning Han's avatar
Jingning Han committed
677
                               cm->subsampling_x, cm->subsampling_y,
678
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
679 680
                               cm->use_highbitdepth,
#endif
Adrian Grange's avatar
Adrian Grange committed
681
                               AOM_ENC_BORDER_IN_PIXELS, cm->byte_alignment,
Jingning Han's avatar
Jingning Han committed
682
                               NULL, NULL, NULL))
Adrian Grange's avatar
Adrian Grange committed
683
    aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR,
Jingning Han's avatar
Jingning Han committed
684 685 686
                       "Failed to allocate altref buffer");
}

687 688
static void alloc_util_frame_buffers(AV1_COMP *cpi) {
  AV1_COMMON *const cm = &cpi->common;
Adrian Grange's avatar
Adrian Grange committed
689
  if (aom_realloc_frame_buffer(&cpi->last_frame_uf, cm->width, cm->height,
Jingning Han's avatar
Jingning Han committed
690
                               cm->subsampling_x, cm->subsampling_y,
691
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
692 693
                               cm->use_highbitdepth,
#endif
Adrian Grange's avatar
Adrian Grange committed
694
                               AOM_ENC_BORDER_IN_PIXELS, cm->byte_alignment,
Jingning Han's avatar
Jingning Han committed
695
                               NULL, NULL, NULL))
Adrian Grange's avatar
Adrian Grange committed
696
    aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR,
Jingning Han's avatar
Jingning Han committed
697 698
                       "Failed to allocate last frame buffer");

Adrian Grange's avatar
Adrian Grange committed
699
  if (aom_realloc_frame_buffer(&cpi->scaled_source, cm->width, cm->height,
Jingning Han's avatar
Jingning Han committed
700
                               cm->subsampling_x, cm->subsampling_y,
701
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
702 703
                               cm->use_highbitdepth,
#endif
Adrian Grange's avatar
Adrian Grange committed
704
                               AOM_ENC_BORDER_IN_PIXELS, cm->byte_alignment,
Jingning Han's avatar
Jingning Han committed
705
                               NULL, NULL, NULL))
Adrian Grange's avatar
Adrian Grange committed
706
    aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR,
Jingning Han's avatar
Jingning Han committed
707 708
                       "Failed to allocate scaled source buffer");

Adrian Grange's avatar
Adrian Grange committed
709
  if (aom_realloc_frame_buffer(&cpi->scaled_last_source, cm->width, cm->height,
Jingning Han's avatar
Jingning Han committed
710
                               cm->subsampling_x, cm->subsampling_y,
711
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
712 713
                               cm->use_highbitdepth,
#endif
Adrian Grange's avatar
Adrian Grange committed
714
                               AOM_ENC_BORDER_IN_PIXELS, cm->byte_alignment,
Jingning Han's avatar
Jingning Han committed
715
                               NULL, NULL, NULL))
Adrian Grange's avatar
Adrian Grange committed
716
    aom_internal_error(&cm->error, AOM_CODEC_MEM_ERROR,
Jingning Han's avatar
Jingning Han committed
717 718 719
                       "Failed to allocate scaled last source buffer");
}

720 721
static int alloc_context_buffers_ext(AV1_COMP *cpi) {
  AV1_COMMON *cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
722 723
  int mi_size = cm->mi_cols * cm->mi_rows;

Adrian Grange's avatar
Adrian Grange committed
724
  cpi->mbmi_ext_base = aom_calloc(mi_size, sizeof(*cpi->mbmi_ext_base));
clang-format's avatar
clang-format committed
725
  if (!cpi->mbmi_ext_base) return 1;
Jingning Han's avatar
Jingning Han committed
726 727 728 729

  return 0;
}

730 731
void av1_alloc_compressor_data(AV1_COMP *cpi) {
  AV1_COMMON *cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
732

733
  av1_alloc_context_buffers(cm, cm->width, cm->height);
Jingning Han's avatar
Jingning Han committed
734 735 736

  alloc_context_buffers_ext(cpi);

Adrian Grange's avatar
Adrian Grange committed
737
  aom_free(cpi->tile_tok[0][0]);
Jingning Han's avatar
Jingning Han committed
738 739 740 741

  {
    unsigned int tokens = get_token_alloc(cm->mb_rows, cm->mb_cols);
    CHECK_MEM_ERROR(cm, cpi->tile_tok[0][0],
Adrian Grange's avatar
Adrian Grange committed
742
                    aom_calloc(tokens, sizeof(*cpi->tile_tok[0][0])));
Jingning Han's avatar
Jingning Han committed
743 744
  }

745
  av1_setup_pc_tree(&cpi->common, &cpi->td);
Jingning Han's avatar
Jingning Han committed
746 747
}

748
void av1_new_framerate(AV1_COMP *cpi, double framerate) {
Jingning Han's avatar
Jingning Han committed
749
  cpi->framerate = framerate < 0.1 ? 30 : framerate;
750
  av1_rc_update_framerate(cpi);
Jingning Han's avatar
Jingning Han committed
751 752
}

753 754
static void set_tile_limits(AV1_COMP *cpi) {
  AV1_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
755 756

  int min_log2_tile_cols, max_log2_tile_cols;
757
  av1_get_tile_n_bits(cm->mi_cols, &min_log2_tile_cols, &max_log2_tile_cols);
Jingning Han's avatar
Jingning Han committed
758

clang-format's avatar
clang-format committed
759 760
  cm->log2_tile_cols =
      clamp(cpi->oxcf.tile_columns, min_log2_tile_cols, max_log2_tile_cols);
Yunqing Wang's avatar
Yunqing Wang committed
761
  cm->log2_tile_rows = cpi->oxcf.tile_rows;
Jingning Han's avatar
Jingning Han committed
762 763
}

764 765
static void update_frame_size(AV1_COMP *cpi) {
  AV1_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
766 767
  MACROBLOCKD *const xd = &cpi->td.mb.e_mbd;

768 769 770
  av1_set_mb_mi(cm, cm->width, cm->height);
  av1_init_context_buffers(cm);
  av1_init_macroblockd(cm, xd, NULL);
Jingning Han's avatar
Jingning Han committed
771 772 773 774 775 776
  memset(cpi->mbmi_ext_base, 0,
         cm->mi_rows * cm->mi_cols * sizeof(*cpi->mbmi_ext_base));

  set_tile_limits(cpi);
}

777
static void init_buffer_indices(AV1_COMP *cpi) {
Jingning Han's avatar
Jingning Han committed
778 779 780 781 782
  cpi->lst_fb_idx = 0;
  cpi->gld_fb_idx = 1;
  cpi->alt_fb_idx = 2;
}

783 784
static void init_config(struct AV1_COMP *cpi, AV1EncoderConfig *oxcf) {
  AV1_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
785 786 787 788 789 790

  cpi->oxcf = *oxcf;
  cpi->framerate = oxcf->init_framerate;

  cm->profile = oxcf->profile;
  cm->bit_depth = oxcf->bit_depth;
791
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
792 793 794
  cm->use_highbitdepth = oxcf->use_highbitdepth;
#endif
  cm->color_space = oxcf->color_space;
795
  cm->color_range = oxcf->color_range;
Jingning Han's avatar
Jingning Han committed
796 797 798

  cm->width = oxcf->width;
  cm->height = oxcf->height;
799
  av1_alloc_compressor_data(cpi);
Jingning Han's avatar
Jingning Han committed
800 801 802 803 804

  // Single thread case: use counts in common.
  cpi->td.counts = &cm->counts;

  // change includes all joint functionality
805
  av1_change_config(cpi, oxcf);
Jingning Han's avatar
Jingning Han committed
806 807 808 809 810 811 812 813

  cpi->static_mb_pct = 0;
  cpi->ref_frame_flags = 0;

  init_buffer_indices(cpi);
}

static void set_rc_buffer_sizes(RATE_CONTROL *rc,
814
                                const AV1EncoderConfig *oxcf) {
Jingning Han's avatar
Jingning Han committed
815 816 817 818 819 820
  const int64_t bandwidth = oxcf->target_bandwidth;
  const int64_t starting = oxcf->starting_buffer_level_ms;
  const int64_t optimal = oxcf->optimal_buffer_level_ms;
  const int64_t maximum = oxcf->maximum_buffer_size_ms;

  rc->starting_buffer_level = starting * bandwidth / 1000;
clang-format's avatar
clang-format committed
821 822 823 824
  rc->optimal_buffer_level =
      (optimal == 0) ? bandwidth / 8 : optimal * bandwidth / 1000;
  rc->maximum_buffer_size =
      (maximum == 0) ? bandwidth / 8 : maximum * bandwidth / 1000;
Jingning Han's avatar
Jingning Han committed
825 826
}

827
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
828
#define HIGHBD_BFP(BT, SDF, SDAF, VF, SVF, SVAF, SDX3F, SDX8F, SDX4DF) \
clang-format's avatar
clang-format committed
829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936
  cpi->fn_ptr[BT].sdf = SDF;                                           \
  cpi->fn_ptr[BT].sdaf = SDAF;                                         \
  cpi->fn_ptr[BT].vf = VF;                                             \
  cpi->fn_ptr[BT].svf = SVF;                                           \
  cpi->fn_ptr[BT].svaf = SVAF;                                         \
  cpi->fn_ptr[BT].sdx3f = SDX3F;                                       \
  cpi->fn_ptr[BT].sdx8f = SDX8F;                                       \
  cpi->fn_ptr[BT].sdx4df = SDX4DF;

#define MAKE_BFP_SAD_WRAPPER(fnname)                                           \
  static unsigned int fnname##_bits8(const uint8_t *src_ptr,                   \
                                     int source_stride,                        \
                                     const uint8_t *ref_ptr, int ref_stride) { \
    return fnname(src_ptr, source_stride, ref_ptr, ref_stride);                \
  }                                                                            \
  static unsigned int fnname##_bits10(                                         \
      const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr,       \
      int ref_stride) {                                                        \
    return fnname(src_ptr, source_stride, ref_ptr, ref_stride) >> 2;           \
  }                                                                            \
  static unsigned int fnname##_bits12(                                         \
      const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr,       \
      int ref_stride) {                                                        \
    return fnname(src_ptr, source_stride, ref_ptr, ref_stride) >> 4;           \
  }

#define MAKE_BFP_SADAVG_WRAPPER(fnname)                                        \
  static unsigned int fnname##_bits8(                                          \
      const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr,       \
      int ref_stride, const uint8_t *second_pred) {                            \
    return fnname(src_ptr, source_stride, ref_ptr, ref_stride, second_pred);   \
  }                                                                            \
  static unsigned int fnname##_bits10(                                         \
      const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr,       \
      int ref_stride, const uint8_t *second_pred) {                            \
    return fnname(src_ptr, source_stride, ref_ptr, ref_stride, second_pred) >> \
           2;                                                                  \
  }                                                                            \
  static unsigned int fnname##_bits12(                                         \
      const uint8_t *src_ptr, int source_stride, const uint8_t *ref_ptr,       \
      int ref_stride, const uint8_t *second_pred) {                            \
    return fnname(src_ptr, source_stride, ref_ptr, ref_stride, second_pred) >> \
           4;                                                                  \
  }

#define MAKE_BFP_SAD3_WRAPPER(fnname)                                    \
  static void fnname##_bits8(const uint8_t *src_ptr, int source_stride,  \
                             const uint8_t *ref_ptr, int ref_stride,     \
                             unsigned int *sad_array) {                  \
    fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array);      \
  }                                                                      \
  static void fnname##_bits10(const uint8_t *src_ptr, int source_stride, \
                              const uint8_t *ref_ptr, int ref_stride,    \
                              unsigned int *sad_array) {                 \
    int i;                                                               \
    fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array);      \
    for (i = 0; i < 3; i++) sad_array[i] >>= 2;                          \
  }                                                                      \
  static void fnname##_bits12(const uint8_t *src_ptr, int source_stride, \
                              const uint8_t *ref_ptr, int ref_stride,    \
                              unsigned int *sad_array) {                 \
    int i;                                                               \
    fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array);      \
    for (i = 0; i < 3; i++) sad_array[i] >>= 4;                          \
  }

#define MAKE_BFP_SAD8_WRAPPER(fnname)                                    \
  static void fnname##_bits8(const uint8_t *src_ptr, int source_stride,  \
                             const uint8_t *ref_ptr, int ref_stride,     \
                             unsigned int *sad_array) {                  \
    fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array);      \
  }                                                                      \
  static void fnname##_bits10(const uint8_t *src_ptr, int source_stride, \
                              const uint8_t *ref_ptr, int ref_stride,    \
                              unsigned int *sad_array) {                 \
    int i;                                                               \
    fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array);      \
    for (i = 0; i < 8; i++) sad_array[i] >>= 2;                          \
  }                                                                      \
  static void fnname##_bits12(const uint8_t *src_ptr, int source_stride, \
                              const uint8_t *ref_ptr, int ref_stride,    \
                              unsigned int *sad_array) {                 \
    int i;                                                               \
    fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array);      \
    for (i = 0; i < 8; i++) sad_array[i] >>= 4;                          \
  }
#define MAKE_BFP_SAD4D_WRAPPER(fnname)                                        \
  static void fnname##_bits8(const uint8_t *src_ptr, int source_stride,       \
                             const uint8_t *const ref_ptr[], int ref_stride,  \
                             unsigned int *sad_array) {                       \
    fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array);           \
  }                                                                           \
  static void fnname##_bits10(const uint8_t *src_ptr, int source_stride,      \
                              const uint8_t *const ref_ptr[], int ref_stride, \
                              unsigned int *sad_array) {                      \
    int i;                                                                    \
    fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array);           \
    for (i = 0; i < 4; i++) sad_array[i] >>= 2;                               \
  }                                                                           \
  static void fnname##_bits12(const uint8_t *src_ptr, int source_stride,      \
                              const uint8_t *const ref_ptr[], int ref_stride, \
                              unsigned int *sad_array) {                      \
    int i;                                                                    \
    fnname(src_ptr, source_stride, ref_ptr, ref_stride, sad_array);           \
    for (i = 0; i < 4; i++) sad_array[i] >>= 4;                               \
  }

/* clang-format off */
Adrian Grange's avatar
Adrian Grange committed
937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991
MAKE_BFP_SAD_WRAPPER(aom_highbd_sad32x16)
MAKE_BFP_SADAVG_WRAPPER(aom_highbd_sad32x16_avg)
MAKE_BFP_SAD4D_WRAPPER(aom_highbd_sad32x16x4d)
MAKE_BFP_SAD_WRAPPER(aom_highbd_sad16x32)
MAKE_BFP_SADAVG_WRAPPER(aom_highbd_sad16x32_avg)
MAKE_BFP_SAD4D_WRAPPER(aom_highbd_sad16x32x4d)
MAKE_BFP_SAD_WRAPPER(aom_highbd_sad64x32)
MAKE_BFP_SADAVG_WRAPPER(aom_highbd_sad64x32_avg)
MAKE_BFP_SAD4D_WRAPPER(aom_highbd_sad64x32x4d)
MAKE_BFP_SAD_WRAPPER(aom_highbd_sad32x64)
MAKE_BFP_SADAVG_WRAPPER(aom_highbd_sad32x64_avg)
MAKE_BFP_SAD4D_WRAPPER(aom_highbd_sad32x64x4d)
MAKE_BFP_SAD_WRAPPER(aom_highbd_sad32x32)
MAKE_BFP_SADAVG_WRAPPER(aom_highbd_sad32x32_avg)
MAKE_BFP_SAD3_WRAPPER(aom_highbd_sad32x32x3)
MAKE_BFP_SAD8_WRAPPER(aom_highbd_sad32x32x8)
MAKE_BFP_SAD4D_WRAPPER(aom_highbd_sad32x32x4d)
MAKE_BFP_SAD_WRAPPER(aom_highbd_sad64x64)
MAKE_BFP_SADAVG_WRAPPER(aom_highbd_sad64x64_avg)
MAKE_BFP_SAD3_WRAPPER(aom_highbd_sad64x64x3)
MAKE_BFP_SAD8_WRAPPER(aom_highbd_sad64x64x8)
MAKE_BFP_SAD4D_WRAPPER(aom_highbd_sad64x64x4d)
MAKE_BFP_SAD_WRAPPER(aom_highbd_sad16x16)
MAKE_BFP_SADAVG_WRAPPER(aom_highbd_sad16x16_avg)
MAKE_BFP_SAD3_WRAPPER(aom_highbd_sad16x16x3)
MAKE_BFP_SAD8_WRAPPER(aom_highbd_sad16x16x8)
MAKE_BFP_SAD4D_WRAPPER(aom_highbd_sad16x16x4d)
MAKE_BFP_SAD_WRAPPER(aom_highbd_sad16x8)
MAKE_BFP_SADAVG_WRAPPER(aom_highbd_sad16x8_avg)
MAKE_BFP_SAD3_WRAPPER(aom_highbd_sad16x8x3)
MAKE_BFP_SAD8_WRAPPER(aom_highbd_sad16x8x8)
MAKE_BFP_SAD4D_WRAPPER(aom_highbd_sad16x8x4d)
MAKE_BFP_SAD_WRAPPER(aom_highbd_sad8x16)
MAKE_BFP_SADAVG_WRAPPER(aom_highbd_sad8x16_avg)
MAKE_BFP_SAD3_WRAPPER(aom_highbd_sad8x16x3)
MAKE_BFP_SAD8_WRAPPER(aom_highbd_sad8x16x8)
MAKE_BFP_SAD4D_WRAPPER(aom_highbd_sad8x16x4d)
MAKE_BFP_SAD_WRAPPER(aom_highbd_sad8x8)
MAKE_BFP_SADAVG_WRAPPER(aom_highbd_sad8x8_avg)
MAKE_BFP_SAD3_WRAPPER(aom_highbd_sad8x8x3)
MAKE_BFP_SAD8_WRAPPER(aom_highbd_sad8x8x8)
MAKE_BFP_SAD4D_WRAPPER(aom_highbd_sad8x8x4d)
MAKE_BFP_SAD_WRAPPER(aom_highbd_sad8x4)
MAKE_BFP_SADAVG_WRAPPER(aom_highbd_sad8x4_avg)
MAKE_BFP_SAD8_WRAPPER(aom_highbd_sad8x4x8)
MAKE_BFP_SAD4D_WRAPPER(aom_highbd_sad8x4x4d)
MAKE_BFP_SAD_WRAPPER(aom_highbd_sad4x8)
MAKE_BFP_SADAVG_WRAPPER(aom_highbd_sad4x8_avg)
MAKE_BFP_SAD8_WRAPPER(aom_highbd_sad4x8x8)
MAKE_BFP_SAD4D_WRAPPER(aom_highbd_sad4x8x4d)
MAKE_BFP_SAD_WRAPPER(aom_highbd_sad4x4)
MAKE_BFP_SADAVG_WRAPPER(aom_highbd_sad4x4_avg)
MAKE_BFP_SAD3_WRAPPER(aom_highbd_sad4x4x3)
MAKE_BFP_SAD8_WRAPPER(aom_highbd_sad4x4x8)
MAKE_BFP_SAD4D_WRAPPER(aom_highbd_sad4x4x4d)
clang-format's avatar
clang-format committed
992
/* clang-format on */
Jingning Han's avatar
Jingning Han committed
993

994 995
static void highbd_set_var_fns(AV1_COMP *const cpi) {
  AV1_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
996 997
  if (cm->use_highbitdepth) {
    switch (cm->bit_depth) {
Adrian Grange's avatar
Adrian Grange committed
998
      case AOM_BITS_8:
Adrian Grange's avatar
Adrian Grange committed
999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042
        HIGHBD_BFP(BLOCK_32X16, aom_highbd_sad32x16_bits8,
                   aom_highbd_sad32x16_avg_bits8, aom_highbd_8_variance32x16,
                   aom_highbd_8_sub_pixel_variance32x16,
                   aom_highbd_8_sub_pixel_avg_variance32x16, NULL, NULL,
                   aom_highbd_sad32x16x4d_bits8)

        HIGHBD_BFP(BLOCK_16X32, aom_highbd_sad16x32_bits8,
                   aom_highbd_sad16x32_avg_bits8, aom_highbd_8_variance16x32,
                   aom_highbd_8_sub_pixel_variance16x32,
                   aom_highbd_8_sub_pixel_avg_variance16x32, NULL, NULL,
                   aom_highbd_sad16x32x4d_bits8)

        HIGHBD_BFP(BLOCK_64X32, aom_highbd_sad64x32_bits8,
                   aom_highbd_sad64x32_avg_bits8, aom_highbd_8_variance64x32,
                   aom_highbd_8_sub_pixel_variance64x32,
                   aom_highbd_8_sub_pixel_avg_variance64x32, NULL, NULL,
                   aom_highbd_sad64x32x4d_bits8)

        HIGHBD_BFP(BLOCK_32X64, aom_highbd_sad32x64_bits8,
                   aom_highbd_sad32x64_avg_bits8, aom_highbd_8_variance32x64,
                   aom_highbd_8_sub_pixel_variance32x64,
                   aom_highbd_8_sub_pixel_avg_variance32x64, NULL, NULL,
                   aom_highbd_sad32x64x4d_bits8)

        HIGHBD_BFP(BLOCK_32X32, aom_highbd_sad32x32_bits8,
                   aom_highbd_sad32x32_avg_bits8, aom_highbd_8_variance32x32,
                   aom_highbd_8_sub_pixel_variance32x32,
                   aom_highbd_8_sub_pixel_avg_variance32x32,
                   aom_highbd_sad32x32x3_bits8, aom_highbd_sad32x32x8_bits8,
                   aom_highbd_sad32x32x4d_bits8)

        HIGHBD_BFP(BLOCK_64X64, aom_highbd_sad64x64_bits8,
                   aom_highbd_sad64x64_avg_bits8, aom_highbd_8_variance64x64,
                   aom_highbd_8_sub_pixel_variance64x64,
                   aom_highbd_8_sub_pixel_avg_variance64x64,
                   aom_highbd_sad64x64x3_bits8, aom_highbd_sad64x64x8_bits8,
                   aom_highbd_sad64x64x4d_bits8)

        HIGHBD_BFP(BLOCK_16X16, aom_highbd_sad16x16_bits8,
                   aom_highbd_sad16x16_avg_bits8, aom_highbd_8_variance16x16,
                   aom_highbd_8_sub_pixel_variance16x16,
                   aom_highbd_8_sub_pixel_avg_variance16x16,
                   aom_highbd_sad16x16x3_bits8, aom_highbd_sad16x16x8_bits8,
                   aom_highbd_sad16x16x4d_bits8)
Jingning Han's avatar
Jingning Han committed
1043

clang-format's avatar
clang-format committed
1044
        HIGHBD_BFP(
Adrian Grange's avatar
Adrian Grange committed
1045 1046 1047 1048
            BLOCK_16X8, aom_highbd_sad16x8_bits8, aom_highbd_sad16x8_avg_bits8,
            aom_highbd_8_variance16x8, aom_highbd_8_sub_pixel_variance16x8,
            aom_highbd_8_sub_pixel_avg_variance16x8, aom_highbd_sad16x8x3_bits8,
            aom_highbd_sad16x8x8_bits8, aom_highbd_sad16x8x4d_bits8)
clang-format's avatar
clang-format committed
1049 1050

        HIGHBD_BFP(
Adrian Grange's avatar
Adrian Grange committed
1051 1052 1053 1054
            BLOCK_8X16, aom_highbd_sad8x16_bits8, aom_highbd_sad8x16_avg_bits8,
            aom_highbd_8_variance8x16, aom_highbd_8_sub_pixel_variance8x16,
            aom_highbd_8_sub_pixel_avg_variance8x16, aom_highbd_sad8x16x3_bits8,
            aom_highbd_sad8x16x8_bits8, aom_highbd_sad8x16x4d_bits8)
clang-format's avatar
clang-format committed
1055 1056

        HIGHBD_BFP(
Adrian Grange's avatar
Adrian Grange committed
1057 1058 1059 1060 1061 1062