rd.c 57.5 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4 5 6 7 8 9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10 11 12 13 14 15
 */

#include <assert.h>
#include <math.h>
#include <stdio.h>

Yaowu Xu's avatar
Yaowu Xu committed
16
#include "./av1_rtcd.h"
Jingning Han's avatar
Jingning Han committed
17

Yaowu Xu's avatar
Yaowu Xu committed
18 19
#include "aom_dsp/aom_dsp_common.h"
#include "aom_mem/aom_mem.h"
20 21 22
#include "aom_ports/bitops.h"
#include "aom_ports/mem.h"
#include "aom_ports/system_state.h"
Jingning Han's avatar
Jingning Han committed
23

24 25 26 27 28 29 30 31 32
#include "av1/common/common.h"
#include "av1/common/entropy.h"
#include "av1/common/entropymode.h"
#include "av1/common/mvref_common.h"
#include "av1/common/pred_common.h"
#include "av1/common/quant_common.h"
#include "av1/common/reconinter.h"
#include "av1/common/reconintra.h"
#include "av1/common/seg_common.h"
33

34
#include "av1/encoder/av1_quantize.h"
35 36 37 38 39 40 41 42
#include "av1/encoder/cost.h"
#include "av1/encoder/encodemb.h"
#include "av1/encoder/encodemv.h"
#include "av1/encoder/encoder.h"
#include "av1/encoder/mcomp.h"
#include "av1/encoder/ratectrl.h"
#include "av1/encoder/rd.h"
#include "av1/encoder/tokenize.h"
Jingning Han's avatar
Jingning Han committed
43

44
#define RD_THRESH_POW 1.25
Jingning Han's avatar
Jingning Han committed
45 46 47 48 49 50 51 52

// Factor to weigh the rate for switchable interp filters.
#define SWITCHABLE_INTERP_RATE_FACTOR 1

// The baseline rd thresholds for breaking out of the rd loop for
// certain modes are assumed to be based on 8x8 blocks.
// This table is used to correct for block size.
// The factors here are << 2 (2 = x0.5, 32 = x8 etc).
53
static const uint8_t rd_thresh_block_size_factor[BLOCK_SIZES_ALL] = {
54
#if CONFIG_CHROMA_2X2 || CONFIG_CHROMA_SUB8X8
Jingning Han's avatar
Jingning Han committed
55 56
  2,  2,  2,
#endif
57
  2,  3,  3,  4, 6,  6, 8, 12, 12, 16, 24, 24, 32,
58
#if CONFIG_EXT_PARTITION
59
  48, 48, 64,
60
#endif  // CONFIG_EXT_PARTITION
61
  4,  4,  8,  8, 16, 16
Jingning Han's avatar
Jingning Han committed
62 63
};

64 65
void av1_fill_mode_rates(AV1_COMMON *const cm, MACROBLOCK *x,
                         FRAME_CONTEXT *fc) {
Jingning Han's avatar
Jingning Han committed
66 67
  int i, j;

68 69
  if (cm->frame_type == KEY_FRAME) {
    for (i = 0; i < PARTITION_CONTEXTS_PRIMARY; ++i)
Yue Chen's avatar
Yue Chen committed
70 71
      av1_cost_tokens_from_cdf(x->partition_cost[i], fc->partition_cdf[i],
                               NULL);
72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95
#if CONFIG_UNPOISON_PARTITION_CTX
    for (; i < PARTITION_CONTEXTS_PRIMARY + PARTITION_BLOCK_SIZES; ++i) {
      aom_prob p = fc->partition_prob[i][PARTITION_VERT];
      assert(p > 0);
      x->partition_cost[i][PARTITION_NONE] = INT_MAX;
      x->partition_cost[i][PARTITION_HORZ] = INT_MAX;
      x->partition_cost[i][PARTITION_VERT] = av1_cost_bit(p, 0);
      x->partition_cost[i][PARTITION_SPLIT] = av1_cost_bit(p, 1);
    }
    for (; i < PARTITION_CONTEXTS_PRIMARY + 2 * PARTITION_BLOCK_SIZES; ++i) {
      aom_prob p = fc->partition_prob[i][PARTITION_HORZ];
      assert(p > 0);
      x->partition_cost[i][PARTITION_NONE] = INT_MAX;
      x->partition_cost[i][PARTITION_HORZ] = av1_cost_bit(p, 0);
      x->partition_cost[i][PARTITION_VERT] = INT_MAX;
      x->partition_cost[i][PARTITION_SPLIT] = av1_cost_bit(p, 1);
    }
    x->partition_cost[PARTITION_CONTEXTS][PARTITION_NONE] = INT_MAX;
    x->partition_cost[PARTITION_CONTEXTS][PARTITION_HORZ] = INT_MAX;
    x->partition_cost[PARTITION_CONTEXTS][PARTITION_VERT] = INT_MAX;
    x->partition_cost[PARTITION_CONTEXTS][PARTITION_SPLIT] = 0;
#endif  // CONFIG_UNPOISON_PARTITION_CTX
  }

Jingning Han's avatar
Jingning Han committed
96 97
  for (i = 0; i < INTRA_MODES; ++i)
    for (j = 0; j < INTRA_MODES; ++j)
98
      av1_cost_tokens_from_cdf(x->y_mode_costs[i][j], av1_kf_y_mode_cdf[i][j],
99
                               av1_intra_mode_inv);
Jingning Han's avatar
Jingning Han committed
100

101
  for (i = 0; i < BLOCK_SIZE_GROUPS; ++i)
102
    av1_cost_tokens_from_cdf(x->mbmode_cost[i], fc->y_mode_cdf[i],
103
                             av1_intra_mode_inv);
104 105 106 107 108 109 110
  const int *uv_mode_inv_map =
#if CONFIG_CFL
      // CfL codes the uv_mode without reordering it
      NULL;
#else
      av1_intra_mode_inv;
#endif
111
  for (i = 0; i < INTRA_MODES; ++i)
112
    av1_cost_tokens_from_cdf(x->intra_uv_mode_cost[i], fc->uv_mode_cdf[i],
113
                             uv_mode_inv_map);
Jingning Han's avatar
Jingning Han committed
114 115

  for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; ++i)
Yue Chen's avatar
Yue Chen committed
116 117 118
    av1_cost_tokens_from_cdf(x->switchable_interp_costs[i],
                             fc->switchable_interp_cdf[i],
                             av1_switchable_interp_inv);
hui su's avatar
hui su committed
119 120

  for (i = 0; i < PALETTE_BLOCK_SIZES; ++i) {
121
    av1_cost_tokens_from_cdf(x->palette_y_size_cost[i],
122
                             fc->palette_y_size_cdf[i], NULL);
123
    av1_cost_tokens_from_cdf(x->palette_uv_size_cost[i],
124
                             fc->palette_uv_size_cdf[i], NULL);
hui su's avatar
hui su committed
125 126
  }

127
  for (i = 0; i < PALETTE_SIZES; ++i) {
128
    for (j = 0; j < PALETTE_COLOR_INDEX_CONTEXTS; ++j) {
129
      av1_cost_tokens_from_cdf(x->palette_y_color_cost[i][j],
130
                               fc->palette_y_color_index_cdf[i][j], NULL);
131
      av1_cost_tokens_from_cdf(x->palette_uv_color_cost[i][j],
132
                               fc->palette_uv_color_index_cdf[i][j], NULL);
hui su's avatar
hui su committed
133
    }
134
  }
135 136 137 138 139 140 141 142 143 144
#if CONFIG_MRC_TX
  for (i = 0; i < PALETTE_SIZES; ++i) {
    for (j = 0; j < PALETTE_COLOR_INDEX_CONTEXTS; ++j) {
      av1_cost_tokens_from_cdf(x->mrc_mask_inter_cost[i][j],
                               fc->mrc_mask_inter_cdf[i][j], NULL);
      av1_cost_tokens_from_cdf(x->mrc_mask_intra_cost[i][j],
                               fc->mrc_mask_intra_cdf[i][j], NULL);
    }
  }
#endif  // CONFIG_MRC_TX
145

146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166
#if CONFIG_CFL
  int sign_cost[CFL_JOINT_SIGNS];
  av1_cost_tokens_from_cdf(sign_cost, fc->cfl_sign_cdf, NULL);
  for (int joint_sign = 0; joint_sign < CFL_JOINT_SIGNS; joint_sign++) {
    const aom_cdf_prob *cdf_u = fc->cfl_alpha_cdf[CFL_CONTEXT_U(joint_sign)];
    const aom_cdf_prob *cdf_v = fc->cfl_alpha_cdf[CFL_CONTEXT_V(joint_sign)];
    int *cost_u = x->cfl_cost[joint_sign][CFL_PRED_U];
    int *cost_v = x->cfl_cost[joint_sign][CFL_PRED_V];
    if (CFL_SIGN_U(joint_sign) == CFL_SIGN_ZERO)
      memset(cost_u, 0, CFL_ALPHABET_SIZE * sizeof(*cost_u));
    else
      av1_cost_tokens_from_cdf(cost_u, cdf_u, NULL);
    if (CFL_SIGN_V(joint_sign) == CFL_SIGN_ZERO)
      memset(cost_v, 0, CFL_ALPHABET_SIZE * sizeof(*cost_v));
    else
      av1_cost_tokens_from_cdf(cost_v, cdf_v, NULL);
    for (int u = 0; u < CFL_ALPHABET_SIZE; u++)
      cost_u[u] += sign_cost[joint_sign];
  }
#endif  // CONFIG_CFL

Jingning Han's avatar
Jingning Han committed
167
  for (i = 0; i < MAX_TX_DEPTH; ++i)
168
    for (j = 0; j < TX_SIZE_CONTEXTS; ++j)
Yue Chen's avatar
Yue Chen committed
169 170
      av1_cost_tokens_from_cdf(x->tx_size_cost[i][j], fc->tx_size_cdf[i][j],
                               NULL);
171

172
#if CONFIG_EXT_TX
173 174 175
  for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
    int s;
    for (s = 1; s < EXT_TX_SETS_INTER; ++s) {
176
      if (use_inter_ext_tx_for_txsize[s][i]) {
Yue Chen's avatar
Yue Chen committed
177 178 179
        av1_cost_tokens_from_cdf(x->inter_tx_type_costs[s][i],
                                 fc->inter_ext_tx_cdf[s][i],
                                 av1_ext_tx_inter_inv[s]);
180 181 182
      }
    }
    for (s = 1; s < EXT_TX_SETS_INTRA; ++s) {
183
      if (use_intra_ext_tx_for_txsize[s][i]) {
184
        for (j = 0; j < INTRA_MODES; ++j)
Yue Chen's avatar
Yue Chen committed
185 186 187
          av1_cost_tokens_from_cdf(x->intra_tx_type_costs[s][i][j],
                                   fc->intra_ext_tx_cdf[s][i][j],
                                   av1_ext_tx_intra_inv[s]);
188 189
      }
    }
190
  }
191
#else
192 193
  for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
    for (j = 0; j < TX_TYPES; ++j)
Yue Chen's avatar
Yue Chen committed
194 195
      av1_cost_tokens_from_cdf(x->intra_tx_type_costs[i][j],
                               fc->intra_ext_tx_cdf[i][j], av1_ext_tx_inv);
196 197
  }
  for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
Yue Chen's avatar
Yue Chen committed
198 199
    av1_cost_tokens_from_cdf(x->inter_tx_type_costs[i], fc->inter_ext_tx_cdf[i],
                             av1_ext_tx_inv);
200
  }
201
#endif  // CONFIG_EXT_TX
202
#if CONFIG_EXT_INTRA
hui su's avatar
hui su committed
203
#if CONFIG_INTRA_INTERP
204
  for (i = 0; i < INTRA_FILTERS + 1; ++i)
Yue Chen's avatar
Yue Chen committed
205 206
    av1_cost_tokens_from_cdf(x->intra_filter_cost[i], fc->intra_filter_cdf[i],
                             NULL);
hui su's avatar
hui su committed
207
#endif  // CONFIG_INTRA_INTERP
208
#endif  // CONFIG_EXT_INTRA
209
#if CONFIG_LOOP_RESTORATION
210
  av1_cost_tokens(x->switchable_restore_cost, fc->switchable_restore_prob,
211 212
                  av1_switchable_restore_tree);
#endif  // CONFIG_LOOP_RESTORATION
213 214 215

  if (!frame_is_intra_only(cm)) {
    for (i = 0; i < NEWMV_MODE_CONTEXTS; ++i) {
Yue Chen's avatar
Yue Chen committed
216 217 218
#if CONFIG_NEW_MULTISYMBOL
      av1_cost_tokens_from_cdf(x->newmv_mode_cost[i], fc->newmv_cdf[i], NULL);
#else
219 220
      x->newmv_mode_cost[i][0] = av1_cost_bit(fc->newmv_prob[i], 0);
      x->newmv_mode_cost[i][1] = av1_cost_bit(fc->newmv_prob[i], 1);
Yue Chen's avatar
Yue Chen committed
221
#endif
222 223 224
    }

    for (i = 0; i < ZEROMV_MODE_CONTEXTS; ++i) {
Yue Chen's avatar
Yue Chen committed
225 226 227
#if CONFIG_NEW_MULTISYMBOL
      av1_cost_tokens_from_cdf(x->zeromv_mode_cost[i], fc->zeromv_cdf[i], NULL);
#else
228 229
      x->zeromv_mode_cost[i][0] = av1_cost_bit(fc->zeromv_prob[i], 0);
      x->zeromv_mode_cost[i][1] = av1_cost_bit(fc->zeromv_prob[i], 1);
Yue Chen's avatar
Yue Chen committed
230
#endif
231 232 233
    }

    for (i = 0; i < REFMV_MODE_CONTEXTS; ++i) {
Yue Chen's avatar
Yue Chen committed
234 235 236
#if CONFIG_NEW_MULTISYMBOL
      av1_cost_tokens_from_cdf(x->refmv_mode_cost[i], fc->refmv_cdf[i], NULL);
#else
237 238
      x->refmv_mode_cost[i][0] = av1_cost_bit(fc->refmv_prob[i], 0);
      x->refmv_mode_cost[i][1] = av1_cost_bit(fc->refmv_prob[i], 1);
Yue Chen's avatar
Yue Chen committed
239
#endif
240 241 242
    }

    for (i = 0; i < DRL_MODE_CONTEXTS; ++i) {
Yue Chen's avatar
Yue Chen committed
243 244 245
#if CONFIG_NEW_MULTISYMBOL
      av1_cost_tokens_from_cdf(x->drl_mode_cost0[i], fc->drl_cdf[i], NULL);
#else
246 247
      x->drl_mode_cost0[i][0] = av1_cost_bit(fc->drl_prob[i], 0);
      x->drl_mode_cost0[i][1] = av1_cost_bit(fc->drl_prob[i], 1);
Yue Chen's avatar
Yue Chen committed
248
#endif
249 250 251
    }
#if CONFIG_EXT_INTER
    for (i = 0; i < INTER_MODE_CONTEXTS; ++i)
Yue Chen's avatar
Yue Chen committed
252 253
      av1_cost_tokens_from_cdf(x->inter_compound_mode_cost[i],
                               fc->inter_compound_mode_cdf[i], NULL);
254 255 256
    for (i = 0; i < BLOCK_SIZES_ALL; ++i)
      av1_cost_tokens_from_cdf(x->compound_type_cost[i],
                               fc->compound_type_cdf[i], NULL);
257 258
#if CONFIG_COMPOUND_SINGLEREF
    for (i = 0; i < INTER_MODE_CONTEXTS; ++i)
Yue Chen's avatar
Yue Chen committed
259 260
      av1_cost_tokens_from_cdf(x->inter_singleref_comp_mode_cost[i],
                               fc->inter_singleref_comp_mode_cdf[i], NULL);
261 262 263
#endif  // CONFIG_COMPOUND_SINGLEREF
#if CONFIG_INTERINTRA
    for (i = 0; i < BLOCK_SIZE_GROUPS; ++i)
Yue Chen's avatar
Yue Chen committed
264 265
      av1_cost_tokens_from_cdf(x->interintra_mode_cost[i],
                               fc->interintra_mode_cdf[i], NULL);
266 267 268 269
#endif  // CONFIG_INTERINTRA
#endif  // CONFIG_EXT_INTER
#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
    for (i = BLOCK_8X8; i < BLOCK_SIZES_ALL; i++) {
270 271
      av1_cost_tokens_from_cdf(x->motion_mode_cost[i], fc->motion_mode_cdf[i],
                               NULL);
272 273 274
    }
#if CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION
    for (i = BLOCK_8X8; i < BLOCK_SIZES_ALL; i++) {
Yue Chen's avatar
Yue Chen committed
275 276 277
#if CONFIG_NEW_MULTISYMBOL
      av1_cost_tokens_from_cdf(x->motion_mode_cost1[i], fc->obmc_cdf[i], NULL);
#else
278 279
      x->motion_mode_cost1[i][0] = av1_cost_bit(fc->obmc_prob[i], 0);
      x->motion_mode_cost1[i][1] = av1_cost_bit(fc->obmc_prob[i], 1);
Yue Chen's avatar
Yue Chen committed
280
#endif
281 282 283 284
    }
#endif  // CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION
#if CONFIG_MOTION_VAR && CONFIG_NCOBMC_ADAPT_WEIGHT
    for (i = ADAPT_OVERLAP_BLOCK_8X8; i < ADAPT_OVERLAP_BLOCKS; ++i) {
285 286
      av1_cost_tokens_from_cdf(x->ncobmc_mode_cost[i], fc->ncobmc_mode_cdf[i],
                               NULL);
287 288 289 290
    }
#endif  // CONFIG_MOTION_VAR && CONFIG_NCOBMC_ADAPT_WEIGHT
#endif  // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
  }
Jingning Han's avatar
Jingning Han committed
291 292 293 294 295 296
}

// Values are now correlated to quantizer.
static int sad_per_bit16lut_8[QINDEX_RANGE];
static int sad_per_bit4lut_8[QINDEX_RANGE];

297
#if CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
298 299 300 301 302 303 304
static int sad_per_bit16lut_10[QINDEX_RANGE];
static int sad_per_bit4lut_10[QINDEX_RANGE];
static int sad_per_bit16lut_12[QINDEX_RANGE];
static int sad_per_bit4lut_12[QINDEX_RANGE];
#endif

static void init_me_luts_bd(int *bit16lut, int *bit4lut, int range,
Yaowu Xu's avatar
Yaowu Xu committed
305
                            aom_bit_depth_t bit_depth) {
Jingning Han's avatar
Jingning Han committed
306 307 308 309 310
  int i;
  // Initialize the sad lut tables using a formulaic calculation for now.
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < range; i++) {
Yaowu Xu's avatar
Yaowu Xu committed
311
    const double q = av1_convert_qindex_to_q(i, bit_depth);
Jingning Han's avatar
Jingning Han committed
312 313 314 315 316
    bit16lut[i] = (int)(0.0418 * q + 2.4107);
    bit4lut[i] = (int)(0.063 * q + 2.742);
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
317
void av1_init_me_luts(void) {
Jingning Han's avatar
Jingning Han committed
318
  init_me_luts_bd(sad_per_bit16lut_8, sad_per_bit4lut_8, QINDEX_RANGE,
Yaowu Xu's avatar
Yaowu Xu committed
319
                  AOM_BITS_8);
320
#if CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
321
  init_me_luts_bd(sad_per_bit16lut_10, sad_per_bit4lut_10, QINDEX_RANGE,
Yaowu Xu's avatar
Yaowu Xu committed
322
                  AOM_BITS_10);
Jingning Han's avatar
Jingning Han committed
323
  init_me_luts_bd(sad_per_bit16lut_12, sad_per_bit4lut_12, QINDEX_RANGE,
Yaowu Xu's avatar
Yaowu Xu committed
324
                  AOM_BITS_12);
Jingning Han's avatar
Jingning Han committed
325 326 327
#endif
}

328 329
static const int rd_boost_factor[16] = { 64, 32, 32, 32, 24, 16, 12, 12,
                                         8,  8,  4,  4,  2,  2,  1,  0 };
Jingning Han's avatar
Jingning Han committed
330
static const int rd_frame_type_factor[FRAME_UPDATE_TYPES] = {
331
  128, 144, 128, 128, 144,
332
#if CONFIG_EXT_REFS
333
  // TODO(zoeliu): To adjust further following factor values.
334
  128, 128, 128,
335 336
  // TODO(weitinglin): We should investigate if the values should be the same
  //                   as the value used by OVERLAY frame
337
  144,  // INTNL_OVERLAY_UPDATE
Zoe Liu's avatar
Zoe Liu committed
338
  128   // INTNL_ARF_UPDATE
339
#endif  // CONFIG_EXT_REFS
Jingning Han's avatar
Jingning Han committed
340 341
};

Yaowu Xu's avatar
Yaowu Xu committed
342 343
int av1_compute_rd_mult(const AV1_COMP *cpi, int qindex) {
  const int64_t q = av1_dc_quant(qindex, 0, cpi->common.bit_depth);
344
#if CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
345 346
  int64_t rdmult = 0;
  switch (cpi->common.bit_depth) {
Yaowu Xu's avatar
Yaowu Xu committed
347 348 349
    case AOM_BITS_8: rdmult = 88 * q * q / 24; break;
    case AOM_BITS_10: rdmult = ROUND_POWER_OF_TWO(88 * q * q / 24, 4); break;
    case AOM_BITS_12: rdmult = ROUND_POWER_OF_TWO(88 * q * q / 24, 8); break;
Jingning Han's avatar
Jingning Han committed
350
    default:
Yaowu Xu's avatar
Yaowu Xu committed
351
      assert(0 && "bit_depth should be AOM_BITS_8, AOM_BITS_10 or AOM_BITS_12");
Jingning Han's avatar
Jingning Han committed
352 353 354 355
      return -1;
  }
#else
  int64_t rdmult = 88 * q * q / 24;
356
#endif  // CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
357 358 359
  if (cpi->oxcf.pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    const GF_GROUP *const gf_group = &cpi->twopass.gf_group;
    const FRAME_UPDATE_TYPE frame_type = gf_group->update_type[gf_group->index];
Yaowu Xu's avatar
Yaowu Xu committed
360
    const int boost_index = AOMMIN(15, (cpi->rc.gfu_boost / 100));
Jingning Han's avatar
Jingning Han committed
361 362 363 364

    rdmult = (rdmult * rd_frame_type_factor[frame_type]) >> 7;
    rdmult += ((rdmult * rd_boost_factor[boost_index]) >> 7);
  }
365
  if (rdmult < 1) rdmult = 1;
Jingning Han's avatar
Jingning Han committed
366 367 368
  return (int)rdmult;
}

Yaowu Xu's avatar
Yaowu Xu committed
369
static int compute_rd_thresh_factor(int qindex, aom_bit_depth_t bit_depth) {
Jingning Han's avatar
Jingning Han committed
370
  double q;
371
#if CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
372
  switch (bit_depth) {
Yaowu Xu's avatar
Yaowu Xu committed
373 374 375
    case AOM_BITS_8: q = av1_dc_quant(qindex, 0, AOM_BITS_8) / 4.0; break;
    case AOM_BITS_10: q = av1_dc_quant(qindex, 0, AOM_BITS_10) / 16.0; break;
    case AOM_BITS_12: q = av1_dc_quant(qindex, 0, AOM_BITS_12) / 64.0; break;
Jingning Han's avatar
Jingning Han committed
376
    default:
Yaowu Xu's avatar
Yaowu Xu committed
377
      assert(0 && "bit_depth should be AOM_BITS_8, AOM_BITS_10 or AOM_BITS_12");
Jingning Han's avatar
Jingning Han committed
378 379 380
      return -1;
  }
#else
381
  (void)bit_depth;
Yaowu Xu's avatar
Yaowu Xu committed
382
  q = av1_dc_quant(qindex, 0, AOM_BITS_8) / 4.0;
383
#endif  // CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
384
  // TODO(debargha): Adjust the function below.
Yaowu Xu's avatar
Yaowu Xu committed
385
  return AOMMAX((int)(pow(q, RD_THRESH_POW) * 5.12), 8);
Jingning Han's avatar
Jingning Han committed
386 387
}

Yaowu Xu's avatar
Yaowu Xu committed
388
void av1_initialize_me_consts(const AV1_COMP *cpi, MACROBLOCK *x, int qindex) {
389
#if CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
390
  switch (cpi->common.bit_depth) {
Yaowu Xu's avatar
Yaowu Xu committed
391
    case AOM_BITS_8:
Jingning Han's avatar
Jingning Han committed
392 393 394
      x->sadperbit16 = sad_per_bit16lut_8[qindex];
      x->sadperbit4 = sad_per_bit4lut_8[qindex];
      break;
Yaowu Xu's avatar
Yaowu Xu committed
395
    case AOM_BITS_10:
Jingning Han's avatar
Jingning Han committed
396 397 398
      x->sadperbit16 = sad_per_bit16lut_10[qindex];
      x->sadperbit4 = sad_per_bit4lut_10[qindex];
      break;
Yaowu Xu's avatar
Yaowu Xu committed
399
    case AOM_BITS_12:
Jingning Han's avatar
Jingning Han committed
400 401 402 403
      x->sadperbit16 = sad_per_bit16lut_12[qindex];
      x->sadperbit4 = sad_per_bit4lut_12[qindex];
      break;
    default:
Yaowu Xu's avatar
Yaowu Xu committed
404
      assert(0 && "bit_depth should be AOM_BITS_8, AOM_BITS_10 or AOM_BITS_12");
Jingning Han's avatar
Jingning Han committed
405 406 407 408 409
  }
#else
  (void)cpi;
  x->sadperbit16 = sad_per_bit16lut_8[qindex];
  x->sadperbit4 = sad_per_bit4lut_8[qindex];
410
#endif  // CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
411 412
}

Yaowu Xu's avatar
Yaowu Xu committed
413
static void set_block_thresholds(const AV1_COMMON *cm, RD_OPT *rd) {
Jingning Han's avatar
Jingning Han committed
414 415 416 417
  int i, bsize, segment_id;

  for (segment_id = 0; segment_id < MAX_SEGMENTS; ++segment_id) {
    const int qindex =
Yaowu Xu's avatar
Yaowu Xu committed
418
        clamp(av1_get_qindex(&cm->seg, segment_id, cm->base_qindex) +
419 420
                  cm->y_dc_delta_q,
              0, MAXQ);
Jingning Han's avatar
Jingning Han committed
421 422
    const int q = compute_rd_thresh_factor(qindex, cm->bit_depth);

423
    for (bsize = 0; bsize < BLOCK_SIZES_ALL; ++bsize) {
Jingning Han's avatar
Jingning Han committed
424 425 426 427 428
      // Threshold here seems unnecessarily harsh but fine given actual
      // range of values used for cpi->sf.thresh_mult[].
      const int t = q * rd_thresh_block_size_factor[bsize];
      const int thresh_max = INT_MAX / t;

429 430 431 432 433 434
#if CONFIG_CB4X4
      for (i = 0; i < MAX_MODES; ++i)
        rd->threshes[segment_id][bsize][i] = rd->thresh_mult[i] < thresh_max
                                                 ? rd->thresh_mult[i] * t / 4
                                                 : INT_MAX;
#else
Jingning Han's avatar
Jingning Han committed
435 436
      if (bsize >= BLOCK_8X8) {
        for (i = 0; i < MAX_MODES; ++i)
437 438 439
          rd->threshes[segment_id][bsize][i] = rd->thresh_mult[i] < thresh_max
                                                   ? rd->thresh_mult[i] * t / 4
                                                   : INT_MAX;
Jingning Han's avatar
Jingning Han committed
440 441 442 443 444 445 446
      } else {
        for (i = 0; i < MAX_REFS; ++i)
          rd->threshes[segment_id][bsize][i] =
              rd->thresh_mult_sub8x8[i] < thresh_max
                  ? rd->thresh_mult_sub8x8[i] * t / 4
                  : INT_MAX;
      }
447
#endif
Jingning Han's avatar
Jingning Han committed
448 449 450 451
    }
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
452 453
void av1_set_mvcost(MACROBLOCK *x, MV_REFERENCE_FRAME ref_frame, int ref,
                    int ref_mv_idx) {
454
  MB_MODE_INFO_EXT *mbmi_ext = x->mbmi_ext;
Yaowu Xu's avatar
Yaowu Xu committed
455 456 457 458
  int8_t rf_type = av1_ref_frame_type(x->e_mbd.mi[0]->mbmi.ref_frame);
  int nmv_ctx = av1_nmv_ctx(mbmi_ext->ref_mv_count[rf_type],
                            mbmi_ext->ref_mv_stack[rf_type], ref, ref_mv_idx);
  (void)ref_frame;
459 460 461 462
  x->mvcost = x->mv_cost_stack[nmv_ctx];
  x->nmvjointcost = x->nmv_vec_cost[nmv_ctx];
}

463
#if CONFIG_LV_MAP
464
#if !LV_MAP_PROB
465 466 467 468
static void get_rate_cost(aom_prob p, int cost[2]) {
  cost[0] = av1_cost_bit(p, 0);
  cost[1] = av1_cost_bit(p, 1);
}
469
#endif  // !LV_MAP_PROB
470 471 472 473 474 475

void av1_fill_coeff_costs(MACROBLOCK *x, FRAME_CONTEXT *fc) {
  for (TX_SIZE tx_size = 0; tx_size < TX_SIZES; ++tx_size) {
    for (int plane = 0; plane < PLANE_TYPES; ++plane) {
      LV_MAP_COEFF_COST *pcost = &x->coeff_costs[tx_size][plane];

476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502
#if LV_MAP_PROB
      for (int ctx = 0; ctx < TXB_SKIP_CONTEXTS; ++ctx)
        av1_cost_tokens_from_cdf(pcost->txb_skip_cost[ctx],
                                 fc->txb_skip_cdf[tx_size][ctx], NULL);

      for (int ctx = 0; ctx < SIG_COEF_CONTEXTS; ++ctx)
        av1_cost_tokens_from_cdf(pcost->nz_map_cost[ctx],
                                 fc->nz_map_cdf[tx_size][plane][ctx], NULL);

      for (int ctx = 0; ctx < EOB_COEF_CONTEXTS; ++ctx)
        av1_cost_tokens_from_cdf(pcost->eob_cost[ctx],
                                 fc->eob_flag_cdf[tx_size][plane][ctx], NULL);

      for (int ctx = 0; ctx < DC_SIGN_CONTEXTS; ++ctx)
        av1_cost_tokens_from_cdf(pcost->dc_sign_cost[ctx],
                                 fc->dc_sign_cdf[plane][ctx], NULL);

      for (int layer = 0; layer < NUM_BASE_LEVELS; ++layer)
        for (int ctx = 0; ctx < COEFF_BASE_CONTEXTS; ++ctx)
          av1_cost_tokens_from_cdf(
              pcost->base_cost[layer][ctx],
              fc->coeff_base_cdf[tx_size][plane][layer][ctx], NULL);

      for (int ctx = 0; ctx < LEVEL_CONTEXTS; ++ctx)
        av1_cost_tokens_from_cdf(pcost->lps_cost[ctx],
                                 fc->coeff_lps_cdf[tx_size][plane][ctx], NULL);
#else
503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521
      for (int ctx = 0; ctx < TXB_SKIP_CONTEXTS; ++ctx)
        get_rate_cost(fc->txb_skip[tx_size][ctx], pcost->txb_skip_cost[ctx]);

      for (int ctx = 0; ctx < SIG_COEF_CONTEXTS; ++ctx)
        get_rate_cost(fc->nz_map[tx_size][plane][ctx], pcost->nz_map_cost[ctx]);

      for (int ctx = 0; ctx < EOB_COEF_CONTEXTS; ++ctx)
        get_rate_cost(fc->eob_flag[tx_size][plane][ctx], pcost->eob_cost[ctx]);

      for (int ctx = 0; ctx < DC_SIGN_CONTEXTS; ++ctx)
        get_rate_cost(fc->dc_sign[plane][ctx], pcost->dc_sign_cost[ctx]);

      for (int layer = 0; layer < NUM_BASE_LEVELS; ++layer)
        for (int ctx = 0; ctx < COEFF_BASE_CONTEXTS; ++ctx)
          get_rate_cost(fc->coeff_base[tx_size][plane][layer][ctx],
                        pcost->base_cost[layer][ctx]);

      for (int ctx = 0; ctx < LEVEL_CONTEXTS; ++ctx)
        get_rate_cost(fc->coeff_lps[tx_size][plane][ctx], pcost->lps_cost[ctx]);
522
#endif
523 524 525 526 527
    }
  }
}
#endif

528 529
void av1_fill_token_costs_from_cdf(av1_coeff_cost *cost,
                                   coeff_cdf_model (*cdf)[PLANE_TYPES]) {
hui su's avatar
hui su committed
530 531 532 533 534 535 536 537 538 539 540 541 542 543
  for (int tx = 0; tx < TX_SIZES; ++tx) {
    for (int pt = 0; pt < PLANE_TYPES; ++pt) {
      for (int rt = 0; rt < REF_TYPES; ++rt) {
        for (int band = 0; band < COEF_BANDS; ++band) {
          for (int ctx = 0; ctx < BAND_COEFF_CONTEXTS(band); ++ctx) {
            av1_cost_tokens_from_cdf(cost[tx][pt][rt][band][ctx],
                                     cdf[tx][pt][rt][band][ctx], NULL);
          }
        }
      }
    }
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
544 545
void av1_initialize_rd_consts(AV1_COMP *cpi) {
  AV1_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
546 547
  MACROBLOCK *const x = &cpi->td.mb;
  RD_OPT *const rd = &cpi->rd;
548
  int nmv_ctx;
Jingning Han's avatar
Jingning Han committed
549

Yaowu Xu's avatar
Yaowu Xu committed
550
  aom_clear_system_state();
Jingning Han's avatar
Jingning Han committed
551

Yaowu Xu's avatar
Yaowu Xu committed
552
  rd->RDMULT = av1_compute_rd_mult(cpi, cm->base_qindex + cm->y_dc_delta_q);
Jingning Han's avatar
Jingning Han committed
553

554
  set_error_per_bit(x, rd->RDMULT);
Jingning Han's avatar
Jingning Han committed
555 556 557

  set_block_thresholds(cm, rd);

558 559 560 561 562 563 564 565 566
  for (nmv_ctx = 0; nmv_ctx < NMV_CONTEXTS; ++nmv_ctx) {
    av1_build_nmv_cost_table(
        x->nmv_vec_cost[nmv_ctx],
        cm->allow_high_precision_mv ? x->nmvcost_hp[nmv_ctx]
                                    : x->nmvcost[nmv_ctx],
        &cm->fc->nmvc[nmv_ctx], cm->allow_high_precision_mv);
  }
  x->mvcost = x->mv_cost_stack[0];
  x->nmvjointcost = x->nmv_vec_cost[0];
Jingning Han's avatar
Jingning Han committed
567

Alex Converse's avatar
Alex Converse committed
568 569 570 571 572 573 574 575 576 577
#if CONFIG_INTRABC
  if (frame_is_intra_only(cm) && cm->allow_screen_content_tools &&
      cpi->oxcf.pass != 1) {
    av1_build_nmv_cost_table(
        x->nmv_vec_cost[0],
        cm->allow_high_precision_mv ? x->nmvcost_hp[0] : x->nmvcost[0],
        &cm->fc->ndvc, MV_SUBPEL_NONE);
  }
#endif

578
#if CONFIG_GLOBAL_MOTION
579
  if (cpi->oxcf.pass != 1) {
580
    for (int i = 0; i < TRANS_TYPES; ++i)
581
#if GLOBAL_TRANS_TYPES > 4
582 583
      cpi->gmtype_cost[i] = (1 + (i > 0 ? GLOBAL_TYPE_BITS : 0))
                            << AV1_PROB_COST_SHIFT;
584 585 586 587 588 589 590 591
#else
      // IDENTITY: 1 bit
      // TRANSLATION: 3 bits
      // ROTZOOM: 2 bits
      // AFFINE: 3 bits
      cpi->gmtype_cost[i] = (1 + (i > 0 ? (i == ROTZOOM ? 1 : 2) : 0))
                            << AV1_PROB_COST_SHIFT;
#endif  // GLOBAL_TRANS_TYPES > 4
Jingning Han's avatar
Jingning Han committed
592
  }
593
#endif  // CONFIG_GLOBAL_MOTION
Jingning Han's avatar
Jingning Han committed
594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609
}

static void model_rd_norm(int xsq_q10, int *r_q10, int *d_q10) {
  // NOTE: The tables below must be of the same size.

  // The functions described below are sampled at the four most significant
  // bits of x^2 + 8 / 256.

  // Normalized rate:
  // This table models the rate for a Laplacian source with given variance
  // when quantized with a uniform quantizer with given stepsize. The
  // closed form expression is:
  // Rn(x) = H(sqrt(r)) + sqrt(r)*[1 + H(r)/(1 - r)],
  // where r = exp(-sqrt(2) * x) and x = qpstep / sqrt(variance),
  // and H(x) is the binary entropy function.
  static const int rate_tab_q10[] = {
610 611 612 613 614 615 616 617 618
    65536, 6086, 5574, 5275, 5063, 4899, 4764, 4651, 4553, 4389, 4255, 4142,
    4044,  3958, 3881, 3811, 3748, 3635, 3538, 3453, 3376, 3307, 3244, 3186,
    3133,  3037, 2952, 2877, 2809, 2747, 2690, 2638, 2589, 2501, 2423, 2353,
    2290,  2232, 2179, 2130, 2084, 2001, 1928, 1862, 1802, 1748, 1698, 1651,
    1608,  1530, 1460, 1398, 1342, 1290, 1243, 1199, 1159, 1086, 1021, 963,
    911,   864,  821,  781,  745,  680,  623,  574,  530,  490,  455,  424,
    395,   345,  304,  269,  239,  213,  190,  171,  154,  126,  104,  87,
    73,    61,   52,   44,   38,   28,   21,   16,   12,   10,   8,    6,
    5,     3,    2,    1,    1,    1,    0,    0,
Jingning Han's avatar
Jingning Han committed
619 620 621 622 623 624 625 626 627
  };
  // Normalized distortion:
  // This table models the normalized distortion for a Laplacian source
  // with given variance when quantized with a uniform quantizer
  // with given stepsize. The closed form expression is:
  // Dn(x) = 1 - 1/sqrt(2) * x / sinh(x/sqrt(2))
  // where x = qpstep / sqrt(variance).
  // Note the actual distortion is Dn * variance.
  static const int dist_tab_q10[] = {
628 629 630 631 632 633 634 635 636
    0,    0,    1,    1,    1,    2,    2,    2,    3,    3,    4,    5,
    5,    6,    7,    7,    8,    9,    11,   12,   13,   15,   16,   17,
    18,   21,   24,   26,   29,   31,   34,   36,   39,   44,   49,   54,
    59,   64,   69,   73,   78,   88,   97,   106,  115,  124,  133,  142,
    151,  167,  184,  200,  215,  231,  245,  260,  274,  301,  327,  351,
    375,  397,  418,  439,  458,  495,  528,  559,  587,  613,  637,  659,
    680,  717,  749,  777,  801,  823,  842,  859,  874,  899,  919,  936,
    949,  960,  969,  977,  983,  994,  1001, 1006, 1010, 1013, 1015, 1017,
    1018, 1020, 1022, 1022, 1023, 1023, 1023, 1024,
Jingning Han's avatar
Jingning Han committed
637 638
  };
  static const int xsq_iq_q10[] = {
639 640 641 642 643 644 645 646 647 648 649 650
    0,      4,      8,      12,     16,     20,     24,     28,     32,
    40,     48,     56,     64,     72,     80,     88,     96,     112,
    128,    144,    160,    176,    192,    208,    224,    256,    288,
    320,    352,    384,    416,    448,    480,    544,    608,    672,
    736,    800,    864,    928,    992,    1120,   1248,   1376,   1504,
    1632,   1760,   1888,   2016,   2272,   2528,   2784,   3040,   3296,
    3552,   3808,   4064,   4576,   5088,   5600,   6112,   6624,   7136,
    7648,   8160,   9184,   10208,  11232,  12256,  13280,  14304,  15328,
    16352,  18400,  20448,  22496,  24544,  26592,  28640,  30688,  32736,
    36832,  40928,  45024,  49120,  53216,  57312,  61408,  65504,  73696,
    81888,  90080,  98272,  106464, 114656, 122848, 131040, 147424, 163808,
    180192, 196576, 212960, 229344, 245728,
Jingning Han's avatar
Jingning Han committed
651 652 653 654 655 656 657 658 659 660 661
  };
  const int tmp = (xsq_q10 >> 2) + 8;
  const int k = get_msb(tmp) - 3;
  const int xq = (k << 3) + ((tmp >> k) & 0x7);
  const int one_q10 = 1 << 10;
  const int a_q10 = ((xsq_q10 - xsq_iq_q10[xq]) << 10) >> (2 + k);
  const int b_q10 = one_q10 - a_q10;
  *r_q10 = (rate_tab_q10[xq] * b_q10 + rate_tab_q10[xq + 1] * a_q10) >> 10;
  *d_q10 = (dist_tab_q10[xq] * b_q10 + dist_tab_q10[xq + 1] * a_q10) >> 10;
}

Yaowu Xu's avatar
Yaowu Xu committed
662 663 664
void av1_model_rd_from_var_lapndz(int64_t var, unsigned int n_log2,
                                  unsigned int qstep, int *rate,
                                  int64_t *dist) {
Jingning Han's avatar
Jingning Han committed
665 666 667 668 669 670 671 672 673 674 675 676 677 678
  // This function models the rate and distortion for a Laplacian
  // source with given variance when quantized with a uniform quantizer
  // with given stepsize. The closed form expressions are in:
  // Hang and Chen, "Source Model for transform video coder and its
  // application - Part I: Fundamental Theory", IEEE Trans. Circ.
  // Sys. for Video Tech., April 1997.
  if (var == 0) {
    *rate = 0;
    *dist = 0;
  } else {
    int d_q10, r_q10;
    static const uint32_t MAX_XSQ_Q10 = 245727;
    const uint64_t xsq_q10_64 =
        (((uint64_t)qstep * qstep << (n_log2 + 10)) + (var >> 1)) / var;
Yaowu Xu's avatar
Yaowu Xu committed
679
    const int xsq_q10 = (int)AOMMIN(xsq_q10_64, MAX_XSQ_Q10);
Jingning Han's avatar
Jingning Han committed
680
    model_rd_norm(xsq_q10, &r_q10, &d_q10);
Yaowu Xu's avatar
Yaowu Xu committed
681
    *rate = ROUND_POWER_OF_TWO(r_q10 << n_log2, 10 - AV1_PROB_COST_SHIFT);
Jingning Han's avatar
Jingning Han committed
682 683 684 685
    *dist = (var * (int64_t)d_q10 + 512) >> 10;
  }
}

686 687 688 689
static void get_entropy_contexts_plane(
    BLOCK_SIZE plane_bsize, TX_SIZE tx_size, const struct macroblockd_plane *pd,
    ENTROPY_CONTEXT t_above[2 * MAX_MIB_SIZE],
    ENTROPY_CONTEXT t_left[2 * MAX_MIB_SIZE]) {
690 691
  const int num_4x4_w = block_size_wide[plane_bsize] >> tx_size_wide_log2[0];
  const int num_4x4_h = block_size_high[plane_bsize] >> tx_size_high_log2[0];
Jingning Han's avatar
Jingning Han committed
692 693 694
  const ENTROPY_CONTEXT *const above = pd->above_context;
  const ENTROPY_CONTEXT *const left = pd->left_context;

695 696 697 698 699 700
#if CONFIG_LV_MAP
  memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
  memcpy(t_left, left, sizeof(ENTROPY_CONTEXT) * num_4x4_h);
  return;
#endif  // CONFIG_LV_MAP

Jingning Han's avatar
Jingning Han committed
701
  int i;
702

703
#if CONFIG_CHROMA_2X2
704
  switch (tx_size) {
705
    case TX_2X2:
706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734
      memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
      memcpy(t_left, left, sizeof(ENTROPY_CONTEXT) * num_4x4_h);
      break;
    case TX_4X4:
      for (i = 0; i < num_4x4_w; i += 2)
        t_above[i] = !!*(const uint16_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 2)
        t_left[i] = !!*(const uint16_t *)&left[i];
      break;
    case TX_8X8:
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
    case TX_16X16:
      for (i = 0; i < num_4x4_w; i += 8)
        t_above[i] = !!*(const uint64_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 8)
        t_left[i] = !!*(const uint64_t *)&left[i];
      break;
    case TX_32X32:
      for (i = 0; i < num_4x4_w; i += 16)
        t_above[i] =
            !!(*(const uint64_t *)&above[i] | *(const uint64_t *)&above[i + 8]);
      for (i = 0; i < num_4x4_h; i += 16)
        t_left[i] =
            !!(*(const uint64_t *)&left[i] | *(const uint64_t *)&left[i + 8]);
      break;
735 736 737 738 739 740 741 742 743 744 745 746 747 748
#if CONFIG_TX64X64
    case TX_64X64:
      for (i = 0; i < num_4x4_w; i += 32)
        t_above[i] =
            !!(*(const uint64_t *)&above[i] | *(const uint64_t *)&above[i + 8] |
               *(const uint64_t *)&above[i + 16] |
               *(const uint64_t *)&above[i + 24]);
      for (i = 0; i < num_4x4_h; i += 32)
        t_left[i] =
            !!(*(const uint64_t *)&left[i] | *(const uint64_t *)&left[i + 8] |
               *(const uint64_t *)&left[i + 16] |
               *(const uint64_t *)&left[i + 24]);
      break;
#endif  // CONFIG_TX64X64
749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785
    case TX_4X8:
      for (i = 0; i < num_4x4_w; i += 2)
        t_above[i] = !!*(const uint16_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
    case TX_8X4:
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 2)
        t_left[i] = !!*(const uint16_t *)&left[i];
      break;
    case TX_8X16:
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 8)
        t_left[i] = !!*(const uint64_t *)&left[i];
      break;
    case TX_16X8:
      for (i = 0; i < num_4x4_w; i += 8)
        t_above[i] = !!*(const uint64_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
    case TX_16X32:
      for (i = 0; i < num_4x4_w; i += 8)
        t_above[i] = !!*(const uint64_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 16)
        t_left[i] =
            !!(*(const uint64_t *)&left[i] | *(const uint64_t *)&left[i + 8]);
      break;
    case TX_32X16:
      for (i = 0; i < num_4x4_w; i += 16)
        t_above[i] =
            !!(*(const uint64_t *)&above[i] | *(const uint64_t *)&above[i + 8]);
      for (i = 0; i < num_4x4_h; i += 8)
        t_left[i] = !!*(const uint64_t *)&left[i];
786
      break;
Yue Chen's avatar
Yue Chen committed
787
#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813
    case TX_4X16:
      for (i = 0; i < num_4x4_w; i += 2)
        t_above[i] = !!*(const uint16_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 8)
        t_left[i] = !!*(const uint64_t *)&left[i];
      break;
    case TX_16X4:
      for (i = 0; i < num_4x4_w; i += 8)
        t_above[i] = !!*(const uint64_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 2)
        t_left[i] = !!*(const uint16_t *)&left[i];
      break;
    case TX_8X32:
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 16)
        t_left[i] =
            !!(*(const uint64_t *)&left[i] | *(const uint64_t *)&left[i + 8]);
      break;
    case TX_32X8:
      for (i = 0; i < num_4x4_w; i += 16)
        t_above[i] =
            !!(*(const uint64_t *)&above[i] | *(const uint64_t *)&above[i + 8]);
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
Yue Chen's avatar
Yue Chen committed
814
#endif
815 816 817 818

    default: assert(0 && "Invalid transform size."); break;
  }
  return;
819
#endif  // CONFIG_CHROMA_2X2
820 821

  switch (tx_size) {
Jingning Han's avatar
Jingning Han committed
822 823 824 825
    case TX_4X4:
      memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
      memcpy(t_left, left, sizeof(ENTROPY_CONTEXT) * num_4x4_h);
      break;
826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843
    case TX_8X8:
      for (i = 0; i < num_4x4_w; i += 2)
        t_above[i] = !!*(const uint16_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 2)
        t_left[i] = !!*(const uint16_t *)&left[i];
      break;
    case TX_16X16:
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
    case TX_32X32:
      for (i = 0; i < num_4x4_w; i += 8)
        t_above[i] = !!*(const uint64_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 8)
        t_left[i] = !!*(const uint64_t *)&left[i];
      break;
844 845 846 847 848 849
#if CONFIG_TX64X64
    case TX_64X64:
      for (i = 0; i < num_4x4_w; i += 16)
        t_above[i] =
            !!(*(const uint64_t *)&above[i] | *(const uint64_t *)&above[i + 8]);
      for (i = 0; i < num_4x4_h; i += 16)
850 851
        t_left[i] =
            !!(*(const uint64_t *)&left[i] | *(const uint64_t *)&left[i + 8]);
852 853
      break;
#endif  // CONFIG_TX64X64
854 855 856 857 858 859 860 861 862 863
    case TX_4X8:
      memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
      for (i = 0; i < num_4x4_h; i += 2)
        t_left[i] = !!*(const uint16_t *)&left[i];
      break;
    case TX_8X4:
      for (i = 0; i < num_4x4_w; i += 2)
        t_above[i] = !!*(const uint16_t *)&above[i];
      memcpy(t_left, left, sizeof(ENTROPY_CONTEXT) * num_4x4_h);
      break;
864
    case TX_8X16:
Jingning Han's avatar
Jingning Han committed
865 866
      for (i = 0; i < num_4x4_w; i += 2)
        t_above[i] = !!*(const uint16_t *)&above[i];
867 868 869 870 871 872
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
    case TX_16X8:
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
Jingning Han's avatar
Jingning Han committed
873 874 875
      for (i = 0; i < num_4x4_h; i += 2)
        t_left[i] = !!*(const uint16_t *)&left[i];
      break;
876
    case TX_16X32:
Jingning Han's avatar
Jingning Han committed
877 878
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
879 880
      for (i = 0; i < num_4x4_h; i += 8)
        t_left[i] = !!*(const uint64_t *)&left[i];
Jingning Han's avatar
Jingning Han committed
881
      break;
882
    case TX_32X16:
Jingning Han's avatar
Jingning Han committed
883 884
      for (i = 0; i < num_4x4_w; i += 8)
        t_above[i] = !!*(const uint64_t *)&above[i];
885 886 887
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
Yue Chen's avatar
Yue Chen committed
888
#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910
    case TX_4X16:
      memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
    case TX_16X4:
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
      memcpy(t_left, left, sizeof(ENTROPY_CONTEXT) * num_4x4_h);
      break;
    case TX_8X32:
      for (i = 0; i < num_4x4_w; i += 2)
        t_above[i] = !!*(const uint16_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 8)
        t_left[i] = !!*(const uint64_t *)&left[i];
      break;
    case TX_32X8:
      for (i = 0; i < num_4x4_w; i += 8)
        t_above[i] = !!*(const uint64_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 2)
        t_left[i] = !!*(const uint16_t *)&left[i];
      break;
Yue Chen's avatar
Yue Chen committed
911
#endif
clang-format's avatar
clang-format committed
912
    default: assert(0 && "Invalid transform size."); break;
Jingning Han's avatar
Jingning Han committed
913 914 915
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
916 917 918 919
void av1_get_entropy_contexts(BLOCK_SIZE bsize, TX_SIZE tx_size,
                              const struct macroblockd_plane *pd,
                              ENTROPY_CONTEXT t_above[2 * MAX_MIB_SIZE],
                              ENTROPY_CONTEXT t_left[2 * MAX_MIB_SIZE]) {
920
#if CONFIG_CHROMA_SUB8X8
921 922 923
  const BLOCK_SIZE plane_bsize =
      AOMMAX(BLOCK_4X4, get_plane_block_size(bsize, pd));
#else
924
  const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd);
925
#endif
926
  get_entropy_contexts_plane(plane_bsize, tx_size, pd, t_above, t_left);
927 928
}

929
void av1_mv_pred(const AV1_COMP *cpi, MACROBLOCK *x, uint8_t *ref_y_buffer,
Yaowu Xu's avatar
Yaowu Xu committed
930
                 int ref_y_stride, int ref_frame, BLOCK_SIZE block_size) {
Jingning Han's avatar
Jingning Han committed
931 932 933 934 935 936 937 938
  int i;
  int zero_seen = 0;
  int best_index = 0;
  int best_sad = INT_MAX;
  int this_sad = INT_MAX;
  int max_mv = 0;
  uint8_t *src_y_ptr = x->plane[0].src.buf;
  uint8_t *ref_y_ptr;
939
  MV pred_mv[MAX_MV_REF_CANDIDATES + 1];
940 941 942 943 944 945
  int num_mv_refs = 0;

  pred_mv[num_mv_refs++] = x->mbmi_ext->ref_mvs[ref_frame][0].as_mv;
  if (x->mbmi_ext->ref_mvs[ref_frame][0].as_int !=
      x->mbmi_ext->ref_mvs[ref_frame][1].as_int) {
    pred_mv[num_mv_refs++] = x->mbmi_ext->ref_mvs[ref_frame][1].as_mv;
946
  }
947 948 949
  if (cpi->sf.adaptive_motion_search && block_size < x->max_partition_size)
    pred_mv[num_mv_refs++] = x->pred_mv[ref_frame];

Jingning Han's avatar
Jingning Han committed
950
  assert(num_mv_refs <= (int)(sizeof(pred_mv) / sizeof(pred_mv[0])));
951

Jingning Han's avatar
Jingning Han committed
952 953 954 955 956 957
  // Get the sad for each candidate reference mv.
  for (i = 0; i < num_mv_refs; ++i) {
    const MV *this_mv = &pred_mv[i];
    int fp_row, fp_col;
    fp_row = (this_mv->row + 3 + (this_mv->row >= 0)) >> 3;
    fp_col = (this_mv->col + 3 + (this_mv->col >= 0)) >> 3;
Yaowu Xu's avatar
Yaowu Xu committed
958
    max_mv = AOMMAX(max_mv, AOMMAX(abs(this_mv->row), abs(this_mv->col)) >> 3);
Jingning Han's avatar
Jingning Han committed
959

960 961
    if (fp_row == 0 && fp_col == 0 && zero_seen) continue;
    zero_seen |= (fp_row == 0 && fp_col == 0);
Jingning Han's avatar
Jingning Han committed
962

963
    ref_y_ptr = &ref_y_buffer[ref_y_stride * fp_row + fp_col];
Jingning Han's avatar
Jingning Han committed
964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979
    // Find sad for current vector.
    this_sad = cpi->fn_ptr[block_size].sdf(src_y_ptr, x->plane[0].src.stride,
                                           ref_y_ptr, ref_y_stride);
    // Note if it is the best so far.
    if (this_sad < best_sad) {
      best_sad = this_sad;
      best_index = i;
    }
  }

  // Note the index of the mv that worked best in the reference list.
  x->mv_best_ref_index[ref_frame] = best_index;
  x->max_mv_context[ref_frame] = max_mv;
  x->pred_mv_sad[ref_frame] = best_sad;
}

Yaowu Xu's avatar
Yaowu Xu committed
980 981 982 983 984
void av1_setup_pred_block(const MACROBLOCKD *xd,
                          struct buf_2d dst[MAX_MB_PLANE],
                          const YV12_BUFFER_CONFIG *src, int mi_row, int mi_col,
                          const struct scale_factors *scale,
                          const struct scale_factors *scale_uv) {
Jingning Han's avatar
Jingning Han committed
985 986 987 988 989 990 991 992 993
  int i;

  dst[0].buf = src->y_buffer;
  dst[0].stride = src->y_stride;
  dst[1].buf = src->u_buffer;
  dst[2].buf = src->v_buffer;
  dst[1].stride = dst[2].stride = src->uv_stride;

  for (i = 0; i < MAX_MB_PLANE; ++i) {
994
    setup_pred_plane(dst + i, xd->mi[0]->mbmi.sb_type, dst[i].buf,
995 996
                     i ? src->uv_crop_width : src->y_crop_width,
                     i ? src->uv_crop_height : src->y_crop_height,
997
                     dst[i].stride, mi_row, mi_col, i ? scale_uv : scale,
Jingning Han's avatar
Jingning Han committed
998 999 1000 1001
                     xd->plane[i].subsampling_x, xd->plane[i].subsampling_y);
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
1002 1003
int av1_raster_block_offset(BLOCK_SIZE plane_bsize, int raster_block,
                            int stride) {
Jingning Han's avatar
Jingning Han committed
1004 1005 1006 1007 1008 1009
  const int bw = b_width_log2_lookup[plane_bsize];
  const int y = 4 * (raster_block >> bw);
  const int x = 4 * (raster_block & ((1 << bw) - 1));
  return y * stride + x;
}

Yaowu Xu's avatar
Yaowu Xu committed
1010 1011
int16_t *av1_raster_block_offset_int16(BLOCK_SIZE plane_bsize, int raster_block,
                                       int16_t *base) {
1012
  const int stride = block_size_wide[plane_bsize];
Yaowu Xu's avatar
Yaowu Xu committed
1013
  return base + av1_raster_block_offset(plane_bsize, raster_block, stride);
Jingning Han's avatar
Jingning Han committed
1014 1015
}

Yaowu Xu's avatar
Yaowu Xu committed
1016 1017 1018
YV12_BUFFER_CONFIG *av1_get_scaled_ref_frame(const AV1_COMP *cpi,
                                             int ref_frame) {
  const AV1_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
1019 1020
  const int scaled_idx = cpi->scaled_ref_idx[ref_frame - 1];
  const int ref_idx = get_ref_frame_buf_idx(cpi, ref_frame);
1021 1022 1023
  return (scaled_idx != ref_idx && scaled_idx != INVALID_IDX)
             ? &cm->buffer_pool->frame_bufs[scaled_idx].buf
             : NULL;
Jingning Han's avatar
Jingning Han committed
1024 1025
}

1026
#if CONFIG_DUAL_FILTER
1027 1028
int av1_get_switchable_rate(const AV1_COMMON *const cm, MACROBLOCK *x,
                            const MACROBLOCKD *xd) {
1029 1030 1031 1032 1033 1034 1035 1036