rd.c 62.5 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4 5 6 7 8 9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10 11 12 13 14 15
 */

#include <assert.h>
#include <math.h>
#include <stdio.h>

Yaowu Xu's avatar
Yaowu Xu committed
16
#include "./av1_rtcd.h"
Jingning Han's avatar
Jingning Han committed
17

Yaowu Xu's avatar
Yaowu Xu committed
18 19
#include "aom_dsp/aom_dsp_common.h"
#include "aom_mem/aom_mem.h"
20 21 22
#include "aom_ports/bitops.h"
#include "aom_ports/mem.h"
#include "aom_ports/system_state.h"
Jingning Han's avatar
Jingning Han committed
23

24 25 26 27 28 29 30 31 32
#include "av1/common/common.h"
#include "av1/common/entropy.h"
#include "av1/common/entropymode.h"
#include "av1/common/mvref_common.h"
#include "av1/common/pred_common.h"
#include "av1/common/quant_common.h"
#include "av1/common/reconinter.h"
#include "av1/common/reconintra.h"
#include "av1/common/seg_common.h"
33

34
#include "av1/encoder/av1_quantize.h"
35 36 37 38
#include "av1/encoder/cost.h"
#include "av1/encoder/encodemb.h"
#include "av1/encoder/encodemv.h"
#include "av1/encoder/encoder.h"
39 40 41
#if CONFIG_LV_MAP
#include "av1/encoder/encodetxb.h"
#endif
42 43 44 45
#include "av1/encoder/mcomp.h"
#include "av1/encoder/ratectrl.h"
#include "av1/encoder/rd.h"
#include "av1/encoder/tokenize.h"
Jingning Han's avatar
Jingning Han committed
46

47
#define RD_THRESH_POW 1.25
Jingning Han's avatar
Jingning Han committed
48 49 50 51 52 53 54 55

// Factor to weigh the rate for switchable interp filters.
#define SWITCHABLE_INTERP_RATE_FACTOR 1

// The baseline rd thresholds for breaking out of the rd loop for
// certain modes are assumed to be based on 8x8 blocks.
// This table is used to correct for block size.
// The factors here are << 2 (2 = x0.5, 32 = x8 etc).
56
static const uint8_t rd_thresh_block_size_factor[BLOCK_SIZES_ALL] = {
57
#if CONFIG_CHROMA_2X2 || CONFIG_CHROMA_SUB8X8
Jingning Han's avatar
Jingning Han committed
58 59
  2,  2,  2,
#endif
60
  2,  3,  3,  4, 6,  6,  8, 12, 12, 16, 24, 24, 32,
61
#if CONFIG_EXT_PARTITION
62
  48, 48, 64,
63
#endif  // CONFIG_EXT_PARTITION
64 65 66 67
  4,  4,  8,  8, 16, 16,
#if CONFIG_EXT_PARTITION
  32, 32
#endif  // CONFIG_EXT_PARTITION
Jingning Han's avatar
Jingning Han committed
68 69
};

Hui Su's avatar
Hui Su committed
70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107
#if CONFIG_EXT_TX
static const int use_intra_ext_tx_for_txsize[EXT_TX_SETS_INTRA][EXT_TX_SIZES] =
    {
#if CONFIG_CHROMA_2X2
      { 1, 1, 1, 1, 1 },  // unused
      { 0, 1, 1, 0, 0 },
      { 0, 0, 0, 1, 0 },
#if CONFIG_MRC_TX
      { 0, 0, 0, 0, 1 },
#endif  // CONFIG_MRC_TX
#else   // CONFIG_CHROMA_2X2
      { 1, 1, 1, 1 },  // unused
      { 1, 1, 0, 0 },
      { 0, 0, 1, 0 },
#if CONFIG_MRC_TX
      { 0, 0, 0, 1 },
#endif  // CONFIG_MRC_TX
#endif  // CONFIG_CHROMA_2X2
    };

static const int use_inter_ext_tx_for_txsize[EXT_TX_SETS_INTER][EXT_TX_SIZES] =
    {
#if CONFIG_CHROMA_2X2
      { 1, 1, 1, 1, 1 },  // unused
      { 0, 1, 1, 0, 0 }, { 0, 0, 0, 1, 0 }, { 0, 0, 0, 0, 1 },
#if CONFIG_MRC_TX
      { 0, 0, 0, 0, 1 },
#endif  // CONFIG_MRC_TX
#else   // CONFIG_CHROMA_2X2
      { 1, 1, 1, 1 },  // unused
      { 1, 1, 0, 0 }, { 0, 0, 1, 0 }, { 0, 0, 0, 1 },
#if CONFIG_MRC_TX
      { 0, 0, 0, 1 },
#endif  // CONFIG_MRC_TX
#endif  // CONFIG_CHROMA_2X2
    };
#endif  // CONFIG_EXT_TX

108 109
void av1_fill_mode_rates(AV1_COMMON *const cm, MACROBLOCK *x,
                         FRAME_CONTEXT *fc) {
Jingning Han's avatar
Jingning Han committed
110 111
  int i, j;

112 113
  if (cm->frame_type == KEY_FRAME) {
    for (i = 0; i < PARTITION_CONTEXTS_PRIMARY; ++i)
Yue Chen's avatar
Yue Chen committed
114 115
      av1_cost_tokens_from_cdf(x->partition_cost[i], fc->partition_cdf[i],
                               NULL);
116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139
#if CONFIG_UNPOISON_PARTITION_CTX
    for (; i < PARTITION_CONTEXTS_PRIMARY + PARTITION_BLOCK_SIZES; ++i) {
      aom_prob p = fc->partition_prob[i][PARTITION_VERT];
      assert(p > 0);
      x->partition_cost[i][PARTITION_NONE] = INT_MAX;
      x->partition_cost[i][PARTITION_HORZ] = INT_MAX;
      x->partition_cost[i][PARTITION_VERT] = av1_cost_bit(p, 0);
      x->partition_cost[i][PARTITION_SPLIT] = av1_cost_bit(p, 1);
    }
    for (; i < PARTITION_CONTEXTS_PRIMARY + 2 * PARTITION_BLOCK_SIZES; ++i) {
      aom_prob p = fc->partition_prob[i][PARTITION_HORZ];
      assert(p > 0);
      x->partition_cost[i][PARTITION_NONE] = INT_MAX;
      x->partition_cost[i][PARTITION_HORZ] = av1_cost_bit(p, 0);
      x->partition_cost[i][PARTITION_VERT] = INT_MAX;
      x->partition_cost[i][PARTITION_SPLIT] = av1_cost_bit(p, 1);
    }
    x->partition_cost[PARTITION_CONTEXTS][PARTITION_NONE] = INT_MAX;
    x->partition_cost[PARTITION_CONTEXTS][PARTITION_HORZ] = INT_MAX;
    x->partition_cost[PARTITION_CONTEXTS][PARTITION_VERT] = INT_MAX;
    x->partition_cost[PARTITION_CONTEXTS][PARTITION_SPLIT] = 0;
#endif  // CONFIG_UNPOISON_PARTITION_CTX
  }

Jingning Han's avatar
Jingning Han committed
140 141
  for (i = 0; i < INTRA_MODES; ++i)
    for (j = 0; j < INTRA_MODES; ++j)
Debargha Mukherjee's avatar
Debargha Mukherjee committed
142
      av1_cost_tokens_from_cdf(x->y_mode_costs[i][j], fc->kf_y_cdf[i][j],
143
                               av1_intra_mode_inv);
Jingning Han's avatar
Jingning Han committed
144

145
  for (i = 0; i < BLOCK_SIZE_GROUPS; ++i)
146
    av1_cost_tokens_from_cdf(x->mbmode_cost[i], fc->y_mode_cdf[i],
147
                             av1_intra_mode_inv);
148 149 150 151 152 153 154
  const int *uv_mode_inv_map =
#if CONFIG_CFL
      // CfL codes the uv_mode without reordering it
      NULL;
#else
      av1_intra_mode_inv;
#endif
155
  for (i = 0; i < INTRA_MODES; ++i)
156
    av1_cost_tokens_from_cdf(x->intra_uv_mode_cost[i], fc->uv_mode_cdf[i],
157
                             uv_mode_inv_map);
Jingning Han's avatar
Jingning Han committed
158 159

  for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; ++i)
Yue Chen's avatar
Yue Chen committed
160
    av1_cost_tokens_from_cdf(x->switchable_interp_costs[i],
161
                             fc->switchable_interp_cdf[i], NULL);
hui su's avatar
hui su committed
162 163

  for (i = 0; i < PALETTE_BLOCK_SIZES; ++i) {
164
    av1_cost_tokens_from_cdf(x->palette_y_size_cost[i],
165
                             fc->palette_y_size_cdf[i], NULL);
166
    av1_cost_tokens_from_cdf(x->palette_uv_size_cost[i],
167
                             fc->palette_uv_size_cdf[i], NULL);
hui su's avatar
hui su committed
168 169
  }

170
  for (i = 0; i < PALETTE_SIZES; ++i) {
171
    for (j = 0; j < PALETTE_COLOR_INDEX_CONTEXTS; ++j) {
172
      av1_cost_tokens_from_cdf(x->palette_y_color_cost[i][j],
173
                               fc->palette_y_color_index_cdf[i][j], NULL);
174
      av1_cost_tokens_from_cdf(x->palette_uv_color_cost[i][j],
175
                               fc->palette_uv_color_index_cdf[i][j], NULL);
hui su's avatar
hui su committed
176
    }
177
  }
178 179 180 181 182 183 184 185 186 187
#if CONFIG_MRC_TX
  for (i = 0; i < PALETTE_SIZES; ++i) {
    for (j = 0; j < PALETTE_COLOR_INDEX_CONTEXTS; ++j) {
      av1_cost_tokens_from_cdf(x->mrc_mask_inter_cost[i][j],
                               fc->mrc_mask_inter_cdf[i][j], NULL);
      av1_cost_tokens_from_cdf(x->mrc_mask_intra_cost[i][j],
                               fc->mrc_mask_intra_cdf[i][j], NULL);
    }
  }
#endif  // CONFIG_MRC_TX
188

189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209
#if CONFIG_CFL
  int sign_cost[CFL_JOINT_SIGNS];
  av1_cost_tokens_from_cdf(sign_cost, fc->cfl_sign_cdf, NULL);
  for (int joint_sign = 0; joint_sign < CFL_JOINT_SIGNS; joint_sign++) {
    const aom_cdf_prob *cdf_u = fc->cfl_alpha_cdf[CFL_CONTEXT_U(joint_sign)];
    const aom_cdf_prob *cdf_v = fc->cfl_alpha_cdf[CFL_CONTEXT_V(joint_sign)];
    int *cost_u = x->cfl_cost[joint_sign][CFL_PRED_U];
    int *cost_v = x->cfl_cost[joint_sign][CFL_PRED_V];
    if (CFL_SIGN_U(joint_sign) == CFL_SIGN_ZERO)
      memset(cost_u, 0, CFL_ALPHABET_SIZE * sizeof(*cost_u));
    else
      av1_cost_tokens_from_cdf(cost_u, cdf_u, NULL);
    if (CFL_SIGN_V(joint_sign) == CFL_SIGN_ZERO)
      memset(cost_v, 0, CFL_ALPHABET_SIZE * sizeof(*cost_v));
    else
      av1_cost_tokens_from_cdf(cost_v, cdf_v, NULL);
    for (int u = 0; u < CFL_ALPHABET_SIZE; u++)
      cost_u[u] += sign_cost[joint_sign];
  }
#endif  // CONFIG_CFL

Jingning Han's avatar
Jingning Han committed
210
  for (i = 0; i < MAX_TX_DEPTH; ++i)
211
    for (j = 0; j < TX_SIZE_CONTEXTS; ++j)
Yue Chen's avatar
Yue Chen committed
212 213
      av1_cost_tokens_from_cdf(x->tx_size_cost[i][j], fc->tx_size_cdf[i][j],
                               NULL);
214

215
#if CONFIG_EXT_TX
216 217 218
  for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
    int s;
    for (s = 1; s < EXT_TX_SETS_INTER; ++s) {
219
      if (use_inter_ext_tx_for_txsize[s][i]) {
220 221 222
        av1_cost_tokens_from_cdf(
            x->inter_tx_type_costs[s][i], fc->inter_ext_tx_cdf[s][i],
            av1_ext_tx_inv[av1_ext_tx_set_idx_to_type[1][s]]);
223 224 225
      }
    }
    for (s = 1; s < EXT_TX_SETS_INTRA; ++s) {
226
      if (use_intra_ext_tx_for_txsize[s][i]) {
Hui Su's avatar
Hui Su committed
227 228 229
        for (j = 0; j < INTRA_MODES; ++j) {
          av1_cost_tokens_from_cdf(
              x->intra_tx_type_costs[s][i][j], fc->intra_ext_tx_cdf[s][i][j],
230
              av1_ext_tx_inv[av1_ext_tx_set_idx_to_type[0][s]]);
Hui Su's avatar
Hui Su committed
231
        }
232 233
      }
    }
234
  }
235
#else
236 237
  for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
    for (j = 0; j < TX_TYPES; ++j)
Yue Chen's avatar
Yue Chen committed
238 239
      av1_cost_tokens_from_cdf(x->intra_tx_type_costs[i][j],
                               fc->intra_ext_tx_cdf[i][j], av1_ext_tx_inv);
240 241
  }
  for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
Yue Chen's avatar
Yue Chen committed
242 243
    av1_cost_tokens_from_cdf(x->inter_tx_type_costs[i], fc->inter_ext_tx_cdf[i],
                             av1_ext_tx_inv);
244
  }
245
#endif  // CONFIG_EXT_TX
246
#if CONFIG_EXT_INTRA
hui su's avatar
hui su committed
247
#if CONFIG_INTRA_INTERP
248
  for (i = 0; i < INTRA_FILTERS + 1; ++i)
Yue Chen's avatar
Yue Chen committed
249 250
    av1_cost_tokens_from_cdf(x->intra_filter_cost[i], fc->intra_filter_cdf[i],
                             NULL);
hui su's avatar
hui su committed
251
#endif  // CONFIG_INTRA_INTERP
252
#endif  // CONFIG_EXT_INTRA
253
#if CONFIG_LOOP_RESTORATION
254
  av1_cost_tokens(x->switchable_restore_cost, fc->switchable_restore_prob,
255 256
                  av1_switchable_restore_tree);
#endif  // CONFIG_LOOP_RESTORATION
Hui Su's avatar
Hui Su committed
257 258 259
#if CONFIG_INTRABC
  av1_cost_tokens_from_cdf(x->intrabc_cost, fc->intrabc_cdf, NULL);
#endif  // CONFIG_INTRABC
260 261 262

  if (!frame_is_intra_only(cm)) {
    for (i = 0; i < NEWMV_MODE_CONTEXTS; ++i) {
Yue Chen's avatar
Yue Chen committed
263 264 265
#if CONFIG_NEW_MULTISYMBOL
      av1_cost_tokens_from_cdf(x->newmv_mode_cost[i], fc->newmv_cdf[i], NULL);
#else
266 267
      x->newmv_mode_cost[i][0] = av1_cost_bit(fc->newmv_prob[i], 0);
      x->newmv_mode_cost[i][1] = av1_cost_bit(fc->newmv_prob[i], 1);
Yue Chen's avatar
Yue Chen committed
268
#endif
269 270 271
    }

    for (i = 0; i < ZEROMV_MODE_CONTEXTS; ++i) {
Yue Chen's avatar
Yue Chen committed
272 273 274
#if CONFIG_NEW_MULTISYMBOL
      av1_cost_tokens_from_cdf(x->zeromv_mode_cost[i], fc->zeromv_cdf[i], NULL);
#else
275 276
      x->zeromv_mode_cost[i][0] = av1_cost_bit(fc->zeromv_prob[i], 0);
      x->zeromv_mode_cost[i][1] = av1_cost_bit(fc->zeromv_prob[i], 1);
Yue Chen's avatar
Yue Chen committed
277
#endif
278 279 280
    }

    for (i = 0; i < REFMV_MODE_CONTEXTS; ++i) {
Yue Chen's avatar
Yue Chen committed
281 282 283
#if CONFIG_NEW_MULTISYMBOL
      av1_cost_tokens_from_cdf(x->refmv_mode_cost[i], fc->refmv_cdf[i], NULL);
#else
284 285
      x->refmv_mode_cost[i][0] = av1_cost_bit(fc->refmv_prob[i], 0);
      x->refmv_mode_cost[i][1] = av1_cost_bit(fc->refmv_prob[i], 1);
Yue Chen's avatar
Yue Chen committed
286
#endif
287 288 289
    }

    for (i = 0; i < DRL_MODE_CONTEXTS; ++i) {
Yue Chen's avatar
Yue Chen committed
290 291 292
#if CONFIG_NEW_MULTISYMBOL
      av1_cost_tokens_from_cdf(x->drl_mode_cost0[i], fc->drl_cdf[i], NULL);
#else
293 294
      x->drl_mode_cost0[i][0] = av1_cost_bit(fc->drl_prob[i], 0);
      x->drl_mode_cost0[i][1] = av1_cost_bit(fc->drl_prob[i], 1);
Yue Chen's avatar
Yue Chen committed
295
#endif
296 297 298
    }
#if CONFIG_EXT_INTER
    for (i = 0; i < INTER_MODE_CONTEXTS; ++i)
Yue Chen's avatar
Yue Chen committed
299 300
      av1_cost_tokens_from_cdf(x->inter_compound_mode_cost[i],
                               fc->inter_compound_mode_cdf[i], NULL);
301
#if CONFIG_WEDGE || CONFIG_COMPOUND_SEGMENT
302 303 304
    for (i = 0; i < BLOCK_SIZES_ALL; ++i)
      av1_cost_tokens_from_cdf(x->compound_type_cost[i],
                               fc->compound_type_cdf[i], NULL);
305
#endif  // CONFIG_WEDGE || CONFIG_COMPOUND_SEGMENT
306 307
#if CONFIG_COMPOUND_SINGLEREF
    for (i = 0; i < INTER_MODE_CONTEXTS; ++i)
Yue Chen's avatar
Yue Chen committed
308 309
      av1_cost_tokens_from_cdf(x->inter_singleref_comp_mode_cost[i],
                               fc->inter_singleref_comp_mode_cdf[i], NULL);
310 311 312
#endif  // CONFIG_COMPOUND_SINGLEREF
#if CONFIG_INTERINTRA
    for (i = 0; i < BLOCK_SIZE_GROUPS; ++i)
Yue Chen's avatar
Yue Chen committed
313 314
      av1_cost_tokens_from_cdf(x->interintra_mode_cost[i],
                               fc->interintra_mode_cdf[i], NULL);
315 316 317 318
#endif  // CONFIG_INTERINTRA
#endif  // CONFIG_EXT_INTER
#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
    for (i = BLOCK_8X8; i < BLOCK_SIZES_ALL; i++) {
319 320
      av1_cost_tokens_from_cdf(x->motion_mode_cost[i], fc->motion_mode_cdf[i],
                               NULL);
321 322 323
    }
#if CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION
    for (i = BLOCK_8X8; i < BLOCK_SIZES_ALL; i++) {
324 325 326 327 328
#if CONFIG_NCOBMC_ADAPT_WEIGHT
      av1_cost_tokens_from_cdf(x->motion_mode_cost2[i], fc->ncobmc_cdf[i],
                               NULL);
#endif
#if CONFIG_NEW_MULTISYMBOL || CONFIG_NCOBMC_ADAPT_WEIGHT
Yue Chen's avatar
Yue Chen committed
329 330
      av1_cost_tokens_from_cdf(x->motion_mode_cost1[i], fc->obmc_cdf[i], NULL);
#else
331 332
      x->motion_mode_cost1[i][0] = av1_cost_bit(fc->obmc_prob[i], 0);
      x->motion_mode_cost1[i][1] = av1_cost_bit(fc->obmc_prob[i], 1);
Yue Chen's avatar
Yue Chen committed
333
#endif
334 335 336 337
    }
#endif  // CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION
#if CONFIG_MOTION_VAR && CONFIG_NCOBMC_ADAPT_WEIGHT
    for (i = ADAPT_OVERLAP_BLOCK_8X8; i < ADAPT_OVERLAP_BLOCKS; ++i) {
338 339
      av1_cost_tokens_from_cdf(x->ncobmc_mode_cost[i], fc->ncobmc_mode_cdf[i],
                               NULL);
340 341 342 343
    }
#endif  // CONFIG_MOTION_VAR && CONFIG_NCOBMC_ADAPT_WEIGHT
#endif  // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
  }
Jingning Han's avatar
Jingning Han committed
344 345 346 347 348 349
}

// Values are now correlated to quantizer.
static int sad_per_bit16lut_8[QINDEX_RANGE];
static int sad_per_bit4lut_8[QINDEX_RANGE];

350
#if CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
351 352 353 354 355 356 357
static int sad_per_bit16lut_10[QINDEX_RANGE];
static int sad_per_bit4lut_10[QINDEX_RANGE];
static int sad_per_bit16lut_12[QINDEX_RANGE];
static int sad_per_bit4lut_12[QINDEX_RANGE];
#endif

static void init_me_luts_bd(int *bit16lut, int *bit4lut, int range,
Yaowu Xu's avatar
Yaowu Xu committed
358
                            aom_bit_depth_t bit_depth) {
Jingning Han's avatar
Jingning Han committed
359 360 361 362 363
  int i;
  // Initialize the sad lut tables using a formulaic calculation for now.
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < range; i++) {
Yaowu Xu's avatar
Yaowu Xu committed
364
    const double q = av1_convert_qindex_to_q(i, bit_depth);
Jingning Han's avatar
Jingning Han committed
365 366 367 368 369
    bit16lut[i] = (int)(0.0418 * q + 2.4107);
    bit4lut[i] = (int)(0.063 * q + 2.742);
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
370
void av1_init_me_luts(void) {
Jingning Han's avatar
Jingning Han committed
371
  init_me_luts_bd(sad_per_bit16lut_8, sad_per_bit4lut_8, QINDEX_RANGE,
Yaowu Xu's avatar
Yaowu Xu committed
372
                  AOM_BITS_8);
373
#if CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
374
  init_me_luts_bd(sad_per_bit16lut_10, sad_per_bit4lut_10, QINDEX_RANGE,
Yaowu Xu's avatar
Yaowu Xu committed
375
                  AOM_BITS_10);
Jingning Han's avatar
Jingning Han committed
376
  init_me_luts_bd(sad_per_bit16lut_12, sad_per_bit4lut_12, QINDEX_RANGE,
Yaowu Xu's avatar
Yaowu Xu committed
377
                  AOM_BITS_12);
Jingning Han's avatar
Jingning Han committed
378 379 380
#endif
}

381 382
static const int rd_boost_factor[16] = { 64, 32, 32, 32, 24, 16, 12, 12,
                                         8,  8,  4,  4,  2,  2,  1,  0 };
Jingning Han's avatar
Jingning Han committed
383
static const int rd_frame_type_factor[FRAME_UPDATE_TYPES] = {
384
  128, 144, 128, 128, 144,
385
#if CONFIG_EXT_REFS
386
  // TODO(zoeliu): To adjust further following factor values.
387
  128, 128, 128,
388 389
  // TODO(weitinglin): We should investigate if the values should be the same
  //                   as the value used by OVERLAY frame
390
  144,  // INTNL_OVERLAY_UPDATE
Zoe Liu's avatar
Zoe Liu committed
391
  128   // INTNL_ARF_UPDATE
392
#endif  // CONFIG_EXT_REFS
Jingning Han's avatar
Jingning Han committed
393 394
};

Yaowu Xu's avatar
Yaowu Xu committed
395 396
int av1_compute_rd_mult(const AV1_COMP *cpi, int qindex) {
  const int64_t q = av1_dc_quant(qindex, 0, cpi->common.bit_depth);
397
#if CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
398 399
  int64_t rdmult = 0;
  switch (cpi->common.bit_depth) {
Yaowu Xu's avatar
Yaowu Xu committed
400 401 402
    case AOM_BITS_8: rdmult = 88 * q * q / 24; break;
    case AOM_BITS_10: rdmult = ROUND_POWER_OF_TWO(88 * q * q / 24, 4); break;
    case AOM_BITS_12: rdmult = ROUND_POWER_OF_TWO(88 * q * q / 24, 8); break;
Jingning Han's avatar
Jingning Han committed
403
    default:
Yaowu Xu's avatar
Yaowu Xu committed
404
      assert(0 && "bit_depth should be AOM_BITS_8, AOM_BITS_10 or AOM_BITS_12");
Jingning Han's avatar
Jingning Han committed
405 406 407 408
      return -1;
  }
#else
  int64_t rdmult = 88 * q * q / 24;
409
#endif  // CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
410 411 412
  if (cpi->oxcf.pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    const GF_GROUP *const gf_group = &cpi->twopass.gf_group;
    const FRAME_UPDATE_TYPE frame_type = gf_group->update_type[gf_group->index];
Yaowu Xu's avatar
Yaowu Xu committed
413
    const int boost_index = AOMMIN(15, (cpi->rc.gfu_boost / 100));
Jingning Han's avatar
Jingning Han committed
414 415 416 417

    rdmult = (rdmult * rd_frame_type_factor[frame_type]) >> 7;
    rdmult += ((rdmult * rd_boost_factor[boost_index]) >> 7);
  }
418
  if (rdmult < 1) rdmult = 1;
Jingning Han's avatar
Jingning Han committed
419 420 421
  return (int)rdmult;
}

Yaowu Xu's avatar
Yaowu Xu committed
422
static int compute_rd_thresh_factor(int qindex, aom_bit_depth_t bit_depth) {
Jingning Han's avatar
Jingning Han committed
423
  double q;
424
#if CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
425
  switch (bit_depth) {
Yaowu Xu's avatar
Yaowu Xu committed
426 427 428
    case AOM_BITS_8: q = av1_dc_quant(qindex, 0, AOM_BITS_8) / 4.0; break;
    case AOM_BITS_10: q = av1_dc_quant(qindex, 0, AOM_BITS_10) / 16.0; break;
    case AOM_BITS_12: q = av1_dc_quant(qindex, 0, AOM_BITS_12) / 64.0; break;
Jingning Han's avatar
Jingning Han committed
429
    default:
Yaowu Xu's avatar
Yaowu Xu committed
430
      assert(0 && "bit_depth should be AOM_BITS_8, AOM_BITS_10 or AOM_BITS_12");
Jingning Han's avatar
Jingning Han committed
431 432 433
      return -1;
  }
#else
434
  (void)bit_depth;
Yaowu Xu's avatar
Yaowu Xu committed
435
  q = av1_dc_quant(qindex, 0, AOM_BITS_8) / 4.0;
436
#endif  // CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
437
  // TODO(debargha): Adjust the function below.
Yaowu Xu's avatar
Yaowu Xu committed
438
  return AOMMAX((int)(pow(q, RD_THRESH_POW) * 5.12), 8);
Jingning Han's avatar
Jingning Han committed
439 440
}

Yaowu Xu's avatar
Yaowu Xu committed
441
void av1_initialize_me_consts(const AV1_COMP *cpi, MACROBLOCK *x, int qindex) {
442
#if CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
443
  switch (cpi->common.bit_depth) {
Yaowu Xu's avatar
Yaowu Xu committed
444
    case AOM_BITS_8:
Jingning Han's avatar
Jingning Han committed
445 446 447
      x->sadperbit16 = sad_per_bit16lut_8[qindex];
      x->sadperbit4 = sad_per_bit4lut_8[qindex];
      break;
Yaowu Xu's avatar
Yaowu Xu committed
448
    case AOM_BITS_10:
Jingning Han's avatar
Jingning Han committed
449 450 451
      x->sadperbit16 = sad_per_bit16lut_10[qindex];
      x->sadperbit4 = sad_per_bit4lut_10[qindex];
      break;
Yaowu Xu's avatar
Yaowu Xu committed
452
    case AOM_BITS_12:
Jingning Han's avatar
Jingning Han committed
453 454 455 456
      x->sadperbit16 = sad_per_bit16lut_12[qindex];
      x->sadperbit4 = sad_per_bit4lut_12[qindex];
      break;
    default:
Yaowu Xu's avatar
Yaowu Xu committed
457
      assert(0 && "bit_depth should be AOM_BITS_8, AOM_BITS_10 or AOM_BITS_12");
Jingning Han's avatar
Jingning Han committed
458 459 460 461 462
  }
#else
  (void)cpi;
  x->sadperbit16 = sad_per_bit16lut_8[qindex];
  x->sadperbit4 = sad_per_bit4lut_8[qindex];
463
#endif  // CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
464 465
}

Yaowu Xu's avatar
Yaowu Xu committed
466
static void set_block_thresholds(const AV1_COMMON *cm, RD_OPT *rd) {
Jingning Han's avatar
Jingning Han committed
467 468 469 470
  int i, bsize, segment_id;

  for (segment_id = 0; segment_id < MAX_SEGMENTS; ++segment_id) {
    const int qindex =
Yaowu Xu's avatar
Yaowu Xu committed
471
        clamp(av1_get_qindex(&cm->seg, segment_id, cm->base_qindex) +
472 473
                  cm->y_dc_delta_q,
              0, MAXQ);
Jingning Han's avatar
Jingning Han committed
474 475
    const int q = compute_rd_thresh_factor(qindex, cm->bit_depth);

476
    for (bsize = 0; bsize < BLOCK_SIZES_ALL; ++bsize) {
Jingning Han's avatar
Jingning Han committed
477 478 479 480 481
      // Threshold here seems unnecessarily harsh but fine given actual
      // range of values used for cpi->sf.thresh_mult[].
      const int t = q * rd_thresh_block_size_factor[bsize];
      const int thresh_max = INT_MAX / t;

482 483 484 485 486 487
#if CONFIG_CB4X4
      for (i = 0; i < MAX_MODES; ++i)
        rd->threshes[segment_id][bsize][i] = rd->thresh_mult[i] < thresh_max
                                                 ? rd->thresh_mult[i] * t / 4
                                                 : INT_MAX;
#else
Jingning Han's avatar
Jingning Han committed
488 489
      if (bsize >= BLOCK_8X8) {
        for (i = 0; i < MAX_MODES; ++i)
490 491 492
          rd->threshes[segment_id][bsize][i] = rd->thresh_mult[i] < thresh_max
                                                   ? rd->thresh_mult[i] * t / 4
                                                   : INT_MAX;
Jingning Han's avatar
Jingning Han committed
493 494 495 496 497 498 499
      } else {
        for (i = 0; i < MAX_REFS; ++i)
          rd->threshes[segment_id][bsize][i] =
              rd->thresh_mult_sub8x8[i] < thresh_max
                  ? rd->thresh_mult_sub8x8[i] * t / 4
                  : INT_MAX;
      }
500
#endif
Jingning Han's avatar
Jingning Han committed
501 502 503 504
    }
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
505 506
void av1_set_mvcost(MACROBLOCK *x, MV_REFERENCE_FRAME ref_frame, int ref,
                    int ref_mv_idx) {
507
  MB_MODE_INFO_EXT *mbmi_ext = x->mbmi_ext;
Yaowu Xu's avatar
Yaowu Xu committed
508 509 510 511
  int8_t rf_type = av1_ref_frame_type(x->e_mbd.mi[0]->mbmi.ref_frame);
  int nmv_ctx = av1_nmv_ctx(mbmi_ext->ref_mv_count[rf_type],
                            mbmi_ext->ref_mv_stack[rf_type], ref, ref_mv_idx);
  (void)ref_frame;
512 513 514 515
  x->mvcost = x->mv_cost_stack[nmv_ctx];
  x->nmvjointcost = x->nmv_vec_cost[nmv_ctx];
}

516
#if CONFIG_LV_MAP
517
#if !LV_MAP_PROB
518 519 520 521
static void get_rate_cost(aom_prob p, int cost[2]) {
  cost[0] = av1_cost_bit(p, 0);
  cost[1] = av1_cost_bit(p, 1);
}
522
#endif  // !LV_MAP_PROB
523 524 525 526 527 528

void av1_fill_coeff_costs(MACROBLOCK *x, FRAME_CONTEXT *fc) {
  for (TX_SIZE tx_size = 0; tx_size < TX_SIZES; ++tx_size) {
    for (int plane = 0; plane < PLANE_TYPES; ++plane) {
      LV_MAP_COEFF_COST *pcost = &x->coeff_costs[tx_size][plane];

529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551
#if LV_MAP_PROB
      for (int ctx = 0; ctx < TXB_SKIP_CONTEXTS; ++ctx)
        av1_cost_tokens_from_cdf(pcost->txb_skip_cost[ctx],
                                 fc->txb_skip_cdf[tx_size][ctx], NULL);

      for (int ctx = 0; ctx < SIG_COEF_CONTEXTS; ++ctx)
        av1_cost_tokens_from_cdf(pcost->nz_map_cost[ctx],
                                 fc->nz_map_cdf[tx_size][plane][ctx], NULL);

      for (int ctx = 0; ctx < EOB_COEF_CONTEXTS; ++ctx)
        av1_cost_tokens_from_cdf(pcost->eob_cost[ctx],
                                 fc->eob_flag_cdf[tx_size][plane][ctx], NULL);

      for (int ctx = 0; ctx < DC_SIGN_CONTEXTS; ++ctx)
        av1_cost_tokens_from_cdf(pcost->dc_sign_cost[ctx],
                                 fc->dc_sign_cdf[plane][ctx], NULL);

      for (int layer = 0; layer < NUM_BASE_LEVELS; ++layer)
        for (int ctx = 0; ctx < COEFF_BASE_CONTEXTS; ++ctx)
          av1_cost_tokens_from_cdf(
              pcost->base_cost[layer][ctx],
              fc->coeff_base_cdf[tx_size][plane][layer][ctx], NULL);

552 553 554 555 556 557 558 559
#if BR_NODE
      for (int br = 0; br < BASE_RANGE_SETS; ++br)
        for (int ctx = 0; ctx < LEVEL_CONTEXTS; ++ctx)
          av1_cost_tokens_from_cdf(pcost->br_cost[br][ctx],
                                   fc->coeff_br_cdf[tx_size][plane][br][ctx],
                                   NULL);
#endif  // BR_NODE

560 561 562
      for (int ctx = 0; ctx < LEVEL_CONTEXTS; ++ctx) {
        int lps_rate[2];
        av1_cost_tokens_from_cdf(lps_rate,
563
                                 fc->coeff_lps_cdf[tx_size][plane][ctx], NULL);
564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595

        for (int base_range = 0; base_range < COEFF_BASE_RANGE + 1;
             ++base_range) {
          int br_set_idx = base_range < COEFF_BASE_RANGE
                               ? coeff_to_br_index[base_range]
                               : BASE_RANGE_SETS;

          pcost->lps_cost[ctx][base_range] = 0;

          for (int idx = 0; idx < BASE_RANGE_SETS; ++idx) {
            if (idx == br_set_idx) {
              pcost->lps_cost[ctx][base_range] += pcost->br_cost[idx][ctx][1];

              int br_base = br_index_to_coeff[br_set_idx];
              int br_offset = base_range - br_base;
              int extra_bits = (1 << br_extra_bits[idx]) - 1;
              for (int tok = 0; tok < extra_bits; ++tok) {
                if (tok == br_offset) {
                  pcost->lps_cost[ctx][base_range] += lps_rate[1];
                  break;
                } else {
                  pcost->lps_cost[ctx][base_range] += lps_rate[0];
                }
              }
              break;
            } else {
              pcost->lps_cost[ctx][base_range] += pcost->br_cost[idx][ctx][0];
            }
          }
          // load the base range cost
        }
      }
596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613
#if CONFIG_CTX1D
      for (int tx_class = 0; tx_class < TX_CLASSES; ++tx_class)
        av1_cost_tokens_from_cdf(pcost->eob_mode_cost[tx_class],
                                 fc->eob_mode_cdf[tx_size][plane][tx_class],
                                 NULL);

      for (int tx_class = 0; tx_class < TX_CLASSES; ++tx_class)
        for (int ctx = 0; ctx < EMPTY_LINE_CONTEXTS; ++ctx)
          av1_cost_tokens_from_cdf(
              pcost->empty_line_cost[tx_class][ctx],
              fc->empty_line_cdf[tx_size][plane][tx_class][ctx], NULL);

      for (int tx_class = 0; tx_class < TX_CLASSES; ++tx_class)
        for (int ctx = 0; ctx < HV_EOB_CONTEXTS; ++ctx)
          av1_cost_tokens_from_cdf(
              pcost->hv_eob_cost[tx_class][ctx],
              fc->hv_eob_cdf[tx_size][plane][tx_class][ctx], NULL);
#endif  // CONFIG_CTX1D
614
#else   // LV_MAP_PROB
615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633
      for (int ctx = 0; ctx < TXB_SKIP_CONTEXTS; ++ctx)
        get_rate_cost(fc->txb_skip[tx_size][ctx], pcost->txb_skip_cost[ctx]);

      for (int ctx = 0; ctx < SIG_COEF_CONTEXTS; ++ctx)
        get_rate_cost(fc->nz_map[tx_size][plane][ctx], pcost->nz_map_cost[ctx]);

      for (int ctx = 0; ctx < EOB_COEF_CONTEXTS; ++ctx)
        get_rate_cost(fc->eob_flag[tx_size][plane][ctx], pcost->eob_cost[ctx]);

      for (int ctx = 0; ctx < DC_SIGN_CONTEXTS; ++ctx)
        get_rate_cost(fc->dc_sign[plane][ctx], pcost->dc_sign_cost[ctx]);

      for (int layer = 0; layer < NUM_BASE_LEVELS; ++layer)
        for (int ctx = 0; ctx < COEFF_BASE_CONTEXTS; ++ctx)
          get_rate_cost(fc->coeff_base[tx_size][plane][layer][ctx],
                        pcost->base_cost[layer][ctx]);

      for (int ctx = 0; ctx < LEVEL_CONTEXTS; ++ctx)
        get_rate_cost(fc->coeff_lps[tx_size][plane][ctx], pcost->lps_cost[ctx]);
634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649

#if CONFIG_CTX1D
      for (int tx_class = 0; tx_class < TX_CLASSES; ++tx_class)
        get_rate_cost(fc->eob_mode[tx_size][plane][tx_class],
                      pcost->eob_mode_cost[tx_class]);

      for (int tx_class = 0; tx_class < TX_CLASSES; ++tx_class)
        for (int ctx = 0; ctx < EMPTY_LINE_CONTEXTS; ++ctx)
          get_rate_cost(fc->empty_line[tx_size][plane][tx_class][ctx],
                        pcost->empty_line_cost[tx_class][ctx]);

      for (int tx_class = 0; tx_class < TX_CLASSES; ++tx_class)
        for (int ctx = 0; ctx < HV_EOB_CONTEXTS; ++ctx)
          get_rate_cost(fc->hv_eob[tx_size][plane][tx_class][ctx],
                        pcost->hv_eob_cost[tx_class][ctx]);
#endif  // CONFIG_CTX1D
650
#endif  // LV_MAP_PROB
651 652 653
    }
  }
}
654
#endif  // CONFIG_LV_MAP
655

656 657
void av1_fill_token_costs_from_cdf(av1_coeff_cost *cost,
                                   coeff_cdf_model (*cdf)[PLANE_TYPES]) {
hui su's avatar
hui su committed
658 659 660 661 662 663 664 665 666 667 668 669 670 671
  for (int tx = 0; tx < TX_SIZES; ++tx) {
    for (int pt = 0; pt < PLANE_TYPES; ++pt) {
      for (int rt = 0; rt < REF_TYPES; ++rt) {
        for (int band = 0; band < COEF_BANDS; ++band) {
          for (int ctx = 0; ctx < BAND_COEFF_CONTEXTS(band); ++ctx) {
            av1_cost_tokens_from_cdf(cost[tx][pt][rt][band][ctx],
                                     cdf[tx][pt][rt][band][ctx], NULL);
          }
        }
      }
    }
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
672 673
void av1_initialize_rd_consts(AV1_COMP *cpi) {
  AV1_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
674 675
  MACROBLOCK *const x = &cpi->td.mb;
  RD_OPT *const rd = &cpi->rd;
676
  int nmv_ctx;
Jingning Han's avatar
Jingning Han committed
677

Yaowu Xu's avatar
Yaowu Xu committed
678
  aom_clear_system_state();
Jingning Han's avatar
Jingning Han committed
679

Yaowu Xu's avatar
Yaowu Xu committed
680
  rd->RDMULT = av1_compute_rd_mult(cpi, cm->base_qindex + cm->y_dc_delta_q);
Jingning Han's avatar
Jingning Han committed
681

682
  set_error_per_bit(x, rd->RDMULT);
Jingning Han's avatar
Jingning Han committed
683 684 685

  set_block_thresholds(cm, rd);

686
  for (nmv_ctx = 0; nmv_ctx < NMV_CONTEXTS; ++nmv_ctx) {
RogerZhou's avatar
RogerZhou committed
687 688 689 690 691 692 693 694 695 696 697 698 699
#if CONFIG_AMVR
    if (cm->cur_frame_mv_precision_level) {
      av1_build_nmv_cost_table(x->nmv_vec_cost[nmv_ctx], x->nmvcost[nmv_ctx],
                               &cm->fc->nmvc[nmv_ctx], MV_SUBPEL_NONE);
    } else {
      av1_build_nmv_cost_table(
          x->nmv_vec_cost[nmv_ctx],
          cm->allow_high_precision_mv ? x->nmvcost_hp[nmv_ctx]
                                      : x->nmvcost[nmv_ctx],
          &cm->fc->nmvc[nmv_ctx], cm->allow_high_precision_mv);
    }

#else
700 701 702 703 704
    av1_build_nmv_cost_table(
        x->nmv_vec_cost[nmv_ctx],
        cm->allow_high_precision_mv ? x->nmvcost_hp[nmv_ctx]
                                    : x->nmvcost[nmv_ctx],
        &cm->fc->nmvc[nmv_ctx], cm->allow_high_precision_mv);
RogerZhou's avatar
RogerZhou committed
705
#endif
706 707 708
  }
  x->mvcost = x->mv_cost_stack[0];
  x->nmvjointcost = x->nmv_vec_cost[0];
Jingning Han's avatar
Jingning Han committed
709

Alex Converse's avatar
Alex Converse committed
710 711 712 713 714 715 716 717 718 719
#if CONFIG_INTRABC
  if (frame_is_intra_only(cm) && cm->allow_screen_content_tools &&
      cpi->oxcf.pass != 1) {
    av1_build_nmv_cost_table(
        x->nmv_vec_cost[0],
        cm->allow_high_precision_mv ? x->nmvcost_hp[0] : x->nmvcost[0],
        &cm->fc->ndvc, MV_SUBPEL_NONE);
  }
#endif

720
#if CONFIG_GLOBAL_MOTION
721
  if (cpi->oxcf.pass != 1) {
722
    for (int i = 0; i < TRANS_TYPES; ++i)
723
#if GLOBAL_TRANS_TYPES > 4
724 725
      cpi->gmtype_cost[i] = (1 + (i > 0 ? GLOBAL_TYPE_BITS : 0))
                            << AV1_PROB_COST_SHIFT;
726 727 728 729 730 731 732 733
#else
      // IDENTITY: 1 bit
      // TRANSLATION: 3 bits
      // ROTZOOM: 2 bits
      // AFFINE: 3 bits
      cpi->gmtype_cost[i] = (1 + (i > 0 ? (i == ROTZOOM ? 1 : 2) : 0))
                            << AV1_PROB_COST_SHIFT;
#endif  // GLOBAL_TRANS_TYPES > 4
Jingning Han's avatar
Jingning Han committed
734
  }
735
#endif  // CONFIG_GLOBAL_MOTION
Jingning Han's avatar
Jingning Han committed
736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751
}

static void model_rd_norm(int xsq_q10, int *r_q10, int *d_q10) {
  // NOTE: The tables below must be of the same size.

  // The functions described below are sampled at the four most significant
  // bits of x^2 + 8 / 256.

  // Normalized rate:
  // This table models the rate for a Laplacian source with given variance
  // when quantized with a uniform quantizer with given stepsize. The
  // closed form expression is:
  // Rn(x) = H(sqrt(r)) + sqrt(r)*[1 + H(r)/(1 - r)],
  // where r = exp(-sqrt(2) * x) and x = qpstep / sqrt(variance),
  // and H(x) is the binary entropy function.
  static const int rate_tab_q10[] = {
752 753 754 755 756 757 758 759 760
    65536, 6086, 5574, 5275, 5063, 4899, 4764, 4651, 4553, 4389, 4255, 4142,
    4044,  3958, 3881, 3811, 3748, 3635, 3538, 3453, 3376, 3307, 3244, 3186,
    3133,  3037, 2952, 2877, 2809, 2747, 2690, 2638, 2589, 2501, 2423, 2353,
    2290,  2232, 2179, 2130, 2084, 2001, 1928, 1862, 1802, 1748, 1698, 1651,
    1608,  1530, 1460, 1398, 1342, 1290, 1243, 1199, 1159, 1086, 1021, 963,
    911,   864,  821,  781,  745,  680,  623,  574,  530,  490,  455,  424,
    395,   345,  304,  269,  239,  213,  190,  171,  154,  126,  104,  87,
    73,    61,   52,   44,   38,   28,   21,   16,   12,   10,   8,    6,
    5,     3,    2,    1,    1,    1,    0,    0,
Jingning Han's avatar
Jingning Han committed
761 762 763 764 765 766 767 768 769
  };
  // Normalized distortion:
  // This table models the normalized distortion for a Laplacian source
  // with given variance when quantized with a uniform quantizer
  // with given stepsize. The closed form expression is:
  // Dn(x) = 1 - 1/sqrt(2) * x / sinh(x/sqrt(2))
  // where x = qpstep / sqrt(variance).
  // Note the actual distortion is Dn * variance.
  static const int dist_tab_q10[] = {
770 771 772 773 774 775 776 777 778
    0,    0,    1,    1,    1,    2,    2,    2,    3,    3,    4,    5,
    5,    6,    7,    7,    8,    9,    11,   12,   13,   15,   16,   17,
    18,   21,   24,   26,   29,   31,   34,   36,   39,   44,   49,   54,
    59,   64,   69,   73,   78,   88,   97,   106,  115,  124,  133,  142,
    151,  167,  184,  200,  215,  231,  245,  260,  274,  301,  327,  351,
    375,  397,  418,  439,  458,  495,  528,  559,  587,  613,  637,  659,
    680,  717,  749,  777,  801,  823,  842,  859,  874,  899,  919,  936,
    949,  960,  969,  977,  983,  994,  1001, 1006, 1010, 1013, 1015, 1017,
    1018, 1020, 1022, 1022, 1023, 1023, 1023, 1024,
Jingning Han's avatar
Jingning Han committed
779 780
  };
  static const int xsq_iq_q10[] = {
781 782 783 784 785 786 787 788 789 790 791 792
    0,      4,      8,      12,     16,     20,     24,     28,     32,
    40,     48,     56,     64,     72,     80,     88,     96,     112,
    128,    144,    160,    176,    192,    208,    224,    256,    288,
    320,    352,    384,    416,    448,    480,    544,    608,    672,
    736,    800,    864,    928,    992,    1120,   1248,   1376,   1504,
    1632,   1760,   1888,   2016,   2272,   2528,   2784,   3040,   3296,
    3552,   3808,   4064,   4576,   5088,   5600,   6112,   6624,   7136,
    7648,   8160,   9184,   10208,  11232,  12256,  13280,  14304,  15328,
    16352,  18400,  20448,  22496,  24544,  26592,  28640,  30688,  32736,
    36832,  40928,  45024,  49120,  53216,  57312,  61408,  65504,  73696,
    81888,  90080,  98272,  106464, 114656, 122848, 131040, 147424, 163808,
    180192, 196576, 212960, 229344, 245728,
Jingning Han's avatar
Jingning Han committed
793 794 795 796 797 798 799 800 801 802 803
  };
  const int tmp = (xsq_q10 >> 2) + 8;
  const int k = get_msb(tmp) - 3;
  const int xq = (k << 3) + ((tmp >> k) & 0x7);
  const int one_q10 = 1 << 10;
  const int a_q10 = ((xsq_q10 - xsq_iq_q10[xq]) << 10) >> (2 + k);
  const int b_q10 = one_q10 - a_q10;
  *r_q10 = (rate_tab_q10[xq] * b_q10 + rate_tab_q10[xq + 1] * a_q10) >> 10;
  *d_q10 = (dist_tab_q10[xq] * b_q10 + dist_tab_q10[xq + 1] * a_q10) >> 10;
}

Yaowu Xu's avatar
Yaowu Xu committed
804 805 806
void av1_model_rd_from_var_lapndz(int64_t var, unsigned int n_log2,
                                  unsigned int qstep, int *rate,
                                  int64_t *dist) {
Jingning Han's avatar
Jingning Han committed
807 808 809 810 811 812 813 814 815 816 817 818 819 820
  // This function models the rate and distortion for a Laplacian
  // source with given variance when quantized with a uniform quantizer
  // with given stepsize. The closed form expressions are in:
  // Hang and Chen, "Source Model for transform video coder and its
  // application - Part I: Fundamental Theory", IEEE Trans. Circ.
  // Sys. for Video Tech., April 1997.
  if (var == 0) {
    *rate = 0;
    *dist = 0;
  } else {
    int d_q10, r_q10;
    static const uint32_t MAX_XSQ_Q10 = 245727;
    const uint64_t xsq_q10_64 =
        (((uint64_t)qstep * qstep << (n_log2 + 10)) + (var >> 1)) / var;
Yaowu Xu's avatar
Yaowu Xu committed
821
    const int xsq_q10 = (int)AOMMIN(xsq_q10_64, MAX_XSQ_Q10);
Jingning Han's avatar
Jingning Han committed
822
    model_rd_norm(xsq_q10, &r_q10, &d_q10);
Yaowu Xu's avatar
Yaowu Xu committed
823
    *rate = ROUND_POWER_OF_TWO(r_q10 << n_log2, 10 - AV1_PROB_COST_SHIFT);
Jingning Han's avatar
Jingning Han committed
824 825 826 827
    *dist = (var * (int64_t)d_q10 + 512) >> 10;
  }
}

828 829 830 831
static void get_entropy_contexts_plane(
    BLOCK_SIZE plane_bsize, TX_SIZE tx_size, const struct macroblockd_plane *pd,
    ENTROPY_CONTEXT t_above[2 * MAX_MIB_SIZE],
    ENTROPY_CONTEXT t_left[2 * MAX_MIB_SIZE]) {
832 833
  const int num_4x4_w = block_size_wide[plane_bsize] >> tx_size_wide_log2[0];
  const int num_4x4_h = block_size_high[plane_bsize] >> tx_size_high_log2[0];
Jingning Han's avatar
Jingning Han committed
834 835 836
  const ENTROPY_CONTEXT *const above = pd->above_context;
  const ENTROPY_CONTEXT *const left = pd->left_context;

837 838 839 840 841 842
#if CONFIG_LV_MAP
  memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
  memcpy(t_left, left, sizeof(ENTROPY_CONTEXT) * num_4x4_h);
  return;
#endif  // CONFIG_LV_MAP

Jingning Han's avatar
Jingning Han committed
843
  int i;
844

845
#if CONFIG_CHROMA_2X2
846
  switch (tx_size) {
847
    case TX_2X2:
848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876
      memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
      memcpy(t_left, left, sizeof(ENTROPY_CONTEXT) * num_4x4_h);
      break;
    case TX_4X4:
      for (i = 0; i < num_4x4_w; i += 2)
        t_above[i] = !!*(const uint16_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 2)
        t_left[i] = !!*(const uint16_t *)&left[i];
      break;
    case TX_8X8:
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
    case TX_16X16:
      for (i = 0; i < num_4x4_w; i += 8)
        t_above[i] = !!*(const uint64_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 8)
        t_left[i] = !!*(const uint64_t *)&left[i];
      break;
    case TX_32X32:
      for (i = 0; i < num_4x4_w; i += 16)
        t_above[i] =
            !!(*(const uint64_t *)&above[i] | *(const uint64_t *)&above[i + 8]);
      for (i = 0; i < num_4x4_h; i += 16)
        t_left[i] =
            !!(*(const uint64_t *)&left[i] | *(const uint64_t *)&left[i + 8]);
      break;
877 878 879 880 881 882 883 884 885 886 887 888 889 890
#if CONFIG_TX64X64
    case TX_64X64:
      for (i = 0; i < num_4x4_w; i += 32)
        t_above[i] =
            !!(*(const uint64_t *)&above[i] | *(const uint64_t *)&above[i + 8] |
               *(const uint64_t *)&above[i + 16] |
               *(const uint64_t *)&above[i + 24]);
      for (i = 0; i < num_4x4_h; i += 32)
        t_left[i] =
            !!(*(const uint64_t *)&left[i] | *(const uint64_t *)&left[i + 8] |
               *(const uint64_t *)&left[i + 16] |
               *(const uint64_t *)&left[i + 24]);
      break;
#endif  // CONFIG_TX64X64
891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927
    case TX_4X8:
      for (i = 0; i < num_4x4_w; i += 2)
        t_above[i] = !!*(const uint16_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
    case TX_8X4:
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 2)
        t_left[i] = !!*(const uint16_t *)&left[i];
      break;
    case TX_8X16:
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 8)
        t_left[i] = !!*(const uint64_t *)&left[i];
      break;
    case TX_16X8:
      for (i = 0; i < num_4x4_w; i += 8)
        t_above[i] = !!*(const uint64_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
    case TX_16X32:
      for (i = 0; i < num_4x4_w; i += 8)
        t_above[i] = !!*(const uint64_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 16)
        t_left[i] =
            !!(*(const uint64_t *)&left[i] | *(const uint64_t *)&left[i + 8]);
      break;
    case TX_32X16:
      for (i = 0; i < num_4x4_w; i += 16)
        t_above[i] =
            !!(*(const uint64_t *)&above[i] | *(const uint64_t *)&above[i + 8]);
      for (i = 0; i < num_4x4_h; i += 8)
        t_left[i] = !!*(const uint64_t *)&left[i];
928
      break;
Yue Chen's avatar
Yue Chen committed
929
#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955
    case TX_4X16:
      for (i = 0; i < num_4x4_w; i += 2)
        t_above[i] = !!*(const uint16_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 8)
        t_left[i] = !!*(const uint64_t *)&left[i];
      break;
    case TX_16X4:
      for (i = 0; i < num_4x4_w; i += 8)
        t_above[i] = !!*(const uint64_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 2)
        t_left[i] = !!*(const uint16_t *)&left[i];
      break;
    case TX_8X32:
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 16)
        t_left[i] =
            !!(*(const uint64_t *)&left[i] | *(const uint64_t *)&left[i + 8]);
      break;
    case TX_32X8:
      for (i = 0; i < num_4x4_w; i += 16)
        t_above[i] =
            !!(*(const uint64_t *)&above[i] | *(const uint64_t *)&above[i + 8]);
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
Yue Chen's avatar
Yue Chen committed
956
#endif
957 958 959 960

    default: assert(0 && "Invalid transform size."); break;
  }
  return;
961
#endif  // CONFIG_CHROMA_2X2
962 963

  switch (tx_size) {
Jingning Han's avatar
Jingning Han committed
964 965 966 967
    case TX_4X4:
      memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
      memcpy(t_left, left, sizeof(ENTROPY_CONTEXT) * num_4x4_h);
      break;
968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985
    case TX_8X8:
      for (i = 0; i < num_4x4_w; i += 2)
        t_above[i] = !!*(const uint16_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 2)
        t_left[i] = !!*(const uint16_t *)&left[i];
      break;
    case TX_16X16:
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
    case TX_32X32:
      for (i = 0; i < num_4x4_w; i += 8)
        t_above[i] = !!*(const uint64_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 8)
        t_left[i] = !!*(const uint64_t *)&left[i];
      break;
986 987 988 989 990 991
#if CONFIG_TX64X64
    case TX_64X64:
      for (i = 0; i < num_4x4_w; i += 16)
        t_above[i] =
            !!(*(const uint64_t *)&above[i] | *(const uint64_t *)&above[i + 8]);
      for (i = 0; i < num_4x4_h; i += 16)
992 993
        t_left[i] =
            !!(*(const uint64_t *)&left[i] | *(const uint64_t *)&left[i + 8]);
994 995
      break;
#endif  // CONFIG_TX64X64
996 997 998 999 1000 1001 1002 1003 1004 1005
    case TX_4X8:
      memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
      for (i = 0; i < num_4x4_h; i += 2)
        t_left[i] = !!*(const uint16_t *)&left[i];
      break;
    case TX_8X4:
      for (i = 0; i < num_4x4_w; i += 2)
        t_above[i] = !!*(const uint16_t *)&above[i];
      memcpy(t_left, left, sizeof(ENTROPY_CONTEXT) * num_4x4_h);
      break;
1006
    case TX_8X16:
Jingning Han's avatar
Jingning Han committed
1007 1008
      for (i = 0; i < num_4x4_w; i += 2)
        t_above[i] = !!*(const uint16_t *)&above[i];
1009 1010 1011 1012 1013 1014
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
    case TX_16X8:
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
Jingning Han's avatar
Jingning Han committed
1015 1016 1017
      for (i = 0; i < num_4x4_h; i += 2)
        t_left[i] = !!*(const uint16_t *)&left[i];
      break;
1018
    case TX_16X32:
Jingning Han's avatar
Jingning Han committed
1019 1020
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
1021 1022
      for (i = 0; i < num_4x4_h; i += 8)
        t_left[i] = !!*(const uint64_t *)&left[i];
Jingning Han's avatar
Jingning Han committed
1023
      break;
1024
    case TX_32X16:
Jingning Han's avatar
Jingning Han committed
1025 1026
      for (i = 0; i < num_4x4_w; i += 8)
        t_above[i] = !!*(const uint64_t *)&above[i];
1027 1028 1029
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
Yue Chen's avatar
Yue Chen committed
1030
#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052
    case TX_4X16:
      memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
    case TX_16X4:
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
      memcpy(t_left, left, sizeof(ENTROPY_CONTEXT) * num_4x4_h);
      break;
    case TX_8X32:
      for (i = 0; i < num_4x4_w; i += 2)
        t_above[i] = !!*(const uint16_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 8)
        t_left[i] = !!*(const uint64_t *)&left[i];
      break;
    case TX_32X8:
      for (i = 0; i < num_4x4_w; i += 8)
        t_above[i] = !!*(const uint64_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 2)
        t_left[i] = !!*(const uint16_t *)&left[i];
      break;
Yue Chen's avatar
Yue Chen committed
1053
#endif
clang-format's avatar
clang-format committed
1054
    default: assert(0 && "Invalid transform size."); break;
Jingning Han's avatar
Jingning Han committed
1055 1056 1057
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
1058 1059 1060 1061
void av1_get_entropy_contexts(BLOCK_SIZE bsize, TX_SIZE tx_size,
                              const struct macroblockd_plane *pd,
                              ENTROPY_CONTEXT t_above[2 * MAX_MIB_SIZE],
                              ENTROPY_CONTEXT t_left[2 * MAX_MIB_SIZE]) {
1062
#if CONFIG_CHROMA_SUB8X8
1063 1064 1065
  const BLOCK_SIZE plane_bsize =
      AOMMAX(BLOCK_4X4, get_plane_block_size(bsize, pd));
#else
1066
  const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd);
1067
#endif
1068
  get_entropy_contexts_plane(plane_bsize, tx_size, pd, t_above, t_left);
1069 1070
}

1071
void av1_mv_pred(const AV1_COMP *cpi, MACROBLOCK *x, uint8_t *ref_y_buffer,
Yaowu Xu's avatar
Yaowu Xu committed
1072
                 int ref_y_stride, int ref_frame, BLOCK_SIZE block_size) {
Jingning Han's avatar