rd.c 55.3 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4 5 6 7 8 9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10 11 12 13 14 15
 */

#include <assert.h>
#include <math.h>
#include <stdio.h>

Yaowu Xu's avatar
Yaowu Xu committed
16
#include "./av1_rtcd.h"
Jingning Han's avatar
Jingning Han committed
17

Yaowu Xu's avatar
Yaowu Xu committed
18 19
#include "aom_dsp/aom_dsp_common.h"
#include "aom_mem/aom_mem.h"
20 21 22
#include "aom_ports/bitops.h"
#include "aom_ports/mem.h"
#include "aom_ports/system_state.h"
Jingning Han's avatar
Jingning Han committed
23

24 25 26 27 28 29 30 31 32
#include "av1/common/common.h"
#include "av1/common/entropy.h"
#include "av1/common/entropymode.h"
#include "av1/common/mvref_common.h"
#include "av1/common/pred_common.h"
#include "av1/common/quant_common.h"
#include "av1/common/reconinter.h"
#include "av1/common/reconintra.h"
#include "av1/common/seg_common.h"
33

34
#include "av1/encoder/av1_quantize.h"
35 36 37 38
#include "av1/encoder/cost.h"
#include "av1/encoder/encodemb.h"
#include "av1/encoder/encodemv.h"
#include "av1/encoder/encoder.h"
39 40 41
#if CONFIG_LV_MAP
#include "av1/encoder/encodetxb.h"
#endif
42 43 44 45
#include "av1/encoder/mcomp.h"
#include "av1/encoder/ratectrl.h"
#include "av1/encoder/rd.h"
#include "av1/encoder/tokenize.h"
Jingning Han's avatar
Jingning Han committed
46

47
#define RD_THRESH_POW 1.25
Jingning Han's avatar
Jingning Han committed
48 49 50 51 52 53 54 55

// Factor to weigh the rate for switchable interp filters.
#define SWITCHABLE_INTERP_RATE_FACTOR 1

// The baseline rd thresholds for breaking out of the rd loop for
// certain modes are assumed to be based on 8x8 blocks.
// This table is used to correct for block size.
// The factors here are << 2 (2 = x0.5, 32 = x8 etc).
56
static const uint8_t rd_thresh_block_size_factor[BLOCK_SIZES_ALL] = {
57
  2,  2,  2,  2, 3,  3,  4, 6, 6, 8, 12, 12, 16, 24, 24, 32,
58
#if CONFIG_EXT_PARTITION
59
  48, 48, 64,
60
#endif  // CONFIG_EXT_PARTITION
61 62 63 64
  4,  4,  8,  8, 16, 16,
#if CONFIG_EXT_PARTITION
  32, 32
#endif  // CONFIG_EXT_PARTITION
Jingning Han's avatar
Jingning Han committed
65 66
};

Hui Su's avatar
Hui Su committed
67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87
#if CONFIG_EXT_TX
static const int use_intra_ext_tx_for_txsize[EXT_TX_SETS_INTRA][EXT_TX_SIZES] =
    {
      { 1, 1, 1, 1 },  // unused
      { 1, 1, 0, 0 },
      { 0, 0, 1, 0 },
#if CONFIG_MRC_TX
      { 0, 0, 0, 1 },
#endif  // CONFIG_MRC_TX
    };

static const int use_inter_ext_tx_for_txsize[EXT_TX_SETS_INTER][EXT_TX_SIZES] =
    {
      { 1, 1, 1, 1 },  // unused
      { 1, 1, 0, 0 }, { 0, 0, 1, 0 }, { 0, 0, 0, 1 },
#if CONFIG_MRC_TX
      { 0, 0, 0, 1 },
#endif  // CONFIG_MRC_TX
    };
#endif  // CONFIG_EXT_TX

88 89
void av1_fill_mode_rates(AV1_COMMON *const cm, MACROBLOCK *x,
                         FRAME_CONTEXT *fc) {
Jingning Han's avatar
Jingning Han committed
90 91
  int i, j;

92 93
  if (cm->frame_type == KEY_FRAME) {
    for (i = 0; i < PARTITION_CONTEXTS_PRIMARY; ++i)
Yue Chen's avatar
Yue Chen committed
94 95
      av1_cost_tokens_from_cdf(x->partition_cost[i], fc->partition_cdf[i],
                               NULL);
96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119
#if CONFIG_UNPOISON_PARTITION_CTX
    for (; i < PARTITION_CONTEXTS_PRIMARY + PARTITION_BLOCK_SIZES; ++i) {
      aom_prob p = fc->partition_prob[i][PARTITION_VERT];
      assert(p > 0);
      x->partition_cost[i][PARTITION_NONE] = INT_MAX;
      x->partition_cost[i][PARTITION_HORZ] = INT_MAX;
      x->partition_cost[i][PARTITION_VERT] = av1_cost_bit(p, 0);
      x->partition_cost[i][PARTITION_SPLIT] = av1_cost_bit(p, 1);
    }
    for (; i < PARTITION_CONTEXTS_PRIMARY + 2 * PARTITION_BLOCK_SIZES; ++i) {
      aom_prob p = fc->partition_prob[i][PARTITION_HORZ];
      assert(p > 0);
      x->partition_cost[i][PARTITION_NONE] = INT_MAX;
      x->partition_cost[i][PARTITION_HORZ] = av1_cost_bit(p, 0);
      x->partition_cost[i][PARTITION_VERT] = INT_MAX;
      x->partition_cost[i][PARTITION_SPLIT] = av1_cost_bit(p, 1);
    }
    x->partition_cost[PARTITION_CONTEXTS][PARTITION_NONE] = INT_MAX;
    x->partition_cost[PARTITION_CONTEXTS][PARTITION_HORZ] = INT_MAX;
    x->partition_cost[PARTITION_CONTEXTS][PARTITION_VERT] = INT_MAX;
    x->partition_cost[PARTITION_CONTEXTS][PARTITION_SPLIT] = 0;
#endif  // CONFIG_UNPOISON_PARTITION_CTX
  }

120 121 122 123 124 125 126 127 128
  for (i = 0; i < SKIP_CONTEXTS; ++i) {
#if CONFIG_NEW_MULTISYMBOL
    av1_cost_tokens_from_cdf(x->skip_cost[i], fc->skip_cdfs[i], NULL);
#else
    x->skip_cost[i][0] = av1_cost_bit(fc->skip_probs[i], 0);
    x->skip_cost[i][1] = av1_cost_bit(fc->skip_probs[i], 1);
#endif  // CONFIG_NEW_MULTISYMBOL
  }

129 130 131 132 133
#if CONFIG_KF_CTX
  for (i = 0; i < KF_MODE_CONTEXTS; ++i)
    for (j = 0; j < KF_MODE_CONTEXTS; ++j)
      av1_cost_tokens_from_cdf(x->y_mode_costs[i][j], fc->kf_y_cdf[i][j], NULL);
#else
Jingning Han's avatar
Jingning Han committed
134 135
  for (i = 0; i < INTRA_MODES; ++i)
    for (j = 0; j < INTRA_MODES; ++j)
Hui Su's avatar
Hui Su committed
136
      av1_cost_tokens_from_cdf(x->y_mode_costs[i][j], fc->kf_y_cdf[i][j], NULL);
137
#endif
Jingning Han's avatar
Jingning Han committed
138

139
  for (i = 0; i < BLOCK_SIZE_GROUPS; ++i)
Hui Su's avatar
Hui Su committed
140
    av1_cost_tokens_from_cdf(x->mbmode_cost[i], fc->y_mode_cdf[i], NULL);
141
  for (i = 0; i < INTRA_MODES; ++i)
142
    av1_cost_tokens_from_cdf(x->intra_uv_mode_cost[i], fc->uv_mode_cdf[i],
Hui Su's avatar
Hui Su committed
143
                             NULL);
Jingning Han's avatar
Jingning Han committed
144

145
#if CONFIG_FILTER_INTRA
Yue Chen's avatar
Yue Chen committed
146 147
  av1_cost_tokens_from_cdf(x->filter_intra_mode_cost[0],
                           fc->filter_intra_mode_cdf[0], NULL);
148 149
#endif

Jingning Han's avatar
Jingning Han committed
150
  for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; ++i)
Yue Chen's avatar
Yue Chen committed
151
    av1_cost_tokens_from_cdf(x->switchable_interp_costs[i],
152
                             fc->switchable_interp_cdf[i], NULL);
hui su's avatar
hui su committed
153 154

  for (i = 0; i < PALETTE_BLOCK_SIZES; ++i) {
155
    av1_cost_tokens_from_cdf(x->palette_y_size_cost[i],
156
                             fc->palette_y_size_cdf[i], NULL);
157
    av1_cost_tokens_from_cdf(x->palette_uv_size_cost[i],
158
                             fc->palette_uv_size_cdf[i], NULL);
159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181
    for (j = 0; j < PALETTE_Y_MODE_CONTEXTS; ++j) {
#if CONFIG_NEW_MULTISYMBOL
      av1_cost_tokens_from_cdf(x->palette_y_mode_cost[i][j],
                               fc->palette_y_mode_cdf[i][j], NULL);
#else
      x->palette_y_mode_cost[i][j][0] =
          av1_cost_bit(av1_default_palette_y_mode_prob[i][j], 0);
      x->palette_y_mode_cost[i][j][1] =
          av1_cost_bit(av1_default_palette_y_mode_prob[i][j], 1);
#endif
    }
  }

  for (i = 0; i < PALETTE_UV_MODE_CONTEXTS; ++i) {
#if CONFIG_NEW_MULTISYMBOL
    av1_cost_tokens_from_cdf(x->palette_uv_mode_cost[i],
                             fc->palette_uv_mode_cdf[i], NULL);
#else
    x->palette_uv_mode_cost[i][0] =
        av1_cost_bit(av1_default_palette_uv_mode_prob[i], 0);
    x->palette_uv_mode_cost[i][1] =
        av1_cost_bit(av1_default_palette_uv_mode_prob[i], 1);
#endif
hui su's avatar
hui su committed
182 183
  }

184
  for (i = 0; i < PALETTE_SIZES; ++i) {
185
    for (j = 0; j < PALETTE_COLOR_INDEX_CONTEXTS; ++j) {
186
      av1_cost_tokens_from_cdf(x->palette_y_color_cost[i][j],
187
                               fc->palette_y_color_index_cdf[i][j], NULL);
188
      av1_cost_tokens_from_cdf(x->palette_uv_color_cost[i][j],
189
                               fc->palette_uv_color_index_cdf[i][j], NULL);
hui su's avatar
hui su committed
190
    }
191
  }
192 193 194 195 196 197 198 199 200 201
#if CONFIG_MRC_TX
  for (i = 0; i < PALETTE_SIZES; ++i) {
    for (j = 0; j < PALETTE_COLOR_INDEX_CONTEXTS; ++j) {
      av1_cost_tokens_from_cdf(x->mrc_mask_inter_cost[i][j],
                               fc->mrc_mask_inter_cdf[i][j], NULL);
      av1_cost_tokens_from_cdf(x->mrc_mask_intra_cost[i][j],
                               fc->mrc_mask_intra_cdf[i][j], NULL);
    }
  }
#endif  // CONFIG_MRC_TX
202

203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223
#if CONFIG_CFL
  int sign_cost[CFL_JOINT_SIGNS];
  av1_cost_tokens_from_cdf(sign_cost, fc->cfl_sign_cdf, NULL);
  for (int joint_sign = 0; joint_sign < CFL_JOINT_SIGNS; joint_sign++) {
    const aom_cdf_prob *cdf_u = fc->cfl_alpha_cdf[CFL_CONTEXT_U(joint_sign)];
    const aom_cdf_prob *cdf_v = fc->cfl_alpha_cdf[CFL_CONTEXT_V(joint_sign)];
    int *cost_u = x->cfl_cost[joint_sign][CFL_PRED_U];
    int *cost_v = x->cfl_cost[joint_sign][CFL_PRED_V];
    if (CFL_SIGN_U(joint_sign) == CFL_SIGN_ZERO)
      memset(cost_u, 0, CFL_ALPHABET_SIZE * sizeof(*cost_u));
    else
      av1_cost_tokens_from_cdf(cost_u, cdf_u, NULL);
    if (CFL_SIGN_V(joint_sign) == CFL_SIGN_ZERO)
      memset(cost_v, 0, CFL_ALPHABET_SIZE * sizeof(*cost_v));
    else
      av1_cost_tokens_from_cdf(cost_v, cdf_v, NULL);
    for (int u = 0; u < CFL_ALPHABET_SIZE; u++)
      cost_u[u] += sign_cost[joint_sign];
  }
#endif  // CONFIG_CFL

Jingning Han's avatar
Jingning Han committed
224
  for (i = 0; i < MAX_TX_DEPTH; ++i)
225
    for (j = 0; j < TX_SIZE_CONTEXTS; ++j)
Yue Chen's avatar
Yue Chen committed
226 227
      av1_cost_tokens_from_cdf(x->tx_size_cost[i][j], fc->tx_size_cdf[i][j],
                               NULL);
228

229
#if CONFIG_RECT_TX_EXT
230 231 232 233 234 235 236 237 238
#if CONFIG_NEW_MULTISYMBOL
  av1_cost_tokens_from_cdf(x->quarter_tx_size_cost, fc->quarter_tx_size_cdf,
                           NULL);
#else
  x->quarter_tx_size_cost[0] = av1_cost_bit(fc->quarter_tx_size_prob, 0);
  x->quarter_tx_size_cost[1] = av1_cost_bit(fc->quarter_tx_size_prob, 1);
#endif
#endif

239 240 241 242 243 244 245 246 247 248
  for (i = 0; i < TXFM_PARTITION_CONTEXTS; ++i) {
#if CONFIG_NEW_MULTISYMBOL
    av1_cost_tokens_from_cdf(x->txfm_partition_cost[i],
                             fc->txfm_partition_cdf[i], NULL);
#else
    x->txfm_partition_cost[i][0] = av1_cost_bit(fc->txfm_partition_prob[i], 0);
    x->txfm_partition_cost[i][1] = av1_cost_bit(fc->txfm_partition_prob[i], 1);
#endif
  }

249
#if CONFIG_EXT_TX
250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265
#if CONFIG_LGT_FROM_PRED
  if (LGT_FROM_PRED_INTRA) {
    for (i = 0; i < LGT_SIZES; ++i) {
      for (j = 0; j < INTRA_MODES; ++j) {
        x->intra_lgt_cost[i][j][0] = av1_cost_bit(fc->intra_lgt_prob[i][j], 0);
        x->intra_lgt_cost[i][j][1] = av1_cost_bit(fc->intra_lgt_prob[i][j], 1);
      }
    }
  }
  if (LGT_FROM_PRED_INTER) {
    for (i = 0; i < LGT_SIZES; ++i) {
      x->inter_lgt_cost[i][0] = av1_cost_bit(fc->inter_lgt_prob[i], 0);
      x->inter_lgt_cost[i][1] = av1_cost_bit(fc->inter_lgt_prob[i], 1);
    }
  }
#endif  // CONFIG_LGT_FROM_PRED
266 267 268
  for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
    int s;
    for (s = 1; s < EXT_TX_SETS_INTER; ++s) {
269
      if (use_inter_ext_tx_for_txsize[s][i]) {
270 271 272
        av1_cost_tokens_from_cdf(
            x->inter_tx_type_costs[s][i], fc->inter_ext_tx_cdf[s][i],
            av1_ext_tx_inv[av1_ext_tx_set_idx_to_type[1][s]]);
273 274 275
      }
    }
    for (s = 1; s < EXT_TX_SETS_INTRA; ++s) {
276
      if (use_intra_ext_tx_for_txsize[s][i]) {
Hui Su's avatar
Hui Su committed
277 278 279
        for (j = 0; j < INTRA_MODES; ++j) {
          av1_cost_tokens_from_cdf(
              x->intra_tx_type_costs[s][i][j], fc->intra_ext_tx_cdf[s][i][j],
280
              av1_ext_tx_inv[av1_ext_tx_set_idx_to_type[0][s]]);
Hui Su's avatar
Hui Su committed
281
        }
282 283
      }
    }
284
  }
285
#else
286 287
  for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
    for (j = 0; j < TX_TYPES; ++j)
Yue Chen's avatar
Yue Chen committed
288 289
      av1_cost_tokens_from_cdf(x->intra_tx_type_costs[i][j],
                               fc->intra_ext_tx_cdf[i][j], av1_ext_tx_inv);
290 291
  }
  for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
Yue Chen's avatar
Yue Chen committed
292 293
    av1_cost_tokens_from_cdf(x->inter_tx_type_costs[i], fc->inter_ext_tx_cdf[i],
                             av1_ext_tx_inv);
294
  }
295
#endif  // CONFIG_EXT_TX
Joe Young's avatar
Joe Young committed
296 297 298 299 300 301
#if CONFIG_EXT_INTRA && CONFIG_EXT_INTRA_MOD
  for (i = 0; i < DIRECTIONAL_MODES; ++i) {
    av1_cost_tokens_from_cdf(x->angle_delta_cost[i], fc->angle_delta_cdf[i],
                             NULL);
  }
#endif  // CONFIG_EXT_INTRA && CONFIG_EXT_INTRA_MOD
302
#if CONFIG_LOOP_RESTORATION
303 304 305 306 307 308 309 310 311 312 313 314 315
  av1_cost_tokens_from_cdf(x->switchable_restore_cost,
                           fc->switchable_restore_cdf, NULL);
#if CONFIG_NEW_MULTISYMBOL
  av1_cost_tokens_from_cdf(x->wiener_restore_cost, fc->wiener_restore_cdf,
                           NULL);
  av1_cost_tokens_from_cdf(x->sgrproj_restore_cost, fc->sgrproj_restore_cdf,
                           NULL);
#else
  x->wiener_restore_cost[0] = av1_cost_bit(RESTORE_NONE_WIENER_PROB, 0);
  x->wiener_restore_cost[1] = av1_cost_bit(RESTORE_NONE_WIENER_PROB, 1);
  x->sgrproj_restore_cost[0] = av1_cost_bit(RESTORE_NONE_SGRPROJ_PROB, 0);
  x->sgrproj_restore_cost[1] = av1_cost_bit(RESTORE_NONE_SGRPROJ_PROB, 1);
#endif  // CONFIG_NEW_MULTISYMBOL
316
#endif  // CONFIG_LOOP_RESTORATION
Hui Su's avatar
Hui Su committed
317 318 319
#if CONFIG_INTRABC
  av1_cost_tokens_from_cdf(x->intrabc_cost, fc->intrabc_cdf, NULL);
#endif  // CONFIG_INTRABC
320 321

  if (!frame_is_intra_only(cm)) {
322 323 324 325 326 327 328 329 330 331
    for (i = 0; i < INTRA_INTER_CONTEXTS; ++i) {
#if CONFIG_NEW_MULTISYMBOL
      av1_cost_tokens_from_cdf(x->intra_inter_cost[i], fc->intra_inter_cdf[i],
                               NULL);
#else
      x->intra_inter_cost[i][0] = av1_cost_bit(fc->intra_inter_prob[i], 0);
      x->intra_inter_cost[i][1] = av1_cost_bit(fc->intra_inter_prob[i], 1);
#endif
    }

332
    for (i = 0; i < NEWMV_MODE_CONTEXTS; ++i) {
Yue Chen's avatar
Yue Chen committed
333 334 335
#if CONFIG_NEW_MULTISYMBOL
      av1_cost_tokens_from_cdf(x->newmv_mode_cost[i], fc->newmv_cdf[i], NULL);
#else
336 337
      x->newmv_mode_cost[i][0] = av1_cost_bit(fc->newmv_prob[i], 0);
      x->newmv_mode_cost[i][1] = av1_cost_bit(fc->newmv_prob[i], 1);
Yue Chen's avatar
Yue Chen committed
338
#endif
339 340
    }

Sarah Parker's avatar
Sarah Parker committed
341
    for (i = 0; i < GLOBALMV_MODE_CONTEXTS; ++i) {
Yue Chen's avatar
Yue Chen committed
342 343 344
#if CONFIG_NEW_MULTISYMBOL
      av1_cost_tokens_from_cdf(x->zeromv_mode_cost[i], fc->zeromv_cdf[i], NULL);
#else
345 346
      x->zeromv_mode_cost[i][0] = av1_cost_bit(fc->zeromv_prob[i], 0);
      x->zeromv_mode_cost[i][1] = av1_cost_bit(fc->zeromv_prob[i], 1);
Yue Chen's avatar
Yue Chen committed
347
#endif
348 349 350
    }

    for (i = 0; i < REFMV_MODE_CONTEXTS; ++i) {
Yue Chen's avatar
Yue Chen committed
351 352 353
#if CONFIG_NEW_MULTISYMBOL
      av1_cost_tokens_from_cdf(x->refmv_mode_cost[i], fc->refmv_cdf[i], NULL);
#else
354 355
      x->refmv_mode_cost[i][0] = av1_cost_bit(fc->refmv_prob[i], 0);
      x->refmv_mode_cost[i][1] = av1_cost_bit(fc->refmv_prob[i], 1);
Yue Chen's avatar
Yue Chen committed
356
#endif
357 358 359
    }

    for (i = 0; i < DRL_MODE_CONTEXTS; ++i) {
Yue Chen's avatar
Yue Chen committed
360 361 362
#if CONFIG_NEW_MULTISYMBOL
      av1_cost_tokens_from_cdf(x->drl_mode_cost0[i], fc->drl_cdf[i], NULL);
#else
363 364
      x->drl_mode_cost0[i][0] = av1_cost_bit(fc->drl_prob[i], 0);
      x->drl_mode_cost0[i][1] = av1_cost_bit(fc->drl_prob[i], 1);
Yue Chen's avatar
Yue Chen committed
365
#endif
366 367
    }
    for (i = 0; i < INTER_MODE_CONTEXTS; ++i)
Yue Chen's avatar
Yue Chen committed
368 369
      av1_cost_tokens_from_cdf(x->inter_compound_mode_cost[i],
                               fc->inter_compound_mode_cdf[i], NULL);
370 371 372
    for (i = 0; i < BLOCK_SIZES_ALL; ++i)
      av1_cost_tokens_from_cdf(x->compound_type_cost[i],
                               fc->compound_type_cdf[i], NULL);
373 374
#if CONFIG_COMPOUND_SINGLEREF
    for (i = 0; i < INTER_MODE_CONTEXTS; ++i)
Yue Chen's avatar
Yue Chen committed
375 376
      av1_cost_tokens_from_cdf(x->inter_singleref_comp_mode_cost[i],
                               fc->inter_singleref_comp_mode_cdf[i], NULL);
377
#endif  // CONFIG_COMPOUND_SINGLEREF
378 379 380 381 382 383 384 385
    for (i = 0; i < BLOCK_SIZE_GROUPS; ++i) {
#if CONFIG_NEW_MULTISYMBOL
      av1_cost_tokens_from_cdf(x->interintra_cost[i], fc->interintra_cdf[i],
                               NULL);
#else
      x->interintra_cost[i][0] = av1_cost_bit(fc->interintra_prob[i], 0);
      x->interintra_cost[i][1] = av1_cost_bit(fc->interintra_prob[i], 1);
#endif
Yue Chen's avatar
Yue Chen committed
386 387
      av1_cost_tokens_from_cdf(x->interintra_mode_cost[i],
                               fc->interintra_mode_cdf[i], NULL);
388 389 390 391 392 393 394 395 396 397 398 399
    }
    for (i = 0; i < BLOCK_SIZES_ALL; ++i) {
#if CONFIG_NEW_MULTISYMBOL
      av1_cost_tokens_from_cdf(x->wedge_interintra_cost[i],
                               fc->wedge_interintra_cdf[i], NULL);
#else
      x->wedge_interintra_cost[i][0] =
          av1_cost_bit(fc->wedge_interintra_prob[i], 0);
      x->wedge_interintra_cost[i][1] =
          av1_cost_bit(fc->wedge_interintra_prob[i], 1);
#endif
    }
400
    for (i = BLOCK_8X8; i < BLOCK_SIZES_ALL; i++) {
401 402
      av1_cost_tokens_from_cdf(x->motion_mode_cost[i], fc->motion_mode_cdf[i],
                               NULL);
403 404
    }
    for (i = BLOCK_8X8; i < BLOCK_SIZES_ALL; i++) {
405 406 407 408 409
#if CONFIG_NCOBMC_ADAPT_WEIGHT
      av1_cost_tokens_from_cdf(x->motion_mode_cost2[i], fc->ncobmc_cdf[i],
                               NULL);
#endif
#if CONFIG_NEW_MULTISYMBOL || CONFIG_NCOBMC_ADAPT_WEIGHT
Yue Chen's avatar
Yue Chen committed
410 411
      av1_cost_tokens_from_cdf(x->motion_mode_cost1[i], fc->obmc_cdf[i], NULL);
#else
412 413
      x->motion_mode_cost1[i][0] = av1_cost_bit(fc->obmc_prob[i], 0);
      x->motion_mode_cost1[i][1] = av1_cost_bit(fc->obmc_prob[i], 1);
Yue Chen's avatar
Yue Chen committed
414
#endif
415
    }
416
#if CONFIG_NCOBMC_ADAPT_WEIGHT
417
    for (i = ADAPT_OVERLAP_BLOCK_8X8; i < ADAPT_OVERLAP_BLOCKS; ++i) {
418 419
      av1_cost_tokens_from_cdf(x->ncobmc_mode_cost[i], fc->ncobmc_mode_cdf[i],
                               NULL);
420
    }
421
#endif  // CONFIG_NCOBMC_ADAPT_WEIGHT
422
  }
Jingning Han's avatar
Jingning Han committed
423 424 425 426 427 428
}

// Values are now correlated to quantizer.
static int sad_per_bit16lut_8[QINDEX_RANGE];
static int sad_per_bit4lut_8[QINDEX_RANGE];

429
#if CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
430 431 432 433 434 435 436
static int sad_per_bit16lut_10[QINDEX_RANGE];
static int sad_per_bit4lut_10[QINDEX_RANGE];
static int sad_per_bit16lut_12[QINDEX_RANGE];
static int sad_per_bit4lut_12[QINDEX_RANGE];
#endif

static void init_me_luts_bd(int *bit16lut, int *bit4lut, int range,
Yaowu Xu's avatar
Yaowu Xu committed
437
                            aom_bit_depth_t bit_depth) {
Jingning Han's avatar
Jingning Han committed
438 439 440 441 442
  int i;
  // Initialize the sad lut tables using a formulaic calculation for now.
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < range; i++) {
Yaowu Xu's avatar
Yaowu Xu committed
443
    const double q = av1_convert_qindex_to_q(i, bit_depth);
Jingning Han's avatar
Jingning Han committed
444 445 446 447 448
    bit16lut[i] = (int)(0.0418 * q + 2.4107);
    bit4lut[i] = (int)(0.063 * q + 2.742);
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
449
void av1_init_me_luts(void) {
Jingning Han's avatar
Jingning Han committed
450
  init_me_luts_bd(sad_per_bit16lut_8, sad_per_bit4lut_8, QINDEX_RANGE,
Yaowu Xu's avatar
Yaowu Xu committed
451
                  AOM_BITS_8);
452
#if CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
453
  init_me_luts_bd(sad_per_bit16lut_10, sad_per_bit4lut_10, QINDEX_RANGE,
Yaowu Xu's avatar
Yaowu Xu committed
454
                  AOM_BITS_10);
Jingning Han's avatar
Jingning Han committed
455
  init_me_luts_bd(sad_per_bit16lut_12, sad_per_bit4lut_12, QINDEX_RANGE,
Yaowu Xu's avatar
Yaowu Xu committed
456
                  AOM_BITS_12);
Jingning Han's avatar
Jingning Han committed
457 458 459
#endif
}

460 461
static const int rd_boost_factor[16] = { 64, 32, 32, 32, 24, 16, 12, 12,
                                         8,  8,  4,  4,  2,  2,  1,  0 };
Jingning Han's avatar
Jingning Han committed
462
static const int rd_frame_type_factor[FRAME_UPDATE_TYPES] = {
463 464
  128, 144, 128, 128, 144,
  // TODO(zoeliu): To adjust further following factor values.
465
  128, 128, 128,
466 467
  // TODO(weitinglin): We should investigate if the values should be the same
  //                   as the value used by OVERLAY frame
468
  144,  // INTNL_OVERLAY_UPDATE
Zoe Liu's avatar
Zoe Liu committed
469
  128   // INTNL_ARF_UPDATE
Jingning Han's avatar
Jingning Han committed
470 471
};

Yaowu Xu's avatar
Yaowu Xu committed
472 473
int av1_compute_rd_mult(const AV1_COMP *cpi, int qindex) {
  const int64_t q = av1_dc_quant(qindex, 0, cpi->common.bit_depth);
474
#if CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
475 476
  int64_t rdmult = 0;
  switch (cpi->common.bit_depth) {
Yaowu Xu's avatar
Yaowu Xu committed
477 478 479
    case AOM_BITS_8: rdmult = 88 * q * q / 24; break;
    case AOM_BITS_10: rdmult = ROUND_POWER_OF_TWO(88 * q * q / 24, 4); break;
    case AOM_BITS_12: rdmult = ROUND_POWER_OF_TWO(88 * q * q / 24, 8); break;
Jingning Han's avatar
Jingning Han committed
480
    default:
Yaowu Xu's avatar
Yaowu Xu committed
481
      assert(0 && "bit_depth should be AOM_BITS_8, AOM_BITS_10 or AOM_BITS_12");
Jingning Han's avatar
Jingning Han committed
482 483 484 485
      return -1;
  }
#else
  int64_t rdmult = 88 * q * q / 24;
486
#endif  // CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
487 488 489
  if (cpi->oxcf.pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    const GF_GROUP *const gf_group = &cpi->twopass.gf_group;
    const FRAME_UPDATE_TYPE frame_type = gf_group->update_type[gf_group->index];
Yaowu Xu's avatar
Yaowu Xu committed
490
    const int boost_index = AOMMIN(15, (cpi->rc.gfu_boost / 100));
Jingning Han's avatar
Jingning Han committed
491 492 493 494

    rdmult = (rdmult * rd_frame_type_factor[frame_type]) >> 7;
    rdmult += ((rdmult * rd_boost_factor[boost_index]) >> 7);
  }
495
  if (rdmult < 1) rdmult = 1;
Jingning Han's avatar
Jingning Han committed
496 497 498
  return (int)rdmult;
}

Yaowu Xu's avatar
Yaowu Xu committed
499
static int compute_rd_thresh_factor(int qindex, aom_bit_depth_t bit_depth) {
Jingning Han's avatar
Jingning Han committed
500
  double q;
501
#if CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
502
  switch (bit_depth) {
Yaowu Xu's avatar
Yaowu Xu committed
503 504 505
    case AOM_BITS_8: q = av1_dc_quant(qindex, 0, AOM_BITS_8) / 4.0; break;
    case AOM_BITS_10: q = av1_dc_quant(qindex, 0, AOM_BITS_10) / 16.0; break;
    case AOM_BITS_12: q = av1_dc_quant(qindex, 0, AOM_BITS_12) / 64.0; break;
Jingning Han's avatar
Jingning Han committed
506
    default:
Yaowu Xu's avatar
Yaowu Xu committed
507
      assert(0 && "bit_depth should be AOM_BITS_8, AOM_BITS_10 or AOM_BITS_12");
Jingning Han's avatar
Jingning Han committed
508 509 510
      return -1;
  }
#else
511
  (void)bit_depth;
Yaowu Xu's avatar
Yaowu Xu committed
512
  q = av1_dc_quant(qindex, 0, AOM_BITS_8) / 4.0;
513
#endif  // CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
514
  // TODO(debargha): Adjust the function below.
Yaowu Xu's avatar
Yaowu Xu committed
515
  return AOMMAX((int)(pow(q, RD_THRESH_POW) * 5.12), 8);
Jingning Han's avatar
Jingning Han committed
516 517
}

Yaowu Xu's avatar
Yaowu Xu committed
518
void av1_initialize_me_consts(const AV1_COMP *cpi, MACROBLOCK *x, int qindex) {
519
#if CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
520
  switch (cpi->common.bit_depth) {
Yaowu Xu's avatar
Yaowu Xu committed
521
    case AOM_BITS_8:
Jingning Han's avatar
Jingning Han committed
522 523 524
      x->sadperbit16 = sad_per_bit16lut_8[qindex];
      x->sadperbit4 = sad_per_bit4lut_8[qindex];
      break;
Yaowu Xu's avatar
Yaowu Xu committed
525
    case AOM_BITS_10:
Jingning Han's avatar
Jingning Han committed
526 527 528
      x->sadperbit16 = sad_per_bit16lut_10[qindex];
      x->sadperbit4 = sad_per_bit4lut_10[qindex];
      break;
Yaowu Xu's avatar
Yaowu Xu committed
529
    case AOM_BITS_12:
Jingning Han's avatar
Jingning Han committed
530 531 532 533
      x->sadperbit16 = sad_per_bit16lut_12[qindex];
      x->sadperbit4 = sad_per_bit4lut_12[qindex];
      break;
    default:
Yaowu Xu's avatar
Yaowu Xu committed
534
      assert(0 && "bit_depth should be AOM_BITS_8, AOM_BITS_10 or AOM_BITS_12");
Jingning Han's avatar
Jingning Han committed
535 536 537 538 539
  }
#else
  (void)cpi;
  x->sadperbit16 = sad_per_bit16lut_8[qindex];
  x->sadperbit4 = sad_per_bit4lut_8[qindex];
540
#endif  // CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
541 542
}

Yaowu Xu's avatar
Yaowu Xu committed
543
static void set_block_thresholds(const AV1_COMMON *cm, RD_OPT *rd) {
Jingning Han's avatar
Jingning Han committed
544 545 546 547
  int i, bsize, segment_id;

  for (segment_id = 0; segment_id < MAX_SEGMENTS; ++segment_id) {
    const int qindex =
Yaowu Xu's avatar
Yaowu Xu committed
548
        clamp(av1_get_qindex(&cm->seg, segment_id, cm->base_qindex) +
549 550
                  cm->y_dc_delta_q,
              0, MAXQ);
Jingning Han's avatar
Jingning Han committed
551 552
    const int q = compute_rd_thresh_factor(qindex, cm->bit_depth);

553
    for (bsize = 0; bsize < BLOCK_SIZES_ALL; ++bsize) {
Jingning Han's avatar
Jingning Han committed
554 555 556 557 558
      // Threshold here seems unnecessarily harsh but fine given actual
      // range of values used for cpi->sf.thresh_mult[].
      const int t = q * rd_thresh_block_size_factor[bsize];
      const int thresh_max = INT_MAX / t;

559 560 561 562
      for (i = 0; i < MAX_MODES; ++i)
        rd->threshes[segment_id][bsize][i] = rd->thresh_mult[i] < thresh_max
                                                 ? rd->thresh_mult[i] * t / 4
                                                 : INT_MAX;
Jingning Han's avatar
Jingning Han committed
563 564 565 566
    }
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
567 568
void av1_set_mvcost(MACROBLOCK *x, MV_REFERENCE_FRAME ref_frame, int ref,
                    int ref_mv_idx) {
569
  MB_MODE_INFO_EXT *mbmi_ext = x->mbmi_ext;
Yaowu Xu's avatar
Yaowu Xu committed
570 571 572 573
  int8_t rf_type = av1_ref_frame_type(x->e_mbd.mi[0]->mbmi.ref_frame);
  int nmv_ctx = av1_nmv_ctx(mbmi_ext->ref_mv_count[rf_type],
                            mbmi_ext->ref_mv_stack[rf_type], ref, ref_mv_idx);
  (void)ref_frame;
574 575 576 577
  x->mvcost = x->mv_cost_stack[nmv_ctx];
  x->nmvjointcost = x->nmv_vec_cost[nmv_ctx];
}

578 579
#if CONFIG_LV_MAP
void av1_fill_coeff_costs(MACROBLOCK *x, FRAME_CONTEXT *fc) {
580
  for (int tx_size = 0; tx_size < TX_SIZES; ++tx_size) {
581 582 583
    for (int plane = 0; plane < PLANE_TYPES; ++plane) {
      LV_MAP_COEFF_COST *pcost = &x->coeff_costs[tx_size][plane];

584 585 586 587 588 589 590 591 592 593 594 595
      for (int ctx = 0; ctx < TXB_SKIP_CONTEXTS; ++ctx)
        av1_cost_tokens_from_cdf(pcost->txb_skip_cost[ctx],
                                 fc->txb_skip_cdf[tx_size][ctx], NULL);

      for (int ctx = 0; ctx < SIG_COEF_CONTEXTS; ++ctx)
        av1_cost_tokens_from_cdf(pcost->nz_map_cost[ctx],
                                 fc->nz_map_cdf[tx_size][plane][ctx], NULL);

      for (int ctx = 0; ctx < EOB_COEF_CONTEXTS; ++ctx)
        av1_cost_tokens_from_cdf(pcost->eob_cost[ctx],
                                 fc->eob_flag_cdf[tx_size][plane][ctx], NULL);

Angie Chiang's avatar
Angie Chiang committed
596 597 598 599
      for (int ctx = 0; ctx < EOB_COEF_CONTEXTS; ++ctx)
        av1_cost_tokens_from_cdf(pcost->eob_extra_cost[ctx],
                                 fc->eob_extra_cdf[tx_size][plane][ctx], NULL);

600 601 602 603 604 605 606 607 608 609
      for (int ctx = 0; ctx < DC_SIGN_CONTEXTS; ++ctx)
        av1_cost_tokens_from_cdf(pcost->dc_sign_cost[ctx],
                                 fc->dc_sign_cdf[plane][ctx], NULL);

      for (int layer = 0; layer < NUM_BASE_LEVELS; ++layer)
        for (int ctx = 0; ctx < COEFF_BASE_CONTEXTS; ++ctx)
          av1_cost_tokens_from_cdf(
              pcost->base_cost[layer][ctx],
              fc->coeff_base_cdf[tx_size][plane][layer][ctx], NULL);

610 611 612 613 614 615
      for (int br = 0; br < BASE_RANGE_SETS; ++br)
        for (int ctx = 0; ctx < LEVEL_CONTEXTS; ++ctx)
          av1_cost_tokens_from_cdf(pcost->br_cost[br][ctx],
                                   fc->coeff_br_cdf[tx_size][plane][br][ctx],
                                   NULL);

616 617 618
      for (int ctx = 0; ctx < LEVEL_CONTEXTS; ++ctx) {
        int lps_rate[2];
        av1_cost_tokens_from_cdf(lps_rate,
619
                                 fc->coeff_lps_cdf[tx_size][plane][ctx], NULL);
620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651

        for (int base_range = 0; base_range < COEFF_BASE_RANGE + 1;
             ++base_range) {
          int br_set_idx = base_range < COEFF_BASE_RANGE
                               ? coeff_to_br_index[base_range]
                               : BASE_RANGE_SETS;

          pcost->lps_cost[ctx][base_range] = 0;

          for (int idx = 0; idx < BASE_RANGE_SETS; ++idx) {
            if (idx == br_set_idx) {
              pcost->lps_cost[ctx][base_range] += pcost->br_cost[idx][ctx][1];

              int br_base = br_index_to_coeff[br_set_idx];
              int br_offset = base_range - br_base;
              int extra_bits = (1 << br_extra_bits[idx]) - 1;
              for (int tok = 0; tok < extra_bits; ++tok) {
                if (tok == br_offset) {
                  pcost->lps_cost[ctx][base_range] += lps_rate[1];
                  break;
                } else {
                  pcost->lps_cost[ctx][base_range] += lps_rate[0];
                }
              }
              break;
            } else {
              pcost->lps_cost[ctx][base_range] += pcost->br_cost[idx][ctx][0];
            }
          }
          // load the base range cost
        }
      }
652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669
#if CONFIG_CTX1D
      for (int tx_class = 0; tx_class < TX_CLASSES; ++tx_class)
        av1_cost_tokens_from_cdf(pcost->eob_mode_cost[tx_class],
                                 fc->eob_mode_cdf[tx_size][plane][tx_class],
                                 NULL);

      for (int tx_class = 0; tx_class < TX_CLASSES; ++tx_class)
        for (int ctx = 0; ctx < EMPTY_LINE_CONTEXTS; ++ctx)
          av1_cost_tokens_from_cdf(
              pcost->empty_line_cost[tx_class][ctx],
              fc->empty_line_cdf[tx_size][plane][tx_class][ctx], NULL);

      for (int tx_class = 0; tx_class < TX_CLASSES; ++tx_class)
        for (int ctx = 0; ctx < HV_EOB_CONTEXTS; ++ctx)
          av1_cost_tokens_from_cdf(
              pcost->hv_eob_cost[tx_class][ctx],
              fc->hv_eob_cdf[tx_size][plane][tx_class][ctx], NULL);
#endif  // CONFIG_CTX1D
670 671 672
    }
  }
}
673
#endif  // CONFIG_LV_MAP
674

675 676
void av1_fill_token_costs_from_cdf(av1_coeff_cost *cost,
                                   coeff_cdf_model (*cdf)[PLANE_TYPES]) {
hui su's avatar
hui su committed
677 678 679 680 681 682 683 684 685 686 687 688 689 690
  for (int tx = 0; tx < TX_SIZES; ++tx) {
    for (int pt = 0; pt < PLANE_TYPES; ++pt) {
      for (int rt = 0; rt < REF_TYPES; ++rt) {
        for (int band = 0; band < COEF_BANDS; ++band) {
          for (int ctx = 0; ctx < BAND_COEFF_CONTEXTS(band); ++ctx) {
            av1_cost_tokens_from_cdf(cost[tx][pt][rt][band][ctx],
                                     cdf[tx][pt][rt][band][ctx], NULL);
          }
        }
      }
    }
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
691 692
void av1_initialize_rd_consts(AV1_COMP *cpi) {
  AV1_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
693 694
  MACROBLOCK *const x = &cpi->td.mb;
  RD_OPT *const rd = &cpi->rd;
695
  int nmv_ctx;
Jingning Han's avatar
Jingning Han committed
696

Yaowu Xu's avatar
Yaowu Xu committed
697
  aom_clear_system_state();
Jingning Han's avatar
Jingning Han committed
698

Yaowu Xu's avatar
Yaowu Xu committed
699
  rd->RDMULT = av1_compute_rd_mult(cpi, cm->base_qindex + cm->y_dc_delta_q);
Jingning Han's avatar
Jingning Han committed
700

701
  set_error_per_bit(x, rd->RDMULT);
Jingning Han's avatar
Jingning Han committed
702 703 704

  set_block_thresholds(cm, rd);

705
  for (nmv_ctx = 0; nmv_ctx < NMV_CONTEXTS; ++nmv_ctx) {
RogerZhou's avatar
RogerZhou committed
706
#if CONFIG_AMVR
707
    if (cm->cur_frame_force_integer_mv) {
RogerZhou's avatar
RogerZhou committed
708 709 710 711 712 713 714 715 716 717 718
      av1_build_nmv_cost_table(x->nmv_vec_cost[nmv_ctx], x->nmvcost[nmv_ctx],
                               &cm->fc->nmvc[nmv_ctx], MV_SUBPEL_NONE);
    } else {
      av1_build_nmv_cost_table(
          x->nmv_vec_cost[nmv_ctx],
          cm->allow_high_precision_mv ? x->nmvcost_hp[nmv_ctx]
                                      : x->nmvcost[nmv_ctx],
          &cm->fc->nmvc[nmv_ctx], cm->allow_high_precision_mv);
    }

#else
719 720 721 722 723
    av1_build_nmv_cost_table(
        x->nmv_vec_cost[nmv_ctx],
        cm->allow_high_precision_mv ? x->nmvcost_hp[nmv_ctx]
                                    : x->nmvcost[nmv_ctx],
        &cm->fc->nmvc[nmv_ctx], cm->allow_high_precision_mv);
RogerZhou's avatar
RogerZhou committed
724
#endif
725 726 727
  }
  x->mvcost = x->mv_cost_stack[0];
  x->nmvjointcost = x->nmv_vec_cost[0];
Jingning Han's avatar
Jingning Han committed
728

Alex Converse's avatar
Alex Converse committed
729 730 731 732 733 734 735 736 737 738
#if CONFIG_INTRABC
  if (frame_is_intra_only(cm) && cm->allow_screen_content_tools &&
      cpi->oxcf.pass != 1) {
    av1_build_nmv_cost_table(
        x->nmv_vec_cost[0],
        cm->allow_high_precision_mv ? x->nmvcost_hp[0] : x->nmvcost[0],
        &cm->fc->ndvc, MV_SUBPEL_NONE);
  }
#endif

739
  if (cpi->oxcf.pass != 1) {
740
    for (int i = 0; i < TRANS_TYPES; ++i)
741
#if GLOBAL_TRANS_TYPES > 4
742 743
      cpi->gmtype_cost[i] = (1 + (i > 0 ? GLOBAL_TYPE_BITS : 0))
                            << AV1_PROB_COST_SHIFT;
744 745 746 747 748 749 750 751
#else
      // IDENTITY: 1 bit
      // TRANSLATION: 3 bits
      // ROTZOOM: 2 bits
      // AFFINE: 3 bits
      cpi->gmtype_cost[i] = (1 + (i > 0 ? (i == ROTZOOM ? 1 : 2) : 0))
                            << AV1_PROB_COST_SHIFT;
#endif  // GLOBAL_TRANS_TYPES > 4
Jingning Han's avatar
Jingning Han committed
752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768
  }
}

static void model_rd_norm(int xsq_q10, int *r_q10, int *d_q10) {
  // NOTE: The tables below must be of the same size.

  // The functions described below are sampled at the four most significant
  // bits of x^2 + 8 / 256.

  // Normalized rate:
  // This table models the rate for a Laplacian source with given variance
  // when quantized with a uniform quantizer with given stepsize. The
  // closed form expression is:
  // Rn(x) = H(sqrt(r)) + sqrt(r)*[1 + H(r)/(1 - r)],
  // where r = exp(-sqrt(2) * x) and x = qpstep / sqrt(variance),
  // and H(x) is the binary entropy function.
  static const int rate_tab_q10[] = {
769 770 771 772 773 774 775 776 777
    65536, 6086, 5574, 5275, 5063, 4899, 4764, 4651, 4553, 4389, 4255, 4142,
    4044,  3958, 3881, 3811, 3748, 3635, 3538, 3453, 3376, 3307, 3244, 3186,
    3133,  3037, 2952, 2877, 2809, 2747, 2690, 2638, 2589, 2501, 2423, 2353,
    2290,  2232, 2179, 2130, 2084, 2001, 1928, 1862, 1802, 1748, 1698, 1651,
    1608,  1530, 1460, 1398, 1342, 1290, 1243, 1199, 1159, 1086, 1021, 963,
    911,   864,  821,  781,  745,  680,  623,  574,  530,  490,  455,  424,
    395,   345,  304,  269,  239,  213,  190,  171,  154,  126,  104,  87,
    73,    61,   52,   44,   38,   28,   21,   16,   12,   10,   8,    6,
    5,     3,    2,    1,    1,    1,    0,    0,
Jingning Han's avatar
Jingning Han committed
778 779 780 781 782 783 784 785 786
  };
  // Normalized distortion:
  // This table models the normalized distortion for a Laplacian source
  // with given variance when quantized with a uniform quantizer
  // with given stepsize. The closed form expression is:
  // Dn(x) = 1 - 1/sqrt(2) * x / sinh(x/sqrt(2))
  // where x = qpstep / sqrt(variance).
  // Note the actual distortion is Dn * variance.
  static const int dist_tab_q10[] = {
787 788 789 790 791 792 793 794 795
    0,    0,    1,    1,    1,    2,    2,    2,    3,    3,    4,    5,
    5,    6,    7,    7,    8,    9,    11,   12,   13,   15,   16,   17,
    18,   21,   24,   26,   29,   31,   34,   36,   39,   44,   49,   54,
    59,   64,   69,   73,   78,   88,   97,   106,  115,  124,  133,  142,
    151,  167,  184,  200,  215,  231,  245,  260,  274,  301,  327,  351,
    375,  397,  418,  439,  458,  495,  528,  559,  587,  613,  637,  659,
    680,  717,  749,  777,  801,  823,  842,  859,  874,  899,  919,  936,
    949,  960,  969,  977,  983,  994,  1001, 1006, 1010, 1013, 1015, 1017,
    1018, 1020, 1022, 1022, 1023, 1023, 1023, 1024,
Jingning Han's avatar
Jingning Han committed
796 797
  };
  static const int xsq_iq_q10[] = {
798 799 800 801 802 803 804 805 806 807 808 809
    0,      4,      8,      12,     16,     20,     24,     28,     32,
    40,     48,     56,     64,     72,     80,     88,     96,     112,
    128,    144,    160,    176,    192,    208,    224,    256,    288,
    320,    352,    384,    416,    448,    480,    544,    608,    672,
    736,    800,    864,    928,    992,    1120,   1248,   1376,   1504,
    1632,   1760,   1888,   2016,   2272,   2528,   2784,   3040,   3296,
    3552,   3808,   4064,   4576,   5088,   5600,   6112,   6624,   7136,
    7648,   8160,   9184,   10208,  11232,  12256,  13280,  14304,  15328,
    16352,  18400,  20448,  22496,  24544,  26592,  28640,  30688,  32736,
    36832,  40928,  45024,  49120,  53216,  57312,  61408,  65504,  73696,
    81888,  90080,  98272,  106464, 114656, 122848, 131040, 147424, 163808,
    180192, 196576, 212960, 229344, 245728,
Jingning Han's avatar
Jingning Han committed
810 811 812 813 814 815 816 817 818 819 820
  };
  const int tmp = (xsq_q10 >> 2) + 8;
  const int k = get_msb(tmp) - 3;
  const int xq = (k << 3) + ((tmp >> k) & 0x7);
  const int one_q10 = 1 << 10;
  const int a_q10 = ((xsq_q10 - xsq_iq_q10[xq]) << 10) >> (2 + k);
  const int b_q10 = one_q10 - a_q10;
  *r_q10 = (rate_tab_q10[xq] * b_q10 + rate_tab_q10[xq + 1] * a_q10) >> 10;
  *d_q10 = (dist_tab_q10[xq] * b_q10 + dist_tab_q10[xq + 1] * a_q10) >> 10;
}

Yaowu Xu's avatar
Yaowu Xu committed
821 822 823
void av1_model_rd_from_var_lapndz(int64_t var, unsigned int n_log2,
                                  unsigned int qstep, int *rate,
                                  int64_t *dist) {
Jingning Han's avatar
Jingning Han committed
824 825 826 827 828 829 830 831 832 833 834 835 836 837
  // This function models the rate and distortion for a Laplacian
  // source with given variance when quantized with a uniform quantizer
  // with given stepsize. The closed form expressions are in:
  // Hang and Chen, "Source Model for transform video coder and its
  // application - Part I: Fundamental Theory", IEEE Trans. Circ.
  // Sys. for Video Tech., April 1997.
  if (var == 0) {
    *rate = 0;
    *dist = 0;
  } else {
    int d_q10, r_q10;
    static const uint32_t MAX_XSQ_Q10 = 245727;
    const uint64_t xsq_q10_64 =
        (((uint64_t)qstep * qstep << (n_log2 + 10)) + (var >> 1)) / var;
Yaowu Xu's avatar
Yaowu Xu committed
838
    const int xsq_q10 = (int)AOMMIN(xsq_q10_64, MAX_XSQ_Q10);
Jingning Han's avatar
Jingning Han committed
839
    model_rd_norm(xsq_q10, &r_q10, &d_q10);
Yaowu Xu's avatar
Yaowu Xu committed
840
    *rate = ROUND_POWER_OF_TWO(r_q10 << n_log2, 10 - AV1_PROB_COST_SHIFT);
Jingning Han's avatar
Jingning Han committed
841 842 843 844
    *dist = (var * (int64_t)d_q10 + 512) >> 10;
  }
}

845 846 847 848
static void get_entropy_contexts_plane(
    BLOCK_SIZE plane_bsize, TX_SIZE tx_size, const struct macroblockd_plane *pd,
    ENTROPY_CONTEXT t_above[2 * MAX_MIB_SIZE],
    ENTROPY_CONTEXT t_left[2 * MAX_MIB_SIZE]) {
849 850
  const int num_4x4_w = block_size_wide[plane_bsize] >> tx_size_wide_log2[0];
  const int num_4x4_h = block_size_high[plane_bsize] >> tx_size_high_log2[0];
Jingning Han's avatar
Jingning Han committed
851 852 853
  const ENTROPY_CONTEXT *const above = pd->above_context;
  const ENTROPY_CONTEXT *const left = pd->left_context;

854 855 856 857 858 859
#if CONFIG_LV_MAP
  memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
  memcpy(t_left, left, sizeof(ENTROPY_CONTEXT) * num_4x4_h);
  return;
#endif  // CONFIG_LV_MAP

Jingning Han's avatar
Jingning Han committed
860
  int i;
861 862

  switch (tx_size) {
Jingning Han's avatar
Jingning Han committed
863 864 865 866
    case TX_4X4:
      memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
      memcpy(t_left, left, sizeof(ENTROPY_CONTEXT) * num_4x4_h);
      break;
867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884
    case TX_8X8:
      for (i = 0; i < num_4x4_w; i += 2)
        t_above[i] = !!*(const uint16_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 2)
        t_left[i] = !!*(const uint16_t *)&left[i];
      break;
    case TX_16X16:
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
    case TX_32X32:
      for (i = 0; i < num_4x4_w; i += 8)
        t_above[i] = !!*(const uint64_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 8)
        t_left[i] = !!*(const uint64_t *)&left[i];
      break;
885
#if CONFIG_TX64X64
886 887 888 889 890 891 892 893 894 895 896 897 898 899
    case TX_32X64:
      for (i = 0; i < num_4x4_w; i += 8)
        t_above[i] = !!*(const uint64_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 16)
        t_left[i] =
            !!(*(const uint64_t *)&left[i] | *(const uint64_t *)&left[i + 8]);
      break;
    case TX_64X32:
      for (i = 0; i < num_4x4_w; i += 16)
        t_above[i] =
            !!(*(const uint64_t *)&above[i] | *(const uint64_t *)&above[i + 8]);
      for (i = 0; i < num_4x4_h; i += 8)
        t_left[i] = !!*(const uint64_t *)&left[i];
      break;
900 901 902 903 904
    case TX_64X64:
      for (i = 0; i < num_4x4_w; i += 16)
        t_above[i] =
            !!(*(const uint64_t *)&above[i] | *(const uint64_t *)&above[i + 8]);
      for (i = 0; i < num_4x4_h; i += 16)
905 906
        t_left[i] =
            !!(*(const uint64_t *)&left[i] | *(const uint64_t *)&left[i + 8]);
907 908
      break;
#endif  // CONFIG_TX64X64
909 910 911 912 913 914 915 916 917 918
    case TX_4X8:
      memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
      for (i = 0; i < num_4x4_h; i += 2)
        t_left[i] = !!*(const uint16_t *)&left[i];
      break;
    case TX_8X4:
      for (i = 0; i < num_4x4_w; i += 2)
        t_above[i] = !!*(const uint16_t *)&above[i];
      memcpy(t_left, left, sizeof(ENTROPY_CONTEXT) * num_4x4_h);
      break;
919
    case TX_8X16:
Jingning Han's avatar
Jingning Han committed
920 921
      for (i = 0; i < num_4x4_w; i += 2)
        t_above[i] = !!*(const uint16_t *)&above[i];
922 923 924 925 926 927
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
    case TX_16X8:
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
Jingning Han's avatar
Jingning Han committed
928 929 930
      for (i = 0; i < num_4x4_h; i += 2)
        t_left[i] = !!*(const uint16_t *)&left[i];
      break;
931
    case TX_16X32:
Jingning Han's avatar
Jingning Han committed
932 933
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
934 935
      for (i = 0; i < num_4x4_h; i += 8)
        t_left[i] = !!*(const uint64_t *)&left[i];
Jingning Han's avatar
Jingning Han committed
936
      break;
937
    case TX_32X16:
Jingning Han's avatar
Jingning Han committed
938 939
      for (i = 0; i < num_4x4_w; i += 8)
        t_above[i] = !!*(const uint64_t *)&above[i];
940 941 942
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
943
#if CONFIG_RECT_TX_EXT
944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965
    case TX_4X16:
      memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
    case TX_16X4:
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
      memcpy(t_left, left, sizeof(ENTROPY_CONTEXT) * num_4x4_h);
      break;
    case TX_8X32:
      for (i = 0; i < num_4x4_w; i += 2)
        t_above[i] = !!*(const uint16_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 8)
        t_left[i] = !!*(const uint64_t *)&left[i];
      break;
    case TX_32X8:
      for (i = 0; i < num_4x4_w; i += 8)
        t_above[i] = !!*(const uint64_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 2)
        t_left[i] = !!*(const uint16_t *)&left[i];
      break;
Yue Chen's avatar
Yue Chen committed
966
#endif
clang-format's avatar
clang-format committed
967
    default: assert(0 && "Invalid transform size."); break;
Jingning Han's avatar
Jingning Han committed
968 969 970
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
971 972 973 974
void av1_get_entropy_contexts(BLOCK_SIZE bsize, TX_SIZE tx_size,
                              const struct macroblockd_plane *pd,
                              ENTROPY_CONTEXT t_above[2 * MAX_MIB_SIZE],
                              ENTROPY_CONTEXT t_left[2 * MAX_MIB_SIZE]) {
975 976
  const BLOCK_SIZE plane_bsize =
      AOMMAX(BLOCK_4X4, get_plane_block_size(bsize, pd));
977
  get_entropy_contexts_plane(plane_bsize, tx_size, pd, t_above, t_left);
978 979
}

980
void av1_mv_pred(const AV1_COMP *cpi, MACROBLOCK *x, uint8_t *ref_y_buffer,