rd.c 58.4 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4
5
6
7
8
9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10
11
12
13
14
15
 */

#include <assert.h>
#include <math.h>
#include <stdio.h>

Yaowu Xu's avatar
Yaowu Xu committed
16
#include "./av1_rtcd.h"
Jingning Han's avatar
Jingning Han committed
17

Yaowu Xu's avatar
Yaowu Xu committed
18
19
#include "aom_dsp/aom_dsp_common.h"
#include "aom_mem/aom_mem.h"
20
21
22
#include "aom_ports/bitops.h"
#include "aom_ports/mem.h"
#include "aom_ports/system_state.h"
Jingning Han's avatar
Jingning Han committed
23

24
25
26
27
28
29
30
31
32
#include "av1/common/common.h"
#include "av1/common/entropy.h"
#include "av1/common/entropymode.h"
#include "av1/common/mvref_common.h"
#include "av1/common/pred_common.h"
#include "av1/common/quant_common.h"
#include "av1/common/reconinter.h"
#include "av1/common/reconintra.h"
#include "av1/common/seg_common.h"
33

34
#include "av1/encoder/av1_quantize.h"
35
36
37
38
39
40
41
42
#include "av1/encoder/cost.h"
#include "av1/encoder/encodemb.h"
#include "av1/encoder/encodemv.h"
#include "av1/encoder/encoder.h"
#include "av1/encoder/mcomp.h"
#include "av1/encoder/ratectrl.h"
#include "av1/encoder/rd.h"
#include "av1/encoder/tokenize.h"
Jingning Han's avatar
Jingning Han committed
43

44
#define RD_THRESH_POW 1.25
Jingning Han's avatar
Jingning Han committed
45
46
47
48
49
50
51
52

// Factor to weigh the rate for switchable interp filters.
#define SWITCHABLE_INTERP_RATE_FACTOR 1

// The baseline rd thresholds for breaking out of the rd loop for
// certain modes are assumed to be based on 8x8 blocks.
// This table is used to correct for block size.
// The factors here are << 2 (2 = x0.5, 32 = x8 etc).
53
static const uint8_t rd_thresh_block_size_factor[BLOCK_SIZES_ALL] = {
54
#if CONFIG_CHROMA_2X2 || CONFIG_CHROMA_SUB8X8
Jingning Han's avatar
Jingning Han committed
55
56
  2,  2,  2,
#endif
57
  2,  3,  3,  4, 6,  6, 8, 12, 12, 16, 24, 24, 32,
58
#if CONFIG_EXT_PARTITION
59
  48, 48, 64,
60
#endif  // CONFIG_EXT_PARTITION
61
  4,  4,  8,  8, 16, 16
Jingning Han's avatar
Jingning Han committed
62
63
};

64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
void av1_fill_nmv_costs(AV1_COMMON *const cm, MACROBLOCK *x,
                        FRAME_CONTEXT *fc) {
  for (int i = 0; i < NMV_CONTEXTS; ++i) {
    memset(x->nmvcost_array, 0, sizeof(x->nmvcost_array));
    memset(x->nmvcost_hp_array, 0, sizeof(x->nmvcost_hp_array));
    x->nmvcost[i][0] = &x->nmvcost_array[i][0][MV_MAX];
    x->nmvcost[i][1] = &x->nmvcost_array[i][1][MV_MAX];
    x->nmvcost_hp[i][0] = &x->nmvcost_hp_array[i][0][MV_MAX];
    x->nmvcost_hp[i][1] = &x->nmvcost_hp_array[i][1][MV_MAX];
    if (cm->allow_high_precision_mv)
      x->mv_cost_stack[i] = x->nmvcost_hp[i];
    else
      x->mv_cost_stack[i] = x->nmvcost[i];

    av1_build_nmv_cost_table(
        x->nmv_vec_cost[i],
        cm->allow_high_precision_mv ? x->nmvcost_hp[i] : x->nmvcost[i],
        &fc->nmvc[i], cm->allow_high_precision_mv);
  }
  x->mvcost = x->mv_cost_stack[0];
  x->nmvjointcost = x->nmv_vec_cost[0];
}

87
88
void av1_fill_mode_rates(AV1_COMMON *const cm, MACROBLOCK *x,
                         FRAME_CONTEXT *fc) {
Jingning Han's avatar
Jingning Han committed
89
90
  int i, j;

91
92
  if (cm->frame_type == KEY_FRAME) {
    for (i = 0; i < PARTITION_CONTEXTS_PRIMARY; ++i)
Yue Chen's avatar
Yue Chen committed
93
94
      av1_cost_tokens_from_cdf(x->partition_cost[i], fc->partition_cdf[i],
                               NULL);
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
#if CONFIG_UNPOISON_PARTITION_CTX
    for (; i < PARTITION_CONTEXTS_PRIMARY + PARTITION_BLOCK_SIZES; ++i) {
      aom_prob p = fc->partition_prob[i][PARTITION_VERT];
      assert(p > 0);
      x->partition_cost[i][PARTITION_NONE] = INT_MAX;
      x->partition_cost[i][PARTITION_HORZ] = INT_MAX;
      x->partition_cost[i][PARTITION_VERT] = av1_cost_bit(p, 0);
      x->partition_cost[i][PARTITION_SPLIT] = av1_cost_bit(p, 1);
    }
    for (; i < PARTITION_CONTEXTS_PRIMARY + 2 * PARTITION_BLOCK_SIZES; ++i) {
      aom_prob p = fc->partition_prob[i][PARTITION_HORZ];
      assert(p > 0);
      x->partition_cost[i][PARTITION_NONE] = INT_MAX;
      x->partition_cost[i][PARTITION_HORZ] = av1_cost_bit(p, 0);
      x->partition_cost[i][PARTITION_VERT] = INT_MAX;
      x->partition_cost[i][PARTITION_SPLIT] = av1_cost_bit(p, 1);
    }
    x->partition_cost[PARTITION_CONTEXTS][PARTITION_NONE] = INT_MAX;
    x->partition_cost[PARTITION_CONTEXTS][PARTITION_HORZ] = INT_MAX;
    x->partition_cost[PARTITION_CONTEXTS][PARTITION_VERT] = INT_MAX;
    x->partition_cost[PARTITION_CONTEXTS][PARTITION_SPLIT] = 0;
#endif  // CONFIG_UNPOISON_PARTITION_CTX
  }

Jingning Han's avatar
Jingning Han committed
119
120
  for (i = 0; i < INTRA_MODES; ++i)
    for (j = 0; j < INTRA_MODES; ++j)
121
      av1_cost_tokens_from_cdf(x->y_mode_costs[i][j], av1_kf_y_mode_cdf[i][j],
122
                               av1_intra_mode_inv);
Jingning Han's avatar
Jingning Han committed
123

124
  for (i = 0; i < BLOCK_SIZE_GROUPS; ++i)
125
    av1_cost_tokens_from_cdf(x->mbmode_cost[i], fc->y_mode_cdf[i],
126
                             av1_intra_mode_inv);
127
128
129
130
131
132
133
  const int *uv_mode_inv_map =
#if CONFIG_CFL
      // CfL codes the uv_mode without reordering it
      NULL;
#else
      av1_intra_mode_inv;
#endif
134
  for (i = 0; i < INTRA_MODES; ++i)
135
    av1_cost_tokens_from_cdf(x->intra_uv_mode_cost[i], fc->uv_mode_cdf[i],
136
                             uv_mode_inv_map);
Jingning Han's avatar
Jingning Han committed
137
138

  for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; ++i)
Yue Chen's avatar
Yue Chen committed
139
140
141
    av1_cost_tokens_from_cdf(x->switchable_interp_costs[i],
                             fc->switchable_interp_cdf[i],
                             av1_switchable_interp_inv);
hui su's avatar
hui su committed
142
143

  for (i = 0; i < PALETTE_BLOCK_SIZES; ++i) {
144
    av1_cost_tokens_from_cdf(x->palette_y_size_cost[i],
145
                             fc->palette_y_size_cdf[i], NULL);
146
    av1_cost_tokens_from_cdf(x->palette_uv_size_cost[i],
147
                             fc->palette_uv_size_cdf[i], NULL);
hui su's avatar
hui su committed
148
149
  }

150
  for (i = 0; i < PALETTE_SIZES; ++i) {
151
    for (j = 0; j < PALETTE_COLOR_INDEX_CONTEXTS; ++j) {
152
      av1_cost_tokens_from_cdf(x->palette_y_color_cost[i][j],
153
                               fc->palette_y_color_index_cdf[i][j], NULL);
154
      av1_cost_tokens_from_cdf(x->palette_uv_color_cost[i][j],
155
                               fc->palette_uv_color_index_cdf[i][j], NULL);
hui su's avatar
hui su committed
156
    }
157
  }
158

159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
#if CONFIG_CFL
  int sign_cost[CFL_JOINT_SIGNS];
  av1_cost_tokens_from_cdf(sign_cost, fc->cfl_sign_cdf, NULL);
  for (int joint_sign = 0; joint_sign < CFL_JOINT_SIGNS; joint_sign++) {
    const aom_cdf_prob *cdf_u = fc->cfl_alpha_cdf[CFL_CONTEXT_U(joint_sign)];
    const aom_cdf_prob *cdf_v = fc->cfl_alpha_cdf[CFL_CONTEXT_V(joint_sign)];
    int *cost_u = x->cfl_cost[joint_sign][CFL_PRED_U];
    int *cost_v = x->cfl_cost[joint_sign][CFL_PRED_V];
    if (CFL_SIGN_U(joint_sign) == CFL_SIGN_ZERO)
      memset(cost_u, 0, CFL_ALPHABET_SIZE * sizeof(*cost_u));
    else
      av1_cost_tokens_from_cdf(cost_u, cdf_u, NULL);
    if (CFL_SIGN_V(joint_sign) == CFL_SIGN_ZERO)
      memset(cost_v, 0, CFL_ALPHABET_SIZE * sizeof(*cost_v));
    else
      av1_cost_tokens_from_cdf(cost_v, cdf_v, NULL);
    for (int u = 0; u < CFL_ALPHABET_SIZE; u++)
      cost_u[u] += sign_cost[joint_sign];
  }
#endif  // CONFIG_CFL

Jingning Han's avatar
Jingning Han committed
180
  for (i = 0; i < MAX_TX_DEPTH; ++i)
181
    for (j = 0; j < TX_SIZE_CONTEXTS; ++j)
Yue Chen's avatar
Yue Chen committed
182
183
      av1_cost_tokens_from_cdf(x->tx_size_cost[i][j], fc->tx_size_cdf[i][j],
                               NULL);
184

185
#if CONFIG_EXT_TX
186
187
188
  for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
    int s;
    for (s = 1; s < EXT_TX_SETS_INTER; ++s) {
189
      if (use_inter_ext_tx_for_txsize[s][i]) {
Yue Chen's avatar
Yue Chen committed
190
191
192
        av1_cost_tokens_from_cdf(x->inter_tx_type_costs[s][i],
                                 fc->inter_ext_tx_cdf[s][i],
                                 av1_ext_tx_inter_inv[s]);
193
194
195
      }
    }
    for (s = 1; s < EXT_TX_SETS_INTRA; ++s) {
196
      if (use_intra_ext_tx_for_txsize[s][i]) {
197
        for (j = 0; j < INTRA_MODES; ++j)
Yue Chen's avatar
Yue Chen committed
198
199
200
          av1_cost_tokens_from_cdf(x->intra_tx_type_costs[s][i][j],
                                   fc->intra_ext_tx_cdf[s][i][j],
                                   av1_ext_tx_intra_inv[s]);
201
202
      }
    }
203
  }
204
#else
205
206
  for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
    for (j = 0; j < TX_TYPES; ++j)
Yue Chen's avatar
Yue Chen committed
207
208
      av1_cost_tokens_from_cdf(x->intra_tx_type_costs[i][j],
                               fc->intra_ext_tx_cdf[i][j], av1_ext_tx_inv);
209
210
  }
  for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
Yue Chen's avatar
Yue Chen committed
211
212
    av1_cost_tokens_from_cdf(x->inter_tx_type_costs[i], fc->inter_ext_tx_cdf[i],
                             av1_ext_tx_inv);
213
  }
214
#endif  // CONFIG_EXT_TX
215
#if CONFIG_EXT_INTRA
hui su's avatar
hui su committed
216
#if CONFIG_INTRA_INTERP
217
  for (i = 0; i < INTRA_FILTERS + 1; ++i)
Yue Chen's avatar
Yue Chen committed
218
219
    av1_cost_tokens_from_cdf(x->intra_filter_cost[i], fc->intra_filter_cdf[i],
                             NULL);
hui su's avatar
hui su committed
220
#endif  // CONFIG_INTRA_INTERP
221
#endif  // CONFIG_EXT_INTRA
222
#if CONFIG_LOOP_RESTORATION
223
  av1_cost_tokens(x->switchable_restore_cost, fc->switchable_restore_prob,
224
225
                  av1_switchable_restore_tree);
#endif  // CONFIG_LOOP_RESTORATION
226
227
228
#if CONFIG_INTRABC
  av1_fill_nmv_costs(cm, x, fc);
#endif
229
230

  if (!frame_is_intra_only(cm)) {
231
232
233
234
#if !CONFIG_INTRABC
    av1_fill_nmv_costs(cm, x, fc);
#endif

235
    for (i = 0; i < NEWMV_MODE_CONTEXTS; ++i) {
Yue Chen's avatar
Yue Chen committed
236
237
238
#if CONFIG_NEW_MULTISYMBOL
      av1_cost_tokens_from_cdf(x->newmv_mode_cost[i], fc->newmv_cdf[i], NULL);
#else
239
240
      x->newmv_mode_cost[i][0] = av1_cost_bit(fc->newmv_prob[i], 0);
      x->newmv_mode_cost[i][1] = av1_cost_bit(fc->newmv_prob[i], 1);
Yue Chen's avatar
Yue Chen committed
241
#endif
242
243
244
    }

    for (i = 0; i < ZEROMV_MODE_CONTEXTS; ++i) {
Yue Chen's avatar
Yue Chen committed
245
246
247
#if CONFIG_NEW_MULTISYMBOL
      av1_cost_tokens_from_cdf(x->zeromv_mode_cost[i], fc->zeromv_cdf[i], NULL);
#else
248
249
      x->zeromv_mode_cost[i][0] = av1_cost_bit(fc->zeromv_prob[i], 0);
      x->zeromv_mode_cost[i][1] = av1_cost_bit(fc->zeromv_prob[i], 1);
Yue Chen's avatar
Yue Chen committed
250
#endif
251
252
253
    }

    for (i = 0; i < REFMV_MODE_CONTEXTS; ++i) {
Yue Chen's avatar
Yue Chen committed
254
255
256
#if CONFIG_NEW_MULTISYMBOL
      av1_cost_tokens_from_cdf(x->refmv_mode_cost[i], fc->refmv_cdf[i], NULL);
#else
257
258
      x->refmv_mode_cost[i][0] = av1_cost_bit(fc->refmv_prob[i], 0);
      x->refmv_mode_cost[i][1] = av1_cost_bit(fc->refmv_prob[i], 1);
Yue Chen's avatar
Yue Chen committed
259
#endif
260
261
262
    }

    for (i = 0; i < DRL_MODE_CONTEXTS; ++i) {
Yue Chen's avatar
Yue Chen committed
263
264
265
#if CONFIG_NEW_MULTISYMBOL
      av1_cost_tokens_from_cdf(x->drl_mode_cost0[i], fc->drl_cdf[i], NULL);
#else
266
267
      x->drl_mode_cost0[i][0] = av1_cost_bit(fc->drl_prob[i], 0);
      x->drl_mode_cost0[i][1] = av1_cost_bit(fc->drl_prob[i], 1);
Yue Chen's avatar
Yue Chen committed
268
#endif
269
270
271
    }
#if CONFIG_EXT_INTER
    for (i = 0; i < INTER_MODE_CONTEXTS; ++i)
Yue Chen's avatar
Yue Chen committed
272
273
      av1_cost_tokens_from_cdf(x->inter_compound_mode_cost[i],
                               fc->inter_compound_mode_cdf[i], NULL);
274
275
276
    for (i = 0; i < BLOCK_SIZES_ALL; ++i)
      av1_cost_tokens_from_cdf(x->compound_type_cost[i],
                               fc->compound_type_cdf[i], NULL);
277
278
#if CONFIG_COMPOUND_SINGLEREF
    for (i = 0; i < INTER_MODE_CONTEXTS; ++i)
Yue Chen's avatar
Yue Chen committed
279
280
      av1_cost_tokens_from_cdf(x->inter_singleref_comp_mode_cost[i],
                               fc->inter_singleref_comp_mode_cdf[i], NULL);
281
282
283
#endif  // CONFIG_COMPOUND_SINGLEREF
#if CONFIG_INTERINTRA
    for (i = 0; i < BLOCK_SIZE_GROUPS; ++i)
Yue Chen's avatar
Yue Chen committed
284
285
      av1_cost_tokens_from_cdf(x->interintra_mode_cost[i],
                               fc->interintra_mode_cdf[i], NULL);
286
287
288
289
#endif  // CONFIG_INTERINTRA
#endif  // CONFIG_EXT_INTER
#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
    for (i = BLOCK_8X8; i < BLOCK_SIZES_ALL; i++) {
290
291
      av1_cost_tokens_from_cdf(x->motion_mode_cost[i], fc->motion_mode_cdf[i],
                               NULL);
292
293
294
    }
#if CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION
    for (i = BLOCK_8X8; i < BLOCK_SIZES_ALL; i++) {
Yue Chen's avatar
Yue Chen committed
295
296
297
#if CONFIG_NEW_MULTISYMBOL
      av1_cost_tokens_from_cdf(x->motion_mode_cost1[i], fc->obmc_cdf[i], NULL);
#else
298
299
      x->motion_mode_cost1[i][0] = av1_cost_bit(fc->obmc_prob[i], 0);
      x->motion_mode_cost1[i][1] = av1_cost_bit(fc->obmc_prob[i], 1);
Yue Chen's avatar
Yue Chen committed
300
#endif
301
302
303
304
    }
#endif  // CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION
#if CONFIG_MOTION_VAR && CONFIG_NCOBMC_ADAPT_WEIGHT
    for (i = ADAPT_OVERLAP_BLOCK_8X8; i < ADAPT_OVERLAP_BLOCKS; ++i) {
305
306
      av1_cost_tokens_from_cdf(x->ncobmc_mode_cost[i], fc->ncobmc_mode_cdf[i],
                               NULL);
307
308
309
310
    }
#endif  // CONFIG_MOTION_VAR && CONFIG_NCOBMC_ADAPT_WEIGHT
#endif  // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
  }
Jingning Han's avatar
Jingning Han committed
311
312
313
314
315
316
}

// Values are now correlated to quantizer.
static int sad_per_bit16lut_8[QINDEX_RANGE];
static int sad_per_bit4lut_8[QINDEX_RANGE];

317
#if CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
318
319
320
321
322
323
324
static int sad_per_bit16lut_10[QINDEX_RANGE];
static int sad_per_bit4lut_10[QINDEX_RANGE];
static int sad_per_bit16lut_12[QINDEX_RANGE];
static int sad_per_bit4lut_12[QINDEX_RANGE];
#endif

static void init_me_luts_bd(int *bit16lut, int *bit4lut, int range,
Yaowu Xu's avatar
Yaowu Xu committed
325
                            aom_bit_depth_t bit_depth) {
Jingning Han's avatar
Jingning Han committed
326
327
328
329
330
  int i;
  // Initialize the sad lut tables using a formulaic calculation for now.
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < range; i++) {
Yaowu Xu's avatar
Yaowu Xu committed
331
    const double q = av1_convert_qindex_to_q(i, bit_depth);
Jingning Han's avatar
Jingning Han committed
332
333
334
335
336
    bit16lut[i] = (int)(0.0418 * q + 2.4107);
    bit4lut[i] = (int)(0.063 * q + 2.742);
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
337
void av1_init_me_luts(void) {
Jingning Han's avatar
Jingning Han committed
338
  init_me_luts_bd(sad_per_bit16lut_8, sad_per_bit4lut_8, QINDEX_RANGE,
Yaowu Xu's avatar
Yaowu Xu committed
339
                  AOM_BITS_8);
340
#if CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
341
  init_me_luts_bd(sad_per_bit16lut_10, sad_per_bit4lut_10, QINDEX_RANGE,
Yaowu Xu's avatar
Yaowu Xu committed
342
                  AOM_BITS_10);
Jingning Han's avatar
Jingning Han committed
343
  init_me_luts_bd(sad_per_bit16lut_12, sad_per_bit4lut_12, QINDEX_RANGE,
Yaowu Xu's avatar
Yaowu Xu committed
344
                  AOM_BITS_12);
Jingning Han's avatar
Jingning Han committed
345
346
347
#endif
}

348
349
static const int rd_boost_factor[16] = { 64, 32, 32, 32, 24, 16, 12, 12,
                                         8,  8,  4,  4,  2,  2,  1,  0 };
Jingning Han's avatar
Jingning Han committed
350
static const int rd_frame_type_factor[FRAME_UPDATE_TYPES] = {
351
  128, 144, 128, 128, 144,
352
#if CONFIG_EXT_REFS
353
354
  // TODO(zoeliu): To adjust further following factor values.
  128, 128, 128
355
356
  // TODO(weitinglin): We should investigate if the values should be the same
  //                   as the value used by OVERLAY frame
Yaowu Xu's avatar
Yaowu Xu committed
357
  ,
Zoe Liu's avatar
Zoe Liu committed
358
359
360
361
362
  144  // INTNL_OVERLAY_UPDATE
#if CONFIG_ALTREF2
  ,
  128   // INTNL_ARF_UPDATE
#endif  // CONFIG_ALTREF2
363
#endif  // CONFIG_EXT_REFS
Jingning Han's avatar
Jingning Han committed
364
365
};

Yaowu Xu's avatar
Yaowu Xu committed
366
367
int av1_compute_rd_mult(const AV1_COMP *cpi, int qindex) {
  const int64_t q = av1_dc_quant(qindex, 0, cpi->common.bit_depth);
368
#if CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
369
370
  int64_t rdmult = 0;
  switch (cpi->common.bit_depth) {
Yaowu Xu's avatar
Yaowu Xu committed
371
372
373
    case AOM_BITS_8: rdmult = 88 * q * q / 24; break;
    case AOM_BITS_10: rdmult = ROUND_POWER_OF_TWO(88 * q * q / 24, 4); break;
    case AOM_BITS_12: rdmult = ROUND_POWER_OF_TWO(88 * q * q / 24, 8); break;
Jingning Han's avatar
Jingning Han committed
374
    default:
Yaowu Xu's avatar
Yaowu Xu committed
375
      assert(0 && "bit_depth should be AOM_BITS_8, AOM_BITS_10 or AOM_BITS_12");
Jingning Han's avatar
Jingning Han committed
376
377
378
379
      return -1;
  }
#else
  int64_t rdmult = 88 * q * q / 24;
380
#endif  // CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
381
382
383
  if (cpi->oxcf.pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    const GF_GROUP *const gf_group = &cpi->twopass.gf_group;
    const FRAME_UPDATE_TYPE frame_type = gf_group->update_type[gf_group->index];
Yaowu Xu's avatar
Yaowu Xu committed
384
    const int boost_index = AOMMIN(15, (cpi->rc.gfu_boost / 100));
Jingning Han's avatar
Jingning Han committed
385
386
387
388

    rdmult = (rdmult * rd_frame_type_factor[frame_type]) >> 7;
    rdmult += ((rdmult * rd_boost_factor[boost_index]) >> 7);
  }
389
  if (rdmult < 1) rdmult = 1;
Jingning Han's avatar
Jingning Han committed
390
391
392
  return (int)rdmult;
}

Yaowu Xu's avatar
Yaowu Xu committed
393
static int compute_rd_thresh_factor(int qindex, aom_bit_depth_t bit_depth) {
Jingning Han's avatar
Jingning Han committed
394
  double q;
395
#if CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
396
  switch (bit_depth) {
Yaowu Xu's avatar
Yaowu Xu committed
397
398
399
    case AOM_BITS_8: q = av1_dc_quant(qindex, 0, AOM_BITS_8) / 4.0; break;
    case AOM_BITS_10: q = av1_dc_quant(qindex, 0, AOM_BITS_10) / 16.0; break;
    case AOM_BITS_12: q = av1_dc_quant(qindex, 0, AOM_BITS_12) / 64.0; break;
Jingning Han's avatar
Jingning Han committed
400
    default:
Yaowu Xu's avatar
Yaowu Xu committed
401
      assert(0 && "bit_depth should be AOM_BITS_8, AOM_BITS_10 or AOM_BITS_12");
Jingning Han's avatar
Jingning Han committed
402
403
404
      return -1;
  }
#else
405
  (void)bit_depth;
Yaowu Xu's avatar
Yaowu Xu committed
406
  q = av1_dc_quant(qindex, 0, AOM_BITS_8) / 4.0;
407
#endif  // CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
408
  // TODO(debargha): Adjust the function below.
Yaowu Xu's avatar
Yaowu Xu committed
409
  return AOMMAX((int)(pow(q, RD_THRESH_POW) * 5.12), 8);
Jingning Han's avatar
Jingning Han committed
410
411
}

Yaowu Xu's avatar
Yaowu Xu committed
412
void av1_initialize_me_consts(const AV1_COMP *cpi, MACROBLOCK *x, int qindex) {
413
#if CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
414
  switch (cpi->common.bit_depth) {
Yaowu Xu's avatar
Yaowu Xu committed
415
    case AOM_BITS_8:
Jingning Han's avatar
Jingning Han committed
416
417
418
      x->sadperbit16 = sad_per_bit16lut_8[qindex];
      x->sadperbit4 = sad_per_bit4lut_8[qindex];
      break;
Yaowu Xu's avatar
Yaowu Xu committed
419
    case AOM_BITS_10:
Jingning Han's avatar
Jingning Han committed
420
421
422
      x->sadperbit16 = sad_per_bit16lut_10[qindex];
      x->sadperbit4 = sad_per_bit4lut_10[qindex];
      break;
Yaowu Xu's avatar
Yaowu Xu committed
423
    case AOM_BITS_12:
Jingning Han's avatar
Jingning Han committed
424
425
426
427
      x->sadperbit16 = sad_per_bit16lut_12[qindex];
      x->sadperbit4 = sad_per_bit4lut_12[qindex];
      break;
    default:
Yaowu Xu's avatar
Yaowu Xu committed
428
      assert(0 && "bit_depth should be AOM_BITS_8, AOM_BITS_10 or AOM_BITS_12");
Jingning Han's avatar
Jingning Han committed
429
430
431
432
433
  }
#else
  (void)cpi;
  x->sadperbit16 = sad_per_bit16lut_8[qindex];
  x->sadperbit4 = sad_per_bit4lut_8[qindex];
434
#endif  // CONFIG_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
435
436
}

Yaowu Xu's avatar
Yaowu Xu committed
437
static void set_block_thresholds(const AV1_COMMON *cm, RD_OPT *rd) {
Jingning Han's avatar
Jingning Han committed
438
439
440
441
  int i, bsize, segment_id;

  for (segment_id = 0; segment_id < MAX_SEGMENTS; ++segment_id) {
    const int qindex =
Yaowu Xu's avatar
Yaowu Xu committed
442
        clamp(av1_get_qindex(&cm->seg, segment_id, cm->base_qindex) +
443
444
                  cm->y_dc_delta_q,
              0, MAXQ);
Jingning Han's avatar
Jingning Han committed
445
446
    const int q = compute_rd_thresh_factor(qindex, cm->bit_depth);

447
    for (bsize = 0; bsize < BLOCK_SIZES_ALL; ++bsize) {
Jingning Han's avatar
Jingning Han committed
448
449
450
451
452
      // Threshold here seems unnecessarily harsh but fine given actual
      // range of values used for cpi->sf.thresh_mult[].
      const int t = q * rd_thresh_block_size_factor[bsize];
      const int thresh_max = INT_MAX / t;

453
454
455
456
457
458
#if CONFIG_CB4X4
      for (i = 0; i < MAX_MODES; ++i)
        rd->threshes[segment_id][bsize][i] = rd->thresh_mult[i] < thresh_max
                                                 ? rd->thresh_mult[i] * t / 4
                                                 : INT_MAX;
#else
Jingning Han's avatar
Jingning Han committed
459
460
      if (bsize >= BLOCK_8X8) {
        for (i = 0; i < MAX_MODES; ++i)
461
462
463
          rd->threshes[segment_id][bsize][i] = rd->thresh_mult[i] < thresh_max
                                                   ? rd->thresh_mult[i] * t / 4
                                                   : INT_MAX;
Jingning Han's avatar
Jingning Han committed
464
465
466
467
468
469
470
      } else {
        for (i = 0; i < MAX_REFS; ++i)
          rd->threshes[segment_id][bsize][i] =
              rd->thresh_mult_sub8x8[i] < thresh_max
                  ? rd->thresh_mult_sub8x8[i] * t / 4
                  : INT_MAX;
      }
471
#endif
Jingning Han's avatar
Jingning Han committed
472
473
474
475
    }
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
476
477
void av1_set_mvcost(MACROBLOCK *x, MV_REFERENCE_FRAME ref_frame, int ref,
                    int ref_mv_idx) {
478
  MB_MODE_INFO_EXT *mbmi_ext = x->mbmi_ext;
Yaowu Xu's avatar
Yaowu Xu committed
479
480
481
482
  int8_t rf_type = av1_ref_frame_type(x->e_mbd.mi[0]->mbmi.ref_frame);
  int nmv_ctx = av1_nmv_ctx(mbmi_ext->ref_mv_count[rf_type],
                            mbmi_ext->ref_mv_stack[rf_type], ref, ref_mv_idx);
  (void)ref_frame;
483
484
485
486
  x->mvcost = x->mv_cost_stack[nmv_ctx];
  x->nmvjointcost = x->nmv_vec_cost[nmv_ctx];
}

487
#if CONFIG_LV_MAP
488
#if !LV_MAP_PROB
489
490
491
492
static void get_rate_cost(aom_prob p, int cost[2]) {
  cost[0] = av1_cost_bit(p, 0);
  cost[1] = av1_cost_bit(p, 1);
}
493
#endif  // !LV_MAP_PROB
494
495
496
497
498
499

void av1_fill_coeff_costs(MACROBLOCK *x, FRAME_CONTEXT *fc) {
  for (TX_SIZE tx_size = 0; tx_size < TX_SIZES; ++tx_size) {
    for (int plane = 0; plane < PLANE_TYPES; ++plane) {
      LV_MAP_COEFF_COST *pcost = &x->coeff_costs[tx_size][plane];

500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
#if LV_MAP_PROB
      for (int ctx = 0; ctx < TXB_SKIP_CONTEXTS; ++ctx)
        av1_cost_tokens_from_cdf(pcost->txb_skip_cost[ctx],
                                 fc->txb_skip_cdf[tx_size][ctx], NULL);

      for (int ctx = 0; ctx < SIG_COEF_CONTEXTS; ++ctx)
        av1_cost_tokens_from_cdf(pcost->nz_map_cost[ctx],
                                 fc->nz_map_cdf[tx_size][plane][ctx], NULL);

      for (int ctx = 0; ctx < EOB_COEF_CONTEXTS; ++ctx)
        av1_cost_tokens_from_cdf(pcost->eob_cost[ctx],
                                 fc->eob_flag_cdf[tx_size][plane][ctx], NULL);

      for (int ctx = 0; ctx < DC_SIGN_CONTEXTS; ++ctx)
        av1_cost_tokens_from_cdf(pcost->dc_sign_cost[ctx],
                                 fc->dc_sign_cdf[plane][ctx], NULL);

      for (int layer = 0; layer < NUM_BASE_LEVELS; ++layer)
        for (int ctx = 0; ctx < COEFF_BASE_CONTEXTS; ++ctx)
          av1_cost_tokens_from_cdf(
              pcost->base_cost[layer][ctx],
              fc->coeff_base_cdf[tx_size][plane][layer][ctx], NULL);

      for (int ctx = 0; ctx < LEVEL_CONTEXTS; ++ctx)
        av1_cost_tokens_from_cdf(pcost->lps_cost[ctx],
                                 fc->coeff_lps_cdf[tx_size][plane][ctx], NULL);
#else
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
      for (int ctx = 0; ctx < TXB_SKIP_CONTEXTS; ++ctx)
        get_rate_cost(fc->txb_skip[tx_size][ctx], pcost->txb_skip_cost[ctx]);

      for (int ctx = 0; ctx < SIG_COEF_CONTEXTS; ++ctx)
        get_rate_cost(fc->nz_map[tx_size][plane][ctx], pcost->nz_map_cost[ctx]);

      for (int ctx = 0; ctx < EOB_COEF_CONTEXTS; ++ctx)
        get_rate_cost(fc->eob_flag[tx_size][plane][ctx], pcost->eob_cost[ctx]);

      for (int ctx = 0; ctx < DC_SIGN_CONTEXTS; ++ctx)
        get_rate_cost(fc->dc_sign[plane][ctx], pcost->dc_sign_cost[ctx]);

      for (int layer = 0; layer < NUM_BASE_LEVELS; ++layer)
        for (int ctx = 0; ctx < COEFF_BASE_CONTEXTS; ++ctx)
          get_rate_cost(fc->coeff_base[tx_size][plane][layer][ctx],
                        pcost->base_cost[layer][ctx]);

      for (int ctx = 0; ctx < LEVEL_CONTEXTS; ++ctx)
        get_rate_cost(fc->coeff_lps[tx_size][plane][ctx], pcost->lps_cost[ctx]);
546
#endif
547
548
549
550
551
    }
  }
}
#endif

552
553
void av1_fill_token_costs_from_cdf(av1_coeff_cost *cost,
                                   coeff_cdf_model (*cdf)[PLANE_TYPES]) {
hui su's avatar
hui su committed
554
555
556
557
558
559
560
561
562
563
564
565
566
567
  for (int tx = 0; tx < TX_SIZES; ++tx) {
    for (int pt = 0; pt < PLANE_TYPES; ++pt) {
      for (int rt = 0; rt < REF_TYPES; ++rt) {
        for (int band = 0; band < COEF_BANDS; ++band) {
          for (int ctx = 0; ctx < BAND_COEFF_CONTEXTS(band); ++ctx) {
            av1_cost_tokens_from_cdf(cost[tx][pt][rt][band][ctx],
                                     cdf[tx][pt][rt][band][ctx], NULL);
          }
        }
      }
    }
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
568
569
void av1_initialize_rd_consts(AV1_COMP *cpi) {
  AV1_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
570
571
572
  MACROBLOCK *const x = &cpi->td.mb;
  RD_OPT *const rd = &cpi->rd;

Yaowu Xu's avatar
Yaowu Xu committed
573
  aom_clear_system_state();
Jingning Han's avatar
Jingning Han committed
574

Yaowu Xu's avatar
Yaowu Xu committed
575
  rd->RDMULT = av1_compute_rd_mult(cpi, cm->base_qindex + cm->y_dc_delta_q);
Jingning Han's avatar
Jingning Han committed
576

577
  set_error_per_bit(x, rd->RDMULT);
Jingning Han's avatar
Jingning Han committed
578
579
580

  set_block_thresholds(cm, rd);

581
  av1_fill_nmv_costs(cm, x, cm->fc);
Jingning Han's avatar
Jingning Han committed
582

Alex Converse's avatar
Alex Converse committed
583
584
585
586
587
588
589
590
591
592
#if CONFIG_INTRABC
  if (frame_is_intra_only(cm) && cm->allow_screen_content_tools &&
      cpi->oxcf.pass != 1) {
    av1_build_nmv_cost_table(
        x->nmv_vec_cost[0],
        cm->allow_high_precision_mv ? x->nmvcost_hp[0] : x->nmvcost[0],
        &cm->fc->ndvc, MV_SUBPEL_NONE);
  }
#endif

593
#if CONFIG_GLOBAL_MOTION
594
  if (cpi->oxcf.pass != 1) {
595
    for (int i = 0; i < TRANS_TYPES; ++i)
596
#if GLOBAL_TRANS_TYPES > 4
597
598
      cpi->gmtype_cost[i] = (1 + (i > 0 ? GLOBAL_TYPE_BITS : 0))
                            << AV1_PROB_COST_SHIFT;
599
600
601
602
603
604
605
606
#else
      // IDENTITY: 1 bit
      // TRANSLATION: 3 bits
      // ROTZOOM: 2 bits
      // AFFINE: 3 bits
      cpi->gmtype_cost[i] = (1 + (i > 0 ? (i == ROTZOOM ? 1 : 2) : 0))
                            << AV1_PROB_COST_SHIFT;
#endif  // GLOBAL_TRANS_TYPES > 4
Jingning Han's avatar
Jingning Han committed
607
  }
608
#endif  // CONFIG_GLOBAL_MOTION
Jingning Han's avatar
Jingning Han committed
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
}

static void model_rd_norm(int xsq_q10, int *r_q10, int *d_q10) {
  // NOTE: The tables below must be of the same size.

  // The functions described below are sampled at the four most significant
  // bits of x^2 + 8 / 256.

  // Normalized rate:
  // This table models the rate for a Laplacian source with given variance
  // when quantized with a uniform quantizer with given stepsize. The
  // closed form expression is:
  // Rn(x) = H(sqrt(r)) + sqrt(r)*[1 + H(r)/(1 - r)],
  // where r = exp(-sqrt(2) * x) and x = qpstep / sqrt(variance),
  // and H(x) is the binary entropy function.
  static const int rate_tab_q10[] = {
625
626
627
628
629
630
631
632
633
    65536, 6086, 5574, 5275, 5063, 4899, 4764, 4651, 4553, 4389, 4255, 4142,
    4044,  3958, 3881, 3811, 3748, 3635, 3538, 3453, 3376, 3307, 3244, 3186,
    3133,  3037, 2952, 2877, 2809, 2747, 2690, 2638, 2589, 2501, 2423, 2353,
    2290,  2232, 2179, 2130, 2084, 2001, 1928, 1862, 1802, 1748, 1698, 1651,
    1608,  1530, 1460, 1398, 1342, 1290, 1243, 1199, 1159, 1086, 1021, 963,
    911,   864,  821,  781,  745,  680,  623,  574,  530,  490,  455,  424,
    395,   345,  304,  269,  239,  213,  190,  171,  154,  126,  104,  87,
    73,    61,   52,   44,   38,   28,   21,   16,   12,   10,   8,    6,
    5,     3,    2,    1,    1,    1,    0,    0,
Jingning Han's avatar
Jingning Han committed
634
635
636
637
638
639
640
641
642
  };
  // Normalized distortion:
  // This table models the normalized distortion for a Laplacian source
  // with given variance when quantized with a uniform quantizer
  // with given stepsize. The closed form expression is:
  // Dn(x) = 1 - 1/sqrt(2) * x / sinh(x/sqrt(2))
  // where x = qpstep / sqrt(variance).
  // Note the actual distortion is Dn * variance.
  static const int dist_tab_q10[] = {
643
644
645
646
647
648
649
650
651
    0,    0,    1,    1,    1,    2,    2,    2,    3,    3,    4,    5,
    5,    6,    7,    7,    8,    9,    11,   12,   13,   15,   16,   17,
    18,   21,   24,   26,   29,   31,   34,   36,   39,   44,   49,   54,
    59,   64,   69,   73,   78,   88,   97,   106,  115,  124,  133,  142,
    151,  167,  184,  200,  215,  231,  245,  260,  274,  301,  327,  351,
    375,  397,  418,  439,  458,  495,  528,  559,  587,  613,  637,  659,
    680,  717,  749,  777,  801,  823,  842,  859,  874,  899,  919,  936,
    949,  960,  969,  977,  983,  994,  1001, 1006, 1010, 1013, 1015, 1017,
    1018, 1020, 1022, 1022, 1023, 1023, 1023, 1024,
Jingning Han's avatar
Jingning Han committed
652
653
  };
  static const int xsq_iq_q10[] = {
654
655
656
657
658
659
660
661
662
663
664
665
    0,      4,      8,      12,     16,     20,     24,     28,     32,
    40,     48,     56,     64,     72,     80,     88,     96,     112,
    128,    144,    160,    176,    192,    208,    224,    256,    288,
    320,    352,    384,    416,    448,    480,    544,    608,    672,
    736,    800,    864,    928,    992,    1120,   1248,   1376,   1504,
    1632,   1760,   1888,   2016,   2272,   2528,   2784,   3040,   3296,
    3552,   3808,   4064,   4576,   5088,   5600,   6112,   6624,   7136,
    7648,   8160,   9184,   10208,  11232,  12256,  13280,  14304,  15328,
    16352,  18400,  20448,  22496,  24544,  26592,  28640,  30688,  32736,
    36832,  40928,  45024,  49120,  53216,  57312,  61408,  65504,  73696,
    81888,  90080,  98272,  106464, 114656, 122848, 131040, 147424, 163808,
    180192, 196576, 212960, 229344, 245728,
Jingning Han's avatar
Jingning Han committed
666
667
668
669
670
671
672
673
674
675
676
  };
  const int tmp = (xsq_q10 >> 2) + 8;
  const int k = get_msb(tmp) - 3;
  const int xq = (k << 3) + ((tmp >> k) & 0x7);
  const int one_q10 = 1 << 10;
  const int a_q10 = ((xsq_q10 - xsq_iq_q10[xq]) << 10) >> (2 + k);
  const int b_q10 = one_q10 - a_q10;
  *r_q10 = (rate_tab_q10[xq] * b_q10 + rate_tab_q10[xq + 1] * a_q10) >> 10;
  *d_q10 = (dist_tab_q10[xq] * b_q10 + dist_tab_q10[xq + 1] * a_q10) >> 10;
}

Yaowu Xu's avatar
Yaowu Xu committed
677
678
679
void av1_model_rd_from_var_lapndz(int64_t var, unsigned int n_log2,
                                  unsigned int qstep, int *rate,
                                  int64_t *dist) {
Jingning Han's avatar
Jingning Han committed
680
681
682
683
684
685
686
687
688
689
690
691
692
693
  // This function models the rate and distortion for a Laplacian
  // source with given variance when quantized with a uniform quantizer
  // with given stepsize. The closed form expressions are in:
  // Hang and Chen, "Source Model for transform video coder and its
  // application - Part I: Fundamental Theory", IEEE Trans. Circ.
  // Sys. for Video Tech., April 1997.
  if (var == 0) {
    *rate = 0;
    *dist = 0;
  } else {
    int d_q10, r_q10;
    static const uint32_t MAX_XSQ_Q10 = 245727;
    const uint64_t xsq_q10_64 =
        (((uint64_t)qstep * qstep << (n_log2 + 10)) + (var >> 1)) / var;
Yaowu Xu's avatar
Yaowu Xu committed
694
    const int xsq_q10 = (int)AOMMIN(xsq_q10_64, MAX_XSQ_Q10);
Jingning Han's avatar
Jingning Han committed
695
    model_rd_norm(xsq_q10, &r_q10, &d_q10);
Yaowu Xu's avatar
Yaowu Xu committed
696
    *rate = ROUND_POWER_OF_TWO(r_q10 << n_log2, 10 - AV1_PROB_COST_SHIFT);
Jingning Han's avatar
Jingning Han committed
697
698
699
700
    *dist = (var * (int64_t)d_q10 + 512) >> 10;
  }
}

701
702
703
704
static void get_entropy_contexts_plane(
    BLOCK_SIZE plane_bsize, TX_SIZE tx_size, const struct macroblockd_plane *pd,
    ENTROPY_CONTEXT t_above[2 * MAX_MIB_SIZE],
    ENTROPY_CONTEXT t_left[2 * MAX_MIB_SIZE]) {
705
706
  const int num_4x4_w = block_size_wide[plane_bsize] >> tx_size_wide_log2[0];
  const int num_4x4_h = block_size_high[plane_bsize] >> tx_size_high_log2[0];
Jingning Han's avatar
Jingning Han committed
707
708
709
  const ENTROPY_CONTEXT *const above = pd->above_context;
  const ENTROPY_CONTEXT *const left = pd->left_context;

710
711
712
713
714
715
#if CONFIG_LV_MAP
  memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
  memcpy(t_left, left, sizeof(ENTROPY_CONTEXT) * num_4x4_h);
  return;
#endif  // CONFIG_LV_MAP

Jingning Han's avatar
Jingning Han committed
716
  int i;
717

718
#if CONFIG_CHROMA_2X2
719
  switch (tx_size) {
720
    case TX_2X2:
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
      memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
      memcpy(t_left, left, sizeof(ENTROPY_CONTEXT) * num_4x4_h);
      break;
    case TX_4X4:
      for (i = 0; i < num_4x4_w; i += 2)
        t_above[i] = !!*(const uint16_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 2)
        t_left[i] = !!*(const uint16_t *)&left[i];
      break;
    case TX_8X8:
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
    case TX_16X16:
      for (i = 0; i < num_4x4_w; i += 8)
        t_above[i] = !!*(const uint64_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 8)
        t_left[i] = !!*(const uint64_t *)&left[i];
      break;
    case TX_32X32:
      for (i = 0; i < num_4x4_w; i += 16)
        t_above[i] =
            !!(*(const uint64_t *)&above[i] | *(const uint64_t *)&above[i + 8]);
      for (i = 0; i < num_4x4_h; i += 16)
        t_left[i] =
            !!(*(const uint64_t *)&left[i] | *(const uint64_t *)&left[i + 8]);
      break;
750
751
752
753
754
755
756
757
758
759
760
761
762
763
#if CONFIG_TX64X64
    case TX_64X64:
      for (i = 0; i < num_4x4_w; i += 32)
        t_above[i] =
            !!(*(const uint64_t *)&above[i] | *(const uint64_t *)&above[i + 8] |
               *(const uint64_t *)&above[i + 16] |
               *(const uint64_t *)&above[i + 24]);
      for (i = 0; i < num_4x4_h; i += 32)
        t_left[i] =
            !!(*(const uint64_t *)&left[i] | *(const uint64_t *)&left[i + 8] |
               *(const uint64_t *)&left[i + 16] |
               *(const uint64_t *)&left[i + 24]);
      break;
#endif  // CONFIG_TX64X64
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
    case TX_4X8:
      for (i = 0; i < num_4x4_w; i += 2)
        t_above[i] = !!*(const uint16_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
    case TX_8X4:
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 2)
        t_left[i] = !!*(const uint16_t *)&left[i];
      break;
    case TX_8X16:
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 8)
        t_left[i] = !!*(const uint64_t *)&left[i];
      break;
    case TX_16X8:
      for (i = 0; i < num_4x4_w; i += 8)
        t_above[i] = !!*(const uint64_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
    case TX_16X32:
      for (i = 0; i < num_4x4_w; i += 8)
        t_above[i] = !!*(const uint64_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 16)
        t_left[i] =
            !!(*(const uint64_t *)&left[i] | *(const uint64_t *)&left[i + 8]);
      break;
    case TX_32X16:
      for (i = 0; i < num_4x4_w; i += 16)
        t_above[i] =
            !!(*(const uint64_t *)&above[i] | *(const uint64_t *)&above[i + 8]);
      for (i = 0; i < num_4x4_h; i += 8)
        t_left[i] = !!*(const uint64_t *)&left[i];
801
      break;
Yue Chen's avatar
Yue Chen committed
802
#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
    case TX_4X16:
      for (i = 0; i < num_4x4_w; i += 2)
        t_above[i] = !!*(const uint16_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 8)
        t_left[i] = !!*(const uint64_t *)&left[i];
      break;
    case TX_16X4:
      for (i = 0; i < num_4x4_w; i += 8)
        t_above[i] = !!*(const uint64_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 2)
        t_left[i] = !!*(const uint16_t *)&left[i];
      break;
    case TX_8X32:
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 16)
        t_left[i] =
            !!(*(const uint64_t *)&left[i] | *(const uint64_t *)&left[i + 8]);
      break;
    case TX_32X8:
      for (i = 0; i < num_4x4_w; i += 16)
        t_above[i] =
            !!(*(const uint64_t *)&above[i] | *(const uint64_t *)&above[i + 8]);
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
Yue Chen's avatar
Yue Chen committed
829
#endif
830
831
832
833

    default: assert(0 && "Invalid transform size."); break;
  }
  return;
834
#endif  // CONFIG_CHROMA_2X2
835
836

  switch (tx_size) {
Jingning Han's avatar
Jingning Han committed
837
838
839
840
    case TX_4X4:
      memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
      memcpy(t_left, left, sizeof(ENTROPY_CONTEXT) * num_4x4_h);
      break;
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
    case TX_8X8:
      for (i = 0; i < num_4x4_w; i += 2)
        t_above[i] = !!*(const uint16_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 2)
        t_left[i] = !!*(const uint16_t *)&left[i];
      break;
    case TX_16X16:
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
    case TX_32X32:
      for (i = 0; i < num_4x4_w; i += 8)
        t_above[i] = !!*(const uint64_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 8)
        t_left[i] = !!*(const uint64_t *)&left[i];
      break;
859
860
861
862
863
864
#if CONFIG_TX64X64
    case TX_64X64:
      for (i = 0; i < num_4x4_w; i += 16)
        t_above[i] =
            !!(*(const uint64_t *)&above[i] | *(const uint64_t *)&above[i + 8]);
      for (i = 0; i < num_4x4_h; i += 16)
865
866
        t_left[i] =
            !!(*(const uint64_t *)&left[i] | *(const uint64_t *)&left[i + 8]);
867
868
      break;
#endif  // CONFIG_TX64X64
869
870
871
872
873
874
875
876
877
878
    case TX_4X8:
      memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
      for (i = 0; i < num_4x4_h; i += 2)
        t_left[i] = !!*(const uint16_t *)&left[i];
      break;
    case TX_8X4:
      for (i = 0; i < num_4x4_w; i += 2)
        t_above[i] = !!*(const uint16_t *)&above[i];
      memcpy(t_left, left, sizeof(ENTROPY_CONTEXT) * num_4x4_h);
      break;
879
    case TX_8X16:
Jingning Han's avatar
Jingning Han committed
880
881
      for (i = 0; i < num_4x4_w; i += 2)
        t_above[i] = !!*(const uint16_t *)&above[i];
882
883
884
885
886
887
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
    case TX_16X8:
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
Jingning Han's avatar
Jingning Han committed
888
889
890
      for (i = 0; i < num_4x4_h; i += 2)
        t_left[i] = !!*(const uint16_t *)&left[i];
      break;
891
    case TX_16X32:
Jingning Han's avatar
Jingning Han committed
892
893
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
894
895
      for (i = 0; i < num_4x4_h; i += 8)
        t_left[i] = !!*(const uint64_t *)&left[i];
Jingning Han's avatar
Jingning Han committed
896
      break;
897
    case TX_32X16:
Jingning Han's avatar
Jingning Han committed
898
899
      for (i = 0; i < num_4x4_w; i += 8)
        t_above[i] = !!*(const uint64_t *)&above[i];
900
901
902
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
Yue Chen's avatar
Yue Chen committed
903
#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
    case TX_4X16:
      memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
    case TX_16X4:
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
      memcpy(t_left, left, sizeof(ENTROPY_CONTEXT) * num_4x4_h);
      break;
    case TX_8X32:
      for (i = 0; i < num_4x4_w; i += 2)
        t_above[i] = !!*(const uint16_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 8)
        t_left[i] = !!*(const uint64_t *)&left[i];
      break;
    case TX_32X8:
      for (i = 0; i < num_4x4_w; i += 8)
        t_above[i] = !!*(const uint64_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 2)
        t_left[i] = !!*(const uint16_t *)&left[i];
      break;
Yue Chen's avatar
Yue Chen committed
926
#endif
clang-format's avatar
clang-format committed
927
    default: assert(0 && "Invalid transform size."); break;
Jingning Han's avatar
Jingning Han committed
928
929
930
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
931
932
933
934
void av1_get_entropy_contexts(BLOCK_SIZE bsize, TX_SIZE tx_size,
                              const struct macroblockd_plane *pd,
                              ENTROPY_CONTEXT t_above[2 * MAX_MIB_SIZE],
                              ENTROPY_CONTEXT t_left[2 * MAX_MIB_SIZE]) {
935
#if CONFIG_CHROMA_SUB8X8
936
937
938
  const BLOCK_SIZE plane_bsize =
      AOMMAX(BLOCK_4X4, get_plane_block_size(bsize, pd));
#else
939
  const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd);
940
#endif
941
  get_entropy_contexts_plane(plane_bsize, tx_size, pd, t_above, t_left);
942
943
}

944
void av1_mv_pred(const AV1_COMP *cpi, MACROBLOCK *x, uint8_t *ref_y_buffer,
Yaowu Xu's avatar
Yaowu Xu committed
945
                 int ref_y_stride, int ref_frame, BLOCK_SIZE block_size) {
Jingning Han's avatar
Jingning Han committed
946
947
948
949
950
951
952
953
  int i;
  int zero_seen = 0;
  int best_index = 0;
  int best_sad = INT_MAX;
  int this_sad = INT_MAX;
  int max_mv = 0;
  uint8_t *src_y_ptr = x->plane[0].src.buf;
  uint8_t *ref_y_ptr;
954
  MV pred_mv[MAX_MV_REF_CANDIDATES + 1];
955
956
957
958
959
960
  int num_mv_refs = 0;

  pred_mv[num_mv_refs++] = x->mbmi_ext->ref_mvs[ref_frame][0].as_mv;
  if (x->mbmi_ext->ref_mvs[ref_frame][0].as_int !=
      x->mbmi_ext->ref_mvs[ref_frame][1].as_int) {
    pred_mv[num_mv_refs++] = x->mbmi_ext->ref_mvs[ref_frame][1].as_mv;
961
  }
962
963
964
  if (cpi->sf.adaptive_motion_search && block_size < x->max_partition_size)
    pred_mv[num_mv_refs++] = x->pred_mv[ref_frame];

Jingning Han's avatar
Jingning Han committed
965
  assert(num_mv_refs <= (int)(sizeof(pred_mv) / sizeof(pred_mv[0])));
966

Jingning Han's avatar
Jingning Han committed
967
968
969
970
971
972
  // Get the sad for each candidate reference mv.
  for (i = 0; i < num_mv_refs; ++i) {
    const MV *this_mv = &pred_mv[i];
    int fp_row, fp_col;
    fp_row = (this_mv->row + 3 + (this_mv->row >= 0)) >> 3;
    fp_col = (this_mv->col + 3 + (this_mv->col >= 0)) >> 3;
Yaowu Xu's avatar
Yaowu Xu committed
973
    max_mv = AOMMAX(max_mv, AOMMAX(abs(this_mv->row), abs(this_mv->col)) >> 3);
Jingning Han's avatar
Jingning Han committed
974

975
976
    if (fp_row == 0 && fp_col == 0 && zero_seen) continue;
    zero_seen |= (fp_row == 0 && fp_col == 0);
Jingning Han's avatar
Jingning Han committed
977

978
    ref_y_ptr = &ref_y_buffer[ref_y_stride * fp_row + fp_col];
Jingning Han's avatar
Jingning Han committed
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
    // Find sad for current vector.
    this_sad = cpi->fn_ptr[block_size].sdf(src_y_ptr, x->plane[0].src.stride,
                                           ref_y_ptr, ref_y_stride);
    // Note if it is the best so far.
    if (this_sad < best_sad) {
      best_sad = this_sad;
      best_index = i;
    }
  }

  // Note the index of the mv that worked best in the reference list.
  x->mv_best_ref_index[ref_frame] = best_index;
  x->max_mv_context[ref_frame] = max_mv;
  x->pred_mv_sad[ref_frame] = best_sad;
}

Yaowu Xu's avatar
Yaowu Xu committed
995
996
997
998
999
void av1_setup_pred_block(const MACROBLOCKD *xd,
                          struct buf_2d dst[MAX_MB_PLANE],
                          const YV12_BUFFER_CONFIG *src, int mi_row, int mi_col,
                          const struct scale_factors *scale,
                          const struct scale_factors *scale_uv) {
Jingning Han's avatar
Jingning Han committed
1000
1001
1002
1003
1004
1005
1006
1007
1008
  int i;

  dst[0].buf = src->y_buffer;
  dst[0].stride = src->y_stride;
  dst[1].buf = src->u_buffer;
  dst[2].buf = src->v_buffer;
  dst[1].stride = dst[2].stride = src->uv_stride;

  for (i = 0; i < MAX_MB_PLANE; ++i) {
1009
    setup_pred_plane(dst + i, xd->mi[0]->mbmi.sb_type, dst[i].buf,
1010
1011
                     i ? src->uv_crop_width : src->y_crop_width,
                     i ? src->uv_crop_height : src->y_crop_height,
1012
                     dst[i].stride, mi_row, mi_col, i ? scale_uv : scale,
Jingning Han's avatar
Jingning Han committed
1013
1014
1015
1016
                     xd->plane[i].subsampling_x, xd->plane[i].subsampling_y);
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
1017
1018
int av1_raster_block_offset(BLOCK_SIZE plane_bsize, int raster_block,
                            int stride) {
Jingning Han's avatar
Jingning Han committed
1019
1020
1021
1022
1023
1024
  const int bw = b_width_log2_lookup[plane_bsize];
  const int y = 4 * (raster_block >> bw);
  const int x = 4 * (raster_block & ((1 << bw) - 1));
  return y * stride + x;
}

Yaowu Xu's avatar
Yaowu Xu committed
1025
1026
int16_t *av1_raster_block_offset_int16(BLOCK_SIZE plane_bsize, int raster_block,
                                       int16_t *base) {
1027
  const int stride = block_size_wide[plane_bsize];
Yaowu Xu's avatar
Yaowu Xu committed
1028
  return base + av1_raster_block_offset(plane_bsize, raster_block, stride);
Jingning Han's avatar
Jingning Han committed
1029
1030
}

Yaowu Xu's avatar
Yaowu Xu committed
1031
1032
1033
YV12_BUFFER_CONFIG *av1_get_scaled_ref_frame(const AV1_COMP *cpi,
                                             int ref_frame) {
  const AV1_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
1034
1035
  const int scaled_idx = cpi->scaled_ref_idx[ref_frame - 1];
  const int ref_idx = get_ref_frame_buf_idx(cpi, ref_frame);
1036
1037
1038
  return (scaled_idx != ref_idx && scaled_idx != INVALID_IDX)
             ? &cm->buffer_pool->frame_bufs[scaled_idx].buf
             : NULL;
Jingning Han's avatar
Jingning Han committed
1039
1040
}

1041
#if CONFIG_DUAL_FILTER
1042
1043
int av1_get_switchable_rate(const AV1_COMMON *const cm, MACROBLOCK *x,
                            const MACROBLOCKD *xd) {
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
  if (cm->interp_filter == SWITCHABLE) {
    const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
    int inter_filter_cost = 0;
    int dir;

    for (dir = 0; dir < 2; ++dir) {
      if (has_subpel_mv_component(xd->mi[0], xd, dir) ||
          (mbmi->ref_frame[1] > INTRA_FRAME &&
           has_subpel_mv_component(xd->mi[0], xd, dir + 2))) {
        const int ctx = av1_get_pred_context_switchable_interp(xd, dir);
        inter_filter_cost +=
1055
            x->switchable_interp_costs[ctx][mbmi->interp_filter[dir]];
1056
      }
1057
    }
1058
1059
1060
    return SWITCHABLE_INTERP_RATE_FACTOR * inter_filter_cost;
  } else {
    return 0;
1061
1062
1063
  }
}
#else
1064
1065
int av1_get_switchable_rate(const AV1_COMMON *const cm, MACROBLOCK *x,
                            const MACROBLOCKD *xd) {
1066
  if (cm->interp_filter == SWITCHABLE) {
1067
1068
1069
    const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
    const int ctx = av1_get_pred_context_switchable_interp(xd);
    return SWITCHABLE_INTERP_RATE_FACTOR *
1070
           x->switchable_interp_costs[ctx][mbmi->interp_filter];
1071
1072
  }
  return 0;
Jingning Han's avatar
Jingning Han committed
1073
}
1074
#endif
Jingning Han's avatar
Jingning Han committed
1075

Yaowu Xu's avatar
Yaowu Xu committed
1076
void av1_set_rd_speed_thresholds(AV1_COMP *cpi) {
Jingning Han's avatar
Jingning Han committed
1077
1078
1079
1080
1081
  int i;
  RD_OPT *const rd = &cpi->rd;
  SPEED_FEATURES *const sf = &cpi->sf;

  // Set baseline threshold values.
1082
  for (i = 0; i < MAX_MODES; ++i) rd->thresh_mult[i] = cpi->oxcf.mode == 0;
Jingning Han's avatar
Jingning Han committed
1083
1084
1085

  if (sf->adaptive_rd_thresh) {
    rd->thresh_mult[THR_NEARESTMV] = 300;
1086
1087
1088
#if CONFIG_EXT_REFS
    rd->thresh_mult[THR_NEARESTL2] = 300;
    rd->thresh_mult[THR_NEARESTL3] = 300;
1089
    rd->thresh_mult[THR_NEARESTB] = 300<