vp9_rdopt.c 163 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5
6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9
10
11
 */

#include <assert.h>
12
13
14
#include <limits.h>
#include <math.h>
#include <stdio.h>
15

16
17
18
19
20
21
#include "./vp9_rtcd.h"

#include "vpx_mem/vpx_mem.h"

#include "vp9/common/vp9_common.h"
#include "vp9/common/vp9_entropy.h"
22
#include "vp9/common/vp9_entropymode.h"
23
24
25
26
27
#include "vp9/common/vp9_idct.h"
#include "vp9/common/vp9_mvref_common.h"
#include "vp9/common/vp9_pragmas.h"
#include "vp9/common/vp9_pred_common.h"
#include "vp9/common/vp9_quant_common.h"
28
29
#include "vp9/common/vp9_reconinter.h"
#include "vp9/common/vp9_reconintra.h"
30
31
32
#include "vp9/common/vp9_seg_common.h"
#include "vp9/common/vp9_systemdependent.h"

Dmitry Kovalev's avatar
Dmitry Kovalev committed
33
#include "vp9/encoder/vp9_cost.h"
34
#include "vp9/encoder/vp9_encodemb.h"
35
#include "vp9/encoder/vp9_encodemv.h"
36
#include "vp9/encoder/vp9_mcomp.h"
37
38
#include "vp9/encoder/vp9_onyx_int.h"
#include "vp9/encoder/vp9_quantize.h"
39
#include "vp9/encoder/vp9_ratectrl.h"
40
41
42
#include "vp9/encoder/vp9_rdopt.h"
#include "vp9/encoder/vp9_tokenize.h"
#include "vp9/encoder/vp9_variance.h"
Paul Wilkins's avatar
Paul Wilkins committed
43

44
45
46
47
48
#define RD_THRESH_MAX_FACT 64
#define RD_THRESH_INC      1
#define RD_THRESH_POW      1.25
#define RD_MULT_EPB_RATIO  64

49
50
51
/* Factor to weigh the rate for switchable interp filters */
#define SWITCHABLE_INTERP_RATE_FACTOR 1

52
53
54
#define LAST_FRAME_MODE_MASK    0xFFEDCD60
#define GOLDEN_FRAME_MODE_MASK  0xFFDA3BB0
#define ALT_REF_MODE_MASK       0xFFC648D0
55

Paul Wilkins's avatar
Paul Wilkins committed
56
57
#define MIN_EARLY_TERM_INDEX    3

58
59
60
61
62
63
64
65
66
typedef struct {
  MB_PREDICTION_MODE mode;
  MV_REFERENCE_FRAME ref_frame[2];
} MODE_DEFINITION;

typedef struct {
  MV_REFERENCE_FRAME ref_frame[2];
} REF_DEFINITION;

Alex Converse's avatar
Alex Converse committed
67
68
69
70
71
72
73
74
75
76
77
78
79
struct rdcost_block_args {
  MACROBLOCK *x;
  ENTROPY_CONTEXT t_above[16];
  ENTROPY_CONTEXT t_left[16];
  int rate;
  int64_t dist;
  int64_t sse;
  int this_rate;
  int64_t this_dist;
  int64_t this_sse;
  int64_t this_rd;
  int64_t best_rd;
  int skip;
80
  int use_fast_coef_costing;
81
  const scan_order *so;
Alex Converse's avatar
Alex Converse committed
82
83
};

84
static const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
  {NEARESTMV, {LAST_FRAME,   NONE}},
  {NEARESTMV, {ALTREF_FRAME, NONE}},
  {NEARESTMV, {GOLDEN_FRAME, NONE}},

  {DC_PRED,   {INTRA_FRAME,  NONE}},

  {NEWMV,     {LAST_FRAME,   NONE}},
  {NEWMV,     {ALTREF_FRAME, NONE}},
  {NEWMV,     {GOLDEN_FRAME, NONE}},

  {NEARMV,    {LAST_FRAME,   NONE}},
  {NEARMV,    {ALTREF_FRAME, NONE}},
  {NEARESTMV, {LAST_FRAME,   ALTREF_FRAME}},
  {NEARESTMV, {GOLDEN_FRAME, ALTREF_FRAME}},

  {TM_PRED,   {INTRA_FRAME,  NONE}},

  {NEARMV,    {LAST_FRAME,   ALTREF_FRAME}},
  {NEWMV,     {LAST_FRAME,   ALTREF_FRAME}},
  {NEARMV,    {GOLDEN_FRAME, NONE}},
  {NEARMV,    {GOLDEN_FRAME, ALTREF_FRAME}},
  {NEWMV,     {GOLDEN_FRAME, ALTREF_FRAME}},

  {ZEROMV,    {LAST_FRAME,   NONE}},
  {ZEROMV,    {GOLDEN_FRAME, NONE}},
  {ZEROMV,    {ALTREF_FRAME, NONE}},
  {ZEROMV,    {LAST_FRAME,   ALTREF_FRAME}},
  {ZEROMV,    {GOLDEN_FRAME, ALTREF_FRAME}},

  {H_PRED,    {INTRA_FRAME,  NONE}},
  {V_PRED,    {INTRA_FRAME,  NONE}},
  {D135_PRED, {INTRA_FRAME,  NONE}},
  {D207_PRED, {INTRA_FRAME,  NONE}},
  {D153_PRED, {INTRA_FRAME,  NONE}},
  {D63_PRED,  {INTRA_FRAME,  NONE}},
  {D117_PRED, {INTRA_FRAME,  NONE}},
  {D45_PRED,  {INTRA_FRAME,  NONE}},
122
123
};

124
static const REF_DEFINITION vp9_ref_order[MAX_REFS] = {
125
126
127
128
129
130
  {{LAST_FRAME,   NONE}},
  {{GOLDEN_FRAME, NONE}},
  {{ALTREF_FRAME, NONE}},
  {{LAST_FRAME,   ALTREF_FRAME}},
  {{GOLDEN_FRAME, ALTREF_FRAME}},
  {{INTRA_FRAME,  NONE}},
John Koleszar's avatar
John Koleszar committed
131
132
};

133
134
135
136
// The baseline rd thresholds for breaking out of the rd loop for
// certain modes are assumed to be based on 8x8 blocks.
// This table is used to correct for blocks size.
// The factors here are << 2 (2 = x0.5, 32 = x8 etc).
137
138
139
static const uint8_t rd_thresh_block_size_factor[BLOCK_SIZES] = {
  2, 3, 3, 4, 6, 6, 8, 12, 12, 16, 24, 24, 32
};
140

141
142
143
144
145
146
147
148
149
static int raster_block_offset(BLOCK_SIZE plane_bsize,
                               int raster_block, int stride) {
  const int bw = b_width_log2(plane_bsize);
  const int y = 4 * (raster_block >> bw);
  const int x = 4 * (raster_block & ((1 << bw) - 1));
  return y * stride + x;
}
static int16_t* raster_block_offset_int16(BLOCK_SIZE plane_bsize,
                                          int raster_block, int16_t *base) {
150
  const int stride = 4 * num_4x4_blocks_wide_lookup[plane_bsize];
151
152
153
  return base + raster_block_offset(plane_bsize, raster_block, stride);
}

Dmitry Kovalev's avatar
Dmitry Kovalev committed
154
155
static void fill_mode_costs(VP9_COMP *cpi) {
  MACROBLOCK *const x = &cpi->mb;
156
  const FRAME_CONTEXT *const fc = &cpi->common.fc;
157
158
159
160
  int i, j;

  for (i = 0; i < INTRA_MODES; i++)
    for (j = 0; j < INTRA_MODES; j++)
Dmitry Kovalev's avatar
Dmitry Kovalev committed
161
      vp9_cost_tokens((int *)x->y_mode_costs[i][j], vp9_kf_y_mode_prob[i][j],
162
163
164
                      vp9_intra_mode_tree);

  // TODO(rbultje) separate tables for superblock costing?
Dmitry Kovalev's avatar
Dmitry Kovalev committed
165
  vp9_cost_tokens(x->mbmode_cost, fc->y_mode_prob[1], vp9_intra_mode_tree);
166
167
168
169
  vp9_cost_tokens(x->intra_uv_mode_cost[KEY_FRAME],
                  vp9_kf_uv_mode_prob[TM_PRED], vp9_intra_mode_tree);
  vp9_cost_tokens(x->intra_uv_mode_cost[INTER_FRAME],
                  fc->uv_mode_prob[TM_PRED], vp9_intra_mode_tree);
170
171

  for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; ++i)
Dmitry Kovalev's avatar
Dmitry Kovalev committed
172
    vp9_cost_tokens((int *)x->switchable_interp_costs[i],
173
                    fc->switchable_interp_prob[i], vp9_switchable_interp_tree);
174
175
}

176
static void fill_token_costs(vp9_coeff_cost *c,
177
                             vp9_coeff_probs_model (*p)[PLANE_TYPES]) {
178
  int i, j, k, l;
179
  TX_SIZE t;
180
  for (t = TX_4X4; t <= TX_32X32; ++t)
181
    for (i = 0; i < PLANE_TYPES; ++i)
182
183
184
      for (j = 0; j < REF_TYPES; ++j)
        for (k = 0; k < COEF_BANDS; ++k)
          for (l = 0; l < BAND_COEFF_CONTEXTS(k); ++l) {
185
186
            vp9_prob probs[ENTROPY_NODES];
            vp9_model_to_full_probs(p[t][i][j][k][l], probs);
187
            vp9_cost_tokens((int *)c[t][i][j][k][0][l], probs,
188
                            vp9_coef_tree);
189
            vp9_cost_tokens_skip((int *)c[t][i][j][k][1][l], probs,
190
                                 vp9_coef_tree);
191
192
            assert(c[t][i][j][k][0][l][EOB_TOKEN] ==
                   c[t][i][j][k][1][l][EOB_TOKEN]);
193
          }
194
195
}

196
static const uint8_t rd_iifactor[32] = {
197
198
199
200
201
  4, 4, 3, 2, 1, 0, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 0,
};
John Koleszar's avatar
John Koleszar committed
202

203
// 3* dc_qlookup[Q]*dc_qlookup[Q];
204

205
/* values are now correlated to quantizer */
Paul Wilkins's avatar
Paul Wilkins committed
206
207
208
static int sad_per_bit16lut[QINDEX_RANGE];
static int sad_per_bit4lut[QINDEX_RANGE];

209
void vp9_init_me_luts() {
John Koleszar's avatar
John Koleszar committed
210
211
212
213
214
215
  int i;

  // Initialize the sad lut tables using a formulaic calculation for now
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < QINDEX_RANGE; i++) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
216
217
218
    const double q = vp9_convert_qindex_to_q(i);
    sad_per_bit16lut[i] = (int)(0.0418 * q + 2.4107);
    sad_per_bit4lut[i] = (int)(0.063 * q + 2.742);
John Koleszar's avatar
John Koleszar committed
219
  }
Paul Wilkins's avatar
Paul Wilkins committed
220
}
John Koleszar's avatar
John Koleszar committed
221

222
int vp9_compute_rd_mult(const VP9_COMP *cpi, int qindex) {
223
  const int q = vp9_dc_quant(qindex, 0);
224
  // TODO(debargha): Adjust the function below
225
226
227
228
229
230
231
232
  int rdmult = 88 * q * q / 25;
  if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    if (cpi->twopass.next_iiratio > 31)
      rdmult += (rdmult * rd_iifactor[31]) >> 4;
    else
      rdmult += (rdmult * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
  }
  return rdmult;
233
234
235
236
}

static int compute_rd_thresh_factor(int qindex) {
  // TODO(debargha): Adjust the function below
237
238
  const int q = (int)(pow(vp9_dc_quant(qindex, 0) / 4.0, RD_THRESH_POW) * 5.12);
  return MAX(q, 8);
239
240
}

Dmitry Kovalev's avatar
Dmitry Kovalev committed
241
242
243
void vp9_initialize_me_consts(VP9_COMP *cpi, int qindex) {
  cpi->mb.sadperbit16 = sad_per_bit16lut[qindex];
  cpi->mb.sadperbit4 = sad_per_bit4lut[qindex];
John Koleszar's avatar
John Koleszar committed
244
245
}

246
static void set_block_thresholds(VP9_COMP *cpi) {
247
  const VP9_COMMON *const cm = &cpi->common;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
248
  RD_OPT *const rd = &cpi->rd;
249
250
251
  int i, bsize, segment_id;

  for (segment_id = 0; segment_id < MAX_SEGMENTS; ++segment_id) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
252
253
254
255
    const int qindex = clamp(vp9_get_qindex(&cm->seg, segment_id,
                                            cm->base_qindex) + cm->y_dc_delta_q,
                             0, MAXQ);
    const int q = compute_rd_thresh_factor(qindex);
256

257
    for (bsize = 0; bsize < BLOCK_SIZES; ++bsize) {
258
259
      // Threshold here seems unnecessarily harsh but fine given actual
      // range of values used for cpi->sf.thresh_mult[].
Dmitry Kovalev's avatar
Dmitry Kovalev committed
260
261
262
263
      const int t = q * rd_thresh_block_size_factor[bsize];
      const int thresh_max = INT_MAX / t;

      for (i = 0; i < MAX_MODES; ++i)
Dmitry Kovalev's avatar
Dmitry Kovalev committed
264
265
        rd->threshes[segment_id][bsize][i] =
            rd->thresh_mult[i] < thresh_max ? rd->thresh_mult[i] * t / 4
Dmitry Kovalev's avatar
Dmitry Kovalev committed
266
                                            : INT_MAX;
267

268
      for (i = 0; i < MAX_REFS; ++i) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
269
270
271
        rd->thresh_sub8x8[segment_id][bsize][i] =
            rd->thresh_mult_sub8x8[i] < thresh_max
                ? rd->thresh_mult_sub8x8[i] * t / 4
Dmitry Kovalev's avatar
Dmitry Kovalev committed
272
                : INT_MAX;
273
274
      }
    }
John Koleszar's avatar
John Koleszar committed
275
  }
Deb Mukherjee's avatar
Deb Mukherjee committed
276
277
}

278
void vp9_initialize_rd_consts(VP9_COMP *cpi) {
279
280
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCK *const x = &cpi->mb;
281
  int i;
Deb Mukherjee's avatar
Deb Mukherjee committed
282

283
  vp9_clear_system_state();
Deb Mukherjee's avatar
Deb Mukherjee committed
284

Dmitry Kovalev's avatar
Dmitry Kovalev committed
285
286
  cpi->rd.RDDIV = RDDIV_BITS;  // in bits (to multiply D by 128)
  cpi->rd.RDMULT = vp9_compute_rd_mult(cpi, cm->base_qindex + cm->y_dc_delta_q);
287

Dmitry Kovalev's avatar
Dmitry Kovalev committed
288
  x->errorperbit = cpi->rd.RDMULT / RD_MULT_EPB_RATIO;
289
  x->errorperbit += (x->errorperbit == 0);
Deb Mukherjee's avatar
Deb Mukherjee committed
290

Dmitry Kovalev's avatar
Dmitry Kovalev committed
291
292
  x->select_txfm_size = (cpi->sf.tx_size_search_method == USE_LARGESTALL &&
                         cm->frame_type != KEY_FRAME) ? 0 : 1;
293

294
  set_block_thresholds(cpi);
John Koleszar's avatar
John Koleszar committed
295

296
  if (!cpi->sf.use_nonrd_pick_mode || cm->frame_type == KEY_FRAME) {
297
298
    fill_token_costs(x->token_costs, cm->fc.coef_probs);

Jim Bankoski's avatar
Jim Bankoski committed
299
300
301
    for (i = 0; i < PARTITION_CONTEXTS; i++)
      vp9_cost_tokens(x->partition_cost[i], get_partition_probs(cm, i),
                      vp9_partition_tree);
302
  }
Jim Bankoski's avatar
Jim Bankoski committed
303

304
305
  if (!cpi->sf.use_nonrd_pick_mode || (cm->current_video_frame & 0x07) == 1 ||
      cm->frame_type == KEY_FRAME) {
Jim Bankoski's avatar
Jim Bankoski committed
306
307
308
309
310
311
    fill_mode_costs(cpi);

    if (!frame_is_intra_only(cm)) {
      vp9_build_nmv_cost_table(x->nmvjointcost,
                               cm->allow_high_precision_mv ? x->nmvcost_hp
                                                           : x->nmvcost,
312
                               &cm->fc.nmvc, cm->allow_high_precision_mv);
Jim Bankoski's avatar
Jim Bankoski committed
313
314
315
316
317

      for (i = 0; i < INTER_MODE_CONTEXTS; ++i)
        vp9_cost_tokens((int *)x->inter_mode_cost[i],
                        cm->fc.inter_mode_probs[i], vp9_inter_mode_tree);
    }
318
  }
John Koleszar's avatar
John Koleszar committed
319
320
}

321
static const int MAX_XSQ_Q10 = 245727;
Deb Mukherjee's avatar
Deb Mukherjee committed
322

323
static void model_rd_norm(int xsq_q10, int *r_q10, int *d_q10) {
324
  // NOTE: The tables below must be of the same size
325
326
327
328

  // The functions described below are sampled at the four most significant
  // bits of x^2 + 8 / 256

Deb Mukherjee's avatar
Deb Mukherjee committed
329
  // Normalized rate
330
  // This table models the rate for a Laplacian source
Deb Mukherjee's avatar
Deb Mukherjee committed
331
332
333
334
335
  // source with given variance when quantized with a uniform quantizer
  // with given stepsize. The closed form expression is:
  // Rn(x) = H(sqrt(r)) + sqrt(r)*[1 + H(r)/(1 - r)],
  // where r = exp(-sqrt(2) * x) and x = qpstep / sqrt(variance),
  // and H(x) is the binary entropy function.
336
337
338
339
340
341
342
343
344
345
346
347
348
349
  static const int rate_tab_q10[] = {
    65536,  6086,  5574,  5275,  5063,  4899,  4764,  4651,
     4553,  4389,  4255,  4142,  4044,  3958,  3881,  3811,
     3748,  3635,  3538,  3453,  3376,  3307,  3244,  3186,
     3133,  3037,  2952,  2877,  2809,  2747,  2690,  2638,
     2589,  2501,  2423,  2353,  2290,  2232,  2179,  2130,
     2084,  2001,  1928,  1862,  1802,  1748,  1698,  1651,
     1608,  1530,  1460,  1398,  1342,  1290,  1243,  1199,
     1159,  1086,  1021,   963,   911,   864,   821,   781,
      745,   680,   623,   574,   530,   490,   455,   424,
      395,   345,   304,   269,   239,   213,   190,   171,
      154,   126,   104,    87,    73,    61,    52,    44,
       38,    28,    21,    16,    12,    10,     8,     6,
        5,     3,     2,     1,     1,     1,     0,     0,
Deb Mukherjee's avatar
Deb Mukherjee committed
350
351
  };
  // Normalized distortion
352
  // This table models the normalized distortion for a Laplacian source
Deb Mukherjee's avatar
Deb Mukherjee committed
353
354
355
356
357
  // source with given variance when quantized with a uniform quantizer
  // with given stepsize. The closed form expression is:
  // Dn(x) = 1 - 1/sqrt(2) * x / sinh(x/sqrt(2))
  // where x = qpstep / sqrt(variance)
  // Note the actual distortion is Dn * variance.
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
  static const int dist_tab_q10[] = {
       0,     0,     1,     1,     1,     2,     2,     2,
       3,     3,     4,     5,     5,     6,     7,     7,
       8,     9,    11,    12,    13,    15,    16,    17,
      18,    21,    24,    26,    29,    31,    34,    36,
      39,    44,    49,    54,    59,    64,    69,    73,
      78,    88,    97,   106,   115,   124,   133,   142,
     151,   167,   184,   200,   215,   231,   245,   260,
     274,   301,   327,   351,   375,   397,   418,   439,
     458,   495,   528,   559,   587,   613,   637,   659,
     680,   717,   749,   777,   801,   823,   842,   859,
     874,   899,   919,   936,   949,   960,   969,   977,
     983,   994,  1001,  1006,  1010,  1013,  1015,  1017,
    1018,  1020,  1022,  1022,  1023,  1023,  1023,  1024,
  };
  static const int xsq_iq_q10[] = {
         0,      4,      8,     12,     16,     20,     24,     28,
        32,     40,     48,     56,     64,     72,     80,     88,
        96,    112,    128,    144,    160,    176,    192,    208,
       224,    256,    288,    320,    352,    384,    416,    448,
       480,    544,    608,    672,    736,    800,    864,    928,
       992,   1120,   1248,   1376,   1504,   1632,   1760,   1888,
      2016,   2272,   2528,   2784,   3040,   3296,   3552,   3808,
      4064,   4576,   5088,   5600,   6112,   6624,   7136,   7648,
      8160,   9184,  10208,  11232,  12256,  13280,  14304,  15328,
     16352,  18400,  20448,  22496,  24544,  26592,  28640,  30688,
     32736,  36832,  40928,  45024,  49120,  53216,  57312,  61408,
     65504,  73696,  81888,  90080,  98272, 106464, 114656, 122848,
    131040, 147424, 163808, 180192, 196576, 212960, 229344, 245728,
Deb Mukherjee's avatar
Deb Mukherjee committed
387
  };
388
  /*
389
390
391
392
  static const int tab_size = sizeof(rate_tab_q10) / sizeof(rate_tab_q10[0]);
  assert(sizeof(dist_tab_q10) / sizeof(dist_tab_q10[0]) == tab_size);
  assert(sizeof(xsq_iq_q10) / sizeof(xsq_iq_q10[0]) == tab_size);
  assert(MAX_XSQ_Q10 + 1 == xsq_iq_q10[tab_size - 1]);
393
  */
394
395
396
397
398
399
400
401
  int tmp = (xsq_q10 >> 2) + 8;
  int k = get_msb(tmp) - 3;
  int xq = (k << 3) + ((tmp >> k) & 0x7);
  const int one_q10 = 1 << 10;
  const int a_q10 = ((xsq_q10 - xsq_iq_q10[xq]) << 10) >> (2 + k);
  const int b_q10 = one_q10 - a_q10;
  *r_q10 = (rate_tab_q10[xq] * b_q10 + rate_tab_q10[xq + 1] * a_q10) >> 10;
  *d_q10 = (dist_tab_q10[xq] * b_q10 + dist_tab_q10[xq + 1] * a_q10) >> 10;
Deb Mukherjee's avatar
Deb Mukherjee committed
402
403
}

404
405
406
void vp9_model_rd_from_var_lapndz(unsigned int var, unsigned int n,
                                  unsigned int qstep, int *rate,
                                  int64_t *dist) {
Deb Mukherjee's avatar
Deb Mukherjee committed
407
408
409
410
411
412
  // This function models the rate and distortion for a Laplacian
  // source with given variance when quantized with a uniform quantizer
  // with given stepsize. The closed form expressions are in:
  // Hang and Chen, "Source Model for transform video coder and its
  // application - Part I: Fundamental Theory", IEEE Trans. Circ.
  // Sys. for Video Tech., April 1997.
413
  if (var == 0) {
Deb Mukherjee's avatar
Deb Mukherjee committed
414
415
416
    *rate = 0;
    *dist = 0;
  } else {
417
    int d_q10, r_q10;
418
    const uint64_t xsq_q10_64 =
419
        ((((uint64_t)qstep * qstep * n) << 10) + (var >> 1)) / var;
420
421
    const int xsq_q10 = xsq_q10_64 > MAX_XSQ_Q10 ?
                        MAX_XSQ_Q10 : (int)xsq_q10_64;
422
423
424
    model_rd_norm(xsq_q10, &r_q10, &d_q10);
    *rate = (n * r_q10 + 2) >> 2;
    *dist = (var * (int64_t)d_q10 + 512) >> 10;
Deb Mukherjee's avatar
Deb Mukherjee committed
425
426
427
  }
}

428
static void model_rd_for_sb(VP9_COMP *cpi, BLOCK_SIZE bsize,
Deb Mukherjee's avatar
Deb Mukherjee committed
429
430
431
432
433
                            MACROBLOCK *x, MACROBLOCKD *xd,
                            int *out_rate_sum, int64_t *out_dist_sum) {
  // Note our transform coeffs are 8 times an orthogonal transform.
  // Hence quantizer step is also 8 times. To get effective quantizer
  // we need to divide by 8 before sending to modeling function.
434
435
436
  int i;
  int64_t rate_sum = 0;
  int64_t dist_sum = 0;
437
  const int ref = xd->mi[0]->mbmi.ref_frame[0];
438
  unsigned int sse;
Deb Mukherjee's avatar
Deb Mukherjee committed
439
440
441
442

  for (i = 0; i < MAX_MB_PLANE; ++i) {
    struct macroblock_plane *const p = &x->plane[i];
    struct macroblockd_plane *const pd = &xd->plane[i];
443
    const BLOCK_SIZE bs = get_plane_block_size(bsize, pd);
444

Deb Mukherjee's avatar
Deb Mukherjee committed
445
    (void) cpi->fn_ptr[bs].vf(p->src.buf, p->src.stride,
446
                              pd->dst.buf, pd->dst.stride, &sse);
447

448
449
    if (i == 0)
      x->pred_sse[ref] = sse;
450
451

    // Fast approximate the modelling function.
452
    if (cpi->speed > 4) {
453
      int64_t rate;
454
455
456
457
      int64_t dist;
      int64_t square_error = sse;
      int quantizer = (pd->dequant[1] >> 3);

458
459
      if (quantizer < 120)
        rate = (square_error * (280 - quantizer)) >> 8;
460
461
462
463
464
      else
        rate = 0;
      dist = (square_error * quantizer) >> 8;
      rate_sum += rate;
      dist_sum += dist;
465
466
467
    } else {
      int rate;
      int64_t dist;
468
469
      vp9_model_rd_from_var_lapndz(sse, 1 << num_pels_log2_lookup[bs],
                                   pd->dequant[1] >> 3, &rate, &dist);
470
      rate_sum += rate;
471
      dist_sum += dist;
472
    }
Deb Mukherjee's avatar
Deb Mukherjee committed
473
474
  }

475
476
  *out_rate_sum = (int)rate_sum;
  *out_dist_sum = dist_sum << 4;
Deb Mukherjee's avatar
Deb Mukherjee committed
477
478
}

479
static void model_rd_for_sb_y_tx(VP9_COMP *cpi, BLOCK_SIZE bsize,
Deb Mukherjee's avatar
Deb Mukherjee committed
480
481
482
483
                                 TX_SIZE tx_size,
                                 MACROBLOCK *x, MACROBLOCKD *xd,
                                 int *out_rate_sum, int64_t *out_dist_sum,
                                 int *out_skip) {
484
  int j, k;
485
  BLOCK_SIZE bs;
486
487
  const struct macroblock_plane *const p = &x->plane[0];
  const struct macroblockd_plane *const pd = &xd->plane[0];
488
489
  const int width = 4 * num_4x4_blocks_wide_lookup[bsize];
  const int height = 4 * num_4x4_blocks_high_lookup[bsize];
Deb Mukherjee's avatar
Deb Mukherjee committed
490
491
  int rate_sum = 0;
  int64_t dist_sum = 0;
492
  const int t = 4 << tx_size;
Deb Mukherjee's avatar
Deb Mukherjee committed
493
494
495
496
497
498
499
500
501
502
503
504

  if (tx_size == TX_4X4) {
    bs = BLOCK_4X4;
  } else if (tx_size == TX_8X8) {
    bs = BLOCK_8X8;
  } else if (tx_size == TX_16X16) {
    bs = BLOCK_16X16;
  } else if (tx_size == TX_32X32) {
    bs = BLOCK_32X32;
  } else {
    assert(0);
  }
505

Deb Mukherjee's avatar
Deb Mukherjee committed
506
  *out_skip = 1;
Jim Bankoski's avatar
Jim Bankoski committed
507
508
  for (j = 0; j < height; j += t) {
    for (k = 0; k < width; k += t) {
Deb Mukherjee's avatar
Deb Mukherjee committed
509
510
511
      int rate;
      int64_t dist;
      unsigned int sse;
512
513
514
      cpi->fn_ptr[bs].vf(&p->src.buf[j * p->src.stride + k], p->src.stride,
                         &pd->dst.buf[j * pd->dst.stride + k], pd->dst.stride,
                         &sse);
Deb Mukherjee's avatar
Deb Mukherjee committed
515
      // sse works better than var, since there is no dc prediction used
516
517
      vp9_model_rd_from_var_lapndz(sse, t * t, pd->dequant[1] >> 3,
                                   &rate, &dist);
Deb Mukherjee's avatar
Deb Mukherjee committed
518
519
520
521
522
      rate_sum += rate;
      dist_sum += dist;
      *out_skip &= (rate < 1024);
    }
  }
523

Deb Mukherjee's avatar
Deb Mukherjee committed
524
  *out_rate_sum = rate_sum;
525
  *out_dist_sum = dist_sum << 4;
Deb Mukherjee's avatar
Deb Mukherjee committed
526
527
}

528
int64_t vp9_block_error_c(const int16_t *coeff, const int16_t *dqcoeff,
529
                          intptr_t block_size, int64_t *ssz) {
Ronald S. Bultje's avatar
Ronald S. Bultje committed
530
  int i;
531
  int64_t error = 0, sqcoeff = 0;
John Koleszar's avatar
John Koleszar committed
532

533
  for (i = 0; i < block_size; i++) {
534
535
536
    const int diff = coeff[i] - dqcoeff[i];
    error +=  diff * diff;
    sqcoeff += coeff[i] * coeff[i];
John Koleszar's avatar
John Koleszar committed
537
  }
John Koleszar's avatar
John Koleszar committed
538

539
  *ssz = sqcoeff;
John Koleszar's avatar
John Koleszar committed
540
  return error;
John Koleszar's avatar
John Koleszar committed
541
542
}

543
544
545
546
547
/* The trailing '0' is a terminator which is used inside cost_coeffs() to
 * decide whether to include cost of a trailing EOB node or not (i.e. we
 * can skip this if the last coefficient in this transform block, e.g. the
 * 16th coefficient in a 4x4 block or the 64th coefficient in a 8x8 block,
 * were non-zero). */
548
static const int16_t band_counts[TX_SIZES][8] = {
549
550
551
552
  { 1, 2, 3, 4,  3,   16 - 13, 0 },
  { 1, 2, 3, 4, 11,   64 - 21, 0 },
  { 1, 2, 3, 4, 11,  256 - 21, 0 },
  { 1, 2, 3, 4, 11, 1024 - 21, 0 },
553
};
554
static INLINE int cost_coeffs(MACROBLOCK *x,
555
                              int plane, int block,
556
                              ENTROPY_CONTEXT *A, ENTROPY_CONTEXT *L,
John Koleszar's avatar
John Koleszar committed
557
                              TX_SIZE tx_size,
558
559
                              const int16_t *scan, const int16_t *nb,
                              int use_fast_coef_costing) {
560
  MACROBLOCKD *const xd = &x->e_mbd;
561
  MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
562
563
  const struct macroblock_plane *p = &x->plane[plane];
  const struct macroblockd_plane *pd = &xd->plane[plane];
564
  const PLANE_TYPE type = pd->plane_type;
565
  const int16_t *band_count = &band_counts[tx_size][1];
566
  const int eob = p->eobs[block];
Dmitry Kovalev's avatar
Dmitry Kovalev committed
567
  const int16_t *const qcoeff = BLOCK_OFFSET(p->qcoeff, block);
568
  unsigned int (*token_costs)[2][COEFF_CONTEXTS][ENTROPY_TOKENS] =
Dmitry Kovalev's avatar
Dmitry Kovalev committed
569
                   x->token_costs[tx_size][type][is_inter_block(mbmi)];
570
  uint8_t token_cache[32 * 32];
Dmitry Kovalev's avatar
Dmitry Kovalev committed
571
  int pt = combine_entropy_contexts(*A, *L);
572
  int c, cost;
573
  // Check for consistency of tx_size with mode info
574
  assert(type == PLANE_TYPE_Y ? mbmi->tx_size == tx_size
Dmitry Kovalev's avatar
Dmitry Kovalev committed
575
                              : get_uv_tx_size(mbmi) == tx_size);
576

577
578
  if (eob == 0) {
    // single eob token
579
    cost = token_costs[0][0][pt][EOB_TOKEN];
580
    c = 0;
581
  } else {
582
    int band_left = *band_count++;
583
584

    // dc token
Dmitry Kovalev's avatar
Dmitry Kovalev committed
585
    int v = qcoeff[0];
586
    int prev_t = vp9_dct_value_tokens_ptr[v].token;
587
    cost = (*token_costs)[0][pt][prev_t] + vp9_dct_value_cost_ptr[v];
588
    token_cache[0] = vp9_pt_energy_class[prev_t];
589
    ++token_costs;
590
591
592
593

    // ac tokens
    for (c = 1; c < eob; c++) {
      const int rc = scan[c];
594
      int t;
595

Dmitry Kovalev's avatar
Dmitry Kovalev committed
596
      v = qcoeff[rc];
597
      t = vp9_dct_value_tokens_ptr[v].token;
598
599
600
      if (use_fast_coef_costing) {
        cost += (*token_costs)[!prev_t][!prev_t][t] + vp9_dct_value_cost_ptr[v];
      } else {
601
        pt = get_coef_context(nb, token_cache, c);
602
        cost += (*token_costs)[!prev_t][pt][t] + vp9_dct_value_cost_ptr[v];
603
        token_cache[rc] = vp9_pt_energy_class[t];
604
      }
605
      prev_t = t;
606
      if (!--band_left) {
607
608
        band_left = *band_count++;
        ++token_costs;
609
      }
610
    }
611
612

    // eob token
613
    if (band_left) {
614
615
616
      if (use_fast_coef_costing) {
        cost += (*token_costs)[0][!prev_t][EOB_TOKEN];
      } else {
617
        pt = get_coef_context(nb, token_cache, c);
618
619
        cost += (*token_costs)[0][pt][EOB_TOKEN];
      }
620
    }
621
622
  }

623
  // is eob first coefficient;
624
  *A = *L = (c > 0);
625

626
627
  return cost;
}
Alex Converse's avatar
Alex Converse committed
628
629
static void dist_block(int plane, int block, TX_SIZE tx_size,
                       struct rdcost_block_args* args) {
630
  const int ss_txfrm_size = tx_size << 1;
Deb Mukherjee's avatar
Deb Mukherjee committed
631
632
  MACROBLOCK* const x = args->x;
  MACROBLOCKD* const xd = &x->e_mbd;
633
634
  const struct macroblock_plane *const p = &x->plane[plane];
  const struct macroblockd_plane *const pd = &xd->plane[plane];
Deb Mukherjee's avatar
Deb Mukherjee committed
635
  int64_t this_sse;
Alex Converse's avatar
Alex Converse committed
636
  int shift = tx_size == TX_32X32 ? 0 : 2;
637
638
  int16_t *const coeff = BLOCK_OFFSET(p->coeff, block);
  int16_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
639
640
641
  args->dist = vp9_block_error(coeff, dqcoeff, 16 << ss_txfrm_size,
                               &this_sse) >> shift;
  args->sse  = this_sse >> shift;
642

643
  if (x->skip_encode && !is_inter_block(&xd->mi[0]->mbmi)) {
644
645
    // TODO(jingning): tune the model to better capture the distortion.
    int64_t p = (pd->dequant[1] * pd->dequant[1] *
646
                    (1 << ss_txfrm_size)) >> (shift + 2);
647
648
    args->dist += (p >> 4);
    args->sse  += p;
649
  }
Deb Mukherjee's avatar
Deb Mukherjee committed
650
651
}

652
static void rate_block(int plane, int block, BLOCK_SIZE plane_bsize,
Alex Converse's avatar
Alex Converse committed
653
                       TX_SIZE tx_size, struct rdcost_block_args* args) {
Deb Mukherjee's avatar
Deb Mukherjee committed
654
  int x_idx, y_idx;
Alex Converse's avatar
Alex Converse committed
655
  txfrm_block_to_raster_xy(plane_bsize, tx_size, block, &x_idx, &y_idx);
Deb Mukherjee's avatar
Deb Mukherjee committed
656

657
  args->rate = cost_coeffs(args->x, plane, block, args->t_above + x_idx,
Alex Converse's avatar
Alex Converse committed
658
                           args->t_left + y_idx, tx_size,
659
660
                           args->so->scan, args->so->neighbors,
                           args->use_fast_coef_costing);
Deb Mukherjee's avatar
Deb Mukherjee committed
661
662
}

663
664
static void block_rd_txfm(int plane, int block, BLOCK_SIZE plane_bsize,
                          TX_SIZE tx_size, void *arg) {
Deb Mukherjee's avatar
Deb Mukherjee committed
665
666
667
  struct rdcost_block_args *args = arg;
  MACROBLOCK *const x = args->x;
  MACROBLOCKD *const xd = &x->e_mbd;
668
  MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
669
  int64_t rd1, rd2, rd;
Deb Mukherjee's avatar
Deb Mukherjee committed
670

671
672
673
  if (args->skip)
    return;

674
  if (!is_inter_block(mbmi))
675
    vp9_encode_block_intra(x, plane, block, plane_bsize, tx_size, &mbmi->skip);
Deb Mukherjee's avatar
Deb Mukherjee committed
676
  else
677
    vp9_xform_quant(x, plane, block, plane_bsize, tx_size);
Deb Mukherjee's avatar
Deb Mukherjee committed
678

679
680
  dist_block(plane, block, tx_size, args);
  rate_block(plane, block, plane_bsize, tx_size, args);
681
682
  rd1 = RDCOST(x->rdmult, x->rddiv, args->rate, args->dist);
  rd2 = RDCOST(x->rdmult, x->rddiv, 0, args->sse);
683
684

  // TODO(jingning): temporarily enabled only for luma component
685
  rd = MIN(rd1, rd2);
Yaowu Xu's avatar
Yaowu Xu committed
686
  if (plane == 0)
687
    x->zcoeff_blk[tx_size][block] = !x->plane[plane].eobs[block] ||
Yaowu Xu's avatar
Yaowu Xu committed
688
                                    (rd1 > rd2 && !xd->lossless);
689

690
691
692
  args->this_rate += args->rate;
  args->this_dist += args->dist;
  args->this_sse  += args->sse;
693
694
695
696
697
698
  args->this_rd += rd;

  if (args->this_rd > args->best_rd) {
    args->skip = 1;
    return;
  }
Deb Mukherjee's avatar
Deb Mukherjee committed
699
700
}

701
702
703
704
705
706
707
708
709
710
void vp9_get_entropy_contexts(BLOCK_SIZE bsize, TX_SIZE tx_size,
                              const struct macroblockd_plane *pd,
                              ENTROPY_CONTEXT t_above[16],
                              ENTROPY_CONTEXT t_left[16]) {
  const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd);
  const int num_4x4_w = num_4x4_blocks_wide_lookup[plane_bsize];
  const int num_4x4_h = num_4x4_blocks_high_lookup[plane_bsize];
  const ENTROPY_CONTEXT *const above = pd->above_context;
  const ENTROPY_CONTEXT *const left = pd->left_context;

711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
  int i;
  switch (tx_size) {
    case TX_4X4:
      vpx_memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
      vpx_memcpy(t_left, left, sizeof(ENTROPY_CONTEXT) * num_4x4_h);
      break;
    case TX_8X8:
      for (i = 0; i < num_4x4_w; i += 2)
        t_above[i] = !!*(const uint16_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 2)
        t_left[i] = !!*(const uint16_t *)&left[i];
      break;
    case TX_16X16:
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
    case TX_32X32:
      for (i = 0; i < num_4x4_w; i += 8)
        t_above[i] = !!*(const uint64_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 8)
        t_left[i] = !!*(const uint64_t *)&left[i];
      break;
    default:
James Zern's avatar
James Zern committed
736
      assert(0 && "Invalid transform size.");
737
738
739
  }
}

740
static void txfm_rd_in_plane(MACROBLOCK *x,
741
742
743
                             int *rate, int64_t *distortion,
                             int *skippable, int64_t *sse,
                             int64_t ref_best_rd, int plane,
744
745
                             BLOCK_SIZE bsize, TX_SIZE tx_size,
                             int use_fast_coef_casting) {
Deb Mukherjee's avatar
Deb Mukherjee committed
746
  MACROBLOCKD *const xd = &x->e_mbd;
747
  const struct macroblockd_plane *const pd = &xd->plane[plane];
748
749
750
  struct rdcost_block_args args = { 0 };
  args.x = x;
  args.best_rd = ref_best_rd;
751
  args.use_fast_coef_costing = use_fast_coef_casting;
752

753
  if (plane == 0)
754
    xd->mi[0]->mbmi.tx_size = tx_size;
755

756
  vp9_get_entropy_contexts(bsize, tx_size, pd, args.t_above, args.t_left);
757

758
  args.so = get_scan(xd, tx_size, pd->plane_type, 0);
Deb Mukherjee's avatar
Deb Mukherjee committed
759

760
  vp9_foreach_transformed_block_in_plane(xd, bsize, plane,
761
762
                                         block_rd_txfm, &args);
  if (args.skip) {
763
764
765
766
767
    *rate       = INT_MAX;
    *distortion = INT64_MAX;
    *sse        = INT64_MAX;
    *skippable  = 0;
  } else {
768
769
770
    *distortion = args.this_dist;
    *rate       = args.this_rate;
    *sse        = args.this_sse;
771
    *skippable  = vp9_is_skippable_in_plane(x, bsize, plane);
772
  }
Deb Mukherjee's avatar
Deb Mukherjee committed
773
774
775
776
777
}

static void choose_largest_txfm_size(VP9_COMP *cpi, MACROBLOCK *x,
                                     int *rate, int64_t *distortion,
                                     int *skip, int64_t *sse,
778
                                     int64_t ref_best_rd,
779
                                     BLOCK_SIZE bs) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
780
  const TX_SIZE max_tx_size = max_txsize_lookup[bs];
Deb Mukherjee's avatar
Deb Mukherjee committed
781
  VP9_COMMON *const cm = &cpi->common;
Yaowu Xu's avatar
Yaowu Xu committed
782
  const TX_SIZE largest_tx_size = tx_mode_to_biggest_tx_size[cm->tx_mode];
Deb Mukherjee's avatar
Deb Mukherjee committed
783
  MACROBLOCKD *const xd = &x->e_mbd;
784
  MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
Yaowu Xu's avatar
Yaowu Xu committed
785
786
787

  mbmi->tx_size = MIN(max_tx_size, largest_tx_size);

Alex Converse's avatar
Alex Converse committed
788
  txfm_rd_in_plane(x, rate, distortion, skip,
789
                   &sse[mbmi->tx_size], ref_best_rd, 0, bs,
790
                   mbmi->tx_size, cpi->sf.use_fast_coef_costing);
Dmitry Kovalev's avatar
Dmitry Kovalev committed
791
  cpi->tx_stepdown_count[0]++;
Deb Mukherjee's avatar
Deb Mukherjee committed
792
793
}

794
static void choose_txfm_size_from_rd(VP9_COMP *cpi, MACROBLOCK *x,
795
                                     int (*r)[2], int *rate,
Ronald S. Bultje's avatar
Ronald S. Bultje committed
796
                                     int64_t *d, int64_t *distortion,
797
                                     int *s, int *skip,
798
                                     int64_t tx_cache[TX_MODES],
799
                                     BLOCK_SIZE bs) {
800
  const TX_SIZE max_tx_size = max_txsize_lookup[bs];
801
802
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
803
  MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
804
  vp9_prob skip_prob = vp9_get_skip_prob(cm, xd);
805
806
807
808
  int64_t rd[TX_SIZES][2] = {{INT64_MAX, INT64_MAX},
                             {INT64_MAX, INT64_MAX},
                             {INT64_MAX, INT64_MAX},
                             {INT64_MAX, INT64_MAX}};
809
  int n, m;
810
  int s0, s1;
811
812
813
  const TX_SIZE max_mode_tx_size = tx_mode_to_biggest_tx_size[cm->tx_mode];
  int64_t best_rd = INT64_MAX;
  TX_SIZE best_tx = TX_4X4;
814

815
  const vp9_prob *tx_probs = get_tx_probs2(max_tx_size, xd, &cm->fc.tx_probs);
816
817
818
  assert(skip_prob > 0);
  s0 = vp9_cost_bit(skip_prob, 0);
  s1 = vp9_cost_bit(skip_prob, 1);
819

820
  for (n = TX_4X4; n <= max_tx_size; n++) {
821
822
823
824
825
826
827
828
829
    r[n][1] = r[n][0];
    if (r[n][0] < INT_MAX) {
      for (m = 0; m <= n - (n == max_tx_size); m++) {
        if (m == n)
          r[n][1] += vp9_cost_zero(tx_probs[m]);
        else
          r[n][1] += vp9_cost_one(tx_probs[m]);
      }
    }
830
831
    if (d[n] == INT64_MAX) {
      rd[n][0] = rd[n][1] = INT64_MAX;
832
    } else if (s[n]) {
833
834
835
836
      rd[n][0] = rd[n][1] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
    } else {
      rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0] + s0, d[n]);
      rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1] + s0, d[n]);
837
    }
838
839
840
841
842

    if (rd[n][1] < best_rd) {
      best_tx = n;
      best_rd = rd[n][1];
    }
843
  }
844
845
  mbmi->tx_size = cm->tx_mode == TX_MODE_SELECT ?
                      best_tx : MIN(max_tx_size, max_mode_tx_size);
846
847


848
849
850
  *distortion = d[mbmi->tx_size];
  *rate       = r[mbmi->tx_size][cm->tx_mode == TX_MODE_SELECT];
  *skip       = s[mbmi->tx_size];
851

852
853
854
855
  tx_cache[ONLY_4X4] = rd[TX_4X4][0];
  tx_cache[ALLOW_8X8] = rd[TX_8X8][0];
  tx_cache[ALLOW_16X16] = rd[MIN(max_tx_size, TX_16X16)][0];
  tx_cache[ALLOW_32X32] = rd[MIN(max_tx_size, TX_32X32)][0];
856

857
858
  if (max_tx_size == TX_32X32 && best_tx == TX_32X32) {
    tx_cache[TX_MODE_SELECT] = rd[TX_32X32][1];
Dmitry Kovalev's avatar
Dmitry Kovalev committed
859
    cpi->tx_stepdown_count[0]++;
860
861
  } else if (max_tx_size >= TX_16X16 && best_tx == TX_16X16) {
    tx_cache[TX_MODE_SELECT] = rd[TX_16X16][1];
Dmitry Kovalev's avatar
Dmitry Kovalev committed
862
    cpi->tx_stepdown_count[max_tx_size - TX_16X16]++;
Deb Mukherjee's avatar
Deb Mukherjee committed
863
  } else if (rd[TX_8X8][1] < rd[TX_4X4][1]) {
864
    tx_cache[TX_MODE_SELECT] = rd[TX_8X8][1];
Dmitry Kovalev's avatar
Dmitry Kovalev committed
865
    cpi->tx_stepdown_count[max_tx_size - TX_8X8]++;
Deb Mukherjee's avatar
Deb Mukherjee committed
866
  } else {
867
    tx_cache[TX_MODE_SELECT] = rd[TX_4X4][1];
Dmitry Kovalev's avatar
Dmitry Kovalev committed
868
    cpi->tx_stepdown_count[max_tx_size - TX_4X4]++;
869
  }
Deb Mukherjee's avatar
Deb Mukherjee committed
870
}
871

872
873
874
875
876
static int64_t scaled_rd_cost(int rdmult, int rddiv,
                              int rate, int64_t dist, double scale) {
  return (int64_t) (RDCOST(rdmult, rddiv, rate, dist) * scale);
}

Deb Mukherjee's avatar
Deb Mukherjee committed
877
878
879
880
static void choose_txfm_size_from_modelrd(VP9_COMP *cpi, MACROBLOCK *x,
                                          int (*r)[2], int *rate,
                                          int64_t *d, int64_t *distortion,
                                          int *s, int *skip, int64_t *sse,
881
                                          int64_t ref_best_rd,
882
                                          BLOCK_SIZE bs) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
883
  const TX_SIZE max_tx_size = max_txsize_lookup[bs];
Deb Mukherjee's avatar
Deb Mukherjee committed
884
885
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;