vp9_rdopt.c 147 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5
6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9
10
11
12
13
14
 */

#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
15

16
#include "vp9/common/vp9_pragmas.h"
17
18
19
20
21
#include "vp9/encoder/vp9_tokenize.h"
#include "vp9/encoder/vp9_treewriter.h"
#include "vp9/encoder/vp9_onyx_int.h"
#include "vp9/encoder/vp9_modecosts.h"
#include "vp9/encoder/vp9_encodeintra.h"
22
23
24
25
26
#include "vp9/common/vp9_entropymode.h"
#include "vp9/common/vp9_reconinter.h"
#include "vp9/common/vp9_reconintra.h"
#include "vp9/common/vp9_findnearmv.h"
#include "vp9/common/vp9_quant_common.h"
27
28
29
30
31
32
#include "vp9/encoder/vp9_encodemb.h"
#include "vp9/encoder/vp9_quantize.h"
#include "vp9/encoder/vp9_variance.h"
#include "vp9/encoder/vp9_mcomp.h"
#include "vp9/encoder/vp9_rdopt.h"
#include "vp9/encoder/vp9_ratectrl.h"
John Koleszar's avatar
John Koleszar committed
33
#include "vpx_mem/vpx_mem.h"
34
35
36
37
38
#include "vp9/common/vp9_systemdependent.h"
#include "vp9/encoder/vp9_encodemv.h"
#include "vp9/common/vp9_seg_common.h"
#include "vp9/common/vp9_pred_common.h"
#include "vp9/common/vp9_entropy.h"
39
#include "vp9_rtcd.h"
40
#include "vp9/common/vp9_mvref_common.h"
Ronald S. Bultje's avatar
Ronald S. Bultje committed
41
#include "vp9/common/vp9_common.h"
Paul Wilkins's avatar
Paul Wilkins committed
42

43
44
#define INVALID_MV 0x80008000

45
46
47
/* Factor to weigh the rate for switchable interp filters */
#define SWITCHABLE_INTERP_RATE_FACTOR 1

48
49
50
DECLARE_ALIGNED(16, extern const uint8_t,
                vp9_pt_energy_class[MAX_ENTROPY_TOKENS]);

Ronald S. Bultje's avatar
Ronald S. Bultje committed
51
#define I4X4_PRED 0x8000
Ronald S. Bultje's avatar
Ronald S. Bultje committed
52
#define SPLITMV 0x10000
Ronald S. Bultje's avatar
Ronald S. Bultje committed
53

54
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
55
  {NEARESTMV, LAST_FRAME,   NONE},
56
57
  {DC_PRED,   INTRA_FRAME,  NONE},

Paul Wilkins's avatar
Paul Wilkins committed
58
59
60
61
  {NEARESTMV, ALTREF_FRAME, NONE},
  {NEARESTMV, GOLDEN_FRAME, NONE},
  {NEWMV,     LAST_FRAME,   NONE},
  {NEARESTMV, LAST_FRAME,   ALTREF_FRAME},
62
  {NEARMV,    LAST_FRAME,   NONE},
Paul Wilkins's avatar
Paul Wilkins committed
63
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},
John Koleszar's avatar
John Koleszar committed
64

Paul Wilkins's avatar
Paul Wilkins committed
65
66
  {NEWMV,     GOLDEN_FRAME, NONE},
  {NEWMV,     ALTREF_FRAME, NONE},
67
  {NEARMV,    ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
68

69
70
  {TM_PRED,   INTRA_FRAME,  NONE},

Paul Wilkins's avatar
Paul Wilkins committed
71
72
73
74
75
  {NEARMV,    LAST_FRAME,   ALTREF_FRAME},
  {NEWMV,     LAST_FRAME,   ALTREF_FRAME},
  {NEARMV,    GOLDEN_FRAME, NONE},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME},
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME},
John Koleszar's avatar
John Koleszar committed
76

77
78
79
  {SPLITMV,   LAST_FRAME,   NONE},
  {SPLITMV,   GOLDEN_FRAME, NONE},
  {SPLITMV,   ALTREF_FRAME, NONE},
Paul Wilkins's avatar
Paul Wilkins committed
80
81
  {SPLITMV,   LAST_FRAME,   ALTREF_FRAME},
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME},
82

Paul Wilkins's avatar
Paul Wilkins committed
83
84
85
  {ZEROMV,    LAST_FRAME,   NONE},
  {ZEROMV,    GOLDEN_FRAME, NONE},
  {ZEROMV,    ALTREF_FRAME, NONE},
Ronald S. Bultje's avatar
Ronald S. Bultje committed
86
  {ZEROMV,    LAST_FRAME,   ALTREF_FRAME},
John Koleszar's avatar
John Koleszar committed
87
  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME},
88

Paul Wilkins's avatar
Paul Wilkins committed
89
90
91
92
  {I4X4_PRED, INTRA_FRAME,  NONE},
  {H_PRED,    INTRA_FRAME,  NONE},
  {V_PRED,    INTRA_FRAME,  NONE},
  {D135_PRED, INTRA_FRAME,  NONE},
Dmitry Kovalev's avatar
Dmitry Kovalev committed
93
  {D207_PRED,  INTRA_FRAME,  NONE},
Paul Wilkins's avatar
Paul Wilkins committed
94
95
96
97
  {D153_PRED, INTRA_FRAME,  NONE},
  {D63_PRED,  INTRA_FRAME,  NONE},
  {D117_PRED, INTRA_FRAME,  NONE},
  {D45_PRED,  INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
98
99
};

100
101
102
103
// The baseline rd thresholds for breaking out of the rd loop for
// certain modes are assumed to be based on 8x8 blocks.
// This table is used to correct for blocks size.
// The factors here are << 2 (2 = x0.5, 32 = x8 etc).
104
static int rd_thresh_block_size_factor[BLOCK_SIZES] =
105
106
  {2, 3, 3, 4, 6, 6, 8, 12, 12, 16, 24, 24, 32};

107
108
#define MAX_RD_THRESH_FACT 64
#define RD_THRESH_INC 1
109

110
static void fill_token_costs(vp9_coeff_cost *c,
111
                             vp9_coeff_probs_model (*p)[BLOCK_TYPES]) {
112
  int i, j, k, l;
113
114
115
116
117
118
119
120
  TX_SIZE t;
  for (t = TX_4X4; t <= TX_32X32; t++)
    for (i = 0; i < BLOCK_TYPES; i++)
      for (j = 0; j < REF_TYPES; j++)
        for (k = 0; k < COEF_BANDS; k++)
          for (l = 0; l < PREV_COEF_CONTEXTS; l++) {
            vp9_prob probs[ENTROPY_NODES];
            vp9_model_to_full_probs(p[t][i][j][k][l], probs);
121
            vp9_cost_tokens((int *)c[t][i][j][k][0][l], probs,
122
                            vp9_coef_tree);
123
            vp9_cost_tokens_skip((int *)c[t][i][j][k][1][l], probs,
124
                                 vp9_coef_tree);
125
126
            assert(c[t][i][j][k][0][l][DCT_EOB_TOKEN] ==
                   c[t][i][j][k][1][l][DCT_EOB_TOKEN]);
127
          }
128
129
}

130
131
132
133
134
135
static const int rd_iifactor[32] = {
  4, 4, 3, 2, 1, 0, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 0,
};
John Koleszar's avatar
John Koleszar committed
136

137
// 3* dc_qlookup[Q]*dc_qlookup[Q];
138

139
/* values are now correlated to quantizer */
Paul Wilkins's avatar
Paul Wilkins committed
140
141
142
static int sad_per_bit16lut[QINDEX_RANGE];
static int sad_per_bit4lut[QINDEX_RANGE];

143
void vp9_init_me_luts() {
John Koleszar's avatar
John Koleszar committed
144
145
146
147
148
149
150
  int i;

  // Initialize the sad lut tables using a formulaic calculation for now
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < QINDEX_RANGE; i++) {
    sad_per_bit16lut[i] =
151
      (int)((0.0418 * vp9_convert_qindex_to_q(i)) + 2.4107);
152
    sad_per_bit4lut[i] = (int)(0.063 * vp9_convert_qindex_to_q(i) + 2.742);
John Koleszar's avatar
John Koleszar committed
153
  }
Paul Wilkins's avatar
Paul Wilkins committed
154
}
John Koleszar's avatar
John Koleszar committed
155

156
static int compute_rd_mult(int qindex) {
157
  const int q = vp9_dc_quant(qindex, 0);
158
  return (11 * q * q) >> 2;
159
160
}

Dmitry Kovalev's avatar
Dmitry Kovalev committed
161
162
163
void vp9_initialize_me_consts(VP9_COMP *cpi, int qindex) {
  cpi->mb.sadperbit16 = sad_per_bit16lut[qindex];
  cpi->mb.sadperbit4 = sad_per_bit4lut[qindex];
John Koleszar's avatar
John Koleszar committed
164
165
}

166

Dmitry Kovalev's avatar
Dmitry Kovalev committed
167
void vp9_initialize_rd_consts(VP9_COMP *cpi, int qindex) {
168
  int q, i, bsize;
John Koleszar's avatar
John Koleszar committed
169

170
  vp9_clear_system_state();  // __asm emms;
John Koleszar's avatar
John Koleszar committed
171

John Koleszar's avatar
John Koleszar committed
172
173
174
175
  // Further tests required to see if optimum is different
  // for key frames, golden frames and arf frames.
  // if (cpi->common.refresh_golden_frame ||
  //     cpi->common.refresh_alt_ref_frame)
176
  qindex = clamp(qindex, 0, MAXQ);
177

Dmitry Kovalev's avatar
Dmitry Kovalev committed
178
  cpi->RDMULT = compute_rd_mult(qindex);
John Koleszar's avatar
John Koleszar committed
179
180
181
182
183
  if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    if (cpi->twopass.next_iiratio > 31)
      cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
    else
      cpi->RDMULT +=
Dmitry Kovalev's avatar
Dmitry Kovalev committed
184
          (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
John Koleszar's avatar
John Koleszar committed
185
  }
186
  cpi->mb.errorperbit = cpi->RDMULT >> 6;
John Koleszar's avatar
John Koleszar committed
187
  cpi->mb.errorperbit += (cpi->mb.errorperbit == 0);
188

189
  vp9_set_speed_features(cpi);
John Koleszar's avatar
John Koleszar committed
190

Dmitry Kovalev's avatar
Dmitry Kovalev committed
191
192
  q = (int)pow(vp9_dc_quant(qindex, 0) >> 2, 1.25);
  q <<= 2;
John Koleszar's avatar
John Koleszar committed
193
194
  if (q < 8)
    q = 8;
195

John Koleszar's avatar
John Koleszar committed
196
197
198
  if (cpi->RDMULT > 1000) {
    cpi->RDDIV = 1;
    cpi->RDMULT /= 100;
John Koleszar's avatar
John Koleszar committed
199

200
    for (bsize = 0; bsize < BLOCK_SIZES; ++bsize) {
201
202
203
204
205
206
207
208
209
210
211
212
213
      for (i = 0; i < MAX_MODES; ++i) {
        // Threshold here seem unecessarily harsh but fine given actual
        // range of values used for cpi->sf.thresh_mult[]
        int thresh_max = INT_MAX / (q * rd_thresh_block_size_factor[bsize]);

        // *4 relates to the scaling of rd_thresh_block_size_factor[]
        if ((int64_t)cpi->sf.thresh_mult[i] < thresh_max) {
          cpi->rd_threshes[bsize][i] =
            cpi->sf.thresh_mult[i] * q *
            rd_thresh_block_size_factor[bsize] / (4 * 100);
        } else {
          cpi->rd_threshes[bsize][i] = INT_MAX;
        }
John Koleszar's avatar
John Koleszar committed
214
      }
John Koleszar's avatar
John Koleszar committed
215
    }
John Koleszar's avatar
John Koleszar committed
216
217
  } else {
    cpi->RDDIV = 100;
John Koleszar's avatar
John Koleszar committed
218

219
    for (bsize = 0; bsize < BLOCK_SIZES; ++bsize) {
220
221
222
223
224
225
226
227
228
229
230
231
      for (i = 0; i < MAX_MODES; i++) {
        // Threshold here seem unecessarily harsh but fine given actual
        // range of values used for cpi->sf.thresh_mult[]
        int thresh_max = INT_MAX / (q * rd_thresh_block_size_factor[bsize]);

        if (cpi->sf.thresh_mult[i] < thresh_max) {
          cpi->rd_threshes[bsize][i] =
            cpi->sf.thresh_mult[i] * q *
            rd_thresh_block_size_factor[bsize] / 4;
        } else {
          cpi->rd_threshes[bsize][i] = INT_MAX;
        }
John Koleszar's avatar
John Koleszar committed
232
      }
John Koleszar's avatar
John Koleszar committed
233
    }
John Koleszar's avatar
John Koleszar committed
234
  }
John Koleszar's avatar
John Koleszar committed
235

236
  fill_token_costs(cpi->mb.token_costs, cpi->common.fc.coef_probs);
237

238
  for (i = 0; i < NUM_PARTITION_CONTEXTS; i++)
239
    vp9_cost_tokens(cpi->mb.partition_cost[i],
240
                    cpi->common.fc.partition_prob[cpi->common.frame_type][i],
241
242
                    vp9_partition_tree);

John Koleszar's avatar
John Koleszar committed
243
  /*rough estimate for costing*/
244
  vp9_init_mode_costs(cpi);
John Koleszar's avatar
John Koleszar committed
245

246
  if (cpi->common.frame_type != KEY_FRAME) {
247
    vp9_build_nmv_cost_table(
248
249
250
251
252
        cpi->mb.nmvjointcost,
        cpi->mb.e_mbd.allow_high_precision_mv ?
        cpi->mb.nmvcost_hp : cpi->mb.nmvcost,
        &cpi->common.fc.nmvc,
        cpi->mb.e_mbd.allow_high_precision_mv, 1, 1);
253
254
255
256
257
258

    for (i = 0; i < INTER_MODE_CONTEXTS; i++) {
      MB_PREDICTION_MODE m;

      for (m = NEARESTMV; m < MB_MODE_COUNT; m++)
        cpi->mb.inter_mode_cost[i][m - NEARESTMV] =
259
            cost_token(vp9_inter_mode_tree,
260
                       cpi->common.fc.inter_mode_probs[i],
261
                       vp9_inter_mode_encodings - NEARESTMV + m);
262
    }
263
  }
John Koleszar's avatar
John Koleszar committed
264
265
}

Yaowu Xu's avatar
Yaowu Xu committed
266
static INLINE void linear_interpolate2(double x, int ntab, int inv_step,
267
268
                                       const double *tab1, const double *tab2,
                                       double *v1, double *v2) {
Deb Mukherjee's avatar
Deb Mukherjee committed
269
270
271
  double y = x * inv_step;
  int d = (int) y;
  if (d >= ntab - 1) {
272
273
    *v1 = tab1[ntab - 1];
    *v2 = tab2[ntab - 1];
Deb Mukherjee's avatar
Deb Mukherjee committed
274
275
  } else {
    double a = y - d;
276
277
    *v1 = tab1[d] * (1 - a) + tab1[d + 1] * a;
    *v2 = tab2[d] * (1 - a) + tab2[d + 1] * a;
Deb Mukherjee's avatar
Deb Mukherjee committed
278
279
280
  }
}

281
282
283
284
285
static void model_rd_norm(double x, double *R, double *D) {
  static const int inv_tab_step = 8;
  static const int tab_size = 120;
  // NOTE: The tables below must be of the same size
  //
Deb Mukherjee's avatar
Deb Mukherjee committed
286
  // Normalized rate
287
  // This table models the rate for a Laplacian source
Deb Mukherjee's avatar
Deb Mukherjee committed
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
  // source with given variance when quantized with a uniform quantizer
  // with given stepsize. The closed form expression is:
  // Rn(x) = H(sqrt(r)) + sqrt(r)*[1 + H(r)/(1 - r)],
  // where r = exp(-sqrt(2) * x) and x = qpstep / sqrt(variance),
  // and H(x) is the binary entropy function.
  static const double rate_tab[] = {
    64.00, 4.944, 3.949, 3.372, 2.966, 2.655, 2.403, 2.194,
    2.014, 1.858, 1.720, 1.596, 1.485, 1.384, 1.291, 1.206,
    1.127, 1.054, 0.986, 0.923, 0.863, 0.808, 0.756, 0.708,
    0.662, 0.619, 0.579, 0.541, 0.506, 0.473, 0.442, 0.412,
    0.385, 0.359, 0.335, 0.313, 0.291, 0.272, 0.253, 0.236,
    0.220, 0.204, 0.190, 0.177, 0.165, 0.153, 0.142, 0.132,
    0.123, 0.114, 0.106, 0.099, 0.091, 0.085, 0.079, 0.073,
    0.068, 0.063, 0.058, 0.054, 0.050, 0.047, 0.043, 0.040,
    0.037, 0.034, 0.032, 0.029, 0.027, 0.025, 0.023, 0.022,
    0.020, 0.019, 0.017, 0.016, 0.015, 0.014, 0.013, 0.012,
    0.011, 0.010, 0.009, 0.008, 0.008, 0.007, 0.007, 0.006,
    0.006, 0.005, 0.005, 0.005, 0.004, 0.004, 0.004, 0.003,
    0.003, 0.003, 0.003, 0.002, 0.002, 0.002, 0.002, 0.002,
    0.002, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001,
    0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.000,
  };
  // Normalized distortion
311
  // This table models the normalized distortion for a Laplacian source
Deb Mukherjee's avatar
Deb Mukherjee committed
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
  // source with given variance when quantized with a uniform quantizer
  // with given stepsize. The closed form expression is:
  // Dn(x) = 1 - 1/sqrt(2) * x / sinh(x/sqrt(2))
  // where x = qpstep / sqrt(variance)
  // Note the actual distortion is Dn * variance.
  static const double dist_tab[] = {
    0.000, 0.001, 0.005, 0.012, 0.021, 0.032, 0.045, 0.061,
    0.079, 0.098, 0.119, 0.142, 0.166, 0.190, 0.216, 0.242,
    0.269, 0.296, 0.324, 0.351, 0.378, 0.405, 0.432, 0.458,
    0.484, 0.509, 0.534, 0.557, 0.580, 0.603, 0.624, 0.645,
    0.664, 0.683, 0.702, 0.719, 0.735, 0.751, 0.766, 0.780,
    0.794, 0.807, 0.819, 0.830, 0.841, 0.851, 0.861, 0.870,
    0.878, 0.886, 0.894, 0.901, 0.907, 0.913, 0.919, 0.925,
    0.930, 0.935, 0.939, 0.943, 0.947, 0.951, 0.954, 0.957,
    0.960, 0.963, 0.966, 0.968, 0.971, 0.973, 0.975, 0.976,
    0.978, 0.980, 0.981, 0.982, 0.984, 0.985, 0.986, 0.987,
    0.988, 0.989, 0.990, 0.990, 0.991, 0.992, 0.992, 0.993,
    0.993, 0.994, 0.994, 0.995, 0.995, 0.996, 0.996, 0.996,
    0.996, 0.997, 0.997, 0.997, 0.997, 0.998, 0.998, 0.998,
    0.998, 0.998, 0.998, 0.999, 0.999, 0.999, 0.999, 0.999,
    0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 1.000,
  };
334
335
336
337
338
  /*
  assert(sizeof(rate_tab) == tab_size * sizeof(rate_tab[0]);
  assert(sizeof(dist_tab) == tab_size * sizeof(dist_tab[0]);
  assert(sizeof(rate_tab) == sizeof(dist_tab));
  */
Deb Mukherjee's avatar
Deb Mukherjee committed
339
  assert(x >= 0.0);
340
341
  linear_interpolate2(x, tab_size, inv_tab_step,
                      rate_tab, dist_tab, R, D);
Deb Mukherjee's avatar
Deb Mukherjee committed
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
}

static void model_rd_from_var_lapndz(int var, int n, int qstep,
                                     int *rate, int64_t *dist) {
  // This function models the rate and distortion for a Laplacian
  // source with given variance when quantized with a uniform quantizer
  // with given stepsize. The closed form expressions are in:
  // Hang and Chen, "Source Model for transform video coder and its
  // application - Part I: Fundamental Theory", IEEE Trans. Circ.
  // Sys. for Video Tech., April 1997.
  vp9_clear_system_state();
  if (var == 0 || n == 0) {
    *rate = 0;
    *dist = 0;
  } else {
    double D, R;
    double s2 = (double) var / n;
    double x = qstep / sqrt(s2);
360
361
362
    model_rd_norm(x, &R, &D);
    *rate = ((n << 8) * R + 0.5);
    *dist = (var * D + 0.5);
Deb Mukherjee's avatar
Deb Mukherjee committed
363
364
365
366
  }
  vp9_clear_system_state();
}

367
static void model_rd_for_sb(VP9_COMP *cpi, BLOCK_SIZE bsize,
Deb Mukherjee's avatar
Deb Mukherjee committed
368
369
370
371
372
373
374
375
376
377
                            MACROBLOCK *x, MACROBLOCKD *xd,
                            int *out_rate_sum, int64_t *out_dist_sum) {
  // Note our transform coeffs are 8 times an orthogonal transform.
  // Hence quantizer step is also 8 times. To get effective quantizer
  // we need to divide by 8 before sending to modeling function.
  int i, rate_sum = 0, dist_sum = 0;

  for (i = 0; i < MAX_MB_PLANE; ++i) {
    struct macroblock_plane *const p = &x->plane[i];
    struct macroblockd_plane *const pd = &xd->plane[i];
378
    const BLOCK_SIZE bs = get_plane_block_size(bsize, pd);
Deb Mukherjee's avatar
Deb Mukherjee committed
379
380
381
382
383
384
    unsigned int sse;
    int rate;
    int64_t dist;
    (void) cpi->fn_ptr[bs].vf(p->src.buf, p->src.stride,
                              pd->dst.buf, pd->dst.stride, &sse);
    // sse works better than var, since there is no dc prediction used
Dmitry Kovalev's avatar
Dmitry Kovalev committed
385
    model_rd_from_var_lapndz(sse, 1 << num_pels_log2_lookup[bs],
386
                             pd->dequant[1] >> 3, &rate, &dist);
Deb Mukherjee's avatar
Deb Mukherjee committed
387
388
389
390
391
392
393
394
395

    rate_sum += rate;
    dist_sum += dist;
  }

  *out_rate_sum = rate_sum;
  *out_dist_sum = dist_sum << 4;
}

396
static void model_rd_for_sb_y_tx(VP9_COMP *cpi, BLOCK_SIZE bsize,
Deb Mukherjee's avatar
Deb Mukherjee committed
397
398
399
400
                                 TX_SIZE tx_size,
                                 MACROBLOCK *x, MACROBLOCKD *xd,
                                 int *out_rate_sum, int64_t *out_dist_sum,
                                 int *out_skip) {
401
  int j, k;
402
  BLOCK_SIZE bs;
Deb Mukherjee's avatar
Deb Mukherjee committed
403
404
  struct macroblock_plane *const p = &x->plane[0];
  struct macroblockd_plane *const pd = &xd->plane[0];
405
406
  const int width = 4 << num_4x4_blocks_wide_lookup[bsize];
  const int height = 4 << num_4x4_blocks_high_lookup[bsize];
Deb Mukherjee's avatar
Deb Mukherjee committed
407
408
  int rate_sum = 0;
  int64_t dist_sum = 0;
409
  const int t = 4 << tx_size;
Deb Mukherjee's avatar
Deb Mukherjee committed
410
411
412
413
414
415
416
417
418
419
420
421

  if (tx_size == TX_4X4) {
    bs = BLOCK_4X4;
  } else if (tx_size == TX_8X8) {
    bs = BLOCK_8X8;
  } else if (tx_size == TX_16X16) {
    bs = BLOCK_16X16;
  } else if (tx_size == TX_32X32) {
    bs = BLOCK_32X32;
  } else {
    assert(0);
  }
422

Deb Mukherjee's avatar
Deb Mukherjee committed
423
  *out_skip = 1;
Jim Bankoski's avatar
Jim Bankoski committed
424
425
  for (j = 0; j < height; j += t) {
    for (k = 0; k < width; k += t) {
Deb Mukherjee's avatar
Deb Mukherjee committed
426
427
428
      int rate;
      int64_t dist;
      unsigned int sse;
429
430
431
      cpi->fn_ptr[bs].vf(&p->src.buf[j * p->src.stride + k], p->src.stride,
                         &pd->dst.buf[j * pd->dst.stride + k], pd->dst.stride,
                         &sse);
Deb Mukherjee's avatar
Deb Mukherjee committed
432
      // sse works better than var, since there is no dc prediction used
433
      model_rd_from_var_lapndz(sse, t * t, pd->dequant[1] >> 3, &rate, &dist);
Deb Mukherjee's avatar
Deb Mukherjee committed
434
435
436
437
438
      rate_sum += rate;
      dist_sum += dist;
      *out_skip &= (rate < 1024);
    }
  }
439

Deb Mukherjee's avatar
Deb Mukherjee committed
440
  *out_rate_sum = rate_sum;
441
  *out_dist_sum = dist_sum << 4;
Deb Mukherjee's avatar
Deb Mukherjee committed
442
443
}

Ronald S. Bultje's avatar
Ronald S. Bultje committed
444
int64_t vp9_block_error_c(int16_t *coeff, int16_t *dqcoeff,
445
                          intptr_t block_size, int64_t *ssz) {
Ronald S. Bultje's avatar
Ronald S. Bultje committed
446
  int i;
447
  int64_t error = 0, sqcoeff = 0;
John Koleszar's avatar
John Koleszar committed
448

449
  for (i = 0; i < block_size; i++) {
John Koleszar's avatar
John Koleszar committed
450
    int this_diff = coeff[i] - dqcoeff[i];
Ronald S. Bultje's avatar
Ronald S. Bultje committed
451
    error += (unsigned)this_diff * this_diff;
452
    sqcoeff += (unsigned) coeff[i] * coeff[i];
John Koleszar's avatar
John Koleszar committed
453
  }
John Koleszar's avatar
John Koleszar committed
454

455
  *ssz = sqcoeff;
John Koleszar's avatar
John Koleszar committed
456
  return error;
John Koleszar's avatar
John Koleszar committed
457
458
}

459
460
461
462
463
/* The trailing '0' is a terminator which is used inside cost_coeffs() to
 * decide whether to include cost of a trailing EOB node or not (i.e. we
 * can skip this if the last coefficient in this transform block, e.g. the
 * 16th coefficient in a 4x4 block or the 64th coefficient in a 8x8 block,
 * were non-zero). */
464
static const int16_t band_counts[TX_SIZES][8] = {
465
466
467
468
  { 1, 2, 3, 4,  3,   16 - 13, 0 },
  { 1, 2, 3, 4, 11,   64 - 21, 0 },
  { 1, 2, 3, 4, 11,  256 - 21, 0 },
  { 1, 2, 3, 4, 11, 1024 - 21, 0 },
469
470
};

471
static INLINE int cost_coeffs(MACROBLOCK *mb,
472
                              int plane, int block,
473
                              ENTROPY_CONTEXT *A, ENTROPY_CONTEXT *L,
John Koleszar's avatar
John Koleszar committed
474
                              TX_SIZE tx_size,
475
                              const int16_t *scan, const int16_t *nb) {
476
  MACROBLOCKD *const xd = &mb->e_mbd;
477
  MB_MODE_INFO *mbmi = &xd->mode_info_context->mbmi;
478
479
  struct macroblockd_plane *pd = &xd->plane[plane];
  const PLANE_TYPE type = pd->plane_type;
480
  const int16_t *band_count = &band_counts[tx_size][1];
481
482
  const int eob = pd->eobs[block];
  const int16_t *const qcoeff_ptr = BLOCK_OFFSET(pd->qcoeff, block);
Ronald S. Bultje's avatar
Ronald S. Bultje committed
483
  const int ref = mbmi->ref_frame[0] != INTRA_FRAME;
484
485
486
  unsigned int (*token_costs)[2][PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS] =
                   mb->token_costs[tx_size][type][ref];
  const ENTROPY_CONTEXT above_ec = !!*A, left_ec = !!*L;
487
  uint8_t token_cache[1024];
488
489
  int pt = combine_entropy_contexts(above_ec, left_ec);
  int c, cost;
490
491

  // Check for consistency of tx_size with mode info
492
493
  assert(type == PLANE_TYPE_Y_WITH_DC ? mbmi->txfm_size == tx_size
                                      : get_uv_tx_size(mbmi) == tx_size);
494

495
496
  if (eob == 0) {
    // single eob token
497
498
    cost = token_costs[0][0][pt][DCT_EOB_TOKEN];
    c = 0;
499
  } else {
500
    int band_left = *band_count++;
501
502

    // dc token
503
504
    int v = qcoeff_ptr[0];
    int prev_t = vp9_dct_value_tokens_ptr[v].token;
505
    cost = (*token_costs)[0][pt][prev_t] + vp9_dct_value_cost_ptr[v];
506
    token_cache[0] = vp9_pt_energy_class[prev_t];
507
    ++token_costs;
508
509
510
511

    // ac tokens
    for (c = 1; c < eob; c++) {
      const int rc = scan[c];
512
      int t;
513
514
515

      v = qcoeff_ptr[rc];
      t = vp9_dct_value_tokens_ptr[v].token;
516
      pt = get_coef_context(nb, token_cache, c);
517
      cost += (*token_costs)[!prev_t][pt][t] + vp9_dct_value_cost_ptr[v];
518
      token_cache[rc] = vp9_pt_energy_class[t];
519
      prev_t = t;
520
      if (!--band_left) {
521
522
        band_left = *band_count++;
        ++token_costs;
523
      }
524
    }
525
526

    // eob token
527
    if (band_left) {
528
      pt = get_coef_context(nb, token_cache, c);
529
      cost += (*token_costs)[0][pt][DCT_EOB_TOKEN];
530
    }
531
532
  }

533
  // is eob first coefficient;
534
  *A = *L = (c > 0);
535

536
537
538
  return cost;
}

Deb Mukherjee's avatar
Deb Mukherjee committed
539
540
541
542
543
544
545
546
547
548
549
550
struct rdcost_block_args {
  MACROBLOCK *x;
  ENTROPY_CONTEXT t_above[16];
  ENTROPY_CONTEXT t_left[16];
  TX_SIZE tx_size;
  int bw;
  int bh;
  int rate;
  int64_t dist;
  int64_t sse;
  int64_t best_rd;
  int skip;
551
  const int16_t *scan, *nb;
Deb Mukherjee's avatar
Deb Mukherjee committed
552
553
};

554
static void dist_block(int plane, int block, TX_SIZE tx_size, void *arg) {
555
  const int ss_txfrm_size = tx_size << 1;
Deb Mukherjee's avatar
Deb Mukherjee committed
556
557
558
  struct rdcost_block_args* args = arg;
  MACROBLOCK* const x = args->x;
  MACROBLOCKD* const xd = &x->e_mbd;
559
560
  struct macroblock_plane *const p = &x->plane[plane];
  struct macroblockd_plane *const pd = &xd->plane[plane];
Deb Mukherjee's avatar
Deb Mukherjee committed
561
562
  int64_t this_sse;
  int shift = args->tx_size == TX_32X32 ? 0 : 2;
563
564
  int16_t *const coeff = BLOCK_OFFSET(p->coeff, block);
  int16_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
Deb Mukherjee's avatar
Deb Mukherjee committed
565
566
567
  args->dist += vp9_block_error(coeff, dqcoeff, 16 << ss_txfrm_size,
                                &this_sse) >> shift;
  args->sse += this_sse >> shift;
568
569
570
571
572
573
574
575
576

  if (x->skip_encode &&
      xd->mode_info_context->mbmi.ref_frame[0] == INTRA_FRAME) {
    // TODO(jingning): tune the model to better capture the distortion.
    int64_t p = (pd->dequant[1] * pd->dequant[1] *
                    (1 << ss_txfrm_size)) >> shift;
    args->dist += p;
    args->sse  += p;
  }
Deb Mukherjee's avatar
Deb Mukherjee committed
577
578
}

579
static void rate_block(int plane, int block, BLOCK_SIZE plane_bsize,
580
                       TX_SIZE tx_size, void *arg) {
Deb Mukherjee's avatar
Deb Mukherjee committed
581
  struct rdcost_block_args* args = arg;
582

Deb Mukherjee's avatar
Deb Mukherjee committed
583
  int x_idx, y_idx;
584
  txfrm_block_to_raster_xy(plane_bsize, args->tx_size, block, &x_idx, &y_idx);
Deb Mukherjee's avatar
Deb Mukherjee committed
585

586
  args->rate += cost_coeffs(args->x, plane, block,
587
                            args->t_above + x_idx,
Deb Mukherjee's avatar
Deb Mukherjee committed
588
                            args->t_left + y_idx, args->tx_size,
589
                            args->scan, args->nb);
Deb Mukherjee's avatar
Deb Mukherjee committed
590
591
}

592
static void block_yrd_txfm(int plane, int block, BLOCK_SIZE plane_bsize,
593
                           TX_SIZE tx_size, void *arg) {
Deb Mukherjee's avatar
Deb Mukherjee committed
594
595
596
  struct rdcost_block_args *args = arg;
  MACROBLOCK *const x = args->x;
  MACROBLOCKD *const xd = &x->e_mbd;
597
  struct encode_b_args encode_args = {x, NULL};
598
  int64_t rd1, rd2, rd;
Deb Mukherjee's avatar
Deb Mukherjee committed
599

600
601
  if (args->skip)
    return;
602
603
604
605
  rd1 = RDCOST(x->rdmult, x->rddiv, args->rate, args->dist);
  rd2 = RDCOST(x->rdmult, x->rddiv, 0, args->sse);
  rd = MIN(rd1, rd2);
  if (rd > args->best_rd) {
606
607
608
609
610
611
612
    args->skip = 1;
    args->rate = INT_MAX;
    args->dist = INT64_MAX;
    args->sse  = INT64_MAX;
    return;
  }

613
  if (!is_inter_block(&xd->mode_info_context->mbmi))
614
    vp9_encode_block_intra(plane, block, plane_bsize, tx_size, &encode_args);
Deb Mukherjee's avatar
Deb Mukherjee committed
615
  else
616
    vp9_xform_quant(plane, block, plane_bsize, tx_size, &encode_args);
Deb Mukherjee's avatar
Deb Mukherjee committed
617

618
619
  dist_block(plane, block, tx_size, args);
  rate_block(plane, block, plane_bsize, tx_size, args);
Deb Mukherjee's avatar
Deb Mukherjee committed
620
621
}

622
static void txfm_rd_in_plane(MACROBLOCK *x,
623
624
625
                             int *rate, int64_t *distortion,
                             int *skippable, int64_t *sse,
                             int64_t ref_best_rd, int plane,
626
                             BLOCK_SIZE bsize, TX_SIZE tx_size) {
Deb Mukherjee's avatar
Deb Mukherjee committed
627
  MACROBLOCKD *const xd = &x->e_mbd;
628
  struct macroblockd_plane *const pd = &xd->plane[plane];
629
  const BLOCK_SIZE bs = get_plane_block_size(bsize, pd);
Dmitry Kovalev's avatar
Dmitry Kovalev committed
630
631
  const int num_4x4_blocks_wide = num_4x4_blocks_wide_lookup[bs];
  const int num_4x4_blocks_high = num_4x4_blocks_high_lookup[bs];
632
  int i;
633
  struct rdcost_block_args args = { x, { 0 }, { 0 }, tx_size,
Dmitry Kovalev's avatar
Dmitry Kovalev committed
634
                                    num_4x4_blocks_wide, num_4x4_blocks_high,
635
                                    0, 0, 0, ref_best_rd, 0 };
636
637
638
  if (plane == 0)
    xd->mode_info_context->mbmi.txfm_size = tx_size;

639
640
641
  switch (tx_size) {
    case TX_4X4:
      vpx_memcpy(&args.t_above, pd->above_context,
Dmitry Kovalev's avatar
Dmitry Kovalev committed
642
                 sizeof(ENTROPY_CONTEXT) * num_4x4_blocks_wide);
643
      vpx_memcpy(&args.t_left, pd->left_context,
Dmitry Kovalev's avatar
Dmitry Kovalev committed
644
                 sizeof(ENTROPY_CONTEXT) * num_4x4_blocks_high);
645
      get_scan_nb_4x4(get_tx_type_4x4(pd->plane_type, xd, 0),
646
                      &args.scan, &args.nb);
647
648
      break;
    case TX_8X8:
Dmitry Kovalev's avatar
Dmitry Kovalev committed
649
      for (i = 0; i < num_4x4_blocks_wide; i += 2)
650
        args.t_above[i] = !!*(uint16_t *)&pd->above_context[i];
Dmitry Kovalev's avatar
Dmitry Kovalev committed
651
      for (i = 0; i < num_4x4_blocks_high; i += 2)
652
        args.t_left[i] = !!*(uint16_t *)&pd->left_context[i];
653
      get_scan_nb_8x8(get_tx_type_8x8(pd->plane_type, xd),
654
                      &args.scan, &args.nb);
655
656
      break;
    case TX_16X16:
Dmitry Kovalev's avatar
Dmitry Kovalev committed
657
      for (i = 0; i < num_4x4_blocks_wide; i += 4)
658
        args.t_above[i] = !!*(uint32_t *)&pd->above_context[i];
Dmitry Kovalev's avatar
Dmitry Kovalev committed
659
      for (i = 0; i < num_4x4_blocks_high; i += 4)
660
        args.t_left[i] = !!*(uint32_t *)&pd->left_context[i];
661
      get_scan_nb_16x16(get_tx_type_16x16(pd->plane_type, xd),
662
                        &args.scan, &args.nb);
663
664
      break;
    case TX_32X32:
Dmitry Kovalev's avatar
Dmitry Kovalev committed
665
      for (i = 0; i < num_4x4_blocks_wide; i += 8)
666
        args.t_above[i] = !!*(uint64_t *)&pd->above_context[i];
Dmitry Kovalev's avatar
Dmitry Kovalev committed
667
      for (i = 0; i < num_4x4_blocks_high; i += 8)
668
669
670
671
672
673
674
        args.t_left[i] = !!*(uint64_t *)&pd->left_context[i];
      args.scan = vp9_default_scan_32x32;
      args.nb = vp9_default_scan_32x32_neighbors;
      break;
    default:
      assert(0);
  }
Deb Mukherjee's avatar
Deb Mukherjee committed
675

676
  foreach_transformed_block_in_plane(xd, bsize, plane, block_yrd_txfm, &args);
Deb Mukherjee's avatar
Deb Mukherjee committed
677
678
679
  *distortion = args.dist;
  *rate       = args.rate;
  *sse        = args.sse;
680
  *skippable  = vp9_is_skippable_in_plane(xd, bsize, plane) && (!args.skip);
Deb Mukherjee's avatar
Deb Mukherjee committed
681
682
683
684
685
}

static void choose_largest_txfm_size(VP9_COMP *cpi, MACROBLOCK *x,
                                     int *rate, int64_t *distortion,
                                     int *skip, int64_t *sse,
686
                                     int64_t ref_best_rd,
687
                                     BLOCK_SIZE bs) {
688
  const TX_SIZE max_txfm_size = max_txsize_lookup[bs];
Deb Mukherjee's avatar
Deb Mukherjee committed
689
690
691
692
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
  if (max_txfm_size == TX_32X32 &&
693
694
      (cm->tx_mode == ALLOW_32X32 ||
       cm->tx_mode == TX_MODE_SELECT)) {
Deb Mukherjee's avatar
Deb Mukherjee committed
695
696
    mbmi->txfm_size = TX_32X32;
  } else if (max_txfm_size >= TX_16X16 &&
697
698
699
             (cm->tx_mode == ALLOW_16X16 ||
              cm->tx_mode == ALLOW_32X32 ||
              cm->tx_mode == TX_MODE_SELECT)) {
Deb Mukherjee's avatar
Deb Mukherjee committed
700
    mbmi->txfm_size = TX_16X16;
701
  } else if (cm->tx_mode != ONLY_4X4) {
Deb Mukherjee's avatar
Deb Mukherjee committed
702
703
704
705
    mbmi->txfm_size = TX_8X8;
  } else {
    mbmi->txfm_size = TX_4X4;
  }
706
  txfm_rd_in_plane(x, rate, distortion, skip,
707
708
                   &sse[mbmi->txfm_size], ref_best_rd, 0, bs,
                   mbmi->txfm_size);
Deb Mukherjee's avatar
Deb Mukherjee committed
709
710
711
  cpi->txfm_stepdown_count[0]++;
}

712
static void choose_txfm_size_from_rd(VP9_COMP *cpi, MACROBLOCK *x,
713
                                     int (*r)[2], int *rate,
Ronald S. Bultje's avatar
Ronald S. Bultje committed
714
                                     int64_t *d, int64_t *distortion,
715
                                     int *s, int *skip,
716
                                     int64_t tx_cache[TX_MODES],
717
                                     BLOCK_SIZE bs) {
718
  const TX_SIZE max_tx_size = max_txsize_lookup[bs];
719
720
721
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
722
  vp9_prob skip_prob = vp9_get_pred_prob_mbskip(cm, xd);
723
  int64_t rd[TX_SIZES][2];
724
  int n, m;
725
  int s0, s1;
726

727
  const vp9_prob *tx_probs = get_tx_probs2(xd, &cm->fc.tx_probs);
728

729
  for (n = TX_4X4; n <= max_tx_size; n++) {
730
    r[n][1] = r[n][0];
731
732
    if (r[n][0] == INT_MAX)
      continue;
733
    for (m = 0; m <= n - (n == max_tx_size); m++) {
734
      if (m == n)
735
        r[n][1] += vp9_cost_zero(tx_probs[m]);
736
      else
737
        r[n][1] += vp9_cost_one(tx_probs[m]);
738
739
    }
  }
740

741
742
743
  assert(skip_prob > 0);
  s0 = vp9_cost_bit(skip_prob, 0);
  s1 = vp9_cost_bit(skip_prob, 1);
744

745
  for (n = TX_4X4; n <= max_tx_size; n++) {
746
747
748
749
    if (d[n] == INT64_MAX) {
      rd[n][0] = rd[n][1] = INT64_MAX;
      continue;
    }
750
751
752
753
754
    if (s[n]) {
      rd[n][0] = rd[n][1] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
    } else {
      rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0] + s0, d[n]);
      rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1] + s0, d[n]);
755
756
757
    }
  }

758
  if (max_tx_size == TX_32X32 &&
759
760
      (cm->tx_mode == ALLOW_32X32 ||
       (cm->tx_mode == TX_MODE_SELECT &&
761
762
763
        rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
        rd[TX_32X32][1] < rd[TX_4X4][1]))) {
    mbmi->txfm_size = TX_32X32;
764
  } else if (max_tx_size >= TX_16X16 &&
765
766
767
             (cm->tx_mode == ALLOW_16X16 ||
              cm->tx_mode == ALLOW_32X32 ||
              (cm->tx_mode == TX_MODE_SELECT &&
768
769
               rd[TX_16X16][1] < rd[TX_8X8][1] &&
               rd[TX_16X16][1] < rd[TX_4X4][1]))) {
770
    mbmi->txfm_size = TX_16X16;
771
772
773
774
  } else if (cm->tx_mode == ALLOW_8X8 ||
             cm->tx_mode == ALLOW_16X16 ||
             cm->tx_mode == ALLOW_32X32 ||
           (cm->tx_mode == TX_MODE_SELECT && rd[TX_8X8][1] < rd[TX_4X4][1])) {
775
776
777
778
779
    mbmi->txfm_size = TX_8X8;
  } else {
    mbmi->txfm_size = TX_4X4;
  }

780
  *distortion = d[mbmi->txfm_size];
781
  *rate       = r[mbmi->txfm_size][cm->tx_mode == TX_MODE_SELECT];
782
783
  *skip       = s[mbmi->txfm_size];

784
785
786
787
788
  tx_cache[ONLY_4X4] = rd[TX_4X4][0];
  tx_cache[ALLOW_8X8] = rd[TX_8X8][0];
  tx_cache[ALLOW_16X16] = rd[MIN(max_tx_size, TX_16X16)][0];
  tx_cache[ALLOW_32X32] = rd[MIN(max_tx_size, TX_32X32)][0];
  if (max_tx_size == TX_32X32 &&
789
790
      rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
      rd[TX_32X32][1] < rd[TX_4X4][1])
791
792
    tx_cache[TX_MODE_SELECT] = rd[TX_32X32][1];
  else if (max_tx_size >= TX_16X16 &&
793
           rd[TX_16X16][1] < rd[TX_8X8][1] && rd[TX_16X16][1] < rd[TX_4X4][1])
794
    tx_cache[TX_MODE_SELECT] = rd[TX_16X16][1];
795
  else
796
    tx_cache[TX_MODE_SELECT] = rd[TX_4X4][1] < rd[TX_8X8][1] ?
797
                                 rd[TX_4X4][1] : rd[TX_8X8][1];
798

799
  if (max_tx_size == TX_32X32 &&
Deb Mukherjee's avatar
Deb Mukherjee committed
800
801
802
803
      rd[TX_32X32][1] < rd[TX_16X16][1] &&
      rd[TX_32X32][1] < rd[TX_8X8][1] &&
      rd[TX_32X32][1] < rd[TX_4X4][1]) {
    cpi->txfm_stepdown_count[0]++;
804
  } else if (max_tx_size >= TX_16X16 &&
Deb Mukherjee's avatar
Deb Mukherjee committed
805
806
             rd[TX_16X16][1] < rd[TX_8X8][1] &&
             rd[TX_16X16][1] < rd[TX_4X4][1]) {
807
    cpi->txfm_stepdown_count[max_tx_size - TX_16X16]++;
Deb Mukherjee's avatar
Deb Mukherjee committed
808
  } else if (rd[TX_8X8][1] < rd[TX_4X4][1]) {
809
    cpi->txfm_stepdown_count[max_tx_size - TX_8X8]++;
Deb Mukherjee's avatar
Deb Mukherjee committed
810
  } else {
811
    cpi->txfm_stepdown_count[max_tx_size - TX_4X4]++;
812
  }
Deb Mukherjee's avatar
Deb Mukherjee committed
813
}
814

Deb Mukherjee's avatar
Deb Mukherjee committed
815
816
817
818
static void choose_txfm_size_from_modelrd(VP9_COMP *cpi, MACROBLOCK *x,
                                          int (*r)[2], int *rate,
                                          int64_t *d, int64_t *distortion,
                                          int *s, int *skip, int64_t *sse,
819
                                          int64_t ref_best_rd,
820
                                          BLOCK_SIZE bs) {
821
  const TX_SIZE max_txfm_size = max_txsize_lookup[bs];
Deb Mukherjee's avatar
Deb Mukherjee committed
822
823
824
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
825
  vp9_prob skip_prob = vp9_get_pred_prob_mbskip(cm, xd);
826
  int64_t rd[TX_SIZES][2];
Deb Mukherjee's avatar
Deb Mukherjee committed
827
828
  int n, m;
  int s0, s1;
829
830
  double scale_rd[TX_SIZES] = {1.73, 1.44, 1.20, 1.00};
  // double scale_r[TX_SIZES] = {2.82, 2.00, 1.41, 1.00};
831

832
  const vp9_prob *tx_probs = get_tx_probs2(xd, &cm->fc.tx_probs);
833

Deb Mukherjee's avatar
Deb Mukherjee committed
834
835
  // for (n = TX_4X4; n <= max_txfm_size; n++)
  //   r[n][0] = (r[n][0] * scale_r[n]);
836

Deb Mukherjee's avatar
Deb Mukherjee committed
837
838
839
840
841
842
843
844
  for (n = TX_4X4; n <= max_txfm_size; n++) {
    r[n][1] = r[n][0];
    for (m = 0; m <= n - (n == max_txfm_size); m++) {
      if (m == n)
        r[n][1] += vp9_cost_zero(tx_probs[m]);
      else
        r[n][1] += vp9_cost_one(tx_probs[m]);
    }
845
  }
846

Deb Mukherjee's avatar
Deb Mukherjee committed
847
848
849
  assert(skip_prob > 0);
  s0 = vp9_cost_bit(skip_prob, 0