vp9_rdopt.c 166 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5
6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9
10
11
12
13
14
 */

#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
15

16
#include "vp9/common/vp9_pragmas.h"
17
18
19
20
21
#include "vp9/encoder/vp9_tokenize.h"
#include "vp9/encoder/vp9_treewriter.h"
#include "vp9/encoder/vp9_onyx_int.h"
#include "vp9/encoder/vp9_modecosts.h"
#include "vp9/encoder/vp9_encodeintra.h"
22
23
24
25
26
#include "vp9/common/vp9_entropymode.h"
#include "vp9/common/vp9_reconinter.h"
#include "vp9/common/vp9_reconintra.h"
#include "vp9/common/vp9_findnearmv.h"
#include "vp9/common/vp9_quant_common.h"
27
28
29
30
31
32
#include "vp9/encoder/vp9_encodemb.h"
#include "vp9/encoder/vp9_quantize.h"
#include "vp9/encoder/vp9_variance.h"
#include "vp9/encoder/vp9_mcomp.h"
#include "vp9/encoder/vp9_rdopt.h"
#include "vp9/encoder/vp9_ratectrl.h"
John Koleszar's avatar
John Koleszar committed
33
#include "vpx_mem/vpx_mem.h"
34
35
36
37
38
#include "vp9/common/vp9_systemdependent.h"
#include "vp9/encoder/vp9_encodemv.h"
#include "vp9/common/vp9_seg_common.h"
#include "vp9/common/vp9_pred_common.h"
#include "vp9/common/vp9_entropy.h"
39
#include "./vp9_rtcd.h"
40
#include "vp9/common/vp9_mvref_common.h"
Ronald S. Bultje's avatar
Ronald S. Bultje committed
41
#include "vp9/common/vp9_common.h"
Paul Wilkins's avatar
Paul Wilkins committed
42

43
44
#define INVALID_MV 0x80008000

45
46
47
/* Factor to weigh the rate for switchable interp filters */
#define SWITCHABLE_INTERP_RATE_FACTOR 1

48
49
50
#define LAST_FRAME_MODE_MASK    0xFFEDCD60
#define GOLDEN_FRAME_MODE_MASK  0xFFDA3BB0
#define ALT_REF_MODE_MASK       0xFFC648D0
51

Paul Wilkins's avatar
Paul Wilkins committed
52
53
#define MIN_EARLY_TERM_INDEX    3

54
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
  {NEARESTMV, LAST_FRAME,   NONE},
  {NEARESTMV, ALTREF_FRAME, NONE},
  {NEARESTMV, GOLDEN_FRAME, NONE},

  {DC_PRED,   INTRA_FRAME,  NONE},

  {NEWMV,     LAST_FRAME,   NONE},
  {NEWMV,     ALTREF_FRAME, NONE},
  {NEWMV,     GOLDEN_FRAME, NONE},

  {NEARMV,    LAST_FRAME,   NONE},
  {NEARMV,    ALTREF_FRAME, NONE},
  {NEARESTMV, LAST_FRAME,   ALTREF_FRAME},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},

  {TM_PRED,   INTRA_FRAME,  NONE},

  {NEARMV,    LAST_FRAME,   ALTREF_FRAME},
  {NEWMV,     LAST_FRAME,   ALTREF_FRAME},
  {NEARMV,    GOLDEN_FRAME, NONE},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME},
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME},

  {ZEROMV,    LAST_FRAME,   NONE},
  {ZEROMV,    GOLDEN_FRAME, NONE},
  {ZEROMV,    ALTREF_FRAME, NONE},
  {ZEROMV,    LAST_FRAME,   ALTREF_FRAME},
  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME},

  {H_PRED,    INTRA_FRAME,  NONE},
  {V_PRED,    INTRA_FRAME,  NONE},
  {D135_PRED, INTRA_FRAME,  NONE},
  {D207_PRED, INTRA_FRAME,  NONE},
  {D153_PRED, INTRA_FRAME,  NONE},
  {D63_PRED,  INTRA_FRAME,  NONE},
  {D117_PRED, INTRA_FRAME,  NONE},
  {D45_PRED,  INTRA_FRAME,  NONE},
};

const REF_DEFINITION vp9_ref_order[MAX_REFS] = {
  {LAST_FRAME,   NONE},
  {GOLDEN_FRAME, NONE},
  {ALTREF_FRAME, NONE},
  {LAST_FRAME,   ALTREF_FRAME},
  {GOLDEN_FRAME, ALTREF_FRAME},
  {INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
101
102
};

103
104
105
106
// The baseline rd thresholds for breaking out of the rd loop for
// certain modes are assumed to be based on 8x8 blocks.
// This table is used to correct for blocks size.
// The factors here are << 2 (2 = x0.5, 32 = x8 etc).
107
static int rd_thresh_block_size_factor[BLOCK_SIZES] =
108
109
  {2, 3, 3, 4, 6, 6, 8, 12, 12, 16, 24, 24, 32};

110
111
#define MAX_RD_THRESH_FACT 64
#define RD_THRESH_INC 1
112

113
static void fill_token_costs(vp9_coeff_cost *c,
114
                             vp9_coeff_probs_model (*p)[BLOCK_TYPES]) {
115
  int i, j, k, l;
116
117
118
119
120
121
122
123
  TX_SIZE t;
  for (t = TX_4X4; t <= TX_32X32; t++)
    for (i = 0; i < BLOCK_TYPES; i++)
      for (j = 0; j < REF_TYPES; j++)
        for (k = 0; k < COEF_BANDS; k++)
          for (l = 0; l < PREV_COEF_CONTEXTS; l++) {
            vp9_prob probs[ENTROPY_NODES];
            vp9_model_to_full_probs(p[t][i][j][k][l], probs);
124
            vp9_cost_tokens((int *)c[t][i][j][k][0][l], probs,
125
                            vp9_coef_tree);
126
            vp9_cost_tokens_skip((int *)c[t][i][j][k][1][l], probs,
127
                                 vp9_coef_tree);
128
129
            assert(c[t][i][j][k][0][l][DCT_EOB_TOKEN] ==
                   c[t][i][j][k][1][l][DCT_EOB_TOKEN]);
130
          }
131
132
}

133
134
135
136
137
138
static const int rd_iifactor[32] = {
  4, 4, 3, 2, 1, 0, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 0,
};
John Koleszar's avatar
John Koleszar committed
139

140
// 3* dc_qlookup[Q]*dc_qlookup[Q];
141

142
/* values are now correlated to quantizer */
Paul Wilkins's avatar
Paul Wilkins committed
143
144
145
static int sad_per_bit16lut[QINDEX_RANGE];
static int sad_per_bit4lut[QINDEX_RANGE];

146
void vp9_init_me_luts() {
John Koleszar's avatar
John Koleszar committed
147
148
149
150
151
152
153
  int i;

  // Initialize the sad lut tables using a formulaic calculation for now
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < QINDEX_RANGE; i++) {
    sad_per_bit16lut[i] =
154
      (int)((0.0418 * vp9_convert_qindex_to_q(i)) + 2.4107);
155
    sad_per_bit4lut[i] = (int)(0.063 * vp9_convert_qindex_to_q(i) + 2.742);
John Koleszar's avatar
John Koleszar committed
156
  }
Paul Wilkins's avatar
Paul Wilkins committed
157
}
John Koleszar's avatar
John Koleszar committed
158

159
static int compute_rd_mult(int qindex) {
160
  const int q = vp9_dc_quant(qindex, 0);
161
  return (11 * q * q) >> 2;
162
163
}

Dmitry Kovalev's avatar
Dmitry Kovalev committed
164
165
166
void vp9_initialize_me_consts(VP9_COMP *cpi, int qindex) {
  cpi->mb.sadperbit16 = sad_per_bit16lut[qindex];
  cpi->mb.sadperbit4 = sad_per_bit4lut[qindex];
John Koleszar's avatar
John Koleszar committed
167
168
}

Dmitry Kovalev's avatar
Dmitry Kovalev committed
169
void vp9_initialize_rd_consts(VP9_COMP *cpi, int qindex) {
170
  int q, i, bsize;
John Koleszar's avatar
John Koleszar committed
171

172
  vp9_clear_system_state();  // __asm emms;
John Koleszar's avatar
John Koleszar committed
173

John Koleszar's avatar
John Koleszar committed
174
175
176
177
  // Further tests required to see if optimum is different
  // for key frames, golden frames and arf frames.
  // if (cpi->common.refresh_golden_frame ||
  //     cpi->common.refresh_alt_ref_frame)
178
  qindex = clamp(qindex, 0, MAXQ);
179

180
  cpi->RDDIV = 100;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
181
  cpi->RDMULT = compute_rd_mult(qindex);
John Koleszar's avatar
John Koleszar committed
182
183
184
185
186
  if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    if (cpi->twopass.next_iiratio > 31)
      cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
    else
      cpi->RDMULT +=
Dmitry Kovalev's avatar
Dmitry Kovalev committed
187
          (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
John Koleszar's avatar
John Koleszar committed
188
  }
189
  cpi->mb.errorperbit = cpi->RDMULT >> 6;
John Koleszar's avatar
John Koleszar committed
190
  cpi->mb.errorperbit += (cpi->mb.errorperbit == 0);
191

192
  vp9_set_speed_features(cpi);
John Koleszar's avatar
John Koleszar committed
193

Dmitry Kovalev's avatar
Dmitry Kovalev committed
194
195
  q = (int)pow(vp9_dc_quant(qindex, 0) >> 2, 1.25);
  q <<= 2;
John Koleszar's avatar
John Koleszar committed
196
197
  if (q < 8)
    q = 8;
198

199
  for (bsize = 0; bsize < BLOCK_SIZES; ++bsize) {
200
    for (i = 0; i < MAX_MODES; ++i) {
201
202
203
      // Threshold here seem unecessarily harsh but fine given actual
      // range of values used for cpi->sf.thresh_mult[]
      int thresh_max = INT_MAX / (q * rd_thresh_block_size_factor[bsize]);
John Koleszar's avatar
John Koleszar committed
204

205
206
      if (cpi->sf.thresh_mult[i] < thresh_max) {
        cpi->rd_threshes[bsize][i] =
207
208
            cpi->sf.thresh_mult[i] * q *
            rd_thresh_block_size_factor[bsize] / 4;
209
210
      } else {
        cpi->rd_threshes[bsize][i] = INT_MAX;
John Koleszar's avatar
John Koleszar committed
211
      }
John Koleszar's avatar
John Koleszar committed
212
    }
213
214
215
216
217
218
219
220
221
222
223
224

    for (i = 0; i < MAX_REFS; ++i) {
      int thresh_max = INT_MAX / (q * rd_thresh_block_size_factor[bsize]);

      if (cpi->sf.thresh_mult_sub8x8[i] < thresh_max) {
        cpi->rd_thresh_sub8x8[bsize][i] =
            cpi->sf.thresh_mult_sub8x8[i] * q *
            rd_thresh_block_size_factor[bsize] / 4;
      } else {
        cpi->rd_thresh_sub8x8[bsize][i] = INT_MAX;
      }
    }
John Koleszar's avatar
John Koleszar committed
225
  }
John Koleszar's avatar
John Koleszar committed
226

227
  fill_token_costs(cpi->mb.token_costs, cpi->common.fc.coef_probs);
228

229
  for (i = 0; i < NUM_PARTITION_CONTEXTS; i++)
230
    vp9_cost_tokens(cpi->mb.partition_cost[i],
231
                    cpi->common.fc.partition_prob[cpi->common.frame_type][i],
232
233
                    vp9_partition_tree);

John Koleszar's avatar
John Koleszar committed
234
  /*rough estimate for costing*/
235
  vp9_init_mode_costs(cpi);
John Koleszar's avatar
John Koleszar committed
236

237
  if (cpi->common.frame_type != KEY_FRAME) {
238
    vp9_build_nmv_cost_table(
239
240
241
242
243
        cpi->mb.nmvjointcost,
        cpi->mb.e_mbd.allow_high_precision_mv ?
        cpi->mb.nmvcost_hp : cpi->mb.nmvcost,
        &cpi->common.fc.nmvc,
        cpi->mb.e_mbd.allow_high_precision_mv, 1, 1);
244
245
246
247
248

    for (i = 0; i < INTER_MODE_CONTEXTS; i++) {
      MB_PREDICTION_MODE m;

      for (m = NEARESTMV; m < MB_MODE_COUNT; m++)
249
        cpi->mb.inter_mode_cost[i][inter_mode_offset(m)] =
250
            cost_token(vp9_inter_mode_tree,
251
                       cpi->common.fc.inter_mode_probs[i],
252
                       vp9_inter_mode_encodings + inter_mode_offset(m));
253
    }
254
  }
John Koleszar's avatar
John Koleszar committed
255
256
}

Yaowu Xu's avatar
Yaowu Xu committed
257
static INLINE void linear_interpolate2(double x, int ntab, int inv_step,
258
259
                                       const double *tab1, const double *tab2,
                                       double *v1, double *v2) {
Deb Mukherjee's avatar
Deb Mukherjee committed
260
261
262
  double y = x * inv_step;
  int d = (int) y;
  if (d >= ntab - 1) {
263
264
    *v1 = tab1[ntab - 1];
    *v2 = tab2[ntab - 1];
Deb Mukherjee's avatar
Deb Mukherjee committed
265
266
  } else {
    double a = y - d;
267
268
    *v1 = tab1[d] * (1 - a) + tab1[d + 1] * a;
    *v2 = tab2[d] * (1 - a) + tab2[d + 1] * a;
Deb Mukherjee's avatar
Deb Mukherjee committed
269
270
271
  }
}

272
273
274
275
276
static void model_rd_norm(double x, double *R, double *D) {
  static const int inv_tab_step = 8;
  static const int tab_size = 120;
  // NOTE: The tables below must be of the same size
  //
Deb Mukherjee's avatar
Deb Mukherjee committed
277
  // Normalized rate
278
  // This table models the rate for a Laplacian source
Deb Mukherjee's avatar
Deb Mukherjee committed
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
  // source with given variance when quantized with a uniform quantizer
  // with given stepsize. The closed form expression is:
  // Rn(x) = H(sqrt(r)) + sqrt(r)*[1 + H(r)/(1 - r)],
  // where r = exp(-sqrt(2) * x) and x = qpstep / sqrt(variance),
  // and H(x) is the binary entropy function.
  static const double rate_tab[] = {
    64.00, 4.944, 3.949, 3.372, 2.966, 2.655, 2.403, 2.194,
    2.014, 1.858, 1.720, 1.596, 1.485, 1.384, 1.291, 1.206,
    1.127, 1.054, 0.986, 0.923, 0.863, 0.808, 0.756, 0.708,
    0.662, 0.619, 0.579, 0.541, 0.506, 0.473, 0.442, 0.412,
    0.385, 0.359, 0.335, 0.313, 0.291, 0.272, 0.253, 0.236,
    0.220, 0.204, 0.190, 0.177, 0.165, 0.153, 0.142, 0.132,
    0.123, 0.114, 0.106, 0.099, 0.091, 0.085, 0.079, 0.073,
    0.068, 0.063, 0.058, 0.054, 0.050, 0.047, 0.043, 0.040,
    0.037, 0.034, 0.032, 0.029, 0.027, 0.025, 0.023, 0.022,
    0.020, 0.019, 0.017, 0.016, 0.015, 0.014, 0.013, 0.012,
    0.011, 0.010, 0.009, 0.008, 0.008, 0.007, 0.007, 0.006,
    0.006, 0.005, 0.005, 0.005, 0.004, 0.004, 0.004, 0.003,
    0.003, 0.003, 0.003, 0.002, 0.002, 0.002, 0.002, 0.002,
    0.002, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001,
    0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.000,
  };
  // Normalized distortion
302
  // This table models the normalized distortion for a Laplacian source
Deb Mukherjee's avatar
Deb Mukherjee committed
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
  // source with given variance when quantized with a uniform quantizer
  // with given stepsize. The closed form expression is:
  // Dn(x) = 1 - 1/sqrt(2) * x / sinh(x/sqrt(2))
  // where x = qpstep / sqrt(variance)
  // Note the actual distortion is Dn * variance.
  static const double dist_tab[] = {
    0.000, 0.001, 0.005, 0.012, 0.021, 0.032, 0.045, 0.061,
    0.079, 0.098, 0.119, 0.142, 0.166, 0.190, 0.216, 0.242,
    0.269, 0.296, 0.324, 0.351, 0.378, 0.405, 0.432, 0.458,
    0.484, 0.509, 0.534, 0.557, 0.580, 0.603, 0.624, 0.645,
    0.664, 0.683, 0.702, 0.719, 0.735, 0.751, 0.766, 0.780,
    0.794, 0.807, 0.819, 0.830, 0.841, 0.851, 0.861, 0.870,
    0.878, 0.886, 0.894, 0.901, 0.907, 0.913, 0.919, 0.925,
    0.930, 0.935, 0.939, 0.943, 0.947, 0.951, 0.954, 0.957,
    0.960, 0.963, 0.966, 0.968, 0.971, 0.973, 0.975, 0.976,
    0.978, 0.980, 0.981, 0.982, 0.984, 0.985, 0.986, 0.987,
    0.988, 0.989, 0.990, 0.990, 0.991, 0.992, 0.992, 0.993,
    0.993, 0.994, 0.994, 0.995, 0.995, 0.996, 0.996, 0.996,
    0.996, 0.997, 0.997, 0.997, 0.997, 0.998, 0.998, 0.998,
    0.998, 0.998, 0.998, 0.999, 0.999, 0.999, 0.999, 0.999,
    0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 1.000,
  };
325
326
327
328
329
  /*
  assert(sizeof(rate_tab) == tab_size * sizeof(rate_tab[0]);
  assert(sizeof(dist_tab) == tab_size * sizeof(dist_tab[0]);
  assert(sizeof(rate_tab) == sizeof(dist_tab));
  */
Deb Mukherjee's avatar
Deb Mukherjee committed
330
  assert(x >= 0.0);
331
332
  linear_interpolate2(x, tab_size, inv_tab_step,
                      rate_tab, dist_tab, R, D);
Deb Mukherjee's avatar
Deb Mukherjee committed
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
}

static void model_rd_from_var_lapndz(int var, int n, int qstep,
                                     int *rate, int64_t *dist) {
  // This function models the rate and distortion for a Laplacian
  // source with given variance when quantized with a uniform quantizer
  // with given stepsize. The closed form expressions are in:
  // Hang and Chen, "Source Model for transform video coder and its
  // application - Part I: Fundamental Theory", IEEE Trans. Circ.
  // Sys. for Video Tech., April 1997.
  vp9_clear_system_state();
  if (var == 0 || n == 0) {
    *rate = 0;
    *dist = 0;
  } else {
    double D, R;
    double s2 = (double) var / n;
    double x = qstep / sqrt(s2);
351
    model_rd_norm(x, &R, &D);
Yaowu Xu's avatar
Yaowu Xu committed
352
353
    *rate = (int)((n << 8) * R + 0.5);
    *dist = (int)(var * D + 0.5);
Deb Mukherjee's avatar
Deb Mukherjee committed
354
355
356
357
  }
  vp9_clear_system_state();
}

358
static void model_rd_for_sb(VP9_COMP *cpi, BLOCK_SIZE bsize,
Deb Mukherjee's avatar
Deb Mukherjee committed
359
360
361
362
363
364
365
366
367
368
                            MACROBLOCK *x, MACROBLOCKD *xd,
                            int *out_rate_sum, int64_t *out_dist_sum) {
  // Note our transform coeffs are 8 times an orthogonal transform.
  // Hence quantizer step is also 8 times. To get effective quantizer
  // we need to divide by 8 before sending to modeling function.
  int i, rate_sum = 0, dist_sum = 0;

  for (i = 0; i < MAX_MB_PLANE; ++i) {
    struct macroblock_plane *const p = &x->plane[i];
    struct macroblockd_plane *const pd = &xd->plane[i];
369
    const BLOCK_SIZE bs = get_plane_block_size(bsize, pd);
Deb Mukherjee's avatar
Deb Mukherjee committed
370
371
372
373
374
375
    unsigned int sse;
    int rate;
    int64_t dist;
    (void) cpi->fn_ptr[bs].vf(p->src.buf, p->src.stride,
                              pd->dst.buf, pd->dst.stride, &sse);
    // sse works better than var, since there is no dc prediction used
Dmitry Kovalev's avatar
Dmitry Kovalev committed
376
    model_rd_from_var_lapndz(sse, 1 << num_pels_log2_lookup[bs],
377
                             pd->dequant[1] >> 3, &rate, &dist);
Deb Mukherjee's avatar
Deb Mukherjee committed
378
379

    rate_sum += rate;
Yaowu Xu's avatar
Yaowu Xu committed
380
    dist_sum += (int)dist;
Deb Mukherjee's avatar
Deb Mukherjee committed
381
382
383
384
385
386
  }

  *out_rate_sum = rate_sum;
  *out_dist_sum = dist_sum << 4;
}

387
static void model_rd_for_sb_y_tx(VP9_COMP *cpi, BLOCK_SIZE bsize,
Deb Mukherjee's avatar
Deb Mukherjee committed
388
389
390
391
                                 TX_SIZE tx_size,
                                 MACROBLOCK *x, MACROBLOCKD *xd,
                                 int *out_rate_sum, int64_t *out_dist_sum,
                                 int *out_skip) {
392
  int j, k;
393
  BLOCK_SIZE bs;
Deb Mukherjee's avatar
Deb Mukherjee committed
394
395
  struct macroblock_plane *const p = &x->plane[0];
  struct macroblockd_plane *const pd = &xd->plane[0];
396
397
  const int width = 4 << num_4x4_blocks_wide_lookup[bsize];
  const int height = 4 << num_4x4_blocks_high_lookup[bsize];
Deb Mukherjee's avatar
Deb Mukherjee committed
398
399
  int rate_sum = 0;
  int64_t dist_sum = 0;
400
  const int t = 4 << tx_size;
Deb Mukherjee's avatar
Deb Mukherjee committed
401
402
403
404
405
406
407
408
409
410
411
412

  if (tx_size == TX_4X4) {
    bs = BLOCK_4X4;
  } else if (tx_size == TX_8X8) {
    bs = BLOCK_8X8;
  } else if (tx_size == TX_16X16) {
    bs = BLOCK_16X16;
  } else if (tx_size == TX_32X32) {
    bs = BLOCK_32X32;
  } else {
    assert(0);
  }
413

Deb Mukherjee's avatar
Deb Mukherjee committed
414
  *out_skip = 1;
Jim Bankoski's avatar
Jim Bankoski committed
415
416
  for (j = 0; j < height; j += t) {
    for (k = 0; k < width; k += t) {
Deb Mukherjee's avatar
Deb Mukherjee committed
417
418
419
      int rate;
      int64_t dist;
      unsigned int sse;
420
421
422
      cpi->fn_ptr[bs].vf(&p->src.buf[j * p->src.stride + k], p->src.stride,
                         &pd->dst.buf[j * pd->dst.stride + k], pd->dst.stride,
                         &sse);
Deb Mukherjee's avatar
Deb Mukherjee committed
423
      // sse works better than var, since there is no dc prediction used
424
      model_rd_from_var_lapndz(sse, t * t, pd->dequant[1] >> 3, &rate, &dist);
Deb Mukherjee's avatar
Deb Mukherjee committed
425
426
427
428
429
      rate_sum += rate;
      dist_sum += dist;
      *out_skip &= (rate < 1024);
    }
  }
430

Deb Mukherjee's avatar
Deb Mukherjee committed
431
  *out_rate_sum = rate_sum;
432
  *out_dist_sum = dist_sum << 4;
Deb Mukherjee's avatar
Deb Mukherjee committed
433
434
}

Ronald S. Bultje's avatar
Ronald S. Bultje committed
435
int64_t vp9_block_error_c(int16_t *coeff, int16_t *dqcoeff,
436
                          intptr_t block_size, int64_t *ssz) {
Ronald S. Bultje's avatar
Ronald S. Bultje committed
437
  int i;
438
  int64_t error = 0, sqcoeff = 0;
John Koleszar's avatar
John Koleszar committed
439

440
  for (i = 0; i < block_size; i++) {
John Koleszar's avatar
John Koleszar committed
441
    int this_diff = coeff[i] - dqcoeff[i];
Ronald S. Bultje's avatar
Ronald S. Bultje committed
442
    error += (unsigned)this_diff * this_diff;
443
    sqcoeff += (unsigned) coeff[i] * coeff[i];
John Koleszar's avatar
John Koleszar committed
444
  }
John Koleszar's avatar
John Koleszar committed
445

446
  *ssz = sqcoeff;
John Koleszar's avatar
John Koleszar committed
447
  return error;
John Koleszar's avatar
John Koleszar committed
448
449
}

450
451
452
453
454
/* The trailing '0' is a terminator which is used inside cost_coeffs() to
 * decide whether to include cost of a trailing EOB node or not (i.e. we
 * can skip this if the last coefficient in this transform block, e.g. the
 * 16th coefficient in a 4x4 block or the 64th coefficient in a 8x8 block,
 * were non-zero). */
455
static const int16_t band_counts[TX_SIZES][8] = {
456
457
458
459
  { 1, 2, 3, 4,  3,   16 - 13, 0 },
  { 1, 2, 3, 4, 11,   64 - 21, 0 },
  { 1, 2, 3, 4, 11,  256 - 21, 0 },
  { 1, 2, 3, 4, 11, 1024 - 21, 0 },
460
461
};

462
static INLINE int cost_coeffs(MACROBLOCK *mb,
463
                              int plane, int block,
464
                              ENTROPY_CONTEXT *A, ENTROPY_CONTEXT *L,
John Koleszar's avatar
John Koleszar committed
465
                              TX_SIZE tx_size,
466
                              const int16_t *scan, const int16_t *nb) {
467
  MACROBLOCKD *const xd = &mb->e_mbd;
468
  MB_MODE_INFO *mbmi = &xd->this_mi->mbmi;
469
470
  struct macroblockd_plane *pd = &xd->plane[plane];
  const PLANE_TYPE type = pd->plane_type;
471
  const int16_t *band_count = &band_counts[tx_size][1];
472
473
  const int eob = pd->eobs[block];
  const int16_t *const qcoeff_ptr = BLOCK_OFFSET(pd->qcoeff, block);
Ronald S. Bultje's avatar
Ronald S. Bultje committed
474
  const int ref = mbmi->ref_frame[0] != INTRA_FRAME;
475
476
477
  unsigned int (*token_costs)[2][PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS] =
                   mb->token_costs[tx_size][type][ref];
  const ENTROPY_CONTEXT above_ec = !!*A, left_ec = !!*L;
478
  uint8_t token_cache[1024];
479
480
  int pt = combine_entropy_contexts(above_ec, left_ec);
  int c, cost;
481
482

  // Check for consistency of tx_size with mode info
483
  assert(type == PLANE_TYPE_Y_WITH_DC ? mbmi->tx_size == tx_size
484
                                      : get_uv_tx_size(mbmi) == tx_size);
485

486
487
  if (eob == 0) {
    // single eob token
488
489
    cost = token_costs[0][0][pt][DCT_EOB_TOKEN];
    c = 0;
490
  } else {
491
    int band_left = *band_count++;
492
493

    // dc token
494
495
    int v = qcoeff_ptr[0];
    int prev_t = vp9_dct_value_tokens_ptr[v].token;
496
    cost = (*token_costs)[0][pt][prev_t] + vp9_dct_value_cost_ptr[v];
497
    token_cache[0] = vp9_pt_energy_class[prev_t];
498
    ++token_costs;
499
500
501
502

    // ac tokens
    for (c = 1; c < eob; c++) {
      const int rc = scan[c];
503
      int t;
504
505
506

      v = qcoeff_ptr[rc];
      t = vp9_dct_value_tokens_ptr[v].token;
507
      pt = get_coef_context(nb, token_cache, c);
508
      cost += (*token_costs)[!prev_t][pt][t] + vp9_dct_value_cost_ptr[v];
509
      token_cache[rc] = vp9_pt_energy_class[t];
510
      prev_t = t;
511
      if (!--band_left) {
512
513
        band_left = *band_count++;
        ++token_costs;
514
      }
515
    }
516
517

    // eob token
518
    if (band_left) {
519
      pt = get_coef_context(nb, token_cache, c);
520
      cost += (*token_costs)[0][pt][DCT_EOB_TOKEN];
521
    }
522
523
  }

524
  // is eob first coefficient;
525
  *A = *L = (c > 0);
526

527
528
529
  return cost;
}

Deb Mukherjee's avatar
Deb Mukherjee committed
530
531
532
533
534
535
536
struct rdcost_block_args {
  MACROBLOCK *x;
  ENTROPY_CONTEXT t_above[16];
  ENTROPY_CONTEXT t_left[16];
  TX_SIZE tx_size;
  int bw;
  int bh;
537
538
539
540
541
542
543
  int rate[256];
  int64_t dist[256];
  int64_t sse[256];
  int this_rate;
  int64_t this_dist;
  int64_t this_sse;
  int64_t this_rd;
Deb Mukherjee's avatar
Deb Mukherjee committed
544
545
  int64_t best_rd;
  int skip;
546
  const int16_t *scan, *nb;
Deb Mukherjee's avatar
Deb Mukherjee committed
547
548
};

549
static void dist_block(int plane, int block, TX_SIZE tx_size, void *arg) {
550
  const int ss_txfrm_size = tx_size << 1;
Deb Mukherjee's avatar
Deb Mukherjee committed
551
552
553
  struct rdcost_block_args* args = arg;
  MACROBLOCK* const x = args->x;
  MACROBLOCKD* const xd = &x->e_mbd;
554
555
  struct macroblock_plane *const p = &x->plane[plane];
  struct macroblockd_plane *const pd = &xd->plane[plane];
Deb Mukherjee's avatar
Deb Mukherjee committed
556
557
  int64_t this_sse;
  int shift = args->tx_size == TX_32X32 ? 0 : 2;
558
559
  int16_t *const coeff = BLOCK_OFFSET(p->coeff, block);
  int16_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
560
  args->dist[block] = vp9_block_error(coeff, dqcoeff, 16 << ss_txfrm_size,
Deb Mukherjee's avatar
Deb Mukherjee committed
561
                                &this_sse) >> shift;
562
  args->sse[block]  = this_sse >> shift;
563
564

  if (x->skip_encode &&
565
      xd->this_mi->mbmi.ref_frame[0] == INTRA_FRAME) {
566
567
568
    // TODO(jingning): tune the model to better capture the distortion.
    int64_t p = (pd->dequant[1] * pd->dequant[1] *
                    (1 << ss_txfrm_size)) >> shift;
569
570
    args->dist[block] = p;
    args->sse[block]  = p;
571
  }
Deb Mukherjee's avatar
Deb Mukherjee committed
572
573
}

574
static void rate_block(int plane, int block, BLOCK_SIZE plane_bsize,
575
                       TX_SIZE tx_size, void *arg) {
Deb Mukherjee's avatar
Deb Mukherjee committed
576
  struct rdcost_block_args* args = arg;
577

Deb Mukherjee's avatar
Deb Mukherjee committed
578
  int x_idx, y_idx;
579
  txfrm_block_to_raster_xy(plane_bsize, args->tx_size, block, &x_idx, &y_idx);
Deb Mukherjee's avatar
Deb Mukherjee committed
580

581
582
583
584
  args->rate[block] = cost_coeffs(args->x, plane, block,
                                  args->t_above + x_idx,
                                  args->t_left + y_idx, args->tx_size,
                                  args->scan, args->nb);
Deb Mukherjee's avatar
Deb Mukherjee committed
585
586
}

587
static void block_yrd_txfm(int plane, int block, BLOCK_SIZE plane_bsize,
588
                           TX_SIZE tx_size, void *arg) {
Deb Mukherjee's avatar
Deb Mukherjee committed
589
590
591
  struct rdcost_block_args *args = arg;
  MACROBLOCK *const x = args->x;
  MACROBLOCKD *const xd = &x->e_mbd;
592
  struct encode_b_args encode_args = {x, NULL};
593
  int64_t rd1, rd2, rd;
Deb Mukherjee's avatar
Deb Mukherjee committed
594

595
596
597
  if (args->skip)
    return;

598
  if (!is_inter_block(&xd->this_mi->mbmi))
599
    vp9_encode_block_intra(plane, block, plane_bsize, tx_size, &encode_args);
Deb Mukherjee's avatar
Deb Mukherjee committed
600
  else
601
    vp9_xform_quant(plane, block, plane_bsize, tx_size, &encode_args);
Deb Mukherjee's avatar
Deb Mukherjee committed
602

603
604
  dist_block(plane, block, tx_size, args);
  rate_block(plane, block, plane_bsize, tx_size, args);
605
606
  rd1 = RDCOST(x->rdmult, x->rddiv, args->rate[block], args->dist[block]);
  rd2 = RDCOST(x->rdmult, x->rddiv, 0, args->sse[block]);
607
608

  // TODO(jingning): temporarily enabled only for luma component
609
  rd = MIN(rd1, rd2);
610
611
612
  if (plane == 0)
    x->zcoeff_blk[tx_size][block] = rd1 > rd2;

613
614
615
616
617
618
619
620
621
  args->this_rate += args->rate[block];
  args->this_dist += args->dist[block];
  args->this_sse  += args->sse[block];
  args->this_rd += rd;

  if (args->this_rd > args->best_rd) {
    args->skip = 1;
    return;
  }
Deb Mukherjee's avatar
Deb Mukherjee committed
622
623
}

624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
void vp9_get_entropy_contexts(TX_SIZE tx_size,
    ENTROPY_CONTEXT t_above[16], ENTROPY_CONTEXT t_left[16],
    const ENTROPY_CONTEXT *above, const ENTROPY_CONTEXT *left,
    int num_4x4_w, int num_4x4_h) {
  int i;
  switch (tx_size) {
    case TX_4X4:
      vpx_memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
      vpx_memcpy(t_left, left, sizeof(ENTROPY_CONTEXT) * num_4x4_h);
      break;
    case TX_8X8:
      for (i = 0; i < num_4x4_w; i += 2)
        t_above[i] = !!*(const uint16_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 2)
        t_left[i] = !!*(const uint16_t *)&left[i];
      break;
    case TX_16X16:
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
    case TX_32X32:
      for (i = 0; i < num_4x4_w; i += 8)
        t_above[i] = !!*(const uint64_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 8)
        t_left[i] = !!*(const uint64_t *)&left[i];
      break;
    default:
      assert(!"Invalid transform size.");
  }
}

657
static void txfm_rd_in_plane(MACROBLOCK *x,
658
659
660
                             int *rate, int64_t *distortion,
                             int *skippable, int64_t *sse,
                             int64_t ref_best_rd, int plane,
661
                             BLOCK_SIZE bsize, TX_SIZE tx_size) {
Deb Mukherjee's avatar
Deb Mukherjee committed
662
  MACROBLOCKD *const xd = &x->e_mbd;
663
  struct macroblockd_plane *const pd = &xd->plane[plane];
664
  const BLOCK_SIZE bs = get_plane_block_size(bsize, pd);
665
666
  const int num_4x4_w = num_4x4_blocks_wide_lookup[bs];
  const int num_4x4_h = num_4x4_blocks_high_lookup[bs];
667
  const uint8_t *band_translate;  // just for the get_scan_and_band call
668

669
  struct rdcost_block_args args = { x, { 0 }, { 0 }, tx_size,
670
                                    num_4x4_w, num_4x4_h,
671
672
                                    { 0 }, { 0 }, { 0 },
                                    0, 0, 0, 0, ref_best_rd, 0 };
673
  if (plane == 0)
674
    xd->this_mi->mbmi.tx_size = tx_size;
675

676
677
678
  vp9_get_entropy_contexts(tx_size, args.t_above, args.t_left,
                           pd->above_context, pd->left_context,
                           num_4x4_w, num_4x4_h);
679
680
681

  get_scan_and_band(xd, tx_size, pd->plane_type, 0, &args.scan, &args.nb,
                    &band_translate);
Deb Mukherjee's avatar
Deb Mukherjee committed
682

683
  foreach_transformed_block_in_plane(xd, bsize, plane, block_yrd_txfm, &args);
684
685
686
687
688
689
690
691
692
693
694
  if (args.skip) {
    *rate       = INT_MAX;
    *distortion = INT64_MAX;
    *sse        = INT64_MAX;
    *skippable  = 0;
  } else {
    *distortion = args.this_dist;
    *rate       = args.this_rate;
    *sse        = args.this_sse;
    *skippable  = vp9_is_skippable_in_plane(xd, bsize, plane);
  }
Deb Mukherjee's avatar
Deb Mukherjee committed
695
696
697
698
699
}

static void choose_largest_txfm_size(VP9_COMP *cpi, MACROBLOCK *x,
                                     int *rate, int64_t *distortion,
                                     int *skip, int64_t *sse,
700
                                     int64_t ref_best_rd,
701
                                     BLOCK_SIZE bs) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
702
  const TX_SIZE max_tx_size = max_txsize_lookup[bs];
Deb Mukherjee's avatar
Deb Mukherjee committed
703
704
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
705
  MB_MODE_INFO *const mbmi = &xd->this_mi->mbmi;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
706
  if (max_tx_size == TX_32X32 &&
707
708
      (cm->tx_mode == ALLOW_32X32 ||
       cm->tx_mode == TX_MODE_SELECT)) {
709
    mbmi->tx_size = TX_32X32;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
710
  } else if (max_tx_size >= TX_16X16 &&
711
712
713
             (cm->tx_mode == ALLOW_16X16 ||
              cm->tx_mode == ALLOW_32X32 ||
              cm->tx_mode == TX_MODE_SELECT)) {
714
    mbmi->tx_size = TX_16X16;
715
  } else if (cm->tx_mode != ONLY_4X4) {
716
    mbmi->tx_size = TX_8X8;
Deb Mukherjee's avatar
Deb Mukherjee committed
717
  } else {
718
    mbmi->tx_size = TX_4X4;
Deb Mukherjee's avatar
Deb Mukherjee committed
719
  }
720
  txfm_rd_in_plane(x, rate, distortion, skip,
721
722
                   &sse[mbmi->tx_size], ref_best_rd, 0, bs,
                   mbmi->tx_size);
Dmitry Kovalev's avatar
Dmitry Kovalev committed
723
  cpi->tx_stepdown_count[0]++;
Deb Mukherjee's avatar
Deb Mukherjee committed
724
725
}

726
static void choose_txfm_size_from_rd(VP9_COMP *cpi, MACROBLOCK *x,
727
                                     int (*r)[2], int *rate,
Ronald S. Bultje's avatar
Ronald S. Bultje committed
728
                                     int64_t *d, int64_t *distortion,
729
                                     int *s, int *skip,
730
                                     int64_t tx_cache[TX_MODES],
731
                                     BLOCK_SIZE bs) {
732
  const TX_SIZE max_tx_size = max_txsize_lookup[bs];
733
734
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
735
  MB_MODE_INFO *const mbmi = &xd->this_mi->mbmi;
736
  vp9_prob skip_prob = vp9_get_pred_prob_mbskip(cm, xd);
737
  int64_t rd[TX_SIZES][2];
738
  int n, m;
739
  int s0, s1;
740

741
  const vp9_prob *tx_probs = get_tx_probs2(xd, &cm->fc.tx_probs, xd->this_mi);
742

743
  for (n = TX_4X4; n <= max_tx_size; n++) {
744
    r[n][1] = r[n][0];
745
746
    if (r[n][0] == INT_MAX)
      continue;
747
    for (m = 0; m <= n - (n == max_tx_size); m++) {
748
      if (m == n)
749
        r[n][1] += vp9_cost_zero(tx_probs[m]);
750
      else
751
        r[n][1] += vp9_cost_one(tx_probs[m]);
752
753
    }
  }
754

755
756
757
  assert(skip_prob > 0);
  s0 = vp9_cost_bit(skip_prob, 0);
  s1 = vp9_cost_bit(skip_prob, 1);
758

759
  for (n = TX_4X4; n <= max_tx_size; n++) {
760
761
762
763
    if (d[n] == INT64_MAX) {
      rd[n][0] = rd[n][1] = INT64_MAX;
      continue;
    }
764
765
766
767
768
    if (s[n]) {
      rd[n][0] = rd[n][1] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
    } else {
      rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0] + s0, d[n]);
      rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1] + s0, d[n]);
769
770
771
    }
  }

772
  if (max_tx_size == TX_32X32 &&
773
774
      (cm->tx_mode == ALLOW_32X32 ||
       (cm->tx_mode == TX_MODE_SELECT &&
775
776
        rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
        rd[TX_32X32][1] < rd[TX_4X4][1]))) {
777
    mbmi->tx_size = TX_32X32;
778
  } else if (max_tx_size >= TX_16X16 &&
779
780
781
             (cm->tx_mode == ALLOW_16X16 ||
              cm->tx_mode == ALLOW_32X32 ||
              (cm->tx_mode == TX_MODE_SELECT &&
782
783
               rd[TX_16X16][1] < rd[TX_8X8][1] &&
               rd[TX_16X16][1] < rd[TX_4X4][1]))) {
784
    mbmi->tx_size = TX_16X16;
785
786
787
788
  } else if (cm->tx_mode == ALLOW_8X8 ||
             cm->tx_mode == ALLOW_16X16 ||
             cm->tx_mode == ALLOW_32X32 ||
           (cm->tx_mode == TX_MODE_SELECT && rd[TX_8X8][1] < rd[TX_4X4][1])) {
789
    mbmi->tx_size = TX_8X8;
790
  } else {
791
    mbmi->tx_size = TX_4X4;
792
793
  }

794
795
796
  *distortion = d[mbmi->tx_size];
  *rate       = r[mbmi->tx_size][cm->tx_mode == TX_MODE_SELECT];
  *skip       = s[mbmi->tx_size];
797

798
799
800
801
802
  tx_cache[ONLY_4X4] = rd[TX_4X4][0];
  tx_cache[ALLOW_8X8] = rd[TX_8X8][0];
  tx_cache[ALLOW_16X16] = rd[MIN(max_tx_size, TX_16X16)][0];
  tx_cache[ALLOW_32X32] = rd[MIN(max_tx_size, TX_32X32)][0];
  if (max_tx_size == TX_32X32 &&
803
804
      rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
      rd[TX_32X32][1] < rd[TX_4X4][1])
805
806
    tx_cache[TX_MODE_SELECT] = rd[TX_32X32][1];
  else if (max_tx_size >= TX_16X16 &&
807
           rd[TX_16X16][1] < rd[TX_8X8][1] && rd[TX_16X16][1] < rd[TX_4X4][1])
808
    tx_cache[TX_MODE_SELECT] = rd[TX_16X16][1];
809
  else
810
    tx_cache[TX_MODE_SELECT] = rd[TX_4X4][1] < rd[TX_8X8][1] ?
811
                                 rd[TX_4X4][1] : rd[TX_8X8][1];
812

813
  if (max_tx_size == TX_32X32 &&
Deb Mukherjee's avatar
Deb Mukherjee committed
814
815
816
      rd[TX_32X32][1] < rd[TX_16X16][1] &&
      rd[TX_32X32][1] < rd[TX_8X8][1] &&
      rd[TX_32X32][1] < rd[TX_4X4][1]) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
817
    cpi->tx_stepdown_count[0]++;
818
  } else if (max_tx_size >= TX_16X16 &&
Deb Mukherjee's avatar
Deb Mukherjee committed
819
820
             rd[TX_16X16][1] < rd[TX_8X8][1] &&
             rd[TX_16X16][1] < rd[TX_4X4][1]) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
821
    cpi->tx_stepdown_count[max_tx_size - TX_16X16]++;
Deb Mukherjee's avatar
Deb Mukherjee committed
822
  } else if (rd[TX_8X8][1] < rd[TX_4X4][1]) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
823
    cpi->tx_stepdown_count[max_tx_size - TX_8X8]++;
Deb Mukherjee's avatar
Deb Mukherjee committed
824
  } else {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
825
    cpi->tx_stepdown_count[max_tx_size - TX_4X4]++;
826
  }
Deb Mukherjee's avatar
Deb Mukherjee committed
827
}
828

Deb Mukherjee's avatar
Deb Mukherjee committed
829
830
831
832
static void choose_txfm_size_from_modelrd(VP9_COMP *cpi, MACROBLOCK *x,
                                          int (*r)[2], int *rate,
                                          int64_t *d, int64_t *distortion,
                                          int *s, int *skip, int64_t *sse,
833
                                          int64_t ref_best_rd,
834
                                          BLOCK_SIZE bs) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
835
  const TX_SIZE max_tx_size = max_txsize_lookup[bs];
Deb Mukherjee's avatar
Deb Mukherjee committed
836
837
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
838
  MB_MODE_INFO *const mbmi = &xd->this_mi->mbmi;
839
  vp9_prob skip_prob = vp9_get_pred_prob_mbskip(cm, xd);
840
  int64_t rd[TX_SIZES][2];
Deb Mukherjee's avatar
Deb Mukherjee committed
841
842
  int n, m;
  int s0, s1;
843
844
  double scale_rd[TX_SIZES] = {1.73, 1.44, 1.20, 1.00};
  // double scale_r[TX_SIZES] = {2.82, 2.00, 1.41, 1.00};
845

846
  const vp9_prob *tx_probs = get_tx_probs2(xd, &cm->fc.tx_probs,  xd->this_mi);
847

Deb Mukherjee's avatar
Deb Mukherjee committed
848
849
  // for (n = TX_4X4; n <= max_txfm_size; n++)
  //   r[n][0] = (r[n][0] * scale_r[n]);
850

Dmitry Kovalev's avatar
Dmitry Kovalev committed
851
  for (n = TX_4X4; n <= max_tx_size; n++) {
Deb Mukherjee's avatar
Deb Mukherjee committed
852
    r[n][1] = r[n][0];
Dmitry Kovalev's avatar
Dmitry Kovalev committed
853
    for (m = 0; m <= n - (n == max_tx_size); m++) {
Deb Mukherjee's avatar
Deb Mukherjee committed
854
855
856
857
858
      if (m == n)
        r[n][1] += vp9_cost_zero(tx_probs[m]);
      else
        r[n][1] += vp9_cost_one(tx_probs[m]);
    }
859
  }
860

Deb Mukherjee's avatar
Deb Mukherjee committed
861
862
863
  assert(skip_prob > 0);
  s0 = vp9_cost_bit(skip_prob, 0);
  s1 = vp9_cost_bit(skip_prob, 1);
864

Dmitry Kovalev's avatar
Dmitry Kovalev committed
865
  for (n = TX_4X4; n <= max_tx_size; n++) {
Deb Mukherjee's avatar
Deb Mukherjee committed
866
867
868
869
870
871
872
    if (s[n]) {
      rd[n][0] = rd[n][1] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
    } else {
      rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0] + s0, d[n]);
      rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1] + s0, d[n]);
    }
  }
Dmitry Kovalev's avatar
Dmitry Kovalev committed
873
  for (n = TX_4X4; n <= max_tx_size; n++) {
Yaowu Xu's avatar
Yaowu Xu committed
874
875
    rd[n][0] = (int64_t)(scale_rd[n] * rd[n][0]);
    rd[n][1] = (int64_t)(scale_rd[n] * rd[n][1]);
Deb Mukherjee's avatar
Deb Mukherjee committed
876
  }
877

Dmitry Kovalev's avatar
Dmitry Kovalev committed
878
  if (max_tx_size == TX_32X32 &&
879
880
      (cm->tx_mode == ALLOW_32X32 ||
       (cm->tx_mode == TX_MODE_SELECT &&
Deb Mukherjee's avatar
Deb Mukherjee committed
881
882
883
        rd[TX_32X32][1] <= rd[TX_16X16][1] &&
        rd[TX_32X32][1] <= rd[TX_8X8][1] &&
        rd[TX_32X32][1] <= rd[TX_4X4][1]))) {
884
    mbmi->tx_size = TX_32X32;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
885
  } else if (max_tx_size >= TX_16X16 &&
886
887
888
             (cm->tx_mode == ALLOW_16X16 ||
              cm->tx_mode == ALLOW_32X32 ||
              (cm->tx_mode == TX_MODE_SELECT &&
Deb Mukherjee's avatar
Deb Mukherjee committed
889
890
               rd[TX_16X16][1] <= rd[TX_8X8][1] &&
               rd[TX_16X16][1] <= rd[TX_4X4][1]))) {
891
    mbmi->tx_size = TX_16X16;
892
893
894
895
  } else if (cm->tx_mode == ALLOW_8X8 ||
             cm->tx_mode == ALLOW_16X16 ||
             cm->tx_mode == ALLOW_32X32 ||
           (cm->tx_mode == TX_MODE_SELECT &&
Deb Mukherjee's avatar
Deb Mukherjee committed
896
            rd[TX_8X8][1] <= rd[TX_4X4][1])) {