vp9_rdopt.c 166 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5
6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9
10
11
12
13
14
 */

#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
15

16
#include "vp9/common/vp9_pragmas.h"
17
18
19
20
21
#include "vp9/encoder/vp9_tokenize.h"
#include "vp9/encoder/vp9_treewriter.h"
#include "vp9/encoder/vp9_onyx_int.h"
#include "vp9/encoder/vp9_modecosts.h"
#include "vp9/encoder/vp9_encodeintra.h"
22
23
24
25
26
#include "vp9/common/vp9_entropymode.h"
#include "vp9/common/vp9_reconinter.h"
#include "vp9/common/vp9_reconintra.h"
#include "vp9/common/vp9_findnearmv.h"
#include "vp9/common/vp9_quant_common.h"
27
28
29
30
31
32
#include "vp9/encoder/vp9_encodemb.h"
#include "vp9/encoder/vp9_quantize.h"
#include "vp9/encoder/vp9_variance.h"
#include "vp9/encoder/vp9_mcomp.h"
#include "vp9/encoder/vp9_rdopt.h"
#include "vp9/encoder/vp9_ratectrl.h"
John Koleszar's avatar
John Koleszar committed
33
#include "vpx_mem/vpx_mem.h"
34
35
36
37
38
#include "vp9/common/vp9_systemdependent.h"
#include "vp9/encoder/vp9_encodemv.h"
#include "vp9/common/vp9_seg_common.h"
#include "vp9/common/vp9_pred_common.h"
#include "vp9/common/vp9_entropy.h"
39
#include "./vp9_rtcd.h"
40
#include "vp9/common/vp9_mvref_common.h"
Ronald S. Bultje's avatar
Ronald S. Bultje committed
41
#include "vp9/common/vp9_common.h"
Paul Wilkins's avatar
Paul Wilkins committed
42

43
44
#define INVALID_MV 0x80008000

45
46
47
/* Factor to weigh the rate for switchable interp filters */
#define SWITCHABLE_INTERP_RATE_FACTOR 1

48
49
50
DECLARE_ALIGNED(16, extern const uint8_t,
                vp9_pt_energy_class[MAX_ENTROPY_TOKENS]);

51
52
53
#define LAST_FRAME_MODE_MASK    0xFFEDCD60
#define GOLDEN_FRAME_MODE_MASK  0xFFDA3BB0
#define ALT_REF_MODE_MASK       0xFFC648D0
54

Paul Wilkins's avatar
Paul Wilkins committed
55
56
#define MIN_EARLY_TERM_INDEX    3

57
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
  {NEARESTMV, LAST_FRAME,   NONE},
  {NEARESTMV, ALTREF_FRAME, NONE},
  {NEARESTMV, GOLDEN_FRAME, NONE},

  {DC_PRED,   INTRA_FRAME,  NONE},

  {NEWMV,     LAST_FRAME,   NONE},
  {NEWMV,     ALTREF_FRAME, NONE},
  {NEWMV,     GOLDEN_FRAME, NONE},

  {NEARMV,    LAST_FRAME,   NONE},
  {NEARMV,    ALTREF_FRAME, NONE},
  {NEARESTMV, LAST_FRAME,   ALTREF_FRAME},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},

  {TM_PRED,   INTRA_FRAME,  NONE},

  {NEARMV,    LAST_FRAME,   ALTREF_FRAME},
  {NEWMV,     LAST_FRAME,   ALTREF_FRAME},
  {NEARMV,    GOLDEN_FRAME, NONE},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME},
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME},

  {ZEROMV,    LAST_FRAME,   NONE},
  {ZEROMV,    GOLDEN_FRAME, NONE},
  {ZEROMV,    ALTREF_FRAME, NONE},
  {ZEROMV,    LAST_FRAME,   ALTREF_FRAME},
  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME},

  {H_PRED,    INTRA_FRAME,  NONE},
  {V_PRED,    INTRA_FRAME,  NONE},
  {D135_PRED, INTRA_FRAME,  NONE},
  {D207_PRED, INTRA_FRAME,  NONE},
  {D153_PRED, INTRA_FRAME,  NONE},
  {D63_PRED,  INTRA_FRAME,  NONE},
  {D117_PRED, INTRA_FRAME,  NONE},
  {D45_PRED,  INTRA_FRAME,  NONE},
};

const REF_DEFINITION vp9_ref_order[MAX_REFS] = {
  {LAST_FRAME,   NONE},
  {GOLDEN_FRAME, NONE},
  {ALTREF_FRAME, NONE},
  {LAST_FRAME,   ALTREF_FRAME},
  {GOLDEN_FRAME, ALTREF_FRAME},
  {INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
104
105
};

106
107
108
109
// The baseline rd thresholds for breaking out of the rd loop for
// certain modes are assumed to be based on 8x8 blocks.
// This table is used to correct for blocks size.
// The factors here are << 2 (2 = x0.5, 32 = x8 etc).
110
static int rd_thresh_block_size_factor[BLOCK_SIZES] =
111
112
  {2, 3, 3, 4, 6, 6, 8, 12, 12, 16, 24, 24, 32};

113
114
#define MAX_RD_THRESH_FACT 64
#define RD_THRESH_INC 1
115

116
static void fill_token_costs(vp9_coeff_cost *c,
117
                             vp9_coeff_probs_model (*p)[BLOCK_TYPES]) {
118
  int i, j, k, l;
119
120
121
122
123
124
125
126
  TX_SIZE t;
  for (t = TX_4X4; t <= TX_32X32; t++)
    for (i = 0; i < BLOCK_TYPES; i++)
      for (j = 0; j < REF_TYPES; j++)
        for (k = 0; k < COEF_BANDS; k++)
          for (l = 0; l < PREV_COEF_CONTEXTS; l++) {
            vp9_prob probs[ENTROPY_NODES];
            vp9_model_to_full_probs(p[t][i][j][k][l], probs);
127
            vp9_cost_tokens((int *)c[t][i][j][k][0][l], probs,
128
                            vp9_coef_tree);
129
            vp9_cost_tokens_skip((int *)c[t][i][j][k][1][l], probs,
130
                                 vp9_coef_tree);
131
132
            assert(c[t][i][j][k][0][l][DCT_EOB_TOKEN] ==
                   c[t][i][j][k][1][l][DCT_EOB_TOKEN]);
133
          }
134
135
}

136
137
138
139
140
141
static const int rd_iifactor[32] = {
  4, 4, 3, 2, 1, 0, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 0,
};
John Koleszar's avatar
John Koleszar committed
142

143
// 3* dc_qlookup[Q]*dc_qlookup[Q];
144

145
/* values are now correlated to quantizer */
Paul Wilkins's avatar
Paul Wilkins committed
146
147
148
static int sad_per_bit16lut[QINDEX_RANGE];
static int sad_per_bit4lut[QINDEX_RANGE];

149
void vp9_init_me_luts() {
John Koleszar's avatar
John Koleszar committed
150
151
152
153
154
155
156
  int i;

  // Initialize the sad lut tables using a formulaic calculation for now
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < QINDEX_RANGE; i++) {
    sad_per_bit16lut[i] =
157
      (int)((0.0418 * vp9_convert_qindex_to_q(i)) + 2.4107);
158
    sad_per_bit4lut[i] = (int)(0.063 * vp9_convert_qindex_to_q(i) + 2.742);
John Koleszar's avatar
John Koleszar committed
159
  }
Paul Wilkins's avatar
Paul Wilkins committed
160
}
John Koleszar's avatar
John Koleszar committed
161

162
static int compute_rd_mult(int qindex) {
163
  const int q = vp9_dc_quant(qindex, 0);
164
  return (11 * q * q) >> 2;
165
166
}

Dmitry Kovalev's avatar
Dmitry Kovalev committed
167
168
169
void vp9_initialize_me_consts(VP9_COMP *cpi, int qindex) {
  cpi->mb.sadperbit16 = sad_per_bit16lut[qindex];
  cpi->mb.sadperbit4 = sad_per_bit4lut[qindex];
John Koleszar's avatar
John Koleszar committed
170
171
}

Dmitry Kovalev's avatar
Dmitry Kovalev committed
172
void vp9_initialize_rd_consts(VP9_COMP *cpi, int qindex) {
173
  int q, i, bsize;
John Koleszar's avatar
John Koleszar committed
174

175
  vp9_clear_system_state();  // __asm emms;
John Koleszar's avatar
John Koleszar committed
176

John Koleszar's avatar
John Koleszar committed
177
178
179
180
  // Further tests required to see if optimum is different
  // for key frames, golden frames and arf frames.
  // if (cpi->common.refresh_golden_frame ||
  //     cpi->common.refresh_alt_ref_frame)
181
  qindex = clamp(qindex, 0, MAXQ);
182

183
  cpi->RDDIV = 100;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
184
  cpi->RDMULT = compute_rd_mult(qindex);
John Koleszar's avatar
John Koleszar committed
185
186
187
188
189
  if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    if (cpi->twopass.next_iiratio > 31)
      cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
    else
      cpi->RDMULT +=
Dmitry Kovalev's avatar
Dmitry Kovalev committed
190
          (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
John Koleszar's avatar
John Koleszar committed
191
  }
192
  cpi->mb.errorperbit = cpi->RDMULT >> 6;
John Koleszar's avatar
John Koleszar committed
193
  cpi->mb.errorperbit += (cpi->mb.errorperbit == 0);
194

195
  vp9_set_speed_features(cpi);
John Koleszar's avatar
John Koleszar committed
196

Dmitry Kovalev's avatar
Dmitry Kovalev committed
197
198
  q = (int)pow(vp9_dc_quant(qindex, 0) >> 2, 1.25);
  q <<= 2;
John Koleszar's avatar
John Koleszar committed
199
200
  if (q < 8)
    q = 8;
201

202
  for (bsize = 0; bsize < BLOCK_SIZES; ++bsize) {
203
    for (i = 0; i < MAX_MODES; ++i) {
204
205
206
      // Threshold here seem unecessarily harsh but fine given actual
      // range of values used for cpi->sf.thresh_mult[]
      int thresh_max = INT_MAX / (q * rd_thresh_block_size_factor[bsize]);
John Koleszar's avatar
John Koleszar committed
207

208
209
      if (cpi->sf.thresh_mult[i] < thresh_max) {
        cpi->rd_threshes[bsize][i] =
210
211
            cpi->sf.thresh_mult[i] * q *
            rd_thresh_block_size_factor[bsize] / 4;
212
213
      } else {
        cpi->rd_threshes[bsize][i] = INT_MAX;
John Koleszar's avatar
John Koleszar committed
214
      }
John Koleszar's avatar
John Koleszar committed
215
    }
216
217
218
219
220
221
222
223
224
225
226
227

    for (i = 0; i < MAX_REFS; ++i) {
      int thresh_max = INT_MAX / (q * rd_thresh_block_size_factor[bsize]);

      if (cpi->sf.thresh_mult_sub8x8[i] < thresh_max) {
        cpi->rd_thresh_sub8x8[bsize][i] =
            cpi->sf.thresh_mult_sub8x8[i] * q *
            rd_thresh_block_size_factor[bsize] / 4;
      } else {
        cpi->rd_thresh_sub8x8[bsize][i] = INT_MAX;
      }
    }
John Koleszar's avatar
John Koleszar committed
228
  }
John Koleszar's avatar
John Koleszar committed
229

230
  fill_token_costs(cpi->mb.token_costs, cpi->common.fc.coef_probs);
231

232
  for (i = 0; i < NUM_PARTITION_CONTEXTS; i++)
233
    vp9_cost_tokens(cpi->mb.partition_cost[i],
234
                    cpi->common.fc.partition_prob[cpi->common.frame_type][i],
235
236
                    vp9_partition_tree);

John Koleszar's avatar
John Koleszar committed
237
  /*rough estimate for costing*/
238
  vp9_init_mode_costs(cpi);
John Koleszar's avatar
John Koleszar committed
239

240
  if (cpi->common.frame_type != KEY_FRAME) {
241
    vp9_build_nmv_cost_table(
242
243
244
245
246
        cpi->mb.nmvjointcost,
        cpi->mb.e_mbd.allow_high_precision_mv ?
        cpi->mb.nmvcost_hp : cpi->mb.nmvcost,
        &cpi->common.fc.nmvc,
        cpi->mb.e_mbd.allow_high_precision_mv, 1, 1);
247
248
249
250
251

    for (i = 0; i < INTER_MODE_CONTEXTS; i++) {
      MB_PREDICTION_MODE m;

      for (m = NEARESTMV; m < MB_MODE_COUNT; m++)
252
        cpi->mb.inter_mode_cost[i][inter_mode_offset(m)] =
253
            cost_token(vp9_inter_mode_tree,
254
                       cpi->common.fc.inter_mode_probs[i],
255
                       vp9_inter_mode_encodings + inter_mode_offset(m));
256
    }
257
  }
John Koleszar's avatar
John Koleszar committed
258
259
}

Yaowu Xu's avatar
Yaowu Xu committed
260
static INLINE void linear_interpolate2(double x, int ntab, int inv_step,
261
262
                                       const double *tab1, const double *tab2,
                                       double *v1, double *v2) {
Deb Mukherjee's avatar
Deb Mukherjee committed
263
264
265
  double y = x * inv_step;
  int d = (int) y;
  if (d >= ntab - 1) {
266
267
    *v1 = tab1[ntab - 1];
    *v2 = tab2[ntab - 1];
Deb Mukherjee's avatar
Deb Mukherjee committed
268
269
  } else {
    double a = y - d;
270
271
    *v1 = tab1[d] * (1 - a) + tab1[d + 1] * a;
    *v2 = tab2[d] * (1 - a) + tab2[d + 1] * a;
Deb Mukherjee's avatar
Deb Mukherjee committed
272
273
274
  }
}

275
276
277
278
279
static void model_rd_norm(double x, double *R, double *D) {
  static const int inv_tab_step = 8;
  static const int tab_size = 120;
  // NOTE: The tables below must be of the same size
  //
Deb Mukherjee's avatar
Deb Mukherjee committed
280
  // Normalized rate
281
  // This table models the rate for a Laplacian source
Deb Mukherjee's avatar
Deb Mukherjee committed
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
  // source with given variance when quantized with a uniform quantizer
  // with given stepsize. The closed form expression is:
  // Rn(x) = H(sqrt(r)) + sqrt(r)*[1 + H(r)/(1 - r)],
  // where r = exp(-sqrt(2) * x) and x = qpstep / sqrt(variance),
  // and H(x) is the binary entropy function.
  static const double rate_tab[] = {
    64.00, 4.944, 3.949, 3.372, 2.966, 2.655, 2.403, 2.194,
    2.014, 1.858, 1.720, 1.596, 1.485, 1.384, 1.291, 1.206,
    1.127, 1.054, 0.986, 0.923, 0.863, 0.808, 0.756, 0.708,
    0.662, 0.619, 0.579, 0.541, 0.506, 0.473, 0.442, 0.412,
    0.385, 0.359, 0.335, 0.313, 0.291, 0.272, 0.253, 0.236,
    0.220, 0.204, 0.190, 0.177, 0.165, 0.153, 0.142, 0.132,
    0.123, 0.114, 0.106, 0.099, 0.091, 0.085, 0.079, 0.073,
    0.068, 0.063, 0.058, 0.054, 0.050, 0.047, 0.043, 0.040,
    0.037, 0.034, 0.032, 0.029, 0.027, 0.025, 0.023, 0.022,
    0.020, 0.019, 0.017, 0.016, 0.015, 0.014, 0.013, 0.012,
    0.011, 0.010, 0.009, 0.008, 0.008, 0.007, 0.007, 0.006,
    0.006, 0.005, 0.005, 0.005, 0.004, 0.004, 0.004, 0.003,
    0.003, 0.003, 0.003, 0.002, 0.002, 0.002, 0.002, 0.002,
    0.002, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001,
    0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.000,
  };
  // Normalized distortion
305
  // This table models the normalized distortion for a Laplacian source
Deb Mukherjee's avatar
Deb Mukherjee committed
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
  // source with given variance when quantized with a uniform quantizer
  // with given stepsize. The closed form expression is:
  // Dn(x) = 1 - 1/sqrt(2) * x / sinh(x/sqrt(2))
  // where x = qpstep / sqrt(variance)
  // Note the actual distortion is Dn * variance.
  static const double dist_tab[] = {
    0.000, 0.001, 0.005, 0.012, 0.021, 0.032, 0.045, 0.061,
    0.079, 0.098, 0.119, 0.142, 0.166, 0.190, 0.216, 0.242,
    0.269, 0.296, 0.324, 0.351, 0.378, 0.405, 0.432, 0.458,
    0.484, 0.509, 0.534, 0.557, 0.580, 0.603, 0.624, 0.645,
    0.664, 0.683, 0.702, 0.719, 0.735, 0.751, 0.766, 0.780,
    0.794, 0.807, 0.819, 0.830, 0.841, 0.851, 0.861, 0.870,
    0.878, 0.886, 0.894, 0.901, 0.907, 0.913, 0.919, 0.925,
    0.930, 0.935, 0.939, 0.943, 0.947, 0.951, 0.954, 0.957,
    0.960, 0.963, 0.966, 0.968, 0.971, 0.973, 0.975, 0.976,
    0.978, 0.980, 0.981, 0.982, 0.984, 0.985, 0.986, 0.987,
    0.988, 0.989, 0.990, 0.990, 0.991, 0.992, 0.992, 0.993,
    0.993, 0.994, 0.994, 0.995, 0.995, 0.996, 0.996, 0.996,
    0.996, 0.997, 0.997, 0.997, 0.997, 0.998, 0.998, 0.998,
    0.998, 0.998, 0.998, 0.999, 0.999, 0.999, 0.999, 0.999,
    0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 1.000,
  };
328
329
330
331
332
  /*
  assert(sizeof(rate_tab) == tab_size * sizeof(rate_tab[0]);
  assert(sizeof(dist_tab) == tab_size * sizeof(dist_tab[0]);
  assert(sizeof(rate_tab) == sizeof(dist_tab));
  */
Deb Mukherjee's avatar
Deb Mukherjee committed
333
  assert(x >= 0.0);
334
335
  linear_interpolate2(x, tab_size, inv_tab_step,
                      rate_tab, dist_tab, R, D);
Deb Mukherjee's avatar
Deb Mukherjee committed
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
}

static void model_rd_from_var_lapndz(int var, int n, int qstep,
                                     int *rate, int64_t *dist) {
  // This function models the rate and distortion for a Laplacian
  // source with given variance when quantized with a uniform quantizer
  // with given stepsize. The closed form expressions are in:
  // Hang and Chen, "Source Model for transform video coder and its
  // application - Part I: Fundamental Theory", IEEE Trans. Circ.
  // Sys. for Video Tech., April 1997.
  vp9_clear_system_state();
  if (var == 0 || n == 0) {
    *rate = 0;
    *dist = 0;
  } else {
    double D, R;
    double s2 = (double) var / n;
    double x = qstep / sqrt(s2);
354
    model_rd_norm(x, &R, &D);
Yaowu Xu's avatar
Yaowu Xu committed
355
356
    *rate = (int)((n << 8) * R + 0.5);
    *dist = (int)(var * D + 0.5);
Deb Mukherjee's avatar
Deb Mukherjee committed
357
358
359
360
  }
  vp9_clear_system_state();
}

361
static void model_rd_for_sb(VP9_COMP *cpi, BLOCK_SIZE bsize,
Deb Mukherjee's avatar
Deb Mukherjee committed
362
363
364
365
366
367
368
369
370
371
                            MACROBLOCK *x, MACROBLOCKD *xd,
                            int *out_rate_sum, int64_t *out_dist_sum) {
  // Note our transform coeffs are 8 times an orthogonal transform.
  // Hence quantizer step is also 8 times. To get effective quantizer
  // we need to divide by 8 before sending to modeling function.
  int i, rate_sum = 0, dist_sum = 0;

  for (i = 0; i < MAX_MB_PLANE; ++i) {
    struct macroblock_plane *const p = &x->plane[i];
    struct macroblockd_plane *const pd = &xd->plane[i];
372
    const BLOCK_SIZE bs = get_plane_block_size(bsize, pd);
Deb Mukherjee's avatar
Deb Mukherjee committed
373
374
375
376
377
378
    unsigned int sse;
    int rate;
    int64_t dist;
    (void) cpi->fn_ptr[bs].vf(p->src.buf, p->src.stride,
                              pd->dst.buf, pd->dst.stride, &sse);
    // sse works better than var, since there is no dc prediction used
Dmitry Kovalev's avatar
Dmitry Kovalev committed
379
    model_rd_from_var_lapndz(sse, 1 << num_pels_log2_lookup[bs],
380
                             pd->dequant[1] >> 3, &rate, &dist);
Deb Mukherjee's avatar
Deb Mukherjee committed
381
382

    rate_sum += rate;
Yaowu Xu's avatar
Yaowu Xu committed
383
    dist_sum += (int)dist;
Deb Mukherjee's avatar
Deb Mukherjee committed
384
385
386
387
388
389
  }

  *out_rate_sum = rate_sum;
  *out_dist_sum = dist_sum << 4;
}

390
static void model_rd_for_sb_y_tx(VP9_COMP *cpi, BLOCK_SIZE bsize,
Deb Mukherjee's avatar
Deb Mukherjee committed
391
392
393
394
                                 TX_SIZE tx_size,
                                 MACROBLOCK *x, MACROBLOCKD *xd,
                                 int *out_rate_sum, int64_t *out_dist_sum,
                                 int *out_skip) {
395
  int j, k;
396
  BLOCK_SIZE bs;
Deb Mukherjee's avatar
Deb Mukherjee committed
397
398
  struct macroblock_plane *const p = &x->plane[0];
  struct macroblockd_plane *const pd = &xd->plane[0];
399
400
  const int width = 4 << num_4x4_blocks_wide_lookup[bsize];
  const int height = 4 << num_4x4_blocks_high_lookup[bsize];
Deb Mukherjee's avatar
Deb Mukherjee committed
401
402
  int rate_sum = 0;
  int64_t dist_sum = 0;
403
  const int t = 4 << tx_size;
Deb Mukherjee's avatar
Deb Mukherjee committed
404
405
406
407
408
409
410
411
412
413
414
415

  if (tx_size == TX_4X4) {
    bs = BLOCK_4X4;
  } else if (tx_size == TX_8X8) {
    bs = BLOCK_8X8;
  } else if (tx_size == TX_16X16) {
    bs = BLOCK_16X16;
  } else if (tx_size == TX_32X32) {
    bs = BLOCK_32X32;
  } else {
    assert(0);
  }
416

Deb Mukherjee's avatar
Deb Mukherjee committed
417
  *out_skip = 1;
Jim Bankoski's avatar
Jim Bankoski committed
418
419
  for (j = 0; j < height; j += t) {
    for (k = 0; k < width; k += t) {
Deb Mukherjee's avatar
Deb Mukherjee committed
420
421
422
      int rate;
      int64_t dist;
      unsigned int sse;
423
424
425
      cpi->fn_ptr[bs].vf(&p->src.buf[j * p->src.stride + k], p->src.stride,
                         &pd->dst.buf[j * pd->dst.stride + k], pd->dst.stride,
                         &sse);
Deb Mukherjee's avatar
Deb Mukherjee committed
426
      // sse works better than var, since there is no dc prediction used
427
      model_rd_from_var_lapndz(sse, t * t, pd->dequant[1] >> 3, &rate, &dist);
Deb Mukherjee's avatar
Deb Mukherjee committed
428
429
430
431
432
      rate_sum += rate;
      dist_sum += dist;
      *out_skip &= (rate < 1024);
    }
  }
433

Deb Mukherjee's avatar
Deb Mukherjee committed
434
  *out_rate_sum = rate_sum;
435
  *out_dist_sum = dist_sum << 4;
Deb Mukherjee's avatar
Deb Mukherjee committed
436
437
}

Ronald S. Bultje's avatar
Ronald S. Bultje committed
438
int64_t vp9_block_error_c(int16_t *coeff, int16_t *dqcoeff,
439
                          intptr_t block_size, int64_t *ssz) {
Ronald S. Bultje's avatar
Ronald S. Bultje committed
440
  int i;
441
  int64_t error = 0, sqcoeff = 0;
John Koleszar's avatar
John Koleszar committed
442

443
  for (i = 0; i < block_size; i++) {
John Koleszar's avatar
John Koleszar committed
444
    int this_diff = coeff[i] - dqcoeff[i];
Ronald S. Bultje's avatar
Ronald S. Bultje committed
445
    error += (unsigned)this_diff * this_diff;
446
    sqcoeff += (unsigned) coeff[i] * coeff[i];
John Koleszar's avatar
John Koleszar committed
447
  }
John Koleszar's avatar
John Koleszar committed
448

449
  *ssz = sqcoeff;
John Koleszar's avatar
John Koleszar committed
450
  return error;
John Koleszar's avatar
John Koleszar committed
451
452
}

453
454
455
456
457
/* The trailing '0' is a terminator which is used inside cost_coeffs() to
 * decide whether to include cost of a trailing EOB node or not (i.e. we
 * can skip this if the last coefficient in this transform block, e.g. the
 * 16th coefficient in a 4x4 block or the 64th coefficient in a 8x8 block,
 * were non-zero). */
458
static const int16_t band_counts[TX_SIZES][8] = {
459
460
461
462
  { 1, 2, 3, 4,  3,   16 - 13, 0 },
  { 1, 2, 3, 4, 11,   64 - 21, 0 },
  { 1, 2, 3, 4, 11,  256 - 21, 0 },
  { 1, 2, 3, 4, 11, 1024 - 21, 0 },
463
464
};

465
static INLINE int cost_coeffs(MACROBLOCK *mb,
466
                              int plane, int block,
467
                              ENTROPY_CONTEXT *A, ENTROPY_CONTEXT *L,
John Koleszar's avatar
John Koleszar committed
468
                              TX_SIZE tx_size,
469
                              const int16_t *scan, const int16_t *nb) {
470
  MACROBLOCKD *const xd = &mb->e_mbd;
471
  MB_MODE_INFO *mbmi = &xd->this_mi->mbmi;
472
473
  struct macroblockd_plane *pd = &xd->plane[plane];
  const PLANE_TYPE type = pd->plane_type;
474
  const int16_t *band_count = &band_counts[tx_size][1];
475
476
  const int eob = pd->eobs[block];
  const int16_t *const qcoeff_ptr = BLOCK_OFFSET(pd->qcoeff, block);
Ronald S. Bultje's avatar
Ronald S. Bultje committed
477
  const int ref = mbmi->ref_frame[0] != INTRA_FRAME;
478
479
480
  unsigned int (*token_costs)[2][PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS] =
                   mb->token_costs[tx_size][type][ref];
  const ENTROPY_CONTEXT above_ec = !!*A, left_ec = !!*L;
481
  uint8_t token_cache[1024];
482
483
  int pt = combine_entropy_contexts(above_ec, left_ec);
  int c, cost;
484
485

  // Check for consistency of tx_size with mode info
486
  assert(type == PLANE_TYPE_Y_WITH_DC ? mbmi->tx_size == tx_size
487
                                      : get_uv_tx_size(mbmi) == tx_size);
488

489
490
  if (eob == 0) {
    // single eob token
491
492
    cost = token_costs[0][0][pt][DCT_EOB_TOKEN];
    c = 0;
493
  } else {
494
    int band_left = *band_count++;
495
496

    // dc token
497
498
    int v = qcoeff_ptr[0];
    int prev_t = vp9_dct_value_tokens_ptr[v].token;
499
    cost = (*token_costs)[0][pt][prev_t] + vp9_dct_value_cost_ptr[v];
500
    token_cache[0] = vp9_pt_energy_class[prev_t];
501
    ++token_costs;
502
503
504
505

    // ac tokens
    for (c = 1; c < eob; c++) {
      const int rc = scan[c];
506
      int t;
507
508
509

      v = qcoeff_ptr[rc];
      t = vp9_dct_value_tokens_ptr[v].token;
510
      pt = get_coef_context(nb, token_cache, c);
511
      cost += (*token_costs)[!prev_t][pt][t] + vp9_dct_value_cost_ptr[v];
512
      token_cache[rc] = vp9_pt_energy_class[t];
513
      prev_t = t;
514
      if (!--band_left) {
515
516
        band_left = *band_count++;
        ++token_costs;
517
      }
518
    }
519
520

    // eob token
521
    if (band_left) {
522
      pt = get_coef_context(nb, token_cache, c);
523
      cost += (*token_costs)[0][pt][DCT_EOB_TOKEN];
524
    }
525
526
  }

527
  // is eob first coefficient;
528
  *A = *L = (c > 0);
529

530
531
532
  return cost;
}

Deb Mukherjee's avatar
Deb Mukherjee committed
533
534
535
536
537
538
539
struct rdcost_block_args {
  MACROBLOCK *x;
  ENTROPY_CONTEXT t_above[16];
  ENTROPY_CONTEXT t_left[16];
  TX_SIZE tx_size;
  int bw;
  int bh;
540
541
542
543
544
545
546
  int rate[256];
  int64_t dist[256];
  int64_t sse[256];
  int this_rate;
  int64_t this_dist;
  int64_t this_sse;
  int64_t this_rd;
Deb Mukherjee's avatar
Deb Mukherjee committed
547
548
  int64_t best_rd;
  int skip;
549
  const int16_t *scan, *nb;
Deb Mukherjee's avatar
Deb Mukherjee committed
550
551
};

552
static void dist_block(int plane, int block, TX_SIZE tx_size, void *arg) {
553
  const int ss_txfrm_size = tx_size << 1;
Deb Mukherjee's avatar
Deb Mukherjee committed
554
555
556
  struct rdcost_block_args* args = arg;
  MACROBLOCK* const x = args->x;
  MACROBLOCKD* const xd = &x->e_mbd;
557
558
  struct macroblock_plane *const p = &x->plane[plane];
  struct macroblockd_plane *const pd = &xd->plane[plane];
Deb Mukherjee's avatar
Deb Mukherjee committed
559
560
  int64_t this_sse;
  int shift = args->tx_size == TX_32X32 ? 0 : 2;
561
562
  int16_t *const coeff = BLOCK_OFFSET(p->coeff, block);
  int16_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
563
  args->dist[block] = vp9_block_error(coeff, dqcoeff, 16 << ss_txfrm_size,
Deb Mukherjee's avatar
Deb Mukherjee committed
564
                                &this_sse) >> shift;
565
  args->sse[block]  = this_sse >> shift;
566
567

  if (x->skip_encode &&
568
      xd->this_mi->mbmi.ref_frame[0] == INTRA_FRAME) {
569
570
571
    // TODO(jingning): tune the model to better capture the distortion.
    int64_t p = (pd->dequant[1] * pd->dequant[1] *
                    (1 << ss_txfrm_size)) >> shift;
572
573
    args->dist[block] = p;
    args->sse[block]  = p;
574
  }
Deb Mukherjee's avatar
Deb Mukherjee committed
575
576
}

577
static void rate_block(int plane, int block, BLOCK_SIZE plane_bsize,
578
                       TX_SIZE tx_size, void *arg) {
Deb Mukherjee's avatar
Deb Mukherjee committed
579
  struct rdcost_block_args* args = arg;
580

Deb Mukherjee's avatar
Deb Mukherjee committed
581
  int x_idx, y_idx;
582
  txfrm_block_to_raster_xy(plane_bsize, args->tx_size, block, &x_idx, &y_idx);
Deb Mukherjee's avatar
Deb Mukherjee committed
583

584
585
586
587
  args->rate[block] = cost_coeffs(args->x, plane, block,
                                  args->t_above + x_idx,
                                  args->t_left + y_idx, args->tx_size,
                                  args->scan, args->nb);
Deb Mukherjee's avatar
Deb Mukherjee committed
588
589
}

590
static void block_yrd_txfm(int plane, int block, BLOCK_SIZE plane_bsize,
591
                           TX_SIZE tx_size, void *arg) {
Deb Mukherjee's avatar
Deb Mukherjee committed
592
593
594
  struct rdcost_block_args *args = arg;
  MACROBLOCK *const x = args->x;
  MACROBLOCKD *const xd = &x->e_mbd;
595
  struct encode_b_args encode_args = {x, NULL};
596
  int64_t rd1, rd2, rd;
Deb Mukherjee's avatar
Deb Mukherjee committed
597

598
599
600
  if (args->skip)
    return;

601
  if (!is_inter_block(&xd->this_mi->mbmi))
602
    vp9_encode_block_intra(plane, block, plane_bsize, tx_size, &encode_args);
Deb Mukherjee's avatar
Deb Mukherjee committed
603
  else
604
    vp9_xform_quant(plane, block, plane_bsize, tx_size, &encode_args);
Deb Mukherjee's avatar
Deb Mukherjee committed
605

606
607
  dist_block(plane, block, tx_size, args);
  rate_block(plane, block, plane_bsize, tx_size, args);
608
609
  rd1 = RDCOST(x->rdmult, x->rddiv, args->rate[block], args->dist[block]);
  rd2 = RDCOST(x->rdmult, x->rddiv, 0, args->sse[block]);
610
611

  // TODO(jingning): temporarily enabled only for luma component
612
  rd = MIN(rd1, rd2);
613
614
615
  if (plane == 0)
    x->zcoeff_blk[tx_size][block] = rd1 > rd2;

616
617
618
619
620
621
622
623
624
  args->this_rate += args->rate[block];
  args->this_dist += args->dist[block];
  args->this_sse  += args->sse[block];
  args->this_rd += rd;

  if (args->this_rd > args->best_rd) {
    args->skip = 1;
    return;
  }
Deb Mukherjee's avatar
Deb Mukherjee committed
625
626
}

627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
void vp9_get_entropy_contexts(TX_SIZE tx_size,
    ENTROPY_CONTEXT t_above[16], ENTROPY_CONTEXT t_left[16],
    const ENTROPY_CONTEXT *above, const ENTROPY_CONTEXT *left,
    int num_4x4_w, int num_4x4_h) {
  int i;
  switch (tx_size) {
    case TX_4X4:
      vpx_memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
      vpx_memcpy(t_left, left, sizeof(ENTROPY_CONTEXT) * num_4x4_h);
      break;
    case TX_8X8:
      for (i = 0; i < num_4x4_w; i += 2)
        t_above[i] = !!*(const uint16_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 2)
        t_left[i] = !!*(const uint16_t *)&left[i];
      break;
    case TX_16X16:
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
    case TX_32X32:
      for (i = 0; i < num_4x4_w; i += 8)
        t_above[i] = !!*(const uint64_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 8)
        t_left[i] = !!*(const uint64_t *)&left[i];
      break;
    default:
      assert(!"Invalid transform size.");
  }
}

660
static void txfm_rd_in_plane(MACROBLOCK *x,
661
662
663
                             int *rate, int64_t *distortion,
                             int *skippable, int64_t *sse,
                             int64_t ref_best_rd, int plane,
664
                             BLOCK_SIZE bsize, TX_SIZE tx_size) {
Deb Mukherjee's avatar
Deb Mukherjee committed
665
  MACROBLOCKD *const xd = &x->e_mbd;
666
  struct macroblockd_plane *const pd = &xd->plane[plane];
667
  const BLOCK_SIZE bs = get_plane_block_size(bsize, pd);
668
669
  const int num_4x4_w = num_4x4_blocks_wide_lookup[bs];
  const int num_4x4_h = num_4x4_blocks_high_lookup[bs];
670
  const uint8_t *band_translate;  // just for the get_scan_and_band call
671

672
  struct rdcost_block_args args = { x, { 0 }, { 0 }, tx_size,
673
                                    num_4x4_w, num_4x4_h,
674
675
                                    { 0 }, { 0 }, { 0 },
                                    0, 0, 0, 0, ref_best_rd, 0 };
676
  if (plane == 0)
677
    xd->this_mi->mbmi.tx_size = tx_size;
678

679
680
681
  vp9_get_entropy_contexts(tx_size, args.t_above, args.t_left,
                           pd->above_context, pd->left_context,
                           num_4x4_w, num_4x4_h);
682
683
684

  get_scan_and_band(xd, tx_size, pd->plane_type, 0, &args.scan, &args.nb,
                    &band_translate);
Deb Mukherjee's avatar
Deb Mukherjee committed
685

686
  foreach_transformed_block_in_plane(xd, bsize, plane, block_yrd_txfm, &args);
687
688
689
690
691
692
693
694
695
696
697
  if (args.skip) {
    *rate       = INT_MAX;
    *distortion = INT64_MAX;
    *sse        = INT64_MAX;
    *skippable  = 0;
  } else {
    *distortion = args.this_dist;
    *rate       = args.this_rate;
    *sse        = args.this_sse;
    *skippable  = vp9_is_skippable_in_plane(xd, bsize, plane);
  }
Deb Mukherjee's avatar
Deb Mukherjee committed
698
699
700
701
702
}

static void choose_largest_txfm_size(VP9_COMP *cpi, MACROBLOCK *x,
                                     int *rate, int64_t *distortion,
                                     int *skip, int64_t *sse,
703
                                     int64_t ref_best_rd,
704
                                     BLOCK_SIZE bs) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
705
  const TX_SIZE max_tx_size = max_txsize_lookup[bs];
Deb Mukherjee's avatar
Deb Mukherjee committed
706
707
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
708
  MB_MODE_INFO *const mbmi = &xd->this_mi->mbmi;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
709
  if (max_tx_size == TX_32X32 &&
710
711
      (cm->tx_mode == ALLOW_32X32 ||
       cm->tx_mode == TX_MODE_SELECT)) {
712
    mbmi->tx_size = TX_32X32;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
713
  } else if (max_tx_size >= TX_16X16 &&
714
715
716
             (cm->tx_mode == ALLOW_16X16 ||
              cm->tx_mode == ALLOW_32X32 ||
              cm->tx_mode == TX_MODE_SELECT)) {
717
    mbmi->tx_size = TX_16X16;
718
  } else if (cm->tx_mode != ONLY_4X4) {
719
    mbmi->tx_size = TX_8X8;
Deb Mukherjee's avatar
Deb Mukherjee committed
720
  } else {
721
    mbmi->tx_size = TX_4X4;
Deb Mukherjee's avatar
Deb Mukherjee committed
722
  }
723
  txfm_rd_in_plane(x, rate, distortion, skip,
724
725
                   &sse[mbmi->tx_size], ref_best_rd, 0, bs,
                   mbmi->tx_size);
Dmitry Kovalev's avatar
Dmitry Kovalev committed
726
  cpi->tx_stepdown_count[0]++;
Deb Mukherjee's avatar
Deb Mukherjee committed
727
728
}

729
static void choose_txfm_size_from_rd(VP9_COMP *cpi, MACROBLOCK *x,
730
                                     int (*r)[2], int *rate,
Ronald S. Bultje's avatar
Ronald S. Bultje committed
731
                                     int64_t *d, int64_t *distortion,
732
                                     int *s, int *skip,
733
                                     int64_t tx_cache[TX_MODES],
734
                                     BLOCK_SIZE bs) {
735
  const TX_SIZE max_tx_size = max_txsize_lookup[bs];
736
737
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
738
  MB_MODE_INFO *const mbmi = &xd->this_mi->mbmi;
739
  vp9_prob skip_prob = vp9_get_pred_prob_mbskip(cm, xd);
740
  int64_t rd[TX_SIZES][2];
741
  int n, m;
742
  int s0, s1;
743

744
  const vp9_prob *tx_probs = get_tx_probs2(xd, &cm->fc.tx_probs, xd->this_mi);
745

746
  for (n = TX_4X4; n <= max_tx_size; n++) {
747
    r[n][1] = r[n][0];
748
749
    if (r[n][0] == INT_MAX)
      continue;
750
    for (m = 0; m <= n - (n == max_tx_size); m++) {
751
      if (m == n)
752
        r[n][1] += vp9_cost_zero(tx_probs[m]);
753
      else
754
        r[n][1] += vp9_cost_one(tx_probs[m]);
755
756
    }
  }
757

758
759
760
  assert(skip_prob > 0);
  s0 = vp9_cost_bit(skip_prob, 0);
  s1 = vp9_cost_bit(skip_prob, 1);
761

762
  for (n = TX_4X4; n <= max_tx_size; n++) {
763
764
765
766
    if (d[n] == INT64_MAX) {
      rd[n][0] = rd[n][1] = INT64_MAX;
      continue;
    }
767
768
769
770
771
    if (s[n]) {
      rd[n][0] = rd[n][1] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
    } else {
      rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0] + s0, d[n]);
      rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1] + s0, d[n]);
772
773
774
    }
  }

775
  if (max_tx_size == TX_32X32 &&
776
777
      (cm->tx_mode == ALLOW_32X32 ||
       (cm->tx_mode == TX_MODE_SELECT &&
778
779
        rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
        rd[TX_32X32][1] < rd[TX_4X4][1]))) {
780
    mbmi->tx_size = TX_32X32;
781
  } else if (max_tx_size >= TX_16X16 &&
782
783
784
             (cm->tx_mode == ALLOW_16X16 ||
              cm->tx_mode == ALLOW_32X32 ||
              (cm->tx_mode == TX_MODE_SELECT &&
785
786
               rd[TX_16X16][1] < rd[TX_8X8][1] &&
               rd[TX_16X16][1] < rd[TX_4X4][1]))) {
787
    mbmi->tx_size = TX_16X16;
788
789
790
791
  } else if (cm->tx_mode == ALLOW_8X8 ||
             cm->tx_mode == ALLOW_16X16 ||
             cm->tx_mode == ALLOW_32X32 ||
           (cm->tx_mode == TX_MODE_SELECT && rd[TX_8X8][1] < rd[TX_4X4][1])) {
792
    mbmi->tx_size = TX_8X8;
793
  } else {
794
    mbmi->tx_size = TX_4X4;
795
796
  }

797
798
799
  *distortion = d[mbmi->tx_size];
  *rate       = r[mbmi->tx_size][cm->tx_mode == TX_MODE_SELECT];
  *skip       = s[mbmi->tx_size];
800

801
802
803
804
805
  tx_cache[ONLY_4X4] = rd[TX_4X4][0];
  tx_cache[ALLOW_8X8] = rd[TX_8X8][0];
  tx_cache[ALLOW_16X16] = rd[MIN(max_tx_size, TX_16X16)][0];
  tx_cache[ALLOW_32X32] = rd[MIN(max_tx_size, TX_32X32)][0];
  if (max_tx_size == TX_32X32 &&
806
807
      rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
      rd[TX_32X32][1] < rd[TX_4X4][1])
808
809
    tx_cache[TX_MODE_SELECT] = rd[TX_32X32][1];
  else if (max_tx_size >= TX_16X16 &&
810
           rd[TX_16X16][1] < rd[TX_8X8][1] && rd[TX_16X16][1] < rd[TX_4X4][1])
811
    tx_cache[TX_MODE_SELECT] = rd[TX_16X16][1];
812
  else
813
    tx_cache[TX_MODE_SELECT] = rd[TX_4X4][1] < rd[TX_8X8][1] ?
814
                                 rd[TX_4X4][1] : rd[TX_8X8][1];
815

816
  if (max_tx_size == TX_32X32 &&
Deb Mukherjee's avatar
Deb Mukherjee committed
817
818
819
      rd[TX_32X32][1] < rd[TX_16X16][1] &&
      rd[TX_32X32][1] < rd[TX_8X8][1] &&
      rd[TX_32X32][1] < rd[TX_4X4][1]) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
820
    cpi->tx_stepdown_count[0]++;
821
  } else if (max_tx_size >= TX_16X16 &&
Deb Mukherjee's avatar
Deb Mukherjee committed
822
823
             rd[TX_16X16][1] < rd[TX_8X8][1] &&
             rd[TX_16X16][1] < rd[TX_4X4][1]) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
824
    cpi->tx_stepdown_count[max_tx_size - TX_16X16]++;
Deb Mukherjee's avatar
Deb Mukherjee committed
825
  } else if (rd[TX_8X8][1] < rd[TX_4X4][1]) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
826
    cpi->tx_stepdown_count[max_tx_size - TX_8X8]++;
Deb Mukherjee's avatar
Deb Mukherjee committed
827
  } else {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
828
    cpi->tx_stepdown_count[max_tx_size - TX_4X4]++;
829
  }
Deb Mukherjee's avatar
Deb Mukherjee committed
830
}
831

Deb Mukherjee's avatar
Deb Mukherjee committed
832
833
834
835
static void choose_txfm_size_from_modelrd(VP9_COMP *cpi, MACROBLOCK *x,
                                          int (*r)[2], int *rate,
                                          int64_t *d, int64_t *distortion,
                                          int *s, int *skip, int64_t *sse,
836
                                          int64_t ref_best_rd,
837
                                          BLOCK_SIZE bs) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
838
  const TX_SIZE max_tx_size = max_txsize_lookup[bs];
Deb Mukherjee's avatar
Deb Mukherjee committed
839
840
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
841
  MB_MODE_INFO *const mbmi = &xd->this_mi->mbmi;
842
  vp9_prob skip_prob = vp9_get_pred_prob_mbskip(cm, xd);
843
  int64_t rd[TX_SIZES][2];
Deb Mukherjee's avatar
Deb Mukherjee committed
844
845
  int n, m;
  int s0, s1;
846
847
  double scale_rd[TX_SIZES] = {1.73, 1.44, 1.20, 1.00};
  // double scale_r[TX_SIZES] = {2.82, 2.00, 1.41, 1.00};
848

849
  const vp9_prob *tx_probs = get_tx_probs2(xd, &cm->fc.tx_probs,  xd->this_mi);
850

Deb Mukherjee's avatar
Deb Mukherjee committed
851
852
  // for (n = TX_4X4; n <= max_txfm_size; n++)
  //   r[n][0] = (r[n][0] * scale_r[n]);
853

Dmitry Kovalev's avatar
Dmitry Kovalev committed
854
  for (n = TX_4X4; n <= max_tx_size; n++) {
Deb Mukherjee's avatar
Deb Mukherjee committed
855
    r[n][1] = r[n][0];
Dmitry Kovalev's avatar
Dmitry Kovalev committed
856
    for (m = 0; m <= n - (n == max_tx_size); m++) {
Deb Mukherjee's avatar
Deb Mukherjee committed
857
858
859
860
861
      if (m == n)
        r[n][1] += vp9_cost_zero(tx_probs[m]);
      else
        r[n][1] += vp9_cost_one(tx_probs[m]);
    }
862
  }
863

Deb Mukherjee's avatar
Deb Mukherjee committed
864
865
866
  assert(skip_prob > 0);
  s0 = vp9_cost_bit(skip_prob, 0);
  s1 = vp9_cost_bit(skip_prob, 1);
867

Dmitry Kovalev's avatar
Dmitry Kovalev committed
868
  for (n = TX_4X4; n <= max_tx_size; n++) {
Deb Mukherjee's avatar
Deb Mukherjee committed
869
870
871
872
873
874
875
    if (s[n]) {
      rd[n][0] = rd[n][1] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
    } else {
      rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0] + s0, d[n]);
      rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1] + s0, d[n]);
    }
  }
Dmitry Kovalev's avatar
Dmitry Kovalev committed
876
  for (n = TX_4X4; n <= max_tx_size; n++) {
Yaowu Xu's avatar
Yaowu Xu committed
877
878
    rd[n][0] = (int64_t)(scale_rd[n] * rd[n][0]);
    rd[n][1] = (int64_t)(scale_rd[n] * rd[n][1]);
Deb Mukherjee's avatar
Deb Mukherjee committed
879
  }
880

Dmitry Kovalev's avatar
Dmitry Kovalev committed
881
  if (max_tx_size == TX_32X32 &&
882
883
      (cm->tx_mode == ALLOW_32X32 ||
       (cm->tx_mode == TX_MODE_SELECT &&
Deb Mukherjee's avatar
Deb Mukherjee committed
884
885
886
        rd[TX_32X32][1] <= rd[TX_16X16][1] &&
        rd[TX_32X32][1] <= rd[TX_8X8][1] &&
        rd[TX_32X32][1] <= rd[TX_4X4][1]))) {
887
    mbmi->tx_size = TX_32X32;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
888
  } else if (max_tx_size >= TX_16X16 &&
889
890
891
             (cm->tx_mode == ALLOW_16X16 ||
              cm->tx_mode == ALLOW_32X32 ||
              (cm->tx_mode == TX_MODE_SELECT &&
Deb Mukherjee's avatar
Deb Mukherjee committed
892
893
               rd[TX_16X16][1] <= rd[TX_8X8][1] &&
               rd[TX_16X16][1] <= rd[TX_4X4][1]))) {
894
    mbmi->tx_size = TX_16X16;
895
896
897
898
  } else if (cm->tx_mode == ALLOW_8X8 ||
             cm->tx_mode == ALLOW_16X16 ||
             cm->tx_mode == ALLOW_32X32 ||
           (cm->tx_mode == TX_MODE_SELECT &&