vp9_rdopt.c 166 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5
6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9
10
11
12
13
14
 */

#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
15

16
#include "vp9/common/vp9_pragmas.h"
17
18
19
20
21
#include "vp9/encoder/vp9_tokenize.h"
#include "vp9/encoder/vp9_treewriter.h"
#include "vp9/encoder/vp9_onyx_int.h"
#include "vp9/encoder/vp9_modecosts.h"
#include "vp9/encoder/vp9_encodeintra.h"
22
23
24
25
26
#include "vp9/common/vp9_entropymode.h"
#include "vp9/common/vp9_reconinter.h"
#include "vp9/common/vp9_reconintra.h"
#include "vp9/common/vp9_findnearmv.h"
#include "vp9/common/vp9_quant_common.h"
27
28
29
30
31
32
#include "vp9/encoder/vp9_encodemb.h"
#include "vp9/encoder/vp9_quantize.h"
#include "vp9/encoder/vp9_variance.h"
#include "vp9/encoder/vp9_mcomp.h"
#include "vp9/encoder/vp9_rdopt.h"
#include "vp9/encoder/vp9_ratectrl.h"
John Koleszar's avatar
John Koleszar committed
33
#include "vpx_mem/vpx_mem.h"
34
35
36
37
38
#include "vp9/common/vp9_systemdependent.h"
#include "vp9/encoder/vp9_encodemv.h"
#include "vp9/common/vp9_seg_common.h"
#include "vp9/common/vp9_pred_common.h"
#include "vp9/common/vp9_entropy.h"
39
#include "./vp9_rtcd.h"
40
#include "vp9/common/vp9_mvref_common.h"
Ronald S. Bultje's avatar
Ronald S. Bultje committed
41
#include "vp9/common/vp9_common.h"
Paul Wilkins's avatar
Paul Wilkins committed
42

43
44
#define INVALID_MV 0x80008000

45
46
47
/* Factor to weigh the rate for switchable interp filters */
#define SWITCHABLE_INTERP_RATE_FACTOR 1

48
49
50
#define LAST_FRAME_MODE_MASK    0xFFEDCD60
#define GOLDEN_FRAME_MODE_MASK  0xFFDA3BB0
#define ALT_REF_MODE_MASK       0xFFC648D0
51

Paul Wilkins's avatar
Paul Wilkins committed
52
53
#define MIN_EARLY_TERM_INDEX    3

54
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
  {NEARESTMV, LAST_FRAME,   NONE},
  {NEARESTMV, ALTREF_FRAME, NONE},
  {NEARESTMV, GOLDEN_FRAME, NONE},

  {DC_PRED,   INTRA_FRAME,  NONE},

  {NEWMV,     LAST_FRAME,   NONE},
  {NEWMV,     ALTREF_FRAME, NONE},
  {NEWMV,     GOLDEN_FRAME, NONE},

  {NEARMV,    LAST_FRAME,   NONE},
  {NEARMV,    ALTREF_FRAME, NONE},
  {NEARESTMV, LAST_FRAME,   ALTREF_FRAME},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},

  {TM_PRED,   INTRA_FRAME,  NONE},

  {NEARMV,    LAST_FRAME,   ALTREF_FRAME},
  {NEWMV,     LAST_FRAME,   ALTREF_FRAME},
  {NEARMV,    GOLDEN_FRAME, NONE},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME},
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME},

  {ZEROMV,    LAST_FRAME,   NONE},
  {ZEROMV,    GOLDEN_FRAME, NONE},
  {ZEROMV,    ALTREF_FRAME, NONE},
  {ZEROMV,    LAST_FRAME,   ALTREF_FRAME},
  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME},

  {H_PRED,    INTRA_FRAME,  NONE},
  {V_PRED,    INTRA_FRAME,  NONE},
  {D135_PRED, INTRA_FRAME,  NONE},
  {D207_PRED, INTRA_FRAME,  NONE},
  {D153_PRED, INTRA_FRAME,  NONE},
  {D63_PRED,  INTRA_FRAME,  NONE},
  {D117_PRED, INTRA_FRAME,  NONE},
  {D45_PRED,  INTRA_FRAME,  NONE},
};

const REF_DEFINITION vp9_ref_order[MAX_REFS] = {
  {LAST_FRAME,   NONE},
  {GOLDEN_FRAME, NONE},
  {ALTREF_FRAME, NONE},
  {LAST_FRAME,   ALTREF_FRAME},
  {GOLDEN_FRAME, ALTREF_FRAME},
  {INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
101
102
};

103
104
105
106
// The baseline rd thresholds for breaking out of the rd loop for
// certain modes are assumed to be based on 8x8 blocks.
// This table is used to correct for blocks size.
// The factors here are << 2 (2 = x0.5, 32 = x8 etc).
107
static int rd_thresh_block_size_factor[BLOCK_SIZES] =
108
109
  {2, 3, 3, 4, 6, 6, 8, 12, 12, 16, 24, 24, 32};

Deb Mukherjee's avatar
Deb Mukherjee committed
110
111
112
113
#define RD_THRESH_MAX_FACT 64
#define RD_THRESH_INC      1
#define RD_THRESH_POW      1.25

114
115
#define MV_COST_WEIGHT      108
#define MV_COST_WEIGHT_SUB  120
116

117
static void fill_token_costs(vp9_coeff_cost *c,
118
                             vp9_coeff_probs_model (*p)[BLOCK_TYPES]) {
119
  int i, j, k, l;
120
121
122
123
124
125
126
127
  TX_SIZE t;
  for (t = TX_4X4; t <= TX_32X32; t++)
    for (i = 0; i < BLOCK_TYPES; i++)
      for (j = 0; j < REF_TYPES; j++)
        for (k = 0; k < COEF_BANDS; k++)
          for (l = 0; l < PREV_COEF_CONTEXTS; l++) {
            vp9_prob probs[ENTROPY_NODES];
            vp9_model_to_full_probs(p[t][i][j][k][l], probs);
128
            vp9_cost_tokens((int *)c[t][i][j][k][0][l], probs,
129
                            vp9_coef_tree);
130
            vp9_cost_tokens_skip((int *)c[t][i][j][k][1][l], probs,
131
                                 vp9_coef_tree);
132
133
            assert(c[t][i][j][k][0][l][DCT_EOB_TOKEN] ==
                   c[t][i][j][k][1][l][DCT_EOB_TOKEN]);
134
          }
135
136
}

137
138
139
140
141
142
static const int rd_iifactor[32] = {
  4, 4, 3, 2, 1, 0, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 0,
};
John Koleszar's avatar
John Koleszar committed
143

144
// 3* dc_qlookup[Q]*dc_qlookup[Q];
145

146
/* values are now correlated to quantizer */
Paul Wilkins's avatar
Paul Wilkins committed
147
148
149
static int sad_per_bit16lut[QINDEX_RANGE];
static int sad_per_bit4lut[QINDEX_RANGE];

150
void vp9_init_me_luts() {
John Koleszar's avatar
John Koleszar committed
151
152
153
154
155
156
157
  int i;

  // Initialize the sad lut tables using a formulaic calculation for now
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < QINDEX_RANGE; i++) {
    sad_per_bit16lut[i] =
158
      (int)((0.0418 * vp9_convert_qindex_to_q(i)) + 2.4107);
159
    sad_per_bit4lut[i] = (int)(0.063 * vp9_convert_qindex_to_q(i) + 2.742);
John Koleszar's avatar
John Koleszar committed
160
  }
Paul Wilkins's avatar
Paul Wilkins committed
161
}
John Koleszar's avatar
John Koleszar committed
162

163
static int compute_rd_mult(int qindex) {
164
  const int q = vp9_dc_quant(qindex, 0);
165
  return (11 * q * q) >> 2;
166
167
}

Dmitry Kovalev's avatar
Dmitry Kovalev committed
168
169
170
void vp9_initialize_me_consts(VP9_COMP *cpi, int qindex) {
  cpi->mb.sadperbit16 = sad_per_bit16lut[qindex];
  cpi->mb.sadperbit4 = sad_per_bit4lut[qindex];
John Koleszar's avatar
John Koleszar committed
171
172
}

Deb Mukherjee's avatar
Deb Mukherjee committed
173
static void set_block_thresholds(VP9_COMP *cpi, int qindex) {
174
  int q, i, bsize;
Deb Mukherjee's avatar
Deb Mukherjee committed
175
  q = ((int)pow(vp9_dc_quant(qindex, 0) >> 2, RD_THRESH_POW)) << 2;
John Koleszar's avatar
John Koleszar committed
176
177
  if (q < 8)
    q = 8;
178

179
  for (bsize = 0; bsize < BLOCK_SIZES; ++bsize) {
180
    for (i = 0; i < MAX_MODES; ++i) {
181
182
183
      // Threshold here seem unecessarily harsh but fine given actual
      // range of values used for cpi->sf.thresh_mult[]
      int thresh_max = INT_MAX / (q * rd_thresh_block_size_factor[bsize]);
John Koleszar's avatar
John Koleszar committed
184

185
186
      if (cpi->sf.thresh_mult[i] < thresh_max) {
        cpi->rd_threshes[bsize][i] =
187
188
            cpi->sf.thresh_mult[i] * q *
            rd_thresh_block_size_factor[bsize] / 4;
189
190
      } else {
        cpi->rd_threshes[bsize][i] = INT_MAX;
John Koleszar's avatar
John Koleszar committed
191
      }
John Koleszar's avatar
John Koleszar committed
192
    }
193
194
195
196
197
198
199
200
201
202
203
204

    for (i = 0; i < MAX_REFS; ++i) {
      int thresh_max = INT_MAX / (q * rd_thresh_block_size_factor[bsize]);

      if (cpi->sf.thresh_mult_sub8x8[i] < thresh_max) {
        cpi->rd_thresh_sub8x8[bsize][i] =
            cpi->sf.thresh_mult_sub8x8[i] * q *
            rd_thresh_block_size_factor[bsize] / 4;
      } else {
        cpi->rd_thresh_sub8x8[bsize][i] = INT_MAX;
      }
    }
John Koleszar's avatar
John Koleszar committed
205
  }
Deb Mukherjee's avatar
Deb Mukherjee committed
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
}

void vp9_initialize_rd_consts(VP9_COMP *cpi, int qindex) {
  int i;

  vp9_clear_system_state();  // __asm emms;

  // Further tests required to see if optimum is different
  // for key frames, golden frames and arf frames.
  // if (cpi->common.refresh_golden_frame ||
  //     cpi->common.refresh_alt_ref_frame)
  qindex = clamp(qindex, 0, MAXQ);

  cpi->RDDIV = 100;
  cpi->RDMULT = compute_rd_mult(qindex);
  if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    if (cpi->twopass.next_iiratio > 31)
      cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
    else
      cpi->RDMULT +=
          (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
  }
  cpi->mb.errorperbit = cpi->RDMULT >> 6;
  cpi->mb.errorperbit += (cpi->mb.errorperbit == 0);

  vp9_set_speed_features(cpi);

  set_block_thresholds(cpi, qindex);
John Koleszar's avatar
John Koleszar committed
234

235
  fill_token_costs(cpi->mb.token_costs, cpi->common.fc.coef_probs);
236

237
  for (i = 0; i < NUM_PARTITION_CONTEXTS; i++)
238
    vp9_cost_tokens(cpi->mb.partition_cost[i],
239
                    cpi->common.fc.partition_prob[cpi->common.frame_type][i],
240
241
                    vp9_partition_tree);

John Koleszar's avatar
John Koleszar committed
242
  /*rough estimate for costing*/
243
  vp9_init_mode_costs(cpi);
John Koleszar's avatar
John Koleszar committed
244

245
  if (cpi->common.frame_type != KEY_FRAME) {
246
    vp9_build_nmv_cost_table(
247
248
249
250
251
        cpi->mb.nmvjointcost,
        cpi->mb.e_mbd.allow_high_precision_mv ?
        cpi->mb.nmvcost_hp : cpi->mb.nmvcost,
        &cpi->common.fc.nmvc,
        cpi->mb.e_mbd.allow_high_precision_mv, 1, 1);
252
253
254
255
256

    for (i = 0; i < INTER_MODE_CONTEXTS; i++) {
      MB_PREDICTION_MODE m;

      for (m = NEARESTMV; m < MB_MODE_COUNT; m++)
257
        cpi->mb.inter_mode_cost[i][inter_mode_offset(m)] =
258
            cost_token(vp9_inter_mode_tree,
259
                       cpi->common.fc.inter_mode_probs[i],
260
                       vp9_inter_mode_encodings + inter_mode_offset(m));
261
    }
262
  }
John Koleszar's avatar
John Koleszar committed
263
264
}

Yaowu Xu's avatar
Yaowu Xu committed
265
static INLINE void linear_interpolate2(double x, int ntab, int inv_step,
266
267
                                       const double *tab1, const double *tab2,
                                       double *v1, double *v2) {
Deb Mukherjee's avatar
Deb Mukherjee committed
268
269
270
  double y = x * inv_step;
  int d = (int) y;
  if (d >= ntab - 1) {
271
272
    *v1 = tab1[ntab - 1];
    *v2 = tab2[ntab - 1];
Deb Mukherjee's avatar
Deb Mukherjee committed
273
274
  } else {
    double a = y - d;
275
276
    *v1 = tab1[d] * (1 - a) + tab1[d + 1] * a;
    *v2 = tab2[d] * (1 - a) + tab2[d + 1] * a;
Deb Mukherjee's avatar
Deb Mukherjee committed
277
278
279
  }
}

280
281
282
283
284
static void model_rd_norm(double x, double *R, double *D) {
  static const int inv_tab_step = 8;
  static const int tab_size = 120;
  // NOTE: The tables below must be of the same size
  //
Deb Mukherjee's avatar
Deb Mukherjee committed
285
  // Normalized rate
286
  // This table models the rate for a Laplacian source
Deb Mukherjee's avatar
Deb Mukherjee committed
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
  // source with given variance when quantized with a uniform quantizer
  // with given stepsize. The closed form expression is:
  // Rn(x) = H(sqrt(r)) + sqrt(r)*[1 + H(r)/(1 - r)],
  // where r = exp(-sqrt(2) * x) and x = qpstep / sqrt(variance),
  // and H(x) is the binary entropy function.
  static const double rate_tab[] = {
    64.00, 4.944, 3.949, 3.372, 2.966, 2.655, 2.403, 2.194,
    2.014, 1.858, 1.720, 1.596, 1.485, 1.384, 1.291, 1.206,
    1.127, 1.054, 0.986, 0.923, 0.863, 0.808, 0.756, 0.708,
    0.662, 0.619, 0.579, 0.541, 0.506, 0.473, 0.442, 0.412,
    0.385, 0.359, 0.335, 0.313, 0.291, 0.272, 0.253, 0.236,
    0.220, 0.204, 0.190, 0.177, 0.165, 0.153, 0.142, 0.132,
    0.123, 0.114, 0.106, 0.099, 0.091, 0.085, 0.079, 0.073,
    0.068, 0.063, 0.058, 0.054, 0.050, 0.047, 0.043, 0.040,
    0.037, 0.034, 0.032, 0.029, 0.027, 0.025, 0.023, 0.022,
    0.020, 0.019, 0.017, 0.016, 0.015, 0.014, 0.013, 0.012,
    0.011, 0.010, 0.009, 0.008, 0.008, 0.007, 0.007, 0.006,
    0.006, 0.005, 0.005, 0.005, 0.004, 0.004, 0.004, 0.003,
    0.003, 0.003, 0.003, 0.002, 0.002, 0.002, 0.002, 0.002,
    0.002, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001,
    0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.001, 0.000,
  };
  // Normalized distortion
310
  // This table models the normalized distortion for a Laplacian source
Deb Mukherjee's avatar
Deb Mukherjee committed
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
  // source with given variance when quantized with a uniform quantizer
  // with given stepsize. The closed form expression is:
  // Dn(x) = 1 - 1/sqrt(2) * x / sinh(x/sqrt(2))
  // where x = qpstep / sqrt(variance)
  // Note the actual distortion is Dn * variance.
  static const double dist_tab[] = {
    0.000, 0.001, 0.005, 0.012, 0.021, 0.032, 0.045, 0.061,
    0.079, 0.098, 0.119, 0.142, 0.166, 0.190, 0.216, 0.242,
    0.269, 0.296, 0.324, 0.351, 0.378, 0.405, 0.432, 0.458,
    0.484, 0.509, 0.534, 0.557, 0.580, 0.603, 0.624, 0.645,
    0.664, 0.683, 0.702, 0.719, 0.735, 0.751, 0.766, 0.780,
    0.794, 0.807, 0.819, 0.830, 0.841, 0.851, 0.861, 0.870,
    0.878, 0.886, 0.894, 0.901, 0.907, 0.913, 0.919, 0.925,
    0.930, 0.935, 0.939, 0.943, 0.947, 0.951, 0.954, 0.957,
    0.960, 0.963, 0.966, 0.968, 0.971, 0.973, 0.975, 0.976,
    0.978, 0.980, 0.981, 0.982, 0.984, 0.985, 0.986, 0.987,
    0.988, 0.989, 0.990, 0.990, 0.991, 0.992, 0.992, 0.993,
    0.993, 0.994, 0.994, 0.995, 0.995, 0.996, 0.996, 0.996,
    0.996, 0.997, 0.997, 0.997, 0.997, 0.998, 0.998, 0.998,
    0.998, 0.998, 0.998, 0.999, 0.999, 0.999, 0.999, 0.999,
    0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 0.999, 1.000,
  };
333
334
335
336
337
  /*
  assert(sizeof(rate_tab) == tab_size * sizeof(rate_tab[0]);
  assert(sizeof(dist_tab) == tab_size * sizeof(dist_tab[0]);
  assert(sizeof(rate_tab) == sizeof(dist_tab));
  */
Deb Mukherjee's avatar
Deb Mukherjee committed
338
  assert(x >= 0.0);
339
340
  linear_interpolate2(x, tab_size, inv_tab_step,
                      rate_tab, dist_tab, R, D);
Deb Mukherjee's avatar
Deb Mukherjee committed
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
}

static void model_rd_from_var_lapndz(int var, int n, int qstep,
                                     int *rate, int64_t *dist) {
  // This function models the rate and distortion for a Laplacian
  // source with given variance when quantized with a uniform quantizer
  // with given stepsize. The closed form expressions are in:
  // Hang and Chen, "Source Model for transform video coder and its
  // application - Part I: Fundamental Theory", IEEE Trans. Circ.
  // Sys. for Video Tech., April 1997.
  vp9_clear_system_state();
  if (var == 0 || n == 0) {
    *rate = 0;
    *dist = 0;
  } else {
    double D, R;
    double s2 = (double) var / n;
    double x = qstep / sqrt(s2);
359
    model_rd_norm(x, &R, &D);
Yaowu Xu's avatar
Yaowu Xu committed
360
361
    *rate = (int)((n << 8) * R + 0.5);
    *dist = (int)(var * D + 0.5);
Deb Mukherjee's avatar
Deb Mukherjee committed
362
363
364
365
  }
  vp9_clear_system_state();
}

366
static void model_rd_for_sb(VP9_COMP *cpi, BLOCK_SIZE bsize,
Deb Mukherjee's avatar
Deb Mukherjee committed
367
368
369
370
371
372
373
374
375
376
                            MACROBLOCK *x, MACROBLOCKD *xd,
                            int *out_rate_sum, int64_t *out_dist_sum) {
  // Note our transform coeffs are 8 times an orthogonal transform.
  // Hence quantizer step is also 8 times. To get effective quantizer
  // we need to divide by 8 before sending to modeling function.
  int i, rate_sum = 0, dist_sum = 0;

  for (i = 0; i < MAX_MB_PLANE; ++i) {
    struct macroblock_plane *const p = &x->plane[i];
    struct macroblockd_plane *const pd = &xd->plane[i];
377
    const BLOCK_SIZE bs = get_plane_block_size(bsize, pd);
Deb Mukherjee's avatar
Deb Mukherjee committed
378
379
380
381
382
383
    unsigned int sse;
    int rate;
    int64_t dist;
    (void) cpi->fn_ptr[bs].vf(p->src.buf, p->src.stride,
                              pd->dst.buf, pd->dst.stride, &sse);
    // sse works better than var, since there is no dc prediction used
Dmitry Kovalev's avatar
Dmitry Kovalev committed
384
    model_rd_from_var_lapndz(sse, 1 << num_pels_log2_lookup[bs],
385
                             pd->dequant[1] >> 3, &rate, &dist);
Deb Mukherjee's avatar
Deb Mukherjee committed
386
387

    rate_sum += rate;
Yaowu Xu's avatar
Yaowu Xu committed
388
    dist_sum += (int)dist;
Deb Mukherjee's avatar
Deb Mukherjee committed
389
390
391
392
393
394
  }

  *out_rate_sum = rate_sum;
  *out_dist_sum = dist_sum << 4;
}

395
static void model_rd_for_sb_y_tx(VP9_COMP *cpi, BLOCK_SIZE bsize,
Deb Mukherjee's avatar
Deb Mukherjee committed
396
397
398
399
                                 TX_SIZE tx_size,
                                 MACROBLOCK *x, MACROBLOCKD *xd,
                                 int *out_rate_sum, int64_t *out_dist_sum,
                                 int *out_skip) {
400
  int j, k;
401
  BLOCK_SIZE bs;
Deb Mukherjee's avatar
Deb Mukherjee committed
402
403
  struct macroblock_plane *const p = &x->plane[0];
  struct macroblockd_plane *const pd = &xd->plane[0];
404
405
  const int width = 4 << num_4x4_blocks_wide_lookup[bsize];
  const int height = 4 << num_4x4_blocks_high_lookup[bsize];
Deb Mukherjee's avatar
Deb Mukherjee committed
406
407
  int rate_sum = 0;
  int64_t dist_sum = 0;
408
  const int t = 4 << tx_size;
Deb Mukherjee's avatar
Deb Mukherjee committed
409
410
411
412
413
414
415
416
417
418
419
420

  if (tx_size == TX_4X4) {
    bs = BLOCK_4X4;
  } else if (tx_size == TX_8X8) {
    bs = BLOCK_8X8;
  } else if (tx_size == TX_16X16) {
    bs = BLOCK_16X16;
  } else if (tx_size == TX_32X32) {
    bs = BLOCK_32X32;
  } else {
    assert(0);
  }
421

Deb Mukherjee's avatar
Deb Mukherjee committed
422
  *out_skip = 1;
Jim Bankoski's avatar
Jim Bankoski committed
423
424
  for (j = 0; j < height; j += t) {
    for (k = 0; k < width; k += t) {
Deb Mukherjee's avatar
Deb Mukherjee committed
425
426
427
      int rate;
      int64_t dist;
      unsigned int sse;
428
429
430
      cpi->fn_ptr[bs].vf(&p->src.buf[j * p->src.stride + k], p->src.stride,
                         &pd->dst.buf[j * pd->dst.stride + k], pd->dst.stride,
                         &sse);
Deb Mukherjee's avatar
Deb Mukherjee committed
431
      // sse works better than var, since there is no dc prediction used
432
      model_rd_from_var_lapndz(sse, t * t, pd->dequant[1] >> 3, &rate, &dist);
Deb Mukherjee's avatar
Deb Mukherjee committed
433
434
435
436
437
      rate_sum += rate;
      dist_sum += dist;
      *out_skip &= (rate < 1024);
    }
  }
438

Deb Mukherjee's avatar
Deb Mukherjee committed
439
  *out_rate_sum = rate_sum;
440
  *out_dist_sum = dist_sum << 4;
Deb Mukherjee's avatar
Deb Mukherjee committed
441
442
}

Ronald S. Bultje's avatar
Ronald S. Bultje committed
443
int64_t vp9_block_error_c(int16_t *coeff, int16_t *dqcoeff,
444
                          intptr_t block_size, int64_t *ssz) {
Ronald S. Bultje's avatar
Ronald S. Bultje committed
445
  int i;
446
  int64_t error = 0, sqcoeff = 0;
John Koleszar's avatar
John Koleszar committed
447

448
  for (i = 0; i < block_size; i++) {
John Koleszar's avatar
John Koleszar committed
449
    int this_diff = coeff[i] - dqcoeff[i];
Ronald S. Bultje's avatar
Ronald S. Bultje committed
450
    error += (unsigned)this_diff * this_diff;
451
    sqcoeff += (unsigned) coeff[i] * coeff[i];
John Koleszar's avatar
John Koleszar committed
452
  }
John Koleszar's avatar
John Koleszar committed
453

454
  *ssz = sqcoeff;
John Koleszar's avatar
John Koleszar committed
455
  return error;
John Koleszar's avatar
John Koleszar committed
456
457
}

458
459
460
461
462
/* The trailing '0' is a terminator which is used inside cost_coeffs() to
 * decide whether to include cost of a trailing EOB node or not (i.e. we
 * can skip this if the last coefficient in this transform block, e.g. the
 * 16th coefficient in a 4x4 block or the 64th coefficient in a 8x8 block,
 * were non-zero). */
463
static const int16_t band_counts[TX_SIZES][8] = {
464
465
466
467
  { 1, 2, 3, 4,  3,   16 - 13, 0 },
  { 1, 2, 3, 4, 11,   64 - 21, 0 },
  { 1, 2, 3, 4, 11,  256 - 21, 0 },
  { 1, 2, 3, 4, 11, 1024 - 21, 0 },
468
469
};

470
static INLINE int cost_coeffs(MACROBLOCK *mb,
471
                              int plane, int block,
472
                              ENTROPY_CONTEXT *A, ENTROPY_CONTEXT *L,
John Koleszar's avatar
John Koleszar committed
473
                              TX_SIZE tx_size,
474
                              const int16_t *scan, const int16_t *nb) {
475
  MACROBLOCKD *const xd = &mb->e_mbd;
476
  MB_MODE_INFO *mbmi = &xd->this_mi->mbmi;
477
478
  struct macroblockd_plane *pd = &xd->plane[plane];
  const PLANE_TYPE type = pd->plane_type;
479
  const int16_t *band_count = &band_counts[tx_size][1];
480
481
  const int eob = pd->eobs[block];
  const int16_t *const qcoeff_ptr = BLOCK_OFFSET(pd->qcoeff, block);
Ronald S. Bultje's avatar
Ronald S. Bultje committed
482
  const int ref = mbmi->ref_frame[0] != INTRA_FRAME;
483
484
485
  unsigned int (*token_costs)[2][PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS] =
                   mb->token_costs[tx_size][type][ref];
  const ENTROPY_CONTEXT above_ec = !!*A, left_ec = !!*L;
486
  uint8_t token_cache[1024];
487
488
  int pt = combine_entropy_contexts(above_ec, left_ec);
  int c, cost;
489
490

  // Check for consistency of tx_size with mode info
491
  assert(type == PLANE_TYPE_Y_WITH_DC ? mbmi->tx_size == tx_size
492
                                      : get_uv_tx_size(mbmi) == tx_size);
493

494
495
  if (eob == 0) {
    // single eob token
496
497
    cost = token_costs[0][0][pt][DCT_EOB_TOKEN];
    c = 0;
498
  } else {
499
    int band_left = *band_count++;
500
501

    // dc token
502
503
    int v = qcoeff_ptr[0];
    int prev_t = vp9_dct_value_tokens_ptr[v].token;
504
    cost = (*token_costs)[0][pt][prev_t] + vp9_dct_value_cost_ptr[v];
505
    token_cache[0] = vp9_pt_energy_class[prev_t];
506
    ++token_costs;
507
508
509
510

    // ac tokens
    for (c = 1; c < eob; c++) {
      const int rc = scan[c];
511
      int t;
512
513
514

      v = qcoeff_ptr[rc];
      t = vp9_dct_value_tokens_ptr[v].token;
515
      pt = get_coef_context(nb, token_cache, c);
516
      cost += (*token_costs)[!prev_t][pt][t] + vp9_dct_value_cost_ptr[v];
517
      token_cache[rc] = vp9_pt_energy_class[t];
518
      prev_t = t;
519
      if (!--band_left) {
520
521
        band_left = *band_count++;
        ++token_costs;
522
      }
523
    }
524
525

    // eob token
526
    if (band_left) {
527
      pt = get_coef_context(nb, token_cache, c);
528
      cost += (*token_costs)[0][pt][DCT_EOB_TOKEN];
529
    }
530
531
  }

532
  // is eob first coefficient;
533
  *A = *L = (c > 0);
534

535
536
537
  return cost;
}

538
static void dist_block(int plane, int block, TX_SIZE tx_size, void *arg) {
539
  const int ss_txfrm_size = tx_size << 1;
Deb Mukherjee's avatar
Deb Mukherjee committed
540
541
542
  struct rdcost_block_args* args = arg;
  MACROBLOCK* const x = args->x;
  MACROBLOCKD* const xd = &x->e_mbd;
543
544
  struct macroblock_plane *const p = &x->plane[plane];
  struct macroblockd_plane *const pd = &xd->plane[plane];
Deb Mukherjee's avatar
Deb Mukherjee committed
545
546
  int64_t this_sse;
  int shift = args->tx_size == TX_32X32 ? 0 : 2;
547
548
  int16_t *const coeff = BLOCK_OFFSET(p->coeff, block);
  int16_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
549
550
551
  args->dist = vp9_block_error(coeff, dqcoeff, 16 << ss_txfrm_size,
                               &this_sse) >> shift;
  args->sse  = this_sse >> shift;
552
553

  if (x->skip_encode &&
554
      xd->this_mi->mbmi.ref_frame[0] == INTRA_FRAME) {
555
556
    // TODO(jingning): tune the model to better capture the distortion.
    int64_t p = (pd->dequant[1] * pd->dequant[1] *
557
                    (1 << ss_txfrm_size)) >> (shift + 2);
558
559
    args->dist += (p >> 4);
    args->sse  += p;
560
  }
Deb Mukherjee's avatar
Deb Mukherjee committed
561
562
}

563
static void rate_block(int plane, int block, BLOCK_SIZE plane_bsize,
564
                       TX_SIZE tx_size, void *arg) {
Deb Mukherjee's avatar
Deb Mukherjee committed
565
  struct rdcost_block_args* args = arg;
566

Deb Mukherjee's avatar
Deb Mukherjee committed
567
  int x_idx, y_idx;
568
  txfrm_block_to_raster_xy(plane_bsize, args->tx_size, block, &x_idx, &y_idx);
Deb Mukherjee's avatar
Deb Mukherjee committed
569

570
571
572
  args->rate = cost_coeffs(args->x, plane, block, args->t_above + x_idx,
                           args->t_left + y_idx, args->tx_size,
                           args->scan, args->nb);
Deb Mukherjee's avatar
Deb Mukherjee committed
573
574
}

575
static void block_yrd_txfm(int plane, int block, BLOCK_SIZE plane_bsize,
576
                           TX_SIZE tx_size, void *arg) {
Deb Mukherjee's avatar
Deb Mukherjee committed
577
578
579
  struct rdcost_block_args *args = arg;
  MACROBLOCK *const x = args->x;
  MACROBLOCKD *const xd = &x->e_mbd;
580
  struct encode_b_args encode_args = {x, NULL};
581
  int64_t rd1, rd2, rd;
Deb Mukherjee's avatar
Deb Mukherjee committed
582

583
584
585
  if (args->skip)
    return;

586
  if (!is_inter_block(&xd->this_mi->mbmi))
587
    vp9_encode_block_intra(plane, block, plane_bsize, tx_size, &encode_args);
Deb Mukherjee's avatar
Deb Mukherjee committed
588
  else
589
    vp9_xform_quant(plane, block, plane_bsize, tx_size, &encode_args);
Deb Mukherjee's avatar
Deb Mukherjee committed
590

591
592
  dist_block(plane, block, tx_size, args);
  rate_block(plane, block, plane_bsize, tx_size, args);
593
594
  rd1 = RDCOST(x->rdmult, x->rddiv, args->rate, args->dist);
  rd2 = RDCOST(x->rdmult, x->rddiv, 0, args->sse);
595
596

  // TODO(jingning): temporarily enabled only for luma component
597
  rd = MIN(rd1, rd2);
598
599
600
  if (plane == 0)
    x->zcoeff_blk[tx_size][block] = rd1 > rd2;

601
602
603
  args->this_rate += args->rate;
  args->this_dist += args->dist;
  args->this_sse  += args->sse;
604
605
606
607
608
609
  args->this_rd += rd;

  if (args->this_rd > args->best_rd) {
    args->skip = 1;
    return;
  }
Deb Mukherjee's avatar
Deb Mukherjee committed
610
611
}

612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
void vp9_get_entropy_contexts(TX_SIZE tx_size,
    ENTROPY_CONTEXT t_above[16], ENTROPY_CONTEXT t_left[16],
    const ENTROPY_CONTEXT *above, const ENTROPY_CONTEXT *left,
    int num_4x4_w, int num_4x4_h) {
  int i;
  switch (tx_size) {
    case TX_4X4:
      vpx_memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
      vpx_memcpy(t_left, left, sizeof(ENTROPY_CONTEXT) * num_4x4_h);
      break;
    case TX_8X8:
      for (i = 0; i < num_4x4_w; i += 2)
        t_above[i] = !!*(const uint16_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 2)
        t_left[i] = !!*(const uint16_t *)&left[i];
      break;
    case TX_16X16:
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
    case TX_32X32:
      for (i = 0; i < num_4x4_w; i += 8)
        t_above[i] = !!*(const uint64_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 8)
        t_left[i] = !!*(const uint64_t *)&left[i];
      break;
    default:
      assert(!"Invalid transform size.");
  }
}

645
646
647
648
649
650
651
652
653
654
655
656
static void init_rdcost_stack(MACROBLOCK *x, TX_SIZE tx_size,
                              const int num_4x4_w, const int num_4x4_h,
                              const int64_t ref_rdcost,
                              struct rdcost_block_args *arg) {
  vpx_memset(arg, 0, sizeof(struct rdcost_block_args));
  arg->x = x;
  arg->tx_size = tx_size;
  arg->bw = num_4x4_w;
  arg->bh = num_4x4_h;
  arg->best_rd = ref_rdcost;
}

657
static void txfm_rd_in_plane(MACROBLOCK *x,
658
                             struct rdcost_block_args *rd_stack,
659
660
661
                             int *rate, int64_t *distortion,
                             int *skippable, int64_t *sse,
                             int64_t ref_best_rd, int plane,
662
                             BLOCK_SIZE bsize, TX_SIZE tx_size) {
Deb Mukherjee's avatar
Deb Mukherjee committed
663
  MACROBLOCKD *const xd = &x->e_mbd;
664
  struct macroblockd_plane *const pd = &xd->plane[plane];
665
  const BLOCK_SIZE bs = get_plane_block_size(bsize, pd);
666
667
  const int num_4x4_w = num_4x4_blocks_wide_lookup[bs];
  const int num_4x4_h = num_4x4_blocks_high_lookup[bs];
668
  const uint8_t *band_translate;  // just for the get_scan_and_band call
669

670
671
  init_rdcost_stack(x, tx_size, num_4x4_w, num_4x4_h,
                    ref_best_rd, rd_stack);
672
  if (plane == 0)
673
    xd->this_mi->mbmi.tx_size = tx_size;
674

675
  vp9_get_entropy_contexts(tx_size, rd_stack->t_above, rd_stack->t_left,
676
677
                           pd->above_context, pd->left_context,
                           num_4x4_w, num_4x4_h);
678

679
680
  get_scan_and_band(xd, tx_size, pd->plane_type, 0, &rd_stack->scan,
                    &rd_stack->nb, &band_translate);
Deb Mukherjee's avatar
Deb Mukherjee committed
681

682
683
684
  foreach_transformed_block_in_plane(xd, bsize, plane,
                                     block_yrd_txfm, rd_stack);
  if (rd_stack->skip) {
685
686
687
688
689
    *rate       = INT_MAX;
    *distortion = INT64_MAX;
    *sse        = INT64_MAX;
    *skippable  = 0;
  } else {
690
691
692
    *distortion = rd_stack->this_dist;
    *rate       = rd_stack->this_rate;
    *sse        = rd_stack->this_sse;
693
694
    *skippable  = vp9_is_skippable_in_plane(xd, bsize, plane);
  }
Deb Mukherjee's avatar
Deb Mukherjee committed
695
696
697
698
699
}

static void choose_largest_txfm_size(VP9_COMP *cpi, MACROBLOCK *x,
                                     int *rate, int64_t *distortion,
                                     int *skip, int64_t *sse,
700
                                     int64_t ref_best_rd,
701
                                     BLOCK_SIZE bs) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
702
  const TX_SIZE max_tx_size = max_txsize_lookup[bs];
Deb Mukherjee's avatar
Deb Mukherjee committed
703
704
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
705
  MB_MODE_INFO *const mbmi = &xd->this_mi->mbmi;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
706
  if (max_tx_size == TX_32X32 &&
707
708
      (cm->tx_mode == ALLOW_32X32 ||
       cm->tx_mode == TX_MODE_SELECT)) {
709
    mbmi->tx_size = TX_32X32;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
710
  } else if (max_tx_size >= TX_16X16 &&
711
712
713
             (cm->tx_mode == ALLOW_16X16 ||
              cm->tx_mode == ALLOW_32X32 ||
              cm->tx_mode == TX_MODE_SELECT)) {
714
    mbmi->tx_size = TX_16X16;
715
  } else if (cm->tx_mode != ONLY_4X4) {
716
    mbmi->tx_size = TX_8X8;
Deb Mukherjee's avatar
Deb Mukherjee committed
717
  } else {
718
    mbmi->tx_size = TX_4X4;
Deb Mukherjee's avatar
Deb Mukherjee committed
719
  }
720
  txfm_rd_in_plane(x, &cpi->rdcost_stack, rate, distortion, skip,
721
722
                   &sse[mbmi->tx_size], ref_best_rd, 0, bs,
                   mbmi->tx_size);
Dmitry Kovalev's avatar
Dmitry Kovalev committed
723
  cpi->tx_stepdown_count[0]++;
Deb Mukherjee's avatar
Deb Mukherjee committed
724
725
}

726
static void choose_txfm_size_from_rd(VP9_COMP *cpi, MACROBLOCK *x,
727
                                     int (*r)[2], int *rate,
Ronald S. Bultje's avatar
Ronald S. Bultje committed
728
                                     int64_t *d, int64_t *distortion,
729
                                     int *s, int *skip,
730
                                     int64_t tx_cache[TX_MODES],
731
                                     BLOCK_SIZE bs) {
732
  const TX_SIZE max_tx_size = max_txsize_lookup[bs];
733
734
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
735
  MB_MODE_INFO *const mbmi = &xd->this_mi->mbmi;
736
  vp9_prob skip_prob = vp9_get_pred_prob_mbskip(cm, xd);
737
  int64_t rd[TX_SIZES][2];
738
  int n, m;
739
  int s0, s1;
740

741
  const vp9_prob *tx_probs = get_tx_probs2(xd, &cm->fc.tx_probs, xd->this_mi);
742

743
  for (n = TX_4X4; n <= max_tx_size; n++) {
744
    r[n][1] = r[n][0];
745
746
    if (r[n][0] == INT_MAX)
      continue;
747
    for (m = 0; m <= n - (n == max_tx_size); m++) {
748
      if (m == n)
749
        r[n][1] += vp9_cost_zero(tx_probs[m]);
750
      else
751
        r[n][1] += vp9_cost_one(tx_probs[m]);
752
753
    }
  }
754

755
756
757
  assert(skip_prob > 0);
  s0 = vp9_cost_bit(skip_prob, 0);
  s1 = vp9_cost_bit(skip_prob, 1);
758

759
  for (n = TX_4X4; n <= max_tx_size; n++) {
760
761
762
763
    if (d[n] == INT64_MAX) {
      rd[n][0] = rd[n][1] = INT64_MAX;
      continue;
    }
764
765
766
767
768
    if (s[n]) {
      rd[n][0] = rd[n][1] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
    } else {
      rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0] + s0, d[n]);
      rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1] + s0, d[n]);
769
770
771
    }
  }

772
  if (max_tx_size == TX_32X32 &&
773
774
      (cm->tx_mode == ALLOW_32X32 ||
       (cm->tx_mode == TX_MODE_SELECT &&
775
776
        rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
        rd[TX_32X32][1] < rd[TX_4X4][1]))) {
777
    mbmi->tx_size = TX_32X32;
778
  } else if (max_tx_size >= TX_16X16 &&
779
780
781
             (cm->tx_mode == ALLOW_16X16 ||
              cm->tx_mode == ALLOW_32X32 ||
              (cm->tx_mode == TX_MODE_SELECT &&
782
783
               rd[TX_16X16][1] < rd[TX_8X8][1] &&
               rd[TX_16X16][1] < rd[TX_4X4][1]))) {
784
    mbmi->tx_size = TX_16X16;
785
786
787
788
  } else if (cm->tx_mode == ALLOW_8X8 ||
             cm->tx_mode == ALLOW_16X16 ||
             cm->tx_mode == ALLOW_32X32 ||
           (cm->tx_mode == TX_MODE_SELECT && rd[TX_8X8][1] < rd[TX_4X4][1])) {
789
    mbmi->tx_size = TX_8X8;
790
  } else {
791
    mbmi->tx_size = TX_4X4;
792
793
  }

794
795
796
  *distortion = d[mbmi->tx_size];
  *rate       = r[mbmi->tx_size][cm->tx_mode == TX_MODE_SELECT];
  *skip       = s[mbmi->tx_size];
797

798
799
800
801
802
  tx_cache[ONLY_4X4] = rd[TX_4X4][0];
  tx_cache[ALLOW_8X8] = rd[TX_8X8][0];
  tx_cache[ALLOW_16X16] = rd[MIN(max_tx_size, TX_16X16)][0];
  tx_cache[ALLOW_32X32] = rd[MIN(max_tx_size, TX_32X32)][0];
  if (max_tx_size == TX_32X32 &&
803
804
      rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
      rd[TX_32X32][1] < rd[TX_4X4][1])
805
806
    tx_cache[TX_MODE_SELECT] = rd[TX_32X32][1];
  else if (max_tx_size >= TX_16X16 &&
807
           rd[TX_16X16][1] < rd[TX_8X8][1] && rd[TX_16X16][1] < rd[TX_4X4][1])
808
    tx_cache[TX_MODE_SELECT] = rd[TX_16X16][1];
809
  else
810
    tx_cache[TX_MODE_SELECT] = rd[TX_4X4][1] < rd[TX_8X8][1] ?
811
                                 rd[TX_4X4][1] : rd[TX_8X8][1];
812

813
  if (max_tx_size == TX_32X32 &&
Deb Mukherjee's avatar
Deb Mukherjee committed
814
815
816
      rd[TX_32X32][1] < rd[TX_16X16][1] &&
      rd[TX_32X32][1] < rd[TX_8X8][1] &&
      rd[TX_32X32][1] < rd[TX_4X4][1]) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
817
    cpi->tx_stepdown_count[0]++;
818
  } else if (max_tx_size >= TX_16X16 &&
Deb Mukherjee's avatar
Deb Mukherjee committed
819
820
             rd[TX_16X16][1] < rd[TX_8X8][1] &&
             rd[TX_16X16][1] < rd[TX_4X4][1]) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
821
    cpi->tx_stepdown_count[max_tx_size - TX_16X16]++;
Deb Mukherjee's avatar
Deb Mukherjee committed
822
  } else if (rd[TX_8X8][1] < rd[TX_4X4][1]) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
823
    cpi->tx_stepdown_count[max_tx_size - TX_8X8]++;
Deb Mukherjee's avatar
Deb Mukherjee committed
824
  } else {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
825
    cpi->tx_stepdown_count[max_tx_size - TX_4X4]++;
826
  }
Deb Mukherjee's avatar
Deb Mukherjee committed
827
}
828

Deb Mukherjee's avatar
Deb Mukherjee committed
829
830
831
832
static void choose_txfm_size_from_modelrd(VP9_COMP *cpi, MACROBLOCK *x,
                                          int (*r)[2], int *rate,
                                          int64_t *d, int64_t *distortion,
                                          int *s, int *skip, int64_t *sse,
833
                                          int64_t ref_best_rd,
834
                                          BLOCK_SIZE bs) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
835
  const TX_SIZE max_tx_size = max_txsize_lookup[bs];
Deb Mukherjee's avatar
Deb Mukherjee committed
836
837
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
838
  MB_MODE_INFO *const mbmi = &xd->this_mi->mbmi;
839
  vp9_prob skip_prob = vp9_get_pred_prob_mbskip(cm, xd);
840
  int64_t rd[TX_SIZES][2];
Deb Mukherjee's avatar
Deb Mukherjee committed
841
842
  int n, m;
  int s0, s1;
843
844
  double scale_rd[TX_SIZES] = {1.73, 1.44, 1.20, 1.00};
  // double scale_r[TX_SIZES] = {2.82, 2.00, 1.41, 1.00};
845

846
  const vp9_prob *tx_probs = get_tx_probs2(xd, &cm->fc.tx_probs,  xd->this_mi);
847

Deb Mukherjee's avatar
Deb Mukherjee committed
848
849
  // for (n = TX_4X4; n <= max_txfm_size; n++)
  //   r[n][0] = (r[n][0] * scale_r[n]);
850

Dmitry Kovalev's avatar
Dmitry Kovalev committed
851
  for (n = TX_4X4; n <= max_tx_size; n++) {
Deb Mukherjee's avatar
Deb Mukherjee committed
852
    r[n][1] = r[n][0];
Dmitry Kovalev's avatar
Dmitry Kovalev committed
853
    for (m = 0; m <= n - (n == max_tx_size); m++) {
Deb Mukherjee's avatar
Deb Mukherjee committed
854
855
856
857
858
      if (m == n)
        r[n][1] += vp9_cost_zero(tx_probs[m]);
      else
        r[n][1] += vp9_cost_one(tx_probs[m]);
    }
859
  }
860

Deb Mukherjee's avatar
Deb Mukherjee committed
861
862
863
  assert(skip_prob > 0);
  s0 = vp9_cost_bit(skip_prob, 0);
  s1 = vp9_cost_bit(skip_prob, 1);
864

Dmitry Kovalev's avatar
Dmitry Kovalev committed
865
  for (n = TX_4X4; n <= max_tx_size; n++) {
Deb Mukherjee's avatar
Deb Mukherjee committed
866
867
868
869
870
871
872
    if (s[n]) {
      rd[n][0] = rd[n][1] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
    } else {
      rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0] + s0, d[n]);
      rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1] + s0, d[n]);
    }
  }
Dmitry Kovalev's avatar
Dmitry Kovalev committed
873
  for (n = TX_4X4; n <= max_tx_size; n++) {
Yaowu Xu's avatar
Yaowu Xu committed
874
875
    rd[n][0] = (int64_t)(scale_rd[n] * rd[n][0]);
    rd[n][1] = (int64_t)(scale_rd[n] * rd[n][1]);
Deb Mukherjee's avatar
Deb Mukherjee committed
876
  }
877

Dmitry Kovalev's avatar
Dmitry Kovalev committed
878
  if (max_tx_size == TX_32X32 &&
879
880
      (cm->tx_mode == ALLOW_32X32 ||
       (cm->tx_mode == TX_MODE_SELECT &&
Deb Mukherjee's avatar
Deb Mukherjee committed
881
882
883
        rd[TX_32X32][1] <= rd[TX_16X16][1] &&
        rd[TX_32X32][1] <= rd[TX_8X8][1] &&
        rd[TX_32X32][1] <= rd[TX_4X4][1]))) {
884
    mbmi->tx_size = TX_32X32;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
885
  } else if (max_tx_size >= TX_16X16 &&
886
887
888
             (cm->tx_mode == ALLOW_16X16 ||
              cm->tx_mode == ALLOW_32X32 ||
              (cm->tx_mode == TX_MODE_SELECT &&
Deb Mukherjee's avatar
Deb Mukherjee committed
889
890
               rd[TX_16X16][1] <= rd[TX_8X8][1] &&
               rd[TX_16X16][1] <= rd[TX_4X4][1]))) {
891
    mbmi->tx_size = TX_16X16;
892
893
894
895
  } else if (cm->tx_mode == ALLOW_8X8 ||
             cm->tx_mode == ALLOW_16X16 ||
             cm->tx_mode == ALLOW_32X32 ||
           (cm->tx_mode == TX_MODE_SELECT &&
Deb Mukherjee's avatar
Deb Mukherjee committed
896
            rd[TX_8X8][1] <= rd[TX_4X4][1])) {
897
    mbmi->tx_size = TX_8X8;
Deb Mukherjee's avatar
Deb Mukherjee committed
898
  } else {
899
    mbmi->tx_size = TX_4X4;
Deb Mukherjee's avatar
Deb Mukherjee committed
900
  }
901

902
903
  // Actually encode using the chosen mode if a model was used, but do not
  // update the r, d costs
904
905
  txfm_rd_in_plane(x, &cpi->rdcost_stack, rate, distortion, skip,
                   &sse[mbmi->tx_size], ref_best_rd, 0, bs, mbmi->tx_size);
906

Dmitry Kovalev's avatar
Dmitry Kovalev committed
907
  if (max_tx_size == TX_32X32 &&
Deb Mukherjee's avatar
Deb Mukherjee committed
908
909
910
      rd[TX_32X32][1] <= rd[TX_16X16][1] &&
      rd[TX_32X32][1] <= rd[TX_8X8][1] &&
      rd[TX_32X32][1] <= rd[TX_4X4][1]) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
911
</