vp9_rdopt.c 164 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5
6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9
10
11
12
13
14
 */

#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
15

16
#include "vp9/common/vp9_pragmas.h"
17
18
19
#include "vp9/encoder/vp9_tokenize.h"
#include "vp9/encoder/vp9_treewriter.h"
#include "vp9/encoder/vp9_onyx_int.h"
20
21
22
23
#include "vp9/common/vp9_entropymode.h"
#include "vp9/common/vp9_reconinter.h"
#include "vp9/common/vp9_reconintra.h"
#include "vp9/common/vp9_quant_common.h"
24
25
26
27
28
29
#include "vp9/encoder/vp9_encodemb.h"
#include "vp9/encoder/vp9_quantize.h"
#include "vp9/encoder/vp9_variance.h"
#include "vp9/encoder/vp9_mcomp.h"
#include "vp9/encoder/vp9_rdopt.h"
#include "vp9/encoder/vp9_ratectrl.h"
John Koleszar's avatar
John Koleszar committed
30
#include "vpx_mem/vpx_mem.h"
31
32
33
34
35
#include "vp9/common/vp9_systemdependent.h"
#include "vp9/encoder/vp9_encodemv.h"
#include "vp9/common/vp9_seg_common.h"
#include "vp9/common/vp9_pred_common.h"
#include "vp9/common/vp9_entropy.h"
36
#include "./vp9_rtcd.h"
37
#include "vp9/common/vp9_mvref_common.h"
Ronald S. Bultje's avatar
Ronald S. Bultje committed
38
#include "vp9/common/vp9_common.h"
Paul Wilkins's avatar
Paul Wilkins committed
39

40
41
42
/* Factor to weigh the rate for switchable interp filters */
#define SWITCHABLE_INTERP_RATE_FACTOR 1

43
44
45
#define LAST_FRAME_MODE_MASK    0xFFEDCD60
#define GOLDEN_FRAME_MODE_MASK  0xFFDA3BB0
#define ALT_REF_MODE_MASK       0xFFC648D0
46

Paul Wilkins's avatar
Paul Wilkins committed
47
48
#define MIN_EARLY_TERM_INDEX    3

49
50
51
52
53
54
55
56
57
typedef struct {
  MB_PREDICTION_MODE mode;
  MV_REFERENCE_FRAME ref_frame[2];
} MODE_DEFINITION;

typedef struct {
  MV_REFERENCE_FRAME ref_frame[2];
} REF_DEFINITION;

Alex Converse's avatar
Alex Converse committed
58
59
60
61
62
63
64
65
66
67
68
69
70
struct rdcost_block_args {
  MACROBLOCK *x;
  ENTROPY_CONTEXT t_above[16];
  ENTROPY_CONTEXT t_left[16];
  int rate;
  int64_t dist;
  int64_t sse;
  int this_rate;
  int64_t this_dist;
  int64_t this_sse;
  int64_t this_rd;
  int64_t best_rd;
  int skip;
71
  const scan_order *so;
Alex Converse's avatar
Alex Converse committed
72
73
};

74
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
  {NEARESTMV, {LAST_FRAME,   NONE}},
  {NEARESTMV, {ALTREF_FRAME, NONE}},
  {NEARESTMV, {GOLDEN_FRAME, NONE}},

  {DC_PRED,   {INTRA_FRAME,  NONE}},

  {NEWMV,     {LAST_FRAME,   NONE}},
  {NEWMV,     {ALTREF_FRAME, NONE}},
  {NEWMV,     {GOLDEN_FRAME, NONE}},

  {NEARMV,    {LAST_FRAME,   NONE}},
  {NEARMV,    {ALTREF_FRAME, NONE}},
  {NEARESTMV, {LAST_FRAME,   ALTREF_FRAME}},
  {NEARESTMV, {GOLDEN_FRAME, ALTREF_FRAME}},

  {TM_PRED,   {INTRA_FRAME,  NONE}},

  {NEARMV,    {LAST_FRAME,   ALTREF_FRAME}},
  {NEWMV,     {LAST_FRAME,   ALTREF_FRAME}},
  {NEARMV,    {GOLDEN_FRAME, NONE}},
  {NEARMV,    {GOLDEN_FRAME, ALTREF_FRAME}},
  {NEWMV,     {GOLDEN_FRAME, ALTREF_FRAME}},

  {ZEROMV,    {LAST_FRAME,   NONE}},
  {ZEROMV,    {GOLDEN_FRAME, NONE}},
  {ZEROMV,    {ALTREF_FRAME, NONE}},
  {ZEROMV,    {LAST_FRAME,   ALTREF_FRAME}},
  {ZEROMV,    {GOLDEN_FRAME, ALTREF_FRAME}},

  {H_PRED,    {INTRA_FRAME,  NONE}},
  {V_PRED,    {INTRA_FRAME,  NONE}},
  {D135_PRED, {INTRA_FRAME,  NONE}},
  {D207_PRED, {INTRA_FRAME,  NONE}},
  {D153_PRED, {INTRA_FRAME,  NONE}},
  {D63_PRED,  {INTRA_FRAME,  NONE}},
  {D117_PRED, {INTRA_FRAME,  NONE}},
  {D45_PRED,  {INTRA_FRAME,  NONE}},
112
113
114
};

const REF_DEFINITION vp9_ref_order[MAX_REFS] = {
115
116
117
118
119
120
  {{LAST_FRAME,   NONE}},
  {{GOLDEN_FRAME, NONE}},
  {{ALTREF_FRAME, NONE}},
  {{LAST_FRAME,   ALTREF_FRAME}},
  {{GOLDEN_FRAME, ALTREF_FRAME}},
  {{INTRA_FRAME,  NONE}},
John Koleszar's avatar
John Koleszar committed
121
122
};

123
124
125
126
// The baseline rd thresholds for breaking out of the rd loop for
// certain modes are assumed to be based on 8x8 blocks.
// This table is used to correct for blocks size.
// The factors here are << 2 (2 = x0.5, 32 = x8 etc).
127
static int rd_thresh_block_size_factor[BLOCK_SIZES] =
128
129
  {2, 3, 3, 4, 6, 6, 8, 12, 12, 16, 24, 24, 32};

130
131
132
133
134
135
136
137
138
static int raster_block_offset(BLOCK_SIZE plane_bsize,
                               int raster_block, int stride) {
  const int bw = b_width_log2(plane_bsize);
  const int y = 4 * (raster_block >> bw);
  const int x = 4 * (raster_block & ((1 << bw) - 1));
  return y * stride + x;
}
static int16_t* raster_block_offset_int16(BLOCK_SIZE plane_bsize,
                                          int raster_block, int16_t *base) {
139
  const int stride = 4 * num_4x4_blocks_wide_lookup[plane_bsize];
140
141
142
  return base + raster_block_offset(plane_bsize, raster_block, stride);
}

Dmitry Kovalev's avatar
Dmitry Kovalev committed
143
144
145
146
static void fill_mode_costs(VP9_COMP *cpi) {
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCK *const x = &cpi->mb;
  FRAME_CONTEXT *const fc = &cm->fc;
147
148
149
150
  int i, j;

  for (i = 0; i < INTRA_MODES; i++)
    for (j = 0; j < INTRA_MODES; j++)
Dmitry Kovalev's avatar
Dmitry Kovalev committed
151
      vp9_cost_tokens((int *)x->y_mode_costs[i][j], vp9_kf_y_mode_prob[i][j],
152
153
154
                      vp9_intra_mode_tree);

  // TODO(rbultje) separate tables for superblock costing?
Dmitry Kovalev's avatar
Dmitry Kovalev committed
155
156
157
158
159
  vp9_cost_tokens(x->mbmode_cost, fc->y_mode_prob[1], vp9_intra_mode_tree);
  vp9_cost_tokens(x->intra_uv_mode_cost[1],
                  fc->uv_mode_prob[INTRA_MODES - 1], vp9_intra_mode_tree);
  vp9_cost_tokens(x->intra_uv_mode_cost[0],
                  vp9_kf_uv_mode_prob[INTRA_MODES - 1], vp9_intra_mode_tree);
160
161

  for (i = 0; i < SWITCHABLE_FILTER_CONTEXTS; ++i)
Dmitry Kovalev's avatar
Dmitry Kovalev committed
162
163
    vp9_cost_tokens((int *)x->switchable_interp_costs[i],
                    fc->switchable_interp_prob[i],
164
165
166
                    vp9_switchable_interp_tree);
}

167
static void fill_token_costs(vp9_coeff_cost *c,
168
                             vp9_coeff_probs_model (*p)[PLANE_TYPES]) {
169
  int i, j, k, l;
170
  TX_SIZE t;
171
  for (t = TX_4X4; t <= TX_32X32; ++t)
172
    for (i = 0; i < PLANE_TYPES; ++i)
173
174
175
      for (j = 0; j < REF_TYPES; ++j)
        for (k = 0; k < COEF_BANDS; ++k)
          for (l = 0; l < BAND_COEFF_CONTEXTS(k); ++l) {
176
177
            vp9_prob probs[ENTROPY_NODES];
            vp9_model_to_full_probs(p[t][i][j][k][l], probs);
178
            vp9_cost_tokens((int *)c[t][i][j][k][0][l], probs,
179
                            vp9_coef_tree);
180
            vp9_cost_tokens_skip((int *)c[t][i][j][k][1][l], probs,
181
                                 vp9_coef_tree);
182
183
            assert(c[t][i][j][k][0][l][EOB_TOKEN] ==
                   c[t][i][j][k][1][l][EOB_TOKEN]);
184
          }
185
186
}

187
188
189
190
191
192
static const int rd_iifactor[32] = {
  4, 4, 3, 2, 1, 0, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 0,
  0, 0, 0, 0, 0, 0, 0, 0,
};
John Koleszar's avatar
John Koleszar committed
193

194
// 3* dc_qlookup[Q]*dc_qlookup[Q];
195

196
/* values are now correlated to quantizer */
Paul Wilkins's avatar
Paul Wilkins committed
197
198
199
static int sad_per_bit16lut[QINDEX_RANGE];
static int sad_per_bit4lut[QINDEX_RANGE];

200
void vp9_init_me_luts() {
John Koleszar's avatar
John Koleszar committed
201
202
203
204
205
206
  int i;

  // Initialize the sad lut tables using a formulaic calculation for now
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < QINDEX_RANGE; i++) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
207
208
209
    const double q = vp9_convert_qindex_to_q(i);
    sad_per_bit16lut[i] = (int)(0.0418 * q + 2.4107);
    sad_per_bit4lut[i] = (int)(0.063 * q + 2.742);
John Koleszar's avatar
John Koleszar committed
210
  }
Paul Wilkins's avatar
Paul Wilkins committed
211
}
John Koleszar's avatar
John Koleszar committed
212

213
int vp9_compute_rd_mult(VP9_COMP *cpi, int qindex) {
214
  const int q = vp9_dc_quant(qindex, 0);
215
  // TODO(debargha): Adjust the function below
216
217
218
219
220
221
222
223
  int rdmult = 88 * q * q / 25;
  if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    if (cpi->twopass.next_iiratio > 31)
      rdmult += (rdmult * rd_iifactor[31]) >> 4;
    else
      rdmult += (rdmult * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
  }
  return rdmult;
224
225
226
227
228
229
230
231
232
}

static int compute_rd_thresh_factor(int qindex) {
  int q;
  // TODO(debargha): Adjust the function below
  q = (int)(pow(vp9_dc_quant(qindex, 0) / 4.0, RD_THRESH_POW) * 5.12);
  if (q < 8)
    q = 8;
  return q;
233
234
}

Dmitry Kovalev's avatar
Dmitry Kovalev committed
235
236
237
void vp9_initialize_me_consts(VP9_COMP *cpi, int qindex) {
  cpi->mb.sadperbit16 = sad_per_bit16lut[qindex];
  cpi->mb.sadperbit4 = sad_per_bit4lut[qindex];
John Koleszar's avatar
John Koleszar committed
238
239
}

240
241
242
static void set_block_thresholds(VP9_COMP *cpi) {
  int i, bsize, segment_id;
  VP9_COMMON *cm = &cpi->common;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
243
  SPEED_FEATURES *sf = &cpi->sf;
244
245

  for (segment_id = 0; segment_id < MAX_SEGMENTS; ++segment_id) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
246
247
248
249
    const int qindex = clamp(vp9_get_qindex(&cm->seg, segment_id,
                                            cm->base_qindex) + cm->y_dc_delta_q,
                             0, MAXQ);
    const int q = compute_rd_thresh_factor(qindex);
250

251
    for (bsize = 0; bsize < BLOCK_SIZES; ++bsize) {
252
253
      // Threshold here seems unnecessarily harsh but fine given actual
      // range of values used for cpi->sf.thresh_mult[].
Dmitry Kovalev's avatar
Dmitry Kovalev committed
254
255
256
257
258
259
260
      const int t = q * rd_thresh_block_size_factor[bsize];
      const int thresh_max = INT_MAX / t;

      for (i = 0; i < MAX_MODES; ++i)
        cpi->rd_threshes[segment_id][bsize][i] =
            sf->thresh_mult[i] < thresh_max ? sf->thresh_mult[i] * t / 4
                                            : INT_MAX;
261

262
      for (i = 0; i < MAX_REFS; ++i) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
263
264
265
266
        cpi->rd_thresh_sub8x8[segment_id][bsize][i] =
            sf->thresh_mult_sub8x8[i] < thresh_max
                ? sf->thresh_mult_sub8x8[i] * t / 4
                : INT_MAX;
267
268
      }
    }
John Koleszar's avatar
John Koleszar committed
269
  }
Deb Mukherjee's avatar
Deb Mukherjee committed
270
271
}

272
273
void vp9_initialize_rd_consts(VP9_COMP *cpi) {
  VP9_COMMON *cm = &cpi->common;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
274
  MACROBLOCK *x = &cpi->mb;
275
  int qindex, i;
Deb Mukherjee's avatar
Deb Mukherjee committed
276

277
  vp9_clear_system_state();
Deb Mukherjee's avatar
Deb Mukherjee committed
278
279
280
281
282

  // Further tests required to see if optimum is different
  // for key frames, golden frames and arf frames.
  // if (cpi->common.refresh_golden_frame ||
  //     cpi->common.refresh_alt_ref_frame)
283
  qindex = clamp(cm->base_qindex + cm->y_dc_delta_q, 0, MAXQ);
Deb Mukherjee's avatar
Deb Mukherjee committed
284

285
  cpi->RDDIV = RDDIV_BITS;  // in bits (to multiply D by 128)
286
287
  cpi->RDMULT = vp9_compute_rd_mult(cpi, qindex);

288
289
  x->errorperbit = cpi->RDMULT / RD_MULT_EPB_RATIO;
  x->errorperbit += (x->errorperbit == 0);
Deb Mukherjee's avatar
Deb Mukherjee committed
290
291
292

  vp9_set_speed_features(cpi);

Dmitry Kovalev's avatar
Dmitry Kovalev committed
293
294
  x->select_txfm_size = (cpi->sf.tx_size_search_method == USE_LARGESTALL &&
                         cm->frame_type != KEY_FRAME) ? 0 : 1;
295

296
  set_block_thresholds(cpi);
John Koleszar's avatar
John Koleszar committed
297

Dmitry Kovalev's avatar
Dmitry Kovalev committed
298
  fill_token_costs(x->token_costs, cm->fc.coef_probs);
299

300
  if (!cpi->sf.use_pick_mode) {
Jim Bankoski's avatar
Jim Bankoski committed
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
    for (i = 0; i < PARTITION_CONTEXTS; i++)
      vp9_cost_tokens(x->partition_cost[i], get_partition_probs(cm, i),
                      vp9_partition_tree);

    fill_mode_costs(cpi);

    if (!frame_is_intra_only(cm)) {
      vp9_build_nmv_cost_table(x->nmvjointcost,
                               cm->allow_high_precision_mv ? x->nmvcost_hp
                                                           : x->nmvcost,
                               &cm->fc.nmvc,
                               cm->allow_high_precision_mv, 1, 1);

      for (i = 0; i < INTER_MODE_CONTEXTS; ++i)
        vp9_cost_tokens((int *)x->inter_mode_cost[i],
                        cm->fc.inter_mode_probs[i], vp9_inter_mode_tree);
    }
318
  }
John Koleszar's avatar
John Koleszar committed
319
320
}

321
static const int MAX_XSQ_Q10 = 245727;
Deb Mukherjee's avatar
Deb Mukherjee committed
322

323
static void model_rd_norm(int xsq_q10, int *r_q10, int *d_q10) {
324
  // NOTE: The tables below must be of the same size
325
326
327
328

  // The functions described below are sampled at the four most significant
  // bits of x^2 + 8 / 256

Deb Mukherjee's avatar
Deb Mukherjee committed
329
  // Normalized rate
330
  // This table models the rate for a Laplacian source
Deb Mukherjee's avatar
Deb Mukherjee committed
331
332
333
334
335
  // source with given variance when quantized with a uniform quantizer
  // with given stepsize. The closed form expression is:
  // Rn(x) = H(sqrt(r)) + sqrt(r)*[1 + H(r)/(1 - r)],
  // where r = exp(-sqrt(2) * x) and x = qpstep / sqrt(variance),
  // and H(x) is the binary entropy function.
336
337
338
339
340
341
342
343
344
345
346
347
348
349
  static const int rate_tab_q10[] = {
    65536,  6086,  5574,  5275,  5063,  4899,  4764,  4651,
     4553,  4389,  4255,  4142,  4044,  3958,  3881,  3811,
     3748,  3635,  3538,  3453,  3376,  3307,  3244,  3186,
     3133,  3037,  2952,  2877,  2809,  2747,  2690,  2638,
     2589,  2501,  2423,  2353,  2290,  2232,  2179,  2130,
     2084,  2001,  1928,  1862,  1802,  1748,  1698,  1651,
     1608,  1530,  1460,  1398,  1342,  1290,  1243,  1199,
     1159,  1086,  1021,   963,   911,   864,   821,   781,
      745,   680,   623,   574,   530,   490,   455,   424,
      395,   345,   304,   269,   239,   213,   190,   171,
      154,   126,   104,    87,    73,    61,    52,    44,
       38,    28,    21,    16,    12,    10,     8,     6,
        5,     3,     2,     1,     1,     1,     0,     0,
Deb Mukherjee's avatar
Deb Mukherjee committed
350
351
  };
  // Normalized distortion
352
  // This table models the normalized distortion for a Laplacian source
Deb Mukherjee's avatar
Deb Mukherjee committed
353
354
355
356
357
  // source with given variance when quantized with a uniform quantizer
  // with given stepsize. The closed form expression is:
  // Dn(x) = 1 - 1/sqrt(2) * x / sinh(x/sqrt(2))
  // where x = qpstep / sqrt(variance)
  // Note the actual distortion is Dn * variance.
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
  static const int dist_tab_q10[] = {
       0,     0,     1,     1,     1,     2,     2,     2,
       3,     3,     4,     5,     5,     6,     7,     7,
       8,     9,    11,    12,    13,    15,    16,    17,
      18,    21,    24,    26,    29,    31,    34,    36,
      39,    44,    49,    54,    59,    64,    69,    73,
      78,    88,    97,   106,   115,   124,   133,   142,
     151,   167,   184,   200,   215,   231,   245,   260,
     274,   301,   327,   351,   375,   397,   418,   439,
     458,   495,   528,   559,   587,   613,   637,   659,
     680,   717,   749,   777,   801,   823,   842,   859,
     874,   899,   919,   936,   949,   960,   969,   977,
     983,   994,  1001,  1006,  1010,  1013,  1015,  1017,
    1018,  1020,  1022,  1022,  1023,  1023,  1023,  1024,
  };
  static const int xsq_iq_q10[] = {
         0,      4,      8,     12,     16,     20,     24,     28,
        32,     40,     48,     56,     64,     72,     80,     88,
        96,    112,    128,    144,    160,    176,    192,    208,
       224,    256,    288,    320,    352,    384,    416,    448,
       480,    544,    608,    672,    736,    800,    864,    928,
       992,   1120,   1248,   1376,   1504,   1632,   1760,   1888,
      2016,   2272,   2528,   2784,   3040,   3296,   3552,   3808,
      4064,   4576,   5088,   5600,   6112,   6624,   7136,   7648,
      8160,   9184,  10208,  11232,  12256,  13280,  14304,  15328,
     16352,  18400,  20448,  22496,  24544,  26592,  28640,  30688,
     32736,  36832,  40928,  45024,  49120,  53216,  57312,  61408,
     65504,  73696,  81888,  90080,  98272, 106464, 114656, 122848,
    131040, 147424, 163808, 180192, 196576, 212960, 229344, 245728,
Deb Mukherjee's avatar
Deb Mukherjee committed
387
  };
388
  /*
389
390
391
392
  static const int tab_size = sizeof(rate_tab_q10) / sizeof(rate_tab_q10[0]);
  assert(sizeof(dist_tab_q10) / sizeof(dist_tab_q10[0]) == tab_size);
  assert(sizeof(xsq_iq_q10) / sizeof(xsq_iq_q10[0]) == tab_size);
  assert(MAX_XSQ_Q10 + 1 == xsq_iq_q10[tab_size - 1]);
393
  */
394
395
396
397
398
399
400
401
  int tmp = (xsq_q10 >> 2) + 8;
  int k = get_msb(tmp) - 3;
  int xq = (k << 3) + ((tmp >> k) & 0x7);
  const int one_q10 = 1 << 10;
  const int a_q10 = ((xsq_q10 - xsq_iq_q10[xq]) << 10) >> (2 + k);
  const int b_q10 = one_q10 - a_q10;
  *r_q10 = (rate_tab_q10[xq] * b_q10 + rate_tab_q10[xq + 1] * a_q10) >> 10;
  *d_q10 = (dist_tab_q10[xq] * b_q10 + dist_tab_q10[xq + 1] * a_q10) >> 10;
Deb Mukherjee's avatar
Deb Mukherjee committed
402
403
}

404
405
406
static void model_rd_from_var_lapndz(unsigned int var, unsigned int n,
                                     unsigned int qstep, int *rate,
                                     int64_t *dist) {
Deb Mukherjee's avatar
Deb Mukherjee committed
407
408
409
410
411
412
  // This function models the rate and distortion for a Laplacian
  // source with given variance when quantized with a uniform quantizer
  // with given stepsize. The closed form expressions are in:
  // Hang and Chen, "Source Model for transform video coder and its
  // application - Part I: Fundamental Theory", IEEE Trans. Circ.
  // Sys. for Video Tech., April 1997.
413
  if (var == 0) {
Deb Mukherjee's avatar
Deb Mukherjee committed
414
415
416
    *rate = 0;
    *dist = 0;
  } else {
417
    int d_q10, r_q10;
418
    const uint64_t xsq_q10_64 =
419
        ((((uint64_t)qstep * qstep * n) << 10) + (var >> 1)) / var;
420
421
    const int xsq_q10 = xsq_q10_64 > MAX_XSQ_Q10 ?
                        MAX_XSQ_Q10 : (int)xsq_q10_64;
422
423
424
    model_rd_norm(xsq_q10, &r_q10, &d_q10);
    *rate = (n * r_q10 + 2) >> 2;
    *dist = (var * (int64_t)d_q10 + 512) >> 10;
Deb Mukherjee's avatar
Deb Mukherjee committed
425
426
427
  }
}

428
static void model_rd_for_sb(VP9_COMP *cpi, BLOCK_SIZE bsize,
Deb Mukherjee's avatar
Deb Mukherjee committed
429
430
431
432
433
                            MACROBLOCK *x, MACROBLOCKD *xd,
                            int *out_rate_sum, int64_t *out_dist_sum) {
  // Note our transform coeffs are 8 times an orthogonal transform.
  // Hence quantizer step is also 8 times. To get effective quantizer
  // we need to divide by 8 before sending to modeling function.
434
435
436
  int i;
  int64_t rate_sum = 0;
  int64_t dist_sum = 0;
437
438
  int ref = xd->mi_8x8[0]->mbmi.ref_frame[0];
  unsigned int sse;
Deb Mukherjee's avatar
Deb Mukherjee committed
439
440
441
442

  for (i = 0; i < MAX_MB_PLANE; ++i) {
    struct macroblock_plane *const p = &x->plane[i];
    struct macroblockd_plane *const pd = &xd->plane[i];
443
    const BLOCK_SIZE bs = get_plane_block_size(bsize, pd);
444

Deb Mukherjee's avatar
Deb Mukherjee committed
445
    (void) cpi->fn_ptr[bs].vf(p->src.buf, p->src.stride,
446
                              pd->dst.buf, pd->dst.stride, &sse);
447

448
449
    if (i == 0)
      x->pred_sse[ref] = sse;
450
451

    // Fast approximate the modelling function.
452
    if (cpi->speed > 4) {
453
      int64_t rate;
454
455
456
457
      int64_t dist;
      int64_t square_error = sse;
      int quantizer = (pd->dequant[1] >> 3);

458
459
      if (quantizer < 120)
        rate = (square_error * (280 - quantizer)) >> 8;
460
461
462
463
464
      else
        rate = 0;
      dist = (square_error * quantizer) >> 8;
      rate_sum += rate;
      dist_sum += dist;
465
466
467
468
469
470
    } else {
      int rate;
      int64_t dist;
      model_rd_from_var_lapndz(sse, 1 << num_pels_log2_lookup[bs],
                               pd->dequant[1] >> 3, &rate, &dist);
      rate_sum += rate;
471
      dist_sum += dist;
472
    }
Deb Mukherjee's avatar
Deb Mukherjee committed
473
474
  }

475
476
  *out_rate_sum = (int)rate_sum;
  *out_dist_sum = dist_sum << 4;
Deb Mukherjee's avatar
Deb Mukherjee committed
477
478
}

479
static void model_rd_for_sb_y_tx(VP9_COMP *cpi, BLOCK_SIZE bsize,
Deb Mukherjee's avatar
Deb Mukherjee committed
480
481
482
483
                                 TX_SIZE tx_size,
                                 MACROBLOCK *x, MACROBLOCKD *xd,
                                 int *out_rate_sum, int64_t *out_dist_sum,
                                 int *out_skip) {
484
  int j, k;
485
  BLOCK_SIZE bs;
Deb Mukherjee's avatar
Deb Mukherjee committed
486
487
  struct macroblock_plane *const p = &x->plane[0];
  struct macroblockd_plane *const pd = &xd->plane[0];
488
489
  const int width = 4 * num_4x4_blocks_wide_lookup[bsize];
  const int height = 4 * num_4x4_blocks_high_lookup[bsize];
Deb Mukherjee's avatar
Deb Mukherjee committed
490
491
  int rate_sum = 0;
  int64_t dist_sum = 0;
492
  const int t = 4 << tx_size;
Deb Mukherjee's avatar
Deb Mukherjee committed
493
494
495
496
497
498
499
500
501
502
503
504

  if (tx_size == TX_4X4) {
    bs = BLOCK_4X4;
  } else if (tx_size == TX_8X8) {
    bs = BLOCK_8X8;
  } else if (tx_size == TX_16X16) {
    bs = BLOCK_16X16;
  } else if (tx_size == TX_32X32) {
    bs = BLOCK_32X32;
  } else {
    assert(0);
  }
505

Deb Mukherjee's avatar
Deb Mukherjee committed
506
  *out_skip = 1;
Jim Bankoski's avatar
Jim Bankoski committed
507
508
  for (j = 0; j < height; j += t) {
    for (k = 0; k < width; k += t) {
Deb Mukherjee's avatar
Deb Mukherjee committed
509
510
511
      int rate;
      int64_t dist;
      unsigned int sse;
512
513
514
      cpi->fn_ptr[bs].vf(&p->src.buf[j * p->src.stride + k], p->src.stride,
                         &pd->dst.buf[j * pd->dst.stride + k], pd->dst.stride,
                         &sse);
Deb Mukherjee's avatar
Deb Mukherjee committed
515
      // sse works better than var, since there is no dc prediction used
516
      model_rd_from_var_lapndz(sse, t * t, pd->dequant[1] >> 3, &rate, &dist);
Deb Mukherjee's avatar
Deb Mukherjee committed
517
518
519
520
521
      rate_sum += rate;
      dist_sum += dist;
      *out_skip &= (rate < 1024);
    }
  }
522

Deb Mukherjee's avatar
Deb Mukherjee committed
523
  *out_rate_sum = rate_sum;
524
  *out_dist_sum = dist_sum << 4;
Deb Mukherjee's avatar
Deb Mukherjee committed
525
526
}

Ronald S. Bultje's avatar
Ronald S. Bultje committed
527
int64_t vp9_block_error_c(int16_t *coeff, int16_t *dqcoeff,
528
                          intptr_t block_size, int64_t *ssz) {
Ronald S. Bultje's avatar
Ronald S. Bultje committed
529
  int i;
530
  int64_t error = 0, sqcoeff = 0;
John Koleszar's avatar
John Koleszar committed
531

532
  for (i = 0; i < block_size; i++) {
John Koleszar's avatar
John Koleszar committed
533
    int this_diff = coeff[i] - dqcoeff[i];
Ronald S. Bultje's avatar
Ronald S. Bultje committed
534
    error += (unsigned)this_diff * this_diff;
535
    sqcoeff += (unsigned) coeff[i] * coeff[i];
John Koleszar's avatar
John Koleszar committed
536
  }
John Koleszar's avatar
John Koleszar committed
537

538
  *ssz = sqcoeff;
John Koleszar's avatar
John Koleszar committed
539
  return error;
John Koleszar's avatar
John Koleszar committed
540
541
}

542
543
544
545
546
/* The trailing '0' is a terminator which is used inside cost_coeffs() to
 * decide whether to include cost of a trailing EOB node or not (i.e. we
 * can skip this if the last coefficient in this transform block, e.g. the
 * 16th coefficient in a 4x4 block or the 64th coefficient in a 8x8 block,
 * were non-zero). */
547
static const int16_t band_counts[TX_SIZES][8] = {
548
549
550
551
  { 1, 2, 3, 4,  3,   16 - 13, 0 },
  { 1, 2, 3, 4, 11,   64 - 21, 0 },
  { 1, 2, 3, 4, 11,  256 - 21, 0 },
  { 1, 2, 3, 4, 11, 1024 - 21, 0 },
552
553
};

554
static INLINE int cost_coeffs(MACROBLOCK *x,
555
                              int plane, int block,
556
                              ENTROPY_CONTEXT *A, ENTROPY_CONTEXT *L,
John Koleszar's avatar
John Koleszar committed
557
                              TX_SIZE tx_size,
558
                              const int16_t *scan, const int16_t *nb) {
559
  MACROBLOCKD *const xd = &x->e_mbd;
560
  MB_MODE_INFO *mbmi = &xd->mi_8x8[0]->mbmi;
561
  struct macroblock_plane *p = &x->plane[plane];
562
563
  struct macroblockd_plane *pd = &xd->plane[plane];
  const PLANE_TYPE type = pd->plane_type;
564
  const int16_t *band_count = &band_counts[tx_size][1];
565
  const int eob = p->eobs[block];
566
  const int16_t *const qcoeff_ptr = BLOCK_OFFSET(p->qcoeff, block);
Ronald S. Bultje's avatar
Ronald S. Bultje committed
567
  const int ref = mbmi->ref_frame[0] != INTRA_FRAME;
568
  unsigned int (*token_costs)[2][COEFF_CONTEXTS][ENTROPY_TOKENS] =
569
                   x->token_costs[tx_size][type][ref];
570
  const ENTROPY_CONTEXT above_ec = !!*A, left_ec = !!*L;
571
  uint8_t *p_tok = x->token_cache;
572
573
  int pt = combine_entropy_contexts(above_ec, left_ec);
  int c, cost;
574
575

  // Check for consistency of tx_size with mode info
576
  assert(type == PLANE_TYPE_Y ? mbmi->tx_size == tx_size
577
                                      : get_uv_tx_size(mbmi) == tx_size);
578

579
580
  if (eob == 0) {
    // single eob token
581
    cost = token_costs[0][0][pt][EOB_TOKEN];
582
    c = 0;
583
  } else {
584
    int band_left = *band_count++;
585
586

    // dc token
587
588
    int v = qcoeff_ptr[0];
    int prev_t = vp9_dct_value_tokens_ptr[v].token;
589
    cost = (*token_costs)[0][pt][prev_t] + vp9_dct_value_cost_ptr[v];
590
    p_tok[0] = vp9_pt_energy_class[prev_t];
591
    ++token_costs;
592
593
594
595

    // ac tokens
    for (c = 1; c < eob; c++) {
      const int rc = scan[c];
596
      int t;
597
598
599

      v = qcoeff_ptr[rc];
      t = vp9_dct_value_tokens_ptr[v].token;
600
      pt = get_coef_context(nb, p_tok, c);
601
      cost += (*token_costs)[!prev_t][pt][t] + vp9_dct_value_cost_ptr[v];
602
      p_tok[rc] = vp9_pt_energy_class[t];
603
      prev_t = t;
604
      if (!--band_left) {
605
606
        band_left = *band_count++;
        ++token_costs;
607
      }
608
    }
609
610

    // eob token
611
    if (band_left) {
612
      pt = get_coef_context(nb, p_tok, c);
613
      cost += (*token_costs)[0][pt][EOB_TOKEN];
614
    }
615
616
  }

617
  // is eob first coefficient;
618
  *A = *L = (c > 0);
619

620
621
622
  return cost;
}

Alex Converse's avatar
Alex Converse committed
623
624
static void dist_block(int plane, int block, TX_SIZE tx_size,
                       struct rdcost_block_args* args) {
625
  const int ss_txfrm_size = tx_size << 1;
Deb Mukherjee's avatar
Deb Mukherjee committed
626
627
  MACROBLOCK* const x = args->x;
  MACROBLOCKD* const xd = &x->e_mbd;
628
629
  struct macroblock_plane *const p = &x->plane[plane];
  struct macroblockd_plane *const pd = &xd->plane[plane];
Deb Mukherjee's avatar
Deb Mukherjee committed
630
  int64_t this_sse;
Alex Converse's avatar
Alex Converse committed
631
  int shift = tx_size == TX_32X32 ? 0 : 2;
632
633
  int16_t *const coeff = BLOCK_OFFSET(p->coeff, block);
  int16_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
634
635
636
  args->dist = vp9_block_error(coeff, dqcoeff, 16 << ss_txfrm_size,
                               &this_sse) >> shift;
  args->sse  = this_sse >> shift;
637

638
  if (x->skip_encode && !is_inter_block(&xd->mi_8x8[0]->mbmi)) {
639
640
    // TODO(jingning): tune the model to better capture the distortion.
    int64_t p = (pd->dequant[1] * pd->dequant[1] *
641
                    (1 << ss_txfrm_size)) >> (shift + 2);
642
643
    args->dist += (p >> 4);
    args->sse  += p;
644
  }
Deb Mukherjee's avatar
Deb Mukherjee committed
645
646
}

647
static void rate_block(int plane, int block, BLOCK_SIZE plane_bsize,
Alex Converse's avatar
Alex Converse committed
648
                       TX_SIZE tx_size, struct rdcost_block_args* args) {
Deb Mukherjee's avatar
Deb Mukherjee committed
649
  int x_idx, y_idx;
Alex Converse's avatar
Alex Converse committed
650
  txfrm_block_to_raster_xy(plane_bsize, tx_size, block, &x_idx, &y_idx);
Deb Mukherjee's avatar
Deb Mukherjee committed
651

652
  args->rate = cost_coeffs(args->x, plane, block, args->t_above + x_idx,
Alex Converse's avatar
Alex Converse committed
653
                           args->t_left + y_idx, tx_size,
654
                           args->so->scan, args->so->neighbors);
Deb Mukherjee's avatar
Deb Mukherjee committed
655
656
}

657
658
static void block_rd_txfm(int plane, int block, BLOCK_SIZE plane_bsize,
                          TX_SIZE tx_size, void *arg) {
Deb Mukherjee's avatar
Deb Mukherjee committed
659
660
661
  struct rdcost_block_args *args = arg;
  MACROBLOCK *const x = args->x;
  MACROBLOCKD *const xd = &x->e_mbd;
Jim Bankoski's avatar
Jim Bankoski committed
662
  MB_MODE_INFO *mbmi = &xd->mi_8x8[0]->mbmi;
663
  int64_t rd1, rd2, rd;
Deb Mukherjee's avatar
Deb Mukherjee committed
664

665
666
667
  if (args->skip)
    return;

668
  if (!is_inter_block(mbmi))
669
    vp9_encode_block_intra(x, plane, block, plane_bsize, tx_size, &mbmi->skip);
Deb Mukherjee's avatar
Deb Mukherjee committed
670
  else
671
    vp9_xform_quant(x, plane, block, plane_bsize, tx_size);
Deb Mukherjee's avatar
Deb Mukherjee committed
672

673
674
  dist_block(plane, block, tx_size, args);
  rate_block(plane, block, plane_bsize, tx_size, args);
675
676
  rd1 = RDCOST(x->rdmult, x->rddiv, args->rate, args->dist);
  rd2 = RDCOST(x->rdmult, x->rddiv, 0, args->sse);
677
678

  // TODO(jingning): temporarily enabled only for luma component
679
  rd = MIN(rd1, rd2);
Yaowu Xu's avatar
Yaowu Xu committed
680
  if (plane == 0)
681
    x->zcoeff_blk[tx_size][block] = !x->plane[plane].eobs[block] ||
Yaowu Xu's avatar
Yaowu Xu committed
682
                                    (rd1 > rd2 && !xd->lossless);
683

684
685
686
  args->this_rate += args->rate;
  args->this_dist += args->dist;
  args->this_sse  += args->sse;
687
688
689
690
691
692
  args->this_rd += rd;

  if (args->this_rd > args->best_rd) {
    args->skip = 1;
    return;
  }
Deb Mukherjee's avatar
Deb Mukherjee committed
693
694
}

695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
void vp9_get_entropy_contexts(TX_SIZE tx_size,
    ENTROPY_CONTEXT t_above[16], ENTROPY_CONTEXT t_left[16],
    const ENTROPY_CONTEXT *above, const ENTROPY_CONTEXT *left,
    int num_4x4_w, int num_4x4_h) {
  int i;
  switch (tx_size) {
    case TX_4X4:
      vpx_memcpy(t_above, above, sizeof(ENTROPY_CONTEXT) * num_4x4_w);
      vpx_memcpy(t_left, left, sizeof(ENTROPY_CONTEXT) * num_4x4_h);
      break;
    case TX_8X8:
      for (i = 0; i < num_4x4_w; i += 2)
        t_above[i] = !!*(const uint16_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 2)
        t_left[i] = !!*(const uint16_t *)&left[i];
      break;
    case TX_16X16:
      for (i = 0; i < num_4x4_w; i += 4)
        t_above[i] = !!*(const uint32_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 4)
        t_left[i] = !!*(const uint32_t *)&left[i];
      break;
    case TX_32X32:
      for (i = 0; i < num_4x4_w; i += 8)
        t_above[i] = !!*(const uint64_t *)&above[i];
      for (i = 0; i < num_4x4_h; i += 8)
        t_left[i] = !!*(const uint64_t *)&left[i];
      break;
    default:
James Zern's avatar
James Zern committed
724
      assert(0 && "Invalid transform size.");
725
726
727
  }
}

728
static void txfm_rd_in_plane(MACROBLOCK *x,
729
730
731
                             int *rate, int64_t *distortion,
                             int *skippable, int64_t *sse,
                             int64_t ref_best_rd, int plane,
732
                             BLOCK_SIZE bsize, TX_SIZE tx_size) {
Deb Mukherjee's avatar
Deb Mukherjee committed
733
  MACROBLOCKD *const xd = &x->e_mbd;
734
  struct macroblockd_plane *const pd = &xd->plane[plane];
735
  const BLOCK_SIZE bs = get_plane_block_size(bsize, pd);
736
737
  const int num_4x4_w = num_4x4_blocks_wide_lookup[bs];
  const int num_4x4_h = num_4x4_blocks_high_lookup[bs];
738
739
740
  struct rdcost_block_args args = { 0 };
  args.x = x;
  args.best_rd = ref_best_rd;
741

742
  if (plane == 0)
743
    xd->mi_8x8[0]->mbmi.tx_size = tx_size;
744

745
  vp9_get_entropy_contexts(tx_size, args.t_above, args.t_left,
746
747
                           pd->above_context, pd->left_context,
                           num_4x4_w, num_4x4_h);
748

749
  args.so = get_scan(xd, tx_size, pd->plane_type, 0);
Deb Mukherjee's avatar
Deb Mukherjee committed
750

751
  vp9_foreach_transformed_block_in_plane(xd, bsize, plane,
752
753
                                         block_rd_txfm, &args);
  if (args.skip) {
754
755
756
757
758
    *rate       = INT_MAX;
    *distortion = INT64_MAX;
    *sse        = INT64_MAX;
    *skippable  = 0;
  } else {
759
760
761
    *distortion = args.this_dist;
    *rate       = args.this_rate;
    *sse        = args.this_sse;
762
    *skippable  = vp9_is_skippable_in_plane(x, bsize, plane);
763
  }
Deb Mukherjee's avatar
Deb Mukherjee committed
764
765
766
767
768
}

static void choose_largest_txfm_size(VP9_COMP *cpi, MACROBLOCK *x,
                                     int *rate, int64_t *distortion,
                                     int *skip, int64_t *sse,
769
                                     int64_t ref_best_rd,
770
                                     BLOCK_SIZE bs) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
771
  const TX_SIZE max_tx_size = max_txsize_lookup[bs];
Deb Mukherjee's avatar
Deb Mukherjee committed
772
  VP9_COMMON *const cm = &cpi->common;
Yaowu Xu's avatar
Yaowu Xu committed
773
  const TX_SIZE largest_tx_size = tx_mode_to_biggest_tx_size[cm->tx_mode];
Deb Mukherjee's avatar
Deb Mukherjee committed
774
  MACROBLOCKD *const xd = &x->e_mbd;
775
  MB_MODE_INFO *const mbmi = &xd->mi_8x8[0]->mbmi;
Yaowu Xu's avatar
Yaowu Xu committed
776
777
778

  mbmi->tx_size = MIN(max_tx_size, largest_tx_size);

Alex Converse's avatar
Alex Converse committed
779
  txfm_rd_in_plane(x, rate, distortion, skip,
780
781
                   &sse[mbmi->tx_size], ref_best_rd, 0, bs,
                   mbmi->tx_size);
Dmitry Kovalev's avatar
Dmitry Kovalev committed
782
  cpi->tx_stepdown_count[0]++;
Deb Mukherjee's avatar
Deb Mukherjee committed
783
784
}

785
static void choose_txfm_size_from_rd(VP9_COMP *cpi, MACROBLOCK *x,
786
                                     int (*r)[2], int *rate,
Ronald S. Bultje's avatar
Ronald S. Bultje committed
787
                                     int64_t *d, int64_t *distortion,
788
                                     int *s, int *skip,
789
                                     int64_t tx_cache[TX_MODES],
790
                                     BLOCK_SIZE bs) {
791
  const TX_SIZE max_tx_size = max_txsize_lookup[bs];
792
793
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
794
  MB_MODE_INFO *const mbmi = &xd->mi_8x8[0]->mbmi;
795
  vp9_prob skip_prob = vp9_get_skip_prob(cm, xd);
796
797
798
799
  int64_t rd[TX_SIZES][2] = {{INT64_MAX, INT64_MAX},
                             {INT64_MAX, INT64_MAX},
                             {INT64_MAX, INT64_MAX},
                             {INT64_MAX, INT64_MAX}};
800
  int n, m;
801
  int s0, s1;
802
803
804
  const TX_SIZE max_mode_tx_size = tx_mode_to_biggest_tx_size[cm->tx_mode];
  int64_t best_rd = INT64_MAX;
  TX_SIZE best_tx = TX_4X4;
805

806
  const vp9_prob *tx_probs = get_tx_probs2(max_tx_size, xd, &cm->fc.tx_probs);
807
808
809
  assert(skip_prob > 0);
  s0 = vp9_cost_bit(skip_prob, 0);
  s1 = vp9_cost_bit(skip_prob, 1);
810

811
  for (n = TX_4X4; n <= max_tx_size; n++) {
812
813
814
815
816
817
818
819
820
    r[n][1] = r[n][0];
    if (r[n][0] < INT_MAX) {
      for (m = 0; m <= n - (n == max_tx_size); m++) {
        if (m == n)
          r[n][1] += vp9_cost_zero(tx_probs[m]);
        else
          r[n][1] += vp9_cost_one(tx_probs[m]);
      }
    }
821
822
    if (d[n] == INT64_MAX) {
      rd[n][0] = rd[n][1] = INT64_MAX;
823
    } else if (s[n]) {
824
825
826
827
      rd[n][0] = rd[n][1] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
    } else {
      rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0] + s0, d[n]);
      rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1] + s0, d[n]);
828
    }
829
830
831
832
833

    if (rd[n][1] < best_rd) {
      best_tx = n;
      best_rd = rd[n][1];
    }
834
  }
835
836
  mbmi->tx_size = cm->tx_mode == TX_MODE_SELECT ?
                      best_tx : MIN(max_tx_size, max_mode_tx_size);
837
838


839
840
841
  *distortion = d[mbmi->tx_size];
  *rate       = r[mbmi->tx_size][cm->tx_mode == TX_MODE_SELECT];
  *skip       = s[mbmi->tx_size];
842

843
844
845
846
  tx_cache[ONLY_4X4] = rd[TX_4X4][0];
  tx_cache[ALLOW_8X8] = rd[TX_8X8][0];
  tx_cache[ALLOW_16X16] = rd[MIN(max_tx_size, TX_16X16)][0];
  tx_cache[ALLOW_32X32] = rd[MIN(max_tx_size, TX_32X32)][0];
847

848
849
  if (max_tx_size == TX_32X32 && best_tx == TX_32X32) {
    tx_cache[TX_MODE_SELECT] = rd[TX_32X32][1];
Dmitry Kovalev's avatar
Dmitry Kovalev committed
850
    cpi->tx_stepdown_count[0]++;
851
852
  } else if (max_tx_size >= TX_16X16 && best_tx == TX_16X16) {
    tx_cache[TX_MODE_SELECT] = rd[TX_16X16][1];
Dmitry Kovalev's avatar
Dmitry Kovalev committed
853
    cpi->tx_stepdown_count[max_tx_size - TX_16X16]++;
Deb Mukherjee's avatar
Deb Mukherjee committed
854
  } else if (rd[TX_8X8][1] < rd[TX_4X4][1]) {
855
    tx_cache[TX_MODE_SELECT] = rd[TX_8X8][1];
Dmitry Kovalev's avatar
Dmitry Kovalev committed
856
    cpi->tx_stepdown_count[max_tx_size - TX_8X8]++;
Deb Mukherjee's avatar
Deb Mukherjee committed
857
  } else {
858
    tx_cache[TX_MODE_SELECT] = rd[TX_4X4][1];
Dmitry Kovalev's avatar
Dmitry Kovalev committed
859
    cpi->tx_stepdown_count[max_tx_size - TX_4X4]++;
860
  }
Deb Mukherjee's avatar
Deb Mukherjee committed
861
}
862

863
864
865
866
867
static int64_t scaled_rd_cost(int rdmult, int rddiv,
                              int rate, int64_t dist, double scale) {
  return (int64_t) (RDCOST(rdmult, rddiv, rate, dist) * scale);
}

Deb Mukherjee's avatar
Deb Mukherjee committed
868
869
870
871
static void choose_txfm_size_from_modelrd(VP9_COMP *cpi, MACROBLOCK *x,
                                          int (*r)[2], int *rate,
                                          int64_t *d, int64_t *distortion,
                                          int *s, int *skip, int64_t *sse,
872
                                          int64_t ref_best_rd,
873
                                          BLOCK_SIZE bs) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
874
  const TX_SIZE max_tx_size = max_txsize_lookup[bs];
Deb Mukherjee's avatar
Deb Mukherjee committed
875
876
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
877
  MB_MODE_INFO *const mbmi = &xd->mi_8x8[0]->mbmi;
878
  vp9_prob skip_prob = vp9_get_skip_prob(cm, xd);
879
880
881
882
  int64_t rd[TX_SIZES][2] = {{INT64_MAX, INT64_MAX},
                             {INT64_MAX, INT64_MAX},
                             {INT64_MAX, INT64_MAX},
                             {INT64_MAX, INT64_MAX}};
Deb Mukherjee's avatar
Deb Mukherjee committed
883
884
  int n, m;
  int s0, s1;
885
  double scale_rd[TX_SIZES] = {1.73, 1.44, 1.20, 1.00};
886
887
888
  const TX_SIZE max_mode_tx_size = tx_mode_to_biggest_tx_size[cm->tx_mode];
  int64_t best_rd = INT64_MAX;
  TX_SIZE best_tx = TX_4X4;
889

890
  const vp9_prob *tx_probs = get_tx_probs2(max_tx_size, xd, &cm->fc.tx_probs);
891
892
893
  assert(skip_prob > 0);
  s0 = vp9_cost_bit(skip_prob, 0);
  s1 = vp9_cost_bit(skip_prob, 1);
894

Dmitry Kovalev's avatar
Dmitry Kovalev committed
895
  for (n = TX_4X4; n <= max_tx_size; n++) {
896
    double scale = scale_rd[n];
Deb Mukherjee's avatar
Deb Mukherjee committed
897
    r[n][1] = r[n][0];
Dmitry Kovalev's avatar
Dmitry Kovalev committed
898
    for (m = 0; m <= n - (n == max_tx_size); m++) {
Deb Mukherjee's avatar
Deb Mukherjee committed
899
900
901
902
903
904
      if (m == n)
        r[n][1] += vp9_cost_zero(tx_probs[m]);
      else
        r[n][1] += vp9_cost_one(tx_probs[m]);
    }
    if (s[n]) {
905
906
      rd[n][0] = rd[n][1] = scaled_rd_cost(x->rdmult, x->rddiv, s1, d[n],
                                           scale);
Deb Mukherjee's avatar
Deb Mukherjee committed
907
    } else {