vp9_rdopt.c 122 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5
6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9
10
11
12
13
14
15
 */


#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
16

17
#include "vp9/common/vp9_pragmas.h"
18
19
20
21
22
#include "vp9/encoder/vp9_tokenize.h"
#include "vp9/encoder/vp9_treewriter.h"
#include "vp9/encoder/vp9_onyx_int.h"
#include "vp9/encoder/vp9_modecosts.h"
#include "vp9/encoder/vp9_encodeintra.h"
23
24
25
26
27
#include "vp9/common/vp9_entropymode.h"
#include "vp9/common/vp9_reconinter.h"
#include "vp9/common/vp9_reconintra.h"
#include "vp9/common/vp9_findnearmv.h"
#include "vp9/common/vp9_quant_common.h"
28
29
30
31
32
33
#include "vp9/encoder/vp9_encodemb.h"
#include "vp9/encoder/vp9_quantize.h"
#include "vp9/encoder/vp9_variance.h"
#include "vp9/encoder/vp9_mcomp.h"
#include "vp9/encoder/vp9_rdopt.h"
#include "vp9/encoder/vp9_ratectrl.h"
John Koleszar's avatar
John Koleszar committed
34
#include "vpx_mem/vpx_mem.h"
35
36
37
38
39
#include "vp9/common/vp9_systemdependent.h"
#include "vp9/encoder/vp9_encodemv.h"
#include "vp9/common/vp9_seg_common.h"
#include "vp9/common/vp9_pred_common.h"
#include "vp9/common/vp9_entropy.h"
40
#include "vp9_rtcd.h"
41
#include "vp9/common/vp9_mvref_common.h"
Ronald S. Bultje's avatar
Ronald S. Bultje committed
42
#include "vp9/common/vp9_common.h"
Paul Wilkins's avatar
Paul Wilkins committed
43

44
45
#define INVALID_MV 0x80008000

46
47
48
/* Factor to weigh the rate for switchable interp filters */
#define SWITCHABLE_INTERP_RATE_FACTOR 1

49
50
51
DECLARE_ALIGNED(16, extern const uint8_t,
                vp9_pt_energy_class[MAX_ENTROPY_TOKENS]);

Ronald S. Bultje's avatar
Ronald S. Bultje committed
52
#define I4X4_PRED 0x8000
Ronald S. Bultje's avatar
Ronald S. Bultje committed
53
#define SPLITMV 0x10000
Ronald S. Bultje's avatar
Ronald S. Bultje committed
54

55
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
56
57
  {ZEROMV,    LAST_FRAME,   NONE},
  {DC_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
58

59
60
  {NEARESTMV, LAST_FRAME,   NONE},
  {NEARMV,    LAST_FRAME,   NONE},
John Koleszar's avatar
John Koleszar committed
61

62
63
  {ZEROMV,    GOLDEN_FRAME, NONE},
  {NEARESTMV, GOLDEN_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
64

65
66
  {ZEROMV,    ALTREF_FRAME, NONE},
  {NEARESTMV, ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
67

68
69
  {NEARMV,    GOLDEN_FRAME, NONE},
  {NEARMV,    ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
70

71
72
73
74
75
76
77
78
  {V_PRED,    INTRA_FRAME,  NONE},
  {H_PRED,    INTRA_FRAME,  NONE},
  {D45_PRED,  INTRA_FRAME,  NONE},
  {D135_PRED, INTRA_FRAME,  NONE},
  {D117_PRED, INTRA_FRAME,  NONE},
  {D153_PRED, INTRA_FRAME,  NONE},
  {D27_PRED,  INTRA_FRAME,  NONE},
  {D63_PRED,  INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
79

80
  {TM_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
81

82
83
84
  {NEWMV,     LAST_FRAME,   NONE},
  {NEWMV,     GOLDEN_FRAME, NONE},
  {NEWMV,     ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
85

86
87
88
  {SPLITMV,   LAST_FRAME,   NONE},
  {SPLITMV,   GOLDEN_FRAME, NONE},
  {SPLITMV,   ALTREF_FRAME, NONE},
89

Ronald S. Bultje's avatar
Ronald S. Bultje committed
90
  {I4X4_PRED, INTRA_FRAME,  NONE},
91

John Koleszar's avatar
John Koleszar committed
92
  /* compound prediction modes */
Ronald S. Bultje's avatar
Ronald S. Bultje committed
93
94
95
  {ZEROMV,    LAST_FRAME,   ALTREF_FRAME},
  {NEARESTMV, LAST_FRAME,   ALTREF_FRAME},
  {NEARMV,    LAST_FRAME,   ALTREF_FRAME},
96

John Koleszar's avatar
John Koleszar committed
97
98
99
  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME},
100

Ronald S. Bultje's avatar
Ronald S. Bultje committed
101
  {NEWMV,     LAST_FRAME,   ALTREF_FRAME},
John Koleszar's avatar
John Koleszar committed
102
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME},
103

Ronald S. Bultje's avatar
Ronald S. Bultje committed
104
  {SPLITMV,   LAST_FRAME,   ALTREF_FRAME},
105
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME},
John Koleszar's avatar
John Koleszar committed
106
107
};

108
109
110
111
112
113
114
115
116
117
118
// The baseline rd thresholds for breaking out of the rd loop for
// certain modes are assumed to be based on 8x8 blocks.
// This table is used to correct for blocks size.
// The factors here are << 2 (2 = x0.5, 32 = x8 etc).
static int rd_thresh_block_size_factor[BLOCK_SIZE_TYPES] =
  {2, 3, 3, 4, 6, 6, 8, 12, 12, 16, 24, 24, 32};

#define BASE_RD_THRESH_FREQ_FACT 16
#define MAX_RD_THRESH_FREQ_FACT 32
#define MAX_RD_THRESH_FREQ_INC 1

119
120
121
static void fill_token_costs(vp9_coeff_count (*c)[BLOCK_TYPES],
                             vp9_coeff_count (*cnoskip)[BLOCK_TYPES],
                             vp9_coeff_probs_model (*p)[BLOCK_TYPES]) {
122
  int i, j, k, l;
123
124
125
126
127
128
129
130
131
132
  TX_SIZE t;
  for (t = TX_4X4; t <= TX_32X32; t++)
    for (i = 0; i < BLOCK_TYPES; i++)
      for (j = 0; j < REF_TYPES; j++)
        for (k = 0; k < COEF_BANDS; k++)
          for (l = 0; l < PREV_COEF_CONTEXTS; l++) {
            vp9_prob probs[ENTROPY_NODES];
            vp9_model_to_full_probs(p[t][i][j][k][l], probs);
            vp9_cost_tokens((int *)cnoskip[t][i][j][k][l], probs,
                            vp9_coef_tree);
133
#if CONFIG_BALANCED_COEFTREE
134
135
136
137
            // Replace the eob node prob with a very small value so that the
            // cost approximately equals the cost without the eob node
            probs[1] = 1;
            vp9_cost_tokens((int *)c[t][i][j][k][l], probs, vp9_coef_tree);
138
#else
139
140
            vp9_cost_tokens_skip((int *)c[t][i][j][k][l], probs,
                                 vp9_coef_tree);
141
142
143
            assert(c[t][i][j][k][l][DCT_EOB_TOKEN] ==
                   cnoskip[t][i][j][k][l][DCT_EOB_TOKEN]);
#endif
144
          }
145
146
}

147
148
149
150
static int rd_iifactor[32] =  { 4, 4, 3, 2, 1, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0, };
John Koleszar's avatar
John Koleszar committed
151

152
// 3* dc_qlookup[Q]*dc_qlookup[Q];
153

154
/* values are now correlated to quantizer */
Paul Wilkins's avatar
Paul Wilkins committed
155
156
157
static int sad_per_bit16lut[QINDEX_RANGE];
static int sad_per_bit4lut[QINDEX_RANGE];

158
void vp9_init_me_luts() {
John Koleszar's avatar
John Koleszar committed
159
160
161
162
163
164
165
  int i;

  // Initialize the sad lut tables using a formulaic calculation for now
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < QINDEX_RANGE; i++) {
    sad_per_bit16lut[i] =
166
      (int)((0.0418 * vp9_convert_qindex_to_q(i)) + 2.4107);
167
    sad_per_bit4lut[i] = (int)(0.063 * vp9_convert_qindex_to_q(i) + 2.742);
John Koleszar's avatar
John Koleszar committed
168
  }
Paul Wilkins's avatar
Paul Wilkins committed
169
}
John Koleszar's avatar
John Koleszar committed
170

171
static int compute_rd_mult(int qindex) {
172
  const int q = vp9_dc_quant(qindex, 0);
173
  return (11 * q * q) >> 2;
174
175
}

Dmitry Kovalev's avatar
Dmitry Kovalev committed
176
177
178
void vp9_initialize_me_consts(VP9_COMP *cpi, int qindex) {
  cpi->mb.sadperbit16 = sad_per_bit16lut[qindex];
  cpi->mb.sadperbit4 = sad_per_bit4lut[qindex];
John Koleszar's avatar
John Koleszar committed
179
180
}

181

Dmitry Kovalev's avatar
Dmitry Kovalev committed
182
void vp9_initialize_rd_consts(VP9_COMP *cpi, int qindex) {
183
  int q, i, bsize;
John Koleszar's avatar
John Koleszar committed
184

185
  vp9_clear_system_state();  // __asm emms;
John Koleszar's avatar
John Koleszar committed
186

John Koleszar's avatar
John Koleszar committed
187
188
189
190
  // Further tests required to see if optimum is different
  // for key frames, golden frames and arf frames.
  // if (cpi->common.refresh_golden_frame ||
  //     cpi->common.refresh_alt_ref_frame)
191
  qindex = clamp(qindex, 0, MAXQ);
192

Dmitry Kovalev's avatar
Dmitry Kovalev committed
193
  cpi->RDMULT = compute_rd_mult(qindex);
John Koleszar's avatar
John Koleszar committed
194
195
196
197
198
  if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    if (cpi->twopass.next_iiratio > 31)
      cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
    else
      cpi->RDMULT +=
Dmitry Kovalev's avatar
Dmitry Kovalev committed
199
          (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
John Koleszar's avatar
John Koleszar committed
200
  }
201
  cpi->mb.errorperbit = cpi->RDMULT >> 6;
John Koleszar's avatar
John Koleszar committed
202
  cpi->mb.errorperbit += (cpi->mb.errorperbit == 0);
203

204
  vp9_set_speed_features(cpi);
John Koleszar's avatar
John Koleszar committed
205

Dmitry Kovalev's avatar
Dmitry Kovalev committed
206
207
  q = (int)pow(vp9_dc_quant(qindex, 0) >> 2, 1.25);
  q <<= 2;
John Koleszar's avatar
John Koleszar committed
208
209
  if (q < 8)
    q = 8;
210

John Koleszar's avatar
John Koleszar committed
211
212
213
  if (cpi->RDMULT > 1000) {
    cpi->RDDIV = 1;
    cpi->RDMULT /= 100;
John Koleszar's avatar
John Koleszar committed
214

215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
    for (bsize = 0; bsize < BLOCK_SIZE_TYPES; ++bsize) {
      for (i = 0; i < MAX_MODES; ++i) {
        // Threshold here seem unecessarily harsh but fine given actual
        // range of values used for cpi->sf.thresh_mult[]
        int thresh_max = INT_MAX / (q * rd_thresh_block_size_factor[bsize]);

        // *4 relates to the scaling of rd_thresh_block_size_factor[]
        if ((int64_t)cpi->sf.thresh_mult[i] < thresh_max) {
          cpi->rd_threshes[bsize][i] =
            cpi->sf.thresh_mult[i] * q *
            rd_thresh_block_size_factor[bsize] / (4 * 100);
        } else {
          cpi->rd_threshes[bsize][i] = INT_MAX;
        }
        cpi->rd_baseline_thresh[bsize][i] = cpi->rd_threshes[bsize][i];
        cpi->rd_thresh_freq_fact[bsize][i] = BASE_RD_THRESH_FREQ_FACT;
John Koleszar's avatar
John Koleszar committed
231
      }
John Koleszar's avatar
John Koleszar committed
232
    }
John Koleszar's avatar
John Koleszar committed
233
234
  } else {
    cpi->RDDIV = 100;
John Koleszar's avatar
John Koleszar committed
235

236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
    for (bsize = 0; bsize < BLOCK_SIZE_TYPES; ++bsize) {
      for (i = 0; i < MAX_MODES; i++) {
        // Threshold here seem unecessarily harsh but fine given actual
        // range of values used for cpi->sf.thresh_mult[]
        int thresh_max = INT_MAX / (q * rd_thresh_block_size_factor[bsize]);

        if (cpi->sf.thresh_mult[i] < thresh_max) {
          cpi->rd_threshes[bsize][i] =
            cpi->sf.thresh_mult[i] * q *
            rd_thresh_block_size_factor[bsize] / 4;
        } else {
          cpi->rd_threshes[bsize][i] = INT_MAX;
        }
        cpi->rd_baseline_thresh[bsize][i] = cpi->rd_threshes[bsize][i];
        cpi->rd_thresh_freq_fact[bsize][i] = BASE_RD_THRESH_FREQ_FACT;
John Koleszar's avatar
John Koleszar committed
251
      }
John Koleszar's avatar
John Koleszar committed
252
    }
John Koleszar's avatar
John Koleszar committed
253
  }
John Koleszar's avatar
John Koleszar committed
254

255
256
257
  fill_token_costs(cpi->mb.token_costs,
                   cpi->mb.token_costs_noskip,
                   cpi->common.fc.coef_probs);
258

259
  for (i = 0; i < NUM_PARTITION_CONTEXTS; i++)
260
    vp9_cost_tokens(cpi->mb.partition_cost[i],
261
                    cpi->common.fc.partition_prob[cpi->common.frame_type][i],
262
263
                    vp9_partition_tree);

John Koleszar's avatar
John Koleszar committed
264
  /*rough estimate for costing*/
265
  vp9_init_mode_costs(cpi);
John Koleszar's avatar
John Koleszar committed
266

267
  if (cpi->common.frame_type != KEY_FRAME) {
268
    vp9_build_nmv_cost_table(
269
270
271
272
273
274
        cpi->mb.nmvjointcost,
        cpi->mb.e_mbd.allow_high_precision_mv ?
        cpi->mb.nmvcost_hp : cpi->mb.nmvcost,
        &cpi->common.fc.nmvc,
        cpi->mb.e_mbd.allow_high_precision_mv, 1, 1);
  }
John Koleszar's avatar
John Koleszar committed
275
276
}

Ronald S. Bultje's avatar
Ronald S. Bultje committed
277
278
279
280
int64_t vp9_block_error_c(int16_t *coeff, int16_t *dqcoeff,
                          intptr_t block_size) {
  int i;
  int64_t error = 0;
John Koleszar's avatar
John Koleszar committed
281

282
  for (i = 0; i < block_size; i++) {
John Koleszar's avatar
John Koleszar committed
283
    int this_diff = coeff[i] - dqcoeff[i];
Ronald S. Bultje's avatar
Ronald S. Bultje committed
284
    error += (unsigned)this_diff * this_diff;
John Koleszar's avatar
John Koleszar committed
285
  }
John Koleszar's avatar
John Koleszar committed
286

John Koleszar's avatar
John Koleszar committed
287
  return error;
John Koleszar's avatar
John Koleszar committed
288
289
}

290
static INLINE int cost_coeffs(VP9_COMMON *const cm, MACROBLOCK *mb,
291
                              int plane, int block, PLANE_TYPE type,
292
293
                              ENTROPY_CONTEXT *A,
                              ENTROPY_CONTEXT *L,
John Koleszar's avatar
John Koleszar committed
294
295
                              TX_SIZE tx_size,
                              int y_blocks) {
296
  MACROBLOCKD *const xd = &mb->e_mbd;
297
298
  MB_MODE_INFO *mbmi = &xd->mode_info_context->mbmi;
  int pt;
299
  int c = 0;
300
301
  int cost = 0, pad;
  const int *scan, *nb;
302
303
304
  const int eob = xd->plane[plane].eobs[block];
  const int16_t *qcoeff_ptr = BLOCK_OFFSET(xd->plane[plane].qcoeff,
                                           block, 16);
Ronald S. Bultje's avatar
Ronald S. Bultje committed
305
  const int ref = mbmi->ref_frame[0] != INTRA_FRAME;
306
  unsigned int (*token_costs)[PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS] =
307
      mb->token_costs[tx_size][type][ref];
308
  ENTROPY_CONTEXT above_ec, left_ec;
309
  TX_TYPE tx_type = DCT_DCT;
310

311
  const int segment_id = xd->mode_info_context->mbmi.segment_id;
312
313
314
  unsigned int (*token_costs_noskip)[PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS] =
      mb->token_costs_noskip[tx_size][type][ref];

315
316
  int seg_eob, default_eob;
  uint8_t token_cache[1024];
Paul Wilkins's avatar
Paul Wilkins committed
317
  const uint8_t * band_translate;
318
319

  // Check for consistency of tx_size with mode info
320
  assert((!type && !plane) || (type && plane));
321
322
323
  if (type == PLANE_TYPE_Y_WITH_DC) {
    assert(xd->mode_info_context->mbmi.txfm_size == tx_size);
  } else {
324
    TX_SIZE tx_size_uv = get_uv_tx_size(mbmi);
325
326
327
    assert(tx_size == tx_size_uv);
  }

328
  switch (tx_size) {
329
    case TX_4X4: {
330
      tx_type = (type == PLANE_TYPE_Y_WITH_DC) ?
331
          get_tx_type_4x4(xd, block) : DCT_DCT;
332
333
      above_ec = A[0] != 0;
      left_ec = L[0] != 0;
334
      seg_eob = 16;
335
      scan = get_scan_4x4(tx_type);
Paul Wilkins's avatar
Paul Wilkins committed
336
      band_translate = vp9_coefband_trans_4x4;
Daniel Kang's avatar
Daniel Kang committed
337
      break;
338
    }
339
    case TX_8X8: {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
340
341
      const TX_TYPE tx_type = type == PLANE_TYPE_Y_WITH_DC ?
                                  get_tx_type_8x8(xd) : DCT_DCT;
342
343
      above_ec = (A[0] + A[1]) != 0;
      left_ec = (L[0] + L[1]) != 0;
344
      scan = get_scan_8x8(tx_type);
345
      seg_eob = 64;
Paul Wilkins's avatar
Paul Wilkins committed
346
      band_translate = vp9_coefband_trans_8x8plus;
Daniel Kang's avatar
Daniel Kang committed
347
      break;
348
349
    }
    case TX_16X16: {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
350
351
      const TX_TYPE tx_type = type == PLANE_TYPE_Y_WITH_DC ?
                                  get_tx_type_16x16(xd) : DCT_DCT;
352
      scan = get_scan_16x16(tx_type);
353
      seg_eob = 256;
354
355
      above_ec = (A[0] + A[1] + A[2] + A[3]) != 0;
      left_ec = (L[0] + L[1] + L[2] + L[3]) != 0;
Paul Wilkins's avatar
Paul Wilkins committed
356
      band_translate = vp9_coefband_trans_8x8plus;
Daniel Kang's avatar
Daniel Kang committed
357
      break;
358
    }
359
    case TX_32X32:
Paul Wilkins's avatar
Paul Wilkins committed
360
      scan = vp9_default_scan_32x32;
361
      seg_eob = 1024;
362
363
      above_ec = (A[0] + A[1] + A[2] + A[3] + A[4] + A[5] + A[6] + A[7]) != 0;
      left_ec = (L[0] + L[1] + L[2] + L[3] + L[4] + L[5] + L[6] + L[7]) != 0;
Paul Wilkins's avatar
Paul Wilkins committed
364
      band_translate = vp9_coefband_trans_8x8plus;
365
      break;
Daniel Kang's avatar
Daniel Kang committed
366
    default:
Dmitry Kovalev's avatar
Dmitry Kovalev committed
367
      assert(0);
Daniel Kang's avatar
Daniel Kang committed
368
369
      break;
  }
John Koleszar's avatar
John Koleszar committed
370
  assert(eob <= seg_eob);
371

372
  pt = combine_entropy_contexts(above_ec, left_ec);
373
374
  nb = vp9_get_coef_neighbors_handle(scan, &pad);
  default_eob = seg_eob;
375

376
377
  if (vp9_segfeature_active(xd, segment_id, SEG_LVL_SKIP))
    seg_eob = 0;
378

379
380
381
382
  /* sanity check to ensure that we do not have spurious non-zero q values */
  if (eob < seg_eob)
    assert(qcoeff_ptr[scan[eob]] == 0);

383
  {
384
    for (c = 0; c < eob; c++) {
385
      int v = qcoeff_ptr[scan[c]];
386
      int t = vp9_dct_value_tokens_ptr[v].token;
Paul Wilkins's avatar
Paul Wilkins committed
387
      int band = get_coef_band(band_translate, c);
388
389
      if (c)
        pt = vp9_get_coef_context(scan, nb, pad, token_cache, c, default_eob);
390

391
392
393
394
      if (!c || token_cache[scan[c - 1]])  // do not skip eob
        cost += token_costs_noskip[band][pt][t] + vp9_dct_value_cost_ptr[v];
      else
        cost += token_costs[band][pt][t] + vp9_dct_value_cost_ptr[v];
395
      token_cache[scan[c]] = vp9_pt_energy_class[t];
396
    }
397
398
399
    if (c < seg_eob) {
      if (c)
        pt = vp9_get_coef_context(scan, nb, pad, token_cache, c, default_eob);
400
401
402
      cost += mb->token_costs_noskip[tx_size][type][ref]
          [get_coef_band(band_translate, c)]
          [pt][DCT_EOB_TOKEN];
403
    }
404
405
  }

406
407
408
  // is eob first coefficient;
  for (pt = 0; pt < (1 << tx_size); pt++) {
    A[pt] = L[pt] = c > 0;
409
  }
410

411
412
413
  return cost;
}

414
static void choose_txfm_size_from_rd(VP9_COMP *cpi, MACROBLOCK *x,
415
                                     int (*r)[2], int *rate,
Ronald S. Bultje's avatar
Ronald S. Bultje committed
416
                                     int64_t *d, int64_t *distortion,
417
418
419
                                     int *s, int *skip,
                                     int64_t txfm_cache[NB_TXFM_MODES],
                                     TX_SIZE max_txfm_size) {
420
421
422
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
423
  vp9_prob skip_prob = vp9_get_pred_prob(cm, xd, PRED_MBSKIP);
424
425
  int64_t rd[TX_SIZE_MAX_SB][2];
  int n, m;
426
  int s0, s1;
427

428
  const vp9_prob *tx_probs = vp9_get_pred_probs(cm, xd, PRED_TX_SIZE);
429

430
431
432
433
  for (n = TX_4X4; n <= max_txfm_size; n++) {
    r[n][1] = r[n][0];
    for (m = 0; m <= n - (n == max_txfm_size); m++) {
      if (m == n)
434
        r[n][1] += vp9_cost_zero(tx_probs[m]);
435
      else
436
        r[n][1] += vp9_cost_one(tx_probs[m]);
437
438
    }
  }
439

440
441
442
  assert(skip_prob > 0);
  s0 = vp9_cost_bit(skip_prob, 0);
  s1 = vp9_cost_bit(skip_prob, 1);
443

444
445
446
447
448
449
  for (n = TX_4X4; n <= max_txfm_size; n++) {
    if (s[n]) {
      rd[n][0] = rd[n][1] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
    } else {
      rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0] + s0, d[n]);
      rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1] + s0, d[n]);
450
451
452
    }
  }

453
454
455
456
457
458
  if (max_txfm_size == TX_32X32 &&
      (cm->txfm_mode == ALLOW_32X32 ||
       (cm->txfm_mode == TX_MODE_SELECT &&
        rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
        rd[TX_32X32][1] < rd[TX_4X4][1]))) {
    mbmi->txfm_size = TX_32X32;
459
460
461
462
463
464
  } else if (max_txfm_size >= TX_16X16 &&
             (cm->txfm_mode == ALLOW_16X16 ||
              cm->txfm_mode == ALLOW_32X32 ||
              (cm->txfm_mode == TX_MODE_SELECT &&
               rd[TX_16X16][1] < rd[TX_8X8][1] &&
               rd[TX_16X16][1] < rd[TX_4X4][1]))) {
465
    mbmi->txfm_size = TX_16X16;
466
  } else if (cm->txfm_mode == ALLOW_8X8 ||
467
468
             cm->txfm_mode == ALLOW_16X16 ||
             cm->txfm_mode == ALLOW_32X32 ||
469
           (cm->txfm_mode == TX_MODE_SELECT && rd[TX_8X8][1] < rd[TX_4X4][1])) {
470
471
472
473
474
    mbmi->txfm_size = TX_8X8;
  } else {
    mbmi->txfm_size = TX_4X4;
  }

475
  *distortion = d[mbmi->txfm_size];
476
  *rate       = r[mbmi->txfm_size][cm->txfm_mode == TX_MODE_SELECT];
477
478
  *skip       = s[mbmi->txfm_size];

479
480
  txfm_cache[ONLY_4X4] = rd[TX_4X4][0];
  txfm_cache[ALLOW_8X8] = rd[TX_8X8][0];
481
482
  txfm_cache[ALLOW_16X16] = rd[MIN(max_txfm_size, TX_16X16)][0];
  txfm_cache[ALLOW_32X32] = rd[MIN(max_txfm_size, TX_32X32)][0];
483
484
485
486
  if (max_txfm_size == TX_32X32 &&
      rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
      rd[TX_32X32][1] < rd[TX_4X4][1])
    txfm_cache[TX_MODE_SELECT] = rd[TX_32X32][1];
487
488
  else if (max_txfm_size >= TX_16X16 &&
           rd[TX_16X16][1] < rd[TX_8X8][1] && rd[TX_16X16][1] < rd[TX_4X4][1])
489
    txfm_cache[TX_MODE_SELECT] = rd[TX_16X16][1];
490
  else
491
492
    txfm_cache[TX_MODE_SELECT] = rd[TX_4X4][1] < rd[TX_8X8][1] ?
                                 rd[TX_4X4][1] : rd[TX_8X8][1];
493
494
}

Ronald S. Bultje's avatar
Ronald S. Bultje committed
495
496
static int64_t block_error_sby(MACROBLOCK *x, BLOCK_SIZE_TYPE bsize,
                               int shift) {
497
  const int bwl = b_width_log2(bsize), bhl = b_height_log2(bsize);
Ronald S. Bultje's avatar
Ronald S. Bultje committed
498
499
  return vp9_block_error(x->plane[0].coeff, x->e_mbd.plane[0].dqcoeff,
                         16 << (bwl + bhl)) >> shift;
500
}
501

Ronald S. Bultje's avatar
Ronald S. Bultje committed
502
503
static int64_t block_error_sbuv(MACROBLOCK *x, BLOCK_SIZE_TYPE bsize,
                                int shift) {
504
505
506
  const int bwl = b_width_log2(bsize), bhl = b_height_log2(bsize);
  int64_t sum = 0;
  int plane;
507

508
509
510
  for (plane = 1; plane < MAX_MB_PLANE; plane++) {
    const int subsampling = x->e_mbd.plane[plane].subsampling_x +
                            x->e_mbd.plane[plane].subsampling_y;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
511
512
    sum += vp9_block_error(x->plane[plane].coeff, x->e_mbd.plane[plane].dqcoeff,
                           16 << (bwl + bhl - subsampling));
513
  }
Ronald S. Bultje's avatar
Ronald S. Bultje committed
514
  return sum >> shift;
515
516
}

517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
struct rdcost_block_args {
  VP9_COMMON *cm;
  MACROBLOCK *x;
  ENTROPY_CONTEXT t_above[16];
  ENTROPY_CONTEXT t_left[16];
  TX_SIZE tx_size;
  int bw;
  int bh;
  int cost;
};

static void rdcost_block(int plane, int block, BLOCK_SIZE_TYPE bsize,
                         int ss_txfrm_size, void *arg) {
  struct rdcost_block_args* args = arg;
  int x_idx, y_idx;
  MACROBLOCKD * const xd = &args->x->e_mbd;

  txfrm_block_to_raster_xy(xd, bsize, plane, block, args->tx_size * 2, &x_idx,
                           &y_idx);

  args->cost += cost_coeffs(args->cm, args->x, plane, block,
                            xd->plane[plane].plane_type, args->t_above + x_idx,
                            args->t_left + y_idx, args->tx_size,
                            args->bw * args->bh);
}

static int rdcost_plane(VP9_COMMON * const cm, MACROBLOCK *x, int plane,
                        BLOCK_SIZE_TYPE bsize, TX_SIZE tx_size) {
  MACROBLOCKD * const xd = &x->e_mbd;
546
547
548
  const int bwl = b_width_log2(bsize) - xd->plane[plane].subsampling_x;
  const int bhl = b_height_log2(bsize) - xd->plane[plane].subsampling_y;
  const int bw = 1 << bwl, bh = 1 << bhl;
549
  struct rdcost_block_args args = { cm, x, { 0 }, { 0 }, tx_size, bw, bh, 0 };
550

551
  vpx_memcpy(&args.t_above, xd->plane[plane].above_context,
552
             sizeof(ENTROPY_CONTEXT) * bw);
553
  vpx_memcpy(&args.t_left, xd->plane[plane].left_context,
554
             sizeof(ENTROPY_CONTEXT) * bh);
555

556
  foreach_transformed_block_in_plane(xd, bsize, plane, rdcost_block, &args);
557

558
  return args.cost;
559
560
}

561
562
563
static int rdcost_uv(VP9_COMMON *const cm, MACROBLOCK *x,
                     BLOCK_SIZE_TYPE bsize, TX_SIZE tx_size) {
  int cost = 0, plane;
564

565
566
  for (plane = 1; plane < MAX_MB_PLANE; plane++) {
    cost += rdcost_plane(cm, x, plane, bsize, tx_size);
567
568
  }
  return cost;
569
570
}

571
static void super_block_yrd_for_txfm(VP9_COMMON *const cm, MACROBLOCK *x,
Ronald S. Bultje's avatar
Ronald S. Bultje committed
572
573
                                     int *rate, int64_t *distortion,
                                     int *skippable,
574
                                     BLOCK_SIZE_TYPE bsize, TX_SIZE tx_size) {
575
  MACROBLOCKD *const xd = &x->e_mbd;
576
  xd->mode_info_context->mbmi.txfm_size = tx_size;
577

Ronald S. Bultje's avatar
Ronald S. Bultje committed
578
  if (xd->mode_info_context->mbmi.ref_frame[0] == INTRA_FRAME)
579
580
581
    vp9_encode_intra_block_y(cm, x, bsize);
  else
    vp9_xform_quant_sby(cm, x, bsize);
582

583
584
  *distortion = block_error_sby(x, bsize, tx_size == TX_32X32 ? 0 : 2);
  *rate       = rdcost_plane(cm, x, 0, bsize, tx_size);
585
  *skippable  = vp9_sby_is_skippable(xd, bsize);
586
587
}

588
static void super_block_yrd(VP9_COMP *cpi,
Ronald S. Bultje's avatar
Ronald S. Bultje committed
589
                            MACROBLOCK *x, int *rate, int64_t *distortion,
590
                            int *skip, BLOCK_SIZE_TYPE bs,
591
                            int64_t txfm_cache[NB_TXFM_MODES]) {
592
  VP9_COMMON *const cm = &cpi->common;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
593
594
  int r[TX_SIZE_MAX_SB][2], s[TX_SIZE_MAX_SB];
  int64_t d[TX_SIZE_MAX_SB];
Jim Bankoski's avatar
Jim Bankoski committed
595
596
  MACROBLOCKD *xd = &x->e_mbd;
  MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
597

598
  assert(bs == mbmi->sb_type);
Ronald S. Bultje's avatar
Ronald S. Bultje committed
599
  if (mbmi->ref_frame[0] > INTRA_FRAME)
600
    vp9_subtract_sby(x, bs);
Ronald S. Bultje's avatar
Ronald S. Bultje committed
601

602
  if (cpi->sf.use_largest_txform) {
Jim Bankoski's avatar
Jim Bankoski committed
603
604
605
606
607
608
609
610
611
    if (bs >= BLOCK_SIZE_SB32X32) {
      mbmi->txfm_size = TX_32X32;
    } else if (bs >= BLOCK_SIZE_MB16X16) {
      mbmi->txfm_size = TX_16X16;
    } else if (bs >= BLOCK_SIZE_SB8X8) {
      mbmi->txfm_size = TX_8X8;
    } else {
      mbmi->txfm_size = TX_4X4;
    }
612
    vpx_memset(txfm_cache, 0, NB_TXFM_MODES * sizeof(int64_t));
Jim Bankoski's avatar
Jim Bankoski committed
613
614
615
616
    super_block_yrd_for_txfm(cm, x, rate, distortion, skip, bs,
                             mbmi->txfm_size);
    return;
  }
617
  if (bs >= BLOCK_SIZE_SB32X32)
618
619
    super_block_yrd_for_txfm(cm, x, &r[TX_32X32][0], &d[TX_32X32], &s[TX_32X32],
                             bs, TX_32X32);
620
  if (bs >= BLOCK_SIZE_MB16X16)
621
622
623
624
625
626
    super_block_yrd_for_txfm(cm, x, &r[TX_16X16][0], &d[TX_16X16], &s[TX_16X16],
                             bs, TX_16X16);
  super_block_yrd_for_txfm(cm, x, &r[TX_8X8][0], &d[TX_8X8], &s[TX_8X8], bs,
                           TX_8X8);
  super_block_yrd_for_txfm(cm, x, &r[TX_4X4][0], &d[TX_4X4], &s[TX_4X4], bs,
                           TX_4X4);
Ronald S. Bultje's avatar
Ronald S. Bultje committed
627

628
  choose_txfm_size_from_rd(cpi, x, r, rate, d, distortion, s,
629
                           skip, txfm_cache,
630
                           TX_32X32 - (bs < BLOCK_SIZE_SB32X32)
Jingning Han's avatar
Jingning Han committed
631
                           - (bs < BLOCK_SIZE_MB16X16));
Ronald S. Bultje's avatar
Ronald S. Bultje committed
632
}
Ronald S. Bultje's avatar
Ronald S. Bultje committed
633

634
static int64_t rd_pick_intra4x4block(VP9_COMP *cpi, MACROBLOCK *x, int ib,
Yaowu Xu's avatar
Yaowu Xu committed
635
                                     MB_PREDICTION_MODE *best_mode,
636
637
638
                                     int *bmode_costs,
                                     ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l,
                                     int *bestrate, int *bestratey,
Ronald S. Bultje's avatar
Ronald S. Bultje committed
639
                                     int64_t *bestdistortion,
640
641
                                     BLOCK_SIZE_TYPE bsize) {
  MB_PREDICTION_MODE mode;
Deb Mukherjee's avatar
Deb Mukherjee committed
642
  MACROBLOCKD *xd = &x->e_mbd;
643
  int64_t best_rd = INT64_MAX;
John Koleszar's avatar
John Koleszar committed
644
  int rate = 0;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
645
  int64_t distortion;
646
  VP9_COMMON *const cm = &cpi->common;
John Koleszar's avatar
John Koleszar committed
647
  const int src_stride = x->plane[0].src.stride;
648
649
650
651
652
  uint8_t *src, *dst;
  int16_t *src_diff, *coeff;

  ENTROPY_CONTEXT ta[2], tempa[2];
  ENTROPY_CONTEXT tl[2], templ[2];
Deb Mukherjee's avatar
Deb Mukherjee committed
653
654
  TX_TYPE tx_type = DCT_DCT;
  TX_TYPE best_tx_type = DCT_DCT;
655
656
657
658
  int bw = 1 << b_width_log2(bsize);
  int bh = 1 << b_height_log2(bsize);
  int idx, idy, block;
  DECLARE_ALIGNED(16, int16_t, best_dqcoeff[4][16]);
John Koleszar's avatar
John Koleszar committed
659

Jingning Han's avatar
Jingning Han committed
660
  assert(ib < 4);
661

662
663
  vpx_memcpy(ta, a, sizeof(ta));
  vpx_memcpy(tl, l, sizeof(tl));
664
  xd->mode_info_context->mbmi.txfm_size = TX_4X4;
665
666

  for (mode = DC_PRED; mode <= TM_PRED; ++mode) {
667
    int64_t this_rd;
668
    int ratey = 0;
669

670
    rate = bmode_costs[mode];
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
    distortion = 0;

    vpx_memcpy(tempa, ta, sizeof(ta));
    vpx_memcpy(templ, tl, sizeof(tl));

    for (idy = 0; idy < bh; ++idy) {
      for (idx = 0; idx < bw; ++idx) {
        block = ib + idy * 2 + idx;
        xd->mode_info_context->bmi[block].as_mode.first = mode;
        src = raster_block_offset_uint8(xd, BLOCK_SIZE_SB8X8, 0, block,
                                        x->plane[0].src.buf, src_stride);
        src_diff = raster_block_offset_int16(xd, BLOCK_SIZE_SB8X8, 0, block,
                                             x->plane[0].src_diff);
        coeff = BLOCK_OFFSET(x->plane[0].coeff, block, 16);
        dst = raster_block_offset_uint8(xd, BLOCK_SIZE_SB8X8, 0, block,
                                        xd->plane[0].dst.buf,
                                        xd->plane[0].dst.stride);
688
689
        vp9_intra4x4_predict(xd, block, BLOCK_SIZE_SB8X8, mode,
                             dst, xd->plane[0].dst.stride);
690
691
692
693
694
695
696
697
698
699
700
701
        vp9_subtract_block(4, 4, src_diff, 8,
                           src, src_stride,
                           dst, xd->plane[0].dst.stride);

        tx_type = get_tx_type_4x4(xd, block);
        if (tx_type != DCT_DCT) {
          vp9_short_fht4x4(src_diff, coeff, 8, tx_type);
          x->quantize_b_4x4(x, block, tx_type, 16);
        } else {
          x->fwd_txm4x4(src_diff, coeff, 16);
          x->quantize_b_4x4(x, block, tx_type, 16);
        }
John Koleszar's avatar
John Koleszar committed
702

703
704
705
706
        ratey += cost_coeffs(cm, x, 0, block, PLANE_TYPE_Y_WITH_DC,
                             tempa + idx, templ + idy, TX_4X4, 16);
        distortion += vp9_block_error(coeff, BLOCK_OFFSET(xd->plane[0].dqcoeff,
                                                         block, 16), 16) >> 2;
John Koleszar's avatar
John Koleszar committed
707

708
709
710
711
712
713
714
715
        if (best_tx_type != DCT_DCT)
          vp9_short_iht4x4_add(BLOCK_OFFSET(xd->plane[0].dqcoeff, block, 16),
                               dst, xd->plane[0].dst.stride, best_tx_type);
        else
          xd->inv_txm4x4_add(BLOCK_OFFSET(xd->plane[0].dqcoeff, block, 16),
                             dst, xd->plane[0].dst.stride);
      }
    }
Jingning Han's avatar
Jingning Han committed
716

717
718
    rate += ratey;
    this_rd = RDCOST(x->rdmult, x->rddiv, rate, distortion);
Jingning Han's avatar
Jingning Han committed
719

720
721
722
723
724
725
726
    if (this_rd < best_rd) {
      *bestrate = rate;
      *bestratey = ratey;
      *bestdistortion = distortion;
      best_rd = this_rd;
      *best_mode = mode;
      best_tx_type = tx_type;
727
728
729
730
731
732
733
734
735
736
      vpx_memcpy(a, tempa, sizeof(tempa));
      vpx_memcpy(l, templ, sizeof(templ));
      for (idy = 0; idy < bh; ++idy) {
        for (idx = 0; idx < bw; ++idx) {
          block = ib + idy * 2 + idx;
          vpx_memcpy(best_dqcoeff[idy * 2 + idx],
                     BLOCK_OFFSET(xd->plane[0].dqcoeff, block, 16),
                     sizeof(best_dqcoeff[0]));
        }
      }
John Koleszar's avatar
John Koleszar committed
737
    }
John Koleszar's avatar
John Koleszar committed
738
  }
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757

  for (idy = 0; idy < bh; ++idy) {
    for (idx = 0; idx < bw; ++idx) {
      block = ib + idy * 2 + idx;
      xd->mode_info_context->bmi[block].as_mode.first = *best_mode;
      dst = raster_block_offset_uint8(xd, BLOCK_SIZE_SB8X8, 0, block,
                                      xd->plane[0].dst.buf,
                                      xd->plane[0].dst.stride);

      vp9_intra4x4_predict(xd, block, BLOCK_SIZE_SB8X8, *best_mode,
                           dst, xd->plane[0].dst.stride);
      // inverse transform
      if (best_tx_type != DCT_DCT)
        vp9_short_iht4x4_add(best_dqcoeff[idy * 2 + idx], dst,
                             xd->plane[0].dst.stride, best_tx_type);
      else
        xd->inv_txm4x4_add(best_dqcoeff[idy * 2 + idx], dst,
                           xd->plane[0].dst.stride);
    }
Scott LaVarnway's avatar
Scott LaVarnway committed
758
  }
John Koleszar's avatar
John Koleszar committed
759

John Koleszar's avatar
John Koleszar committed
760
  return best_rd;
John Koleszar's avatar
John Koleszar committed
761
762
}

763
764
static int64_t rd_pick_intra4x4mby_modes(VP9_COMP *cpi, MACROBLOCK *mb,
                                         int *Rate, int *rate_y,
Ronald S. Bultje's avatar
Ronald S. Bultje committed
765
                                         int64_t *Distortion, int64_t best_rd) {
766
  int i, j;
John Koleszar's avatar
John Koleszar committed
767
  MACROBLOCKD *const xd = &mb->e_mbd;
768
769
770
771
  BLOCK_SIZE_TYPE bsize = xd->mode_info_context->mbmi.sb_type;
  int bw = 1 << b_width_log2(bsize);
  int bh = 1 << b_height_log2(bsize);
  int idx, idy;
772
  int cost = 0;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
773
  int64_t distortion = 0;
John Koleszar's avatar
John Koleszar committed
774
775
  int tot_rate_y = 0;
  int64_t total_rd = 0;
776
  ENTROPY_CONTEXT t_above[4], t_left[4];
John Koleszar's avatar
John Koleszar committed
777
  int *bmode_costs;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
778
  MODE_INFO *const mic = xd->mode_info_context;
John Koleszar's avatar
John Koleszar committed
779

780
781
  vpx_memcpy(t_above, xd->plane[0].above_context, sizeof(t_above));
  vpx_memcpy(t_left, xd->plane[0].left_context, sizeof(t_left));
John Koleszar's avatar
John Koleszar committed
782

783
  bmode_costs = mb->mbmode_cost;
784

785
786
787
  for (idy = 0; idy < 2; idy += bh) {
    for (idx = 0; idx < 2; idx += bw) {
      const int mis = xd->mode_info_stride;
Yaowu Xu's avatar
Yaowu Xu committed
788
      MB_PREDICTION_MODE UNINITIALIZED_IS_SAFE(best_mode);
789
      int UNINITIALIZED_IS_SAFE(r), UNINITIALIZED_IS_SAFE(ry);
Ronald S. Bultje's avatar
Ronald S. Bultje committed
790
      int64_t UNINITIALIZED_IS_SAFE(d);
791
792
793
794
795
796
797
      i = idy * 2 + idx;

      if (xd->frame_type == KEY_FRAME) {
        const MB_PREDICTION_MODE A = above_block_mode(mic, i, mis);
        const MB_PREDICTION_MODE L = (xd->left_available || idx) ?
                                     left_block_mode(mic, i) : DC_PRED;

798
        bmode_costs  = mb->y_mode_costs[A][L];
799
      }
800

801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
      total_rd += rd_pick_intra4x4block(cpi, mb, i, &best_mode, bmode_costs,
                                        t_above + idx, t_left + idy,
                                        &r, &ry, &d, bsize);
      cost += r;
      distortion += d;
      tot_rate_y += ry;

      mic->bmi[i].as_mode.first = best_mode;
      for (j = 1; j < bh; ++j)
        mic->bmi[i + j * 2].as_mode.first = best_mode;
      for (j = 1; j < bw; ++j)
        mic->bmi[i + j].as_mode.first = best_mode;

      if (total_rd >= best_rd)
        break;
John Koleszar's avatar
John Koleszar committed
816
817
    }
  }
John Koleszar's avatar
John Koleszar committed
818

819
  if (total_rd >= best_rd)
820
    return INT64_MAX;
821

John Koleszar's avatar
John Koleszar committed
822
  *Rate = cost;
823
  *rate_y = tot_rate_y;
John Koleszar's avatar
John Koleszar committed
824
  *Distortion = distortion;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
825
  xd->mode_info_context->mbmi.mode = mic->bmi[3].as_mode.first;
John Koleszar's avatar
John Koleszar committed
826

John Koleszar's avatar
John Koleszar committed
827
  return RDCOST(mb->rdmult, mb->rddiv, cost, distortion);
John Koleszar's avatar
John Koleszar committed
828
}
829

830
831
static int64_t rd_pick_intra_sby_mode(VP9_COMP *cpi, MACROBLOCK *x,
                                      int *rate, int *rate_tokenonly,
Ronald S. Bultje's avatar
Ronald S. Bultje committed
832
                                      int64_t *distortion, int *skippable,
833
                                      BLOCK_SIZE_TYPE bsize,
834
                                      int64_t txfm_cache[NB_TXFM_MODES]) {
Ronald S. Bultje's avatar
Ronald S. Bultje committed
835
836
  MB_PREDICTION_MODE mode;
  MB_PREDICTION_MODE UNINITIALIZED_IS_SAFE(mode_selected);
Jim Bankoski's avatar
Jim Bankoski committed
837
  MACROBLOCKD *const xd = &x->e_mbd;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
838
839
  int this_rate, this_rate_tokenonly, s;
  int64_t this_distortion;
840
  int64_t best_rd = INT64_MAX, this_rd;
841
842
  TX_SIZE UNINITIALIZED_IS_SAFE(best_tx);
  int i;
843
  int *bmode_costs = x->mbmode_cost;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
844

845
846
847
848
849
  if (bsize < BLOCK_SIZE_SB8X8) {
    x->e_mbd.mode_info_context->mbmi.txfm_size = TX_4X4;
    return best_rd;
  }

850
851
  for (i = 0; i < NB_TXFM_MODES; i++)
    txfm_cache[i] = INT64_MAX;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
852
853
854

  /* Y Search for 32x32 intra prediction mode */
  for (mode = DC_PRED; mode <= TM_PRED; mode++) {
855
    int64_t local_txfm_cache[NB_TXFM_MODES];
856
857
    MODE_INFO *const mic = xd->mode_info_context;
    const int mis = xd->mode_info_stride;
858

859
860
861
862
863
864
865
    if (cpi->common.frame_type == KEY_FRAME) {
      const MB_PREDICTION_MODE A = above_block_mode(mic, 0, mis);
      const MB_PREDICTION_MODE L = xd->left_available ?
                                   left_block_mode(mic, 0) : DC_PRED;

      bmode_costs = x->y_mode_costs[A][L];
    }
Ronald S. Bultje's avatar
Ronald S. Bultje committed
866
867
    x->e_mbd.mode_info_context->mbmi.mode = mode;

868
869
    super_block_yrd(cpi, x, &this_rate_tokenonly, &this_distortion, &s,
                    bsize, local_txfm_cache);
Jim Bankoski's avatar
Jim Bankoski committed
870

871
    this_rate = this_rate_tokenonly + bmode_costs[mode];
Ronald S. Bultje's avatar
Ronald S. Bultje committed
872
873
874
875
876
    this_rd = RDCOST(x->rdmult, x->rddiv, this_rate, this_distortion);

    if (this_rd < best_rd) {
      mode_selected   = mode;
      best_rd         = this_rd;
877
      best_tx         = x->e_mbd.mode_info_context->mbmi.txfm_size;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
878
879
880
881
882
      *rate           = this_rate;
      *rate_tokenonly = this_rate_tokenonly;
      *distortion     = this_distortion;
      *skippable      = s;
    }
883

884
885
    for (i = 0; i < NB_TXFM_MODES; i++) {
      int64_t adj_rd = this_rd + local_txfm_cache[i] -
886
                       local_txfm_cache[cpi->common.txfm_mode];
887
888
      if (adj_rd < txfm_cache[i]) {
        txfm_cache[i] = adj_rd;
John Koleszar's avatar
John Koleszar committed
889
      }
John Koleszar's avatar
John Koleszar committed
890
    }
John Koleszar's avatar
John Koleszar committed
891
  }
John Koleszar's avatar
John Koleszar committed
892

893
894
  x->e_mbd.mode_info_context->mbmi.mode = mode_selected;
  x->e_mbd.mode_info_context->mbmi.txfm_size = best_tx;
895