vp9_rdopt.c 123 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5
6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9
10
11
12
13
14
15
 */


#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
16

17
#include "vp9/common/vp9_pragmas.h"
18
19
20
21
22
#include "vp9/encoder/vp9_tokenize.h"
#include "vp9/encoder/vp9_treewriter.h"
#include "vp9/encoder/vp9_onyx_int.h"
#include "vp9/encoder/vp9_modecosts.h"
#include "vp9/encoder/vp9_encodeintra.h"
23
24
25
26
27
#include "vp9/common/vp9_entropymode.h"
#include "vp9/common/vp9_reconinter.h"
#include "vp9/common/vp9_reconintra.h"
#include "vp9/common/vp9_findnearmv.h"
#include "vp9/common/vp9_quant_common.h"
28
29
30
31
32
33
#include "vp9/encoder/vp9_encodemb.h"
#include "vp9/encoder/vp9_quantize.h"
#include "vp9/encoder/vp9_variance.h"
#include "vp9/encoder/vp9_mcomp.h"
#include "vp9/encoder/vp9_rdopt.h"
#include "vp9/encoder/vp9_ratectrl.h"
John Koleszar's avatar
John Koleszar committed
34
#include "vpx_mem/vpx_mem.h"
35
36
37
38
39
#include "vp9/common/vp9_systemdependent.h"
#include "vp9/encoder/vp9_encodemv.h"
#include "vp9/common/vp9_seg_common.h"
#include "vp9/common/vp9_pred_common.h"
#include "vp9/common/vp9_entropy.h"
40
#include "vp9_rtcd.h"
41
#include "vp9/common/vp9_mvref_common.h"
Ronald S. Bultje's avatar
Ronald S. Bultje committed
42
#include "vp9/common/vp9_common.h"
Paul Wilkins's avatar
Paul Wilkins committed
43

44
45
#define INVALID_MV 0x80008000

46
47
48
/* Factor to weigh the rate for switchable interp filters */
#define SWITCHABLE_INTERP_RATE_FACTOR 1

49
50
51
DECLARE_ALIGNED(16, extern const uint8_t,
                vp9_pt_energy_class[MAX_ENTROPY_TOKENS]);

Ronald S. Bultje's avatar
Ronald S. Bultje committed
52
#define I4X4_PRED 0x8000
Ronald S. Bultje's avatar
Ronald S. Bultje committed
53
#define SPLITMV 0x10000
Ronald S. Bultje's avatar
Ronald S. Bultje committed
54

55
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
56
57
  {ZEROMV,    LAST_FRAME,   NONE},
  {DC_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
58

59
60
  {NEARESTMV, LAST_FRAME,   NONE},
  {NEARMV,    LAST_FRAME,   NONE},
John Koleszar's avatar
John Koleszar committed
61

62
63
  {ZEROMV,    GOLDEN_FRAME, NONE},
  {NEARESTMV, GOLDEN_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
64

65
66
  {ZEROMV,    ALTREF_FRAME, NONE},
  {NEARESTMV, ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
67

68
69
  {NEARMV,    GOLDEN_FRAME, NONE},
  {NEARMV,    ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
70

71
72
73
74
75
76
77
78
  {V_PRED,    INTRA_FRAME,  NONE},
  {H_PRED,    INTRA_FRAME,  NONE},
  {D45_PRED,  INTRA_FRAME,  NONE},
  {D135_PRED, INTRA_FRAME,  NONE},
  {D117_PRED, INTRA_FRAME,  NONE},
  {D153_PRED, INTRA_FRAME,  NONE},
  {D27_PRED,  INTRA_FRAME,  NONE},
  {D63_PRED,  INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
79

80
  {TM_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
81

82
83
84
  {NEWMV,     LAST_FRAME,   NONE},
  {NEWMV,     GOLDEN_FRAME, NONE},
  {NEWMV,     ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
85

86
87
88
  {SPLITMV,   LAST_FRAME,   NONE},
  {SPLITMV,   GOLDEN_FRAME, NONE},
  {SPLITMV,   ALTREF_FRAME, NONE},
89

Ronald S. Bultje's avatar
Ronald S. Bultje committed
90
  {I4X4_PRED, INTRA_FRAME,  NONE},
91

John Koleszar's avatar
John Koleszar committed
92
  /* compound prediction modes */
Ronald S. Bultje's avatar
Ronald S. Bultje committed
93
94
95
  {ZEROMV,    LAST_FRAME,   ALTREF_FRAME},
  {NEARESTMV, LAST_FRAME,   ALTREF_FRAME},
  {NEARMV,    LAST_FRAME,   ALTREF_FRAME},
96

John Koleszar's avatar
John Koleszar committed
97
98
99
  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME},
100

Ronald S. Bultje's avatar
Ronald S. Bultje committed
101
  {NEWMV,     LAST_FRAME,   ALTREF_FRAME},
John Koleszar's avatar
John Koleszar committed
102
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME},
103

Ronald S. Bultje's avatar
Ronald S. Bultje committed
104
  {SPLITMV,   LAST_FRAME,   ALTREF_FRAME},
105
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME},
John Koleszar's avatar
John Koleszar committed
106
107
};

108
109
110
111
112
113
114
115
116
117
118
// The baseline rd thresholds for breaking out of the rd loop for
// certain modes are assumed to be based on 8x8 blocks.
// This table is used to correct for blocks size.
// The factors here are << 2 (2 = x0.5, 32 = x8 etc).
static int rd_thresh_block_size_factor[BLOCK_SIZE_TYPES] =
  {2, 3, 3, 4, 6, 6, 8, 12, 12, 16, 24, 24, 32};

#define BASE_RD_THRESH_FREQ_FACT 16
#define MAX_RD_THRESH_FREQ_FACT 32
#define MAX_RD_THRESH_FREQ_INC 1

119
120
121
static void fill_token_costs(vp9_coeff_count (*c)[BLOCK_TYPES],
                             vp9_coeff_count (*cnoskip)[BLOCK_TYPES],
                             vp9_coeff_probs_model (*p)[BLOCK_TYPES]) {
122
  int i, j, k, l;
123
124
125
126
127
128
129
130
131
132
  TX_SIZE t;
  for (t = TX_4X4; t <= TX_32X32; t++)
    for (i = 0; i < BLOCK_TYPES; i++)
      for (j = 0; j < REF_TYPES; j++)
        for (k = 0; k < COEF_BANDS; k++)
          for (l = 0; l < PREV_COEF_CONTEXTS; l++) {
            vp9_prob probs[ENTROPY_NODES];
            vp9_model_to_full_probs(p[t][i][j][k][l], probs);
            vp9_cost_tokens((int *)cnoskip[t][i][j][k][l], probs,
                            vp9_coef_tree);
133
#if CONFIG_BALANCED_COEFTREE
134
135
136
137
            // Replace the eob node prob with a very small value so that the
            // cost approximately equals the cost without the eob node
            probs[1] = 1;
            vp9_cost_tokens((int *)c[t][i][j][k][l], probs, vp9_coef_tree);
138
#else
139
140
            vp9_cost_tokens_skip((int *)c[t][i][j][k][l], probs,
                                 vp9_coef_tree);
141
142
143
            assert(c[t][i][j][k][l][DCT_EOB_TOKEN] ==
                   cnoskip[t][i][j][k][l][DCT_EOB_TOKEN]);
#endif
144
          }
145
146
}

147
148
149
150
static int rd_iifactor[32] =  { 4, 4, 3, 2, 1, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0, };
John Koleszar's avatar
John Koleszar committed
151

152
// 3* dc_qlookup[Q]*dc_qlookup[Q];
153

154
/* values are now correlated to quantizer */
Paul Wilkins's avatar
Paul Wilkins committed
155
156
157
static int sad_per_bit16lut[QINDEX_RANGE];
static int sad_per_bit4lut[QINDEX_RANGE];

158
void vp9_init_me_luts() {
John Koleszar's avatar
John Koleszar committed
159
160
161
162
163
164
165
  int i;

  // Initialize the sad lut tables using a formulaic calculation for now
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < QINDEX_RANGE; i++) {
    sad_per_bit16lut[i] =
166
      (int)((0.0418 * vp9_convert_qindex_to_q(i)) + 2.4107);
167
    sad_per_bit4lut[i] = (int)(0.063 * vp9_convert_qindex_to_q(i) + 2.742);
John Koleszar's avatar
John Koleszar committed
168
  }
Paul Wilkins's avatar
Paul Wilkins committed
169
}
John Koleszar's avatar
John Koleszar committed
170

171
static int compute_rd_mult(int qindex) {
172
  const int q = vp9_dc_quant(qindex, 0);
173
  return (11 * q * q) >> 2;
174
175
}

Dmitry Kovalev's avatar
Dmitry Kovalev committed
176
177
178
void vp9_initialize_me_consts(VP9_COMP *cpi, int qindex) {
  cpi->mb.sadperbit16 = sad_per_bit16lut[qindex];
  cpi->mb.sadperbit4 = sad_per_bit4lut[qindex];
John Koleszar's avatar
John Koleszar committed
179
180
}

181

Dmitry Kovalev's avatar
Dmitry Kovalev committed
182
void vp9_initialize_rd_consts(VP9_COMP *cpi, int qindex) {
183
  int q, i, bsize;
John Koleszar's avatar
John Koleszar committed
184

185
  vp9_clear_system_state();  // __asm emms;
John Koleszar's avatar
John Koleszar committed
186

John Koleszar's avatar
John Koleszar committed
187
188
189
190
  // Further tests required to see if optimum is different
  // for key frames, golden frames and arf frames.
  // if (cpi->common.refresh_golden_frame ||
  //     cpi->common.refresh_alt_ref_frame)
191
  qindex = clamp(qindex, 0, MAXQ);
192

Dmitry Kovalev's avatar
Dmitry Kovalev committed
193
  cpi->RDMULT = compute_rd_mult(qindex);
John Koleszar's avatar
John Koleszar committed
194
195
196
197
198
  if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    if (cpi->twopass.next_iiratio > 31)
      cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
    else
      cpi->RDMULT +=
Dmitry Kovalev's avatar
Dmitry Kovalev committed
199
          (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
John Koleszar's avatar
John Koleszar committed
200
  }
201
  cpi->mb.errorperbit = cpi->RDMULT >> 6;
John Koleszar's avatar
John Koleszar committed
202
  cpi->mb.errorperbit += (cpi->mb.errorperbit == 0);
203

204
  vp9_set_speed_features(cpi);
John Koleszar's avatar
John Koleszar committed
205

Dmitry Kovalev's avatar
Dmitry Kovalev committed
206
207
  q = (int)pow(vp9_dc_quant(qindex, 0) >> 2, 1.25);
  q <<= 2;
John Koleszar's avatar
John Koleszar committed
208
209
  if (q < 8)
    q = 8;
210

John Koleszar's avatar
John Koleszar committed
211
212
213
  if (cpi->RDMULT > 1000) {
    cpi->RDDIV = 1;
    cpi->RDMULT /= 100;
John Koleszar's avatar
John Koleszar committed
214

215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
    for (bsize = 0; bsize < BLOCK_SIZE_TYPES; ++bsize) {
      for (i = 0; i < MAX_MODES; ++i) {
        // Threshold here seem unecessarily harsh but fine given actual
        // range of values used for cpi->sf.thresh_mult[]
        int thresh_max = INT_MAX / (q * rd_thresh_block_size_factor[bsize]);

        // *4 relates to the scaling of rd_thresh_block_size_factor[]
        if ((int64_t)cpi->sf.thresh_mult[i] < thresh_max) {
          cpi->rd_threshes[bsize][i] =
            cpi->sf.thresh_mult[i] * q *
            rd_thresh_block_size_factor[bsize] / (4 * 100);
        } else {
          cpi->rd_threshes[bsize][i] = INT_MAX;
        }
        cpi->rd_baseline_thresh[bsize][i] = cpi->rd_threshes[bsize][i];
230

Paul Wilkins's avatar
Paul Wilkins committed
231
232
233
234
        if (cpi->sf.adpative_rd_thresh)
          cpi->rd_thresh_freq_fact[bsize][i] = MAX_RD_THRESH_FREQ_FACT;
        else
          cpi->rd_thresh_freq_fact[bsize][i] = BASE_RD_THRESH_FREQ_FACT;
John Koleszar's avatar
John Koleszar committed
235
      }
John Koleszar's avatar
John Koleszar committed
236
    }
John Koleszar's avatar
John Koleszar committed
237
238
  } else {
    cpi->RDDIV = 100;
John Koleszar's avatar
John Koleszar committed
239

240
241
242
243
244
245
246
247
248
249
250
251
252
253
    for (bsize = 0; bsize < BLOCK_SIZE_TYPES; ++bsize) {
      for (i = 0; i < MAX_MODES; i++) {
        // Threshold here seem unecessarily harsh but fine given actual
        // range of values used for cpi->sf.thresh_mult[]
        int thresh_max = INT_MAX / (q * rd_thresh_block_size_factor[bsize]);

        if (cpi->sf.thresh_mult[i] < thresh_max) {
          cpi->rd_threshes[bsize][i] =
            cpi->sf.thresh_mult[i] * q *
            rd_thresh_block_size_factor[bsize] / 4;
        } else {
          cpi->rd_threshes[bsize][i] = INT_MAX;
        }
        cpi->rd_baseline_thresh[bsize][i] = cpi->rd_threshes[bsize][i];
Paul Wilkins's avatar
Paul Wilkins committed
254
255
256
257
258

        if (cpi->sf.adpative_rd_thresh)
          cpi->rd_thresh_freq_fact[bsize][i] = MAX_RD_THRESH_FREQ_FACT;
        else
          cpi->rd_thresh_freq_fact[bsize][i] = BASE_RD_THRESH_FREQ_FACT;
John Koleszar's avatar
John Koleszar committed
259
      }
John Koleszar's avatar
John Koleszar committed
260
    }
John Koleszar's avatar
John Koleszar committed
261
  }
John Koleszar's avatar
John Koleszar committed
262

263
264
265
  fill_token_costs(cpi->mb.token_costs,
                   cpi->mb.token_costs_noskip,
                   cpi->common.fc.coef_probs);
266

267
  for (i = 0; i < NUM_PARTITION_CONTEXTS; i++)
268
    vp9_cost_tokens(cpi->mb.partition_cost[i],
269
                    cpi->common.fc.partition_prob[cpi->common.frame_type][i],
270
271
                    vp9_partition_tree);

John Koleszar's avatar
John Koleszar committed
272
  /*rough estimate for costing*/
273
  vp9_init_mode_costs(cpi);
John Koleszar's avatar
John Koleszar committed
274

275
  if (cpi->common.frame_type != KEY_FRAME) {
276
    vp9_build_nmv_cost_table(
277
278
279
280
281
282
        cpi->mb.nmvjointcost,
        cpi->mb.e_mbd.allow_high_precision_mv ?
        cpi->mb.nmvcost_hp : cpi->mb.nmvcost,
        &cpi->common.fc.nmvc,
        cpi->mb.e_mbd.allow_high_precision_mv, 1, 1);
  }
John Koleszar's avatar
John Koleszar committed
283
284
}

Ronald S. Bultje's avatar
Ronald S. Bultje committed
285
286
287
288
int64_t vp9_block_error_c(int16_t *coeff, int16_t *dqcoeff,
                          intptr_t block_size) {
  int i;
  int64_t error = 0;
John Koleszar's avatar
John Koleszar committed
289

290
  for (i = 0; i < block_size; i++) {
John Koleszar's avatar
John Koleszar committed
291
    int this_diff = coeff[i] - dqcoeff[i];
Ronald S. Bultje's avatar
Ronald S. Bultje committed
292
    error += (unsigned)this_diff * this_diff;
John Koleszar's avatar
John Koleszar committed
293
  }
John Koleszar's avatar
John Koleszar committed
294

John Koleszar's avatar
John Koleszar committed
295
  return error;
John Koleszar's avatar
John Koleszar committed
296
297
}

298
static INLINE int cost_coeffs(VP9_COMMON *const cm, MACROBLOCK *mb,
299
                              int plane, int block, PLANE_TYPE type,
300
301
                              ENTROPY_CONTEXT *A,
                              ENTROPY_CONTEXT *L,
John Koleszar's avatar
John Koleszar committed
302
303
                              TX_SIZE tx_size,
                              int y_blocks) {
304
  MACROBLOCKD *const xd = &mb->e_mbd;
305
306
  MB_MODE_INFO *mbmi = &xd->mode_info_context->mbmi;
  int pt;
307
  int c = 0;
308
309
  int cost = 0, pad;
  const int *scan, *nb;
310
311
312
  const int eob = xd->plane[plane].eobs[block];
  const int16_t *qcoeff_ptr = BLOCK_OFFSET(xd->plane[plane].qcoeff,
                                           block, 16);
Ronald S. Bultje's avatar
Ronald S. Bultje committed
313
  const int ref = mbmi->ref_frame[0] != INTRA_FRAME;
314
  unsigned int (*token_costs)[PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS] =
315
      mb->token_costs[tx_size][type][ref];
316
  ENTROPY_CONTEXT above_ec, left_ec;
317
  TX_TYPE tx_type = DCT_DCT;
318

319
  const int segment_id = xd->mode_info_context->mbmi.segment_id;
320
321
322
  unsigned int (*token_costs_noskip)[PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS] =
      mb->token_costs_noskip[tx_size][type][ref];

323
324
  int seg_eob, default_eob;
  uint8_t token_cache[1024];
Paul Wilkins's avatar
Paul Wilkins committed
325
  const uint8_t * band_translate;
326
327

  // Check for consistency of tx_size with mode info
328
  assert((!type && !plane) || (type && plane));
329
330
331
  if (type == PLANE_TYPE_Y_WITH_DC) {
    assert(xd->mode_info_context->mbmi.txfm_size == tx_size);
  } else {
332
    TX_SIZE tx_size_uv = get_uv_tx_size(mbmi);
333
334
335
    assert(tx_size == tx_size_uv);
  }

336
  switch (tx_size) {
337
    case TX_4X4: {
338
      tx_type = (type == PLANE_TYPE_Y_WITH_DC) ?
339
          get_tx_type_4x4(xd, block) : DCT_DCT;
340
341
      above_ec = A[0] != 0;
      left_ec = L[0] != 0;
342
      seg_eob = 16;
343
      scan = get_scan_4x4(tx_type);
Paul Wilkins's avatar
Paul Wilkins committed
344
      band_translate = vp9_coefband_trans_4x4;
Daniel Kang's avatar
Daniel Kang committed
345
      break;
346
    }
347
    case TX_8X8: {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
348
349
      const TX_TYPE tx_type = type == PLANE_TYPE_Y_WITH_DC ?
                                  get_tx_type_8x8(xd) : DCT_DCT;
350
351
      above_ec = (A[0] + A[1]) != 0;
      left_ec = (L[0] + L[1]) != 0;
352
      scan = get_scan_8x8(tx_type);
353
      seg_eob = 64;
Paul Wilkins's avatar
Paul Wilkins committed
354
      band_translate = vp9_coefband_trans_8x8plus;
Daniel Kang's avatar
Daniel Kang committed
355
      break;
356
357
    }
    case TX_16X16: {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
358
359
      const TX_TYPE tx_type = type == PLANE_TYPE_Y_WITH_DC ?
                                  get_tx_type_16x16(xd) : DCT_DCT;
360
      scan = get_scan_16x16(tx_type);
361
      seg_eob = 256;
362
363
      above_ec = (A[0] + A[1] + A[2] + A[3]) != 0;
      left_ec = (L[0] + L[1] + L[2] + L[3]) != 0;
Paul Wilkins's avatar
Paul Wilkins committed
364
      band_translate = vp9_coefband_trans_8x8plus;
Daniel Kang's avatar
Daniel Kang committed
365
      break;
366
    }
367
    case TX_32X32:
Paul Wilkins's avatar
Paul Wilkins committed
368
      scan = vp9_default_scan_32x32;
369
      seg_eob = 1024;
370
371
      above_ec = (A[0] + A[1] + A[2] + A[3] + A[4] + A[5] + A[6] + A[7]) != 0;
      left_ec = (L[0] + L[1] + L[2] + L[3] + L[4] + L[5] + L[6] + L[7]) != 0;
Paul Wilkins's avatar
Paul Wilkins committed
372
      band_translate = vp9_coefband_trans_8x8plus;
373
      break;
Daniel Kang's avatar
Daniel Kang committed
374
    default:
Dmitry Kovalev's avatar
Dmitry Kovalev committed
375
      assert(0);
Daniel Kang's avatar
Daniel Kang committed
376
377
      break;
  }
John Koleszar's avatar
John Koleszar committed
378
  assert(eob <= seg_eob);
379

380
  pt = combine_entropy_contexts(above_ec, left_ec);
381
382
  nb = vp9_get_coef_neighbors_handle(scan, &pad);
  default_eob = seg_eob;
383

384
385
  if (vp9_segfeature_active(xd, segment_id, SEG_LVL_SKIP))
    seg_eob = 0;
386

387
388
389
390
  /* sanity check to ensure that we do not have spurious non-zero q values */
  if (eob < seg_eob)
    assert(qcoeff_ptr[scan[eob]] == 0);

391
  {
392
    for (c = 0; c < eob; c++) {
393
      int v = qcoeff_ptr[scan[c]];
394
      int t = vp9_dct_value_tokens_ptr[v].token;
Paul Wilkins's avatar
Paul Wilkins committed
395
      int band = get_coef_band(band_translate, c);
396
397
      if (c)
        pt = vp9_get_coef_context(scan, nb, pad, token_cache, c, default_eob);
398

399
400
401
402
      if (!c || token_cache[scan[c - 1]])  // do not skip eob
        cost += token_costs_noskip[band][pt][t] + vp9_dct_value_cost_ptr[v];
      else
        cost += token_costs[band][pt][t] + vp9_dct_value_cost_ptr[v];
403
      token_cache[scan[c]] = vp9_pt_energy_class[t];
404
    }
405
406
407
    if (c < seg_eob) {
      if (c)
        pt = vp9_get_coef_context(scan, nb, pad, token_cache, c, default_eob);
408
409
410
      cost += mb->token_costs_noskip[tx_size][type][ref]
          [get_coef_band(band_translate, c)]
          [pt][DCT_EOB_TOKEN];
411
    }
412
413
  }

414
415
416
  // is eob first coefficient;
  for (pt = 0; pt < (1 << tx_size); pt++) {
    A[pt] = L[pt] = c > 0;
417
  }
418

419
420
421
  return cost;
}

422
static void choose_txfm_size_from_rd(VP9_COMP *cpi, MACROBLOCK *x,
423
                                     int (*r)[2], int *rate,
Ronald S. Bultje's avatar
Ronald S. Bultje committed
424
                                     int64_t *d, int64_t *distortion,
425
426
427
                                     int *s, int *skip,
                                     int64_t txfm_cache[NB_TXFM_MODES],
                                     TX_SIZE max_txfm_size) {
428
429
430
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
431
  vp9_prob skip_prob = vp9_get_pred_prob(cm, xd, PRED_MBSKIP);
432
433
  int64_t rd[TX_SIZE_MAX_SB][2];
  int n, m;
434
  int s0, s1;
435

436
  const vp9_prob *tx_probs = vp9_get_pred_probs(cm, xd, PRED_TX_SIZE);
437

438
439
440
441
  for (n = TX_4X4; n <= max_txfm_size; n++) {
    r[n][1] = r[n][0];
    for (m = 0; m <= n - (n == max_txfm_size); m++) {
      if (m == n)
442
        r[n][1] += vp9_cost_zero(tx_probs[m]);
443
      else
444
        r[n][1] += vp9_cost_one(tx_probs[m]);
445
446
    }
  }
447

448
449
450
  assert(skip_prob > 0);
  s0 = vp9_cost_bit(skip_prob, 0);
  s1 = vp9_cost_bit(skip_prob, 1);
451

452
453
454
455
456
457
  for (n = TX_4X4; n <= max_txfm_size; n++) {
    if (s[n]) {
      rd[n][0] = rd[n][1] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
    } else {
      rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0] + s0, d[n]);
      rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1] + s0, d[n]);
458
459
460
    }
  }

461
462
463
464
465
466
  if (max_txfm_size == TX_32X32 &&
      (cm->txfm_mode == ALLOW_32X32 ||
       (cm->txfm_mode == TX_MODE_SELECT &&
        rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
        rd[TX_32X32][1] < rd[TX_4X4][1]))) {
    mbmi->txfm_size = TX_32X32;
467
468
469
470
471
472
  } else if (max_txfm_size >= TX_16X16 &&
             (cm->txfm_mode == ALLOW_16X16 ||
              cm->txfm_mode == ALLOW_32X32 ||
              (cm->txfm_mode == TX_MODE_SELECT &&
               rd[TX_16X16][1] < rd[TX_8X8][1] &&
               rd[TX_16X16][1] < rd[TX_4X4][1]))) {
473
    mbmi->txfm_size = TX_16X16;
474
  } else if (cm->txfm_mode == ALLOW_8X8 ||
475
476
             cm->txfm_mode == ALLOW_16X16 ||
             cm->txfm_mode == ALLOW_32X32 ||
477
           (cm->txfm_mode == TX_MODE_SELECT && rd[TX_8X8][1] < rd[TX_4X4][1])) {
478
479
480
481
482
    mbmi->txfm_size = TX_8X8;
  } else {
    mbmi->txfm_size = TX_4X4;
  }

483
  *distortion = d[mbmi->txfm_size];
484
  *rate       = r[mbmi->txfm_size][cm->txfm_mode == TX_MODE_SELECT];
485
486
  *skip       = s[mbmi->txfm_size];

487
488
  txfm_cache[ONLY_4X4] = rd[TX_4X4][0];
  txfm_cache[ALLOW_8X8] = rd[TX_8X8][0];
489
490
  txfm_cache[ALLOW_16X16] = rd[MIN(max_txfm_size, TX_16X16)][0];
  txfm_cache[ALLOW_32X32] = rd[MIN(max_txfm_size, TX_32X32)][0];
491
492
493
494
  if (max_txfm_size == TX_32X32 &&
      rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
      rd[TX_32X32][1] < rd[TX_4X4][1])
    txfm_cache[TX_MODE_SELECT] = rd[TX_32X32][1];
495
496
  else if (max_txfm_size >= TX_16X16 &&
           rd[TX_16X16][1] < rd[TX_8X8][1] && rd[TX_16X16][1] < rd[TX_4X4][1])
497
    txfm_cache[TX_MODE_SELECT] = rd[TX_16X16][1];
498
  else
499
500
    txfm_cache[TX_MODE_SELECT] = rd[TX_4X4][1] < rd[TX_8X8][1] ?
                                 rd[TX_4X4][1] : rd[TX_8X8][1];
501
502
}

Ronald S. Bultje's avatar
Ronald S. Bultje committed
503
504
static int64_t block_error_sby(MACROBLOCK *x, BLOCK_SIZE_TYPE bsize,
                               int shift) {
505
  const int bwl = b_width_log2(bsize), bhl = b_height_log2(bsize);
Ronald S. Bultje's avatar
Ronald S. Bultje committed
506
507
  return vp9_block_error(x->plane[0].coeff, x->e_mbd.plane[0].dqcoeff,
                         16 << (bwl + bhl)) >> shift;
508
}
509

Ronald S. Bultje's avatar
Ronald S. Bultje committed
510
511
static int64_t block_error_sbuv(MACROBLOCK *x, BLOCK_SIZE_TYPE bsize,
                                int shift) {
512
513
514
  const int bwl = b_width_log2(bsize), bhl = b_height_log2(bsize);
  int64_t sum = 0;
  int plane;
515

516
517
518
  for (plane = 1; plane < MAX_MB_PLANE; plane++) {
    const int subsampling = x->e_mbd.plane[plane].subsampling_x +
                            x->e_mbd.plane[plane].subsampling_y;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
519
520
    sum += vp9_block_error(x->plane[plane].coeff, x->e_mbd.plane[plane].dqcoeff,
                           16 << (bwl + bhl - subsampling));
521
  }
Ronald S. Bultje's avatar
Ronald S. Bultje committed
522
  return sum >> shift;
523
524
}

525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
struct rdcost_block_args {
  VP9_COMMON *cm;
  MACROBLOCK *x;
  ENTROPY_CONTEXT t_above[16];
  ENTROPY_CONTEXT t_left[16];
  TX_SIZE tx_size;
  int bw;
  int bh;
  int cost;
};

static void rdcost_block(int plane, int block, BLOCK_SIZE_TYPE bsize,
                         int ss_txfrm_size, void *arg) {
  struct rdcost_block_args* args = arg;
  int x_idx, y_idx;
  MACROBLOCKD * const xd = &args->x->e_mbd;

  txfrm_block_to_raster_xy(xd, bsize, plane, block, args->tx_size * 2, &x_idx,
                           &y_idx);

  args->cost += cost_coeffs(args->cm, args->x, plane, block,
                            xd->plane[plane].plane_type, args->t_above + x_idx,
                            args->t_left + y_idx, args->tx_size,
                            args->bw * args->bh);
}

static int rdcost_plane(VP9_COMMON * const cm, MACROBLOCK *x, int plane,
                        BLOCK_SIZE_TYPE bsize, TX_SIZE tx_size) {
  MACROBLOCKD * const xd = &x->e_mbd;
554
555
556
  const int bwl = b_width_log2(bsize) - xd->plane[plane].subsampling_x;
  const int bhl = b_height_log2(bsize) - xd->plane[plane].subsampling_y;
  const int bw = 1 << bwl, bh = 1 << bhl;
557
  struct rdcost_block_args args = { cm, x, { 0 }, { 0 }, tx_size, bw, bh, 0 };
558

559
  vpx_memcpy(&args.t_above, xd->plane[plane].above_context,
560
             sizeof(ENTROPY_CONTEXT) * bw);
561
  vpx_memcpy(&args.t_left, xd->plane[plane].left_context,
562
             sizeof(ENTROPY_CONTEXT) * bh);
563

564
  foreach_transformed_block_in_plane(xd, bsize, plane, rdcost_block, &args);
565

566
  return args.cost;
567
568
}

569
570
571
static int rdcost_uv(VP9_COMMON *const cm, MACROBLOCK *x,
                     BLOCK_SIZE_TYPE bsize, TX_SIZE tx_size) {
  int cost = 0, plane;
572

573
574
  for (plane = 1; plane < MAX_MB_PLANE; plane++) {
    cost += rdcost_plane(cm, x, plane, bsize, tx_size);
575
576
  }
  return cost;
577
578
}

579
static void super_block_yrd_for_txfm(VP9_COMMON *const cm, MACROBLOCK *x,
Ronald S. Bultje's avatar
Ronald S. Bultje committed
580
581
                                     int *rate, int64_t *distortion,
                                     int *skippable,
582
                                     BLOCK_SIZE_TYPE bsize, TX_SIZE tx_size) {
583
  MACROBLOCKD *const xd = &x->e_mbd;
584
  xd->mode_info_context->mbmi.txfm_size = tx_size;
585

Ronald S. Bultje's avatar
Ronald S. Bultje committed
586
  if (xd->mode_info_context->mbmi.ref_frame[0] == INTRA_FRAME)
587
588
589
    vp9_encode_intra_block_y(cm, x, bsize);
  else
    vp9_xform_quant_sby(cm, x, bsize);
590

591
592
  *distortion = block_error_sby(x, bsize, tx_size == TX_32X32 ? 0 : 2);
  *rate       = rdcost_plane(cm, x, 0, bsize, tx_size);
593
  *skippable  = vp9_sby_is_skippable(xd, bsize);
594
595
}

596
static void super_block_yrd(VP9_COMP *cpi,
Ronald S. Bultje's avatar
Ronald S. Bultje committed
597
                            MACROBLOCK *x, int *rate, int64_t *distortion,
598
                            int *skip, BLOCK_SIZE_TYPE bs,
599
                            int64_t txfm_cache[NB_TXFM_MODES]) {
600
  VP9_COMMON *const cm = &cpi->common;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
601
602
  int r[TX_SIZE_MAX_SB][2], s[TX_SIZE_MAX_SB];
  int64_t d[TX_SIZE_MAX_SB];
Jim Bankoski's avatar
Jim Bankoski committed
603
604
  MACROBLOCKD *xd = &x->e_mbd;
  MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
605

606
  assert(bs == mbmi->sb_type);
Ronald S. Bultje's avatar
Ronald S. Bultje committed
607
  if (mbmi->ref_frame[0] > INTRA_FRAME)
608
    vp9_subtract_sby(x, bs);
Ronald S. Bultje's avatar
Ronald S. Bultje committed
609

610
  if (cpi->sf.use_largest_txform) {
Jim Bankoski's avatar
Jim Bankoski committed
611
612
613
614
615
616
617
618
619
    if (bs >= BLOCK_SIZE_SB32X32) {
      mbmi->txfm_size = TX_32X32;
    } else if (bs >= BLOCK_SIZE_MB16X16) {
      mbmi->txfm_size = TX_16X16;
    } else if (bs >= BLOCK_SIZE_SB8X8) {
      mbmi->txfm_size = TX_8X8;
    } else {
      mbmi->txfm_size = TX_4X4;
    }
620
    vpx_memset(txfm_cache, 0, NB_TXFM_MODES * sizeof(int64_t));
Jim Bankoski's avatar
Jim Bankoski committed
621
622
623
624
    super_block_yrd_for_txfm(cm, x, rate, distortion, skip, bs,
                             mbmi->txfm_size);
    return;
  }
625
  if (bs >= BLOCK_SIZE_SB32X32)
626
627
    super_block_yrd_for_txfm(cm, x, &r[TX_32X32][0], &d[TX_32X32], &s[TX_32X32],
                             bs, TX_32X32);
628
  if (bs >= BLOCK_SIZE_MB16X16)
629
630
631
632
633
634
    super_block_yrd_for_txfm(cm, x, &r[TX_16X16][0], &d[TX_16X16], &s[TX_16X16],
                             bs, TX_16X16);
  super_block_yrd_for_txfm(cm, x, &r[TX_8X8][0], &d[TX_8X8], &s[TX_8X8], bs,
                           TX_8X8);
  super_block_yrd_for_txfm(cm, x, &r[TX_4X4][0], &d[TX_4X4], &s[TX_4X4], bs,
                           TX_4X4);
Ronald S. Bultje's avatar
Ronald S. Bultje committed
635

636
  choose_txfm_size_from_rd(cpi, x, r, rate, d, distortion, s,
637
                           skip, txfm_cache,
638
                           TX_32X32 - (bs < BLOCK_SIZE_SB32X32)
Jingning Han's avatar
Jingning Han committed
639
                           - (bs < BLOCK_SIZE_MB16X16));
Ronald S. Bultje's avatar
Ronald S. Bultje committed
640
}
Ronald S. Bultje's avatar
Ronald S. Bultje committed
641

642
static int64_t rd_pick_intra4x4block(VP9_COMP *cpi, MACROBLOCK *x, int ib,
Yaowu Xu's avatar
Yaowu Xu committed
643
                                     MB_PREDICTION_MODE *best_mode,
644
645
646
                                     int *bmode_costs,
                                     ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l,
                                     int *bestrate, int *bestratey,
Ronald S. Bultje's avatar
Ronald S. Bultje committed
647
                                     int64_t *bestdistortion,
648
649
                                     BLOCK_SIZE_TYPE bsize) {
  MB_PREDICTION_MODE mode;
Deb Mukherjee's avatar
Deb Mukherjee committed
650
  MACROBLOCKD *xd = &x->e_mbd;
651
  int64_t best_rd = INT64_MAX;
John Koleszar's avatar
John Koleszar committed
652
  int rate = 0;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
653
  int64_t distortion;
654
  VP9_COMMON *const cm = &cpi->common;
John Koleszar's avatar
John Koleszar committed
655
  const int src_stride = x->plane[0].src.stride;
656
657
658
659
660
  uint8_t *src, *dst;
  int16_t *src_diff, *coeff;

  ENTROPY_CONTEXT ta[2], tempa[2];
  ENTROPY_CONTEXT tl[2], templ[2];
Deb Mukherjee's avatar
Deb Mukherjee committed
661
662
  TX_TYPE tx_type = DCT_DCT;
  TX_TYPE best_tx_type = DCT_DCT;
663
664
665
666
  int bw = 1 << b_width_log2(bsize);
  int bh = 1 << b_height_log2(bsize);
  int idx, idy, block;
  DECLARE_ALIGNED(16, int16_t, best_dqcoeff[4][16]);
John Koleszar's avatar
John Koleszar committed
667

Jingning Han's avatar
Jingning Han committed
668
  assert(ib < 4);
669

670
671
  vpx_memcpy(ta, a, sizeof(ta));
  vpx_memcpy(tl, l, sizeof(tl));
672
  xd->mode_info_context->mbmi.txfm_size = TX_4X4;
673
674

  for (mode = DC_PRED; mode <= TM_PRED; ++mode) {
675
    int64_t this_rd;
676
    int ratey = 0;
677

678
    rate = bmode_costs[mode];
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
    distortion = 0;

    vpx_memcpy(tempa, ta, sizeof(ta));
    vpx_memcpy(templ, tl, sizeof(tl));

    for (idy = 0; idy < bh; ++idy) {
      for (idx = 0; idx < bw; ++idx) {
        block = ib + idy * 2 + idx;
        xd->mode_info_context->bmi[block].as_mode.first = mode;
        src = raster_block_offset_uint8(xd, BLOCK_SIZE_SB8X8, 0, block,
                                        x->plane[0].src.buf, src_stride);
        src_diff = raster_block_offset_int16(xd, BLOCK_SIZE_SB8X8, 0, block,
                                             x->plane[0].src_diff);
        coeff = BLOCK_OFFSET(x->plane[0].coeff, block, 16);
        dst = raster_block_offset_uint8(xd, BLOCK_SIZE_SB8X8, 0, block,
                                        xd->plane[0].dst.buf,
                                        xd->plane[0].dst.stride);
Jingning Han's avatar
Jingning Han committed
696
697
        vp9_predict_intra_block(xd, block, b_width_log2(BLOCK_SIZE_SB8X8),
                                TX_4X4, mode, dst, xd->plane[0].dst.stride);
698
699
700
701
702
703
704
705
706
707
708
709
        vp9_subtract_block(4, 4, src_diff, 8,
                           src, src_stride,
                           dst, xd->plane[0].dst.stride);

        tx_type = get_tx_type_4x4(xd, block);
        if (tx_type != DCT_DCT) {
          vp9_short_fht4x4(src_diff, coeff, 8, tx_type);
          x->quantize_b_4x4(x, block, tx_type, 16);
        } else {
          x->fwd_txm4x4(src_diff, coeff, 16);
          x->quantize_b_4x4(x, block, tx_type, 16);
        }
John Koleszar's avatar
John Koleszar committed
710

711
712
713
714
        ratey += cost_coeffs(cm, x, 0, block, PLANE_TYPE_Y_WITH_DC,
                             tempa + idx, templ + idy, TX_4X4, 16);
        distortion += vp9_block_error(coeff, BLOCK_OFFSET(xd->plane[0].dqcoeff,
                                                         block, 16), 16) >> 2;
John Koleszar's avatar
John Koleszar committed
715

716
717
718
719
720
721
722
723
        if (best_tx_type != DCT_DCT)
          vp9_short_iht4x4_add(BLOCK_OFFSET(xd->plane[0].dqcoeff, block, 16),
                               dst, xd->plane[0].dst.stride, best_tx_type);
        else
          xd->inv_txm4x4_add(BLOCK_OFFSET(xd->plane[0].dqcoeff, block, 16),
                             dst, xd->plane[0].dst.stride);
      }
    }
Jingning Han's avatar
Jingning Han committed
724

725
726
    rate += ratey;
    this_rd = RDCOST(x->rdmult, x->rddiv, rate, distortion);
Jingning Han's avatar
Jingning Han committed
727

728
729
730
731
732
733
734
    if (this_rd < best_rd) {
      *bestrate = rate;
      *bestratey = ratey;
      *bestdistortion = distortion;
      best_rd = this_rd;
      *best_mode = mode;
      best_tx_type = tx_type;
735
736
737
738
739
740
741
742
743
744
      vpx_memcpy(a, tempa, sizeof(tempa));
      vpx_memcpy(l, templ, sizeof(templ));
      for (idy = 0; idy < bh; ++idy) {
        for (idx = 0; idx < bw; ++idx) {
          block = ib + idy * 2 + idx;
          vpx_memcpy(best_dqcoeff[idy * 2 + idx],
                     BLOCK_OFFSET(xd->plane[0].dqcoeff, block, 16),
                     sizeof(best_dqcoeff[0]));
        }
      }
John Koleszar's avatar
John Koleszar committed
745
    }
John Koleszar's avatar
John Koleszar committed
746
  }
747
748
749
750
751
752
753
754
755

  for (idy = 0; idy < bh; ++idy) {
    for (idx = 0; idx < bw; ++idx) {
      block = ib + idy * 2 + idx;
      xd->mode_info_context->bmi[block].as_mode.first = *best_mode;
      dst = raster_block_offset_uint8(xd, BLOCK_SIZE_SB8X8, 0, block,
                                      xd->plane[0].dst.buf,
                                      xd->plane[0].dst.stride);

Jingning Han's avatar
Jingning Han committed
756
757
      vp9_predict_intra_block(xd, block, b_width_log2(BLOCK_SIZE_SB8X8), TX_4X4,
                              *best_mode, dst, xd->plane[0].dst.stride);
758
759
760
761
762
763
764
765
      // inverse transform
      if (best_tx_type != DCT_DCT)
        vp9_short_iht4x4_add(best_dqcoeff[idy * 2 + idx], dst,
                             xd->plane[0].dst.stride, best_tx_type);
      else
        xd->inv_txm4x4_add(best_dqcoeff[idy * 2 + idx], dst,
                           xd->plane[0].dst.stride);
    }
Scott LaVarnway's avatar
Scott LaVarnway committed
766
  }
John Koleszar's avatar
John Koleszar committed
767

John Koleszar's avatar
John Koleszar committed
768
  return best_rd;
John Koleszar's avatar
John Koleszar committed
769
770
}

771
772
static int64_t rd_pick_intra4x4mby_modes(VP9_COMP *cpi, MACROBLOCK *mb,
                                         int *Rate, int *rate_y,
Ronald S. Bultje's avatar
Ronald S. Bultje committed
773
                                         int64_t *Distortion, int64_t best_rd) {
774
  int i, j;
John Koleszar's avatar
John Koleszar committed
775
  MACROBLOCKD *const xd = &mb->e_mbd;
776
777
778
779
  BLOCK_SIZE_TYPE bsize = xd->mode_info_context->mbmi.sb_type;
  int bw = 1 << b_width_log2(bsize);
  int bh = 1 << b_height_log2(bsize);
  int idx, idy;
780
  int cost = 0;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
781
  int64_t distortion = 0;
John Koleszar's avatar
John Koleszar committed
782
783
  int tot_rate_y = 0;
  int64_t total_rd = 0;
784
  ENTROPY_CONTEXT t_above[4], t_left[4];
John Koleszar's avatar
John Koleszar committed
785
  int *bmode_costs;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
786
  MODE_INFO *const mic = xd->mode_info_context;
John Koleszar's avatar
John Koleszar committed
787

788
789
  vpx_memcpy(t_above, xd->plane[0].above_context, sizeof(t_above));
  vpx_memcpy(t_left, xd->plane[0].left_context, sizeof(t_left));
John Koleszar's avatar
John Koleszar committed
790

791
  bmode_costs = mb->mbmode_cost;
792

793
794
795
  for (idy = 0; idy < 2; idy += bh) {
    for (idx = 0; idx < 2; idx += bw) {
      const int mis = xd->mode_info_stride;
Yaowu Xu's avatar
Yaowu Xu committed
796
      MB_PREDICTION_MODE UNINITIALIZED_IS_SAFE(best_mode);
797
      int UNINITIALIZED_IS_SAFE(r), UNINITIALIZED_IS_SAFE(ry);
Ronald S. Bultje's avatar
Ronald S. Bultje committed
798
      int64_t UNINITIALIZED_IS_SAFE(d);
799
800
801
802
803
804
805
      i = idy * 2 + idx;

      if (xd->frame_type == KEY_FRAME) {
        const MB_PREDICTION_MODE A = above_block_mode(mic, i, mis);
        const MB_PREDICTION_MODE L = (xd->left_available || idx) ?
                                     left_block_mode(mic, i) : DC_PRED;

806
        bmode_costs  = mb->y_mode_costs[A][L];
807
      }
808

809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
      total_rd += rd_pick_intra4x4block(cpi, mb, i, &best_mode, bmode_costs,
                                        t_above + idx, t_left + idy,
                                        &r, &ry, &d, bsize);
      cost += r;
      distortion += d;
      tot_rate_y += ry;

      mic->bmi[i].as_mode.first = best_mode;
      for (j = 1; j < bh; ++j)
        mic->bmi[i + j * 2].as_mode.first = best_mode;
      for (j = 1; j < bw; ++j)
        mic->bmi[i + j].as_mode.first = best_mode;

      if (total_rd >= best_rd)
        break;
John Koleszar's avatar
John Koleszar committed
824
825
    }
  }
John Koleszar's avatar
John Koleszar committed
826

827
  if (total_rd >= best_rd)
828
    return INT64_MAX;
829

John Koleszar's avatar
John Koleszar committed
830
  *Rate = cost;
831
  *rate_y = tot_rate_y;
John Koleszar's avatar
John Koleszar committed
832
  *Distortion = distortion;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
833
  xd->mode_info_context->mbmi.mode = mic->bmi[3].as_mode.first;
John Koleszar's avatar
John Koleszar committed
834

John Koleszar's avatar
John Koleszar committed
835
  return RDCOST(mb->rdmult, mb->rddiv, cost, distortion);
John Koleszar's avatar
John Koleszar committed
836
}
837

838
839
static int64_t rd_pick_intra_sby_mode(VP9_COMP *cpi, MACROBLOCK *x,
                                      int *rate, int *rate_tokenonly,
Ronald S. Bultje's avatar
Ronald S. Bultje committed
840
                                      int64_t *distortion, int *skippable,
841
                                      BLOCK_SIZE_TYPE bsize,
842
                                      int64_t txfm_cache[NB_TXFM_MODES]) {
Ronald S. Bultje's avatar
Ronald S. Bultje committed
843
844
  MB_PREDICTION_MODE mode;
  MB_PREDICTION_MODE UNINITIALIZED_IS_SAFE(mode_selected);
Jim Bankoski's avatar
Jim Bankoski committed
845
  MACROBLOCKD *const xd = &x->e_mbd;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
846
847
  int this_rate, this_rate_tokenonly, s;
  int64_t this_distortion;
848
  int64_t best_rd = INT64_MAX, this_rd;
849
850
  TX_SIZE UNINITIALIZED_IS_SAFE(best_tx);
  int i;
851
  int *bmode_costs = x->mbmode_cost;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
852

853
854
855
856
857
  if (bsize < BLOCK_SIZE_SB8X8) {
    x->e_mbd.mode_info_context->mbmi.txfm_size = TX_4X4;
    return best_rd;
  }

858
859
  for (i = 0; i < NB_TXFM_MODES; i++)
    txfm_cache[i] = INT64_MAX;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
860
861
862

  /* Y Search for 32x32 intra prediction mode */
  for (mode = DC_PRED; mode <= TM_PRED; mode++) {
863
    int64_t local_txfm_cache[NB_TXFM_MODES];
864
865
    MODE_INFO *const mic = xd->mode_info_context;
    const int mis = xd->mode_info_stride;
866

867
868
869
870
871
872
873
    if (cpi->common.frame_type == KEY_FRAME) {
      const MB_PREDICTION_MODE A = above_block_mode(mic, 0, mis);
      const MB_PREDICTION_MODE L = xd->left_available ?
                                   left_block_mode(mic, 0) : DC_PRED;

      bmode_costs = x->y_mode_costs[A][L];
    }
Ronald S. Bultje's avatar
Ronald S. Bultje committed
874
875
    x->e_mbd.mode_info_context->mbmi.mode = mode;

876
877
    super_block_yrd(cpi, x, &this_rate_tokenonly, &this_distortion, &s,
                    bsize, local_txfm_cache);
Jim Bankoski's avatar
Jim Bankoski committed
878

879
    this_rate = this_rate_tokenonly + bmode_costs[mode];
Ronald S. Bultje's avatar
Ronald S. Bultje committed
880
881
882
883
884
    this_rd = RDCOST(x->rdmult, x->rddiv, this_rate, this_distortion);

    if (this_rd < best_rd) {
      mode_selected   = mode;
      best_rd         = this_rd;
885
      best_tx         = x->e_mbd.mode_info_context->mbmi.txfm_size;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
886
887
888
889
890
      *rate           = this_rate;
      *rate_tokenonly = this_rate_tokenonly;
      *distortion     = this_distortion;
      *skippable      = s;
    }
891

892
893
    for (i = 0; i < NB_TXFM_MODES; i++) {
      int64_t adj_rd = this_rd + local_txfm_cache[i] -
894
                       local_txfm_cache[cpi->common.txfm_mode];
895
896
      if (adj_rd < txfm_cache[i]) {