vp9_rdopt.c 112 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5
6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9
10
11
12
13
14
15
 */


#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
16

17
#include "vp9/common/vp9_pragmas.h"
18
19
20
21
22
#include "vp9/encoder/vp9_tokenize.h"
#include "vp9/encoder/vp9_treewriter.h"
#include "vp9/encoder/vp9_onyx_int.h"
#include "vp9/encoder/vp9_modecosts.h"
#include "vp9/encoder/vp9_encodeintra.h"
23
24
25
26
27
#include "vp9/common/vp9_entropymode.h"
#include "vp9/common/vp9_reconinter.h"
#include "vp9/common/vp9_reconintra.h"
#include "vp9/common/vp9_findnearmv.h"
#include "vp9/common/vp9_quant_common.h"
28
29
30
31
32
33
#include "vp9/encoder/vp9_encodemb.h"
#include "vp9/encoder/vp9_quantize.h"
#include "vp9/encoder/vp9_variance.h"
#include "vp9/encoder/vp9_mcomp.h"
#include "vp9/encoder/vp9_rdopt.h"
#include "vp9/encoder/vp9_ratectrl.h"
John Koleszar's avatar
John Koleszar committed
34
#include "vpx_mem/vpx_mem.h"
35
36
37
38
39
#include "vp9/common/vp9_systemdependent.h"
#include "vp9/encoder/vp9_encodemv.h"
#include "vp9/common/vp9_seg_common.h"
#include "vp9/common/vp9_pred_common.h"
#include "vp9/common/vp9_entropy.h"
40
#include "vp9_rtcd.h"
41
#include "vp9/common/vp9_mvref_common.h"
Ronald S. Bultje's avatar
Ronald S. Bultje committed
42
#include "vp9/common/vp9_common.h"
Paul Wilkins's avatar
Paul Wilkins committed
43

44
45
#define INVALID_MV 0x80008000

46
47
48
/* Factor to weigh the rate for switchable interp filters */
#define SWITCHABLE_INTERP_RATE_FACTOR 1

49
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
50
51
  {ZEROMV,    LAST_FRAME,   NONE},
  {DC_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
52

53
54
  {NEARESTMV, LAST_FRAME,   NONE},
  {NEARMV,    LAST_FRAME,   NONE},
John Koleszar's avatar
John Koleszar committed
55

56
57
  {ZEROMV,    GOLDEN_FRAME, NONE},
  {NEARESTMV, GOLDEN_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
58

59
60
  {ZEROMV,    ALTREF_FRAME, NONE},
  {NEARESTMV, ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
61

62
63
  {NEARMV,    GOLDEN_FRAME, NONE},
  {NEARMV,    ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
64

65
66
67
68
69
70
71
72
  {V_PRED,    INTRA_FRAME,  NONE},
  {H_PRED,    INTRA_FRAME,  NONE},
  {D45_PRED,  INTRA_FRAME,  NONE},
  {D135_PRED, INTRA_FRAME,  NONE},
  {D117_PRED, INTRA_FRAME,  NONE},
  {D153_PRED, INTRA_FRAME,  NONE},
  {D27_PRED,  INTRA_FRAME,  NONE},
  {D63_PRED,  INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
73

74
  {TM_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
75

76
77
78
  {NEWMV,     LAST_FRAME,   NONE},
  {NEWMV,     GOLDEN_FRAME, NONE},
  {NEWMV,     ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
79

80
81
82
  {SPLITMV,   LAST_FRAME,   NONE},
  {SPLITMV,   GOLDEN_FRAME, NONE},
  {SPLITMV,   ALTREF_FRAME, NONE},
83

Yaowu Xu's avatar
Yaowu Xu committed
84
  {I4X4_PRED,    INTRA_FRAME,  NONE},
85

John Koleszar's avatar
John Koleszar committed
86
87
88
89
  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME},
90

John Koleszar's avatar
John Koleszar committed
91
92
93
  {ZEROMV,    ALTREF_FRAME, LAST_FRAME},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME},
94

John Koleszar's avatar
John Koleszar committed
95
96
97
  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME},
98

John Koleszar's avatar
John Koleszar committed
99
100
101
  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME  },
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME},
102

John Koleszar's avatar
John Koleszar committed
103
104
  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME  },
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME},

#if CONFIG_COMP_INTERINTRA_PRED
  /* compound inter-intra prediction */
  {ZEROMV,    LAST_FRAME,   INTRA_FRAME},
  {NEARESTMV, LAST_FRAME,   INTRA_FRAME},
  {NEARMV,    LAST_FRAME,   INTRA_FRAME},
  {NEWMV,     LAST_FRAME,   INTRA_FRAME},

  {ZEROMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEARESTMV, GOLDEN_FRAME,   INTRA_FRAME},
  {NEARMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEWMV,     GOLDEN_FRAME,   INTRA_FRAME},

  {ZEROMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEARESTMV, ALTREF_FRAME,   INTRA_FRAME},
  {NEARMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEWMV,     ALTREF_FRAME,   INTRA_FRAME},
#endif
John Koleszar's avatar
John Koleszar committed
124
125
};

126
127
static void fill_token_costs(vp9_coeff_count *c,
                             vp9_coeff_probs *p,
128
                             TX_SIZE tx_size) {
129
  int i, j, k, l;
John Koleszar's avatar
John Koleszar committed
130

131
  for (i = 0; i < BLOCK_TYPES; i++)
132
133
    for (j = 0; j < REF_TYPES; j++)
      for (k = 0; k < COEF_BANDS; k++)
134
135
        for (l = 0; l < PREV_COEF_CONTEXTS; l++)
          vp9_cost_tokens_skip((int *)c[i][j][k][l], p[i][j][k][l],
136
137
138
                               vp9_coef_tree);
}

139
140
141
142
static int rd_iifactor[32] =  { 4, 4, 3, 2, 1, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0, };
John Koleszar's avatar
John Koleszar committed
143

144
// 3* dc_qlookup[Q]*dc_qlookup[Q];
145

146
/* values are now correlated to quantizer */
Paul Wilkins's avatar
Paul Wilkins committed
147
148
149
static int sad_per_bit16lut[QINDEX_RANGE];
static int sad_per_bit4lut[QINDEX_RANGE];

150
void vp9_init_me_luts() {
John Koleszar's avatar
John Koleszar committed
151
152
153
154
155
156
157
  int i;

  // Initialize the sad lut tables using a formulaic calculation for now
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < QINDEX_RANGE; i++) {
    sad_per_bit16lut[i] =
158
      (int)((0.0418 * vp9_convert_qindex_to_q(i)) + 2.4107);
159
    sad_per_bit4lut[i] = (int)(0.063 * vp9_convert_qindex_to_q(i) + 2.742);
John Koleszar's avatar
John Koleszar committed
160
  }
Paul Wilkins's avatar
Paul Wilkins committed
161
}
John Koleszar's avatar
John Koleszar committed
162

163
static int compute_rd_mult(int qindex) {
164
  const int q = vp9_dc_quant(qindex, 0);
165
  return (11 * q * q) >> 2;
166
167
}

Dmitry Kovalev's avatar
Dmitry Kovalev committed
168
169
170
void vp9_initialize_me_consts(VP9_COMP *cpi, int qindex) {
  cpi->mb.sadperbit16 = sad_per_bit16lut[qindex];
  cpi->mb.sadperbit4 = sad_per_bit4lut[qindex];
John Koleszar's avatar
John Koleszar committed
171
172
}

173

Dmitry Kovalev's avatar
Dmitry Kovalev committed
174
void vp9_initialize_rd_consts(VP9_COMP *cpi, int qindex) {
175
  int q, i;
John Koleszar's avatar
John Koleszar committed
176

177
  vp9_clear_system_state();  // __asm emms;
John Koleszar's avatar
John Koleszar committed
178

John Koleszar's avatar
John Koleszar committed
179
180
181
182
  // Further tests required to see if optimum is different
  // for key frames, golden frames and arf frames.
  // if (cpi->common.refresh_golden_frame ||
  //     cpi->common.refresh_alt_ref_frame)
183
  qindex = clamp(qindex, 0, MAXQ);
184

Dmitry Kovalev's avatar
Dmitry Kovalev committed
185
  cpi->RDMULT = compute_rd_mult(qindex);
John Koleszar's avatar
John Koleszar committed
186
187
188
189
190
  if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    if (cpi->twopass.next_iiratio > 31)
      cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
    else
      cpi->RDMULT +=
Dmitry Kovalev's avatar
Dmitry Kovalev committed
191
          (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
John Koleszar's avatar
John Koleszar committed
192
  }
193
  cpi->mb.errorperbit = cpi->RDMULT >> 6;
John Koleszar's avatar
John Koleszar committed
194
  cpi->mb.errorperbit += (cpi->mb.errorperbit == 0);
195

196
  vp9_set_speed_features(cpi);
John Koleszar's avatar
John Koleszar committed
197

Dmitry Kovalev's avatar
Dmitry Kovalev committed
198
199
  q = (int)pow(vp9_dc_quant(qindex, 0) >> 2, 1.25);
  q <<= 2;
John Koleszar's avatar
John Koleszar committed
200
201
  if (q < 8)
    q = 8;
202

John Koleszar's avatar
John Koleszar committed
203
204
205
  if (cpi->RDMULT > 1000) {
    cpi->RDDIV = 1;
    cpi->RDMULT /= 100;
John Koleszar's avatar
John Koleszar committed
206

John Koleszar's avatar
John Koleszar committed
207
208
209
210
211
212
213
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < INT_MAX) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q / 100;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
214
    }
John Koleszar's avatar
John Koleszar committed
215
216
  } else {
    cpi->RDDIV = 100;
John Koleszar's avatar
John Koleszar committed
217

John Koleszar's avatar
John Koleszar committed
218
219
220
221
222
223
224
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < (INT_MAX / q)) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
225
    }
John Koleszar's avatar
John Koleszar committed
226
  }
John Koleszar's avatar
John Koleszar committed
227

228
  fill_token_costs(cpi->mb.token_costs[TX_4X4],
229
                   cpi->common.fc.coef_probs_4x4, TX_4X4);
230
  fill_token_costs(cpi->mb.token_costs[TX_8X8],
231
                   cpi->common.fc.coef_probs_8x8, TX_8X8);
232
  fill_token_costs(cpi->mb.token_costs[TX_16X16],
233
                   cpi->common.fc.coef_probs_16x16, TX_16X16);
234
  fill_token_costs(cpi->mb.token_costs[TX_32X32],
235
                   cpi->common.fc.coef_probs_32x32, TX_32X32);
236

237
  for (i = 0; i < NUM_PARTITION_CONTEXTS; i++)
238
239
240
241
    vp9_cost_tokens(cpi->mb.partition_cost[i],
                    cpi->common.fc.partition_prob[i],
                    vp9_partition_tree);

John Koleszar's avatar
John Koleszar committed
242
243
  /*rough estimate for costing*/
  cpi->common.kf_ymode_probs_index = cpi->common.base_qindex >> 4;
244
  vp9_init_mode_costs(cpi);
John Koleszar's avatar
John Koleszar committed
245

246
  if (cpi->common.frame_type != KEY_FRAME) {
247
    vp9_build_nmv_cost_table(
248
249
250
251
252
253
        cpi->mb.nmvjointcost,
        cpi->mb.e_mbd.allow_high_precision_mv ?
        cpi->mb.nmvcost_hp : cpi->mb.nmvcost,
        &cpi->common.fc.nmvc,
        cpi->mb.e_mbd.allow_high_precision_mv, 1, 1);
  }
John Koleszar's avatar
John Koleszar committed
254
255
}

256
int vp9_block_error_c(int16_t *coeff, int16_t *dqcoeff, int block_size) {
257
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
258

259
  for (i = 0; i < block_size; i++) {
John Koleszar's avatar
John Koleszar committed
260
261
262
    int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }
John Koleszar's avatar
John Koleszar committed
263

John Koleszar's avatar
John Koleszar committed
264
  return error;
John Koleszar's avatar
John Koleszar committed
265
266
}

267
static INLINE int cost_coeffs(VP9_COMMON *const cm, MACROBLOCK *mb,
268
                              int plane, int block, PLANE_TYPE type,
269
270
                              ENTROPY_CONTEXT *A,
                              ENTROPY_CONTEXT *L,
John Koleszar's avatar
John Koleszar committed
271
272
                              TX_SIZE tx_size,
                              int y_blocks) {
273
  MACROBLOCKD *const xd = &mb->e_mbd;
274
275
  MB_MODE_INFO *mbmi = &xd->mode_info_context->mbmi;
  int pt;
276
  int c = 0;
277
278
  int cost = 0, pad;
  const int *scan, *nb;
279
280
281
  const int eob = xd->plane[plane].eobs[block];
  const int16_t *qcoeff_ptr = BLOCK_OFFSET(xd->plane[plane].qcoeff,
                                           block, 16);
282
  const int ref = mbmi->ref_frame != INTRA_FRAME;
283
  unsigned int (*token_costs)[PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS] =
284
      mb->token_costs[tx_size][type][ref];
285
  ENTROPY_CONTEXT above_ec, left_ec;
286
  TX_TYPE tx_type = DCT_DCT;
287

288
289
290
291
292
293
294
295
#if CONFIG_CODE_ZEROGROUP
  int last_nz_pos[3] = {-1, -1, -1};  // Encoder only
  int is_eoo_list[3] = {0, 0, 0};
  int is_eoo_negative[3] = {0, 0, 0};
  int is_last_zero[3] = {0, 0, 0};
  int o, rc, skip_coef_val;
  vp9_zpc_probs *zpc_probs;
  uint8_t token_cache_full[1024];
296
#endif
297
  const int segment_id = xd->mode_info_context->mbmi.segment_id;
298
299
300
301
  vp9_prob (*coef_probs)[REF_TYPES][COEF_BANDS][PREV_COEF_CONTEXTS]
                        [ENTROPY_NODES];
  int seg_eob, default_eob;
  uint8_t token_cache[1024];
302

303
304
305
306
#if CONFIG_CODE_ZEROGROUP
  vpx_memset(token_cache, UNKNOWN_TOKEN, sizeof(token_cache));
#endif

307
  // Check for consistency of tx_size with mode info
308
  assert((!type && !plane) || (type && plane));
309
310
311
312
313
314
315
  if (type == PLANE_TYPE_Y_WITH_DC) {
    assert(xd->mode_info_context->mbmi.txfm_size == tx_size);
  } else {
    TX_SIZE tx_size_uv = get_uv_tx_size(xd);
    assert(tx_size == tx_size_uv);
  }

316
  switch (tx_size) {
317
    case TX_4X4: {
318
      tx_type = (type == PLANE_TYPE_Y_WITH_DC) ?
319
          get_tx_type_4x4(xd, block) : DCT_DCT;
320
321
      above_ec = A[0] != 0;
      left_ec = L[0] != 0;
322
      coef_probs = cm->fc.coef_probs_4x4;
323
      seg_eob = 16;
324
      scan = get_scan_4x4(tx_type);
325
326
327
#if CONFIG_CODE_ZEROGROUP
      zpc_probs = &cm->fc.zpc_probs_4x4;
#endif
Daniel Kang's avatar
Daniel Kang committed
328
      break;
329
    }
330
331
    case TX_8X8: {
      const BLOCK_SIZE_TYPE sb_type = xd->mode_info_context->mbmi.sb_type;
332
      const int sz = 1 + b_width_log2(sb_type);
333
      const int x = block & ((1 << sz) - 1), y = block - x;
334
335
      TX_TYPE tx_type = (type == PLANE_TYPE_Y_WITH_DC) ?
          get_tx_type_8x8(xd, y + (x >> 1)) : DCT_DCT;
336
337
      above_ec = (A[0] + A[1]) != 0;
      left_ec = (L[0] + L[1]) != 0;
338
      scan = get_scan_8x8(tx_type);
339
      coef_probs = cm->fc.coef_probs_8x8;
340
      seg_eob = 64;
341
342
343
#if CONFIG_CODE_ZEROGROUP
      zpc_probs = &cm->fc.zpc_probs_8x8;
#endif
Daniel Kang's avatar
Daniel Kang committed
344
      break;
345
346
347
    }
    case TX_16X16: {
      const BLOCK_SIZE_TYPE sb_type = xd->mode_info_context->mbmi.sb_type;
348
      const int sz = 2 + b_width_log2(sb_type);
349
      const int x = block & ((1 << sz) - 1), y = block - x;
350
351
      TX_TYPE tx_type = (type == PLANE_TYPE_Y_WITH_DC) ?
          get_tx_type_16x16(xd, y + (x >> 2)) : DCT_DCT;
352
      scan = get_scan_16x16(tx_type);
353
      coef_probs = cm->fc.coef_probs_16x16;
354
      seg_eob = 256;
355
356
      above_ec = (A[0] + A[1] + A[2] + A[3]) != 0;
      left_ec = (L[0] + L[1] + L[2] + L[3]) != 0;
357
358
359
#if CONFIG_CODE_ZEROGROUP
      zpc_probs = &cm->fc.zpc_probs_16x16;
#endif
Daniel Kang's avatar
Daniel Kang committed
360
      break;
361
    }
362
363
    case TX_32X32:
      scan = vp9_default_zig_zag1d_32x32;
364
      coef_probs = cm->fc.coef_probs_32x32;
365
      seg_eob = 1024;
366
367
368
      above_ec = (A[0] + A[1] + A[2] + A[3] + A[4] + A[5] + A[6] + A[7]) != 0;
      left_ec = (L[0] + L[1] + L[2] + L[3] + L[4] + L[5] + L[6] + L[7]) != 0;

369
370
371
#if CONFIG_CODE_ZEROGROUP
      zpc_probs = &cm->fc.zpc_probs_32x32;
#endif
372
      break;
Daniel Kang's avatar
Daniel Kang committed
373
    default:
374
      abort();
Daniel Kang's avatar
Daniel Kang committed
375
376
      break;
  }
John Koleszar's avatar
John Koleszar committed
377
  assert(eob <= seg_eob);
378

379
  pt = combine_entropy_contexts(above_ec, left_ec);
380
381
  nb = vp9_get_coef_neighbors_handle(scan, &pad);
  default_eob = seg_eob;
382

383
384
  if (vp9_segfeature_active(xd, segment_id, SEG_LVL_SKIP))
    seg_eob = 0;
385

386
387
388
389
  /* sanity check to ensure that we do not have spurious non-zero q values */
  if (eob < seg_eob)
    assert(qcoeff_ptr[scan[eob]] == 0);

390
391
392
393
394
395
396
397
398
399
#if CONFIG_CODE_ZEROGROUP
  vpx_memset(token_cache_full, ZERO_TOKEN, sizeof(token_cache_full));
  for (c = 0; c < eob; ++c) {
    rc = scan[c];
    token_cache_full[rc] = vp9_dct_value_tokens_ptr[qcoeff_ptr[rc]].token;
    o = vp9_get_orientation(rc, tx_size);
    if (qcoeff_ptr[rc] != 0)
      last_nz_pos[o] = c;
  }
#endif
400
  {
401
    for (c = 0; c < eob; c++) {
402
      int v = qcoeff_ptr[scan[c]];
403
      int t = vp9_dct_value_tokens_ptr[v].token;
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
      int band = get_coef_band(scan, tx_size, c);
      if (c)
        pt = vp9_get_coef_context(scan, nb, pad, token_cache, c, default_eob);
#if CONFIG_CODE_ZEROGROUP
      rc = scan[c];
      o = vp9_get_orientation(rc, tx_size);
      skip_coef_val = (token_cache[rc] == ZERO_TOKEN || is_eoo_list[o]);
      if (!skip_coef_val) {
        cost += token_costs[band][pt][t] + vp9_dct_value_cost_ptr[v];
      } else {
        assert(v == 0);
      }
#else
      cost += token_costs[band][pt][t] + vp9_dct_value_cost_ptr[v];
#endif
419
420
      if (!c || token_cache[scan[c - 1]])
        cost += vp9_cost_bit(coef_probs[type][ref][band][pt][0], 1);
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
      token_cache[scan[c]] = t;
#if CONFIG_CODE_ZEROGROUP
      if (t == ZERO_TOKEN && !skip_coef_val) {
        int eoo = 0, use_eoo;
#if USE_ZPC_EOORIENT == 1
        use_eoo = vp9_use_eoo(c, seg_eob, scan, tx_size,
                              is_last_zero, is_eoo_list);
#else
        use_eoo = 0;
#endif
        if (use_eoo) {
          eoo = vp9_is_eoo(c, eob, scan, tx_size, qcoeff_ptr, last_nz_pos);
          if (eoo && is_eoo_negative[o]) eoo = 0;
          if (eoo) {
            int c_;
            int savings = 0;
            int zsaved = 0;
            savings = vp9_cost_bit((*zpc_probs)[ref]
                                   [coef_to_zpc_band(band)]
                                   [coef_to_zpc_ptok(pt)][0], 1) -
                      vp9_cost_bit((*zpc_probs)[ref]
                                   [coef_to_zpc_band(band)]
                                   [coef_to_zpc_ptok(pt)][0], 0);
            for (c_ = c + 1; c_ < eob; ++c_) {
              if (o == vp9_get_orientation(scan[c_], tx_size)) {
                int pt_ = vp9_get_coef_context(scan, nb, pad,
                                               token_cache_full, c_,
                                               default_eob);
                int band_ = get_coef_band(scan, tx_size, c_);
                assert(token_cache_full[scan[c_]] == ZERO_TOKEN);
                if (!c_ || token_cache_full[scan[c_ - 1]])
                  savings += vp9_cost_bit(
                      coef_probs[type][ref][band_][pt_][0], 1);
                savings += vp9_cost_bit(
                    coef_probs[type][ref][band_][pt_][1], 0);
                zsaved++;
              }
            }
            if (savings < 0) {
            // if (zsaved < ZPC_ZEROSSAVED_EOO) {
              eoo = 0;
              is_eoo_negative[o] = 1;
            }
          }
        }
        if (use_eoo) {
          cost += vp9_cost_bit((*zpc_probs)[ref]
                                           [coef_to_zpc_band(band)]
                                           [coef_to_zpc_ptok(pt)][0], !eoo);
          if (eoo) {
            assert(is_eoo_list[o] == 0);
            is_eoo_list[o] = 1;
          }
        }
      }
      is_last_zero[o] = (t == ZERO_TOKEN);
477
#endif
478
    }
479
480
481
482
483
484
485
    if (c < seg_eob) {
      if (c)
        pt = vp9_get_coef_context(scan, nb, pad, token_cache, c, default_eob);
      cost += mb->token_costs[tx_size][type][ref]
          [get_coef_band(scan, tx_size, c)]
          [pt][DCT_EOB_TOKEN];
    }
486
487
  }

488
489
490
  // is eob first coefficient;
  for (pt = 0; pt < (1 << tx_size); pt++) {
    A[pt] = L[pt] = c > 0;
491
  }
492

493
494
495
  return cost;
}

496
static void choose_txfm_size_from_rd(VP9_COMP *cpi, MACROBLOCK *x,
497
498
499
500
501
                                     int (*r)[2], int *rate,
                                     int *d, int *distortion,
                                     int *s, int *skip,
                                     int64_t txfm_cache[NB_TXFM_MODES],
                                     TX_SIZE max_txfm_size) {
502
503
504
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
505
  vp9_prob skip_prob = vp9_get_pred_prob(cm, xd, PRED_MBSKIP);
506
507
  int64_t rd[TX_SIZE_MAX_SB][2];
  int n, m;
508
  int s0, s1;
509
510
511
512
513
514
515
516
517
518

  for (n = TX_4X4; n <= max_txfm_size; n++) {
    r[n][1] = r[n][0];
    for (m = 0; m <= n - (n == max_txfm_size); m++) {
      if (m == n)
        r[n][1] += vp9_cost_zero(cm->prob_tx[m]);
      else
        r[n][1] += vp9_cost_one(cm->prob_tx[m]);
    }
  }
519

520
521
522
  assert(skip_prob > 0);
  s0 = vp9_cost_bit(skip_prob, 0);
  s1 = vp9_cost_bit(skip_prob, 1);
523

524
525
526
527
528
529
  for (n = TX_4X4; n <= max_txfm_size; n++) {
    if (s[n]) {
      rd[n][0] = rd[n][1] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
    } else {
      rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0] + s0, d[n]);
      rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1] + s0, d[n]);
530
531
532
    }
  }

533
534
535
536
537
538
  if (max_txfm_size == TX_32X32 &&
      (cm->txfm_mode == ALLOW_32X32 ||
       (cm->txfm_mode == TX_MODE_SELECT &&
        rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
        rd[TX_32X32][1] < rd[TX_4X4][1]))) {
    mbmi->txfm_size = TX_32X32;
539
540
541
542
543
544
  } else if (max_txfm_size >= TX_16X16 &&
             (cm->txfm_mode == ALLOW_16X16 ||
              cm->txfm_mode == ALLOW_32X32 ||
              (cm->txfm_mode == TX_MODE_SELECT &&
               rd[TX_16X16][1] < rd[TX_8X8][1] &&
               rd[TX_16X16][1] < rd[TX_4X4][1]))) {
545
    mbmi->txfm_size = TX_16X16;
546
  } else if (cm->txfm_mode == ALLOW_8X8 ||
547
548
             cm->txfm_mode == ALLOW_16X16 ||
             cm->txfm_mode == ALLOW_32X32 ||
549
           (cm->txfm_mode == TX_MODE_SELECT && rd[TX_8X8][1] < rd[TX_4X4][1])) {
550
551
552
553
554
    mbmi->txfm_size = TX_8X8;
  } else {
    mbmi->txfm_size = TX_4X4;
  }

555
  *distortion = d[mbmi->txfm_size];
556
  *rate       = r[mbmi->txfm_size][cm->txfm_mode == TX_MODE_SELECT];
557
558
  *skip       = s[mbmi->txfm_size];

559
560
  txfm_cache[ONLY_4X4] = rd[TX_4X4][0];
  txfm_cache[ALLOW_8X8] = rd[TX_8X8][0];
561
562
  txfm_cache[ALLOW_16X16] = rd[MIN(max_txfm_size, TX_16X16)][0];
  txfm_cache[ALLOW_32X32] = rd[MIN(max_txfm_size, TX_32X32)][0];
563
564
565
566
  if (max_txfm_size == TX_32X32 &&
      rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
      rd[TX_32X32][1] < rd[TX_4X4][1])
    txfm_cache[TX_MODE_SELECT] = rd[TX_32X32][1];
567
568
  else if (max_txfm_size >= TX_16X16 &&
           rd[TX_16X16][1] < rd[TX_8X8][1] && rd[TX_16X16][1] < rd[TX_4X4][1])
569
    txfm_cache[TX_MODE_SELECT] = rd[TX_16X16][1];
570
  else
571
572
    txfm_cache[TX_MODE_SELECT] = rd[TX_4X4][1] < rd[TX_8X8][1] ?
                                 rd[TX_4X4][1] : rd[TX_8X8][1];
573
574
}

575
576
static int block_error(int16_t *coeff, int16_t *dqcoeff,
                       int block_size, int shift) {
577
578
579
580
  int i;
  int64_t error = 0;

  for (i = 0; i < block_size; i++) {
581
    int this_diff = coeff[i] - dqcoeff[i];
582
    error += (unsigned)this_diff * this_diff;
583
  }
584
  error >>= shift;
585

Frank Galligan's avatar
Frank Galligan committed
586
  return error > INT_MAX ? INT_MAX : (int)error;
587
588
}

589
590
static int block_error_sby(MACROBLOCK *x, BLOCK_SIZE_TYPE bsize, int shift) {
  const int bwl = b_width_log2(bsize), bhl = b_height_log2(bsize);
591
  return block_error(x->plane[0].coeff, x->e_mbd.plane[0].dqcoeff,
592
                     16 << (bwl + bhl), shift);
593
}
594

595
596
597
598
static int block_error_sbuv(MACROBLOCK *x, BLOCK_SIZE_TYPE bsize, int shift) {
  const int bwl = b_width_log2(bsize), bhl = b_height_log2(bsize);
  int64_t sum = 0;
  int plane;
599

600
601
602
603
604
605
606
607
  for (plane = 1; plane < MAX_MB_PLANE; plane++) {
    const int subsampling = x->e_mbd.plane[plane].subsampling_x +
                            x->e_mbd.plane[plane].subsampling_y;
    sum += block_error(x->plane[plane].coeff, x->e_mbd.plane[plane].dqcoeff,
                       16 << (bwl + bhl - subsampling), 0);
  }
  sum >>= shift;
  return sum > INT_MAX ? INT_MAX : (int)sum;
608
609
}

610
611
static int rdcost_plane(VP9_COMMON *const cm, MACROBLOCK *x,
                        int plane, BLOCK_SIZE_TYPE bsize, TX_SIZE tx_size) {
612
  MACROBLOCKD *const xd = &x->e_mbd;
613
614
615
  const int bwl = b_width_log2(bsize) - xd->plane[plane].subsampling_x;
  const int bhl = b_height_log2(bsize) - xd->plane[plane].subsampling_y;
  const int bw = 1 << bwl, bh = 1 << bhl;
616
  ENTROPY_CONTEXT t_above[16], t_left[16];
617
  int block, cost;
618

619
  vpx_memcpy(&t_above, xd->plane[plane].above_context,
620
             sizeof(ENTROPY_CONTEXT) * bw);
621
  vpx_memcpy(&t_left,  xd->plane[plane].left_context,
622
             sizeof(ENTROPY_CONTEXT) * bh);
623

624
625
626
  cost = 0;
  for (block = 0; block < bw * bh; block += 1 << (tx_size * 2)) {
    int x_idx, y_idx;
627

628
629
    txfrm_block_to_raster_xy(xd, bsize, plane, block, tx_size * 2,
                             &x_idx, &y_idx);
630

631
632
633
    cost += cost_coeffs(cm, x, plane, block, xd->plane[plane].plane_type,
                        t_above + x_idx, t_left + y_idx,
                        tx_size, bw * bh);
634
  }
635
636
637
638

  return cost;
}

639
640
641
static int rdcost_uv(VP9_COMMON *const cm, MACROBLOCK *x,
                     BLOCK_SIZE_TYPE bsize, TX_SIZE tx_size) {
  int cost = 0, plane;
642

643
644
  for (plane = 1; plane < MAX_MB_PLANE; plane++) {
    cost += rdcost_plane(cm, x, plane, bsize, tx_size);
645
646
  }
  return cost;
647
648
}

649
650
651
static void super_block_yrd_for_txfm(VP9_COMMON *const cm, MACROBLOCK *x,
                                     int *rate, int *distortion, int *skippable,
                                     BLOCK_SIZE_TYPE bsize, TX_SIZE tx_size) {
652
  MACROBLOCKD *const xd = &x->e_mbd;
653
654
  xd->mode_info_context->mbmi.txfm_size = tx_size;
  vp9_xform_quant_sby(cm, x, bsize);
655

656
657
  *distortion = block_error_sby(x, bsize, tx_size == TX_32X32 ? 0 : 2);
  *rate       = rdcost_plane(cm, x, 0, bsize, tx_size);
658
  *skippable  = vp9_sby_is_skippable(xd, bsize);
659
660
}

661
662
static void super_block_yrd(VP9_COMP *cpi,
                            MACROBLOCK *x, int *rate, int *distortion,
663
                            int *skip, BLOCK_SIZE_TYPE bs,
664
                            int64_t txfm_cache[NB_TXFM_MODES]) {
665
666
  VP9_COMMON *const cm = &cpi->common;
  int r[TX_SIZE_MAX_SB][2], d[TX_SIZE_MAX_SB], s[TX_SIZE_MAX_SB];
Ronald S. Bultje's avatar
Ronald S. Bultje committed
667

668
  vp9_subtract_sby(x, bs);
Ronald S. Bultje's avatar
Ronald S. Bultje committed
669

670
  if (bs >= BLOCK_SIZE_SB32X32)
671
672
    super_block_yrd_for_txfm(cm, x, &r[TX_32X32][0], &d[TX_32X32], &s[TX_32X32],
                             bs, TX_32X32);
673
  if (bs >= BLOCK_SIZE_MB16X16)
674
675
676
677
678
679
    super_block_yrd_for_txfm(cm, x, &r[TX_16X16][0], &d[TX_16X16], &s[TX_16X16],
                             bs, TX_16X16);
  super_block_yrd_for_txfm(cm, x, &r[TX_8X8][0], &d[TX_8X8], &s[TX_8X8], bs,
                           TX_8X8);
  super_block_yrd_for_txfm(cm, x, &r[TX_4X4][0], &d[TX_4X4], &s[TX_4X4], bs,
                           TX_4X4);
Ronald S. Bultje's avatar
Ronald S. Bultje committed
680
681

  choose_txfm_size_from_rd(cpi, x, r, rate, d, distortion, s, skip, txfm_cache,
682
                           TX_32X32 - (bs < BLOCK_SIZE_SB32X32)
Jingning Han's avatar
Jingning Han committed
683
                           - (bs < BLOCK_SIZE_MB16X16));
Ronald S. Bultje's avatar
Ronald S. Bultje committed
684
}
Ronald S. Bultje's avatar
Ronald S. Bultje committed
685

686
687
static int64_t rd_pick_intra4x4block(VP9_COMP *cpi, MACROBLOCK *x, int ib,
                                     B_PREDICTION_MODE *best_mode,
688
689
690
691
                                     int *bmode_costs,
                                     ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l,
                                     int *bestrate, int *bestratey,
                                     int *bestdistortion) {
John Koleszar's avatar
John Koleszar committed
692
  B_PREDICTION_MODE mode;
Deb Mukherjee's avatar
Deb Mukherjee committed
693
  MACROBLOCKD *xd = &x->e_mbd;
694
  int64_t best_rd = INT64_MAX;
John Koleszar's avatar
John Koleszar committed
695
696
  int rate = 0;
  int distortion;
697
  VP9_COMMON *const cm = &cpi->common;
John Koleszar's avatar
John Koleszar committed
698
699
  const int src_stride = x->plane[0].src.stride;
  uint8_t* const src =
700
701
702
      raster_block_offset_uint8(xd,
                                BLOCK_SIZE_SB8X8,
                                0, ib,
John Koleszar's avatar
John Koleszar committed
703
                                x->plane[0].src.buf, src_stride);
704
  int16_t* const src_diff =
705
706
707
      raster_block_offset_int16(xd,
                                BLOCK_SIZE_SB8X8,
                                0, ib,
708
                                x->plane[0].src_diff);
John Koleszar's avatar
John Koleszar committed
709
  int16_t* const diff =
710
711
712
      raster_block_offset_int16(xd,
                                BLOCK_SIZE_SB8X8,
                                0, ib,
John Koleszar's avatar
John Koleszar committed
713
                                xd->plane[0].diff);
John Koleszar's avatar
John Koleszar committed
714
  int16_t* const coeff = BLOCK_OFFSET(x->plane[0].coeff, ib, 16);
715
  uint8_t* const dst =
716
717
718
      raster_block_offset_uint8(xd,
                                BLOCK_SIZE_SB8X8,
                                0, ib,
719
                                xd->plane[0].dst.buf, xd->plane[0].dst.stride);
John Koleszar's avatar
John Koleszar committed
720
721
  ENTROPY_CONTEXT ta = *a, tempa = *a;
  ENTROPY_CONTEXT tl = *l, templ = *l;
Deb Mukherjee's avatar
Deb Mukherjee committed
722
723
  TX_TYPE tx_type = DCT_DCT;
  TX_TYPE best_tx_type = DCT_DCT;
John Koleszar's avatar
John Koleszar committed
724
725
726
727
728
  /*
   * The predictor buffer is a 2d buffer with a stride of 16.  Create
   * a temp buffer that meets the stride requirements, but we are only
   * interested in the left 4x4 block
   * */
729
  DECLARE_ALIGNED_ARRAY(16, int16_t, best_dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
730

Jingning Han's avatar
Jingning Han committed
731
  assert(ib < 4);
732

733
  xd->mode_info_context->mbmi.txfm_size = TX_4X4;
734
  for (mode = B_DC_PRED; mode < LEFT4X4; mode++) {
735
736
    int64_t this_rd;
    int ratey;
737

Scott LaVarnway's avatar
Scott LaVarnway committed
738
    xd->mode_info_context->bmi[ib].as_mode.first = mode;
739
    rate = bmode_costs[mode];
John Koleszar's avatar
John Koleszar committed
740

741
742
743
    vp9_intra4x4_predict(xd, ib,
                         BLOCK_SIZE_SB8X8,
                         mode, dst, xd->plane[0].dst.stride);
Jingning Han's avatar
Jingning Han committed
744
    vp9_subtract_block(4, 4, src_diff, 8,
John Koleszar's avatar
John Koleszar committed
745
                       src, src_stride,
746
                       dst, xd->plane[0].dst.stride);
John Koleszar's avatar
John Koleszar committed
747

Scott LaVarnway's avatar
Scott LaVarnway committed
748
    xd->mode_info_context->bmi[ib].as_mode.first = mode;
John Koleszar's avatar
John Koleszar committed
749
    tx_type = get_tx_type_4x4(xd, ib);
750
    if (tx_type != DCT_DCT) {
Jingning Han's avatar
Jingning Han committed
751
      vp9_short_fht4x4(src_diff, coeff, 8, tx_type);
752
      x->quantize_b_4x4(x, ib, tx_type, 16);
753
    } else {
Jingning Han's avatar
Jingning Han committed
754
      x->fwd_txm4x4(src_diff, coeff, 16);
755
      x->quantize_b_4x4(x, ib, tx_type, 16);
756
    }
John Koleszar's avatar
John Koleszar committed
757

758
759
    tempa = ta;
    templ = tl;
Jingning Han's avatar
Jingning Han committed
760

761
    ratey = cost_coeffs(cm, x, 0, ib,
John Koleszar's avatar
John Koleszar committed
762
                        PLANE_TYPE_Y_WITH_DC, &tempa, &templ, TX_4X4, 16);
763
    rate += ratey;
John Koleszar's avatar
John Koleszar committed
764
    distortion = vp9_block_error(coeff,
765
766
                                 BLOCK_OFFSET(xd->plane[0].dqcoeff, ib, 16),
                                 16) >> 2;
Jingning Han's avatar
Jingning Han committed
767

768
    this_rd = RDCOST(x->rdmult, x->rddiv, rate, distortion);
Jingning Han's avatar
Jingning Han committed
769

770
771
772
773
774
775
776
777
778
    if (this_rd < best_rd) {
      *bestrate = rate;
      *bestratey = ratey;
      *bestdistortion = distortion;
      best_rd = this_rd;
      *best_mode = mode;
      best_tx_type = tx_type;
      *a = tempa;
      *l = templ;
779
      vpx_memcpy(best_dqcoeff, BLOCK_OFFSET(xd->plane[0].dqcoeff, ib, 16), 32);
John Koleszar's avatar
John Koleszar committed
780
    }
John Koleszar's avatar
John Koleszar committed
781
  }
Scott LaVarnway's avatar
Scott LaVarnway committed
782
783
  xd->mode_info_context->bmi[ib].as_mode.first =
    (B_PREDICTION_MODE)(*best_mode);
784

Jingning Han's avatar
Jingning Han committed
785
  // inverse transform
Deb Mukherjee's avatar
Deb Mukherjee committed
786
  if (best_tx_type != DCT_DCT)
Jingning Han's avatar
Jingning Han committed
787
    vp9_short_iht4x4(best_dqcoeff, diff, 8, best_tx_type);
788
  else
Jingning Han's avatar
Jingning Han committed
789
    xd->inv_txm4x4(best_dqcoeff, diff, 16);
Jingning Han's avatar
Jingning Han committed
790

791
792
793
  vp9_intra4x4_predict(xd, ib,
                       BLOCK_SIZE_SB8X8,
                       *best_mode,
794
                       dst, xd->plane[0].dst.stride);
Jingning Han's avatar
Jingning Han committed
795
  vp9_recon_b(dst, diff, 8,
796
              dst, xd->plane[0].dst.stride);
John Koleszar's avatar
John Koleszar committed
797

John Koleszar's avatar
John Koleszar committed
798
  return best_rd;
John Koleszar's avatar
John Koleszar committed
799
800
}

801
802
static int64_t rd_pick_intra4x4mby_modes(VP9_COMP *cpi, MACROBLOCK *mb,
                                         int *Rate, int *rate_y,
803
                                         int *Distortion, int64_t best_rd) {
John Koleszar's avatar
John Koleszar committed
804
805
  int i;
  MACROBLOCKD *const xd = &mb->e_mbd;
Yaowu Xu's avatar
Yaowu Xu committed
806
  int cost = mb->mbmode_cost[xd->frame_type][I4X4_PRED];
John Koleszar's avatar
John Koleszar committed
807
808
809
  int distortion = 0;
  int tot_rate_y = 0;
  int64_t total_rd = 0;
Jingning Han's avatar
Jingning Han committed
810
  ENTROPY_CONTEXT t_above[2], t_left[2];
John Koleszar's avatar
John Koleszar committed
811
812
  int *bmode_costs;

813
814
  vpx_memcpy(t_above, xd->plane[0].above_context, sizeof(t_above));
  vpx_memcpy(t_left, xd->plane[0].left_context, sizeof(t_left));
John Koleszar's avatar
John Koleszar committed
815

Yaowu Xu's avatar
Yaowu Xu committed
816
  xd->mode_info_context->mbmi.mode = I4X4_PRED;
John Koleszar's avatar
John Koleszar committed
817
  bmode_costs = mb->inter_bmode_costs;
818

Jingning Han's avatar
Jingning Han committed
819
820
  for (i = 0; i < 4; i++) {
    const int x_idx = i & 1, y_idx = i >> 1;
John Koleszar's avatar
John Koleszar committed
821
822
823
824
    MODE_INFO *const mic = xd->mode_info_context;
    const int mis = xd->mode_info_stride;
    B_PREDICTION_MODE UNINITIALIZED_IS_SAFE(best_mode);
    int UNINITIALIZED_IS_SAFE(r), UNINITIALIZED_IS_SAFE(ry), UNINITIALIZED_IS_SAFE(d);
John Koleszar's avatar
John Koleszar committed
825

Paul Wilkins's avatar
Paul Wilkins committed
826
    if (xd->frame_type == KEY_FRAME) {
John Koleszar's avatar
John Koleszar committed
827
828
      const B_PREDICTION_MODE A = above_block_mode(mic, i, mis);
      const B_PREDICTION_MODE L = left_block_mode(mic, i);
829

John Koleszar's avatar
John Koleszar committed
830
831
      bmode_costs  = mb->bmode_costs[A][L];
    }
832

833
834
835
    total_rd += rd_pick_intra4x4block(cpi, mb, i, &best_mode, bmode_costs,
                                      t_above + x_idx, t_left + y_idx,
                                      &r, &ry, &d);
John Koleszar's avatar
John Koleszar committed
836

John Koleszar's avatar
John Koleszar committed
837
838
839
    cost += r;
    distortion += d;
    tot_rate_y += ry;
Scott LaVarnway's avatar
Scott LaVarnway committed
840

John Koleszar's avatar
John Koleszar committed
841
    mic->bmi[i].as_mode.first = best_mode;
842

843
    if (total_rd >= best_rd)
John Koleszar's avatar
John Koleszar committed
844
845
      break;
  }
John Koleszar's avatar
John Koleszar committed
846

847
  if (total_rd >= best_rd)
848
    return INT64_MAX;
849

John Koleszar's avatar
John Koleszar committed
850
  *Rate = cost;
851
  *rate_y = tot_rate_y;
John Koleszar's avatar
John Koleszar committed
852
  *Distortion = distortion;
John Koleszar's avatar
John Koleszar committed
853

John Koleszar's avatar
John Koleszar committed
854
  return RDCOST(mb->rdmult, mb->rddiv, cost, distortion);
John Koleszar's avatar
John Koleszar committed
855
}
856

857
858
859
860
static int64_t rd_pick_intra_sby_mode(VP9_COMP *cpi, MACROBLOCK *x,
                                      int *rate, int *rate_tokenonly,
                                      int *distortion, int *skippable,
                                      BLOCK_SIZE_TYPE bsize,
861
                                      int64_t txfm_cache[NB_TXFM_MODES]) {
Ronald S. Bultje's avatar
Ronald S. Bultje committed
862
863
864
  MB_PREDICTION_MODE mode;
  MB_PREDICTION_MODE UNINITIALIZED_IS_SAFE(mode_selected);
  int this_rate, this_rate_tokenonly;
865
  int this_distortion, s;
866
  int64_t best_rd = INT64_MAX, this_rd;
867
868
  TX_SIZE UNINITIALIZED_IS_SAFE(best_tx);
  int i;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
869

870
871
  for (i = 0; i < NB_TXFM_MODES; i++)
    txfm_cache[i] = INT64_MAX;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
872
873
874

  /* Y Search for 32x32 intra prediction mode */
  for (mode = DC_PRED; mode <= TM_PRED; mode++) {
875
876
    int64_t local_txfm_cache[NB_TXFM_MODES];

Ronald S. Bultje's avatar
Ronald S. Bultje committed
877
    x->e_mbd.mode_info_context->mbmi.mode = mode;
878
    vp9_build_intra_predictors_sby_s(&x->e_mbd, bsize);
Ronald S. Bultje's avatar
Ronald S. Bultje committed
879

880
881
882
    super_block_yrd(cpi, x, &this_rate_tokenonly, &this_distortion, &s,
                    bsize, local_txfm_cache);
    this_rate = this_rate_tokenonly + x->mbmode_cost[x->e_mbd.frame_type][mode];
Ronald S. Bultje's avatar
Ronald S. Bultje committed
883
884
885
886
887
    this_rd = RDCOST(x->rdmult, x->rddiv, this_rate, this_distortion);

    if (this_rd < best_rd) {
      mode_selected   = mode;
      best_rd         = this_rd;
888
      best_tx         = x->e_mbd.mode_info_context->mbmi.txfm_size;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
889
890
891
892
893
      *rate           = this_rate;
      *rate_tokenonly = this_rate_tokenonly;
      *distortion     = this_distortion;
      *skippable      = s;