vp9_rdopt.c 166 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5 6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9 10 11 12 13 14 15
 */


#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
16 17
#include "vp9/common/vp9_pragmas.h"

18 19 20 21 22
#include "vp9/encoder/vp9_tokenize.h"
#include "vp9/encoder/vp9_treewriter.h"
#include "vp9/encoder/vp9_onyx_int.h"
#include "vp9/encoder/vp9_modecosts.h"
#include "vp9/encoder/vp9_encodeintra.h"
23 24 25 26 27 28
#include "vp9/common/vp9_entropymode.h"
#include "vp9/common/vp9_reconinter.h"
#include "vp9/common/vp9_reconintra.h"
#include "vp9/common/vp9_reconintra4x4.h"
#include "vp9/common/vp9_findnearmv.h"
#include "vp9/common/vp9_quant_common.h"
29 30 31 32 33 34
#include "vp9/encoder/vp9_encodemb.h"
#include "vp9/encoder/vp9_quantize.h"
#include "vp9/encoder/vp9_variance.h"
#include "vp9/encoder/vp9_mcomp.h"
#include "vp9/encoder/vp9_rdopt.h"
#include "vp9/encoder/vp9_ratectrl.h"
John Koleszar's avatar
John Koleszar committed
35
#include "vpx_mem/vpx_mem.h"
36 37
#include "vp9/common/vp9_systemdependent.h"
#include "vp9/encoder/vp9_encodemv.h"
John Koleszar's avatar
John Koleszar committed
38

39 40 41
#include "vp9/common/vp9_seg_common.h"
#include "vp9/common/vp9_pred_common.h"
#include "vp9/common/vp9_entropy.h"
42
#include "vp9_rtcd.h"
43
#include "vp9/common/vp9_mvref_common.h"
Paul Wilkins's avatar
Paul Wilkins committed
44

John Koleszar's avatar
John Koleszar committed
45 46
#define MAXF(a,b)            (((a) > (b)) ? (a) : (b))

47 48
#define INVALID_MV 0x80008000

49 50 51
/* Factor to weigh the rate for switchable interp filters */
#define SWITCHABLE_INTERP_RATE_FACTOR 1

John Koleszar's avatar
John Koleszar committed
52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69
static const int auto_speed_thresh[17] = {
  1000,
  200,
  150,
  130,
  150,
  125,
  120,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  105
John Koleszar's avatar
John Koleszar committed
70 71
};

72
#if CONFIG_PRED_FILTER
73
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121
  {ZEROMV,    LAST_FRAME,   NONE,  0},
  {ZEROMV,    LAST_FRAME,   NONE,  1},
  {DC_PRED,   INTRA_FRAME,  NONE,  0},

  {NEARESTMV, LAST_FRAME,   NONE,  0},
  {NEARESTMV, LAST_FRAME,   NONE,  1},
  {NEARMV,    LAST_FRAME,   NONE,  0},
  {NEARMV,    LAST_FRAME,   NONE,  1},

  {ZEROMV,    GOLDEN_FRAME, NONE,  0},
  {ZEROMV,    GOLDEN_FRAME, NONE,  1},
  {NEARESTMV, GOLDEN_FRAME, NONE,  0},
  {NEARESTMV, GOLDEN_FRAME, NONE,  1},

  {ZEROMV,    ALTREF_FRAME, NONE,  0},
  {ZEROMV,    ALTREF_FRAME, NONE,  1},
  {NEARESTMV, ALTREF_FRAME, NONE,  0},
  {NEARESTMV, ALTREF_FRAME, NONE,  1},

  {NEARMV,    GOLDEN_FRAME, NONE,  0},
  {NEARMV,    GOLDEN_FRAME, NONE,  1},
  {NEARMV,    ALTREF_FRAME, NONE,  0},
  {NEARMV,    ALTREF_FRAME, NONE,  1},

  {V_PRED,    INTRA_FRAME,  NONE,  0},
  {H_PRED,    INTRA_FRAME,  NONE,  0},
  {D45_PRED,  INTRA_FRAME,  NONE,  0},
  {D135_PRED, INTRA_FRAME,  NONE,  0},
  {D117_PRED, INTRA_FRAME,  NONE,  0},
  {D153_PRED, INTRA_FRAME,  NONE,  0},
  {D27_PRED,  INTRA_FRAME,  NONE,  0},
  {D63_PRED,  INTRA_FRAME,  NONE,  0},

  {TM_PRED,   INTRA_FRAME,  NONE,  0},

  {NEWMV,     LAST_FRAME,   NONE,  0},
  {NEWMV,     LAST_FRAME,   NONE,  1},
  {NEWMV,     GOLDEN_FRAME, NONE,  0},
  {NEWMV,     GOLDEN_FRAME, NONE,  1},
  {NEWMV,     ALTREF_FRAME, NONE,  0},
  {NEWMV,     ALTREF_FRAME, NONE,  1},

  {SPLITMV,   LAST_FRAME,   NONE,  0},
  {SPLITMV,   GOLDEN_FRAME, NONE,  0},
  {SPLITMV,   ALTREF_FRAME, NONE,  0},

  {B_PRED,    INTRA_FRAME,  NONE,  0},
  {I8X8_PRED, INTRA_FRAME,  NONE,  0},
John Koleszar's avatar
John Koleszar committed
122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141

  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME, 0},

  {ZEROMV,    ALTREF_FRAME, LAST_FRAME,   0},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME,   0},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME,   0},

  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME, 0},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME, 0},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME, 0},

  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME,   0},
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME, 0},

  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME, 0},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME,   0},
142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME, 0},

#if CONFIG_COMP_INTERINTRA_PRED
  /* compound inter-intra prediction */
  {ZEROMV,    LAST_FRAME,   INTRA_FRAME, 0},
  {NEARESTMV, LAST_FRAME,   INTRA_FRAME, 0},
  {NEARMV,    LAST_FRAME,   INTRA_FRAME, 0},
  {NEWMV,     LAST_FRAME,   INTRA_FRAME, 0},

  {ZEROMV,    GOLDEN_FRAME,   INTRA_FRAME, 0},
  {NEARESTMV, GOLDEN_FRAME,   INTRA_FRAME, 0},
  {NEARMV,    GOLDEN_FRAME,   INTRA_FRAME, 0},
  {NEWMV,     GOLDEN_FRAME,   INTRA_FRAME, 0},

  {ZEROMV,    ALTREF_FRAME,   INTRA_FRAME, 0},
  {NEARESTMV, ALTREF_FRAME,   INTRA_FRAME, 0},
  {NEARMV,    ALTREF_FRAME,   INTRA_FRAME, 0},
  {NEWMV,     ALTREF_FRAME,   INTRA_FRAME, 0},
#endif
John Koleszar's avatar
John Koleszar committed
161
};
162
#else
163
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
164 165
  {ZEROMV,    LAST_FRAME,   NONE},
  {DC_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
166

167 168
  {NEARESTMV, LAST_FRAME,   NONE},
  {NEARMV,    LAST_FRAME,   NONE},
John Koleszar's avatar
John Koleszar committed
169

170 171
  {ZEROMV,    GOLDEN_FRAME, NONE},
  {NEARESTMV, GOLDEN_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
172

173 174
  {ZEROMV,    ALTREF_FRAME, NONE},
  {NEARESTMV, ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
175

176 177
  {NEARMV,    GOLDEN_FRAME, NONE},
  {NEARMV,    ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
178

179 180 181 182 183 184 185 186
  {V_PRED,    INTRA_FRAME,  NONE},
  {H_PRED,    INTRA_FRAME,  NONE},
  {D45_PRED,  INTRA_FRAME,  NONE},
  {D135_PRED, INTRA_FRAME,  NONE},
  {D117_PRED, INTRA_FRAME,  NONE},
  {D153_PRED, INTRA_FRAME,  NONE},
  {D27_PRED,  INTRA_FRAME,  NONE},
  {D63_PRED,  INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
187

188
  {TM_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
189

190 191 192
  {NEWMV,     LAST_FRAME,   NONE},
  {NEWMV,     GOLDEN_FRAME, NONE},
  {NEWMV,     ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
193

194 195 196
  {SPLITMV,   LAST_FRAME,   NONE},
  {SPLITMV,   GOLDEN_FRAME, NONE},
  {SPLITMV,   ALTREF_FRAME, NONE},
197

198 199
  {B_PRED,    INTRA_FRAME,  NONE},
  {I8X8_PRED, INTRA_FRAME,  NONE},
200

John Koleszar's avatar
John Koleszar committed
201 202 203 204
  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME},
205

John Koleszar's avatar
John Koleszar committed
206 207 208
  {ZEROMV,    ALTREF_FRAME, LAST_FRAME},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME},
209

John Koleszar's avatar
John Koleszar committed
210 211 212
  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME},
213

John Koleszar's avatar
John Koleszar committed
214 215 216
  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME  },
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME},
217

John Koleszar's avatar
John Koleszar committed
218 219
  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME  },
220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME},

#if CONFIG_COMP_INTERINTRA_PRED
  /* compound inter-intra prediction */
  {ZEROMV,    LAST_FRAME,   INTRA_FRAME},
  {NEARESTMV, LAST_FRAME,   INTRA_FRAME},
  {NEARMV,    LAST_FRAME,   INTRA_FRAME},
  {NEWMV,     LAST_FRAME,   INTRA_FRAME},

  {ZEROMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEARESTMV, GOLDEN_FRAME,   INTRA_FRAME},
  {NEARMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEWMV,     GOLDEN_FRAME,   INTRA_FRAME},

  {ZEROMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEARESTMV, ALTREF_FRAME,   INTRA_FRAME},
  {NEARMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEWMV,     ALTREF_FRAME,   INTRA_FRAME},
#endif
John Koleszar's avatar
John Koleszar committed
239
};
240
#endif
John Koleszar's avatar
John Koleszar committed
241 242

static void fill_token_costs(
243
  unsigned int (*c)[COEF_BANDS][PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS],
244
  const vp9_prob(*p)[COEF_BANDS][PREV_COEF_CONTEXTS][ENTROPY_NODES],
John Koleszar's avatar
John Koleszar committed
245 246 247 248 249 250 251
  int block_type_counts) {
  int i, j, k;

  for (i = 0; i < block_type_counts; i++)
    for (j = 0; j < COEF_BANDS; j++)
      for (k = 0; k < PREV_COEF_CONTEXTS; k++) {
        if (k == 0 && ((j > 0 && i > 0) || (j > 1 && i == 0)))
252 253
          vp9_cost_tokens_skip((int *)(c[i][j][k]),
                               p[i][j][k],
254
                               vp9_coef_tree);
John Koleszar's avatar
John Koleszar committed
255
        else
256 257
          vp9_cost_tokens((int *)(c[i][j][k]),
                          p[i][j][k],
258
                          vp9_coef_tree);
John Koleszar's avatar
John Koleszar committed
259
      }
John Koleszar's avatar
John Koleszar committed
260 261
}

262

263 264 265 266
static int rd_iifactor[32] =  { 4, 4, 3, 2, 1, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0, };
John Koleszar's avatar
John Koleszar committed
267

268
// 3* dc_qlookup[Q]*dc_qlookup[Q];
269

270
/* values are now correlated to quantizer */
Paul Wilkins's avatar
Paul Wilkins committed
271 272 273
static int sad_per_bit16lut[QINDEX_RANGE];
static int sad_per_bit4lut[QINDEX_RANGE];

274
void vp9_init_me_luts() {
John Koleszar's avatar
John Koleszar committed
275 276 277 278 279 280 281
  int i;

  // Initialize the sad lut tables using a formulaic calculation for now
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < QINDEX_RANGE; i++) {
    sad_per_bit16lut[i] =
282 283
      (int)((0.0418 * vp9_convert_qindex_to_q(i)) + 2.4107);
    sad_per_bit4lut[i] = (int)((0.063 * vp9_convert_qindex_to_q(i)) + 2.742);
John Koleszar's avatar
John Koleszar committed
284
  }
Paul Wilkins's avatar
Paul Wilkins committed
285
}
John Koleszar's avatar
John Koleszar committed
286

287
static int compute_rd_mult(int qindex) {
John Koleszar's avatar
John Koleszar committed
288
  int q;
289

290
  q = vp9_dc_quant(qindex, 0);
John Koleszar's avatar
John Koleszar committed
291
  return (11 * q * q) >> 6;
292 293
}

294
void vp9_initialize_me_consts(VP9_COMP *cpi, int QIndex) {
John Koleszar's avatar
John Koleszar committed
295 296
  cpi->mb.sadperbit16 =  sad_per_bit16lut[QIndex];
  cpi->mb.sadperbit4  =  sad_per_bit4lut[QIndex];
John Koleszar's avatar
John Koleszar committed
297 298
}

299

300
void vp9_initialize_rd_consts(VP9_COMP *cpi, int QIndex) {
301
  int q, i;
John Koleszar's avatar
John Koleszar committed
302

303
  vp9_clear_system_state();  // __asm emms;
John Koleszar's avatar
John Koleszar committed
304

John Koleszar's avatar
John Koleszar committed
305 306 307 308 309
  // Further tests required to see if optimum is different
  // for key frames, golden frames and arf frames.
  // if (cpi->common.refresh_golden_frame ||
  //     cpi->common.refresh_alt_ref_frame)
  QIndex = (QIndex < 0) ? 0 : ((QIndex > MAXQ) ? MAXQ : QIndex);
310

John Koleszar's avatar
John Koleszar committed
311
  cpi->RDMULT = compute_rd_mult(QIndex);
John Koleszar's avatar
John Koleszar committed
312

John Koleszar's avatar
John Koleszar committed
313 314 315
  // Extend rate multiplier along side quantizer zbin increases
  if (cpi->zbin_over_quant  > 0) {
    double oq_factor;
316

John Koleszar's avatar
John Koleszar committed
317 318 319 320 321
    // Experimental code using the same basic equation as used for Q above
    // The units of cpi->zbin_over_quant are 1/128 of Q bin size
    oq_factor = 1.0 + ((double)0.0015625 * cpi->zbin_over_quant);
    cpi->RDMULT = (int)((double)cpi->RDMULT * oq_factor * oq_factor);
  }
John Koleszar's avatar
John Koleszar committed
322

John Koleszar's avatar
John Koleszar committed
323 324 325 326 327 328 329
  if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    if (cpi->twopass.next_iiratio > 31)
      cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
    else
      cpi->RDMULT +=
        (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
  }
John Koleszar's avatar
John Koleszar committed
330

John Koleszar's avatar
John Koleszar committed
331 332
  if (cpi->RDMULT < 7)
    cpi->RDMULT = 7;
333

John Koleszar's avatar
John Koleszar committed
334 335
  cpi->mb.errorperbit = (cpi->RDMULT / 110);
  cpi->mb.errorperbit += (cpi->mb.errorperbit == 0);
336

337
  vp9_set_speed_features(cpi);
John Koleszar's avatar
John Koleszar committed
338

339
  q = (int)pow(vp9_dc_quant(QIndex, 0) >> 2, 1.25);
John Koleszar's avatar
John Koleszar committed
340 341
  q = q << 2;
  cpi->RDMULT = cpi->RDMULT << 4;
Paul Wilkins's avatar
Paul Wilkins committed
342

John Koleszar's avatar
John Koleszar committed
343 344
  if (q < 8)
    q = 8;
345

John Koleszar's avatar
John Koleszar committed
346 347 348
  if (cpi->RDMULT > 1000) {
    cpi->RDDIV = 1;
    cpi->RDMULT /= 100;
John Koleszar's avatar
John Koleszar committed
349

John Koleszar's avatar
John Koleszar committed
350 351 352 353 354 355
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < INT_MAX) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q / 100;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
356

John Koleszar's avatar
John Koleszar committed
357
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
358
    }
John Koleszar's avatar
John Koleszar committed
359 360
  } else {
    cpi->RDDIV = 100;
John Koleszar's avatar
John Koleszar committed
361

John Koleszar's avatar
John Koleszar committed
362 363 364 365 366 367
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < (INT_MAX / q)) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
368

John Koleszar's avatar
John Koleszar committed
369
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
370
    }
John Koleszar's avatar
John Koleszar committed
371
  }
John Koleszar's avatar
John Koleszar committed
372

John Koleszar's avatar
John Koleszar committed
373
  fill_token_costs(
374
    cpi->mb.token_costs[TX_4X4],
375
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs,
John Koleszar's avatar
John Koleszar committed
376
    BLOCK_TYPES);
377 378
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_4X4],
379
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11])
380 381
    cpi->common.fc.hybrid_coef_probs,
    BLOCK_TYPES);
John Koleszar's avatar
John Koleszar committed
382

John Koleszar's avatar
John Koleszar committed
383
  fill_token_costs(
384
    cpi->mb.token_costs[TX_8X8],
385
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs_8x8,
John Koleszar's avatar
John Koleszar committed
386
    BLOCK_TYPES_8X8);
387 388
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_8X8],
389
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11])
390 391
    cpi->common.fc.hybrid_coef_probs_8x8,
    BLOCK_TYPES_8X8);
392

Daniel Kang's avatar
Daniel Kang committed
393
  fill_token_costs(
394
    cpi->mb.token_costs[TX_16X16],
395
    (const vp9_prob(*)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs_16x16,
Daniel Kang's avatar
Daniel Kang committed
396
    BLOCK_TYPES_16X16);
397 398
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_16X16],
399
    (const vp9_prob(*)[8][PREV_COEF_CONTEXTS][11])
400 401
    cpi->common.fc.hybrid_coef_probs_16x16,
    BLOCK_TYPES_16X16);
Daniel Kang's avatar
Daniel Kang committed
402

John Koleszar's avatar
John Koleszar committed
403 404
  /*rough estimate for costing*/
  cpi->common.kf_ymode_probs_index = cpi->common.base_qindex >> 4;
405
  vp9_init_mode_costs(cpi);
John Koleszar's avatar
John Koleszar committed
406

407 408
  if (cpi->common.frame_type != KEY_FRAME)
  {
409
    vp9_build_nmv_cost_table(
410 411 412 413 414 415
        cpi->mb.nmvjointcost,
        cpi->mb.e_mbd.allow_high_precision_mv ?
        cpi->mb.nmvcost_hp : cpi->mb.nmvcost,
        &cpi->common.fc.nmvc,
        cpi->mb.e_mbd.allow_high_precision_mv, 1, 1);
  }
John Koleszar's avatar
John Koleszar committed
416 417
}

418
int vp9_block_error_c(short *coeff, short *dqcoeff, int block_size) {
419
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
420

421
  for (i = 0; i < block_size; i++) {
John Koleszar's avatar
John Koleszar committed
422 423 424
    int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }
John Koleszar's avatar
John Koleszar committed
425

John Koleszar's avatar
John Koleszar committed
426
  return error;
John Koleszar's avatar
John Koleszar committed
427 428
}

429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447
int vp9_mbblock_error_8x8_c(MACROBLOCK *mb, int dc) {
  BLOCK  *be;
  BLOCKD *bd;
  int i, j;
  int berror, error = 0;

  for (i = 0; i < 16; i+=4) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
    berror = 0;
    for (j = dc; j < 64; j++) {
      int this_diff = be->coeff[j] - bd->dqcoeff[j];
      berror += this_diff * this_diff;
    }
    error += berror;
  }
  return error;
}

448
int vp9_mbblock_error_c(MACROBLOCK *mb, int dc) {
John Koleszar's avatar
John Koleszar committed
449 450 451 452
  BLOCK  *be;
  BLOCKD *bd;
  int i, j;
  int berror, error = 0;
John Koleszar's avatar
John Koleszar committed
453

John Koleszar's avatar
John Koleszar committed
454 455 456 457 458 459 460
  for (i = 0; i < 16; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
    berror = 0;
    for (j = dc; j < 16; j++) {
      int this_diff = be->coeff[j] - bd->dqcoeff[j];
      berror += this_diff * this_diff;
John Koleszar's avatar
John Koleszar committed
461
    }
John Koleszar's avatar
John Koleszar committed
462 463 464
    error += berror;
  }
  return error;
John Koleszar's avatar
John Koleszar committed
465 466
}

467
int vp9_mbuverror_c(MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
468 469
  BLOCK  *be;
  BLOCKD *bd;
John Koleszar's avatar
John Koleszar committed
470

471
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
472

John Koleszar's avatar
John Koleszar committed
473 474 475
  for (i = 16; i < 24; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
John Koleszar's avatar
John Koleszar committed
476

477
    error += vp9_block_error_c(be->coeff, bd->dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
478
  }
John Koleszar's avatar
John Koleszar committed
479

John Koleszar's avatar
John Koleszar committed
480
  return error;
John Koleszar's avatar
John Koleszar committed
481 482
}

483
int vp9_uvsse(MACROBLOCK *x) {
John Koleszar's avatar
John Koleszar committed
484 485 486 487 488 489 490
  unsigned char *uptr, *vptr;
  unsigned char *upred_ptr = (*(x->block[16].base_src) + x->block[16].src);
  unsigned char *vpred_ptr = (*(x->block[20].base_src) + x->block[20].src);
  int uv_stride = x->block[16].src_stride;

  unsigned int sse1 = 0;
  unsigned int sse2 = 0;
491 492
  int mv_row = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.row;
  int mv_col = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.col;
John Koleszar's avatar
John Koleszar committed
493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513
  int offset;
  int pre_stride = x->e_mbd.block[16].pre_stride;

  if (mv_row < 0)
    mv_row -= 1;
  else
    mv_row += 1;

  if (mv_col < 0)
    mv_col -= 1;
  else
    mv_col += 1;

  mv_row /= 2;
  mv_col /= 2;

  offset = (mv_row >> 3) * pre_stride + (mv_col >> 3);
  uptr = x->e_mbd.pre.u_buffer + offset;
  vptr = x->e_mbd.pre.v_buffer + offset;

  if ((mv_row | mv_col) & 7) {
514
    vp9_sub_pixel_variance8x8(uptr, pre_stride, (mv_col & 7) << 1,
515
                              (mv_row & 7) << 1, upred_ptr, uv_stride, &sse2);
516
    vp9_sub_pixel_variance8x8(vptr, pre_stride, (mv_col & 7) << 1,
517
                              (mv_row & 7) << 1, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
518 519
    sse2 += sse1;
  } else {
520 521
    vp9_variance8x8(uptr, pre_stride, upred_ptr, uv_stride, &sse2);
    vp9_variance8x8(vptr, pre_stride, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
522 523 524
    sse2 += sse1;
  }
  return sse2;
John Koleszar's avatar
John Koleszar committed
525 526 527

}

528
static int cost_coeffs_2x2(MACROBLOCK *mb,
529
                           BLOCKD *b, PLANE_TYPE type,
530
                           ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l) {
531
  int c = (type == PLANE_TYPE_Y_NO_DC); /* start at coef 0, unless Y with Y2 */
532 533 534 535 536
  int eob = b->eob;
  int pt;    /* surrounding block/prev coef predictor */
  int cost = 0;
  short *qcoeff_ptr = b->qcoeff;

537
  VP9_COMBINEENTROPYCONTEXTS(pt, *a, *l);
538 539 540
  assert(eob <= 4);

  for (; c < eob; c++) {
541 542 543 544 545
    int v = qcoeff_ptr[vp9_default_zig_zag1d[c]];
    int t = vp9_dct_value_tokens_ptr[v].Token;
    cost += mb->token_costs[TX_8X8][type][vp9_coef_bands[c]][pt][t];
    cost += vp9_dct_value_cost_ptr[v];
    pt = vp9_prev_token_class[t];
546 547 548
  }

  if (c < 4)
549
    cost += mb->token_costs[TX_8X8][type][vp9_coef_bands[c]]
550
            [pt] [DCT_EOB_TOKEN];
551 552
  // is eob first coefficient;
  pt = (c > !type);
553 554 555 556
  *a = *l = pt;
  return cost;
}

557
static int cost_coeffs(MACROBLOCK *mb, BLOCKD *b, PLANE_TYPE type,
Daniel Kang's avatar
Daniel Kang committed
558
                       ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l,
559
                       int tx_size) {
Daniel Kang's avatar
Daniel Kang committed
560
  const int eob = b->eob;
561
  int c = (type == PLANE_TYPE_Y_NO_DC); /* start at coef 0, unless Y with Y2 */
562
  int cost = 0, default_eob, seg_eob;
Daniel Kang's avatar
Daniel Kang committed
563 564
  int pt;                     /* surrounding block/prev coef predictor */
  int const *scan, *band;
John Koleszar's avatar
John Koleszar committed
565
  short *qcoeff_ptr = b->qcoeff;
566 567 568
  MACROBLOCKD *xd = &mb->e_mbd;
  MB_MODE_INFO *mbmi = &mb->e_mbd.mode_info_context->mbmi;
  TX_TYPE tx_type = DCT_DCT;
Paul Wilkins's avatar
Paul Wilkins committed
569
  int segment_id = mbmi->segment_id;
Jim Bankoski's avatar
Jim Bankoski committed
570 571 572
  scan = vp9_default_zig_zag1d;
  band = vp9_coef_bands;
  default_eob = 16;
573

574
  switch (tx_size) {
Daniel Kang's avatar
Daniel Kang committed
575
    case TX_4X4:
Deb Mukherjee's avatar
Deb Mukherjee committed
576 577 578 579 580
      if (type == PLANE_TYPE_Y_WITH_DC) {
        tx_type = get_tx_type_4x4(xd, b);
        if (tx_type != DCT_DCT) {
          switch (tx_type) {
            case ADST_DCT:
581
              scan = vp9_row_scan;
Deb Mukherjee's avatar
Deb Mukherjee committed
582 583 584
              break;

            case DCT_ADST:
585
              scan = vp9_col_scan;
Deb Mukherjee's avatar
Deb Mukherjee committed
586 587 588
              break;

            default:
589
              scan = vp9_default_zig_zag1d;
Deb Mukherjee's avatar
Deb Mukherjee committed
590 591
              break;
          }
592
        }
Daniel Kang's avatar
Daniel Kang committed
593
      }
Deb Mukherjee's avatar
Deb Mukherjee committed
594

Daniel Kang's avatar
Daniel Kang committed
595 596
      break;
    case TX_8X8:
597 598
      scan = vp9_default_zig_zag1d_8x8;
      band = vp9_coef_bands_8x8;
Daniel Kang's avatar
Daniel Kang committed
599
      default_eob = 64;
Deb Mukherjee's avatar
Deb Mukherjee committed
600
      if (type == PLANE_TYPE_Y_WITH_DC) {
601
        BLOCKD *bb;
602
        int ib = (int)(b - xd->block);
603 604 605
        if (ib < 16) {
          ib = (ib & 8) + ((ib & 4) >> 1);
          bb = xd->block + ib;
Deb Mukherjee's avatar
Deb Mukherjee committed
606
          tx_type = get_tx_type_8x8(xd, bb);
607
        }
608
      }
Daniel Kang's avatar
Daniel Kang committed
609 610
      break;
    case TX_16X16:
611 612
      scan = vp9_default_zig_zag1d_16x16;
      band = vp9_coef_bands_16x16;
Daniel Kang's avatar
Daniel Kang committed
613
      default_eob = 256;
Deb Mukherjee's avatar
Deb Mukherjee committed
614 615 616
      if (type == PLANE_TYPE_Y_WITH_DC) {
        tx_type = get_tx_type_16x16(xd, b);
      }
Daniel Kang's avatar
Daniel Kang committed
617 618 619 620
      break;
    default:
      break;
  }
621 622
  if (vp9_segfeature_active(&mb->e_mbd, segment_id, SEG_LVL_EOB))
    seg_eob = vp9_get_segdata(&mb->e_mbd, segment_id, SEG_LVL_EOB);
623 624 625
  else
    seg_eob = default_eob;

626
  VP9_COMBINEENTROPYCONTEXTS(pt, *a, *l);
627

628 629 630
  if (tx_type != DCT_DCT) {
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
631
      int t = vp9_dct_value_tokens_ptr[v].Token;
632
      cost += mb->hybrid_token_costs[tx_size][type][band[c]][pt][t];
633 634
      cost += vp9_dct_value_cost_ptr[v];
      pt = vp9_prev_token_class[t];
635 636 637 638
    }
    if (c < seg_eob)
      cost += mb->hybrid_token_costs[tx_size][type][band[c]]
          [pt][DCT_EOB_TOKEN];
639
  } else {
640 641
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
642
      int t = vp9_dct_value_tokens_ptr[v].Token;
643
      cost += mb->token_costs[tx_size][type][band[c]][pt][t];
644 645
      cost += vp9_dct_value_cost_ptr[v];
      pt = vp9_prev_token_class[t];
646 647 648 649
    }
    if (c < seg_eob)
      cost += mb->token_costs[tx_size][type][band[c]]
          [pt][DCT_EOB_TOKEN];
650 651
  }

652 653
  // is eob first coefficient;
  pt = (c > !type);
654 655 656 657
  *a = *l = pt;
  return cost;
}

658
static int rdcost_mby_4x4(MACROBLOCK *mb, int has_2nd_order, int backup) {
John Koleszar's avatar
John Koleszar committed
659 660
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
661
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
662 663 664
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta;
  ENTROPY_CONTEXT *tl;
665

666 667 668
  if (backup) {
    vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
669

670 671 672 673 674 675
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)xd->above_context;
    tl = (ENTROPY_CONTEXT *)xd->left_context;
  }
John Koleszar's avatar
John Koleszar committed
676

John Koleszar's avatar
John Koleszar committed
677
  for (b = 0; b < 16; b++)
678 679 680
    cost += cost_coeffs(mb, xd->block + b,
                        (has_2nd_order ?
                         PLANE_TYPE_Y_NO_DC : PLANE_TYPE_Y_WITH_DC),
681
                        ta + vp9_block2above[b], tl + vp9_block2left[b],
Daniel Kang's avatar
Daniel Kang committed
682
                        TX_4X4);
John Koleszar's avatar
John Koleszar committed
683

684 685 686 687
  if (has_2nd_order)
    cost += cost_coeffs(mb, xd->block + 24, PLANE_TYPE_Y2,
                        ta + vp9_block2above[24], tl + vp9_block2left[24],
                        TX_4X4);
John Koleszar's avatar
John Koleszar committed
688

John Koleszar's avatar
John Koleszar committed
689
  return cost;
John Koleszar's avatar
John Koleszar committed
690 691
}

692 693 694
static void macro_block_yrd_4x4(MACROBLOCK *mb,
                                int *Rate,
                                int *Distortion,
695
                                int *skippable, int backup) {
Paul Wilkins's avatar
Paul Wilkins committed
696
  MACROBLOCKD *const xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
697
  BLOCK   *const mb_y2 = mb->block + 24;
Paul Wilkins's avatar
Paul Wilkins committed
698
  BLOCKD *const x_y2  = xd->block + 24;
Jim Bankoski's avatar
Jim Bankoski committed
699
  int d, has_2nd_order;
700

701 702
  xd->mode_info_context->mbmi.txfm_size = TX_4X4;
  has_2nd_order = get_2nd_order_usage(xd);
John Koleszar's avatar
John Koleszar committed
703
  // Fdct and building the 2nd order block
704 705 706 707 708
  vp9_transform_mby_4x4(mb);
  vp9_quantize_mby_4x4(mb);
  d = vp9_mbblock_error(mb, has_2nd_order);
  if (has_2nd_order)
    d += vp9_block_error(mb_y2->coeff, x_y2->dqcoeff, 16);
709

John Koleszar's avatar
John Koleszar committed
710 711
  *Distortion = (d >> 2);
  // rate
712 713
  *Rate = rdcost_mby_4x4(mb, has_2nd_order, backup);
  *skippable = vp9_mby_is_skippable_4x4(&mb->e_mbd, has_2nd_order);
714
}
John Koleszar's avatar
John Koleszar committed
715

716
static int rdcost_mby_8x8(MACROBLOCK *mb, int has_2nd_order, int backup) {
John Koleszar's avatar
John Koleszar committed
717 718
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
719
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
720 721 722 723
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta;
  ENTROPY_CONTEXT *tl;

Ronald S. Bultje's avatar
Ronald S. Bultje committed
724 725 726
  if (backup) {
    vpx_memcpy(&t_above,xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
727

Ronald S. Bultje's avatar
Ronald S. Bultje committed
728 729 730 731 732 733
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)mb->e_mbd.above_context;
    tl = (ENTROPY_CONTEXT *)mb->e_mbd.left_context;
  }
John Koleszar's avatar
John Koleszar committed
734 735

  for (b = 0; b < 16; b += 4)
736 737 738
    cost += cost_coeffs(mb, xd->block + b,
                        (has_2nd_order ?
                         PLANE_TYPE_Y_NO_DC : PLANE_TYPE_Y_WITH_DC),
739
                        ta + vp9_block2above_8x8[b], tl + vp9_block2left_8x8[b],
Daniel Kang's avatar
Daniel Kang committed
740
                        TX_8X8);
John Koleszar's avatar
John Koleszar committed
741

742 743 744
  if (has_2nd_order)
    cost += cost_coeffs_2x2(mb, xd->block + 24, PLANE_TYPE_Y2,
                            ta + vp9_block2above[24], tl + vp9_block2left[24]);
John Koleszar's avatar
John Koleszar committed
745
  return cost;
746 747
}

John Koleszar's avatar
John Koleszar committed
748 749 750
static void macro_block_yrd_8x8(MACROBLOCK *mb,
                                int *Rate,
                                int *Distortion,
751
                                int *skippable, int backup) {
Paul Wilkins's avatar
Paul Wilkins committed
752
  MACROBLOCKD *const xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
753
  BLOCK   *const mb_y2 = mb->block + 24;
Paul Wilkins's avatar
Paul Wilkins committed
754
  BLOCKD *const x_y2  = xd->block + 24;
755 756 757
  int d, has_2nd_order;

  xd->mode_info_context->mbmi.txfm_size = TX_8X8;
John Koleszar's avatar
John Koleszar committed
758

759 760
  vp9_transform_mby_8x8(mb);
  vp9_quantize_mby_8x8(mb);
761 762 763 764
  has_2nd_order = get_2nd_order_usage(xd);
  d = vp9_mbblock_error_8x8_c(mb, has_2nd_order);
  if (has_2nd_order)
    d += vp9_block_error(mb_y2->coeff, x_y2->dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
765 766 767

  *Distortion = (d >> 2);
  // rate
768 769
  *Rate = rdcost_mby_8x8(mb, has_2nd_order, backup);
  *skippable = vp9_mby_is_skippable_8x8(&mb->e_mbd, has_2nd_order);
770
}
771

772
static int rdcost_mby_16x16(MACROBLOCK *mb, int backup) {
Daniel Kang's avatar
Daniel Kang committed
773
  int cost;
Paul Wilkins's avatar
Paul Wilkins committed
774
  MACROBLOCKD *xd = &mb->e_mbd;
Daniel Kang's avatar
Daniel Kang committed
775 776 777
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta, *tl;

778 779 780
  if (backup) {
    vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
Daniel Kang's avatar
Daniel Kang committed
781

782 783 784 785 786 787
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)xd->above_context;
    tl = (ENTROPY_CONTEXT *)xd->left_context;
  }
Daniel Kang's avatar
Daniel Kang committed
788

Paul Wilkins's avatar
Paul Wilkins committed
789
  cost = cost_coeffs(mb, xd->block, PLANE_TYPE_Y_WITH_DC, ta, tl, TX_16X16);
Daniel Kang's avatar
Daniel Kang committed
790 791
  return cost;
}
792

Daniel Kang's avatar
Daniel Kang committed
793
static void macro_block_yrd_16x16(MACROBLOCK *mb, int *Rate, int *Distortion,
794
                                  int *skippable, int backup) {
Daniel Kang's avatar
Daniel Kang committed
795
  int d;
Deb Mukherjee's avatar
Deb Mukherjee committed
796
  MACROBLOCKD *xd = &mb->e_mbd;
Daniel Kang's avatar
Daniel Kang committed
797

798 799
  xd->mode_info_context->mbmi.txfm_size = TX_16X16;
  vp9_transform_mby_16x16(mb);
800
  vp9_quantize_mby_16x16(mb);
801 802 803
  // TODO(jingning) is it possible to quickly determine whether to force
  //                trailing coefficients to be zero, instead of running trellis
  //                optimization in the rate-distortion optimization loop?
804
  if (mb->e_mbd.mode_info_context->mbmi.mode < I8X8_PRED)
805
    vp9_optimize_mby_16x16(mb);
806

807
  d = vp9_mbblock_error(mb, 0);
Daniel Kang's avatar
Daniel Kang committed
808 809 810

  *Distortion = (d >> 2);
  // rate
811
  *Rate = rdcost_mby_16x16(mb, backup);
812
  *skippable = vp9_mby_is_skippable_16x16(&mb->e_mbd);
Daniel Kang's avatar
Daniel Kang committed
813 814
}

815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847
static void choose_txfm_size_from_rd(VP9_COMP *cpi, MACROBLOCK *x,
                                     int r[2][TX_SIZE_MAX], int *rate,
                                     int d[TX_SIZE_MAX], int *distortion,
                                     int s[TX_SIZE_MAX], int *skip,
                                     int64_t txfm_cache[NB_TXFM_MODES]) {
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
  vp9_prob skip_prob = cm->mb_no_coeff_skip ?
                       vp9_get_pred_prob(cm, xd, PRED_MBSKIP) : 128;
  int64_t rd[2][TX_SIZE_MAX];
  int n;

  r[1][TX_16X16] = r[0][TX_16X16] + vp9_cost_one(cm->prob_tx[0]) +
                   vp9_cost_one(cm->prob_tx[1]);
  r[1][TX_8X8]   = r[0][TX_8X8] + vp9_cost_one(cm->prob_tx[0]) +
                   vp9_cost_zero(cm->prob_tx[1]);
  r[1][TX_4X4]   = r[0][TX_4X4] + vp9_cost_zero(cm->prob_tx[0]);

  if (cm->mb_no_coeff_skip) {
    int s0, s1;

    assert(skip_prob > 0);
    s0 = vp9_cost_bit(skip_prob, 0);
    s1 = vp9_cost_bit(skip_prob, 1);

    for (n = TX_4X4; n <= TX_16X16; n++) {
      if (s[n]) {
        rd[0][n] = rd[1][n] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
      } else {
        rd[0][n] = RDCOST(x->rdmult, x->rddiv, r[0][n] + s0, d[n]);
        rd[1][n] = RDCOST(x->rdmult, x->rddiv, r[1][n] + s0, d[n]);
      }
848 849
    }
  } else {
850 851 852
    for (n = TX_4X4; n <= TX_16X16; n++) {
      rd[0][n] = RDCOST(x->rdmult, x->rddiv, r[0][n], d[n]);
      rd[1][n] = RDCOST(x->rdmult, x->rddiv, r[1][n], d[n]);
853 854 855
    }
  }

856 857 858
  if ( cm->txfm_mode == ALLOW_16X16 ||
      (cm->txfm_mode == TX_MODE_SELECT &&
       rd[1][TX_16X16] < rd[1][TX_8X8] && rd[1][TX_16X16] < rd[1][TX_4X4])) {
859
    mbmi->txfm_size = TX_16X16;
860 861
  } else if (cm->txfm_mode == ALLOW_8X8 ||
           (cm->txfm_mode == TX_MODE_SELECT && rd[1][TX_8X8] < rd[1][TX_4X4])) {
862 863
    mbmi->txfm_size = TX_8X8;
  } else {
864 865
    assert(cm->txfm_mode == ONLY_4X4 ||
          (cm->txfm_mode == TX_MODE_SELECT && rd[1][TX_4X4] <= rd[1][TX_8X8]));
866 867 868
    mbmi->txfm_size = TX_4X4;
  }

869 870 871 872 873 874 875 876 877
  *distortion = d[mbmi->txfm_size];
  *rate       = r[cm->txfm_mode == TX_MODE_SELECT][mbmi->txfm_size];
  *skip       = s[mbmi->txfm_size];

  txfm_cache[ONLY_4X4] = rd[0][TX_4X4];
  txfm_cache[ALLOW_8X8] = rd[0][TX_8X8];
  txfm_cache[ALLOW_16X16] = rd[0][TX_16X16];
  if (rd[1][TX_16X16] < rd[1][TX_8X8] && rd[1][TX_16X16] < rd[1][TX_4X4])
    txfm_cache[TX_MODE_SELECT] = rd[1][TX_16X16];
878
  else
879 880 881 882 883 884 885 886 887 888 889 890
    txfm_cache[TX_MODE_SELECT] = rd[1][TX_4X4] < rd[1][TX_8X8] ?
                                 rd[1][TX_4X4] : rd[1][TX_8X8];
}

static void macro_block_yrd(VP9_COMP *cpi, MACROBLOCK *x, int *rate,
                            int *distortion, int *skippable,
                            int64_t txfm_cache[NB_TXFM_MODES]) {
  MACROBLOCKD *const xd = &x->e_mbd;
  int r[2][TX_SIZE_MAX], d[TX_SIZE_MAX], s[TX_SIZE_MAX];

  vp9_subtract_mby(x->src_diff, *(x->block[0].base_src), xd->predictor,
                   x->block[0].src_stride);