vp9_rdopt.c 164 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5 6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9 10 11 12 13 14 15
 */


#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
16 17
#include "vp9/common/vp9_pragmas.h"

18 19 20 21 22
#include "vp9/encoder/vp9_tokenize.h"
#include "vp9/encoder/vp9_treewriter.h"
#include "vp9/encoder/vp9_onyx_int.h"
#include "vp9/encoder/vp9_modecosts.h"
#include "vp9/encoder/vp9_encodeintra.h"
23 24 25 26 27 28
#include "vp9/common/vp9_entropymode.h"
#include "vp9/common/vp9_reconinter.h"
#include "vp9/common/vp9_reconintra.h"
#include "vp9/common/vp9_reconintra4x4.h"
#include "vp9/common/vp9_findnearmv.h"
#include "vp9/common/vp9_quant_common.h"
29 30 31 32 33 34
#include "vp9/encoder/vp9_encodemb.h"
#include "vp9/encoder/vp9_quantize.h"
#include "vp9/encoder/vp9_variance.h"
#include "vp9/encoder/vp9_mcomp.h"
#include "vp9/encoder/vp9_rdopt.h"
#include "vp9/encoder/vp9_ratectrl.h"
John Koleszar's avatar
John Koleszar committed
35
#include "vpx_mem/vpx_mem.h"
36 37
#include "vp9/common/vp9_systemdependent.h"
#include "vp9/encoder/vp9_encodemv.h"
John Koleszar's avatar
John Koleszar committed
38

39 40 41
#include "vp9/common/vp9_seg_common.h"
#include "vp9/common/vp9_pred_common.h"
#include "vp9/common/vp9_entropy.h"
42
#include "vp9_rtcd.h"
43
#include "vp9/common/vp9_mvref_common.h"
Paul Wilkins's avatar
Paul Wilkins committed
44

John Koleszar's avatar
John Koleszar committed
45 46
#define MAXF(a,b)            (((a) > (b)) ? (a) : (b))

47 48
#define INVALID_MV 0x80008000

49 50 51
/* Factor to weigh the rate for switchable interp filters */
#define SWITCHABLE_INTERP_RATE_FACTOR 1

John Koleszar's avatar
John Koleszar committed
52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69
static const int auto_speed_thresh[17] = {
  1000,
  200,
  150,
  130,
  150,
  125,
  120,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  105
John Koleszar's avatar
John Koleszar committed
70 71
};

72
#if CONFIG_PRED_FILTER
73
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121
  {ZEROMV,    LAST_FRAME,   NONE,  0},
  {ZEROMV,    LAST_FRAME,   NONE,  1},
  {DC_PRED,   INTRA_FRAME,  NONE,  0},

  {NEARESTMV, LAST_FRAME,   NONE,  0},
  {NEARESTMV, LAST_FRAME,   NONE,  1},
  {NEARMV,    LAST_FRAME,   NONE,  0},
  {NEARMV,    LAST_FRAME,   NONE,  1},

  {ZEROMV,    GOLDEN_FRAME, NONE,  0},
  {ZEROMV,    GOLDEN_FRAME, NONE,  1},
  {NEARESTMV, GOLDEN_FRAME, NONE,  0},
  {NEARESTMV, GOLDEN_FRAME, NONE,  1},

  {ZEROMV,    ALTREF_FRAME, NONE,  0},
  {ZEROMV,    ALTREF_FRAME, NONE,  1},
  {NEARESTMV, ALTREF_FRAME, NONE,  0},
  {NEARESTMV, ALTREF_FRAME, NONE,  1},

  {NEARMV,    GOLDEN_FRAME, NONE,  0},
  {NEARMV,    GOLDEN_FRAME, NONE,  1},
  {NEARMV,    ALTREF_FRAME, NONE,  0},
  {NEARMV,    ALTREF_FRAME, NONE,  1},

  {V_PRED,    INTRA_FRAME,  NONE,  0},
  {H_PRED,    INTRA_FRAME,  NONE,  0},
  {D45_PRED,  INTRA_FRAME,  NONE,  0},
  {D135_PRED, INTRA_FRAME,  NONE,  0},
  {D117_PRED, INTRA_FRAME,  NONE,  0},
  {D153_PRED, INTRA_FRAME,  NONE,  0},
  {D27_PRED,  INTRA_FRAME,  NONE,  0},
  {D63_PRED,  INTRA_FRAME,  NONE,  0},

  {TM_PRED,   INTRA_FRAME,  NONE,  0},

  {NEWMV,     LAST_FRAME,   NONE,  0},
  {NEWMV,     LAST_FRAME,   NONE,  1},
  {NEWMV,     GOLDEN_FRAME, NONE,  0},
  {NEWMV,     GOLDEN_FRAME, NONE,  1},
  {NEWMV,     ALTREF_FRAME, NONE,  0},
  {NEWMV,     ALTREF_FRAME, NONE,  1},

  {SPLITMV,   LAST_FRAME,   NONE,  0},
  {SPLITMV,   GOLDEN_FRAME, NONE,  0},
  {SPLITMV,   ALTREF_FRAME, NONE,  0},

  {B_PRED,    INTRA_FRAME,  NONE,  0},
  {I8X8_PRED, INTRA_FRAME,  NONE,  0},
John Koleszar's avatar
John Koleszar committed
122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141

  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME, 0},

  {ZEROMV,    ALTREF_FRAME, LAST_FRAME,   0},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME,   0},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME,   0},

  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME, 0},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME, 0},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME, 0},

  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME,   0},
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME, 0},

  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME, 0},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME,   0},
142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME, 0},

#if CONFIG_COMP_INTERINTRA_PRED
  /* compound inter-intra prediction */
  {ZEROMV,    LAST_FRAME,   INTRA_FRAME, 0},
  {NEARESTMV, LAST_FRAME,   INTRA_FRAME, 0},
  {NEARMV,    LAST_FRAME,   INTRA_FRAME, 0},
  {NEWMV,     LAST_FRAME,   INTRA_FRAME, 0},

  {ZEROMV,    GOLDEN_FRAME,   INTRA_FRAME, 0},
  {NEARESTMV, GOLDEN_FRAME,   INTRA_FRAME, 0},
  {NEARMV,    GOLDEN_FRAME,   INTRA_FRAME, 0},
  {NEWMV,     GOLDEN_FRAME,   INTRA_FRAME, 0},

  {ZEROMV,    ALTREF_FRAME,   INTRA_FRAME, 0},
  {NEARESTMV, ALTREF_FRAME,   INTRA_FRAME, 0},
  {NEARMV,    ALTREF_FRAME,   INTRA_FRAME, 0},
  {NEWMV,     ALTREF_FRAME,   INTRA_FRAME, 0},
#endif
John Koleszar's avatar
John Koleszar committed
161
};
162
#else
163
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
164 165
  {ZEROMV,    LAST_FRAME,   NONE},
  {DC_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
166

167 168
  {NEARESTMV, LAST_FRAME,   NONE},
  {NEARMV,    LAST_FRAME,   NONE},
John Koleszar's avatar
John Koleszar committed
169

170 171
  {ZEROMV,    GOLDEN_FRAME, NONE},
  {NEARESTMV, GOLDEN_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
172

173 174
  {ZEROMV,    ALTREF_FRAME, NONE},
  {NEARESTMV, ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
175

176 177
  {NEARMV,    GOLDEN_FRAME, NONE},
  {NEARMV,    ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
178

179 180 181 182 183 184 185 186
  {V_PRED,    INTRA_FRAME,  NONE},
  {H_PRED,    INTRA_FRAME,  NONE},
  {D45_PRED,  INTRA_FRAME,  NONE},
  {D135_PRED, INTRA_FRAME,  NONE},
  {D117_PRED, INTRA_FRAME,  NONE},
  {D153_PRED, INTRA_FRAME,  NONE},
  {D27_PRED,  INTRA_FRAME,  NONE},
  {D63_PRED,  INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
187

188
  {TM_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
189

190 191 192
  {NEWMV,     LAST_FRAME,   NONE},
  {NEWMV,     GOLDEN_FRAME, NONE},
  {NEWMV,     ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
193

194 195 196
  {SPLITMV,   LAST_FRAME,   NONE},
  {SPLITMV,   GOLDEN_FRAME, NONE},
  {SPLITMV,   ALTREF_FRAME, NONE},
197

198 199
  {B_PRED,    INTRA_FRAME,  NONE},
  {I8X8_PRED, INTRA_FRAME,  NONE},
200

John Koleszar's avatar
John Koleszar committed
201 202 203 204
  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME},
205

John Koleszar's avatar
John Koleszar committed
206 207 208
  {ZEROMV,    ALTREF_FRAME, LAST_FRAME},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME},
209

John Koleszar's avatar
John Koleszar committed
210 211 212
  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME},
213

John Koleszar's avatar
John Koleszar committed
214 215 216
  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME  },
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME},
217

John Koleszar's avatar
John Koleszar committed
218 219
  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME  },
220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME},

#if CONFIG_COMP_INTERINTRA_PRED
  /* compound inter-intra prediction */
  {ZEROMV,    LAST_FRAME,   INTRA_FRAME},
  {NEARESTMV, LAST_FRAME,   INTRA_FRAME},
  {NEARMV,    LAST_FRAME,   INTRA_FRAME},
  {NEWMV,     LAST_FRAME,   INTRA_FRAME},

  {ZEROMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEARESTMV, GOLDEN_FRAME,   INTRA_FRAME},
  {NEARMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEWMV,     GOLDEN_FRAME,   INTRA_FRAME},

  {ZEROMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEARESTMV, ALTREF_FRAME,   INTRA_FRAME},
  {NEARMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEWMV,     ALTREF_FRAME,   INTRA_FRAME},
#endif
John Koleszar's avatar
John Koleszar committed
239
};
240
#endif
John Koleszar's avatar
John Koleszar committed
241 242

static void fill_token_costs(
243
  unsigned int (*c)[COEF_BANDS][PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS],
244
  const vp9_prob(*p)[COEF_BANDS][PREV_COEF_CONTEXTS][ENTROPY_NODES],
John Koleszar's avatar
John Koleszar committed
245 246 247 248 249 250 251
  int block_type_counts) {
  int i, j, k;

  for (i = 0; i < block_type_counts; i++)
    for (j = 0; j < COEF_BANDS; j++)
      for (k = 0; k < PREV_COEF_CONTEXTS; k++) {
        if (k == 0 && ((j > 0 && i > 0) || (j > 1 && i == 0)))
252 253
          vp9_cost_tokens_skip((int *)(c[i][j][k]),
                               p[i][j][k],
254
                               vp9_coef_tree);
John Koleszar's avatar
John Koleszar committed
255
        else
256 257
          vp9_cost_tokens((int *)(c[i][j][k]),
                          p[i][j][k],
258
                          vp9_coef_tree);
John Koleszar's avatar
John Koleszar committed
259
      }
John Koleszar's avatar
John Koleszar committed
260 261
}

262

263 264 265 266
static int rd_iifactor[32] =  { 4, 4, 3, 2, 1, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0, };
John Koleszar's avatar
John Koleszar committed
267

268
// 3* dc_qlookup[Q]*dc_qlookup[Q];
269

270
/* values are now correlated to quantizer */
Paul Wilkins's avatar
Paul Wilkins committed
271 272 273
static int sad_per_bit16lut[QINDEX_RANGE];
static int sad_per_bit4lut[QINDEX_RANGE];

274
void vp9_init_me_luts() {
John Koleszar's avatar
John Koleszar committed
275 276 277 278 279 280 281
  int i;

  // Initialize the sad lut tables using a formulaic calculation for now
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < QINDEX_RANGE; i++) {
    sad_per_bit16lut[i] =
282 283
      (int)((0.0418 * vp9_convert_qindex_to_q(i)) + 2.4107);
    sad_per_bit4lut[i] = (int)((0.063 * vp9_convert_qindex_to_q(i)) + 2.742);
John Koleszar's avatar
John Koleszar committed
284
  }
Paul Wilkins's avatar
Paul Wilkins committed
285
}
John Koleszar's avatar
John Koleszar committed
286

287
static int compute_rd_mult(int qindex) {
John Koleszar's avatar
John Koleszar committed
288
  int q;
289

290
  q = vp9_dc_quant(qindex, 0);
John Koleszar's avatar
John Koleszar committed
291
  return (11 * q * q) >> 6;
292 293
}

294
void vp9_initialize_me_consts(VP9_COMP *cpi, int QIndex) {
John Koleszar's avatar
John Koleszar committed
295 296
  cpi->mb.sadperbit16 =  sad_per_bit16lut[QIndex];
  cpi->mb.sadperbit4  =  sad_per_bit4lut[QIndex];
John Koleszar's avatar
John Koleszar committed
297 298
}

299

300
void vp9_initialize_rd_consts(VP9_COMP *cpi, int QIndex) {
301
  int q, i;
John Koleszar's avatar
John Koleszar committed
302

303
  vp9_clear_system_state();  // __asm emms;
John Koleszar's avatar
John Koleszar committed
304

John Koleszar's avatar
John Koleszar committed
305 306 307 308 309
  // Further tests required to see if optimum is different
  // for key frames, golden frames and arf frames.
  // if (cpi->common.refresh_golden_frame ||
  //     cpi->common.refresh_alt_ref_frame)
  QIndex = (QIndex < 0) ? 0 : ((QIndex > MAXQ) ? MAXQ : QIndex);
310

John Koleszar's avatar
John Koleszar committed
311
  cpi->RDMULT = compute_rd_mult(QIndex);
John Koleszar's avatar
John Koleszar committed
312

John Koleszar's avatar
John Koleszar committed
313 314 315
  // Extend rate multiplier along side quantizer zbin increases
  if (cpi->zbin_over_quant  > 0) {
    double oq_factor;
316

John Koleszar's avatar
John Koleszar committed
317 318 319 320 321
    // Experimental code using the same basic equation as used for Q above
    // The units of cpi->zbin_over_quant are 1/128 of Q bin size
    oq_factor = 1.0 + ((double)0.0015625 * cpi->zbin_over_quant);
    cpi->RDMULT = (int)((double)cpi->RDMULT * oq_factor * oq_factor);
  }
John Koleszar's avatar
John Koleszar committed
322

John Koleszar's avatar
John Koleszar committed
323 324 325 326 327 328 329
  if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    if (cpi->twopass.next_iiratio > 31)
      cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
    else
      cpi->RDMULT +=
        (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
  }
John Koleszar's avatar
John Koleszar committed
330

John Koleszar's avatar
John Koleszar committed
331 332
  if (cpi->RDMULT < 7)
    cpi->RDMULT = 7;
333

John Koleszar's avatar
John Koleszar committed
334 335
  cpi->mb.errorperbit = (cpi->RDMULT / 110);
  cpi->mb.errorperbit += (cpi->mb.errorperbit == 0);
336

337
  vp9_set_speed_features(cpi);
John Koleszar's avatar
John Koleszar committed
338

339
  q = (int)pow(vp9_dc_quant(QIndex, 0) >> 2, 1.25);
John Koleszar's avatar
John Koleszar committed
340 341
  q = q << 2;
  cpi->RDMULT = cpi->RDMULT << 4;
Paul Wilkins's avatar
Paul Wilkins committed
342

John Koleszar's avatar
John Koleszar committed
343 344
  if (q < 8)
    q = 8;
345

John Koleszar's avatar
John Koleszar committed
346 347 348
  if (cpi->RDMULT > 1000) {
    cpi->RDDIV = 1;
    cpi->RDMULT /= 100;
John Koleszar's avatar
John Koleszar committed
349

John Koleszar's avatar
John Koleszar committed
350 351 352 353 354 355
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < INT_MAX) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q / 100;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
356

John Koleszar's avatar
John Koleszar committed
357
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
358
    }
John Koleszar's avatar
John Koleszar committed
359 360
  } else {
    cpi->RDDIV = 100;
John Koleszar's avatar
John Koleszar committed
361

John Koleszar's avatar
John Koleszar committed
362 363 364 365 366 367
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < (INT_MAX / q)) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
368

John Koleszar's avatar
John Koleszar committed
369
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
370
    }
John Koleszar's avatar
John Koleszar committed
371
  }
John Koleszar's avatar
John Koleszar committed
372

John Koleszar's avatar
John Koleszar committed
373
  fill_token_costs(
374
    cpi->mb.token_costs[TX_4X4],
375
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs,
John Koleszar's avatar
John Koleszar committed
376
    BLOCK_TYPES);
377 378
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_4X4],
379
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11])
380 381
    cpi->common.fc.hybrid_coef_probs,
    BLOCK_TYPES);
John Koleszar's avatar
John Koleszar committed
382

John Koleszar's avatar
John Koleszar committed
383
  fill_token_costs(
384
    cpi->mb.token_costs[TX_8X8],
385
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs_8x8,
John Koleszar's avatar
John Koleszar committed
386
    BLOCK_TYPES_8X8);
387 388
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_8X8],
389
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11])
390 391
    cpi->common.fc.hybrid_coef_probs_8x8,
    BLOCK_TYPES_8X8);
392

Daniel Kang's avatar
Daniel Kang committed
393
  fill_token_costs(
394
    cpi->mb.token_costs[TX_16X16],
395
    (const vp9_prob(*)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs_16x16,
Daniel Kang's avatar
Daniel Kang committed
396
    BLOCK_TYPES_16X16);
397 398
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_16X16],
399
    (const vp9_prob(*)[8][PREV_COEF_CONTEXTS][11])
400 401
    cpi->common.fc.hybrid_coef_probs_16x16,
    BLOCK_TYPES_16X16);
Daniel Kang's avatar
Daniel Kang committed
402

John Koleszar's avatar
John Koleszar committed
403 404
  /*rough estimate for costing*/
  cpi->common.kf_ymode_probs_index = cpi->common.base_qindex >> 4;
405
  vp9_init_mode_costs(cpi);
John Koleszar's avatar
John Koleszar committed
406

407 408
  if (cpi->common.frame_type != KEY_FRAME)
  {
409
    vp9_build_nmv_cost_table(
410 411 412 413 414 415
        cpi->mb.nmvjointcost,
        cpi->mb.e_mbd.allow_high_precision_mv ?
        cpi->mb.nmvcost_hp : cpi->mb.nmvcost,
        &cpi->common.fc.nmvc,
        cpi->mb.e_mbd.allow_high_precision_mv, 1, 1);
  }
John Koleszar's avatar
John Koleszar committed
416 417
}

418
int vp9_block_error_c(short *coeff, short *dqcoeff, int block_size) {
419
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
420

421
  for (i = 0; i < block_size; i++) {
John Koleszar's avatar
John Koleszar committed
422 423 424
    int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }
John Koleszar's avatar
John Koleszar committed
425

John Koleszar's avatar
John Koleszar committed
426
  return error;
John Koleszar's avatar
John Koleszar committed
427 428
}

429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447
int vp9_mbblock_error_8x8_c(MACROBLOCK *mb, int dc) {
  BLOCK  *be;
  BLOCKD *bd;
  int i, j;
  int berror, error = 0;

  for (i = 0; i < 16; i+=4) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
    berror = 0;
    for (j = dc; j < 64; j++) {
      int this_diff = be->coeff[j] - bd->dqcoeff[j];
      berror += this_diff * this_diff;
    }
    error += berror;
  }
  return error;
}

448
int vp9_mbblock_error_c(MACROBLOCK *mb, int dc) {
John Koleszar's avatar
John Koleszar committed
449 450 451 452
  BLOCK  *be;
  BLOCKD *bd;
  int i, j;
  int berror, error = 0;
John Koleszar's avatar
John Koleszar committed
453

John Koleszar's avatar
John Koleszar committed
454 455 456 457 458 459 460
  for (i = 0; i < 16; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
    berror = 0;
    for (j = dc; j < 16; j++) {
      int this_diff = be->coeff[j] - bd->dqcoeff[j];
      berror += this_diff * this_diff;
John Koleszar's avatar
John Koleszar committed
461
    }
John Koleszar's avatar
John Koleszar committed
462 463 464
    error += berror;
  }
  return error;
John Koleszar's avatar
John Koleszar committed
465 466
}

467
int vp9_mbuverror_c(MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
468 469
  BLOCK  *be;
  BLOCKD *bd;
John Koleszar's avatar
John Koleszar committed
470

471
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
472

John Koleszar's avatar
John Koleszar committed
473 474 475
  for (i = 16; i < 24; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
John Koleszar's avatar
John Koleszar committed
476

477
    error += vp9_block_error_c(be->coeff, bd->dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
478
  }
John Koleszar's avatar
John Koleszar committed
479

John Koleszar's avatar
John Koleszar committed
480
  return error;
John Koleszar's avatar
John Koleszar committed
481 482
}

483
int vp9_uvsse(MACROBLOCK *x) {
John Koleszar's avatar
John Koleszar committed
484 485 486 487 488 489 490
  unsigned char *uptr, *vptr;
  unsigned char *upred_ptr = (*(x->block[16].base_src) + x->block[16].src);
  unsigned char *vpred_ptr = (*(x->block[20].base_src) + x->block[20].src);
  int uv_stride = x->block[16].src_stride;

  unsigned int sse1 = 0;
  unsigned int sse2 = 0;
491 492
  int mv_row = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.row;
  int mv_col = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.col;
John Koleszar's avatar
John Koleszar committed
493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513
  int offset;
  int pre_stride = x->e_mbd.block[16].pre_stride;

  if (mv_row < 0)
    mv_row -= 1;
  else
    mv_row += 1;

  if (mv_col < 0)
    mv_col -= 1;
  else
    mv_col += 1;

  mv_row /= 2;
  mv_col /= 2;

  offset = (mv_row >> 3) * pre_stride + (mv_col >> 3);
  uptr = x->e_mbd.pre.u_buffer + offset;
  vptr = x->e_mbd.pre.v_buffer + offset;

  if ((mv_row | mv_col) & 7) {
514
    vp9_sub_pixel_variance8x8(uptr, pre_stride, (mv_col & 7) << 1,
515
                              (mv_row & 7) << 1, upred_ptr, uv_stride, &sse2);
516
    vp9_sub_pixel_variance8x8(vptr, pre_stride, (mv_col & 7) << 1,
517
                              (mv_row & 7) << 1, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
518 519
    sse2 += sse1;
  } else {
520 521
    vp9_variance8x8(uptr, pre_stride, upred_ptr, uv_stride, &sse2);
    vp9_variance8x8(vptr, pre_stride, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
522 523 524
    sse2 += sse1;
  }
  return sse2;
John Koleszar's avatar
John Koleszar committed
525 526 527

}

528
static int cost_coeffs_2x2(MACROBLOCK *mb,
529
                           BLOCKD *b, PLANE_TYPE type,
530
                           ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l) {
531
  int c = (type == PLANE_TYPE_Y_NO_DC); /* start at coef 0, unless Y with Y2 */
532 533 534 535 536
  int eob = b->eob;
  int pt;    /* surrounding block/prev coef predictor */
  int cost = 0;
  short *qcoeff_ptr = b->qcoeff;

537
  VP9_COMBINEENTROPYCONTEXTS(pt, *a, *l);
538 539 540
  assert(eob <= 4);

  for (; c < eob; c++) {
541 542 543 544 545
    int v = qcoeff_ptr[vp9_default_zig_zag1d[c]];
    int t = vp9_dct_value_tokens_ptr[v].Token;
    cost += mb->token_costs[TX_8X8][type][vp9_coef_bands[c]][pt][t];
    cost += vp9_dct_value_cost_ptr[v];
    pt = vp9_prev_token_class[t];
546 547 548
  }

  if (c < 4)
549
    cost += mb->token_costs[TX_8X8][type][vp9_coef_bands[c]]
550 551 552 553 554 555 556
            [pt] [DCT_EOB_TOKEN];

  pt = (c != !type); // is eob first coefficient;
  *a = *l = pt;
  return cost;
}

557
static int cost_coeffs(MACROBLOCK *mb, BLOCKD *b, PLANE_TYPE type,
Daniel Kang's avatar
Daniel Kang committed
558
                       ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l,
559
                       int tx_size) {
Daniel Kang's avatar
Daniel Kang committed
560
  const int eob = b->eob;
561
  int c = (type == PLANE_TYPE_Y_NO_DC); /* start at coef 0, unless Y with Y2 */
562
  int cost = 0, default_eob, seg_eob;
Daniel Kang's avatar
Daniel Kang committed
563 564
  int pt;                     /* surrounding block/prev coef predictor */
  int const *scan, *band;
John Koleszar's avatar
John Koleszar committed
565
  short *qcoeff_ptr = b->qcoeff;
566 567 568
  MACROBLOCKD *xd = &mb->e_mbd;
  MB_MODE_INFO *mbmi = &mb->e_mbd.mode_info_context->mbmi;
  TX_TYPE tx_type = DCT_DCT;
Paul Wilkins's avatar
Paul Wilkins committed
569
  int segment_id = mbmi->segment_id;
570

571
  switch (tx_size) {
Daniel Kang's avatar
Daniel Kang committed
572
    case TX_4X4:
573 574
      scan = vp9_default_zig_zag1d;
      band = vp9_coef_bands;
Daniel Kang's avatar
Daniel Kang committed
575
      default_eob = 16;
Deb Mukherjee's avatar
Deb Mukherjee committed
576 577 578 579 580
      if (type == PLANE_TYPE_Y_WITH_DC) {
        tx_type = get_tx_type_4x4(xd, b);
        if (tx_type != DCT_DCT) {
          switch (tx_type) {
            case ADST_DCT:
581
              scan = vp9_row_scan;
Deb Mukherjee's avatar
Deb Mukherjee committed
582 583 584
              break;

            case DCT_ADST:
585
              scan = vp9_col_scan;
Deb Mukherjee's avatar
Deb Mukherjee committed
586 587 588
              break;

            default:
589
              scan = vp9_default_zig_zag1d;
Deb Mukherjee's avatar
Deb Mukherjee committed
590 591
              break;
          }
592
        }
Daniel Kang's avatar
Daniel Kang committed
593
      }
Deb Mukherjee's avatar
Deb Mukherjee committed
594

Daniel Kang's avatar
Daniel Kang committed
595 596
      break;
    case TX_8X8:
597 598
      scan = vp9_default_zig_zag1d_8x8;
      band = vp9_coef_bands_8x8;
Daniel Kang's avatar
Daniel Kang committed
599
      default_eob = 64;
Deb Mukherjee's avatar
Deb Mukherjee committed
600
      if (type == PLANE_TYPE_Y_WITH_DC) {
601
        BLOCKD *bb;
602
        int ib = (int)(b - xd->block);
603 604 605
        if (ib < 16) {
          ib = (ib & 8) + ((ib & 4) >> 1);
          bb = xd->block + ib;
Deb Mukherjee's avatar
Deb Mukherjee committed
606
          tx_type = get_tx_type_8x8(xd, bb);
607
        }
608
      }
Daniel Kang's avatar
Daniel Kang committed
609 610
      break;
    case TX_16X16:
611 612
      scan = vp9_default_zig_zag1d_16x16;
      band = vp9_coef_bands_16x16;
Daniel Kang's avatar
Daniel Kang committed
613
      default_eob = 256;
Deb Mukherjee's avatar
Deb Mukherjee committed
614 615 616
      if (type == PLANE_TYPE_Y_WITH_DC) {
        tx_type = get_tx_type_16x16(xd, b);
      }
Daniel Kang's avatar
Daniel Kang committed
617 618 619 620
      break;
    default:
      break;
  }
621 622
  if (vp9_segfeature_active(&mb->e_mbd, segment_id, SEG_LVL_EOB))
    seg_eob = vp9_get_segdata(&mb->e_mbd, segment_id, SEG_LVL_EOB);
623 624 625
  else
    seg_eob = default_eob;

626
  VP9_COMBINEENTROPYCONTEXTS(pt, *a, *l);
627

628 629 630
  if (tx_type != DCT_DCT) {
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
631
      int t = vp9_dct_value_tokens_ptr[v].Token;
632
      cost += mb->hybrid_token_costs[tx_size][type][band[c]][pt][t];
633 634
      cost += vp9_dct_value_cost_ptr[v];
      pt = vp9_prev_token_class[t];
635 636 637 638
    }
    if (c < seg_eob)
      cost += mb->hybrid_token_costs[tx_size][type][band[c]]
          [pt][DCT_EOB_TOKEN];
639
  } else {
640 641
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
642
      int t = vp9_dct_value_tokens_ptr[v].Token;
643
      cost += mb->token_costs[tx_size][type][band[c]][pt][t];
644 645
      cost += vp9_dct_value_cost_ptr[v];
      pt = vp9_prev_token_class[t];
646 647 648 649
    }
    if (c < seg_eob)
      cost += mb->token_costs[tx_size][type][band[c]]
          [pt][DCT_EOB_TOKEN];
650 651 652 653 654 655 656
  }

  pt = (c != !type); // is eob first coefficient;
  *a = *l = pt;
  return cost;
}

657
static int rdcost_mby_4x4(MACROBLOCK *mb, int has_2nd_order, int backup) {
John Koleszar's avatar
John Koleszar committed
658 659
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
660
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
661 662 663
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta;
  ENTROPY_CONTEXT *tl;
664

665 666 667
  if (backup) {
    vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
668

669 670 671 672 673 674
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)xd->above_context;
    tl = (ENTROPY_CONTEXT *)xd->left_context;
  }
John Koleszar's avatar
John Koleszar committed
675

John Koleszar's avatar
John Koleszar committed
676
  for (b = 0; b < 16; b++)
677 678 679
    cost += cost_coeffs(mb, xd->block + b,
                        (has_2nd_order ?
                         PLANE_TYPE_Y_NO_DC : PLANE_TYPE_Y_WITH_DC),
680
                        ta + vp9_block2above[b], tl + vp9_block2left[b],
Daniel Kang's avatar
Daniel Kang committed
681
                        TX_4X4);
John Koleszar's avatar
John Koleszar committed
682

683 684 685 686
  if (has_2nd_order)
    cost += cost_coeffs(mb, xd->block + 24, PLANE_TYPE_Y2,
                        ta + vp9_block2above[24], tl + vp9_block2left[24],
                        TX_4X4);
John Koleszar's avatar
John Koleszar committed
687

John Koleszar's avatar
John Koleszar committed
688
  return cost;
John Koleszar's avatar
John Koleszar committed
689 690
}

691 692 693
static void macro_block_yrd_4x4(MACROBLOCK *mb,
                                int *Rate,
                                int *Distortion,
694
                                int *skippable, int backup) {
John Koleszar's avatar
John Koleszar committed
695
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
696
  MACROBLOCKD *const xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
697
  BLOCK   *const mb_y2 = mb->block + 24;
Paul Wilkins's avatar
Paul Wilkins committed
698
  BLOCKD *const x_y2  = xd->block + 24;
John Koleszar's avatar
John Koleszar committed
699 700
  short *Y2DCPtr = mb_y2->src_diff;
  BLOCK *beptr;
701
  int d, i, has_2nd_order;
702

703 704
  xd->mode_info_context->mbmi.txfm_size = TX_4X4;
  has_2nd_order = get_2nd_order_usage(xd);
John Koleszar's avatar
John Koleszar committed
705
  // Fdct and building the 2nd order block
706 707 708 709 710
  vp9_transform_mby_4x4(mb);
  vp9_quantize_mby_4x4(mb);
  d = vp9_mbblock_error(mb, has_2nd_order);
  if (has_2nd_order)
    d += vp9_block_error(mb_y2->coeff, x_y2->dqcoeff, 16);
711

John Koleszar's avatar
John Koleszar committed
712 713
  *Distortion = (d >> 2);
  // rate
714 715
  *Rate = rdcost_mby_4x4(mb, has_2nd_order, backup);
  *skippable = vp9_mby_is_skippable_4x4(&mb->e_mbd, has_2nd_order);
716
}
John Koleszar's avatar
John Koleszar committed
717

718
static int rdcost_mby_8x8(MACROBLOCK *mb, int has_2nd_order, int backup) {
John Koleszar's avatar
John Koleszar committed
719 720
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
721
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
722 723 724 725
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta;
  ENTROPY_CONTEXT *tl;

Ronald S. Bultje's avatar
Ronald S. Bultje committed
726 727 728
  if (backup) {
    vpx_memcpy(&t_above,xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
729

Ronald S. Bultje's avatar
Ronald S. Bultje committed
730 731 732 733 734 735
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)mb->e_mbd.above_context;
    tl = (ENTROPY_CONTEXT *)mb->e_mbd.left_context;
  }
John Koleszar's avatar
John Koleszar committed
736 737

  for (b = 0; b < 16; b += 4)
738 739 740
    cost += cost_coeffs(mb, xd->block + b,
                        (has_2nd_order ?
                         PLANE_TYPE_Y_NO_DC : PLANE_TYPE_Y_WITH_DC),
741
                        ta + vp9_block2above_8x8[b], tl + vp9_block2left_8x8[b],
Daniel Kang's avatar
Daniel Kang committed
742
                        TX_8X8);
John Koleszar's avatar
John Koleszar committed
743

744 745 746
  if (has_2nd_order)
    cost += cost_coeffs_2x2(mb, xd->block + 24, PLANE_TYPE_Y2,
                            ta + vp9_block2above[24], tl + vp9_block2left[24]);
John Koleszar's avatar
John Koleszar committed
747
  return cost;
748 749
}

John Koleszar's avatar
John Koleszar committed
750 751 752
static void macro_block_yrd_8x8(MACROBLOCK *mb,
                                int *Rate,
                                int *Distortion,
753
                                int *skippable, int backup) {
Paul Wilkins's avatar
Paul Wilkins committed
754
  MACROBLOCKD *const xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
755
  BLOCK   *const mb_y2 = mb->block + 24;
Paul Wilkins's avatar
Paul Wilkins committed
756
  BLOCKD *const x_y2  = xd->block + 24;
757 758 759
  int d, has_2nd_order;

  xd->mode_info_context->mbmi.txfm_size = TX_8X8;
John Koleszar's avatar
John Koleszar committed
760

761 762
  vp9_transform_mby_8x8(mb);
  vp9_quantize_mby_8x8(mb);
763 764 765 766
  has_2nd_order = get_2nd_order_usage(xd);
  d = vp9_mbblock_error_8x8_c(mb, has_2nd_order);
  if (has_2nd_order)
    d += vp9_block_error(mb_y2->coeff, x_y2->dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
767 768 769

  *Distortion = (d >> 2);
  // rate
770 771
  *Rate = rdcost_mby_8x8(mb, has_2nd_order, backup);
  *skippable = vp9_mby_is_skippable_8x8(&mb->e_mbd, has_2nd_order);
772
}
773

774
static int rdcost_mby_16x16(MACROBLOCK *mb, int backup) {
Daniel Kang's avatar
Daniel Kang committed
775
  int cost;
Paul Wilkins's avatar
Paul Wilkins committed
776
  MACROBLOCKD *xd = &mb->e_mbd;
Daniel Kang's avatar
Daniel Kang committed
777 778 779
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta, *tl;

780 781 782
  if (backup) {
    vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
Daniel Kang's avatar
Daniel Kang committed
783

784 785 786 787 788 789
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)xd->above_context;
    tl = (ENTROPY_CONTEXT *)xd->left_context;
  }
Daniel Kang's avatar
Daniel Kang committed
790

Paul Wilkins's avatar
Paul Wilkins committed
791
  cost = cost_coeffs(mb, xd->block, PLANE_TYPE_Y_WITH_DC, ta, tl, TX_16X16);
Daniel Kang's avatar
Daniel Kang committed
792 793
  return cost;
}
794

Daniel Kang's avatar
Daniel Kang committed
795
static void macro_block_yrd_16x16(MACROBLOCK *mb, int *Rate, int *Distortion,
796
                                  int *skippable, int backup) {
Daniel Kang's avatar
Daniel Kang committed
797
  int d;
Deb Mukherjee's avatar
Deb Mukherjee committed
798 799 800 801
  MACROBLOCKD *xd = &mb->e_mbd;
  BLOCKD *b  = &mb->e_mbd.block[0];
  BLOCK  *be = &mb->block[0];
  TX_TYPE tx_type;
Daniel Kang's avatar
Daniel Kang committed
802

803 804
  xd->mode_info_context->mbmi.txfm_size = TX_16X16;
  vp9_transform_mby_16x16(mb);
805
  vp9_quantize_mby_16x16(mb);
806 807 808
  // TODO(jingning) is it possible to quickly determine whether to force
  //                trailing coefficients to be zero, instead of running trellis
  //                optimization in the rate-distortion optimization loop?
809
  if (mb->e_mbd.mode_info_context->mbmi.mode < I8X8_PRED)
810
    vp9_optimize_mby_16x16(mb);
811

812
  d = vp9_mbblock_error(mb, 0);
Daniel Kang's avatar
Daniel Kang committed
813 814 815

  *Distortion = (d >> 2);
  // rate
816
  *Rate = rdcost_mby_16x16(mb, backup);
817
  *skippable = vp9_mby_is_skippable_16x16(&mb->e_mbd);
Daniel Kang's avatar
Daniel Kang committed
818 819
}

820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852
static void choose_txfm_size_from_rd(VP9_COMP *cpi, MACROBLOCK *x,
                                     int r[2][TX_SIZE_MAX], int *rate,
                                     int d[TX_SIZE_MAX], int *distortion,
                                     int s[TX_SIZE_MAX], int *skip,
                                     int64_t txfm_cache[NB_TXFM_MODES]) {
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
  vp9_prob skip_prob = cm->mb_no_coeff_skip ?
                       vp9_get_pred_prob(cm, xd, PRED_MBSKIP) : 128;
  int64_t rd[2][TX_SIZE_MAX];
  int n;

  r[1][TX_16X16] = r[0][TX_16X16] + vp9_cost_one(cm->prob_tx[0]) +
                   vp9_cost_one(cm->prob_tx[1]);
  r[1][TX_8X8]   = r[0][TX_8X8] + vp9_cost_one(cm->prob_tx[0]) +
                   vp9_cost_zero(cm->prob_tx[1]);
  r[1][TX_4X4]   = r[0][TX_4X4] + vp9_cost_zero(cm->prob_tx[0]);

  if (cm->mb_no_coeff_skip) {
    int s0, s1;

    assert(skip_prob > 0);
    s0 = vp9_cost_bit(skip_prob, 0);
    s1 = vp9_cost_bit(skip_prob, 1);

    for (n = TX_4X4; n <= TX_16X16; n++) {
      if (s[n]) {
        rd[0][n] = rd[1][n] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
      } else {
        rd[0][n] = RDCOST(x->rdmult, x->rddiv, r[0][n] + s0, d[n]);
        rd[1][n] = RDCOST(x->rdmult, x->rddiv, r[1][n] + s0, d[n]);
      }
853 854
    }
  } else {
855 856 857
    for (n = TX_4X4; n <= TX_16X16; n++) {
      rd[0][n] = RDCOST(x->rdmult, x->rddiv, r[0][n], d[n]);
      rd[1][n] = RDCOST(x->rdmult, x->rddiv, r[1][n], d[n]);
858 859 860
    }
  }

861 862 863
  if ( cm->txfm_mode == ALLOW_16X16 ||
      (cm->txfm_mode == TX_MODE_SELECT &&
       rd[1][TX_16X16] < rd[1][TX_8X8] && rd[1][TX_16X16] < rd[1][TX_4X4])) {
864
    mbmi->txfm_size = TX_16X16;
865 866
  } else if (cm->txfm_mode == ALLOW_8X8 ||
           (cm->txfm_mode == TX_MODE_SELECT && rd[1][TX_8X8] < rd[1][TX_4X4])) {
867 868
    mbmi->txfm_size = TX_8X8;
  } else {
869 870
    assert(cm->txfm_mode == ONLY_4X4 ||
          (cm->txfm_mode == TX_MODE_SELECT && rd[1][TX_4X4] <= rd[1][TX_8X8]));
871 872 873
    mbmi->txfm_size = TX_4X4;
  }

874 875 876 877 878 879 880 881 882
  *distortion = d[mbmi->txfm_size];
  *rate       = r[cm->txfm_mode == TX_MODE_SELECT][mbmi->txfm_size];
  *skip       = s[mbmi->txfm_size];

  txfm_cache[ONLY_4X4] = rd[0][TX_4X4];
  txfm_cache[ALLOW_8X8] = rd[0][TX_8X8];
  txfm_cache[ALLOW_16X16] = rd[0][TX_16X16];
  if (rd[1][TX_16X16] < rd[1][TX_8X8] && rd[1][TX_16X16] < rd[1][TX_4X4])
    txfm_cache[TX_MODE_SELECT] = rd[1][TX_16X16];
883
  else
884 885 886 887 888 889 890 891 892 893 894 895
    txfm_cache[TX_MODE_SELECT] = rd[1][TX_4X4] < rd[1][TX_8X8] ?
                                 rd[1][TX_4X4] : rd[1][TX_8X8];
}

static void macro_block_yrd(VP9_COMP *cpi, MACROBLOCK *x, int *rate,
                            int *distortion, int *skippable,
                            int64_t txfm_cache[NB_TXFM_MODES]) {
  MACROBLOCKD *const xd = &x->e_mbd;
  int r[2][TX_SIZE_MAX], d[TX_SIZE_MAX], s[TX_SIZE_MAX];

  vp9_subtract_mby(x->src_diff, *(x->block[0].base_src), xd->predictor,
                   x->block[0].src_stride);