vp9_rdopt.c 165 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5 6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9 10 11 12 13 14 15
 */


#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
#include "vp9/common/vp9_pragmas.h"

#include "vp9_tokenize.h"
#include "vp9_treewriter.h"
#include "vp9_onyx_int.h"
#include "vp9_modecosts.h"
#include "vp9_encodeintra.h"
#include "vp9/common/vp9_entropymode.h"
#include "vp9/common/vp9_reconinter.h"
#include "vp9/common/vp9_reconintra.h"
#include "vp9/common/vp9_reconintra4x4.h"
#include "vp9/common/vp9_findnearmv.h"
#include "vp9/common/vp9_quant_common.h"
#include "vp9_encodemb.h"
#include "vp9_quantize.h"
#include "vp9_variance.h"
#include "vp9_mcomp.h"
#include "vp9_rdopt.h"
#include "vp9_ratectrl.h"
John Koleszar's avatar
John Koleszar committed
35
#include "vpx_mem/vpx_mem.h"
36 37
#include "vp9/common/vp9_systemdependent.h"
#include "vp9/encoder/vp9_encodemv.h"
John Koleszar's avatar
John Koleszar committed
38

39 40 41
#include "vp9/common/vp9_seg_common.h"
#include "vp9/common/vp9_pred_common.h"
#include "vp9/common/vp9_entropy.h"
42
#include "vp9_rtcd.h"
43
#include "vp9/common/vp9_mvref_common.h"
Paul Wilkins's avatar
Paul Wilkins committed
44

John Koleszar's avatar
John Koleszar committed
45 46 47 48 49 50 51 52
#if CONFIG_RUNTIME_CPU_DETECT
#define IF_RTCD(x)  (x)
#else
#define IF_RTCD(x)  NULL
#endif

#define MAXF(a,b)            (((a) > (b)) ? (a) : (b))

53 54
#define INVALID_MV 0x80008000

55 56 57
/* Factor to weigh the rate for switchable interp filters */
#define SWITCHABLE_INTERP_RATE_FACTOR 1

John Koleszar's avatar
John Koleszar committed
58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75
static const int auto_speed_thresh[17] = {
  1000,
  200,
  150,
  130,
  150,
  125,
  120,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  105
John Koleszar's avatar
John Koleszar committed
76 77
};

78
#if CONFIG_PRED_FILTER
79
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
  {ZEROMV,    LAST_FRAME,   NONE,  0},
  {ZEROMV,    LAST_FRAME,   NONE,  1},
  {DC_PRED,   INTRA_FRAME,  NONE,  0},

  {NEARESTMV, LAST_FRAME,   NONE,  0},
  {NEARESTMV, LAST_FRAME,   NONE,  1},
  {NEARMV,    LAST_FRAME,   NONE,  0},
  {NEARMV,    LAST_FRAME,   NONE,  1},

  {ZEROMV,    GOLDEN_FRAME, NONE,  0},
  {ZEROMV,    GOLDEN_FRAME, NONE,  1},
  {NEARESTMV, GOLDEN_FRAME, NONE,  0},
  {NEARESTMV, GOLDEN_FRAME, NONE,  1},

  {ZEROMV,    ALTREF_FRAME, NONE,  0},
  {ZEROMV,    ALTREF_FRAME, NONE,  1},
  {NEARESTMV, ALTREF_FRAME, NONE,  0},
  {NEARESTMV, ALTREF_FRAME, NONE,  1},

  {NEARMV,    GOLDEN_FRAME, NONE,  0},
  {NEARMV,    GOLDEN_FRAME, NONE,  1},
  {NEARMV,    ALTREF_FRAME, NONE,  0},
  {NEARMV,    ALTREF_FRAME, NONE,  1},

  {V_PRED,    INTRA_FRAME,  NONE,  0},
  {H_PRED,    INTRA_FRAME,  NONE,  0},
  {D45_PRED,  INTRA_FRAME,  NONE,  0},
  {D135_PRED, INTRA_FRAME,  NONE,  0},
  {D117_PRED, INTRA_FRAME,  NONE,  0},
  {D153_PRED, INTRA_FRAME,  NONE,  0},
  {D27_PRED,  INTRA_FRAME,  NONE,  0},
  {D63_PRED,  INTRA_FRAME,  NONE,  0},

  {TM_PRED,   INTRA_FRAME,  NONE,  0},

  {NEWMV,     LAST_FRAME,   NONE,  0},
  {NEWMV,     LAST_FRAME,   NONE,  1},
  {NEWMV,     GOLDEN_FRAME, NONE,  0},
  {NEWMV,     GOLDEN_FRAME, NONE,  1},
  {NEWMV,     ALTREF_FRAME, NONE,  0},
  {NEWMV,     ALTREF_FRAME, NONE,  1},

  {SPLITMV,   LAST_FRAME,   NONE,  0},
  {SPLITMV,   GOLDEN_FRAME, NONE,  0},
  {SPLITMV,   ALTREF_FRAME, NONE,  0},

  {B_PRED,    INTRA_FRAME,  NONE,  0},
  {I8X8_PRED, INTRA_FRAME,  NONE,  0},
John Koleszar's avatar
John Koleszar committed
128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147

  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME, 0},

  {ZEROMV,    ALTREF_FRAME, LAST_FRAME,   0},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME,   0},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME,   0},

  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME, 0},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME, 0},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME, 0},

  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME,   0},
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME, 0},

  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME, 0},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME,   0},
148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME, 0},

#if CONFIG_COMP_INTERINTRA_PRED
  /* compound inter-intra prediction */
  {ZEROMV,    LAST_FRAME,   INTRA_FRAME, 0},
  {NEARESTMV, LAST_FRAME,   INTRA_FRAME, 0},
  {NEARMV,    LAST_FRAME,   INTRA_FRAME, 0},
  {NEWMV,     LAST_FRAME,   INTRA_FRAME, 0},

  {ZEROMV,    GOLDEN_FRAME,   INTRA_FRAME, 0},
  {NEARESTMV, GOLDEN_FRAME,   INTRA_FRAME, 0},
  {NEARMV,    GOLDEN_FRAME,   INTRA_FRAME, 0},
  {NEWMV,     GOLDEN_FRAME,   INTRA_FRAME, 0},

  {ZEROMV,    ALTREF_FRAME,   INTRA_FRAME, 0},
  {NEARESTMV, ALTREF_FRAME,   INTRA_FRAME, 0},
  {NEARMV,    ALTREF_FRAME,   INTRA_FRAME, 0},
  {NEWMV,     ALTREF_FRAME,   INTRA_FRAME, 0},
#endif
John Koleszar's avatar
John Koleszar committed
167
};
168
#else
169
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
170 171
  {ZEROMV,    LAST_FRAME,   NONE},
  {DC_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
172

173 174
  {NEARESTMV, LAST_FRAME,   NONE},
  {NEARMV,    LAST_FRAME,   NONE},
John Koleszar's avatar
John Koleszar committed
175

176 177
  {ZEROMV,    GOLDEN_FRAME, NONE},
  {NEARESTMV, GOLDEN_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
178

179 180
  {ZEROMV,    ALTREF_FRAME, NONE},
  {NEARESTMV, ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
181

182 183
  {NEARMV,    GOLDEN_FRAME, NONE},
  {NEARMV,    ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
184

185 186 187 188 189 190 191 192
  {V_PRED,    INTRA_FRAME,  NONE},
  {H_PRED,    INTRA_FRAME,  NONE},
  {D45_PRED,  INTRA_FRAME,  NONE},
  {D135_PRED, INTRA_FRAME,  NONE},
  {D117_PRED, INTRA_FRAME,  NONE},
  {D153_PRED, INTRA_FRAME,  NONE},
  {D27_PRED,  INTRA_FRAME,  NONE},
  {D63_PRED,  INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
193

194
  {TM_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
195

196 197 198
  {NEWMV,     LAST_FRAME,   NONE},
  {NEWMV,     GOLDEN_FRAME, NONE},
  {NEWMV,     ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
199

200 201 202
  {SPLITMV,   LAST_FRAME,   NONE},
  {SPLITMV,   GOLDEN_FRAME, NONE},
  {SPLITMV,   ALTREF_FRAME, NONE},
203

204 205
  {B_PRED,    INTRA_FRAME,  NONE},
  {I8X8_PRED, INTRA_FRAME,  NONE},
206

John Koleszar's avatar
John Koleszar committed
207 208 209 210
  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME},
211

John Koleszar's avatar
John Koleszar committed
212 213 214
  {ZEROMV,    ALTREF_FRAME, LAST_FRAME},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME},
215

John Koleszar's avatar
John Koleszar committed
216 217 218
  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME},
219

John Koleszar's avatar
John Koleszar committed
220 221 222
  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME  },
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME},
223

John Koleszar's avatar
John Koleszar committed
224 225
  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME  },
226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME},

#if CONFIG_COMP_INTERINTRA_PRED
  /* compound inter-intra prediction */
  {ZEROMV,    LAST_FRAME,   INTRA_FRAME},
  {NEARESTMV, LAST_FRAME,   INTRA_FRAME},
  {NEARMV,    LAST_FRAME,   INTRA_FRAME},
  {NEWMV,     LAST_FRAME,   INTRA_FRAME},

  {ZEROMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEARESTMV, GOLDEN_FRAME,   INTRA_FRAME},
  {NEARMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEWMV,     GOLDEN_FRAME,   INTRA_FRAME},

  {ZEROMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEARESTMV, ALTREF_FRAME,   INTRA_FRAME},
  {NEARMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEWMV,     ALTREF_FRAME,   INTRA_FRAME},
#endif
John Koleszar's avatar
John Koleszar committed
245
};
246
#endif
John Koleszar's avatar
John Koleszar committed
247 248

static void fill_token_costs(
249
  unsigned int (*c)[COEF_BANDS][PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS],
250
  const vp9_prob(*p)[COEF_BANDS][PREV_COEF_CONTEXTS][ENTROPY_NODES],
John Koleszar's avatar
John Koleszar committed
251 252 253 254 255 256 257
  int block_type_counts) {
  int i, j, k;

  for (i = 0; i < block_type_counts; i++)
    for (j = 0; j < COEF_BANDS; j++)
      for (k = 0; k < PREV_COEF_CONTEXTS; k++) {
        if (k == 0 && ((j > 0 && i > 0) || (j > 1 && i == 0)))
258 259
          vp9_cost_tokens_skip((int *)(c[i][j][k]),
                               p[i][j][k],
260
                               vp9_coef_tree);
John Koleszar's avatar
John Koleszar committed
261
        else
262 263
          vp9_cost_tokens((int *)(c[i][j][k]),
                          p[i][j][k],
264
                          vp9_coef_tree);
John Koleszar's avatar
John Koleszar committed
265
      }
John Koleszar's avatar
John Koleszar committed
266 267
}

268

269 270 271 272
static int rd_iifactor[32] =  { 4, 4, 3, 2, 1, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0, };
John Koleszar's avatar
John Koleszar committed
273

274
// 3* dc_qlookup[Q]*dc_qlookup[Q];
275

276
/* values are now correlated to quantizer */
Paul Wilkins's avatar
Paul Wilkins committed
277 278 279
static int sad_per_bit16lut[QINDEX_RANGE];
static int sad_per_bit4lut[QINDEX_RANGE];

280
void vp9_init_me_luts() {
John Koleszar's avatar
John Koleszar committed
281 282 283 284 285 286 287
  int i;

  // Initialize the sad lut tables using a formulaic calculation for now
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < QINDEX_RANGE; i++) {
    sad_per_bit16lut[i] =
288 289
      (int)((0.0418 * vp9_convert_qindex_to_q(i)) + 2.4107);
    sad_per_bit4lut[i] = (int)((0.063 * vp9_convert_qindex_to_q(i)) + 2.742);
John Koleszar's avatar
John Koleszar committed
290
  }
Paul Wilkins's avatar
Paul Wilkins committed
291
}
John Koleszar's avatar
John Koleszar committed
292

293
static int compute_rd_mult(int qindex) {
John Koleszar's avatar
John Koleszar committed
294
  int q;
295

296
  q = vp9_dc_quant(qindex, 0);
John Koleszar's avatar
John Koleszar committed
297
  return (11 * q * q) >> 6;
298 299
}

300
void vp9_initialize_me_consts(VP9_COMP *cpi, int QIndex) {
John Koleszar's avatar
John Koleszar committed
301 302
  cpi->mb.sadperbit16 =  sad_per_bit16lut[QIndex];
  cpi->mb.sadperbit4  =  sad_per_bit4lut[QIndex];
John Koleszar's avatar
John Koleszar committed
303 304
}

305

306
void vp9_initialize_rd_consts(VP9_COMP *cpi, int QIndex) {
307
  int q, i;
John Koleszar's avatar
John Koleszar committed
308

309
  vp9_clear_system_state();  // __asm emms;
John Koleszar's avatar
John Koleszar committed
310

John Koleszar's avatar
John Koleszar committed
311 312 313 314 315
  // Further tests required to see if optimum is different
  // for key frames, golden frames and arf frames.
  // if (cpi->common.refresh_golden_frame ||
  //     cpi->common.refresh_alt_ref_frame)
  QIndex = (QIndex < 0) ? 0 : ((QIndex > MAXQ) ? MAXQ : QIndex);
316

John Koleszar's avatar
John Koleszar committed
317
  cpi->RDMULT = compute_rd_mult(QIndex);
John Koleszar's avatar
John Koleszar committed
318

John Koleszar's avatar
John Koleszar committed
319 320 321
  // Extend rate multiplier along side quantizer zbin increases
  if (cpi->zbin_over_quant  > 0) {
    double oq_factor;
322

John Koleszar's avatar
John Koleszar committed
323 324 325 326 327
    // Experimental code using the same basic equation as used for Q above
    // The units of cpi->zbin_over_quant are 1/128 of Q bin size
    oq_factor = 1.0 + ((double)0.0015625 * cpi->zbin_over_quant);
    cpi->RDMULT = (int)((double)cpi->RDMULT * oq_factor * oq_factor);
  }
John Koleszar's avatar
John Koleszar committed
328

John Koleszar's avatar
John Koleszar committed
329 330 331 332 333 334 335
  if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    if (cpi->twopass.next_iiratio > 31)
      cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
    else
      cpi->RDMULT +=
        (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
  }
John Koleszar's avatar
John Koleszar committed
336

John Koleszar's avatar
John Koleszar committed
337 338
  if (cpi->RDMULT < 7)
    cpi->RDMULT = 7;
339

John Koleszar's avatar
John Koleszar committed
340 341
  cpi->mb.errorperbit = (cpi->RDMULT / 110);
  cpi->mb.errorperbit += (cpi->mb.errorperbit == 0);
342

343
  vp9_set_speed_features(cpi);
John Koleszar's avatar
John Koleszar committed
344

345
  q = (int)pow(vp9_dc_quant(QIndex, 0) >> 2, 1.25);
John Koleszar's avatar
John Koleszar committed
346 347
  q = q << 2;
  cpi->RDMULT = cpi->RDMULT << 4;
Paul Wilkins's avatar
Paul Wilkins committed
348

John Koleszar's avatar
John Koleszar committed
349 350
  if (q < 8)
    q = 8;
351

John Koleszar's avatar
John Koleszar committed
352 353 354
  if (cpi->RDMULT > 1000) {
    cpi->RDDIV = 1;
    cpi->RDMULT /= 100;
John Koleszar's avatar
John Koleszar committed
355

John Koleszar's avatar
John Koleszar committed
356 357 358 359 360 361
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < INT_MAX) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q / 100;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
362

John Koleszar's avatar
John Koleszar committed
363
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
364
    }
John Koleszar's avatar
John Koleszar committed
365 366
  } else {
    cpi->RDDIV = 100;
John Koleszar's avatar
John Koleszar committed
367

John Koleszar's avatar
John Koleszar committed
368 369 370 371 372 373
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < (INT_MAX / q)) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
374

John Koleszar's avatar
John Koleszar committed
375
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
376
    }
John Koleszar's avatar
John Koleszar committed
377
  }
John Koleszar's avatar
John Koleszar committed
378

John Koleszar's avatar
John Koleszar committed
379
  fill_token_costs(
380
    cpi->mb.token_costs[TX_4X4],
381
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs,
John Koleszar's avatar
John Koleszar committed
382
    BLOCK_TYPES);
383 384
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_4X4],
385
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11])
386 387
    cpi->common.fc.hybrid_coef_probs,
    BLOCK_TYPES);
John Koleszar's avatar
John Koleszar committed
388

John Koleszar's avatar
John Koleszar committed
389
  fill_token_costs(
390
    cpi->mb.token_costs[TX_8X8],
391
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs_8x8,
John Koleszar's avatar
John Koleszar committed
392
    BLOCK_TYPES_8X8);
393 394
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_8X8],
395
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11])
396 397
    cpi->common.fc.hybrid_coef_probs_8x8,
    BLOCK_TYPES_8X8);
398

Daniel Kang's avatar
Daniel Kang committed
399
  fill_token_costs(
400
    cpi->mb.token_costs[TX_16X16],
401
    (const vp9_prob(*)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs_16x16,
Daniel Kang's avatar
Daniel Kang committed
402
    BLOCK_TYPES_16X16);
403 404
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_16X16],
405
    (const vp9_prob(*)[8][PREV_COEF_CONTEXTS][11])
406 407
    cpi->common.fc.hybrid_coef_probs_16x16,
    BLOCK_TYPES_16X16);
Daniel Kang's avatar
Daniel Kang committed
408

John Koleszar's avatar
John Koleszar committed
409 410
  /*rough estimate for costing*/
  cpi->common.kf_ymode_probs_index = cpi->common.base_qindex >> 4;
411
  vp9_init_mode_costs(cpi);
John Koleszar's avatar
John Koleszar committed
412

413 414
  if (cpi->common.frame_type != KEY_FRAME)
  {
415
    vp9_build_nmv_cost_table(
416 417 418 419 420 421
        cpi->mb.nmvjointcost,
        cpi->mb.e_mbd.allow_high_precision_mv ?
        cpi->mb.nmvcost_hp : cpi->mb.nmvcost,
        &cpi->common.fc.nmvc,
        cpi->mb.e_mbd.allow_high_precision_mv, 1, 1);
  }
John Koleszar's avatar
John Koleszar committed
422 423
}

424
int vp9_block_error_c(short *coeff, short *dqcoeff, int block_size) {
425
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
426

427
  for (i = 0; i < block_size; i++) {
John Koleszar's avatar
John Koleszar committed
428 429 430
    int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }
John Koleszar's avatar
John Koleszar committed
431

John Koleszar's avatar
John Koleszar committed
432
  return error;
John Koleszar's avatar
John Koleszar committed
433 434
}

435
int vp9_mbblock_error_c(MACROBLOCK *mb, int dc) {
John Koleszar's avatar
John Koleszar committed
436 437 438 439
  BLOCK  *be;
  BLOCKD *bd;
  int i, j;
  int berror, error = 0;
John Koleszar's avatar
John Koleszar committed
440

John Koleszar's avatar
John Koleszar committed
441 442 443
  for (i = 0; i < 16; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
John Koleszar's avatar
John Koleszar committed
444

John Koleszar's avatar
John Koleszar committed
445
    berror = 0;
John Koleszar's avatar
John Koleszar committed
446

John Koleszar's avatar
John Koleszar committed
447 448 449
    for (j = dc; j < 16; j++) {
      int this_diff = be->coeff[j] - bd->dqcoeff[j];
      berror += this_diff * this_diff;
John Koleszar's avatar
John Koleszar committed
450 451
    }

John Koleszar's avatar
John Koleszar committed
452 453 454 455
    error += berror;
  }

  return error;
John Koleszar's avatar
John Koleszar committed
456 457
}

458
int vp9_mbuverror_c(MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
459 460
  BLOCK  *be;
  BLOCKD *bd;
John Koleszar's avatar
John Koleszar committed
461

462
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
463

John Koleszar's avatar
John Koleszar committed
464 465 466
  for (i = 16; i < 24; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
John Koleszar's avatar
John Koleszar committed
467

468
    error += vp9_block_error_c(be->coeff, bd->dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
469
  }
John Koleszar's avatar
John Koleszar committed
470

John Koleszar's avatar
John Koleszar committed
471
  return error;
John Koleszar's avatar
John Koleszar committed
472 473
}

474
int vp9_uvsse(MACROBLOCK *x) {
John Koleszar's avatar
John Koleszar committed
475 476 477 478 479 480 481
  unsigned char *uptr, *vptr;
  unsigned char *upred_ptr = (*(x->block[16].base_src) + x->block[16].src);
  unsigned char *vpred_ptr = (*(x->block[20].base_src) + x->block[20].src);
  int uv_stride = x->block[16].src_stride;

  unsigned int sse1 = 0;
  unsigned int sse2 = 0;
482 483
  int mv_row = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.row;
  int mv_col = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.col;
John Koleszar's avatar
John Koleszar committed
484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504
  int offset;
  int pre_stride = x->e_mbd.block[16].pre_stride;

  if (mv_row < 0)
    mv_row -= 1;
  else
    mv_row += 1;

  if (mv_col < 0)
    mv_col -= 1;
  else
    mv_col += 1;

  mv_row /= 2;
  mv_col /= 2;

  offset = (mv_row >> 3) * pre_stride + (mv_col >> 3);
  uptr = x->e_mbd.pre.u_buffer + offset;
  vptr = x->e_mbd.pre.v_buffer + offset;

  if ((mv_row | mv_col) & 7) {
505
    vp9_sub_pixel_variance8x8(uptr, pre_stride, (mv_col & 7) << 1,
506
                              (mv_row & 7) << 1, upred_ptr, uv_stride, &sse2);
507
    vp9_sub_pixel_variance8x8(vptr, pre_stride, (mv_col & 7) << 1,
508
                              (mv_row & 7) << 1, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
509 510
    sse2 += sse1;
  } else {
511 512
    vp9_variance8x8(uptr, pre_stride, upred_ptr, uv_stride, &sse2);
    vp9_variance8x8(vptr, pre_stride, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
513 514 515
    sse2 += sse1;
  }
  return sse2;
John Koleszar's avatar
John Koleszar committed
516 517 518

}

519
static int cost_coeffs_2x2(MACROBLOCK *mb,
520
                           BLOCKD *b, PLANE_TYPE type,
521
                           ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l) {
522
  int c = (type == PLANE_TYPE_Y_NO_DC); /* start at coef 0, unless Y with Y2 */
523 524 525 526 527
  int eob = b->eob;
  int pt;    /* surrounding block/prev coef predictor */
  int cost = 0;
  short *qcoeff_ptr = b->qcoeff;

528
  VP9_COMBINEENTROPYCONTEXTS(pt, *a, *l);
529 530 531
  assert(eob <= 4);

  for (; c < eob; c++) {
532 533 534 535 536
    int v = qcoeff_ptr[vp9_default_zig_zag1d[c]];
    int t = vp9_dct_value_tokens_ptr[v].Token;
    cost += mb->token_costs[TX_8X8][type][vp9_coef_bands[c]][pt][t];
    cost += vp9_dct_value_cost_ptr[v];
    pt = vp9_prev_token_class[t];
537 538 539
  }

  if (c < 4)
540
    cost += mb->token_costs[TX_8X8][type][vp9_coef_bands[c]]
541 542 543 544 545 546 547
            [pt] [DCT_EOB_TOKEN];

  pt = (c != !type); // is eob first coefficient;
  *a = *l = pt;
  return cost;
}

548
static int cost_coeffs(MACROBLOCK *mb, BLOCKD *b, PLANE_TYPE type,
Daniel Kang's avatar
Daniel Kang committed
549
                       ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l,
550
                       int tx_size) {
Daniel Kang's avatar
Daniel Kang committed
551
  const int eob = b->eob;
552
  int c = (type == PLANE_TYPE_Y_NO_DC); /* start at coef 0, unless Y with Y2 */
553
  int cost = 0, default_eob, seg_eob;
Daniel Kang's avatar
Daniel Kang committed
554 555
  int pt;                     /* surrounding block/prev coef predictor */
  int const *scan, *band;
John Koleszar's avatar
John Koleszar committed
556
  short *qcoeff_ptr = b->qcoeff;
557 558 559
  MACROBLOCKD *xd = &mb->e_mbd;
  MB_MODE_INFO *mbmi = &mb->e_mbd.mode_info_context->mbmi;
  TX_TYPE tx_type = DCT_DCT;
Paul Wilkins's avatar
Paul Wilkins committed
560
  int segment_id = mbmi->segment_id;
561

562
  switch (tx_size) {
Daniel Kang's avatar
Daniel Kang committed
563
    case TX_4X4:
564 565
      scan = vp9_default_zig_zag1d;
      band = vp9_coef_bands;
Daniel Kang's avatar
Daniel Kang committed
566
      default_eob = 16;
Deb Mukherjee's avatar
Deb Mukherjee committed
567 568 569 570 571
      if (type == PLANE_TYPE_Y_WITH_DC) {
        tx_type = get_tx_type_4x4(xd, b);
        if (tx_type != DCT_DCT) {
          switch (tx_type) {
            case ADST_DCT:
572
              scan = vp9_row_scan;
Deb Mukherjee's avatar
Deb Mukherjee committed
573 574 575
              break;

            case DCT_ADST:
576
              scan = vp9_col_scan;
Deb Mukherjee's avatar
Deb Mukherjee committed
577 578 579
              break;

            default:
580
              scan = vp9_default_zig_zag1d;
Deb Mukherjee's avatar
Deb Mukherjee committed
581 582
              break;
          }
583
        }
Daniel Kang's avatar
Daniel Kang committed
584
      }
Deb Mukherjee's avatar
Deb Mukherjee committed
585

Daniel Kang's avatar
Daniel Kang committed
586 587
      break;
    case TX_8X8:
588 589
      scan = vp9_default_zig_zag1d_8x8;
      band = vp9_coef_bands_8x8;
Daniel Kang's avatar
Daniel Kang committed
590
      default_eob = 64;
Deb Mukherjee's avatar
Deb Mukherjee committed
591
      if (type == PLANE_TYPE_Y_WITH_DC) {
592
        BLOCKD *bb;
593
        int ib = (int)(b - xd->block);
594 595 596
        if (ib < 16) {
          ib = (ib & 8) + ((ib & 4) >> 1);
          bb = xd->block + ib;
Deb Mukherjee's avatar
Deb Mukherjee committed
597
          tx_type = get_tx_type_8x8(xd, bb);
598
        }
599
      }
Daniel Kang's avatar
Daniel Kang committed
600 601
      break;
    case TX_16X16:
602 603
      scan = vp9_default_zig_zag1d_16x16;
      band = vp9_coef_bands_16x16;
Daniel Kang's avatar
Daniel Kang committed
604
      default_eob = 256;
Deb Mukherjee's avatar
Deb Mukherjee committed
605 606 607
      if (type == PLANE_TYPE_Y_WITH_DC) {
        tx_type = get_tx_type_16x16(xd, b);
      }
Daniel Kang's avatar
Daniel Kang committed
608 609 610 611
      break;
    default:
      break;
  }
612 613
  if (vp9_segfeature_active(&mb->e_mbd, segment_id, SEG_LVL_EOB))
    seg_eob = vp9_get_segdata(&mb->e_mbd, segment_id, SEG_LVL_EOB);
614 615 616
  else
    seg_eob = default_eob;

617
  VP9_COMBINEENTROPYCONTEXTS(pt, *a, *l);
618

619 620 621
  if (tx_type != DCT_DCT) {
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
622
      int t = vp9_dct_value_tokens_ptr[v].Token;
623
      cost += mb->hybrid_token_costs[tx_size][type][band[c]][pt][t];
624 625
      cost += vp9_dct_value_cost_ptr[v];
      pt = vp9_prev_token_class[t];
626 627 628 629
    }
    if (c < seg_eob)
      cost += mb->hybrid_token_costs[tx_size][type][band[c]]
          [pt][DCT_EOB_TOKEN];
630
  } else {
631 632
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
633
      int t = vp9_dct_value_tokens_ptr[v].Token;
634
      cost += mb->token_costs[tx_size][type][band[c]][pt][t];
635 636
      cost += vp9_dct_value_cost_ptr[v];
      pt = vp9_prev_token_class[t];
637 638 639 640
    }
    if (c < seg_eob)
      cost += mb->token_costs[tx_size][type][band[c]]
          [pt][DCT_EOB_TOKEN];
641 642 643 644 645 646 647
  }

  pt = (c != !type); // is eob first coefficient;
  *a = *l = pt;
  return cost;
}

648
static int rdcost_mby_4x4(MACROBLOCK *mb, int backup) {
John Koleszar's avatar
John Koleszar committed
649 650
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
651
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
652 653 654
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta;
  ENTROPY_CONTEXT *tl;
655

656 657 658
  if (backup) {
    vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
659

660 661 662 663 664 665
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)xd->above_context;
    tl = (ENTROPY_CONTEXT *)xd->left_context;
  }
John Koleszar's avatar
John Koleszar committed
666

John Koleszar's avatar
John Koleszar committed
667
  for (b = 0; b < 16; b++)
Paul Wilkins's avatar
Paul Wilkins committed
668
    cost += cost_coeffs(mb, xd->block + b, PLANE_TYPE_Y_NO_DC,
669
                        ta + vp9_block2above[b], tl + vp9_block2left[b],
Daniel Kang's avatar
Daniel Kang committed
670
                        TX_4X4);
John Koleszar's avatar
John Koleszar committed
671

Paul Wilkins's avatar
Paul Wilkins committed
672
  cost += cost_coeffs(mb, xd->block + 24, PLANE_TYPE_Y2,
673
                      ta + vp9_block2above[24], tl + vp9_block2left[24],
Daniel Kang's avatar
Daniel Kang committed
674
                      TX_4X4);
John Koleszar's avatar
John Koleszar committed
675

John Koleszar's avatar
John Koleszar committed
676
  return cost;
John Koleszar's avatar
John Koleszar committed
677 678
}

679 680 681
static void macro_block_yrd_4x4(MACROBLOCK *mb,
                                int *Rate,
                                int *Distortion,
682
                                int *skippable, int backup) {
John Koleszar's avatar
John Koleszar committed
683
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
684
  MACROBLOCKD *const xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
685
  BLOCK   *const mb_y2 = mb->block + 24;
Paul Wilkins's avatar
Paul Wilkins committed
686
  BLOCKD *const x_y2  = xd->block + 24;
John Koleszar's avatar
John Koleszar committed
687 688 689
  short *Y2DCPtr = mb_y2->src_diff;
  BLOCK *beptr;
  int d;
690

John Koleszar's avatar
John Koleszar committed
691 692
  // Fdct and building the 2nd order block
  for (beptr = mb->block; beptr < mb->block + 16; beptr += 2) {
693
    mb->vp9_short_fdct8x4(beptr->src_diff, beptr->coeff, 32);
John Koleszar's avatar
John Koleszar committed
694 695 696 697 698 699 700 701 702
    *Y2DCPtr++ = beptr->coeff[0];
    *Y2DCPtr++ = beptr->coeff[16];
  }

  // 2nd order fdct
  mb->short_walsh4x4(mb_y2->src_diff, mb_y2->coeff, 8);

  // Quantization
  for (b = 0; b < 16; b++) {
703
    mb->quantize_b_4x4(&mb->block[b], &xd->block[b]);
John Koleszar's avatar
John Koleszar committed
704
  }
705

John Koleszar's avatar
John Koleszar committed
706
  // DC predication and Quantization of 2nd Order block
707
  mb->quantize_b_4x4(mb_y2, x_y2);
708

John Koleszar's avatar
John Koleszar committed
709
  // Distortion
710
  d = vp9_mbblock_error(mb, 1);
711

712
  d += vp9_block_error(mb_y2->coeff, x_y2->dqcoeff, 16);
713

John Koleszar's avatar
John Koleszar committed
714 715
  *Distortion = (d >> 2);
  // rate
716
  *Rate = rdcost_mby_4x4(mb, backup);
717
  *skippable = vp9_mby_is_skippable_4x4(&mb->e_mbd, 1);
718
}
John Koleszar's avatar
John Koleszar committed
719

720
static int rdcost_mby_8x8(MACROBLOCK *mb, int backup) {
John Koleszar's avatar
John Koleszar committed
721 722
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
723
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
724 725 726 727
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta;
  ENTROPY_CONTEXT *tl;

Ronald S. Bultje's avatar
Ronald S. Bultje committed
728 729 730
  if (backup) {
    vpx_memcpy(&t_above,xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
731

Ronald S. Bultje's avatar
Ronald S. Bultje committed
732 733 734 735 736 737
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)mb->e_mbd.above_context;
    tl = (ENTROPY_CONTEXT *)mb->e_mbd.left_context;
  }
John Koleszar's avatar
John Koleszar committed
738 739

  for (b = 0; b < 16; b += 4)
Paul Wilkins's avatar
Paul Wilkins committed
740
    cost += cost_coeffs(mb, xd->block + b, PLANE_TYPE_Y_NO_DC,
741
                        ta + vp9_block2above_8x8[b], tl + vp9_block2left_8x8[b],
Daniel Kang's avatar
Daniel Kang committed
742
                        TX_8X8);
John Koleszar's avatar
John Koleszar committed
743

Paul Wilkins's avatar
Paul Wilkins committed
744
  cost += cost_coeffs_2x2(mb, xd->block + 24, PLANE_TYPE_Y2,
745
                          ta + vp9_block2above[24], tl + vp9_block2left[24]);
John Koleszar's avatar
John Koleszar committed
746
  return cost;
747 748
}

John Koleszar's avatar
John Koleszar committed
749 750 751
static void macro_block_yrd_8x8(MACROBLOCK *mb,
                                int *Rate,
                                int *Distortion,
752
                                int *skippable, int backup) {
Paul Wilkins's avatar
Paul Wilkins committed
753
  MACROBLOCKD *const xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
754
  BLOCK   *const mb_y2 = mb->block + 24;
Paul Wilkins's avatar
Paul Wilkins committed
755
  BLOCKD *const x_y2  = xd->block + 24;
John Koleszar's avatar
John Koleszar committed
756 757
  int d;

758 759
  vp9_transform_mby_8x8(mb);
  vp9_quantize_mby_8x8(mb);
John Koleszar's avatar
John Koleszar committed
760 761 762 763 764 765

  /* remove 1st order dc to properly combine 1st/2nd order distortion */
  mb->coeff[0] = 0;
  mb->coeff[64] = 0;
  mb->coeff[128] = 0;
  mb->coeff[192] = 0;
Paul Wilkins's avatar
Paul Wilkins committed
766 767 768 769
  xd->dqcoeff[0] = 0;
  xd->dqcoeff[64] = 0;
  xd->dqcoeff[128] = 0;
  xd->dqcoeff[192] = 0;
John Koleszar's avatar
John Koleszar committed
770

771 772
  d = vp9_mbblock_error(mb, 0);
  d += vp9_block_error(mb_y2->coeff, x_y2->dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
773 774 775

  *Distortion = (d >> 2);
  // rate
776
  *Rate = rdcost_mby_8x8(mb, backup);
777
  *skippable = vp9_mby_is_skippable_8x8(&mb->e_mbd, 1);
778
}
779

780
static int rdcost_mby_16x16(MACROBLOCK *mb, int backup) {
Daniel Kang's avatar
Daniel Kang committed
781
  int cost;
Paul Wilkins's avatar
Paul Wilkins committed
782
  MACROBLOCKD *xd = &mb->e_mbd;
Daniel Kang's avatar
Daniel Kang committed
783 784 785
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta, *tl;

786 787 788
  if (backup) {
    vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
Daniel Kang's avatar
Daniel Kang committed
789

790 791 792 793 794 795
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)xd->above_context;
    tl = (ENTROPY_CONTEXT *)xd->left_context;
  }
Daniel Kang's avatar
Daniel Kang committed
796

Paul Wilkins's avatar
Paul Wilkins committed
797
  cost = cost_coeffs(mb, xd->block, PLANE_TYPE_Y_WITH_DC, ta, tl, TX_16X16);
Daniel Kang's avatar
Daniel Kang committed
798 799
  return cost;
}
800

Daniel Kang's avatar
Daniel Kang committed
801
static void macro_block_yrd_16x16(MACROBLOCK *mb, int *Rate, int *Distortion,
802
                                  int *skippable, int backup) {
Daniel Kang's avatar
Daniel Kang committed
803
  int d;
Deb Mukherjee's avatar
Deb Mukherjee committed
804 805 806 807
  MACROBLOCKD *xd = &mb->e_mbd;
  BLOCKD *b  = &mb->e_mbd.block[0];
  BLOCK  *be = &mb->block[0];
  TX_TYPE tx_type;
Daniel Kang's avatar
Daniel Kang committed
808

Deb Mukherjee's avatar
Deb Mukherjee committed
809 810
  tx_type = get_tx_type_16x16(xd, b);
  if (tx_type != DCT_DCT) {
811
    vp9_fht(be->src_diff, 32, be->coeff, tx_type, 16);
812
  } else
813
    vp9_transform_mby_16x16(mb);
814

815
  vp9_quantize_mby_16x16(mb);
816 817 818
  // TODO(jingning) is it possible to quickly determine whether to force
  //                trailing coefficients to be zero, instead of running trellis
  //                optimization in the rate-distortion optimization loop?
819
  if (mb->e_mbd.mode_info_context->mbmi.mode < I8X8_PRED)
820
    vp9_optimize_mby_16x16(mb);
821

822
  d = vp9_mbblock_error(mb, 0);
Daniel Kang's avatar
Daniel Kang committed
823 824 825

  *Distortion = (d >> 2);
  // rate
826
  *Rate = rdcost_mby_16x16(mb, backup);
827
  *skippable = vp9_mby_is_skippable_16x16(&mb->e_mbd);
Daniel Kang's avatar
Daniel Kang committed
828 829
}

830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862
static void choose_txfm_size_from_rd(VP9_COMP *cpi, MACROBLOCK *x,
                                     int r[2][TX_SIZE_MAX], int *rate,
                                     int d[TX_SIZE_MAX], int *distortion,
                                     int s[TX_SIZE_MAX], int *skip,
                                     int64_t txfm_cache[NB_TXFM_MODES]) {
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
  vp9_prob skip_prob = cm->mb_no_coeff_skip ?
                       vp9_get_pred_prob(cm, xd, PRED_MBSKIP) : 128;
  int64_t rd[2][TX_SIZE_MAX];
  int n;

  r[1][TX_16X16] = r[0][TX_16X16] + vp9_cost_one(cm->prob_tx[0]) +
                   vp9_cost_one(cm->prob_tx[1]);
  r[1][TX_8X8]   = r[0][TX_8X8] + vp9_cost_one(cm->prob_tx[0]) +
                   vp9_cost_zero(cm->prob_tx[1]);
  r[1][TX_4X4]   = r[0][TX_4X4] + vp9_cost_zero(cm->prob_tx[0]);

  if (cm->mb_no_coeff_skip) {
    int s0, s1;

    assert(skip_prob > 0);
    s0 = vp9_cost_bit(skip_prob, 0);
    s1 = vp9_cost_bit(skip_prob, 1);

    for (n = TX_4X4; n <= TX_16X16; n++) {
      if (s[n]) {
        rd[0][n] = rd[1][n] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
      } else {
        rd[0][n] = RDCOST(x->rdmult, x->rddiv, r[0][n] + s0, d[n]);
        rd[1][n] = RDCOST(x->rdmult, x->rddiv, r[1][n] + s0, d[n]);
      }
863 864
    }
  } else {
865 866 867
    for (n = TX_4X4; n <= TX_16X16; n++) {
      rd[0][n] = RDCOST(x->rdmult, x->rddiv, r[0][n], d[n]);
      rd[1][n] = RDCOST(x->rdmult, x->rddiv, r[1][n], d[n]);
868 869 870
    }
  }

871 872 873
  if ( cm->txfm_mode == ALLOW_16X16 ||
      (cm->txfm_mode == TX_MODE_SELECT &&