vp9_rdopt.c 172 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5 6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9 10 11 12 13 14 15
 */


#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
16 17
#include "vp9/common/vp9_pragmas.h"

18 19 20 21 22
#include "vp9/encoder/vp9_tokenize.h"
#include "vp9/encoder/vp9_treewriter.h"
#include "vp9/encoder/vp9_onyx_int.h"
#include "vp9/encoder/vp9_modecosts.h"
#include "vp9/encoder/vp9_encodeintra.h"
23 24 25 26 27 28
#include "vp9/common/vp9_entropymode.h"
#include "vp9/common/vp9_reconinter.h"
#include "vp9/common/vp9_reconintra.h"
#include "vp9/common/vp9_reconintra4x4.h"
#include "vp9/common/vp9_findnearmv.h"
#include "vp9/common/vp9_quant_common.h"
29 30 31 32 33 34
#include "vp9/encoder/vp9_encodemb.h"
#include "vp9/encoder/vp9_quantize.h"
#include "vp9/encoder/vp9_variance.h"
#include "vp9/encoder/vp9_mcomp.h"
#include "vp9/encoder/vp9_rdopt.h"
#include "vp9/encoder/vp9_ratectrl.h"
John Koleszar's avatar
John Koleszar committed
35
#include "vpx_mem/vpx_mem.h"
36 37
#include "vp9/common/vp9_systemdependent.h"
#include "vp9/encoder/vp9_encodemv.h"
John Koleszar's avatar
John Koleszar committed
38

39 40 41
#include "vp9/common/vp9_seg_common.h"
#include "vp9/common/vp9_pred_common.h"
#include "vp9/common/vp9_entropy.h"
42
#include "vp9_rtcd.h"
43
#include "vp9/common/vp9_mvref_common.h"
Paul Wilkins's avatar
Paul Wilkins committed
44

John Koleszar's avatar
John Koleszar committed
45 46
#define MAXF(a,b)            (((a) > (b)) ? (a) : (b))

47 48
#define INVALID_MV 0x80008000

49 50 51
/* Factor to weigh the rate for switchable interp filters */
#define SWITCHABLE_INTERP_RATE_FACTOR 1

John Koleszar's avatar
John Koleszar committed
52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69
static const int auto_speed_thresh[17] = {
  1000,
  200,
  150,
  130,
  150,
  125,
  120,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  105
John Koleszar's avatar
John Koleszar committed
70 71
};

72
#if CONFIG_PRED_FILTER
73
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121
  {ZEROMV,    LAST_FRAME,   NONE,  0},
  {ZEROMV,    LAST_FRAME,   NONE,  1},
  {DC_PRED,   INTRA_FRAME,  NONE,  0},

  {NEARESTMV, LAST_FRAME,   NONE,  0},
  {NEARESTMV, LAST_FRAME,   NONE,  1},
  {NEARMV,    LAST_FRAME,   NONE,  0},
  {NEARMV,    LAST_FRAME,   NONE,  1},

  {ZEROMV,    GOLDEN_FRAME, NONE,  0},
  {ZEROMV,    GOLDEN_FRAME, NONE,  1},
  {NEARESTMV, GOLDEN_FRAME, NONE,  0},
  {NEARESTMV, GOLDEN_FRAME, NONE,  1},

  {ZEROMV,    ALTREF_FRAME, NONE,  0},
  {ZEROMV,    ALTREF_FRAME, NONE,  1},
  {NEARESTMV, ALTREF_FRAME, NONE,  0},
  {NEARESTMV, ALTREF_FRAME, NONE,  1},

  {NEARMV,    GOLDEN_FRAME, NONE,  0},
  {NEARMV,    GOLDEN_FRAME, NONE,  1},
  {NEARMV,    ALTREF_FRAME, NONE,  0},
  {NEARMV,    ALTREF_FRAME, NONE,  1},

  {V_PRED,    INTRA_FRAME,  NONE,  0},
  {H_PRED,    INTRA_FRAME,  NONE,  0},
  {D45_PRED,  INTRA_FRAME,  NONE,  0},
  {D135_PRED, INTRA_FRAME,  NONE,  0},
  {D117_PRED, INTRA_FRAME,  NONE,  0},
  {D153_PRED, INTRA_FRAME,  NONE,  0},
  {D27_PRED,  INTRA_FRAME,  NONE,  0},
  {D63_PRED,  INTRA_FRAME,  NONE,  0},

  {TM_PRED,   INTRA_FRAME,  NONE,  0},

  {NEWMV,     LAST_FRAME,   NONE,  0},
  {NEWMV,     LAST_FRAME,   NONE,  1},
  {NEWMV,     GOLDEN_FRAME, NONE,  0},
  {NEWMV,     GOLDEN_FRAME, NONE,  1},
  {NEWMV,     ALTREF_FRAME, NONE,  0},
  {NEWMV,     ALTREF_FRAME, NONE,  1},

  {SPLITMV,   LAST_FRAME,   NONE,  0},
  {SPLITMV,   GOLDEN_FRAME, NONE,  0},
  {SPLITMV,   ALTREF_FRAME, NONE,  0},

  {B_PRED,    INTRA_FRAME,  NONE,  0},
  {I8X8_PRED, INTRA_FRAME,  NONE,  0},
John Koleszar's avatar
John Koleszar committed
122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141

  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME, 0},

  {ZEROMV,    ALTREF_FRAME, LAST_FRAME,   0},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME,   0},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME,   0},

  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME, 0},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME, 0},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME, 0},

  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME,   0},
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME, 0},

  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME, 0},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME,   0},
142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME, 0},

#if CONFIG_COMP_INTERINTRA_PRED
  /* compound inter-intra prediction */
  {ZEROMV,    LAST_FRAME,   INTRA_FRAME, 0},
  {NEARESTMV, LAST_FRAME,   INTRA_FRAME, 0},
  {NEARMV,    LAST_FRAME,   INTRA_FRAME, 0},
  {NEWMV,     LAST_FRAME,   INTRA_FRAME, 0},

  {ZEROMV,    GOLDEN_FRAME,   INTRA_FRAME, 0},
  {NEARESTMV, GOLDEN_FRAME,   INTRA_FRAME, 0},
  {NEARMV,    GOLDEN_FRAME,   INTRA_FRAME, 0},
  {NEWMV,     GOLDEN_FRAME,   INTRA_FRAME, 0},

  {ZEROMV,    ALTREF_FRAME,   INTRA_FRAME, 0},
  {NEARESTMV, ALTREF_FRAME,   INTRA_FRAME, 0},
  {NEARMV,    ALTREF_FRAME,   INTRA_FRAME, 0},
  {NEWMV,     ALTREF_FRAME,   INTRA_FRAME, 0},
#endif
John Koleszar's avatar
John Koleszar committed
161
};
162
#else
163
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
164 165
  {ZEROMV,    LAST_FRAME,   NONE},
  {DC_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
166

167 168
  {NEARESTMV, LAST_FRAME,   NONE},
  {NEARMV,    LAST_FRAME,   NONE},
John Koleszar's avatar
John Koleszar committed
169

170 171
  {ZEROMV,    GOLDEN_FRAME, NONE},
  {NEARESTMV, GOLDEN_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
172

173 174
  {ZEROMV,    ALTREF_FRAME, NONE},
  {NEARESTMV, ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
175

176 177
  {NEARMV,    GOLDEN_FRAME, NONE},
  {NEARMV,    ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
178

179 180 181 182 183 184 185 186
  {V_PRED,    INTRA_FRAME,  NONE},
  {H_PRED,    INTRA_FRAME,  NONE},
  {D45_PRED,  INTRA_FRAME,  NONE},
  {D135_PRED, INTRA_FRAME,  NONE},
  {D117_PRED, INTRA_FRAME,  NONE},
  {D153_PRED, INTRA_FRAME,  NONE},
  {D27_PRED,  INTRA_FRAME,  NONE},
  {D63_PRED,  INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
187

188
  {TM_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
189

190 191 192
  {NEWMV,     LAST_FRAME,   NONE},
  {NEWMV,     GOLDEN_FRAME, NONE},
  {NEWMV,     ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
193

194 195 196
  {SPLITMV,   LAST_FRAME,   NONE},
  {SPLITMV,   GOLDEN_FRAME, NONE},
  {SPLITMV,   ALTREF_FRAME, NONE},
197

198 199
  {B_PRED,    INTRA_FRAME,  NONE},
  {I8X8_PRED, INTRA_FRAME,  NONE},
200

John Koleszar's avatar
John Koleszar committed
201 202 203 204
  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME},
205

John Koleszar's avatar
John Koleszar committed
206 207 208
  {ZEROMV,    ALTREF_FRAME, LAST_FRAME},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME},
209

John Koleszar's avatar
John Koleszar committed
210 211 212
  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME},
213

John Koleszar's avatar
John Koleszar committed
214 215 216
  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME  },
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME},
217

John Koleszar's avatar
John Koleszar committed
218 219
  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME  },
220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME},

#if CONFIG_COMP_INTERINTRA_PRED
  /* compound inter-intra prediction */
  {ZEROMV,    LAST_FRAME,   INTRA_FRAME},
  {NEARESTMV, LAST_FRAME,   INTRA_FRAME},
  {NEARMV,    LAST_FRAME,   INTRA_FRAME},
  {NEWMV,     LAST_FRAME,   INTRA_FRAME},

  {ZEROMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEARESTMV, GOLDEN_FRAME,   INTRA_FRAME},
  {NEARMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEWMV,     GOLDEN_FRAME,   INTRA_FRAME},

  {ZEROMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEARESTMV, ALTREF_FRAME,   INTRA_FRAME},
  {NEARMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEWMV,     ALTREF_FRAME,   INTRA_FRAME},
#endif
John Koleszar's avatar
John Koleszar committed
239
};
240
#endif
John Koleszar's avatar
John Koleszar committed
241 242

static void fill_token_costs(
243
  unsigned int (*c)[COEF_BANDS][PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS],
244
  const vp9_prob(*p)[COEF_BANDS][PREV_COEF_CONTEXTS][ENTROPY_NODES],
John Koleszar's avatar
John Koleszar committed
245 246 247 248 249 250 251
  int block_type_counts) {
  int i, j, k;

  for (i = 0; i < block_type_counts; i++)
    for (j = 0; j < COEF_BANDS; j++)
      for (k = 0; k < PREV_COEF_CONTEXTS; k++) {
        if (k == 0 && ((j > 0 && i > 0) || (j > 1 && i == 0)))
252 253
          vp9_cost_tokens_skip((int *)(c[i][j][k]),
                               p[i][j][k],
254
                               vp9_coef_tree);
John Koleszar's avatar
John Koleszar committed
255
        else
256 257
          vp9_cost_tokens((int *)(c[i][j][k]),
                          p[i][j][k],
258
                          vp9_coef_tree);
John Koleszar's avatar
John Koleszar committed
259
      }
John Koleszar's avatar
John Koleszar committed
260 261
}

262

263 264 265 266
static int rd_iifactor[32] =  { 4, 4, 3, 2, 1, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0, };
John Koleszar's avatar
John Koleszar committed
267

268
// 3* dc_qlookup[Q]*dc_qlookup[Q];
269

270
/* values are now correlated to quantizer */
Paul Wilkins's avatar
Paul Wilkins committed
271 272 273
static int sad_per_bit16lut[QINDEX_RANGE];
static int sad_per_bit4lut[QINDEX_RANGE];

274
void vp9_init_me_luts() {
John Koleszar's avatar
John Koleszar committed
275 276 277 278 279 280 281
  int i;

  // Initialize the sad lut tables using a formulaic calculation for now
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < QINDEX_RANGE; i++) {
    sad_per_bit16lut[i] =
282 283
      (int)((0.0418 * vp9_convert_qindex_to_q(i)) + 2.4107);
    sad_per_bit4lut[i] = (int)((0.063 * vp9_convert_qindex_to_q(i)) + 2.742);
John Koleszar's avatar
John Koleszar committed
284
  }
Paul Wilkins's avatar
Paul Wilkins committed
285
}
John Koleszar's avatar
John Koleszar committed
286

287
static int compute_rd_mult(int qindex) {
John Koleszar's avatar
John Koleszar committed
288
  int q;
289

290
  q = vp9_dc_quant(qindex, 0);
John Koleszar's avatar
John Koleszar committed
291
  return (11 * q * q) >> 6;
292 293
}

294
void vp9_initialize_me_consts(VP9_COMP *cpi, int QIndex) {
John Koleszar's avatar
John Koleszar committed
295 296
  cpi->mb.sadperbit16 =  sad_per_bit16lut[QIndex];
  cpi->mb.sadperbit4  =  sad_per_bit4lut[QIndex];
John Koleszar's avatar
John Koleszar committed
297 298
}

299

300
void vp9_initialize_rd_consts(VP9_COMP *cpi, int QIndex) {
301
  int q, i;
John Koleszar's avatar
John Koleszar committed
302

303
  vp9_clear_system_state();  // __asm emms;
John Koleszar's avatar
John Koleszar committed
304

John Koleszar's avatar
John Koleszar committed
305 306 307 308 309
  // Further tests required to see if optimum is different
  // for key frames, golden frames and arf frames.
  // if (cpi->common.refresh_golden_frame ||
  //     cpi->common.refresh_alt_ref_frame)
  QIndex = (QIndex < 0) ? 0 : ((QIndex > MAXQ) ? MAXQ : QIndex);
310

John Koleszar's avatar
John Koleszar committed
311
  cpi->RDMULT = compute_rd_mult(QIndex);
John Koleszar's avatar
John Koleszar committed
312

John Koleszar's avatar
John Koleszar committed
313 314 315
  // Extend rate multiplier along side quantizer zbin increases
  if (cpi->zbin_over_quant  > 0) {
    double oq_factor;
316

John Koleszar's avatar
John Koleszar committed
317 318 319 320 321
    // Experimental code using the same basic equation as used for Q above
    // The units of cpi->zbin_over_quant are 1/128 of Q bin size
    oq_factor = 1.0 + ((double)0.0015625 * cpi->zbin_over_quant);
    cpi->RDMULT = (int)((double)cpi->RDMULT * oq_factor * oq_factor);
  }
John Koleszar's avatar
John Koleszar committed
322

John Koleszar's avatar
John Koleszar committed
323 324 325 326 327 328 329
  if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    if (cpi->twopass.next_iiratio > 31)
      cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
    else
      cpi->RDMULT +=
        (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
  }
John Koleszar's avatar
John Koleszar committed
330

John Koleszar's avatar
John Koleszar committed
331 332
  if (cpi->RDMULT < 7)
    cpi->RDMULT = 7;
333

John Koleszar's avatar
John Koleszar committed
334 335
  cpi->mb.errorperbit = (cpi->RDMULT / 110);
  cpi->mb.errorperbit += (cpi->mb.errorperbit == 0);
336

337
  vp9_set_speed_features(cpi);
John Koleszar's avatar
John Koleszar committed
338

339
  q = (int)pow(vp9_dc_quant(QIndex, 0) >> 2, 1.25);
John Koleszar's avatar
John Koleszar committed
340 341
  q = q << 2;
  cpi->RDMULT = cpi->RDMULT << 4;
Paul Wilkins's avatar
Paul Wilkins committed
342

John Koleszar's avatar
John Koleszar committed
343 344
  if (q < 8)
    q = 8;
345

John Koleszar's avatar
John Koleszar committed
346 347 348
  if (cpi->RDMULT > 1000) {
    cpi->RDDIV = 1;
    cpi->RDMULT /= 100;
John Koleszar's avatar
John Koleszar committed
349

John Koleszar's avatar
John Koleszar committed
350 351 352 353 354 355
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < INT_MAX) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q / 100;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
356

John Koleszar's avatar
John Koleszar committed
357
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
358
    }
John Koleszar's avatar
John Koleszar committed
359 360
  } else {
    cpi->RDDIV = 100;
John Koleszar's avatar
John Koleszar committed
361

John Koleszar's avatar
John Koleszar committed
362 363 364 365 366 367
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < (INT_MAX / q)) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
368

John Koleszar's avatar
John Koleszar committed
369
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
370
    }
John Koleszar's avatar
John Koleszar committed
371
  }
John Koleszar's avatar
John Koleszar committed
372

John Koleszar's avatar
John Koleszar committed
373
  fill_token_costs(
374
    cpi->mb.token_costs[TX_4X4],
375
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs,
John Koleszar's avatar
John Koleszar committed
376
    BLOCK_TYPES);
377 378
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_4X4],
379
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11])
380 381
    cpi->common.fc.hybrid_coef_probs,
    BLOCK_TYPES);
John Koleszar's avatar
John Koleszar committed
382

John Koleszar's avatar
John Koleszar committed
383
  fill_token_costs(
384
    cpi->mb.token_costs[TX_8X8],
385
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs_8x8,
John Koleszar's avatar
John Koleszar committed
386
    BLOCK_TYPES_8X8);
387 388
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_8X8],
389
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11])
390 391
    cpi->common.fc.hybrid_coef_probs_8x8,
    BLOCK_TYPES_8X8);
392

Daniel Kang's avatar
Daniel Kang committed
393
  fill_token_costs(
394
    cpi->mb.token_costs[TX_16X16],
395
    (const vp9_prob(*)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs_16x16,
Daniel Kang's avatar
Daniel Kang committed
396
    BLOCK_TYPES_16X16);
397 398
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_16X16],
399
    (const vp9_prob(*)[8][PREV_COEF_CONTEXTS][11])
400 401
    cpi->common.fc.hybrid_coef_probs_16x16,
    BLOCK_TYPES_16X16);
Daniel Kang's avatar
Daniel Kang committed
402

403 404 405 406 407 408 409
#if CONFIG_TX32X32 && CONFIG_SUPERBLOCKS
  fill_token_costs(
    cpi->mb.token_costs[TX_32X32],
    (const vp9_prob(*)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs_32x32,
    BLOCK_TYPES_32X32);
#endif

John Koleszar's avatar
John Koleszar committed
410 411
  /*rough estimate for costing*/
  cpi->common.kf_ymode_probs_index = cpi->common.base_qindex >> 4;
412
  vp9_init_mode_costs(cpi);
John Koleszar's avatar
John Koleszar committed
413

414
  if (cpi->common.frame_type != KEY_FRAME) {
415
    vp9_build_nmv_cost_table(
416 417 418 419 420 421
        cpi->mb.nmvjointcost,
        cpi->mb.e_mbd.allow_high_precision_mv ?
        cpi->mb.nmvcost_hp : cpi->mb.nmvcost,
        &cpi->common.fc.nmvc,
        cpi->mb.e_mbd.allow_high_precision_mv, 1, 1);
  }
John Koleszar's avatar
John Koleszar committed
422 423
}

424
int vp9_block_error_c(short *coeff, short *dqcoeff, int block_size) {
425
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
426

427
  for (i = 0; i < block_size; i++) {
John Koleszar's avatar
John Koleszar committed
428 429 430
    int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }
John Koleszar's avatar
John Koleszar committed
431

John Koleszar's avatar
John Koleszar committed
432
  return error;
John Koleszar's avatar
John Koleszar committed
433 434
}

435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453
int vp9_mbblock_error_8x8_c(MACROBLOCK *mb, int dc) {
  BLOCK  *be;
  BLOCKD *bd;
  int i, j;
  int berror, error = 0;

  for (i = 0; i < 16; i+=4) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
    berror = 0;
    for (j = dc; j < 64; j++) {
      int this_diff = be->coeff[j] - bd->dqcoeff[j];
      berror += this_diff * this_diff;
    }
    error += berror;
  }
  return error;
}

454
int vp9_mbblock_error_c(MACROBLOCK *mb, int dc) {
John Koleszar's avatar
John Koleszar committed
455 456 457 458
  BLOCK  *be;
  BLOCKD *bd;
  int i, j;
  int berror, error = 0;
John Koleszar's avatar
John Koleszar committed
459

John Koleszar's avatar
John Koleszar committed
460 461 462 463 464 465 466
  for (i = 0; i < 16; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
    berror = 0;
    for (j = dc; j < 16; j++) {
      int this_diff = be->coeff[j] - bd->dqcoeff[j];
      berror += this_diff * this_diff;
John Koleszar's avatar
John Koleszar committed
467
    }
John Koleszar's avatar
John Koleszar committed
468 469 470
    error += berror;
  }
  return error;
John Koleszar's avatar
John Koleszar committed
471 472
}

473
int vp9_mbuverror_c(MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
474 475
  BLOCK  *be;
  BLOCKD *bd;
John Koleszar's avatar
John Koleszar committed
476

477
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
478

John Koleszar's avatar
John Koleszar committed
479 480 481
  for (i = 16; i < 24; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
John Koleszar's avatar
John Koleszar committed
482

483
    error += vp9_block_error_c(be->coeff, bd->dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
484
  }
John Koleszar's avatar
John Koleszar committed
485

John Koleszar's avatar
John Koleszar committed
486
  return error;
John Koleszar's avatar
John Koleszar committed
487 488
}

489
int vp9_uvsse(MACROBLOCK *x) {
John Koleszar's avatar
John Koleszar committed
490 491 492 493 494 495 496
  unsigned char *uptr, *vptr;
  unsigned char *upred_ptr = (*(x->block[16].base_src) + x->block[16].src);
  unsigned char *vpred_ptr = (*(x->block[20].base_src) + x->block[20].src);
  int uv_stride = x->block[16].src_stride;

  unsigned int sse1 = 0;
  unsigned int sse2 = 0;
497 498
  int mv_row = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.row;
  int mv_col = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.col;
John Koleszar's avatar
John Koleszar committed
499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519
  int offset;
  int pre_stride = x->e_mbd.block[16].pre_stride;

  if (mv_row < 0)
    mv_row -= 1;
  else
    mv_row += 1;

  if (mv_col < 0)
    mv_col -= 1;
  else
    mv_col += 1;

  mv_row /= 2;
  mv_col /= 2;

  offset = (mv_row >> 3) * pre_stride + (mv_col >> 3);
  uptr = x->e_mbd.pre.u_buffer + offset;
  vptr = x->e_mbd.pre.v_buffer + offset;

  if ((mv_row | mv_col) & 7) {
520
    vp9_sub_pixel_variance8x8(uptr, pre_stride, (mv_col & 7) << 1,
521
                              (mv_row & 7) << 1, upred_ptr, uv_stride, &sse2);
522
    vp9_sub_pixel_variance8x8(vptr, pre_stride, (mv_col & 7) << 1,
523
                              (mv_row & 7) << 1, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
524 525
    sse2 += sse1;
  } else {
526 527
    vp9_variance8x8(uptr, pre_stride, upred_ptr, uv_stride, &sse2);
    vp9_variance8x8(vptr, pre_stride, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
528 529 530
    sse2 += sse1;
  }
  return sse2;
John Koleszar's avatar
John Koleszar committed
531 532 533

}

534
static int cost_coeffs_2x2(MACROBLOCK *mb,
535
                           BLOCKD *b, PLANE_TYPE type,
536
                           ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l) {
537
  int c = (type == PLANE_TYPE_Y_NO_DC); /* start at coef 0, unless Y with Y2 */
538 539 540 541 542
  int eob = b->eob;
  int pt;    /* surrounding block/prev coef predictor */
  int cost = 0;
  short *qcoeff_ptr = b->qcoeff;

543
  VP9_COMBINEENTROPYCONTEXTS(pt, *a, *l);
544 545 546
  assert(eob <= 4);

  for (; c < eob; c++) {
547 548 549 550 551
    int v = qcoeff_ptr[vp9_default_zig_zag1d[c]];
    int t = vp9_dct_value_tokens_ptr[v].Token;
    cost += mb->token_costs[TX_8X8][type][vp9_coef_bands[c]][pt][t];
    cost += vp9_dct_value_cost_ptr[v];
    pt = vp9_prev_token_class[t];
552 553 554
  }

  if (c < 4)
555
    cost += mb->token_costs[TX_8X8][type][vp9_coef_bands[c]]
556
            [pt] [DCT_EOB_TOKEN];
557 558
  // is eob first coefficient;
  pt = (c > !type);
559 560 561 562
  *a = *l = pt;
  return cost;
}

563
static int cost_coeffs(MACROBLOCK *mb, BLOCKD *b, PLANE_TYPE type,
Daniel Kang's avatar
Daniel Kang committed
564
                       ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l,
565
                       TX_SIZE tx_size) {
Daniel Kang's avatar
Daniel Kang committed
566
  const int eob = b->eob;
567
  int c = (type == PLANE_TYPE_Y_NO_DC); /* start at coef 0, unless Y with Y2 */
568
  int cost = 0, default_eob, seg_eob;
Daniel Kang's avatar
Daniel Kang committed
569 570
  int pt;                     /* surrounding block/prev coef predictor */
  int const *scan, *band;
John Koleszar's avatar
John Koleszar committed
571
  short *qcoeff_ptr = b->qcoeff;
572 573 574
  MACROBLOCKD *xd = &mb->e_mbd;
  MB_MODE_INFO *mbmi = &mb->e_mbd.mode_info_context->mbmi;
  TX_TYPE tx_type = DCT_DCT;
Paul Wilkins's avatar
Paul Wilkins committed
575
  int segment_id = mbmi->segment_id;
Jim Bankoski's avatar
Jim Bankoski committed
576 577 578
  scan = vp9_default_zig_zag1d;
  band = vp9_coef_bands;
  default_eob = 16;
579

580
  switch (tx_size) {
Daniel Kang's avatar
Daniel Kang committed
581
    case TX_4X4:
Deb Mukherjee's avatar
Deb Mukherjee committed
582 583 584 585 586
      if (type == PLANE_TYPE_Y_WITH_DC) {
        tx_type = get_tx_type_4x4(xd, b);
        if (tx_type != DCT_DCT) {
          switch (tx_type) {
            case ADST_DCT:
587
              scan = vp9_row_scan;
Deb Mukherjee's avatar
Deb Mukherjee committed
588 589 590
              break;

            case DCT_ADST:
591
              scan = vp9_col_scan;
Deb Mukherjee's avatar
Deb Mukherjee committed
592 593 594
              break;

            default:
595
              scan = vp9_default_zig_zag1d;
Deb Mukherjee's avatar
Deb Mukherjee committed
596 597
              break;
          }
598
        }
Daniel Kang's avatar
Daniel Kang committed
599
      }
Deb Mukherjee's avatar
Deb Mukherjee committed
600

Daniel Kang's avatar
Daniel Kang committed
601 602
      break;
    case TX_8X8:
603 604
      scan = vp9_default_zig_zag1d_8x8;
      band = vp9_coef_bands_8x8;
Daniel Kang's avatar
Daniel Kang committed
605
      default_eob = 64;
Deb Mukherjee's avatar
Deb Mukherjee committed
606
      if (type == PLANE_TYPE_Y_WITH_DC) {
607
        BLOCKD *bb;
608
        int ib = (int)(b - xd->block);
609 610 611
        if (ib < 16) {
          ib = (ib & 8) + ((ib & 4) >> 1);
          bb = xd->block + ib;
Deb Mukherjee's avatar
Deb Mukherjee committed
612
          tx_type = get_tx_type_8x8(xd, bb);
613
        }
614
      }
Daniel Kang's avatar
Daniel Kang committed
615 616
      break;
    case TX_16X16:
617 618
      scan = vp9_default_zig_zag1d_16x16;
      band = vp9_coef_bands_16x16;
Daniel Kang's avatar
Daniel Kang committed
619
      default_eob = 256;
Deb Mukherjee's avatar
Deb Mukherjee committed
620 621
      if (type == PLANE_TYPE_Y_WITH_DC) {
        tx_type = get_tx_type_16x16(xd, b);
622 623 624 625 626 627
#if CONFIG_TX32X32 && CONFIG_SUPERBLOCKS
      } else if (type == PLANE_TYPE_UV) {
        int ib = (int)(b - xd->block) - 16;

        qcoeff_ptr = xd->sb_coeff_data.qcoeff + 1024 + 64 * ib;
#endif
Deb Mukherjee's avatar
Deb Mukherjee committed
628
      }
Daniel Kang's avatar
Daniel Kang committed
629
      break;
630 631 632 633 634 635 636 637
#if CONFIG_TX32X32 && CONFIG_SUPERBLOCKS
    case TX_32X32:
      scan = vp9_default_zig_zag1d_32x32;
      band = vp9_coef_bands_32x32;
      default_eob = 1024;
      qcoeff_ptr = xd->sb_coeff_data.qcoeff;
      break;
#endif
Daniel Kang's avatar
Daniel Kang committed
638
    default:
639
      abort();
Daniel Kang's avatar
Daniel Kang committed
640 641
      break;
  }
642 643
  if (vp9_segfeature_active(&mb->e_mbd, segment_id, SEG_LVL_EOB))
    seg_eob = vp9_get_segdata(&mb->e_mbd, segment_id, SEG_LVL_EOB);
644 645 646
  else
    seg_eob = default_eob;

647
  VP9_COMBINEENTROPYCONTEXTS(pt, *a, *l);
648

649 650 651
  if (tx_type != DCT_DCT) {
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
652
      int t = vp9_dct_value_tokens_ptr[v].Token;
653
      cost += mb->hybrid_token_costs[tx_size][type][band[c]][pt][t];
654 655
      cost += vp9_dct_value_cost_ptr[v];
      pt = vp9_prev_token_class[t];
656 657 658 659
    }
    if (c < seg_eob)
      cost += mb->hybrid_token_costs[tx_size][type][band[c]]
          [pt][DCT_EOB_TOKEN];
660
  } else {
661 662
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
663
      int t = vp9_dct_value_tokens_ptr[v].Token;
664
      cost += mb->token_costs[tx_size][type][band[c]][pt][t];
665 666
      cost += vp9_dct_value_cost_ptr[v];
      pt = vp9_prev_token_class[t];
667 668 669 670
    }
    if (c < seg_eob)
      cost += mb->token_costs[tx_size][type][band[c]]
          [pt][DCT_EOB_TOKEN];
671 672
  }

673 674
  // is eob first coefficient;
  pt = (c > !type);
675 676 677 678
  *a = *l = pt;
  return cost;
}

679
static int rdcost_mby_4x4(MACROBLOCK *mb, int has_2nd_order, int backup) {
John Koleszar's avatar
John Koleszar committed
680 681
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
682
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
683 684 685
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta;
  ENTROPY_CONTEXT *tl;
686

687 688 689
  if (backup) {
    vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
690

691 692 693 694 695 696
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)xd->above_context;
    tl = (ENTROPY_CONTEXT *)xd->left_context;
  }
John Koleszar's avatar
John Koleszar committed
697

John Koleszar's avatar
John Koleszar committed
698
  for (b = 0; b < 16; b++)
699 700 701
    cost += cost_coeffs(mb, xd->block + b,
                        (has_2nd_order ?
                         PLANE_TYPE_Y_NO_DC : PLANE_TYPE_Y_WITH_DC),
702
                        ta + vp9_block2above[b], tl + vp9_block2left[b],
Daniel Kang's avatar
Daniel Kang committed
703
                        TX_4X4);
John Koleszar's avatar
John Koleszar committed
704

705 706 707 708
  if (has_2nd_order)
    cost += cost_coeffs(mb, xd->block + 24, PLANE_TYPE_Y2,
                        ta + vp9_block2above[24], tl + vp9_block2left[24],
                        TX_4X4);
John Koleszar's avatar
John Koleszar committed
709

John Koleszar's avatar
John Koleszar committed
710
  return cost;
John Koleszar's avatar
John Koleszar committed
711 712
}

713 714 715
static void macro_block_yrd_4x4(MACROBLOCK *mb,
                                int *Rate,
                                int *Distortion,
716
                                int *skippable, int backup) {
Paul Wilkins's avatar
Paul Wilkins committed
717
  MACROBLOCKD *const xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
718
  BLOCK   *const mb_y2 = mb->block + 24;
Paul Wilkins's avatar
Paul Wilkins committed
719
  BLOCKD *const x_y2  = xd->block + 24;
Jim Bankoski's avatar
Jim Bankoski committed
720
  int d, has_2nd_order;
721

722 723
  xd->mode_info_context->mbmi.txfm_size = TX_4X4;
  has_2nd_order = get_2nd_order_usage(xd);
John Koleszar's avatar
John Koleszar committed
724
  // Fdct and building the 2nd order block
725 726 727 728 729
  vp9_transform_mby_4x4(mb);
  vp9_quantize_mby_4x4(mb);
  d = vp9_mbblock_error(mb, has_2nd_order);
  if (has_2nd_order)
    d += vp9_block_error(mb_y2->coeff, x_y2->dqcoeff, 16);
730

John Koleszar's avatar
John Koleszar committed
731 732
  *Distortion = (d >> 2);
  // rate
733 734
  *Rate = rdcost_mby_4x4(mb, has_2nd_order, backup);
  *skippable = vp9_mby_is_skippable_4x4(&mb->e_mbd, has_2nd_order);
735
}
John Koleszar's avatar
John Koleszar committed
736

737
static int rdcost_mby_8x8(MACROBLOCK *mb, int has_2nd_order, int backup) {
John Koleszar's avatar
John Koleszar committed
738 739
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
740
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
741 742 743 744
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta;
  ENTROPY_CONTEXT *tl;

Ronald S. Bultje's avatar
Ronald S. Bultje committed
745 746 747
  if (backup) {
    vpx_memcpy(&t_above,xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
748

Ronald S. Bultje's avatar
Ronald S. Bultje committed
749 750 751 752 753 754
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)mb->e_mbd.above_context;
    tl = (ENTROPY_CONTEXT *)mb->e_mbd.left_context;
  }
John Koleszar's avatar
John Koleszar committed
755 756

  for (b = 0; b < 16; b += 4)
757 758 759
    cost += cost_coeffs(mb, xd->block + b,
                        (has_2nd_order ?
                         PLANE_TYPE_Y_NO_DC : PLANE_TYPE_Y_WITH_DC),
760
                        ta + vp9_block2above_8x8[b], tl + vp9_block2left_8x8[b],
Daniel Kang's avatar
Daniel Kang committed
761
                        TX_8X8);
John Koleszar's avatar
John Koleszar committed
762

763 764 765
  if (has_2nd_order)
    cost += cost_coeffs_2x2(mb, xd->block + 24, PLANE_TYPE_Y2,
                            ta + vp9_block2above[24], tl + vp9_block2left[24]);
John Koleszar's avatar
John Koleszar committed
766
  return cost;
767 768
}

John Koleszar's avatar
John Koleszar committed
769 770 771
static void macro_block_yrd_8x8(MACROBLOCK *mb,
                                int *Rate,
                                int *Distortion,
772
                                int *skippable, int backup) {
Paul Wilkins's avatar
Paul Wilkins committed
773
  MACROBLOCKD *const xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
774
  BLOCK   *const mb_y2 = mb->block + 24;
Paul Wilkins's avatar
Paul Wilkins committed
775
  BLOCKD *const x_y2  = xd->block + 24;
776 777 778
  int d, has_2nd_order;

  xd->mode_info_context->mbmi.txfm_size = TX_8X8;
John Koleszar's avatar
John Koleszar committed
779

780 781
  vp9_transform_mby_8x8(mb);
  vp9_quantize_mby_8x8(mb);
782 783 784 785
  has_2nd_order = get_2nd_order_usage(xd);
  d = vp9_mbblock_error_8x8_c(mb, has_2nd_order);
  if (has_2nd_order)
    d += vp9_block_error(mb_y2->coeff, x_y2->dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
786 787 788

  *Distortion = (d >> 2);
  // rate
789 790
  *Rate = rdcost_mby_8x8(mb, has_2nd_order, backup);
  *skippable = vp9_mby_is_skippable_8x8(&mb->e_mbd, has_2nd_order);
791
}
792

793
static int rdcost_mby_16x16(MACROBLOCK *mb, int backup) {
Daniel Kang's avatar
Daniel Kang committed
794
  int cost;
Paul Wilkins's avatar
Paul Wilkins committed
795
  MACROBLOCKD *xd = &mb->e_mbd;
Daniel Kang's avatar
Daniel Kang committed
796 797 798
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta, *tl;

799 800 801
  if (backup) {
    vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
Daniel Kang's avatar
Daniel Kang committed
802

803 804 805 806 807 808
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)xd->above_context;
    tl = (ENTROPY_CONTEXT *)xd->left_context;
  }
Daniel Kang's avatar
Daniel Kang committed
809

Paul Wilkins's avatar
Paul Wilkins committed
810
  cost = cost_coeffs(mb, xd->block, PLANE_TYPE_Y_WITH_DC, ta, tl, TX_16X16);
Daniel Kang's avatar
Daniel Kang committed
811 812
  return cost;
}
813

Daniel Kang's avatar
Daniel Kang committed
814
static void macro_block_yrd_16x16(MACROBLOCK *mb, int *Rate, int *Distortion,
815
                                  int *skippable, int backup) {
Daniel Kang's avatar
Daniel Kang committed
816
  int d;
Deb Mukherjee's avatar
Deb Mukherjee committed
817
  MACROBLOCKD *xd = &mb->e_mbd;
Daniel Kang's avatar
Daniel Kang committed
818

819 820
  xd->mode_info_context->mbmi.txfm_size = TX_16X16;
  vp9_transform_mby_16x16(mb);
821
  vp9_quantize_mby_16x16(mb);
822 823 824
  // TODO(jingning) is it possible to quickly determine whether to force
  //                trailing coefficients to be zero, instead of running trellis
  //                optimization in the rate-distortion optimization loop?
825
  if (mb->e_mbd.mode_info_context->mbmi.mode < I8X8_PRED)
826
    vp9_optimize_mby_16x16(mb);
827

828
  d = vp9_mbblock_error(mb, 0);
Daniel Kang's avatar
Daniel Kang committed
829 830 831

  *Distortion = (d >> 2);
  // rate
832
  *Rate = rdcost_mby_16x16(mb, backup);
833
  *skippable = vp9_mby_is_skippable_16x16(&mb->e_mbd);
Daniel Kang's avatar
Daniel Kang committed
834 835
}

836
static void choose_txfm_size_from_rd(VP9_COMP *cpi, MACROBLOCK *x,
837 838 839 840 841
                                     int (*r)[2], int *rate,
                                     int *d, int *distortion,
                                     int *s, int *skip,
                                     int64_t txfm_cache[NB_TXFM_MODES],
                                     TX_SIZE max_txfm_size) {
842 843 844 845 846
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
  vp9_prob skip_prob = cm->mb_no_coeff_skip ?
                       vp9_get_pred_prob(cm, xd, PRED_MBSKIP) : 128;
847 848 849 850 851 852 853 854 855 856 857 858
  int64_t rd[TX_SIZE_MAX_SB][2];
  int n, m;

  for (n = TX_4X4; n <= max_txfm_size; n++) {
    r[n][1] = r[n][0];
    for (m = 0; m <= n - (n == max_txfm_size); m++) {
      if (m == n)
        r[n][1] += vp9_cost_zero(cm->prob_tx[m]);
      else
        r[n][1] += vp9_cost_one(cm->prob_tx[m]);
    }
  }
859 860 861 862 863 864 865 866

  if (cm->mb_no_coeff_skip) {
    int s0, s1;

    assert(skip_prob > 0);
    s0 = vp9_cost_bit(skip_prob, 0);
    s1 = vp9_cost_bit(skip_prob, 1);

867
    for (n = TX_4X4; n <= max_txfm_size; n++) {
868
      if (s[n]) {
869
        rd[n][0] = rd[n][1] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
870
      } else {
871 872
        rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0] + s0, d[n]);
        rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1] + s0, d[n]);
873
      }
874 875
    }
  } else {
876 877 878
    for (n = TX_4X4; n <= max_txfm_size; n++) {
      rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0], d[n]);
      rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1], d[n]);