rdopt.c 166 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5 6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9 10 11 12 13 14 15
 */


#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
16
#include "vp9/common/pragmas.h"
John Koleszar's avatar
John Koleszar committed
17 18 19 20 21 22

#include "tokenize.h"
#include "treewriter.h"
#include "onyx_int.h"
#include "modecosts.h"
#include "encodeintra.h"
23 24 25 26 27 28
#include "vp9/common/entropymode.h"
#include "vp9/common/reconinter.h"
#include "vp9/common/reconintra.h"
#include "vp9/common/reconintra4x4.h"
#include "vp9/common/findnearmv.h"
#include "vp9/common/quant_common.h"
John Koleszar's avatar
John Koleszar committed
29 30
#include "encodemb.h"
#include "quantize.h"
31
#include "vp9/common/idct.h"
John Koleszar's avatar
John Koleszar committed
32 33
#include "variance.h"
#include "mcomp.h"
Yunqing Wang's avatar
Yunqing Wang committed
34
#include "rdopt.h"
Paul Wilkins's avatar
Paul Wilkins committed
35
#include "ratectrl.h"
John Koleszar's avatar
John Koleszar committed
36
#include "vpx_mem/vpx_mem.h"
37 38
#include "vp9/common/systemdependent.h"
#include "vp9/encoder/encodemv.h"
John Koleszar's avatar
John Koleszar committed
39

40 41 42
#include "vp9/common/seg_common.h"
#include "vp9/common/pred_common.h"
#include "vp9/common/entropy.h"
Jim Bankoski's avatar
Jim Bankoski committed
43
#include "vpx_rtcd.h"
44
#include "vp9/common/mvref_common.h"
Paul Wilkins's avatar
Paul Wilkins committed
45

John Koleszar's avatar
John Koleszar committed
46 47 48 49 50 51 52 53
#if CONFIG_RUNTIME_CPU_DETECT
#define IF_RTCD(x)  (x)
#else
#define IF_RTCD(x)  NULL
#endif

#define MAXF(a,b)            (((a) > (b)) ? (a) : (b))

54 55
#define INVALID_MV 0x80008000

56 57 58
/* Factor to weigh the rate for switchable interp filters */
#define SWITCHABLE_INTERP_RATE_FACTOR 1

John Koleszar's avatar
John Koleszar committed
59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76
static const int auto_speed_thresh[17] = {
  1000,
  200,
  150,
  130,
  150,
  125,
  120,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  105
John Koleszar's avatar
John Koleszar committed
77 78
};

79
#if CONFIG_PRED_FILTER
80
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
John Koleszar's avatar
John Koleszar committed
81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149
  {ZEROMV,    LAST_FRAME,   0,  0},
  {ZEROMV,    LAST_FRAME,   0,  1},
  {DC_PRED,   INTRA_FRAME,  0,  0},

  {NEARESTMV, LAST_FRAME,   0,  0},
  {NEARESTMV, LAST_FRAME,   0,  1},
  {NEARMV,    LAST_FRAME,   0,  0},
  {NEARMV,    LAST_FRAME,   0,  1},

  {ZEROMV,    GOLDEN_FRAME, 0,  0},
  {ZEROMV,    GOLDEN_FRAME, 0,  1},
  {NEARESTMV, GOLDEN_FRAME, 0,  0},
  {NEARESTMV, GOLDEN_FRAME, 0,  1},

  {ZEROMV,    ALTREF_FRAME, 0,  0},
  {ZEROMV,    ALTREF_FRAME, 0,  1},
  {NEARESTMV, ALTREF_FRAME, 0,  0},
  {NEARESTMV, ALTREF_FRAME, 0,  1},

  {NEARMV,    GOLDEN_FRAME, 0,  0},
  {NEARMV,    GOLDEN_FRAME, 0,  1},
  {NEARMV,    ALTREF_FRAME, 0,  0},
  {NEARMV,    ALTREF_FRAME, 0,  1},

  {V_PRED,    INTRA_FRAME,  0,  0},
  {H_PRED,    INTRA_FRAME,  0,  0},
  {D45_PRED,  INTRA_FRAME,  0,  0},
  {D135_PRED, INTRA_FRAME,  0,  0},
  {D117_PRED, INTRA_FRAME,  0,  0},
  {D153_PRED, INTRA_FRAME,  0,  0},
  {D27_PRED,  INTRA_FRAME,  0,  0},
  {D63_PRED,  INTRA_FRAME,  0,  0},

  {TM_PRED,   INTRA_FRAME,  0,  0},

  {NEWMV,     LAST_FRAME,   0,  0},
  {NEWMV,     LAST_FRAME,   0,  1},
  {NEWMV,     GOLDEN_FRAME, 0,  0},
  {NEWMV,     GOLDEN_FRAME, 0,  1},
  {NEWMV,     ALTREF_FRAME, 0,  0},
  {NEWMV,     ALTREF_FRAME, 0,  1},

  {SPLITMV,   LAST_FRAME,   0,  0},
  {SPLITMV,   GOLDEN_FRAME, 0,  0},
  {SPLITMV,   ALTREF_FRAME, 0,  0},

  {B_PRED,    INTRA_FRAME,  0,  0},
  {I8X8_PRED, INTRA_FRAME,  0,  0},

  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME, 0},

  {ZEROMV,    ALTREF_FRAME, LAST_FRAME,   0},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME,   0},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME,   0},

  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME, 0},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME, 0},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME, 0},

  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME,   0},
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME, 0},

  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME, 0},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME,   0},
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME, 0}
John Koleszar's avatar
John Koleszar committed
150
};
151
#else
152
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
John Koleszar's avatar
John Koleszar committed
153 154
  {ZEROMV,    LAST_FRAME,   0},
  {DC_PRED,   INTRA_FRAME,  0},
John Koleszar's avatar
John Koleszar committed
155

John Koleszar's avatar
John Koleszar committed
156 157
  {NEARESTMV, LAST_FRAME,   0},
  {NEARMV,    LAST_FRAME,   0},
John Koleszar's avatar
John Koleszar committed
158

John Koleszar's avatar
John Koleszar committed
159 160
  {ZEROMV,    GOLDEN_FRAME, 0},
  {NEARESTMV, GOLDEN_FRAME, 0},
John Koleszar's avatar
John Koleszar committed
161

John Koleszar's avatar
John Koleszar committed
162 163
  {ZEROMV,    ALTREF_FRAME, 0},
  {NEARESTMV, ALTREF_FRAME, 0},
John Koleszar's avatar
John Koleszar committed
164

John Koleszar's avatar
John Koleszar committed
165 166
  {NEARMV,    GOLDEN_FRAME, 0},
  {NEARMV,    ALTREF_FRAME, 0},
John Koleszar's avatar
John Koleszar committed
167

John Koleszar's avatar
John Koleszar committed
168 169 170 171 172 173 174 175
  {V_PRED,    INTRA_FRAME,  0},
  {H_PRED,    INTRA_FRAME,  0},
  {D45_PRED,  INTRA_FRAME,  0},
  {D135_PRED, INTRA_FRAME,  0},
  {D117_PRED, INTRA_FRAME,  0},
  {D153_PRED, INTRA_FRAME,  0},
  {D27_PRED,  INTRA_FRAME,  0},
  {D63_PRED,  INTRA_FRAME,  0},
John Koleszar's avatar
John Koleszar committed
176

John Koleszar's avatar
John Koleszar committed
177
  {TM_PRED,   INTRA_FRAME,  0},
John Koleszar's avatar
John Koleszar committed
178

John Koleszar's avatar
John Koleszar committed
179 180 181
  {NEWMV,     LAST_FRAME,   0},
  {NEWMV,     GOLDEN_FRAME, 0},
  {NEWMV,     ALTREF_FRAME, 0},
John Koleszar's avatar
John Koleszar committed
182

John Koleszar's avatar
John Koleszar committed
183 184 185
  {SPLITMV,   LAST_FRAME,   0},
  {SPLITMV,   GOLDEN_FRAME, 0},
  {SPLITMV,   ALTREF_FRAME, 0},
186

John Koleszar's avatar
John Koleszar committed
187 188
  {B_PRED,    INTRA_FRAME,  0},
  {I8X8_PRED, INTRA_FRAME,  0},
189

John Koleszar's avatar
John Koleszar committed
190 191 192 193
  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME},
194

John Koleszar's avatar
John Koleszar committed
195 196 197
  {ZEROMV,    ALTREF_FRAME, LAST_FRAME},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME},
198

John Koleszar's avatar
John Koleszar committed
199 200 201
  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME},
202

John Koleszar's avatar
John Koleszar committed
203 204 205
  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME  },
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME},
206

John Koleszar's avatar
John Koleszar committed
207 208 209
  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME  },
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME}
John Koleszar's avatar
John Koleszar committed
210
};
211
#endif
John Koleszar's avatar
John Koleszar committed
212 213

static void fill_token_costs(
214
  unsigned int (*c)[COEF_BANDS][PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS],
215
  const vp9_prob(*p)[COEF_BANDS][PREV_COEF_CONTEXTS][ENTROPY_NODES],
John Koleszar's avatar
John Koleszar committed
216 217 218 219 220 221 222
  int block_type_counts) {
  int i, j, k;

  for (i = 0; i < block_type_counts; i++)
    for (j = 0; j < COEF_BANDS; j++)
      for (k = 0; k < PREV_COEF_CONTEXTS; k++) {
        if (k == 0 && ((j > 0 && i > 0) || (j > 1 && i == 0)))
223 224
          vp9_cost_tokens_skip((int *)(c[i][j][k]),
                               p[i][j][k],
225
                               vp9_coef_tree);
John Koleszar's avatar
John Koleszar committed
226
        else
227 228
          vp9_cost_tokens((int *)(c[i][j][k]),
                          p[i][j][k],
229
                          vp9_coef_tree);
John Koleszar's avatar
John Koleszar committed
230
      }
John Koleszar's avatar
John Koleszar committed
231 232
}

233

234 235 236 237
static int rd_iifactor[32] =  { 4, 4, 3, 2, 1, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0, };
John Koleszar's avatar
John Koleszar committed
238

239
// 3* dc_qlookup[Q]*dc_qlookup[Q];
240

241
/* values are now correlated to quantizer */
Paul Wilkins's avatar
Paul Wilkins committed
242 243 244
static int sad_per_bit16lut[QINDEX_RANGE];
static int sad_per_bit4lut[QINDEX_RANGE];

245
void vp9_init_me_luts() {
John Koleszar's avatar
John Koleszar committed
246 247 248 249 250 251 252
  int i;

  // Initialize the sad lut tables using a formulaic calculation for now
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < QINDEX_RANGE; i++) {
    sad_per_bit16lut[i] =
253 254
      (int)((0.0418 * vp9_convert_qindex_to_q(i)) + 2.4107);
    sad_per_bit4lut[i] = (int)((0.063 * vp9_convert_qindex_to_q(i)) + 2.742);
John Koleszar's avatar
John Koleszar committed
255
  }
Paul Wilkins's avatar
Paul Wilkins committed
256
}
John Koleszar's avatar
John Koleszar committed
257

258
static int compute_rd_mult(int qindex) {
John Koleszar's avatar
John Koleszar committed
259
  int q;
260

261
  q = vp9_dc_quant(qindex, 0);
John Koleszar's avatar
John Koleszar committed
262
  return (11 * q * q) >> 6;
263 264
}

265
void vp9_initialize_me_consts(VP9_COMP *cpi, int QIndex) {
John Koleszar's avatar
John Koleszar committed
266 267
  cpi->mb.sadperbit16 =  sad_per_bit16lut[QIndex];
  cpi->mb.sadperbit4  =  sad_per_bit4lut[QIndex];
John Koleszar's avatar
John Koleszar committed
268 269
}

270

271
void vp9_initialize_rd_consts(VP9_COMP *cpi, int QIndex) {
272
  int q, i;
John Koleszar's avatar
John Koleszar committed
273

274
  vp9_clear_system_state();  // __asm emms;
John Koleszar's avatar
John Koleszar committed
275

John Koleszar's avatar
John Koleszar committed
276 277 278 279 280
  // Further tests required to see if optimum is different
  // for key frames, golden frames and arf frames.
  // if (cpi->common.refresh_golden_frame ||
  //     cpi->common.refresh_alt_ref_frame)
  QIndex = (QIndex < 0) ? 0 : ((QIndex > MAXQ) ? MAXQ : QIndex);
281

John Koleszar's avatar
John Koleszar committed
282
  cpi->RDMULT = compute_rd_mult(QIndex);
John Koleszar's avatar
John Koleszar committed
283

John Koleszar's avatar
John Koleszar committed
284 285 286
  // Extend rate multiplier along side quantizer zbin increases
  if (cpi->zbin_over_quant  > 0) {
    double oq_factor;
287

John Koleszar's avatar
John Koleszar committed
288 289 290 291 292
    // Experimental code using the same basic equation as used for Q above
    // The units of cpi->zbin_over_quant are 1/128 of Q bin size
    oq_factor = 1.0 + ((double)0.0015625 * cpi->zbin_over_quant);
    cpi->RDMULT = (int)((double)cpi->RDMULT * oq_factor * oq_factor);
  }
John Koleszar's avatar
John Koleszar committed
293

John Koleszar's avatar
John Koleszar committed
294 295 296 297 298 299 300
  if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    if (cpi->twopass.next_iiratio > 31)
      cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
    else
      cpi->RDMULT +=
        (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
  }
John Koleszar's avatar
John Koleszar committed
301

John Koleszar's avatar
John Koleszar committed
302 303
  if (cpi->RDMULT < 7)
    cpi->RDMULT = 7;
304

John Koleszar's avatar
John Koleszar committed
305 306
  cpi->mb.errorperbit = (cpi->RDMULT / 110);
  cpi->mb.errorperbit += (cpi->mb.errorperbit == 0);
307

308
  vp9_set_speed_features(cpi);
John Koleszar's avatar
John Koleszar committed
309

310
  q = (int)pow(vp9_dc_quant(QIndex, 0) >> 2, 1.25);
John Koleszar's avatar
John Koleszar committed
311 312
  q = q << 2;
  cpi->RDMULT = cpi->RDMULT << 4;
Paul Wilkins's avatar
Paul Wilkins committed
313

John Koleszar's avatar
John Koleszar committed
314 315
  if (q < 8)
    q = 8;
316

John Koleszar's avatar
John Koleszar committed
317 318 319
  if (cpi->RDMULT > 1000) {
    cpi->RDDIV = 1;
    cpi->RDMULT /= 100;
John Koleszar's avatar
John Koleszar committed
320

John Koleszar's avatar
John Koleszar committed
321 322 323 324 325 326
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < INT_MAX) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q / 100;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
327

John Koleszar's avatar
John Koleszar committed
328
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
329
    }
John Koleszar's avatar
John Koleszar committed
330 331
  } else {
    cpi->RDDIV = 100;
John Koleszar's avatar
John Koleszar committed
332

John Koleszar's avatar
John Koleszar committed
333 334 335 336 337 338
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < (INT_MAX / q)) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
339

John Koleszar's avatar
John Koleszar committed
340
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
341
    }
John Koleszar's avatar
John Koleszar committed
342
  }
John Koleszar's avatar
John Koleszar committed
343

John Koleszar's avatar
John Koleszar committed
344
  fill_token_costs(
345
    cpi->mb.token_costs[TX_4X4],
346
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs,
John Koleszar's avatar
John Koleszar committed
347
    BLOCK_TYPES);
348 349
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_4X4],
350
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11])
351 352
    cpi->common.fc.hybrid_coef_probs,
    BLOCK_TYPES);
John Koleszar's avatar
John Koleszar committed
353

John Koleszar's avatar
John Koleszar committed
354
  fill_token_costs(
355
    cpi->mb.token_costs[TX_8X8],
356
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs_8x8,
John Koleszar's avatar
John Koleszar committed
357
    BLOCK_TYPES_8X8);
358 359
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_8X8],
360
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11])
361 362
    cpi->common.fc.hybrid_coef_probs_8x8,
    BLOCK_TYPES_8X8);
363

Daniel Kang's avatar
Daniel Kang committed
364
  fill_token_costs(
365
    cpi->mb.token_costs[TX_16X16],
366
    (const vp9_prob(*)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs_16x16,
Daniel Kang's avatar
Daniel Kang committed
367
    BLOCK_TYPES_16X16);
368 369
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_16X16],
370
    (const vp9_prob(*)[8][PREV_COEF_CONTEXTS][11])
371 372
    cpi->common.fc.hybrid_coef_probs_16x16,
    BLOCK_TYPES_16X16);
Daniel Kang's avatar
Daniel Kang committed
373

John Koleszar's avatar
John Koleszar committed
374 375
  /*rough estimate for costing*/
  cpi->common.kf_ymode_probs_index = cpi->common.base_qindex >> 4;
376
  vp9_init_mode_costs(cpi);
John Koleszar's avatar
John Koleszar committed
377

378 379
  if (cpi->common.frame_type != KEY_FRAME)
  {
380
    vp9_build_nmv_cost_table(
381 382 383 384 385 386
        cpi->mb.nmvjointcost,
        cpi->mb.e_mbd.allow_high_precision_mv ?
        cpi->mb.nmvcost_hp : cpi->mb.nmvcost,
        &cpi->common.fc.nmvc,
        cpi->mb.e_mbd.allow_high_precision_mv, 1, 1);
  }
John Koleszar's avatar
John Koleszar committed
387 388
}

389
int vp9_block_error_c(short *coeff, short *dqcoeff, int block_size) {
390
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
391

392
  for (i = 0; i < block_size; i++) {
John Koleszar's avatar
John Koleszar committed
393 394 395
    int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }
John Koleszar's avatar
John Koleszar committed
396

John Koleszar's avatar
John Koleszar committed
397
  return error;
John Koleszar's avatar
John Koleszar committed
398 399
}

400
int vp9_mbblock_error_c(MACROBLOCK *mb, int dc) {
John Koleszar's avatar
John Koleszar committed
401 402 403 404
  BLOCK  *be;
  BLOCKD *bd;
  int i, j;
  int berror, error = 0;
John Koleszar's avatar
John Koleszar committed
405

John Koleszar's avatar
John Koleszar committed
406 407 408
  for (i = 0; i < 16; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
John Koleszar's avatar
John Koleszar committed
409

John Koleszar's avatar
John Koleszar committed
410
    berror = 0;
John Koleszar's avatar
John Koleszar committed
411

John Koleszar's avatar
John Koleszar committed
412 413 414
    for (j = dc; j < 16; j++) {
      int this_diff = be->coeff[j] - bd->dqcoeff[j];
      berror += this_diff * this_diff;
John Koleszar's avatar
John Koleszar committed
415 416
    }

John Koleszar's avatar
John Koleszar committed
417 418 419 420
    error += berror;
  }

  return error;
John Koleszar's avatar
John Koleszar committed
421 422
}

423
int vp9_mbuverror_c(MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
424 425
  BLOCK  *be;
  BLOCKD *bd;
John Koleszar's avatar
John Koleszar committed
426

427
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
428

John Koleszar's avatar
John Koleszar committed
429 430 431
  for (i = 16; i < 24; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
John Koleszar's avatar
John Koleszar committed
432

433
    error += vp9_block_error_c(be->coeff, bd->dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
434
  }
John Koleszar's avatar
John Koleszar committed
435

John Koleszar's avatar
John Koleszar committed
436
  return error;
John Koleszar's avatar
John Koleszar committed
437 438
}

439
int vp9_uvsse(MACROBLOCK *x) {
John Koleszar's avatar
John Koleszar committed
440 441 442 443 444 445 446
  unsigned char *uptr, *vptr;
  unsigned char *upred_ptr = (*(x->block[16].base_src) + x->block[16].src);
  unsigned char *vpred_ptr = (*(x->block[20].base_src) + x->block[20].src);
  int uv_stride = x->block[16].src_stride;

  unsigned int sse1 = 0;
  unsigned int sse2 = 0;
447 448
  int mv_row = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.row;
  int mv_col = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.col;
John Koleszar's avatar
John Koleszar committed
449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469
  int offset;
  int pre_stride = x->e_mbd.block[16].pre_stride;

  if (mv_row < 0)
    mv_row -= 1;
  else
    mv_row += 1;

  if (mv_col < 0)
    mv_col -= 1;
  else
    mv_col += 1;

  mv_row /= 2;
  mv_col /= 2;

  offset = (mv_row >> 3) * pre_stride + (mv_col >> 3);
  uptr = x->e_mbd.pre.u_buffer + offset;
  vptr = x->e_mbd.pre.v_buffer + offset;

  if ((mv_row | mv_col) & 7) {
470
    vp9_sub_pixel_variance8x8(uptr, pre_stride, (mv_col & 7) << 1,
471
                              (mv_row & 7) << 1, upred_ptr, uv_stride, &sse2);
472
    vp9_sub_pixel_variance8x8(vptr, pre_stride, (mv_col & 7) << 1,
473
                              (mv_row & 7) << 1, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
474 475
    sse2 += sse1;
  } else {
476 477
    vp9_variance8x8(uptr, pre_stride, upred_ptr, uv_stride, &sse2);
    vp9_variance8x8(vptr, pre_stride, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
478 479 480
    sse2 += sse1;
  }
  return sse2;
John Koleszar's avatar
John Koleszar committed
481 482 483

}

484
static int cost_coeffs_2x2(MACROBLOCK *mb,
485
                           BLOCKD *b, PLANE_TYPE type,
486
                           ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l) {
487
  int c = (type == PLANE_TYPE_Y_NO_DC); /* start at coef 0, unless Y with Y2 */
488 489 490 491 492
  int eob = b->eob;
  int pt;    /* surrounding block/prev coef predictor */
  int cost = 0;
  short *qcoeff_ptr = b->qcoeff;

493
  VP9_COMBINEENTROPYCONTEXTS(pt, *a, *l);
494 495 496
  assert(eob <= 4);

  for (; c < eob; c++) {
497 498 499 500 501
    int v = qcoeff_ptr[vp9_default_zig_zag1d[c]];
    int t = vp9_dct_value_tokens_ptr[v].Token;
    cost += mb->token_costs[TX_8X8][type][vp9_coef_bands[c]][pt][t];
    cost += vp9_dct_value_cost_ptr[v];
    pt = vp9_prev_token_class[t];
502 503 504
  }

  if (c < 4)
505
    cost += mb->token_costs[TX_8X8][type][vp9_coef_bands[c]]
506 507 508 509 510 511 512
            [pt] [DCT_EOB_TOKEN];

  pt = (c != !type); // is eob first coefficient;
  *a = *l = pt;
  return cost;
}

513
static int cost_coeffs(MACROBLOCK *mb, BLOCKD *b, PLANE_TYPE type,
Daniel Kang's avatar
Daniel Kang committed
514
                       ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l,
515
                       int tx_size) {
Daniel Kang's avatar
Daniel Kang committed
516
  const int eob = b->eob;
517
  int c = (type == PLANE_TYPE_Y_NO_DC); /* start at coef 0, unless Y with Y2 */
518
  int cost = 0, default_eob, seg_eob;
Daniel Kang's avatar
Daniel Kang committed
519 520
  int pt;                     /* surrounding block/prev coef predictor */
  int const *scan, *band;
John Koleszar's avatar
John Koleszar committed
521
  short *qcoeff_ptr = b->qcoeff;
522 523 524
  MACROBLOCKD *xd = &mb->e_mbd;
  MB_MODE_INFO *mbmi = &mb->e_mbd.mode_info_context->mbmi;
  TX_TYPE tx_type = DCT_DCT;
Paul Wilkins's avatar
Paul Wilkins committed
525
  int segment_id = mbmi->segment_id;
526

527
  switch (tx_size) {
Daniel Kang's avatar
Daniel Kang committed
528
    case TX_4X4:
529 530
      scan = vp9_default_zig_zag1d;
      band = vp9_coef_bands;
Daniel Kang's avatar
Daniel Kang committed
531
      default_eob = 16;
Deb Mukherjee's avatar
Deb Mukherjee committed
532 533 534 535 536
      if (type == PLANE_TYPE_Y_WITH_DC) {
        tx_type = get_tx_type_4x4(xd, b);
        if (tx_type != DCT_DCT) {
          switch (tx_type) {
            case ADST_DCT:
537
              scan = vp9_row_scan;
Deb Mukherjee's avatar
Deb Mukherjee committed
538 539 540
              break;

            case DCT_ADST:
541
              scan = vp9_col_scan;
Deb Mukherjee's avatar
Deb Mukherjee committed
542 543 544
              break;

            default:
545
              scan = vp9_default_zig_zag1d;
Deb Mukherjee's avatar
Deb Mukherjee committed
546 547
              break;
          }
548
        }
Daniel Kang's avatar
Daniel Kang committed
549
      }
Deb Mukherjee's avatar
Deb Mukherjee committed
550

Daniel Kang's avatar
Daniel Kang committed
551 552
      break;
    case TX_8X8:
553 554
      scan = vp9_default_zig_zag1d_8x8;
      band = vp9_coef_bands_8x8;
Daniel Kang's avatar
Daniel Kang committed
555
      default_eob = 64;
Deb Mukherjee's avatar
Deb Mukherjee committed
556
      if (type == PLANE_TYPE_Y_WITH_DC) {
557
        BLOCKD *bb;
558
        int ib = (int)(b - xd->block);
559 560 561
        if (ib < 16) {
          ib = (ib & 8) + ((ib & 4) >> 1);
          bb = xd->block + ib;
Deb Mukherjee's avatar
Deb Mukherjee committed
562
          tx_type = get_tx_type_8x8(xd, bb);
563
        }
564
      }
Daniel Kang's avatar
Daniel Kang committed
565 566
      break;
    case TX_16X16:
567 568
      scan = vp9_default_zig_zag1d_16x16;
      band = vp9_coef_bands_16x16;
Daniel Kang's avatar
Daniel Kang committed
569
      default_eob = 256;
Deb Mukherjee's avatar
Deb Mukherjee committed
570 571 572
      if (type == PLANE_TYPE_Y_WITH_DC) {
        tx_type = get_tx_type_16x16(xd, b);
      }
Daniel Kang's avatar
Daniel Kang committed
573 574 575 576
      break;
    default:
      break;
  }
577 578
  if (vp9_segfeature_active(&mb->e_mbd, segment_id, SEG_LVL_EOB))
    seg_eob = vp9_get_segdata(&mb->e_mbd, segment_id, SEG_LVL_EOB);
579 580 581
  else
    seg_eob = default_eob;

582
  VP9_COMBINEENTROPYCONTEXTS(pt, *a, *l);
583

584 585 586
  if (tx_type != DCT_DCT) {
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
587
      int t = vp9_dct_value_tokens_ptr[v].Token;
588
      cost += mb->hybrid_token_costs[tx_size][type][band[c]][pt][t];
589 590
      cost += vp9_dct_value_cost_ptr[v];
      pt = vp9_prev_token_class[t];
591 592 593 594
    }
    if (c < seg_eob)
      cost += mb->hybrid_token_costs[tx_size][type][band[c]]
          [pt][DCT_EOB_TOKEN];
595
  } else {
596 597
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
598
      int t = vp9_dct_value_tokens_ptr[v].Token;
599
      cost += mb->token_costs[tx_size][type][band[c]][pt][t];
600 601
      cost += vp9_dct_value_cost_ptr[v];
      pt = vp9_prev_token_class[t];
602 603 604 605
    }
    if (c < seg_eob)
      cost += mb->token_costs[tx_size][type][band[c]]
          [pt][DCT_EOB_TOKEN];
606 607 608 609 610 611 612
  }

  pt = (c != !type); // is eob first coefficient;
  *a = *l = pt;
  return cost;
}

613
static int rdcost_mby_4x4(MACROBLOCK *mb, int backup) {
John Koleszar's avatar
John Koleszar committed
614 615
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
616
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
617 618 619
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta;
  ENTROPY_CONTEXT *tl;
620

621 622 623
  if (backup) {
    vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
624

625 626 627 628 629 630
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)xd->above_context;
    tl = (ENTROPY_CONTEXT *)xd->left_context;
  }
John Koleszar's avatar
John Koleszar committed
631

John Koleszar's avatar
John Koleszar committed
632
  for (b = 0; b < 16; b++)
Paul Wilkins's avatar
Paul Wilkins committed
633
    cost += cost_coeffs(mb, xd->block + b, PLANE_TYPE_Y_NO_DC,
634
                        ta + vp9_block2above[b], tl + vp9_block2left[b],
Daniel Kang's avatar
Daniel Kang committed
635
                        TX_4X4);
John Koleszar's avatar
John Koleszar committed
636

Paul Wilkins's avatar
Paul Wilkins committed
637
  cost += cost_coeffs(mb, xd->block + 24, PLANE_TYPE_Y2,
638
                      ta + vp9_block2above[24], tl + vp9_block2left[24],
Daniel Kang's avatar
Daniel Kang committed
639
                      TX_4X4);
John Koleszar's avatar
John Koleszar committed
640

John Koleszar's avatar
John Koleszar committed
641
  return cost;
John Koleszar's avatar
John Koleszar committed
642 643
}

644 645 646
static void macro_block_yrd_4x4(MACROBLOCK *mb,
                                int *Rate,
                                int *Distortion,
647
                                const VP9_ENCODER_RTCD *rtcd,
648
                                int *skippable, int backup) {
John Koleszar's avatar
John Koleszar committed
649
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
650
  MACROBLOCKD *const xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
651
  BLOCK   *const mb_y2 = mb->block + 24;
Paul Wilkins's avatar
Paul Wilkins committed
652
  BLOCKD *const x_y2  = xd->block + 24;
John Koleszar's avatar
John Koleszar committed
653 654 655
  short *Y2DCPtr = mb_y2->src_diff;
  BLOCK *beptr;
  int d;
656

John Koleszar's avatar
John Koleszar committed
657 658
  // Fdct and building the 2nd order block
  for (beptr = mb->block; beptr < mb->block + 16; beptr += 2) {
659
    mb->vp9_short_fdct8x4(beptr->src_diff, beptr->coeff, 32);
John Koleszar's avatar
John Koleszar committed
660 661 662 663 664 665 666 667 668
    *Y2DCPtr++ = beptr->coeff[0];
    *Y2DCPtr++ = beptr->coeff[16];
  }

  // 2nd order fdct
  mb->short_walsh4x4(mb_y2->src_diff, mb_y2->coeff, 8);

  // Quantization
  for (b = 0; b < 16; b++) {
669
    mb->quantize_b_4x4(&mb->block[b], &xd->block[b]);
John Koleszar's avatar
John Koleszar committed
670
  }
671

John Koleszar's avatar
John Koleszar committed
672
  // DC predication and Quantization of 2nd Order block
673
  mb->quantize_b_4x4(mb_y2, x_y2);
674

John Koleszar's avatar
John Koleszar committed
675
  // Distortion
676
  d = vp9_mbblock_error(mb, 1);
677

678
  d += vp9_block_error(mb_y2->coeff, x_y2->dqcoeff, 16);
679

John Koleszar's avatar
John Koleszar committed
680 681
  *Distortion = (d >> 2);
  // rate
682
  *Rate = rdcost_mby_4x4(mb, backup);
683
  *skippable = vp9_mby_is_skippable_4x4(&mb->e_mbd, 1);
684
}
John Koleszar's avatar
John Koleszar committed
685

686
static int rdcost_mby_8x8(MACROBLOCK *mb, int backup) {
John Koleszar's avatar
John Koleszar committed
687 688
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
689
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
690 691 692 693
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta;
  ENTROPY_CONTEXT *tl;

Ronald S. Bultje's avatar
Ronald S. Bultje committed
694 695 696
  if (backup) {
    vpx_memcpy(&t_above,xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
697

Ronald S. Bultje's avatar
Ronald S. Bultje committed
698 699 700 701 702 703
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)mb->e_mbd.above_context;
    tl = (ENTROPY_CONTEXT *)mb->e_mbd.left_context;
  }
John Koleszar's avatar
John Koleszar committed
704 705

  for (b = 0; b < 16; b += 4)
Paul Wilkins's avatar
Paul Wilkins committed
706
    cost += cost_coeffs(mb, xd->block + b, PLANE_TYPE_Y_NO_DC,
707
                        ta + vp9_block2above_8x8[b], tl + vp9_block2left_8x8[b],
Daniel Kang's avatar
Daniel Kang committed
708
                        TX_8X8);
John Koleszar's avatar
John Koleszar committed
709

Paul Wilkins's avatar
Paul Wilkins committed
710
  cost += cost_coeffs_2x2(mb, xd->block + 24, PLANE_TYPE_Y2,
711
                          ta + vp9_block2above[24], tl + vp9_block2left[24]);
John Koleszar's avatar
John Koleszar committed
712
  return cost;
713 714
}

John Koleszar's avatar
John Koleszar committed
715 716 717
static void macro_block_yrd_8x8(MACROBLOCK *mb,
                                int *Rate,
                                int *Distortion,
718
                                const VP9_ENCODER_RTCD *rtcd,
719
                                int *skippable, int backup) {
Paul Wilkins's avatar
Paul Wilkins committed
720
  MACROBLOCKD *const xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
721
  BLOCK   *const mb_y2 = mb->block + 24;
Paul Wilkins's avatar
Paul Wilkins committed
722
  BLOCKD *const x_y2  = xd->block + 24;
John Koleszar's avatar
John Koleszar committed
723 724
  int d;

725 726
  vp9_transform_mby_8x8(mb);
  vp9_quantize_mby_8x8(mb);
John Koleszar's avatar
John Koleszar committed
727 728 729 730 731 732

  /* remove 1st order dc to properly combine 1st/2nd order distortion */
  mb->coeff[0] = 0;
  mb->coeff[64] = 0;
  mb->coeff[128] = 0;
  mb->coeff[192] = 0;
Paul Wilkins's avatar
Paul Wilkins committed
733 734 735 736
  xd->dqcoeff[0] = 0;
  xd->dqcoeff[64] = 0;
  xd->dqcoeff[128] = 0;
  xd->dqcoeff[192] = 0;
John Koleszar's avatar
John Koleszar committed
737

738 739
  d = vp9_mbblock_error(mb, 0);
  d += vp9_block_error(mb_y2->coeff, x_y2->dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
740 741 742

  *Distortion = (d >> 2);
  // rate
743
  *Rate = rdcost_mby_8x8(mb, backup);
744
  *skippable = vp9_mby_is_skippable_8x8(&mb->e_mbd, 1);
745
}
746

747
static int rdcost_mby_16x16(MACROBLOCK *mb, int backup) {
Daniel Kang's avatar
Daniel Kang committed
748
  int cost;
Paul Wilkins's avatar
Paul Wilkins committed
749
  MACROBLOCKD *xd = &mb->e_mbd;
Daniel Kang's avatar
Daniel Kang committed
750 751 752
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta, *tl;

753 754 755
  if (backup) {
    vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
Daniel Kang's avatar
Daniel Kang committed
756

757 758 759 760 761 762
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)xd->above_context;
    tl = (ENTROPY_CONTEXT *)xd->left_context;
  }
Daniel Kang's avatar
Daniel Kang committed
763

Paul Wilkins's avatar
Paul Wilkins committed
764
  cost = cost_coeffs(mb, xd->block, PLANE_TYPE_Y_WITH_DC, ta, tl, TX_16X16);
Daniel Kang's avatar
Daniel Kang committed
765 766
  return cost;
}
767

Daniel Kang's avatar
Daniel Kang committed
768
static void macro_block_yrd_16x16(MACROBLOCK *mb, int *Rate, int *Distortion,
769 770
                                  const VP9_ENCODER_RTCD *rtcd, int *skippable,
                                  int backup) {
Daniel Kang's avatar
Daniel Kang committed
771
  int d;
Deb Mukherjee's avatar
Deb Mukherjee committed
772 773 774 775
  MACROBLOCKD *xd = &mb->e_mbd;
  BLOCKD *b  = &mb->e_mbd.block[0];
  BLOCK  *be = &mb->block[0];
  TX_TYPE tx_type;
Daniel Kang's avatar
Daniel Kang committed
776

Deb Mukherjee's avatar
Deb Mukherjee committed
777 778
  tx_type = get_tx_type_16x16(xd, b);
  if (tx_type != DCT_DCT) {
779
    vp9_fht(be->src_diff, 32, be->coeff, tx_type, 16);
780
  } else
781
    vp9_transform_mby_16x16(mb);
782

783
  vp9_quantize_mby_16x16(mb);
784 785 786
  // TODO(jingning) is it possible to quickly determine whether to force
  //                trailing coefficients to be zero, instead of running trellis
  //                optimization in the rate-distortion optimization loop?
787
  if (mb->e_mbd.mode_info_context->mbmi.mode < I8X8_PRED)
788
    vp9_optimize_mby_16x16(mb, rtcd);
789

790
  d = vp9_mbblock_error(mb, 0);
Daniel Kang's avatar
Daniel Kang committed
791 792 793

  *Distortion = (d >> 2);
  // rate
794
  *Rate = rdcost_mby_16x16(mb, backup);
795
  *skippable = vp9_mby_is_skippable_16x16(&mb->e_mbd);
Daniel Kang's avatar
Daniel Kang committed
796 797
}

798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830
static void choose_txfm_size_from_rd(VP9_COMP *cpi, MACROBLOCK *x,
                                     int r[2][TX_SIZE_MAX], int *rate,
                                     int d[TX_SIZE_MAX], int *distortion,
                                     int s[TX_SIZE_MAX], int *skip,
                                     int64_t txfm_cache[NB_TXFM_MODES]) {
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
  vp9_prob skip_prob = cm->mb_no_coeff_skip ?
                       vp9_get_pred_prob(cm, xd, PRED_MBSKIP) : 128;
  int64_t rd[2][TX_SIZE_MAX];
  int n;

  r[1][TX_16X16] = r[0][TX_16X16] + vp9_cost_one(cm->prob_tx[0]) +
                   vp9_cost_one(cm->prob_tx[1]);
  r[1][TX_8X8]   = r[0][TX_8X8] + vp9_cost_one(cm->prob_tx[0]) +
                   vp9_cost_zero(cm->prob_tx[1]);
  r[1][TX_4X4]   = r[0][TX_4X4] + vp9_cost_zero(cm->prob_tx[0]);

  if (cm->mb_no_coeff_skip) {
    int s0, s1;

    assert(skip_prob > 0);
    s0 = vp9_cost_bit(skip_prob, 0);
    s1 = vp9_cost_bit(skip_prob, 1);

    for (n = TX_4X4; n <= TX_16X16; n++) {
      if (s[n]) {
        rd[0][n] = rd[1][n] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
      } else {
        rd[0][n] = RDCOST(x->rdmult, x->rddiv, r[0][n] + s0, d[n]);
        rd[1][n] = RDCOST(x->rdmult, x->rddiv, r[1][n] + s0, d[n]);
      }
831 832
    }
  } else {
833 834 835
    for (n = TX_4X4; n <= TX_16X16; n++) {
      rd[0][n] = RDCOST(x->rdmult, x->rddiv, r[0][n], d[n]);
      rd[1][n] = RDCOST(x->rdmult, x->rddiv, r[1][n], d[n]);
836 837 838
    }
  }

839 840 841
  if ( cm->txfm_mode == ALLOW_16X16 ||
      (cm->txfm_mode == TX_MODE_SELECT &&
       rd[1][TX_16X16] < rd[1][TX_8X8] && rd[1][TX_16X16] < rd[1][TX_4X4])) {
842
    mbmi->txfm_size = TX_16X16;
843 844
  } else if (cm->txfm_mode == ALLOW_8X8 ||
           (cm->txfm_mode == TX_MODE_SELECT && rd[1][TX_8X8] < rd[1][TX_4X4])) {
845 846
    mbmi->txfm_size = TX_8X8;
  } else {
847 848
    assert(cm->txfm_mode == ONLY_4X4 ||
          (cm->txfm_mode == TX_MODE_SELECT && rd[1][TX_4X4] <= rd[1][TX_8X8]));
849 850 851
    mbmi->txfm_size = TX_4X4;
  }

852 853 854 855 856 857 858 859 860
  *distortion = d[mbmi->txfm_size];
  *rate       = r[cm->txfm_mode == TX_MODE_SELECT][mbmi->txfm_size];
  *skip       = s[mbmi->txfm_size];

  txfm_cache[ONLY_4X4] = rd[0][TX_4X4];
  txfm_cache[ALLOW_8X8] = rd[0][TX_8X8];
  txfm_cache[ALLOW_16X16] = rd[0][TX_16X16];
  if (rd[1][TX_16X16] < rd[1][TX_8X8] && rd[1][TX_16X16] < rd[1][TX_4X4])
    txfm_cache[TX_MODE_SELECT] = rd[1][TX_16X16];
861
  else
862 863 864 865 866 867 868 869 870 871 872 873 874
    txfm_cache[TX_MODE_SELECT] = rd[1][TX_4X4] < rd[1][TX_8X8] ?
                                 rd[1][TX_4X4] : rd[1][TX_8X8];
}

static void macro_block_yrd(VP9_COMP *cpi, MACROBLOCK *x, int *rate,
                            int *distortion, int *skippable,
                            int64_t txfm_cache[NB_TXFM_MODES]) {
  VP9_COMMON *cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
  int r[2][TX_SIZE_MAX], d[TX_SIZE_MAX], s[TX_SIZE_MAX];

  vp9_subtract_mby(x->src_diff, *(x->block[0].base_src), xd->predictor,
                   x->block[0].src_stride);
875

876 877 878 879 880 881 882 883 884
  macro_block_yrd_16x16(x, &r[0][TX_16X16], &d[TX_16X16],
                        IF_RTCD(&cpi->rtcd), &s[TX_16X16], 1);
  macro_block_yrd_8x8(x, &r[0][TX_8X8], &d[TX_8X8],
                      IF_RTCD(&cpi->rtcd), &s[TX_8X8], 1);
  macro_block_yrd_4x4(x, &r[0][TX_4X4], &d[TX_4X4],
                      IF_RTCD(&cpi->rtcd), &s[TX_4X4], 1);

  choose_txfm_size_from_rd(cpi, x, r, rate, d, distortion, s, skippable,
                           txfm_cache);
885 886
}

John Koleszar's avatar
John Koleszar committed
887 888 889 890 891 892 893
static void copy_predictor(unsigned char *dst, const unsigned char *predictor) {
  const unsigned int *p = (const unsigned int *)predictor;
  unsigned int *d = (unsigned int *)dst;
  d[0] = p[0];
  d[4] = p[4];
  d[8] = p[8];
  d[12] = p[12];
894
}
Yaowu Xu's avatar
Yaowu Xu committed
895

Ronald S. Bultje's avatar
Ronald S. Bultje committed
896
#if CONFIG_SUPERBLOCKS
897 898 899 900
static void super_block_yrd(VP9_COMP *cpi,
                            MACROBLOCK *x, int *rate, int *distortion,
                            const VP9_ENCODER_RTCD *rtcd, int *skip,
                            int64_t txfm_cache[NB_TXFM_MODES]) {
Ronald S. Bultje's avatar
Ronald S. Bultje committed
901
  MACROBLOCKD *const xd = &x->e_mbd;
902
  int r[2][TX_SIZE_MAX], d[TX_SIZE_MAX], s[TX_SIZE_MAX], n;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
903 904
  const uint8_t *src = x->src.y_buffer, *dst = xd->dst.y_buffer;
  int src_y_stride = x->src.y_stride, dst_y_stride = xd->dst.y_stride;
905 906 907 908 909 910 911 912 913 914
  ENTROPY_CONTEXT_PLANES t_above[3][2], *orig_above = xd->above_context;
  ENTROPY_CONTEXT_PLANES t_left[3][2], *orig_left = xd->left_context;

  for (n = TX_4X4; n <= TX_16X16; n++) {
    vpx_memcpy(t_above[n], xd->above_context, sizeof(t_above[n]));
    vpx_memcpy(t_left[n], xd->left_context, sizeof(t_left[n]));
    r[0][n] = 0;
    d[n] = 0;
    s[n] = 1;
  }
Ronald S. Bultje's avatar
Ronald S. Bultje committed
915 916 917

  for (n = 0; n < 4; n++) {
    int x_idx = n & 1, y_idx = n >> 1;
918
    int r_tmp, d_tmp, s_tmp;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
919

920
    vp9_subtract_mby_s_c(x->src_diff,