rdopt.c 164 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5 6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9 10 11 12 13 14 15
 */


#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
16
#include "vp9/common/pragmas.h"
John Koleszar's avatar
John Koleszar committed
17 18 19 20 21 22

#include "tokenize.h"
#include "treewriter.h"
#include "onyx_int.h"
#include "modecosts.h"
#include "encodeintra.h"
23 24 25 26 27 28
#include "vp9/common/entropymode.h"
#include "vp9/common/reconinter.h"
#include "vp9/common/reconintra.h"
#include "vp9/common/reconintra4x4.h"
#include "vp9/common/findnearmv.h"
#include "vp9/common/quant_common.h"
John Koleszar's avatar
John Koleszar committed
29 30
#include "encodemb.h"
#include "quantize.h"
31
#include "vp9/common/idct.h"
John Koleszar's avatar
John Koleszar committed
32 33
#include "variance.h"
#include "mcomp.h"
Yunqing Wang's avatar
Yunqing Wang committed
34
#include "rdopt.h"
Paul Wilkins's avatar
Paul Wilkins committed
35
#include "ratectrl.h"
John Koleszar's avatar
John Koleszar committed
36
#include "vpx_mem/vpx_mem.h"
37 38
#include "vp9/common/systemdependent.h"
#include "vp9/encoder/encodemv.h"
John Koleszar's avatar
John Koleszar committed
39

40 41 42
#include "vp9/common/seg_common.h"
#include "vp9/common/pred_common.h"
#include "vp9/common/entropy.h"
Jim Bankoski's avatar
Jim Bankoski committed
43
#include "vpx_rtcd.h"
44
#if CONFIG_NEWBESTREFMV
45
#include "vp9/common/mvref_common.h"
Paul Wilkins's avatar
Paul Wilkins committed
46 47
#endif

John Koleszar's avatar
John Koleszar committed
48 49 50 51 52 53 54 55
#if CONFIG_RUNTIME_CPU_DETECT
#define IF_RTCD(x)  (x)
#else
#define IF_RTCD(x)  NULL
#endif

#define MAXF(a,b)            (((a) > (b)) ? (a) : (b))

56 57
#define INVALID_MV 0x80008000

58 59 60
/* Factor to weigh the rate for switchable interp filters */
#define SWITCHABLE_INTERP_RATE_FACTOR 1

John Koleszar's avatar
John Koleszar committed
61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78
static const int auto_speed_thresh[17] = {
  1000,
  200,
  150,
  130,
  150,
  125,
  120,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  105
John Koleszar's avatar
John Koleszar committed
79 80
};

81
#if CONFIG_PRED_FILTER
82
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
John Koleszar's avatar
John Koleszar committed
83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151
  {ZEROMV,    LAST_FRAME,   0,  0},
  {ZEROMV,    LAST_FRAME,   0,  1},
  {DC_PRED,   INTRA_FRAME,  0,  0},

  {NEARESTMV, LAST_FRAME,   0,  0},
  {NEARESTMV, LAST_FRAME,   0,  1},
  {NEARMV,    LAST_FRAME,   0,  0},
  {NEARMV,    LAST_FRAME,   0,  1},

  {ZEROMV,    GOLDEN_FRAME, 0,  0},
  {ZEROMV,    GOLDEN_FRAME, 0,  1},
  {NEARESTMV, GOLDEN_FRAME, 0,  0},
  {NEARESTMV, GOLDEN_FRAME, 0,  1},

  {ZEROMV,    ALTREF_FRAME, 0,  0},
  {ZEROMV,    ALTREF_FRAME, 0,  1},
  {NEARESTMV, ALTREF_FRAME, 0,  0},
  {NEARESTMV, ALTREF_FRAME, 0,  1},

  {NEARMV,    GOLDEN_FRAME, 0,  0},
  {NEARMV,    GOLDEN_FRAME, 0,  1},
  {NEARMV,    ALTREF_FRAME, 0,  0},
  {NEARMV,    ALTREF_FRAME, 0,  1},

  {V_PRED,    INTRA_FRAME,  0,  0},
  {H_PRED,    INTRA_FRAME,  0,  0},
  {D45_PRED,  INTRA_FRAME,  0,  0},
  {D135_PRED, INTRA_FRAME,  0,  0},
  {D117_PRED, INTRA_FRAME,  0,  0},
  {D153_PRED, INTRA_FRAME,  0,  0},
  {D27_PRED,  INTRA_FRAME,  0,  0},
  {D63_PRED,  INTRA_FRAME,  0,  0},

  {TM_PRED,   INTRA_FRAME,  0,  0},

  {NEWMV,     LAST_FRAME,   0,  0},
  {NEWMV,     LAST_FRAME,   0,  1},
  {NEWMV,     GOLDEN_FRAME, 0,  0},
  {NEWMV,     GOLDEN_FRAME, 0,  1},
  {NEWMV,     ALTREF_FRAME, 0,  0},
  {NEWMV,     ALTREF_FRAME, 0,  1},

  {SPLITMV,   LAST_FRAME,   0,  0},
  {SPLITMV,   GOLDEN_FRAME, 0,  0},
  {SPLITMV,   ALTREF_FRAME, 0,  0},

  {B_PRED,    INTRA_FRAME,  0,  0},
  {I8X8_PRED, INTRA_FRAME,  0,  0},

  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME, 0},

  {ZEROMV,    ALTREF_FRAME, LAST_FRAME,   0},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME,   0},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME,   0},

  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME, 0},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME, 0},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME, 0},

  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME,   0},
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME, 0},

  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME, 0},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME,   0},
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME, 0}
John Koleszar's avatar
John Koleszar committed
152
};
153
#else
154
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
John Koleszar's avatar
John Koleszar committed
155 156
  {ZEROMV,    LAST_FRAME,   0},
  {DC_PRED,   INTRA_FRAME,  0},
John Koleszar's avatar
John Koleszar committed
157

John Koleszar's avatar
John Koleszar committed
158 159
  {NEARESTMV, LAST_FRAME,   0},
  {NEARMV,    LAST_FRAME,   0},
John Koleszar's avatar
John Koleszar committed
160

John Koleszar's avatar
John Koleszar committed
161 162
  {ZEROMV,    GOLDEN_FRAME, 0},
  {NEARESTMV, GOLDEN_FRAME, 0},
John Koleszar's avatar
John Koleszar committed
163

John Koleszar's avatar
John Koleszar committed
164 165
  {ZEROMV,    ALTREF_FRAME, 0},
  {NEARESTMV, ALTREF_FRAME, 0},
John Koleszar's avatar
John Koleszar committed
166

John Koleszar's avatar
John Koleszar committed
167 168
  {NEARMV,    GOLDEN_FRAME, 0},
  {NEARMV,    ALTREF_FRAME, 0},
John Koleszar's avatar
John Koleszar committed
169

John Koleszar's avatar
John Koleszar committed
170 171 172 173 174 175 176 177
  {V_PRED,    INTRA_FRAME,  0},
  {H_PRED,    INTRA_FRAME,  0},
  {D45_PRED,  INTRA_FRAME,  0},
  {D135_PRED, INTRA_FRAME,  0},
  {D117_PRED, INTRA_FRAME,  0},
  {D153_PRED, INTRA_FRAME,  0},
  {D27_PRED,  INTRA_FRAME,  0},
  {D63_PRED,  INTRA_FRAME,  0},
John Koleszar's avatar
John Koleszar committed
178

John Koleszar's avatar
John Koleszar committed
179
  {TM_PRED,   INTRA_FRAME,  0},
John Koleszar's avatar
John Koleszar committed
180

John Koleszar's avatar
John Koleszar committed
181 182 183
  {NEWMV,     LAST_FRAME,   0},
  {NEWMV,     GOLDEN_FRAME, 0},
  {NEWMV,     ALTREF_FRAME, 0},
John Koleszar's avatar
John Koleszar committed
184

John Koleszar's avatar
John Koleszar committed
185 186 187
  {SPLITMV,   LAST_FRAME,   0},
  {SPLITMV,   GOLDEN_FRAME, 0},
  {SPLITMV,   ALTREF_FRAME, 0},
188

John Koleszar's avatar
John Koleszar committed
189 190
  {B_PRED,    INTRA_FRAME,  0},
  {I8X8_PRED, INTRA_FRAME,  0},
191

John Koleszar's avatar
John Koleszar committed
192 193 194 195
  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME},
196

John Koleszar's avatar
John Koleszar committed
197 198 199
  {ZEROMV,    ALTREF_FRAME, LAST_FRAME},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME},
200

John Koleszar's avatar
John Koleszar committed
201 202 203
  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME},
204

John Koleszar's avatar
John Koleszar committed
205 206 207
  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME  },
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME},
208

John Koleszar's avatar
John Koleszar committed
209 210 211
  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME  },
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME}
John Koleszar's avatar
John Koleszar committed
212
};
213
#endif
John Koleszar's avatar
John Koleszar committed
214 215

static void fill_token_costs(
216
  unsigned int (*c)[COEF_BANDS][PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS],
217
  const vp9_prob(*p)[COEF_BANDS][PREV_COEF_CONTEXTS][ENTROPY_NODES],
John Koleszar's avatar
John Koleszar committed
218 219 220 221 222 223 224
  int block_type_counts) {
  int i, j, k;

  for (i = 0; i < block_type_counts; i++)
    for (j = 0; j < COEF_BANDS; j++)
      for (k = 0; k < PREV_COEF_CONTEXTS; k++) {
        if (k == 0 && ((j > 0 && i > 0) || (j > 1 && i == 0)))
225 226
          vp9_cost_tokens_skip((int *)(c[i][j][k]),
                               p[i][j][k],
227
                               vp9_coef_tree);
John Koleszar's avatar
John Koleszar committed
228
        else
229 230
          vp9_cost_tokens((int *)(c[i][j][k]),
                          p[i][j][k],
231
                          vp9_coef_tree);
John Koleszar's avatar
John Koleszar committed
232
      }
John Koleszar's avatar
John Koleszar committed
233 234
}

235

236 237 238 239
static int rd_iifactor[32] =  { 4, 4, 3, 2, 1, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0, };
John Koleszar's avatar
John Koleszar committed
240

241
// 3* dc_qlookup[Q]*dc_qlookup[Q];
242

243
/* values are now correlated to quantizer */
Paul Wilkins's avatar
Paul Wilkins committed
244 245 246
static int sad_per_bit16lut[QINDEX_RANGE];
static int sad_per_bit4lut[QINDEX_RANGE];

247
void vp9_init_me_luts() {
John Koleszar's avatar
John Koleszar committed
248 249 250 251 252 253 254
  int i;

  // Initialize the sad lut tables using a formulaic calculation for now
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < QINDEX_RANGE; i++) {
    sad_per_bit16lut[i] =
255 256
      (int)((0.0418 * vp9_convert_qindex_to_q(i)) + 2.4107);
    sad_per_bit4lut[i] = (int)((0.063 * vp9_convert_qindex_to_q(i)) + 2.742);
John Koleszar's avatar
John Koleszar committed
257
  }
Paul Wilkins's avatar
Paul Wilkins committed
258
}
John Koleszar's avatar
John Koleszar committed
259

260
static int compute_rd_mult(int qindex) {
John Koleszar's avatar
John Koleszar committed
261
  int q;
262

263
  q = vp9_dc_quant(qindex, 0);
John Koleszar's avatar
John Koleszar committed
264
  return (11 * q * q) >> 6;
265 266
}

267
void vp9_initialize_me_consts(VP9_COMP *cpi, int QIndex) {
John Koleszar's avatar
John Koleszar committed
268 269
  cpi->mb.sadperbit16 =  sad_per_bit16lut[QIndex];
  cpi->mb.sadperbit4  =  sad_per_bit4lut[QIndex];
John Koleszar's avatar
John Koleszar committed
270 271
}

272

273
void vp9_initialize_rd_consts(VP9_COMP *cpi, int QIndex) {
274
  int q, i;
John Koleszar's avatar
John Koleszar committed
275

276
  vp9_clear_system_state();  // __asm emms;
John Koleszar's avatar
John Koleszar committed
277

John Koleszar's avatar
John Koleszar committed
278 279 280 281 282
  // Further tests required to see if optimum is different
  // for key frames, golden frames and arf frames.
  // if (cpi->common.refresh_golden_frame ||
  //     cpi->common.refresh_alt_ref_frame)
  QIndex = (QIndex < 0) ? 0 : ((QIndex > MAXQ) ? MAXQ : QIndex);
283

John Koleszar's avatar
John Koleszar committed
284
  cpi->RDMULT = compute_rd_mult(QIndex);
John Koleszar's avatar
John Koleszar committed
285

John Koleszar's avatar
John Koleszar committed
286 287 288
  // Extend rate multiplier along side quantizer zbin increases
  if (cpi->zbin_over_quant  > 0) {
    double oq_factor;
289

John Koleszar's avatar
John Koleszar committed
290 291 292 293 294
    // Experimental code using the same basic equation as used for Q above
    // The units of cpi->zbin_over_quant are 1/128 of Q bin size
    oq_factor = 1.0 + ((double)0.0015625 * cpi->zbin_over_quant);
    cpi->RDMULT = (int)((double)cpi->RDMULT * oq_factor * oq_factor);
  }
John Koleszar's avatar
John Koleszar committed
295

John Koleszar's avatar
John Koleszar committed
296 297 298 299 300 301 302
  if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    if (cpi->twopass.next_iiratio > 31)
      cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
    else
      cpi->RDMULT +=
        (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
  }
John Koleszar's avatar
John Koleszar committed
303

John Koleszar's avatar
John Koleszar committed
304 305
  if (cpi->RDMULT < 7)
    cpi->RDMULT = 7;
306

John Koleszar's avatar
John Koleszar committed
307 308
  cpi->mb.errorperbit = (cpi->RDMULT / 110);
  cpi->mb.errorperbit += (cpi->mb.errorperbit == 0);
309

310
  vp9_set_speed_features(cpi);
John Koleszar's avatar
John Koleszar committed
311

312
  q = (int)pow(vp9_dc_quant(QIndex, 0) >> 2, 1.25);
John Koleszar's avatar
John Koleszar committed
313 314
  q = q << 2;
  cpi->RDMULT = cpi->RDMULT << 4;
Paul Wilkins's avatar
Paul Wilkins committed
315

John Koleszar's avatar
John Koleszar committed
316 317
  if (q < 8)
    q = 8;
318

John Koleszar's avatar
John Koleszar committed
319 320 321
  if (cpi->RDMULT > 1000) {
    cpi->RDDIV = 1;
    cpi->RDMULT /= 100;
John Koleszar's avatar
John Koleszar committed
322

John Koleszar's avatar
John Koleszar committed
323 324 325 326 327 328
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < INT_MAX) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q / 100;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
329

John Koleszar's avatar
John Koleszar committed
330
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
331
    }
John Koleszar's avatar
John Koleszar committed
332 333
  } else {
    cpi->RDDIV = 100;
John Koleszar's avatar
John Koleszar committed
334

John Koleszar's avatar
John Koleszar committed
335 336 337 338 339 340
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < (INT_MAX / q)) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
341

John Koleszar's avatar
John Koleszar committed
342
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
343
    }
John Koleszar's avatar
John Koleszar committed
344
  }
John Koleszar's avatar
John Koleszar committed
345

John Koleszar's avatar
John Koleszar committed
346
  fill_token_costs(
347
    cpi->mb.token_costs[TX_4X4],
348
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs,
John Koleszar's avatar
John Koleszar committed
349
    BLOCK_TYPES);
350 351
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_4X4],
352
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11])
353 354
    cpi->common.fc.hybrid_coef_probs,
    BLOCK_TYPES);
John Koleszar's avatar
John Koleszar committed
355

John Koleszar's avatar
John Koleszar committed
356
  fill_token_costs(
357
    cpi->mb.token_costs[TX_8X8],
358
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs_8x8,
John Koleszar's avatar
John Koleszar committed
359
    BLOCK_TYPES_8X8);
360 361
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_8X8],
362
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11])
363 364
    cpi->common.fc.hybrid_coef_probs_8x8,
    BLOCK_TYPES_8X8);
365

Daniel Kang's avatar
Daniel Kang committed
366
  fill_token_costs(
367
    cpi->mb.token_costs[TX_16X16],
368
    (const vp9_prob(*)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs_16x16,
Daniel Kang's avatar
Daniel Kang committed
369
    BLOCK_TYPES_16X16);
370 371
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_16X16],
372
    (const vp9_prob(*)[8][PREV_COEF_CONTEXTS][11])
373 374
    cpi->common.fc.hybrid_coef_probs_16x16,
    BLOCK_TYPES_16X16);
Daniel Kang's avatar
Daniel Kang committed
375

John Koleszar's avatar
John Koleszar committed
376 377
  /*rough estimate for costing*/
  cpi->common.kf_ymode_probs_index = cpi->common.base_qindex >> 4;
378
  vp9_init_mode_costs(cpi);
John Koleszar's avatar
John Koleszar committed
379

380 381
  if (cpi->common.frame_type != KEY_FRAME)
  {
382
    vp9_build_nmv_cost_table(
383 384 385 386 387 388
        cpi->mb.nmvjointcost,
        cpi->mb.e_mbd.allow_high_precision_mv ?
        cpi->mb.nmvcost_hp : cpi->mb.nmvcost,
        &cpi->common.fc.nmvc,
        cpi->mb.e_mbd.allow_high_precision_mv, 1, 1);
  }
John Koleszar's avatar
John Koleszar committed
389 390
}

391
int vp9_block_error_c(short *coeff, short *dqcoeff, int block_size) {
392
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
393

394
  for (i = 0; i < block_size; i++) {
John Koleszar's avatar
John Koleszar committed
395 396 397
    int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }
John Koleszar's avatar
John Koleszar committed
398

John Koleszar's avatar
John Koleszar committed
399
  return error;
John Koleszar's avatar
John Koleszar committed
400 401
}

402
int vp9_mbblock_error_c(MACROBLOCK *mb, int dc) {
John Koleszar's avatar
John Koleszar committed
403 404 405 406
  BLOCK  *be;
  BLOCKD *bd;
  int i, j;
  int berror, error = 0;
John Koleszar's avatar
John Koleszar committed
407

John Koleszar's avatar
John Koleszar committed
408 409 410
  for (i = 0; i < 16; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
John Koleszar's avatar
John Koleszar committed
411

John Koleszar's avatar
John Koleszar committed
412
    berror = 0;
John Koleszar's avatar
John Koleszar committed
413

John Koleszar's avatar
John Koleszar committed
414 415 416
    for (j = dc; j < 16; j++) {
      int this_diff = be->coeff[j] - bd->dqcoeff[j];
      berror += this_diff * this_diff;
John Koleszar's avatar
John Koleszar committed
417 418
    }

John Koleszar's avatar
John Koleszar committed
419 420 421 422
    error += berror;
  }

  return error;
John Koleszar's avatar
John Koleszar committed
423 424
}

425
int vp9_mbuverror_c(MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
426 427
  BLOCK  *be;
  BLOCKD *bd;
John Koleszar's avatar
John Koleszar committed
428

429
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
430

John Koleszar's avatar
John Koleszar committed
431 432 433
  for (i = 16; i < 24; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
John Koleszar's avatar
John Koleszar committed
434

435
    error += vp9_block_error_c(be->coeff, bd->dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
436
  }
John Koleszar's avatar
John Koleszar committed
437

John Koleszar's avatar
John Koleszar committed
438
  return error;
John Koleszar's avatar
John Koleszar committed
439 440
}

441
int vp9_uvsse(MACROBLOCK *x) {
John Koleszar's avatar
John Koleszar committed
442 443 444 445 446 447 448
  unsigned char *uptr, *vptr;
  unsigned char *upred_ptr = (*(x->block[16].base_src) + x->block[16].src);
  unsigned char *vpred_ptr = (*(x->block[20].base_src) + x->block[20].src);
  int uv_stride = x->block[16].src_stride;

  unsigned int sse1 = 0;
  unsigned int sse2 = 0;
449 450
  int mv_row = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.row;
  int mv_col = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.col;
John Koleszar's avatar
John Koleszar committed
451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471
  int offset;
  int pre_stride = x->e_mbd.block[16].pre_stride;

  if (mv_row < 0)
    mv_row -= 1;
  else
    mv_row += 1;

  if (mv_col < 0)
    mv_col -= 1;
  else
    mv_col += 1;

  mv_row /= 2;
  mv_col /= 2;

  offset = (mv_row >> 3) * pre_stride + (mv_col >> 3);
  uptr = x->e_mbd.pre.u_buffer + offset;
  vptr = x->e_mbd.pre.v_buffer + offset;

  if ((mv_row | mv_col) & 7) {
472
    vp9_sub_pixel_variance8x8(uptr, pre_stride, (mv_col & 7) << 1,
473
                              (mv_row & 7) << 1, upred_ptr, uv_stride, &sse2);
474
    vp9_sub_pixel_variance8x8(vptr, pre_stride, (mv_col & 7) << 1,
475
                              (mv_row & 7) << 1, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
476 477
    sse2 += sse1;
  } else {
478 479
    vp9_variance8x8(uptr, pre_stride, upred_ptr, uv_stride, &sse2);
    vp9_variance8x8(vptr, pre_stride, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
480 481 482
    sse2 += sse1;
  }
  return sse2;
John Koleszar's avatar
John Koleszar committed
483 484 485

}

486
static int cost_coeffs_2x2(MACROBLOCK *mb,
487
                           BLOCKD *b, PLANE_TYPE type,
488
                           ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l) {
489
  int c = (type == PLANE_TYPE_Y_NO_DC); /* start at coef 0, unless Y with Y2 */
490 491 492 493 494
  int eob = b->eob;
  int pt;    /* surrounding block/prev coef predictor */
  int cost = 0;
  short *qcoeff_ptr = b->qcoeff;

495
  VP9_COMBINEENTROPYCONTEXTS(pt, *a, *l);
496 497 498
  assert(eob <= 4);

  for (; c < eob; c++) {
499 500 501 502 503
    int v = qcoeff_ptr[vp9_default_zig_zag1d[c]];
    int t = vp9_dct_value_tokens_ptr[v].Token;
    cost += mb->token_costs[TX_8X8][type][vp9_coef_bands[c]][pt][t];
    cost += vp9_dct_value_cost_ptr[v];
    pt = vp9_prev_token_class[t];
504 505 506
  }

  if (c < 4)
507
    cost += mb->token_costs[TX_8X8][type][vp9_coef_bands[c]]
508 509 510 511 512 513 514
            [pt] [DCT_EOB_TOKEN];

  pt = (c != !type); // is eob first coefficient;
  *a = *l = pt;
  return cost;
}

515
static int cost_coeffs(MACROBLOCK *mb, BLOCKD *b, PLANE_TYPE type,
Daniel Kang's avatar
Daniel Kang committed
516
                       ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l,
517
                       int tx_size) {
Daniel Kang's avatar
Daniel Kang committed
518
  const int eob = b->eob;
519
  int c = (type == PLANE_TYPE_Y_NO_DC); /* start at coef 0, unless Y with Y2 */
520
  int cost = 0, default_eob, seg_eob;
Daniel Kang's avatar
Daniel Kang committed
521 522
  int pt;                     /* surrounding block/prev coef predictor */
  int const *scan, *band;
John Koleszar's avatar
John Koleszar committed
523
  short *qcoeff_ptr = b->qcoeff;
524 525 526
  MACROBLOCKD *xd = &mb->e_mbd;
  MB_MODE_INFO *mbmi = &mb->e_mbd.mode_info_context->mbmi;
  TX_TYPE tx_type = DCT_DCT;
Paul Wilkins's avatar
Paul Wilkins committed
527
  int segment_id = mbmi->segment_id;
528

529
  switch (tx_size) {
Daniel Kang's avatar
Daniel Kang committed
530
    case TX_4X4:
531 532
      scan = vp9_default_zig_zag1d;
      band = vp9_coef_bands;
Daniel Kang's avatar
Daniel Kang committed
533
      default_eob = 16;
Deb Mukherjee's avatar
Deb Mukherjee committed
534 535 536 537 538
      if (type == PLANE_TYPE_Y_WITH_DC) {
        tx_type = get_tx_type_4x4(xd, b);
        if (tx_type != DCT_DCT) {
          switch (tx_type) {
            case ADST_DCT:
539
              scan = vp9_row_scan;
Deb Mukherjee's avatar
Deb Mukherjee committed
540 541 542
              break;

            case DCT_ADST:
543
              scan = vp9_col_scan;
Deb Mukherjee's avatar
Deb Mukherjee committed
544 545 546
              break;

            default:
547
              scan = vp9_default_zig_zag1d;
Deb Mukherjee's avatar
Deb Mukherjee committed
548 549
              break;
          }
550
        }
Daniel Kang's avatar
Daniel Kang committed
551
      }
Deb Mukherjee's avatar
Deb Mukherjee committed
552

Daniel Kang's avatar
Daniel Kang committed
553 554
      break;
    case TX_8X8:
555 556
      scan = vp9_default_zig_zag1d_8x8;
      band = vp9_coef_bands_8x8;
Daniel Kang's avatar
Daniel Kang committed
557
      default_eob = 64;
Deb Mukherjee's avatar
Deb Mukherjee committed
558
      if (type == PLANE_TYPE_Y_WITH_DC) {
559 560
        BLOCKD *bb;
        int ib = (b - xd->block);
561 562 563
        if (ib < 16) {
          ib = (ib & 8) + ((ib & 4) >> 1);
          bb = xd->block + ib;
Deb Mukherjee's avatar
Deb Mukherjee committed
564
          tx_type = get_tx_type_8x8(xd, bb);
565
        }
566
      }
Daniel Kang's avatar
Daniel Kang committed
567 568
      break;
    case TX_16X16:
569 570
      scan = vp9_default_zig_zag1d_16x16;
      band = vp9_coef_bands_16x16;
Daniel Kang's avatar
Daniel Kang committed
571
      default_eob = 256;
Deb Mukherjee's avatar
Deb Mukherjee committed
572 573 574
      if (type == PLANE_TYPE_Y_WITH_DC) {
        tx_type = get_tx_type_16x16(xd, b);
      }
Daniel Kang's avatar
Daniel Kang committed
575 576 577 578
      break;
    default:
      break;
  }
579 580
  if (vp9_segfeature_active(&mb->e_mbd, segment_id, SEG_LVL_EOB))
    seg_eob = vp9_get_segdata(&mb->e_mbd, segment_id, SEG_LVL_EOB);
581 582 583
  else
    seg_eob = default_eob;

584
  VP9_COMBINEENTROPYCONTEXTS(pt, *a, *l);
585

586 587 588
  if (tx_type != DCT_DCT) {
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
589
      int t = vp9_dct_value_tokens_ptr[v].Token;
590
      cost += mb->hybrid_token_costs[tx_size][type][band[c]][pt][t];
591 592
      cost += vp9_dct_value_cost_ptr[v];
      pt = vp9_prev_token_class[t];
593 594 595 596
    }
    if (c < seg_eob)
      cost += mb->hybrid_token_costs[tx_size][type][band[c]]
          [pt][DCT_EOB_TOKEN];
597
  } else {
598 599
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
600
      int t = vp9_dct_value_tokens_ptr[v].Token;
601
      cost += mb->token_costs[tx_size][type][band[c]][pt][t];
602 603
      cost += vp9_dct_value_cost_ptr[v];
      pt = vp9_prev_token_class[t];
604 605 606 607
    }
    if (c < seg_eob)
      cost += mb->token_costs[tx_size][type][band[c]]
          [pt][DCT_EOB_TOKEN];
608 609 610 611 612 613 614
  }

  pt = (c != !type); // is eob first coefficient;
  *a = *l = pt;
  return cost;
}

615
static int rdcost_mby_4x4(MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
616 617
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
618
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
619 620 621
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta;
  ENTROPY_CONTEXT *tl;
622

Paul Wilkins's avatar
Paul Wilkins committed
623 624
  vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
  vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
625

John Koleszar's avatar
John Koleszar committed
626 627
  ta = (ENTROPY_CONTEXT *)&t_above;
  tl = (ENTROPY_CONTEXT *)&t_left;
John Koleszar's avatar
John Koleszar committed
628

John Koleszar's avatar
John Koleszar committed
629
  for (b = 0; b < 16; b++)
Paul Wilkins's avatar
Paul Wilkins committed
630
    cost += cost_coeffs(mb, xd->block + b, PLANE_TYPE_Y_NO_DC,
631
                        ta + vp9_block2above[b], tl + vp9_block2left[b],
Daniel Kang's avatar
Daniel Kang committed
632
                        TX_4X4);
John Koleszar's avatar
John Koleszar committed
633

Paul Wilkins's avatar
Paul Wilkins committed
634
  cost += cost_coeffs(mb, xd->block + 24, PLANE_TYPE_Y2,
635
                      ta + vp9_block2above[24], tl + vp9_block2left[24],
Daniel Kang's avatar
Daniel Kang committed
636
                      TX_4X4);
John Koleszar's avatar
John Koleszar committed
637

John Koleszar's avatar
John Koleszar committed
638
  return cost;
John Koleszar's avatar
John Koleszar committed
639 640
}

641 642 643
static void macro_block_yrd_4x4(MACROBLOCK *mb,
                                int *Rate,
                                int *Distortion,
644
                                const VP9_ENCODER_RTCD *rtcd,
645
                                int *skippable) {
John Koleszar's avatar
John Koleszar committed
646
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
647
  MACROBLOCKD *const xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
648
  BLOCK   *const mb_y2 = mb->block + 24;
Paul Wilkins's avatar
Paul Wilkins committed
649
  BLOCKD *const x_y2  = xd->block + 24;
John Koleszar's avatar
John Koleszar committed
650 651 652
  short *Y2DCPtr = mb_y2->src_diff;
  BLOCK *beptr;
  int d;
653

654
  vp9_subtract_mby(mb->src_diff, *(mb->block[0].base_src), xd->predictor,
Jim Bankoski's avatar
Jim Bankoski committed
655
                   mb->block[0].src_stride);
656

John Koleszar's avatar
John Koleszar committed
657 658
  // Fdct and building the 2nd order block
  for (beptr = mb->block; beptr < mb->block + 16; beptr += 2) {
659
    mb->vp9_short_fdct8x4(beptr->src_diff, beptr->coeff, 32);
John Koleszar's avatar
John Koleszar committed
660 661 662 663 664 665 666 667 668
    *Y2DCPtr++ = beptr->coeff[0];
    *Y2DCPtr++ = beptr->coeff[16];
  }

  // 2nd order fdct
  mb->short_walsh4x4(mb_y2->src_diff, mb_y2->coeff, 8);

  // Quantization
  for (b = 0; b < 16; b++) {
669
    mb->quantize_b_4x4(&mb->block[b], &xd->block[b]);
John Koleszar's avatar
John Koleszar committed
670
  }
671

John Koleszar's avatar
John Koleszar committed
672
  // DC predication and Quantization of 2nd Order block
673
  mb->quantize_b_4x4(mb_y2, x_y2);
674

John Koleszar's avatar
John Koleszar committed
675
  // Distortion
676
  d = vp9_mbblock_error(mb, 1);
677

678
  d += vp9_block_error(mb_y2->coeff, x_y2->dqcoeff, 16);
679

John Koleszar's avatar
John Koleszar committed
680 681
  *Distortion = (d >> 2);
  // rate
682
  *Rate = rdcost_mby_4x4(mb);
683
  *skippable = vp9_mby_is_skippable_4x4(&mb->e_mbd, 1);
684
}
John Koleszar's avatar
John Koleszar committed
685

686
static int rdcost_mby_8x8(MACROBLOCK *mb, int backup) {
John Koleszar's avatar
John Koleszar committed
687 688
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
689
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
690 691 692 693
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta;
  ENTROPY_CONTEXT *tl;

Ronald S. Bultje's avatar
Ronald S. Bultje committed
694 695 696
  if (backup) {
    vpx_memcpy(&t_above,xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
697

Ronald S. Bultje's avatar
Ronald S. Bultje committed
698 699 700 701 702 703
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)mb->e_mbd.above_context;
    tl = (ENTROPY_CONTEXT *)mb->e_mbd.left_context;
  }
John Koleszar's avatar
John Koleszar committed
704 705

  for (b = 0; b < 16; b += 4)
Paul Wilkins's avatar
Paul Wilkins committed
706
    cost += cost_coeffs(mb, xd->block + b, PLANE_TYPE_Y_NO_DC,
707
                        ta + vp9_block2above_8x8[b], tl + vp9_block2left_8x8[b],
Daniel Kang's avatar
Daniel Kang committed
708
                        TX_8X8);
John Koleszar's avatar
John Koleszar committed
709

Paul Wilkins's avatar
Paul Wilkins committed
710
  cost += cost_coeffs_2x2(mb, xd->block + 24, PLANE_TYPE_Y2,
711
                          ta + vp9_block2above[24], tl + vp9_block2left[24]);
John Koleszar's avatar
John Koleszar committed
712
  return cost;
713 714
}

John Koleszar's avatar
John Koleszar committed
715 716 717
static void macro_block_yrd_8x8(MACROBLOCK *mb,
                                int *Rate,
                                int *Distortion,
718
                                const VP9_ENCODER_RTCD *rtcd,
719
                                int *skippable) {
Paul Wilkins's avatar
Paul Wilkins committed
720
  MACROBLOCKD *const xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
721
  BLOCK   *const mb_y2 = mb->block + 24;
Paul Wilkins's avatar
Paul Wilkins committed
722
  BLOCKD *const x_y2  = xd->block + 24;
John Koleszar's avatar
John Koleszar committed
723 724
  int d;

725
  vp9_subtract_mby(mb->src_diff, *(mb->block[0].base_src), xd->predictor,
Jim Bankoski's avatar
Jim Bankoski committed
726
                   mb->block[0].src_stride);
John Koleszar's avatar
John Koleszar committed
727

728 729
  vp9_transform_mby_8x8(mb);
  vp9_quantize_mby_8x8(mb);
John Koleszar's avatar
John Koleszar committed
730 731 732 733 734 735

  /* remove 1st order dc to properly combine 1st/2nd order distortion */
  mb->coeff[0] = 0;
  mb->coeff[64] = 0;
  mb->coeff[128] = 0;
  mb->coeff[192] = 0;
Paul Wilkins's avatar
Paul Wilkins committed
736 737 738 739
  xd->dqcoeff[0] = 0;
  xd->dqcoeff[64] = 0;
  xd->dqcoeff[128] = 0;
  xd->dqcoeff[192] = 0;
John Koleszar's avatar
John Koleszar committed
740

741 742
  d = vp9_mbblock_error(mb, 0);
  d += vp9_block_error(mb_y2->coeff, x_y2->dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
743 744 745

  *Distortion = (d >> 2);
  // rate
746
  *Rate = rdcost_mby_8x8(mb, 1);
747
  *skippable = vp9_mby_is_skippable_8x8(&mb->e_mbd, 1);
748
}
749

750
static int rdcost_mby_16x16(MACROBLOCK *mb) {
Daniel Kang's avatar
Daniel Kang committed
751
  int cost;
Paul Wilkins's avatar
Paul Wilkins committed
752
  MACROBLOCKD *xd = &mb->e_mbd;
Daniel Kang's avatar
Daniel Kang committed
753 754 755
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta, *tl;

Paul Wilkins's avatar
Paul Wilkins committed
756 757
  vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
  vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
Daniel Kang's avatar
Daniel Kang committed
758 759 760 761

  ta = (ENTROPY_CONTEXT *)&t_above;
  tl = (ENTROPY_CONTEXT *)&t_left;

Paul Wilkins's avatar
Paul Wilkins committed
762
  cost = cost_coeffs(mb, xd->block, PLANE_TYPE_Y_WITH_DC, ta, tl, TX_16X16);
Daniel Kang's avatar
Daniel Kang committed
763 764
  return cost;
}
765

Daniel Kang's avatar
Daniel Kang committed
766
static void macro_block_yrd_16x16(MACROBLOCK *mb, int *Rate, int *Distortion,
767
                                  const VP9_ENCODER_RTCD *rtcd, int *skippable) {
Daniel Kang's avatar
Daniel Kang committed
768
  int d;
Deb Mukherjee's avatar
Deb Mukherjee committed
769 770 771 772
  MACROBLOCKD *xd = &mb->e_mbd;
  BLOCKD *b  = &mb->e_mbd.block[0];
  BLOCK  *be = &mb->block[0];
  TX_TYPE tx_type;
Daniel Kang's avatar
Daniel Kang committed
773

774
  vp9_subtract_mby(mb->src_diff, *(mb->block[0].base_src), mb->e_mbd.predictor,
Jim Bankoski's avatar
Jim Bankoski committed
775
                   mb->block[0].src_stride);
Daniel Kang's avatar
Daniel Kang committed
776

Deb Mukherjee's avatar
Deb Mukherjee committed
777 778
  tx_type = get_tx_type_16x16(xd, b);
  if (tx_type != DCT_DCT) {
779
    vp9_fht(be->src_diff, 32, be->coeff, tx_type, 16);
780
  } else
781
    vp9_transform_mby_16x16(mb);
782

783
  vp9_quantize_mby_16x16(mb);
784 785 786
  // TODO(jingning) is it possible to quickly determine whether to force
  //                trailing coefficients to be zero, instead of running trellis
  //                optimization in the rate-distortion optimization loop?
787
  if (mb->e_mbd.mode_info_context->mbmi.mode < I8X8_PRED)
788
    vp9_optimize_mby_16x16(mb, rtcd);
789

790
  d = vp9_mbblock_error(mb, 0);
Daniel Kang's avatar
Daniel Kang committed
791 792 793

  *Distortion = (d >> 2);
  // rate
794
  *Rate = rdcost_mby_16x16(mb);
795
  *skippable = vp9_mby_is_skippable_16x16(&mb->e_mbd);
Daniel Kang's avatar
Daniel Kang committed
796 797
}

798
static void macro_block_yrd(VP9_COMP *cpi, MACROBLOCK *x, int *rate,
799 800
                            int *distortion, int *skippable,
                            int64_t txfm_cache[NB_TXFM_MODES]) {
801
  VP9_COMMON *cm = &cpi->common;
802 803 804 805
  MB_MODE_INFO *mbmi = &x->e_mbd.mode_info_context->mbmi;

  MACROBLOCKD *xd = &x->e_mbd;
  int can_skip = cm->mb_no_coeff_skip;
806
  vp9_prob skip_prob = can_skip ? vp9_get_pred_prob(cm, xd, PRED_MBSKIP) : 128;
807 808 809 810 811 812 813 814 815
  int s0, s1;
  int r4x4, r4x4s, r8x8, r8x8s, d4x4, d8x8, s4x4, s8x8;
  int64_t rd4x4, rd8x8, rd4x4s, rd8x8s;
  int d16x16, r16x16, r16x16s, s16x16;
  int64_t rd16x16, rd16x16s;

  // FIXME don't do sub x3
  if (skip_prob == 0)
    skip_prob = 1;
816 817
  s0 = vp9_cost_bit(skip_prob, 0);
  s1 = vp9_cost_bit(skip_prob, 1);
818 819 820 821 822 823 824 825 826 827
  macro_block_yrd_16x16(x, &r16x16, &d16x16, IF_RTCD(&cpi->rtcd), &s16x16);
  if (can_skip) {
    if (s16x16) {
      rd16x16 = RDCOST(x->rdmult, x->rddiv, s1, d16x16);
    } else {
      rd16x16 = RDCOST(x->rdmult, x->rddiv, r16x16 + s0, d16x16);
    }
  } else {
    rd16x16 = RDCOST(x->rdmult, x->rddiv, r16x16, d16x16);
  }
828
  r16x16s = r16x16 + vp9_cost_one(cm->prob_tx[0]) + vp9_cost_one(cm->prob_tx[1]);
829 830 831 832