vp9_rdopt.c 198 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5 6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9 10 11 12 13 14 15
 */


#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
16 17
#include "vp9/common/vp9_pragmas.h"

18 19 20 21 22
#include "vp9/encoder/vp9_tokenize.h"
#include "vp9/encoder/vp9_treewriter.h"
#include "vp9/encoder/vp9_onyx_int.h"
#include "vp9/encoder/vp9_modecosts.h"
#include "vp9/encoder/vp9_encodeintra.h"
23 24 25 26 27
#include "vp9/common/vp9_entropymode.h"
#include "vp9/common/vp9_reconinter.h"
#include "vp9/common/vp9_reconintra.h"
#include "vp9/common/vp9_findnearmv.h"
#include "vp9/common/vp9_quant_common.h"
28 29 30 31 32 33
#include "vp9/encoder/vp9_encodemb.h"
#include "vp9/encoder/vp9_quantize.h"
#include "vp9/encoder/vp9_variance.h"
#include "vp9/encoder/vp9_mcomp.h"
#include "vp9/encoder/vp9_rdopt.h"
#include "vp9/encoder/vp9_ratectrl.h"
John Koleszar's avatar
John Koleszar committed
34
#include "vpx_mem/vpx_mem.h"
35 36
#include "vp9/common/vp9_systemdependent.h"
#include "vp9/encoder/vp9_encodemv.h"
John Koleszar's avatar
John Koleszar committed
37

38 39 40
#include "vp9/common/vp9_seg_common.h"
#include "vp9/common/vp9_pred_common.h"
#include "vp9/common/vp9_entropy.h"
41
#include "vp9_rtcd.h"
42
#include "vp9/common/vp9_mvref_common.h"
Ronald S. Bultje's avatar
Ronald S. Bultje committed
43
#include "vp9/common/vp9_common.h"
Paul Wilkins's avatar
Paul Wilkins committed
44

John Koleszar's avatar
John Koleszar committed
45 46
#define MAXF(a,b)            (((a) > (b)) ? (a) : (b))

47 48
#define INVALID_MV 0x80008000

49 50 51
/* Factor to weigh the rate for switchable interp filters */
#define SWITCHABLE_INTERP_RATE_FACTOR 1

John Koleszar's avatar
John Koleszar committed
52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69
static const int auto_speed_thresh[17] = {
  1000,
  200,
  150,
  130,
  150,
  125,
  120,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  105
John Koleszar's avatar
John Koleszar committed
70 71
};

72
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
73 74
  {ZEROMV,    LAST_FRAME,   NONE},
  {DC_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
75

76 77
  {NEARESTMV, LAST_FRAME,   NONE},
  {NEARMV,    LAST_FRAME,   NONE},
John Koleszar's avatar
John Koleszar committed
78

79 80
  {ZEROMV,    GOLDEN_FRAME, NONE},
  {NEARESTMV, GOLDEN_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
81

82 83
  {ZEROMV,    ALTREF_FRAME, NONE},
  {NEARESTMV, ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
84

85 86
  {NEARMV,    GOLDEN_FRAME, NONE},
  {NEARMV,    ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
87

88 89 90 91 92 93 94 95
  {V_PRED,    INTRA_FRAME,  NONE},
  {H_PRED,    INTRA_FRAME,  NONE},
  {D45_PRED,  INTRA_FRAME,  NONE},
  {D135_PRED, INTRA_FRAME,  NONE},
  {D117_PRED, INTRA_FRAME,  NONE},
  {D153_PRED, INTRA_FRAME,  NONE},
  {D27_PRED,  INTRA_FRAME,  NONE},
  {D63_PRED,  INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
96

97
  {TM_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
98

99 100 101
  {NEWMV,     LAST_FRAME,   NONE},
  {NEWMV,     GOLDEN_FRAME, NONE},
  {NEWMV,     ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
102

103 104 105
  {SPLITMV,   LAST_FRAME,   NONE},
  {SPLITMV,   GOLDEN_FRAME, NONE},
  {SPLITMV,   ALTREF_FRAME, NONE},
106

107 108
  {B_PRED,    INTRA_FRAME,  NONE},
  {I8X8_PRED, INTRA_FRAME,  NONE},
109

John Koleszar's avatar
John Koleszar committed
110 111 112 113
  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME},
114

John Koleszar's avatar
John Koleszar committed
115 116 117
  {ZEROMV,    ALTREF_FRAME, LAST_FRAME},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME},
118

John Koleszar's avatar
John Koleszar committed
119 120 121
  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME},
122

John Koleszar's avatar
John Koleszar committed
123 124 125
  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME  },
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME},
126

John Koleszar's avatar
John Koleszar committed
127 128
  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME  },
129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME},

#if CONFIG_COMP_INTERINTRA_PRED
  /* compound inter-intra prediction */
  {ZEROMV,    LAST_FRAME,   INTRA_FRAME},
  {NEARESTMV, LAST_FRAME,   INTRA_FRAME},
  {NEARMV,    LAST_FRAME,   INTRA_FRAME},
  {NEWMV,     LAST_FRAME,   INTRA_FRAME},

  {ZEROMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEARESTMV, GOLDEN_FRAME,   INTRA_FRAME},
  {NEARMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEWMV,     GOLDEN_FRAME,   INTRA_FRAME},

  {ZEROMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEARESTMV, ALTREF_FRAME,   INTRA_FRAME},
  {NEARMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEWMV,     ALTREF_FRAME,   INTRA_FRAME},
#endif
John Koleszar's avatar
John Koleszar committed
148 149
};

150 151 152
static void fill_token_costs(vp9_coeff_count *c,
                             vp9_coeff_probs *p,
                             int block_type_counts) {
153
  int i, j, k, l;
John Koleszar's avatar
John Koleszar committed
154 155

  for (i = 0; i < block_type_counts; i++)
156 157 158 159 160 161 162 163 164 165 166 167
    for (j = 0; j < REF_TYPES; j++)
      for (k = 0; k < COEF_BANDS; k++)
        for (l = 0; l < PREV_COEF_CONTEXTS; l++) {
          if (l == 0 && k > 0)
            vp9_cost_tokens_skip((int *)(c[i][j][k][l]),
                                 p[i][j][k][l],
                                 vp9_coef_tree);
          else
            vp9_cost_tokens((int *)(c[i][j][k][l]),
                            p[i][j][k][l],
                            vp9_coef_tree);
        }
John Koleszar's avatar
John Koleszar committed
168 169
}

170

171 172 173 174
static int rd_iifactor[32] =  { 4, 4, 3, 2, 1, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0, };
John Koleszar's avatar
John Koleszar committed
175

176
// 3* dc_qlookup[Q]*dc_qlookup[Q];
177

178
/* values are now correlated to quantizer */
Paul Wilkins's avatar
Paul Wilkins committed
179 180 181
static int sad_per_bit16lut[QINDEX_RANGE];
static int sad_per_bit4lut[QINDEX_RANGE];

182
void vp9_init_me_luts() {
John Koleszar's avatar
John Koleszar committed
183 184 185 186 187 188 189
  int i;

  // Initialize the sad lut tables using a formulaic calculation for now
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < QINDEX_RANGE; i++) {
    sad_per_bit16lut[i] =
190 191
      (int)((0.0418 * vp9_convert_qindex_to_q(i)) + 2.4107);
    sad_per_bit4lut[i] = (int)((0.063 * vp9_convert_qindex_to_q(i)) + 2.742);
John Koleszar's avatar
John Koleszar committed
192
  }
Paul Wilkins's avatar
Paul Wilkins committed
193
}
John Koleszar's avatar
John Koleszar committed
194

195
static int compute_rd_mult(int qindex) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
196
  int q = vp9_dc_quant(qindex, 0);
John Koleszar's avatar
John Koleszar committed
197
  return (11 * q * q) >> 6;
198 199
}

Dmitry Kovalev's avatar
Dmitry Kovalev committed
200 201 202
void vp9_initialize_me_consts(VP9_COMP *cpi, int qindex) {
  cpi->mb.sadperbit16 = sad_per_bit16lut[qindex];
  cpi->mb.sadperbit4 = sad_per_bit4lut[qindex];
John Koleszar's avatar
John Koleszar committed
203 204
}

205

Dmitry Kovalev's avatar
Dmitry Kovalev committed
206
void vp9_initialize_rd_consts(VP9_COMP *cpi, int qindex) {
207
  int q, i;
John Koleszar's avatar
John Koleszar committed
208

209
  vp9_clear_system_state();  // __asm emms;
John Koleszar's avatar
John Koleszar committed
210

John Koleszar's avatar
John Koleszar committed
211 212 213 214
  // Further tests required to see if optimum is different
  // for key frames, golden frames and arf frames.
  // if (cpi->common.refresh_golden_frame ||
  //     cpi->common.refresh_alt_ref_frame)
Dmitry Kovalev's avatar
Dmitry Kovalev committed
215
  qindex = (qindex < 0) ? 0 : ((qindex > MAXQ) ? MAXQ : qindex);
216

Dmitry Kovalev's avatar
Dmitry Kovalev committed
217
  cpi->RDMULT = compute_rd_mult(qindex);
John Koleszar's avatar
John Koleszar committed
218

John Koleszar's avatar
John Koleszar committed
219 220 221 222 223
  if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    if (cpi->twopass.next_iiratio > 31)
      cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
    else
      cpi->RDMULT +=
Dmitry Kovalev's avatar
Dmitry Kovalev committed
224
          (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
John Koleszar's avatar
John Koleszar committed
225
  }
John Koleszar's avatar
John Koleszar committed
226

John Koleszar's avatar
John Koleszar committed
227 228
  if (cpi->RDMULT < 7)
    cpi->RDMULT = 7;
229

John Koleszar's avatar
John Koleszar committed
230 231
  cpi->mb.errorperbit = (cpi->RDMULT / 110);
  cpi->mb.errorperbit += (cpi->mb.errorperbit == 0);
232

233
  vp9_set_speed_features(cpi);
John Koleszar's avatar
John Koleszar committed
234

Dmitry Kovalev's avatar
Dmitry Kovalev committed
235 236
  q = (int)pow(vp9_dc_quant(qindex, 0) >> 2, 1.25);
  q <<= 2;
John Koleszar's avatar
John Koleszar committed
237
  cpi->RDMULT = cpi->RDMULT << 4;
Paul Wilkins's avatar
Paul Wilkins committed
238

John Koleszar's avatar
John Koleszar committed
239 240
  if (q < 8)
    q = 8;
241

John Koleszar's avatar
John Koleszar committed
242 243 244
  if (cpi->RDMULT > 1000) {
    cpi->RDDIV = 1;
    cpi->RDMULT /= 100;
John Koleszar's avatar
John Koleszar committed
245

John Koleszar's avatar
John Koleszar committed
246 247 248 249 250 251
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < INT_MAX) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q / 100;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
252

John Koleszar's avatar
John Koleszar committed
253
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
254
    }
John Koleszar's avatar
John Koleszar committed
255 256
  } else {
    cpi->RDDIV = 100;
John Koleszar's avatar
John Koleszar committed
257

John Koleszar's avatar
John Koleszar committed
258 259 260 261 262 263
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < (INT_MAX / q)) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
264

John Koleszar's avatar
John Koleszar committed
265
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
266
    }
John Koleszar's avatar
John Koleszar committed
267
  }
John Koleszar's avatar
John Koleszar committed
268

269
  fill_token_costs(cpi->mb.token_costs[TX_4X4],
270
                   cpi->common.fc.coef_probs_4x4, BLOCK_TYPES);
271
  fill_token_costs(cpi->mb.token_costs[TX_8X8],
272
                   cpi->common.fc.coef_probs_8x8, BLOCK_TYPES);
273
  fill_token_costs(cpi->mb.token_costs[TX_16X16],
274
                   cpi->common.fc.coef_probs_16x16, BLOCK_TYPES);
275
  fill_token_costs(cpi->mb.token_costs[TX_32X32],
276
                   cpi->common.fc.coef_probs_32x32, BLOCK_TYPES);
277

John Koleszar's avatar
John Koleszar committed
278 279
  /*rough estimate for costing*/
  cpi->common.kf_ymode_probs_index = cpi->common.base_qindex >> 4;
280
  vp9_init_mode_costs(cpi);
John Koleszar's avatar
John Koleszar committed
281

282
  if (cpi->common.frame_type != KEY_FRAME) {
283
    vp9_build_nmv_cost_table(
284 285 286 287 288 289
        cpi->mb.nmvjointcost,
        cpi->mb.e_mbd.allow_high_precision_mv ?
        cpi->mb.nmvcost_hp : cpi->mb.nmvcost,
        &cpi->common.fc.nmvc,
        cpi->mb.e_mbd.allow_high_precision_mv, 1, 1);
  }
John Koleszar's avatar
John Koleszar committed
290 291
}

292
int vp9_block_error_c(int16_t *coeff, int16_t *dqcoeff, int block_size) {
293
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
294

295
  for (i = 0; i < block_size; i++) {
John Koleszar's avatar
John Koleszar committed
296 297 298
    int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }
John Koleszar's avatar
John Koleszar committed
299

John Koleszar's avatar
John Koleszar committed
300
  return error;
John Koleszar's avatar
John Koleszar committed
301 302
}

303
int vp9_mbblock_error_c(MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
304 305 306 307
  BLOCK  *be;
  BLOCKD *bd;
  int i, j;
  int berror, error = 0;
John Koleszar's avatar
John Koleszar committed
308

John Koleszar's avatar
John Koleszar committed
309 310 311 312
  for (i = 0; i < 16; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
    berror = 0;
313
    for (j = 0; j < 16; j++) {
John Koleszar's avatar
John Koleszar committed
314 315
      int this_diff = be->coeff[j] - bd->dqcoeff[j];
      berror += this_diff * this_diff;
John Koleszar's avatar
John Koleszar committed
316
    }
John Koleszar's avatar
John Koleszar committed
317 318 319
    error += berror;
  }
  return error;
John Koleszar's avatar
John Koleszar committed
320 321
}

322
int vp9_mbuverror_c(MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
323 324
  BLOCK  *be;
  BLOCKD *bd;
John Koleszar's avatar
John Koleszar committed
325

326
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
327

John Koleszar's avatar
John Koleszar committed
328 329 330
  for (i = 16; i < 24; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
John Koleszar's avatar
John Koleszar committed
331

332
    error += vp9_block_error_c(be->coeff, bd->dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
333
  }
John Koleszar's avatar
John Koleszar committed
334

John Koleszar's avatar
John Koleszar committed
335
  return error;
John Koleszar's avatar
John Koleszar committed
336 337
}

338
int vp9_uvsse(MACROBLOCK *x) {
339 340 341
  uint8_t *uptr, *vptr;
  uint8_t *upred_ptr = (*(x->block[16].base_src) + x->block[16].src);
  uint8_t *vpred_ptr = (*(x->block[20].base_src) + x->block[20].src);
John Koleszar's avatar
John Koleszar committed
342 343 344 345
  int uv_stride = x->block[16].src_stride;

  unsigned int sse1 = 0;
  unsigned int sse2 = 0;
346 347
  int mv_row = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.row;
  int mv_col = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.col;
John Koleszar's avatar
John Koleszar committed
348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368
  int offset;
  int pre_stride = x->e_mbd.block[16].pre_stride;

  if (mv_row < 0)
    mv_row -= 1;
  else
    mv_row += 1;

  if (mv_col < 0)
    mv_col -= 1;
  else
    mv_col += 1;

  mv_row /= 2;
  mv_col /= 2;

  offset = (mv_row >> 3) * pre_stride + (mv_col >> 3);
  uptr = x->e_mbd.pre.u_buffer + offset;
  vptr = x->e_mbd.pre.v_buffer + offset;

  if ((mv_row | mv_col) & 7) {
369
    vp9_sub_pixel_variance8x8(uptr, pre_stride, (mv_col & 7) << 1,
370
                              (mv_row & 7) << 1, upred_ptr, uv_stride, &sse2);
371
    vp9_sub_pixel_variance8x8(vptr, pre_stride, (mv_col & 7) << 1,
372
                              (mv_row & 7) << 1, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
373 374
    sse2 += sse1;
  } else {
375 376
    vp9_variance8x8(uptr, pre_stride, upred_ptr, uv_stride, &sse2);
    vp9_variance8x8(vptr, pre_stride, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
377 378 379
    sse2 += sse1;
  }
  return sse2;
John Koleszar's avatar
John Koleszar committed
380 381
}

382
static INLINE int cost_coeffs(MACROBLOCK *mb,
383
                              int ib, PLANE_TYPE type,
384 385 386
                              ENTROPY_CONTEXT *a,
                              ENTROPY_CONTEXT *l,
                              TX_SIZE tx_size) {
387
  MACROBLOCKD *const xd = &mb->e_mbd;
388 389 390
  MB_MODE_INFO *mbmi = &xd->mode_info_context->mbmi;
  const BLOCK_SIZE_TYPE sb_type = mbmi->sb_type;
  int pt;
391
  const int eob = xd->eobs[ib];
392
  int c = 0;
393
  int cost = 0, seg_eob;
394
  const int segment_id = mbmi->segment_id;
395
  const int *scan;
396 397 398 399 400
  const int16_t *qcoeff_ptr = xd->qcoeff + ib * 16;
  const int ref = mbmi->ref_frame != INTRA_FRAME;
  const TX_TYPE tx_type = (sb_type == BLOCK_SIZE_MB16X16 &&
                           type == PLANE_TYPE_Y_WITH_DC) ?
                          get_tx_type(xd, &xd->block[ib]) : DCT_DCT;
401
  unsigned int (*token_costs)[PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS] =
402
      mb->token_costs[tx_size][type][ref];
403
  ENTROPY_CONTEXT a_ec, l_ec;
404 405 406 407
  ENTROPY_CONTEXT *const a1 = a +
      sizeof(ENTROPY_CONTEXT_PLANES)/sizeof(ENTROPY_CONTEXT);
  ENTROPY_CONTEXT *const l1 = l +
      sizeof(ENTROPY_CONTEXT_PLANES)/sizeof(ENTROPY_CONTEXT);
408

409
  switch (tx_size) {
Daniel Kang's avatar
Daniel Kang committed
410
    case TX_4X4:
411 412
      a_ec = *a;
      l_ec = *l;
413 414
      scan = vp9_default_zig_zag1d_4x4;
      seg_eob = 16;
Deb Mukherjee's avatar
Deb Mukherjee committed
415
      if (type == PLANE_TYPE_Y_WITH_DC) {
416 417 418 419
        if (tx_type == ADST_DCT) {
          scan = vp9_row_scan_4x4;
        } else if (tx_type == DCT_ADST) {
          scan = vp9_col_scan_4x4;
420
        }
Daniel Kang's avatar
Daniel Kang committed
421 422 423
      }
      break;
    case TX_8X8:
424 425
      a_ec = (a[0] + a[1]) != 0;
      l_ec = (l[0] + l[1]) != 0;
426 427
      scan = vp9_default_zig_zag1d_8x8;
      seg_eob = 64;
Daniel Kang's avatar
Daniel Kang committed
428 429
      break;
    case TX_16X16:
430
      scan = vp9_default_zig_zag1d_16x16;
431 432
      seg_eob = 256;
      if (type == PLANE_TYPE_UV) {
433 434 435 436 437
        a_ec = (a[0] + a[1] + a1[0] + a1[1]) != 0;
        l_ec = (l[0] + l[1] + l1[0] + l1[1]) != 0;
      } else {
        a_ec = (a[0] + a[1] + a[2] + a[3]) != 0;
        l_ec = (l[0] + l[1] + l[2] + l[3]) != 0;
Deb Mukherjee's avatar
Deb Mukherjee committed
438
      }
Daniel Kang's avatar
Daniel Kang committed
439
      break;
440 441
    case TX_32X32:
      scan = vp9_default_zig_zag1d_32x32;
442
      seg_eob = 1024;
443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458
      if (type == PLANE_TYPE_UV) {
        ENTROPY_CONTEXT *a2, *a3, *l2, *l3;
        a2 = a1 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        a3 = a2 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        l2 = l1 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        l3 = l2 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        a_ec = (a[0] + a[1] + a1[0] + a1[1] +
                a2[0] + a2[1] + a3[0] + a3[1]) != 0;
        l_ec = (l[0] + l[1] + l1[0] + l1[1] +
                l2[0] + l2[1] + l3[0] + l3[1]) != 0;
      } else {
        a_ec = (a[0] + a[1] + a[2] + a[3] +
                a1[0] + a1[1] + a1[2] + a1[3]) != 0;
        l_ec = (l[0] + l[1] + l[2] + l[3] +
                l1[0] + l1[1] + l1[2] + l1[3]) != 0;
      }
459
      break;
Daniel Kang's avatar
Daniel Kang committed
460
    default:
461
      abort();
Daniel Kang's avatar
Daniel Kang committed
462 463
      break;
  }
464

465 466
  VP9_COMBINEENTROPYCONTEXTS(pt, a_ec, l_ec);

Paul Wilkins's avatar
Paul Wilkins committed
467 468
  if (vp9_segfeature_active(xd, segment_id, SEG_LVL_SKIP))
    seg_eob = 0;
469

470
  {
471
    int recent_energy = 0;
472 473
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
474
      int t = vp9_dct_value_tokens_ptr[v].Token;
475
      cost += token_costs[get_coef_band(tx_size, c)][pt][t];
476
      cost += vp9_dct_value_cost_ptr[v];
477
      pt = vp9_get_coef_context(&recent_energy, t);
478 479
    }
    if (c < seg_eob)
480
      cost += mb->token_costs[tx_size][type][ref][get_coef_band(tx_size, c)]
481
          [pt][DCT_EOB_TOKEN];
482 483
  }

484
  // is eob first coefficient;
485
  pt = (c > 0);
486
  *a = *l = pt;
487 488 489 490 491 492 493 494 495 496 497 498 499 500
  if (tx_size >= TX_8X8) {
    a[1] = l[1] = pt;
    if (tx_size >= TX_16X16) {
      if (type == PLANE_TYPE_UV) {
        a1[0] = a1[1] = l1[0] = l1[1] = pt;
      } else {
        a[2] = a[3] = l[2] = l[3] = pt;
        if (tx_size >= TX_32X32) {
          a1[0] = a1[1] = a1[2] = a1[3] = pt;
          l1[0] = l1[1] = l1[2] = l1[3] = pt;
        }
      }
    }
  }
501 502 503
  return cost;
}

504
static int rdcost_mby_4x4(MACROBLOCK *mb, int backup) {
John Koleszar's avatar
John Koleszar committed
505 506
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
507
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
508 509 510
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta;
  ENTROPY_CONTEXT *tl;
511

512 513 514
  if (backup) {
    vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
515

516 517 518 519 520 521
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)xd->above_context;
    tl = (ENTROPY_CONTEXT *)xd->left_context;
  }
John Koleszar's avatar
John Koleszar committed
522

John Koleszar's avatar
John Koleszar committed
523
  for (b = 0; b < 16; b++)
524
    cost += cost_coeffs(mb, b, PLANE_TYPE_Y_WITH_DC,
Yaowu Xu's avatar
Yaowu Xu committed
525 526
                        ta + vp9_block2above[TX_4X4][b],
                        tl + vp9_block2left[TX_4X4][b],
Daniel Kang's avatar
Daniel Kang committed
527
                        TX_4X4);
John Koleszar's avatar
John Koleszar committed
528

John Koleszar's avatar
John Koleszar committed
529
  return cost;
John Koleszar's avatar
John Koleszar committed
530 531
}

532 533 534
static void macro_block_yrd_4x4(MACROBLOCK *mb,
                                int *Rate,
                                int *Distortion,
535
                                int *skippable, int backup) {
Paul Wilkins's avatar
Paul Wilkins committed
536
  MACROBLOCKD *const xd = &mb->e_mbd;
537

538 539 540
  xd->mode_info_context->mbmi.txfm_size = TX_4X4;
  vp9_transform_mby_4x4(mb);
  vp9_quantize_mby_4x4(mb);
541

542 543 544
  *Distortion = vp9_mbblock_error(mb) >> 2;
  *Rate = rdcost_mby_4x4(mb, backup);
  *skippable = vp9_mby_is_skippable_4x4(xd);
545
}
John Koleszar's avatar
John Koleszar committed
546

547
static int rdcost_mby_8x8(MACROBLOCK *mb, int backup) {
John Koleszar's avatar
John Koleszar committed
548 549
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
550
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
551 552 553 554
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta;
  ENTROPY_CONTEXT *tl;

Ronald S. Bultje's avatar
Ronald S. Bultje committed
555 556 557
  if (backup) {
    vpx_memcpy(&t_above,xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
558

Ronald S. Bultje's avatar
Ronald S. Bultje committed
559 560 561 562 563 564
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)mb->e_mbd.above_context;
    tl = (ENTROPY_CONTEXT *)mb->e_mbd.left_context;
  }
John Koleszar's avatar
John Koleszar committed
565 566

  for (b = 0; b < 16; b += 4)
567
    cost += cost_coeffs(mb, b, PLANE_TYPE_Y_WITH_DC,
Yaowu Xu's avatar
Yaowu Xu committed
568 569
                        ta + vp9_block2above[TX_8X8][b],
                        tl + vp9_block2left[TX_8X8][b],
Daniel Kang's avatar
Daniel Kang committed
570
                        TX_8X8);
John Koleszar's avatar
John Koleszar committed
571 572

  return cost;
573 574
}

John Koleszar's avatar
John Koleszar committed
575 576 577
static void macro_block_yrd_8x8(MACROBLOCK *mb,
                                int *Rate,
                                int *Distortion,
578
                                int *skippable, int backup) {
Paul Wilkins's avatar
Paul Wilkins committed
579
  MACROBLOCKD *const xd = &mb->e_mbd;
580 581

  xd->mode_info_context->mbmi.txfm_size = TX_8X8;
582 583
  vp9_transform_mby_8x8(mb);
  vp9_quantize_mby_8x8(mb);
John Koleszar's avatar
John Koleszar committed
584

585 586 587
  *Distortion = vp9_mbblock_error(mb) >> 2;
  *Rate = rdcost_mby_8x8(mb, backup);
  *skippable = vp9_mby_is_skippable_8x8(xd);
588
}
589

590
static int rdcost_mby_16x16(MACROBLOCK *mb, int backup) {
Daniel Kang's avatar
Daniel Kang committed
591
  int cost;
Paul Wilkins's avatar
Paul Wilkins committed
592
  MACROBLOCKD *xd = &mb->e_mbd;
Daniel Kang's avatar
Daniel Kang committed
593 594 595
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta, *tl;

596 597 598
  if (backup) {
    vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
Daniel Kang's avatar
Daniel Kang committed
599

600 601 602 603 604 605
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)xd->above_context;
    tl = (ENTROPY_CONTEXT *)xd->left_context;
  }
Daniel Kang's avatar
Daniel Kang committed
606

607
  cost = cost_coeffs(mb, 0, PLANE_TYPE_Y_WITH_DC, ta, tl, TX_16X16);
Daniel Kang's avatar
Daniel Kang committed
608 609
  return cost;
}
610

Daniel Kang's avatar
Daniel Kang committed
611
static void macro_block_yrd_16x16(MACROBLOCK *mb, int *Rate, int *Distortion,
612
                                  int *skippable, int backup) {
Deb Mukherjee's avatar
Deb Mukherjee committed
613
  MACROBLOCKD *xd = &mb->e_mbd;
Daniel Kang's avatar
Daniel Kang committed
614

615 616
  xd->mode_info_context->mbmi.txfm_size = TX_16X16;
  vp9_transform_mby_16x16(mb);
617
  vp9_quantize_mby_16x16(mb);
618 619 620
  // TODO(jingning) is it possible to quickly determine whether to force
  //                trailing coefficients to be zero, instead of running trellis
  //                optimization in the rate-distortion optimization loop?
621 622
  if (mb->optimize &&
      xd->mode_info_context->mbmi.mode < I8X8_PRED)
623
    vp9_optimize_mby_16x16(mb);
624

625
  *Distortion = vp9_mbblock_error(mb) >> 2;
626
  *Rate = rdcost_mby_16x16(mb, backup);
627
  *skippable = vp9_mby_is_skippable_16x16(xd);
Daniel Kang's avatar
Daniel Kang committed
628 629
}

630
static void choose_txfm_size_from_rd(VP9_COMP *cpi, MACROBLOCK *x,
631 632 633 634 635
                                     int (*r)[2], int *rate,
                                     int *d, int *distortion,
                                     int *s, int *skip,
                                     int64_t txfm_cache[NB_TXFM_MODES],
                                     TX_SIZE max_txfm_size) {
636 637 638 639 640
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
  vp9_prob skip_prob = cm->mb_no_coeff_skip ?
                       vp9_get_pred_prob(cm, xd, PRED_MBSKIP) : 128;
641 642 643 644 645 646 647 648 649 650 651 652
  int64_t rd[TX_SIZE_MAX_SB][2];
  int n, m;

  for (n = TX_4X4; n <= max_txfm_size; n++) {
    r[n][1] = r[n][0];
    for (m = 0; m <= n - (n == max_txfm_size); m++) {
      if (m == n)
        r[n][1] += vp9_cost_zero(cm->prob_tx[m]);
      else
        r[n][1] += vp9_cost_one(cm->prob_tx[m]);
    }
  }
653 654 655 656 657 658 659 660

  if (cm->mb_no_coeff_skip) {
    int s0, s1;

    assert(skip_prob > 0);
    s0 = vp9_cost_bit(skip_prob, 0);
    s1 = vp9_cost_bit(skip_prob, 1);

661
    for (n = TX_4X4; n <= max_txfm_size; n++) {
662
      if (s[n]) {
663
        rd[n][0] = rd[n][1] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
664
      } else {
665 666
        rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0] + s0, d[n]);
        rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1] + s0, d[n]);
667
      }
668 669
    }
  } else {
670 671 672
    for (n = TX_4X4; n <= max_txfm_size; n++) {
      rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0], d[n]);
      rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1], d[n]);
673 674 675
    }
  }

676 677 678 679 680 681
  if (max_txfm_size == TX_32X32 &&
      (cm->txfm_mode == ALLOW_32X32 ||
       (cm->txfm_mode == TX_MODE_SELECT &&
        rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
        rd[TX_32X32][1] < rd[TX_4X4][1]))) {
    mbmi->txfm_size = TX_32X32;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
682 683 684 685 686
  } else if ( cm->txfm_mode == ALLOW_16X16 ||
             (max_txfm_size == TX_16X16 && cm->txfm_mode == ALLOW_32X32) ||
             (cm->txfm_mode == TX_MODE_SELECT &&
              rd[TX_16X16][1] < rd[TX_8X8][1] &&
              rd[TX_16X16][1] < rd[TX_4X4][1])) {
687
    mbmi->txfm_size = TX_16X16;
688
  } else if (cm->txfm_mode == ALLOW_8X8 ||
689
           (cm->txfm_mode == TX_MODE_SELECT && rd[TX_8X8][1] < rd[TX_4X4][1])) {
690 691
    mbmi->txfm_size = TX_8X8;
  } else {
692
    assert(cm->txfm_mode == ONLY_4X4 || cm->txfm_mode == TX_MODE_SELECT);
693 694 695
    mbmi->txfm_size = TX_4X4;
  }

696
  *distortion = d[mbmi->txfm_size];
697
  *rate       = r[mbmi->txfm_size][cm->txfm_mode == TX_MODE_SELECT];
698 699
  *skip       = s[mbmi->txfm_size];

700 701 702 703 704 705 706 707
  txfm_cache[ONLY_4X4] = rd[TX_4X4][0];
  txfm_cache[ALLOW_8X8] = rd[TX_8X8][0];
  txfm_cache[ALLOW_16X16] = rd[TX_16X16][0];
  txfm_cache[ALLOW_32X32] = rd[max_txfm_size][0];
  if (max_txfm_size == TX_32X32 &&
      rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
      rd[TX_32X32][1] < rd[TX_4X4][1])
    txfm_cache[TX_MODE_SELECT] = rd[TX_32X32][1];
Ronald S. Bultje's avatar
Ronald S. Bultje committed
708
  else if (rd[TX_16X16][1] < rd[TX_8X8][1] && rd[TX_16X16][1] < rd[TX_4X4][1])
709
    txfm_cache[TX_MODE_SELECT] = rd[TX_16X16][1];
710
  else
711 712
    txfm_cache[TX_MODE_SELECT] = rd[TX_4X4][1] < rd[TX_8X8][1] ?
                                 rd[TX_4X4][1] : rd[TX_8X8][1];
713 714 715 716 717 718
}

static void macro_block_yrd(VP9_COMP *cpi, MACROBLOCK *x, int *rate,
                            int *distortion, int *skippable,
                            int64_t txfm_cache[NB_TXFM_MODES]) {
  MACROBLOCKD *const xd = &x->e_mbd;
719
  int r[TX_SIZE_MAX_MB][2], d[TX_SIZE_MAX_MB], s[TX_SIZE_MAX_MB];
720 721 722

  vp9_subtract_mby(x->src_diff, *(x->block[0].base_src), xd->predictor,
                   x->block[0].src_stride);
723

724 725 726
  macro_block_yrd_16x16(x, &r[TX_16X16][0], &d[TX_16X16], &s[TX_16X16], 1);
  macro_block_yrd_8x8(x, &r[TX_8X8][0], &d[TX_8X8], &s[TX_8X8], 1);
  macro_block_yrd_4x4(x, &r[TX_4X4][0], &d[TX_4X4], &s[TX_4X4], 1);
727 728

  choose_txfm_size_from_rd(cpi, x, r, rate, d, distortion, s, skippable,
729
                           txfm_cache, TX_16X16);
730 731
}

732
static void copy_predictor(uint8_t *dst, const uint8_t *predictor) {
John Koleszar's avatar
John Koleszar committed
733 734 735 736 737 738
  const unsigned int *p = (const unsigned int *)predictor;
  unsigned int *d = (unsigned int *)dst;
  d[0] = p[0];
  d[4] = p[4];
  d[8] = p[8];
  d[12] = p[12];
739
}
Yaowu Xu's avatar
Yaowu Xu committed
740

Ronald S. Bultje's avatar
Ronald S. Bultje committed
741
static int rdcost_sby_32x32(MACROBLOCK *x, int backup) {
742
  MACROBLOCKD * const xd = &x->e_mbd;
743
  ENTROPY_CONTEXT_PLANES t_above[2], t_left[2];
Ronald S. Bultje's avatar
Ronald S. Bultje committed
744
  ENTROPY_CONTEXT *ta, *tl;
745

Ronald S. Bultje's avatar
Ronald S. Bultje committed
746 747 748 749
  if (backup) {
    ta = (ENTROPY_CONTEXT *) &t_above,
    tl = (ENTROPY_CONTEXT *) &t_left;

750 751
    vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES) * 2);
    vpx_memcpy(&t_left,  xd->left_context,  sizeof(ENTROPY_CONTEXT_PLANES) * 2);
Ronald S. Bultje's avatar
Ronald S. Bultje committed
752 753 754 755
  } else {
    ta = (ENTROPY_CONTEXT *) xd->above_context;
    tl = (ENTROPY_CONTEXT *) xd->left_context;
  }
756

757
  return cost_coeffs(x, 0, PLANE_TYPE_Y_WITH_DC, ta, tl, TX_32X32);
758 759
}

760 761
static int vp9_sb_block_error_c(int16_t *coeff, int16_t *dqcoeff,
                                int block_size) {
762 763 764 765 766 767 768 769
  int i;
  int64_t error = 0;

  for (i = 0; i < block_size; i++) {
    unsigned int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }

Frank Galligan's avatar
Frank Galligan committed
770
  return error > INT_MAX ? INT_MAX : (int)error;
771 772 773 774
}

#define DEBUG_ERROR 0
static void super_block_yrd_32x32(MACROBLOCK *x,
Ronald S. Bultje's avatar
Ronald S. Bultje committed
775 776
                                  int *rate, int *distortion, int *skippable,
                                  int backup) {
777
  MACROBLOCKD *const xd = &x->e_mbd;
778
#if DEBUG_ERROR
779
  int16_t out[1024];
780 781 782 783
#endif

  vp9_transform_sby_32x32(x);
  vp9_quantize_sby_32x32(x);
784
#if DEBUG_ERROR
785
  vp9_short_idct32x32(xd->dqcoeff, out, 64);
786 787
#endif

788
  *distortion = vp9_sb_block_error_c(x->coeff, xd->dqcoeff, 1024);
789

790 791
#if DEBUG_ERROR
  printf("IDCT/FDCT error 32x32: %d (d: %d)\n",
792
         vp9_block_error_c(x->src_diff, out, 1024), *distortion);
793
#endif
Ronald S. Bultje's avatar
Ronald S. Bultje committed
794
  *rate       = rdcost_sby_32x32(x, backup);
795
  *skippable  = vp9_sby_is_skippable_32x32(xd);
796 797
}

798 799
static void super_block_yrd(VP9_COMP *cpi,
                            MACROBLOCK *x, int *rate, int *distortion,
800
                            int *skip,
801
                            int64_t txfm_cache[NB_TXFM_MODES]) {
Ronald S. Bultje's avatar
Ronald S. Bultje committed
802
  MACROBLOCKD *const xd = &x->e_mbd;
803
  int r[TX_SIZE_MAX_SB][2], d[TX_SIZE_MAX_SB], s[TX_SIZE_MAX_SB], n;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
804 805
  const uint8_t *src = x->src.y_buffer, *dst = xd->dst.y_buffer;
  int src_y_stride = x->src.y_stride, dst_y_stride = xd->dst.y_stride;
806 807 808 809
  ENTROPY_CONTEXT_PLANES t_above[TX_SIZE_MAX_MB][2],
                        *orig_above = xd->above_context;
  ENTROPY_CONTEXT_PLANES t_left[TX_SIZE_MAX_MB][2],
                        *orig_left = xd->left_context;
810

811
  for (n = TX_4X4; n < TX_SIZE_MAX_MB; n++) {
812 813
    vpx_memcpy(t_above[n], xd->above_context, sizeof(t_above[n]));
    vpx_memcpy(t_left[n], xd->left_context, sizeof(t_left[n]));
814
    r[n][0] = 0;
815 816 817
    d[n] = 0;
    s[n] = 1;
  }
Ronald S. Bultje's avatar
Ronald S. Bultje committed
818