vp9_rdopt.c 206 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5 6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9 10 11 12 13 14 15
 */


#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
16 17
#include "vp9/common/vp9_pragmas.h"

18 19 20 21 22
#include "vp9/encoder/vp9_tokenize.h"
#include "vp9/encoder/vp9_treewriter.h"
#include "vp9/encoder/vp9_onyx_int.h"
#include "vp9/encoder/vp9_modecosts.h"
#include "vp9/encoder/vp9_encodeintra.h"
23 24 25 26 27
#include "vp9/common/vp9_entropymode.h"
#include "vp9/common/vp9_reconinter.h"
#include "vp9/common/vp9_reconintra.h"
#include "vp9/common/vp9_findnearmv.h"
#include "vp9/common/vp9_quant_common.h"
28 29 30 31 32 33
#include "vp9/encoder/vp9_encodemb.h"
#include "vp9/encoder/vp9_quantize.h"
#include "vp9/encoder/vp9_variance.h"
#include "vp9/encoder/vp9_mcomp.h"
#include "vp9/encoder/vp9_rdopt.h"
#include "vp9/encoder/vp9_ratectrl.h"
John Koleszar's avatar
John Koleszar committed
34
#include "vpx_mem/vpx_mem.h"
35 36
#include "vp9/common/vp9_systemdependent.h"
#include "vp9/encoder/vp9_encodemv.h"
John Koleszar's avatar
John Koleszar committed
37

38 39 40
#include "vp9/common/vp9_seg_common.h"
#include "vp9/common/vp9_pred_common.h"
#include "vp9/common/vp9_entropy.h"
41
#include "vp9_rtcd.h"
42
#include "vp9/common/vp9_mvref_common.h"
Ronald S. Bultje's avatar
Ronald S. Bultje committed
43
#include "vp9/common/vp9_common.h"
Paul Wilkins's avatar
Paul Wilkins committed
44

John Koleszar's avatar
John Koleszar committed
45 46
#define MAXF(a,b)            (((a) > (b)) ? (a) : (b))

47 48
#define INVALID_MV 0x80008000

49 50 51
/* Factor to weigh the rate for switchable interp filters */
#define SWITCHABLE_INTERP_RATE_FACTOR 1

John Koleszar's avatar
John Koleszar committed
52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69
static const int auto_speed_thresh[17] = {
  1000,
  200,
  150,
  130,
  150,
  125,
  120,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  105
John Koleszar's avatar
John Koleszar committed
70 71
};

72
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
73 74
  {ZEROMV,    LAST_FRAME,   NONE},
  {DC_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
75

76 77
  {NEARESTMV, LAST_FRAME,   NONE},
  {NEARMV,    LAST_FRAME,   NONE},
John Koleszar's avatar
John Koleszar committed
78

79 80
  {ZEROMV,    GOLDEN_FRAME, NONE},
  {NEARESTMV, GOLDEN_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
81

82 83
  {ZEROMV,    ALTREF_FRAME, NONE},
  {NEARESTMV, ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
84

85 86
  {NEARMV,    GOLDEN_FRAME, NONE},
  {NEARMV,    ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
87

88 89 90 91 92 93 94 95
  {V_PRED,    INTRA_FRAME,  NONE},
  {H_PRED,    INTRA_FRAME,  NONE},
  {D45_PRED,  INTRA_FRAME,  NONE},
  {D135_PRED, INTRA_FRAME,  NONE},
  {D117_PRED, INTRA_FRAME,  NONE},
  {D153_PRED, INTRA_FRAME,  NONE},
  {D27_PRED,  INTRA_FRAME,  NONE},
  {D63_PRED,  INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
96

97
  {TM_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
98

99 100 101
  {NEWMV,     LAST_FRAME,   NONE},
  {NEWMV,     GOLDEN_FRAME, NONE},
  {NEWMV,     ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
102

103 104 105
  {SPLITMV,   LAST_FRAME,   NONE},
  {SPLITMV,   GOLDEN_FRAME, NONE},
  {SPLITMV,   ALTREF_FRAME, NONE},
106

107 108
  {B_PRED,    INTRA_FRAME,  NONE},
  {I8X8_PRED, INTRA_FRAME,  NONE},
109

John Koleszar's avatar
John Koleszar committed
110 111 112 113
  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME},
114

John Koleszar's avatar
John Koleszar committed
115 116 117
  {ZEROMV,    ALTREF_FRAME, LAST_FRAME},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME},
118

John Koleszar's avatar
John Koleszar committed
119 120 121
  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME},
122

John Koleszar's avatar
John Koleszar committed
123 124 125
  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME  },
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME},
126

John Koleszar's avatar
John Koleszar committed
127 128
  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME  },
129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME},

#if CONFIG_COMP_INTERINTRA_PRED
  /* compound inter-intra prediction */
  {ZEROMV,    LAST_FRAME,   INTRA_FRAME},
  {NEARESTMV, LAST_FRAME,   INTRA_FRAME},
  {NEARMV,    LAST_FRAME,   INTRA_FRAME},
  {NEWMV,     LAST_FRAME,   INTRA_FRAME},

  {ZEROMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEARESTMV, GOLDEN_FRAME,   INTRA_FRAME},
  {NEARMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEWMV,     GOLDEN_FRAME,   INTRA_FRAME},

  {ZEROMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEARESTMV, ALTREF_FRAME,   INTRA_FRAME},
  {NEARMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEWMV,     ALTREF_FRAME,   INTRA_FRAME},
#endif
John Koleszar's avatar
John Koleszar committed
148 149
};

150 151 152
static void fill_token_costs(vp9_coeff_count *c,
                             vp9_coeff_probs *p,
                             int block_type_counts) {
153
  int i, j, k, l;
John Koleszar's avatar
John Koleszar committed
154 155

  for (i = 0; i < block_type_counts; i++)
156 157 158
    for (j = 0; j < REF_TYPES; j++)
      for (k = 0; k < COEF_BANDS; k++)
        for (l = 0; l < PREV_COEF_CONTEXTS; l++) {
159 160 161 162 163 164
#if CONFIG_CODE_NONZEROCOUNT
          // All costs are without the EOB node
          vp9_cost_tokens_skip((int *)(c[i][j][k][l]),
                               p[i][j][k][l],
                               vp9_coef_tree);
#else
165 166 167 168 169 170 171 172
          if (l == 0 && k > 0)
            vp9_cost_tokens_skip((int *)(c[i][j][k][l]),
                                 p[i][j][k][l],
                                 vp9_coef_tree);
          else
            vp9_cost_tokens((int *)(c[i][j][k][l]),
                            p[i][j][k][l],
                            vp9_coef_tree);
173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223
#endif
        }
}

#if CONFIG_CODE_NONZEROCOUNT
static void fill_nzc_costs(VP9_COMP *cpi, int block_size) {
  int nzc_context, r, b, nzc, values;
  int cost[16];
  values = block_size * block_size + 1;

  for (nzc_context = 0; nzc_context < MAX_NZC_CONTEXTS; ++nzc_context) {
    for (r = 0; r < REF_TYPES; ++r) {
      for (b = 0; b < BLOCK_TYPES; ++b) {
        if (block_size == 4)
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_4x4[nzc_context][r][b],
                          vp9_nzc4x4_tree);
        else if (block_size == 8)
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_8x8[nzc_context][r][b],
                          vp9_nzc8x8_tree);
        else if (block_size == 16)
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_16x16[nzc_context][r][b],
                          vp9_nzc16x16_tree);
        else
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_32x32[nzc_context][r][b],
                          vp9_nzc32x32_tree);

        for (nzc = 0; nzc < values; ++nzc) {
          int e, c, totalcost = 0;
          c = codenzc(nzc);
          totalcost = cost[c];
          if ((e = extranzcbits(c))) {
            int x = nzc - basenzcvalue(c);
            while (e--) {
              if ((x >> e) & 1)
                totalcost += vp9_cost_one(Pcat_nzc[nzc_context][c - 3][e]);
              else
                totalcost += vp9_cost_zero(Pcat_nzc[nzc_context][c - 3][e]);
            }
          }
          if (block_size == 4)
            cpi->mb.nzc_costs_4x4[nzc_context][r][b][nzc] = totalcost;
          else if (block_size == 8)
            cpi->mb.nzc_costs_8x8[nzc_context][r][b][nzc] = totalcost;
          else if (block_size == 16)
            cpi->mb.nzc_costs_16x16[nzc_context][r][b][nzc] = totalcost;
          else
            cpi->mb.nzc_costs_32x32[nzc_context][r][b][nzc] = totalcost;
224
        }
225 226 227
      }
    }
  }
John Koleszar's avatar
John Koleszar committed
228
}
229
#endif
John Koleszar's avatar
John Koleszar committed
230

231

232 233 234 235
static int rd_iifactor[32] =  { 4, 4, 3, 2, 1, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0, };
John Koleszar's avatar
John Koleszar committed
236

237
// 3* dc_qlookup[Q]*dc_qlookup[Q];
238

239
/* values are now correlated to quantizer */
Paul Wilkins's avatar
Paul Wilkins committed
240 241 242
static int sad_per_bit16lut[QINDEX_RANGE];
static int sad_per_bit4lut[QINDEX_RANGE];

243
void vp9_init_me_luts() {
John Koleszar's avatar
John Koleszar committed
244 245 246 247 248 249 250
  int i;

  // Initialize the sad lut tables using a formulaic calculation for now
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < QINDEX_RANGE; i++) {
    sad_per_bit16lut[i] =
251 252
      (int)((0.0418 * vp9_convert_qindex_to_q(i)) + 2.4107);
    sad_per_bit4lut[i] = (int)((0.063 * vp9_convert_qindex_to_q(i)) + 2.742);
John Koleszar's avatar
John Koleszar committed
253
  }
Paul Wilkins's avatar
Paul Wilkins committed
254
}
John Koleszar's avatar
John Koleszar committed
255

256
static int compute_rd_mult(int qindex) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
257
  int q = vp9_dc_quant(qindex, 0);
John Koleszar's avatar
John Koleszar committed
258
  return (11 * q * q) >> 6;
259 260
}

Dmitry Kovalev's avatar
Dmitry Kovalev committed
261 262 263
void vp9_initialize_me_consts(VP9_COMP *cpi, int qindex) {
  cpi->mb.sadperbit16 = sad_per_bit16lut[qindex];
  cpi->mb.sadperbit4 = sad_per_bit4lut[qindex];
John Koleszar's avatar
John Koleszar committed
264 265
}

266

Dmitry Kovalev's avatar
Dmitry Kovalev committed
267
void vp9_initialize_rd_consts(VP9_COMP *cpi, int qindex) {
268
  int q, i;
John Koleszar's avatar
John Koleszar committed
269

270
  vp9_clear_system_state();  // __asm emms;
John Koleszar's avatar
John Koleszar committed
271

John Koleszar's avatar
John Koleszar committed
272 273 274 275
  // Further tests required to see if optimum is different
  // for key frames, golden frames and arf frames.
  // if (cpi->common.refresh_golden_frame ||
  //     cpi->common.refresh_alt_ref_frame)
Dmitry Kovalev's avatar
Dmitry Kovalev committed
276
  qindex = (qindex < 0) ? 0 : ((qindex > MAXQ) ? MAXQ : qindex);
277

Dmitry Kovalev's avatar
Dmitry Kovalev committed
278
  cpi->RDMULT = compute_rd_mult(qindex);
John Koleszar's avatar
John Koleszar committed
279

John Koleszar's avatar
John Koleszar committed
280 281 282 283 284
  if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    if (cpi->twopass.next_iiratio > 31)
      cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
    else
      cpi->RDMULT +=
Dmitry Kovalev's avatar
Dmitry Kovalev committed
285
          (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
John Koleszar's avatar
John Koleszar committed
286
  }
John Koleszar's avatar
John Koleszar committed
287

John Koleszar's avatar
John Koleszar committed
288 289
  if (cpi->RDMULT < 7)
    cpi->RDMULT = 7;
290

John Koleszar's avatar
John Koleszar committed
291 292
  cpi->mb.errorperbit = (cpi->RDMULT / 110);
  cpi->mb.errorperbit += (cpi->mb.errorperbit == 0);
293

294
  vp9_set_speed_features(cpi);
John Koleszar's avatar
John Koleszar committed
295

Dmitry Kovalev's avatar
Dmitry Kovalev committed
296 297
  q = (int)pow(vp9_dc_quant(qindex, 0) >> 2, 1.25);
  q <<= 2;
John Koleszar's avatar
John Koleszar committed
298
  cpi->RDMULT = cpi->RDMULT << 4;
Paul Wilkins's avatar
Paul Wilkins committed
299

John Koleszar's avatar
John Koleszar committed
300 301
  if (q < 8)
    q = 8;
302

John Koleszar's avatar
John Koleszar committed
303 304 305
  if (cpi->RDMULT > 1000) {
    cpi->RDDIV = 1;
    cpi->RDMULT /= 100;
John Koleszar's avatar
John Koleszar committed
306

John Koleszar's avatar
John Koleszar committed
307 308 309 310 311 312
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < INT_MAX) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q / 100;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
313

John Koleszar's avatar
John Koleszar committed
314
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
315
    }
John Koleszar's avatar
John Koleszar committed
316 317
  } else {
    cpi->RDDIV = 100;
John Koleszar's avatar
John Koleszar committed
318

John Koleszar's avatar
John Koleszar committed
319 320 321 322 323 324
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < (INT_MAX / q)) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
325

John Koleszar's avatar
John Koleszar committed
326
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
327
    }
John Koleszar's avatar
John Koleszar committed
328
  }
John Koleszar's avatar
John Koleszar committed
329

330
  fill_token_costs(cpi->mb.token_costs[TX_4X4],
331
                   cpi->common.fc.coef_probs_4x4, BLOCK_TYPES);
332
  fill_token_costs(cpi->mb.token_costs[TX_8X8],
333
                   cpi->common.fc.coef_probs_8x8, BLOCK_TYPES);
334
  fill_token_costs(cpi->mb.token_costs[TX_16X16],
335
                   cpi->common.fc.coef_probs_16x16, BLOCK_TYPES);
336
  fill_token_costs(cpi->mb.token_costs[TX_32X32],
337
                   cpi->common.fc.coef_probs_32x32, BLOCK_TYPES);
338 339 340 341 342 343
#if CONFIG_CODE_NONZEROCOUNT
  fill_nzc_costs(cpi, 4);
  fill_nzc_costs(cpi, 8);
  fill_nzc_costs(cpi, 16);
  fill_nzc_costs(cpi, 32);
#endif
344

John Koleszar's avatar
John Koleszar committed
345 346
  /*rough estimate for costing*/
  cpi->common.kf_ymode_probs_index = cpi->common.base_qindex >> 4;
347
  vp9_init_mode_costs(cpi);
John Koleszar's avatar
John Koleszar committed
348

349
  if (cpi->common.frame_type != KEY_FRAME) {
350
    vp9_build_nmv_cost_table(
351 352 353 354 355 356
        cpi->mb.nmvjointcost,
        cpi->mb.e_mbd.allow_high_precision_mv ?
        cpi->mb.nmvcost_hp : cpi->mb.nmvcost,
        &cpi->common.fc.nmvc,
        cpi->mb.e_mbd.allow_high_precision_mv, 1, 1);
  }
John Koleszar's avatar
John Koleszar committed
357 358
}

359
int vp9_block_error_c(int16_t *coeff, int16_t *dqcoeff, int block_size) {
360
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
361

362
  for (i = 0; i < block_size; i++) {
John Koleszar's avatar
John Koleszar committed
363 364 365
    int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }
John Koleszar's avatar
John Koleszar committed
366

John Koleszar's avatar
John Koleszar committed
367
  return error;
John Koleszar's avatar
John Koleszar committed
368 369
}

370
int vp9_mbblock_error_c(MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
371 372 373 374
  BLOCK  *be;
  BLOCKD *bd;
  int i, j;
  int berror, error = 0;
John Koleszar's avatar
John Koleszar committed
375

John Koleszar's avatar
John Koleszar committed
376 377 378 379
  for (i = 0; i < 16; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
    berror = 0;
380
    for (j = 0; j < 16; j++) {
John Koleszar's avatar
John Koleszar committed
381 382
      int this_diff = be->coeff[j] - bd->dqcoeff[j];
      berror += this_diff * this_diff;
John Koleszar's avatar
John Koleszar committed
383
    }
John Koleszar's avatar
John Koleszar committed
384 385 386
    error += berror;
  }
  return error;
John Koleszar's avatar
John Koleszar committed
387 388
}

389
int vp9_mbuverror_c(MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
390 391
  BLOCK  *be;
  BLOCKD *bd;
John Koleszar's avatar
John Koleszar committed
392

393
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
394

John Koleszar's avatar
John Koleszar committed
395 396 397
  for (i = 16; i < 24; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
John Koleszar's avatar
John Koleszar committed
398

399
    error += vp9_block_error_c(be->coeff, bd->dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
400
  }
John Koleszar's avatar
John Koleszar committed
401

John Koleszar's avatar
John Koleszar committed
402
  return error;
John Koleszar's avatar
John Koleszar committed
403 404
}

405
int vp9_uvsse(MACROBLOCK *x) {
406 407 408
  uint8_t *uptr, *vptr;
  uint8_t *upred_ptr = (*(x->block[16].base_src) + x->block[16].src);
  uint8_t *vpred_ptr = (*(x->block[20].base_src) + x->block[20].src);
John Koleszar's avatar
John Koleszar committed
409 410 411 412
  int uv_stride = x->block[16].src_stride;

  unsigned int sse1 = 0;
  unsigned int sse2 = 0;
413 414
  int mv_row = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.row;
  int mv_col = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.col;
John Koleszar's avatar
John Koleszar committed
415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435
  int offset;
  int pre_stride = x->e_mbd.block[16].pre_stride;

  if (mv_row < 0)
    mv_row -= 1;
  else
    mv_row += 1;

  if (mv_col < 0)
    mv_col -= 1;
  else
    mv_col += 1;

  mv_row /= 2;
  mv_col /= 2;

  offset = (mv_row >> 3) * pre_stride + (mv_col >> 3);
  uptr = x->e_mbd.pre.u_buffer + offset;
  vptr = x->e_mbd.pre.v_buffer + offset;

  if ((mv_row | mv_col) & 7) {
436
    vp9_sub_pixel_variance8x8(uptr, pre_stride, (mv_col & 7) << 1,
437
                              (mv_row & 7) << 1, upred_ptr, uv_stride, &sse2);
438
    vp9_sub_pixel_variance8x8(vptr, pre_stride, (mv_col & 7) << 1,
439
                              (mv_row & 7) << 1, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
440 441
    sse2 += sse1;
  } else {
442 443
    vp9_variance8x8(uptr, pre_stride, upred_ptr, uv_stride, &sse2);
    vp9_variance8x8(vptr, pre_stride, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
444 445 446
    sse2 += sse1;
  }
  return sse2;
John Koleszar's avatar
John Koleszar committed
447 448
}

449
static INLINE int cost_coeffs(VP9_COMMON *const cm, MACROBLOCK *mb,
450
                              int ib, PLANE_TYPE type,
451 452 453
                              ENTROPY_CONTEXT *a,
                              ENTROPY_CONTEXT *l,
                              TX_SIZE tx_size) {
454
  MACROBLOCKD *const xd = &mb->e_mbd;
455 456
  MB_MODE_INFO *mbmi = &xd->mode_info_context->mbmi;
  int pt;
457
  const int eob = xd->eobs[ib];
458
  int c = 0;
459
  int cost = 0;
460
  const int *scan;
461 462
  const int16_t *qcoeff_ptr = xd->qcoeff + ib * 16;
  const int ref = mbmi->ref_frame != INTRA_FRAME;
463
  unsigned int (*token_costs)[PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS] =
464
      mb->token_costs[tx_size][type][ref];
465
  ENTROPY_CONTEXT a_ec, l_ec;
466 467 468 469
  ENTROPY_CONTEXT *const a1 = a +
      sizeof(ENTROPY_CONTEXT_PLANES)/sizeof(ENTROPY_CONTEXT);
  ENTROPY_CONTEXT *const l1 = l +
      sizeof(ENTROPY_CONTEXT_PLANES)/sizeof(ENTROPY_CONTEXT);
470

471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486
#if CONFIG_CODE_NONZEROCOUNT
  int nzc_context = vp9_get_nzc_context(cm, xd, ib);
  unsigned int *nzc_cost;
#else
  int seg_eob;
  const int segment_id = xd->mode_info_context->mbmi.segment_id;
#endif

  // Check for consistency of tx_size with mode info
  if (type == PLANE_TYPE_Y_WITH_DC) {
    assert(xd->mode_info_context->mbmi.txfm_size == tx_size);
  } else {
    TX_SIZE tx_size_uv = get_uv_tx_size(xd);
    assert(tx_size == tx_size_uv);
  }

487
  switch (tx_size) {
488 489 490
    case TX_4X4: {
      const TX_TYPE tx_type = (type == PLANE_TYPE_Y_WITH_DC) ?
                              get_tx_type_4x4(xd, ib) : DCT_DCT;
491 492
      a_ec = *a;
      l_ec = *l;
493 494 495
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_4x4[nzc_context][ref][type];
#else
496
      seg_eob = 16;
497
#endif
498 499 500 501 502 503
      if (tx_type == ADST_DCT) {
        scan = vp9_row_scan_4x4;
      } else if (tx_type == DCT_ADST) {
        scan = vp9_col_scan_4x4;
      } else {
        scan = vp9_default_zig_zag1d_4x4;
Daniel Kang's avatar
Daniel Kang committed
504 505
      }
      break;
506
    }
Daniel Kang's avatar
Daniel Kang committed
507
    case TX_8X8:
508 509
      a_ec = (a[0] + a[1]) != 0;
      l_ec = (l[0] + l[1]) != 0;
510
      scan = vp9_default_zig_zag1d_8x8;
511 512 513
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_8x8[nzc_context][ref][type];
#else
514
      seg_eob = 64;
515
#endif
Daniel Kang's avatar
Daniel Kang committed
516 517
      break;
    case TX_16X16:
518
      scan = vp9_default_zig_zag1d_16x16;
519 520 521
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_16x16[nzc_context][ref][type];
#else
522
      seg_eob = 256;
523
#endif
524
      if (type == PLANE_TYPE_UV) {
525 526 527 528 529
        a_ec = (a[0] + a[1] + a1[0] + a1[1]) != 0;
        l_ec = (l[0] + l[1] + l1[0] + l1[1]) != 0;
      } else {
        a_ec = (a[0] + a[1] + a[2] + a[3]) != 0;
        l_ec = (l[0] + l[1] + l[2] + l[3]) != 0;
Deb Mukherjee's avatar
Deb Mukherjee committed
530
      }
Daniel Kang's avatar
Daniel Kang committed
531
      break;
532 533
    case TX_32X32:
      scan = vp9_default_zig_zag1d_32x32;
534 535 536
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_32x32[nzc_context][ref][type];
#else
537
      seg_eob = 1024;
538
#endif
539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554
      if (type == PLANE_TYPE_UV) {
        ENTROPY_CONTEXT *a2, *a3, *l2, *l3;
        a2 = a1 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        a3 = a2 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        l2 = l1 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        l3 = l2 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        a_ec = (a[0] + a[1] + a1[0] + a1[1] +
                a2[0] + a2[1] + a3[0] + a3[1]) != 0;
        l_ec = (l[0] + l[1] + l1[0] + l1[1] +
                l2[0] + l2[1] + l3[0] + l3[1]) != 0;
      } else {
        a_ec = (a[0] + a[1] + a[2] + a[3] +
                a1[0] + a1[1] + a1[2] + a1[3]) != 0;
        l_ec = (l[0] + l[1] + l[2] + l[3] +
                l1[0] + l1[1] + l1[2] + l1[3]) != 0;
      }
555
      break;
Daniel Kang's avatar
Daniel Kang committed
556
    default:
557
      abort();
Daniel Kang's avatar
Daniel Kang committed
558 559
      break;
  }
560

561 562
  VP9_COMBINEENTROPYCONTEXTS(pt, a_ec, l_ec);

563
#if CONFIG_CODE_NONZEROCOUNT == 0
Paul Wilkins's avatar
Paul Wilkins committed
564 565
  if (vp9_segfeature_active(xd, segment_id, SEG_LVL_SKIP))
    seg_eob = 0;
566
#endif
567

568
  {
569
    int recent_energy = 0;
570 571 572
#if CONFIG_CODE_NONZEROCOUNT
    int nzc = 0;
#endif
573 574
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
575
      int t = vp9_dct_value_tokens_ptr[v].Token;
576 577 578
#if CONFIG_CODE_NONZEROCOUNT
      nzc += (v != 0);
#endif
579
      cost += token_costs[get_coef_band(tx_size, c)][pt][t];
580
      cost += vp9_dct_value_cost_ptr[v];
581
      pt = vp9_get_coef_context(&recent_energy, t);
582
    }
583 584 585
#if CONFIG_CODE_NONZEROCOUNT
    cost += nzc_cost[nzc];
#else
586
    if (c < seg_eob)
587
      cost += mb->token_costs[tx_size][type][ref][get_coef_band(tx_size, c)]
588
          [pt][DCT_EOB_TOKEN];
589
#endif
590 591
  }

592
  // is eob first coefficient;
593
  pt = (c > 0);
594
  *a = *l = pt;
595 596 597 598 599 600 601 602 603 604 605 606 607 608
  if (tx_size >= TX_8X8) {
    a[1] = l[1] = pt;
    if (tx_size >= TX_16X16) {
      if (type == PLANE_TYPE_UV) {
        a1[0] = a1[1] = l1[0] = l1[1] = pt;
      } else {
        a[2] = a[3] = l[2] = l[3] = pt;
        if (tx_size >= TX_32X32) {
          a1[0] = a1[1] = a1[2] = a1[3] = pt;
          l1[0] = l1[1] = l1[2] = l1[3] = pt;
        }
      }
    }
  }
609 610 611
  return cost;
}

612
static int rdcost_mby_4x4(VP9_COMMON *const cm, MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
613 614
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
615
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
616
  ENTROPY_CONTEXT_PLANES t_above, t_left;
617 618
  ENTROPY_CONTEXT *ta = (ENTROPY_CONTEXT *)&t_above;
  ENTROPY_CONTEXT *tl = (ENTROPY_CONTEXT *)&t_left;
John Koleszar's avatar
John Koleszar committed
619

620 621
  vpx_memcpy(&t_above, xd->above_context, sizeof(t_above));
  vpx_memcpy(&t_left, xd->left_context, sizeof(t_left));
John Koleszar's avatar
John Koleszar committed
622

John Koleszar's avatar
John Koleszar committed
623
  for (b = 0; b < 16; b++)
624
    cost += cost_coeffs(cm, mb, b, PLANE_TYPE_Y_WITH_DC,
Yaowu Xu's avatar
Yaowu Xu committed
625 626
                        ta + vp9_block2above[TX_4X4][b],
                        tl + vp9_block2left[TX_4X4][b],
Daniel Kang's avatar
Daniel Kang committed
627
                        TX_4X4);
John Koleszar's avatar
John Koleszar committed
628

John Koleszar's avatar
John Koleszar committed
629
  return cost;
John Koleszar's avatar
John Koleszar committed
630 631
}

632 633
static void macro_block_yrd_4x4(VP9_COMMON *const cm,
                                MACROBLOCK *mb,
634 635 636
                                int *rate,
                                int *distortion,
                                int *skippable) {
Paul Wilkins's avatar
Paul Wilkins committed
637
  MACROBLOCKD *const xd = &mb->e_mbd;
638

639 640 641
  xd->mode_info_context->mbmi.txfm_size = TX_4X4;
  vp9_transform_mby_4x4(mb);
  vp9_quantize_mby_4x4(mb);
642

643 644
  *distortion = vp9_mbblock_error(mb) >> 2;
  *rate = rdcost_mby_4x4(cm, mb);
645
  *skippable = vp9_mby_is_skippable_4x4(xd);
646
}
John Koleszar's avatar
John Koleszar committed
647

648
static int rdcost_mby_8x8(VP9_COMMON *const cm, MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
649 650
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
651
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
652
  ENTROPY_CONTEXT_PLANES t_above, t_left;
653 654
  ENTROPY_CONTEXT *ta = (ENTROPY_CONTEXT *)&t_above;
  ENTROPY_CONTEXT *tl = (ENTROPY_CONTEXT *)&t_left;
John Koleszar's avatar
John Koleszar committed
655

656 657
  vpx_memcpy(&t_above, xd->above_context, sizeof(t_above));
  vpx_memcpy(&t_left,  xd->left_context, sizeof(t_left));
John Koleszar's avatar
John Koleszar committed
658 659

  for (b = 0; b < 16; b += 4)
660
    cost += cost_coeffs(cm, mb, b, PLANE_TYPE_Y_WITH_DC,
Yaowu Xu's avatar
Yaowu Xu committed
661 662
                        ta + vp9_block2above[TX_8X8][b],
                        tl + vp9_block2left[TX_8X8][b],
Daniel Kang's avatar
Daniel Kang committed
663
                        TX_8X8);
John Koleszar's avatar
John Koleszar committed
664 665

  return cost;
666 667
}

668 669
static void macro_block_yrd_8x8(VP9_COMMON *const cm,
                                MACROBLOCK *mb,
670 671 672
                                int *rate,
                                int *distortion,
                                int *skippable) {
Paul Wilkins's avatar
Paul Wilkins committed
673
  MACROBLOCKD *const xd = &mb->e_mbd;
674 675

  xd->mode_info_context->mbmi.txfm_size = TX_8X8;
676 677
  vp9_transform_mby_8x8(mb);
  vp9_quantize_mby_8x8(mb);
John Koleszar's avatar
John Koleszar committed
678

679 680
  *distortion = vp9_mbblock_error(mb) >> 2;
  *rate = rdcost_mby_8x8(cm, mb);
681
  *skippable = vp9_mby_is_skippable_8x8(xd);
682
}
683

684 685
static int rdcost_mby_16x16(VP9_COMMON *const cm, MACROBLOCK *mb) {
  MACROBLOCKD *const xd = &mb->e_mbd;
Daniel Kang's avatar
Daniel Kang committed
686
  ENTROPY_CONTEXT_PLANES t_above, t_left;
687 688
  ENTROPY_CONTEXT *ta = (ENTROPY_CONTEXT *)&t_above;
  ENTROPY_CONTEXT *tl = (ENTROPY_CONTEXT *)&t_left;
Daniel Kang's avatar
Daniel Kang committed
689

690 691
  vpx_memcpy(&t_above, xd->above_context, sizeof(t_above));
  vpx_memcpy(&t_left, xd->left_context, sizeof(t_left));
Daniel Kang's avatar
Daniel Kang committed
692

693
  return cost_coeffs(cm, mb, 0, PLANE_TYPE_Y_WITH_DC, ta, tl, TX_16X16);
Daniel Kang's avatar
Daniel Kang committed
694
}
695

696
static void macro_block_yrd_16x16(VP9_COMMON *const cm, MACROBLOCK *mb,
697 698
                                  int *rate, int *distortion, int *skippable) {
  MACROBLOCKD *const xd = &mb->e_mbd;
Daniel Kang's avatar
Daniel Kang committed
699

700 701
  xd->mode_info_context->mbmi.txfm_size = TX_16X16;
  vp9_transform_mby_16x16(mb);
702
  vp9_quantize_mby_16x16(mb);
703 704 705
  // TODO(jingning) is it possible to quickly determine whether to force
  //                trailing coefficients to be zero, instead of running trellis
  //                optimization in the rate-distortion optimization loop?
706 707
  if (mb->optimize &&
      xd->mode_info_context->mbmi.mode < I8X8_PRED)
708
    vp9_optimize_mby_16x16(cm, mb);
709

710 711
  *distortion = vp9_mbblock_error(mb) >> 2;
  *rate = rdcost_mby_16x16(cm, mb);
712
  *skippable = vp9_mby_is_skippable_16x16(xd);
Daniel Kang's avatar
Daniel Kang committed
713 714
}

715
static void choose_txfm_size_from_rd(VP9_COMP *cpi, MACROBLOCK *x,
716 717 718 719 720
                                     int (*r)[2], int *rate,
                                     int *d, int *distortion,
                                     int *s, int *skip,
                                     int64_t txfm_cache[NB_TXFM_MODES],
                                     TX_SIZE max_txfm_size) {
721 722 723 724 725
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
  vp9_prob skip_prob = cm->mb_no_coeff_skip ?
                       vp9_get_pred_prob(cm, xd, PRED_MBSKIP) : 128;
726 727 728 729 730 731 732 733 734 735 736 737
  int64_t rd[TX_SIZE_MAX_SB][2];
  int n, m;

  for (n = TX_4X4; n <= max_txfm_size; n++) {
    r[n][1] = r[n][0];
    for (m = 0; m <= n - (n == max_txfm_size); m++) {
      if (m == n)
        r[n][1] += vp9_cost_zero(cm->prob_tx[m]);
      else
        r[n][1] += vp9_cost_one(cm->prob_tx[m]);
    }
  }
738 739 740 741 742 743 744 745

  if (cm->mb_no_coeff_skip) {
    int s0, s1;

    assert(skip_prob > 0);
    s0 = vp9_cost_bit(skip_prob, 0);
    s1 = vp9_cost_bit(skip_prob, 1);

746
    for (n = TX_4X4; n <= max_txfm_size; n++) {
747
      if (s[n]) {
748
        rd[n][0] = rd[n][1] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
749
      } else {
750 751
        rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0] + s0, d[n]);
        rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1] + s0, d[n]);
752
      }
753 754
    }
  } else {
755 756 757
    for (n = TX_4X4; n <= max_txfm_size; n++) {
      rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0], d[n]);
      rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1], d[n]);
758 759 760
    }
  }

761 762 763 764 765 766
  if (max_txfm_size == TX_32X32 &&
      (cm->txfm_mode == ALLOW_32X32 ||
       (cm->txfm_mode == TX_MODE_SELECT &&
        rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
        rd[TX_32X32][1] < rd[TX_4X4][1]))) {
    mbmi->txfm_size = TX_32X32;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
767 768 769 770 771
  } else if ( cm->txfm_mode == ALLOW_16X16 ||
             (max_txfm_size == TX_16X16 && cm->txfm_mode == ALLOW_32X32) ||
             (cm->txfm_mode == TX_MODE_SELECT &&
              rd[TX_16X16][1] < rd[TX_8X8][1] &&
              rd[TX_16X16][1] < rd[TX_4X4][1])) {
772
    mbmi->txfm_size = TX_16X16;
773
  } else if (cm->txfm_mode == ALLOW_8X8 ||
774
           (cm->txfm_mode == TX_MODE_SELECT && rd[TX_8X8][1] < rd[TX_4X4][1])) {
775 776
    mbmi->txfm_size = TX_8X8;
  } else {
777
    assert(cm->txfm_mode == ONLY_4X4 || cm->txfm_mode == TX_MODE_SELECT);
778 779 780
    mbmi->txfm_size = TX_4X4;
  }

781
  *distortion = d[mbmi->txfm_size];
782
  *rate       = r[mbmi->txfm_size][cm->txfm_mode == TX_MODE_SELECT];
783 784
  *skip       = s[mbmi->txfm_size];

785 786 787 788 789 790 791 792
  txfm_cache[ONLY_4X4] = rd[TX_4X4][0];
  txfm_cache[ALLOW_8X8] = rd[TX_8X8][0];
  txfm_cache[ALLOW_16X16] = rd[TX_16X16][0];
  txfm_cache[ALLOW_32X32] = rd[max_txfm_size][0];
  if (max_txfm_size == TX_32X32 &&
      rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
      rd[TX_32X32][1] < rd[TX_4X4][1])
    txfm_cache[TX_MODE_SELECT] = rd[TX_32X32][1];
Ronald S. Bultje's avatar
Ronald S. Bultje committed
793
  else if (rd[TX_16X16][1] < rd[TX_8X8][1] && rd[TX_16X16][1] < rd[TX_4X4][1])
794
    txfm_cache[TX_MODE_SELECT] = rd[TX_16X16][1];
795
  else
796 797
    txfm_cache[TX_MODE_SELECT] = rd[TX_4X4][1] < rd[TX_8X8][1] ?
                                 rd[TX_4X4][1] : rd[TX_8X8][1];
798 799 800 801 802
}

static void macro_block_yrd(VP9_COMP *cpi, MACROBLOCK *x, int *rate,
                            int *distortion, int *skippable,
                            int64_t txfm_cache[NB_TXFM_MODES]) {
803
  VP9_COMMON *const cm = &cpi->common;
804
  MACROBLOCKD *const xd = &x->e_mbd;
805
  int r[TX_SIZE_MAX_MB][2], d[TX_SIZE_MAX_MB], s[TX_SIZE_MAX_MB];
806 807 808

  vp9_subtract_mby(x->src_diff, *(x->block[0].base_src), xd->predictor,
                   x->block[0].src_stride);
809

810 811 812
  macro_block_yrd_16x16(cm, x, &r[TX_16X16][0], &d[TX_16X16], &s[TX_16X16]);
  macro_block_yrd_8x8(cm, x, &r[TX_8X8][0], &d[TX_8X8], &s[TX_8X8]);
  macro_block_yrd_4x4(cm, x, &r[TX_4X4][0], &d[TX_4X4], &s[TX_4X4]);
813 814

  choose_txfm_size_from_rd(cpi, x, r, rate, d, distortion, s, skippable,
815
                           txfm_cache, TX_16X16);
816 817
}

818
static void copy_predictor(uint8_t *dst, const uint8_t *predictor) {
John Koleszar's avatar
John Koleszar committed
819 820 821 822 823 824
  const unsigned int *p = (const unsigned int *)predictor;
  unsigned int *d = (unsigned int *)dst;
  d[0] = p[0];
  d[4] = p[4];
  d[8] = p[8];
  d[12] = p[12];
825
}
Yaowu Xu's avatar
Yaowu Xu committed
826

827
static int vp9_sb_block_error_c(int16_t *coeff, int16_t *dqcoeff,
828
                                int block_size, int shift) {
829 830 831 832 833 834 835
  int i;
  int64_t error = 0;

  for (i = 0; i < block_size; i++) {
    unsigned int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }
836
  error >>= shift;
837

Frank Galligan's avatar
Frank Galligan committed
838
  return error > INT_MAX ? INT_MAX : (int)error;
839 840
}

841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873