vp9_rdopt.c 203 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5 6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9 10 11 12 13 14 15
 */


#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
16 17
#include "vp9/common/vp9_pragmas.h"

18 19 20 21 22
#include "vp9/encoder/vp9_tokenize.h"
#include "vp9/encoder/vp9_treewriter.h"
#include "vp9/encoder/vp9_onyx_int.h"
#include "vp9/encoder/vp9_modecosts.h"
#include "vp9/encoder/vp9_encodeintra.h"
23 24 25 26 27
#include "vp9/common/vp9_entropymode.h"
#include "vp9/common/vp9_reconinter.h"
#include "vp9/common/vp9_reconintra.h"
#include "vp9/common/vp9_findnearmv.h"
#include "vp9/common/vp9_quant_common.h"
28 29 30 31 32 33
#include "vp9/encoder/vp9_encodemb.h"
#include "vp9/encoder/vp9_quantize.h"
#include "vp9/encoder/vp9_variance.h"
#include "vp9/encoder/vp9_mcomp.h"
#include "vp9/encoder/vp9_rdopt.h"
#include "vp9/encoder/vp9_ratectrl.h"
John Koleszar's avatar
John Koleszar committed
34
#include "vpx_mem/vpx_mem.h"
35 36
#include "vp9/common/vp9_systemdependent.h"
#include "vp9/encoder/vp9_encodemv.h"
John Koleszar's avatar
John Koleszar committed
37

38 39 40
#include "vp9/common/vp9_seg_common.h"
#include "vp9/common/vp9_pred_common.h"
#include "vp9/common/vp9_entropy.h"
41
#include "vp9_rtcd.h"
42
#include "vp9/common/vp9_mvref_common.h"
Ronald S. Bultje's avatar
Ronald S. Bultje committed
43
#include "vp9/common/vp9_common.h"
Paul Wilkins's avatar
Paul Wilkins committed
44

John Koleszar's avatar
John Koleszar committed
45 46
#define MAXF(a,b)            (((a) > (b)) ? (a) : (b))

47 48
#define INVALID_MV 0x80008000

49 50 51
/* Factor to weigh the rate for switchable interp filters */
#define SWITCHABLE_INTERP_RATE_FACTOR 1

John Koleszar's avatar
John Koleszar committed
52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69
static const int auto_speed_thresh[17] = {
  1000,
  200,
  150,
  130,
  150,
  125,
  120,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  105
John Koleszar's avatar
John Koleszar committed
70 71
};

72
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
73 74
  {ZEROMV,    LAST_FRAME,   NONE},
  {DC_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
75

76 77
  {NEARESTMV, LAST_FRAME,   NONE},
  {NEARMV,    LAST_FRAME,   NONE},
John Koleszar's avatar
John Koleszar committed
78

79 80
  {ZEROMV,    GOLDEN_FRAME, NONE},
  {NEARESTMV, GOLDEN_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
81

82 83
  {ZEROMV,    ALTREF_FRAME, NONE},
  {NEARESTMV, ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
84

85 86
  {NEARMV,    GOLDEN_FRAME, NONE},
  {NEARMV,    ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
87

88 89 90 91 92 93 94 95
  {V_PRED,    INTRA_FRAME,  NONE},
  {H_PRED,    INTRA_FRAME,  NONE},
  {D45_PRED,  INTRA_FRAME,  NONE},
  {D135_PRED, INTRA_FRAME,  NONE},
  {D117_PRED, INTRA_FRAME,  NONE},
  {D153_PRED, INTRA_FRAME,  NONE},
  {D27_PRED,  INTRA_FRAME,  NONE},
  {D63_PRED,  INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
96

97
  {TM_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
98

99 100 101
  {NEWMV,     LAST_FRAME,   NONE},
  {NEWMV,     GOLDEN_FRAME, NONE},
  {NEWMV,     ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
102

103 104 105
  {SPLITMV,   LAST_FRAME,   NONE},
  {SPLITMV,   GOLDEN_FRAME, NONE},
  {SPLITMV,   ALTREF_FRAME, NONE},
106

107 108
  {B_PRED,    INTRA_FRAME,  NONE},
  {I8X8_PRED, INTRA_FRAME,  NONE},
109

John Koleszar's avatar
John Koleszar committed
110 111 112 113
  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME},
114

John Koleszar's avatar
John Koleszar committed
115 116 117
  {ZEROMV,    ALTREF_FRAME, LAST_FRAME},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME},
118

John Koleszar's avatar
John Koleszar committed
119 120 121
  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME},
122

John Koleszar's avatar
John Koleszar committed
123 124 125
  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME  },
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME},
126

John Koleszar's avatar
John Koleszar committed
127 128
  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME  },
129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME},

#if CONFIG_COMP_INTERINTRA_PRED
  /* compound inter-intra prediction */
  {ZEROMV,    LAST_FRAME,   INTRA_FRAME},
  {NEARESTMV, LAST_FRAME,   INTRA_FRAME},
  {NEARMV,    LAST_FRAME,   INTRA_FRAME},
  {NEWMV,     LAST_FRAME,   INTRA_FRAME},

  {ZEROMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEARESTMV, GOLDEN_FRAME,   INTRA_FRAME},
  {NEARMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEWMV,     GOLDEN_FRAME,   INTRA_FRAME},

  {ZEROMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEARESTMV, ALTREF_FRAME,   INTRA_FRAME},
  {NEARMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEWMV,     ALTREF_FRAME,   INTRA_FRAME},
#endif
John Koleszar's avatar
John Koleszar committed
148 149
};

150 151 152
static void fill_token_costs(vp9_coeff_count *c,
                             vp9_coeff_probs *p,
                             int block_type_counts) {
153
  int i, j, k, l;
John Koleszar's avatar
John Koleszar committed
154 155

  for (i = 0; i < block_type_counts; i++)
156 157 158
    for (j = 0; j < REF_TYPES; j++)
      for (k = 0; k < COEF_BANDS; k++)
        for (l = 0; l < PREV_COEF_CONTEXTS; l++) {
159 160 161 162 163 164
#if CONFIG_CODE_NONZEROCOUNT
          // All costs are without the EOB node
          vp9_cost_tokens_skip((int *)(c[i][j][k][l]),
                               p[i][j][k][l],
                               vp9_coef_tree);
#else
165 166 167 168 169 170 171 172
          if (l == 0 && k > 0)
            vp9_cost_tokens_skip((int *)(c[i][j][k][l]),
                                 p[i][j][k][l],
                                 vp9_coef_tree);
          else
            vp9_cost_tokens((int *)(c[i][j][k][l]),
                            p[i][j][k][l],
                            vp9_coef_tree);
173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223
#endif
        }
}

#if CONFIG_CODE_NONZEROCOUNT
static void fill_nzc_costs(VP9_COMP *cpi, int block_size) {
  int nzc_context, r, b, nzc, values;
  int cost[16];
  values = block_size * block_size + 1;

  for (nzc_context = 0; nzc_context < MAX_NZC_CONTEXTS; ++nzc_context) {
    for (r = 0; r < REF_TYPES; ++r) {
      for (b = 0; b < BLOCK_TYPES; ++b) {
        if (block_size == 4)
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_4x4[nzc_context][r][b],
                          vp9_nzc4x4_tree);
        else if (block_size == 8)
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_8x8[nzc_context][r][b],
                          vp9_nzc8x8_tree);
        else if (block_size == 16)
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_16x16[nzc_context][r][b],
                          vp9_nzc16x16_tree);
        else
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_32x32[nzc_context][r][b],
                          vp9_nzc32x32_tree);

        for (nzc = 0; nzc < values; ++nzc) {
          int e, c, totalcost = 0;
          c = codenzc(nzc);
          totalcost = cost[c];
          if ((e = extranzcbits(c))) {
            int x = nzc - basenzcvalue(c);
            while (e--) {
              if ((x >> e) & 1)
                totalcost += vp9_cost_one(Pcat_nzc[nzc_context][c - 3][e]);
              else
                totalcost += vp9_cost_zero(Pcat_nzc[nzc_context][c - 3][e]);
            }
          }
          if (block_size == 4)
            cpi->mb.nzc_costs_4x4[nzc_context][r][b][nzc] = totalcost;
          else if (block_size == 8)
            cpi->mb.nzc_costs_8x8[nzc_context][r][b][nzc] = totalcost;
          else if (block_size == 16)
            cpi->mb.nzc_costs_16x16[nzc_context][r][b][nzc] = totalcost;
          else
            cpi->mb.nzc_costs_32x32[nzc_context][r][b][nzc] = totalcost;
224
        }
225 226 227
      }
    }
  }
John Koleszar's avatar
John Koleszar committed
228
}
229
#endif
John Koleszar's avatar
John Koleszar committed
230

231

232 233 234 235
static int rd_iifactor[32] =  { 4, 4, 3, 2, 1, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0, };
John Koleszar's avatar
John Koleszar committed
236

237
// 3* dc_qlookup[Q]*dc_qlookup[Q];
238

239
/* values are now correlated to quantizer */
Paul Wilkins's avatar
Paul Wilkins committed
240 241 242
static int sad_per_bit16lut[QINDEX_RANGE];
static int sad_per_bit4lut[QINDEX_RANGE];

243
void vp9_init_me_luts() {
John Koleszar's avatar
John Koleszar committed
244 245 246 247 248 249 250
  int i;

  // Initialize the sad lut tables using a formulaic calculation for now
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < QINDEX_RANGE; i++) {
    sad_per_bit16lut[i] =
251 252
      (int)((0.0418 * vp9_convert_qindex_to_q(i)) + 2.4107);
    sad_per_bit4lut[i] = (int)((0.063 * vp9_convert_qindex_to_q(i)) + 2.742);
John Koleszar's avatar
John Koleszar committed
253
  }
Paul Wilkins's avatar
Paul Wilkins committed
254
}
John Koleszar's avatar
John Koleszar committed
255

256
static int compute_rd_mult(int qindex) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
257
  int q = vp9_dc_quant(qindex, 0);
John Koleszar's avatar
John Koleszar committed
258
  return (11 * q * q) >> 6;
259 260
}

Dmitry Kovalev's avatar
Dmitry Kovalev committed
261 262 263
void vp9_initialize_me_consts(VP9_COMP *cpi, int qindex) {
  cpi->mb.sadperbit16 = sad_per_bit16lut[qindex];
  cpi->mb.sadperbit4 = sad_per_bit4lut[qindex];
John Koleszar's avatar
John Koleszar committed
264 265
}

266

Dmitry Kovalev's avatar
Dmitry Kovalev committed
267
void vp9_initialize_rd_consts(VP9_COMP *cpi, int qindex) {
268
  int q, i;
John Koleszar's avatar
John Koleszar committed
269

270
  vp9_clear_system_state();  // __asm emms;
John Koleszar's avatar
John Koleszar committed
271

John Koleszar's avatar
John Koleszar committed
272 273 274 275
  // Further tests required to see if optimum is different
  // for key frames, golden frames and arf frames.
  // if (cpi->common.refresh_golden_frame ||
  //     cpi->common.refresh_alt_ref_frame)
Dmitry Kovalev's avatar
Dmitry Kovalev committed
276
  qindex = (qindex < 0) ? 0 : ((qindex > MAXQ) ? MAXQ : qindex);
277

Dmitry Kovalev's avatar
Dmitry Kovalev committed
278
  cpi->RDMULT = compute_rd_mult(qindex);
John Koleszar's avatar
John Koleszar committed
279

John Koleszar's avatar
John Koleszar committed
280 281 282 283 284
  if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    if (cpi->twopass.next_iiratio > 31)
      cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
    else
      cpi->RDMULT +=
Dmitry Kovalev's avatar
Dmitry Kovalev committed
285
          (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
John Koleszar's avatar
John Koleszar committed
286
  }
John Koleszar's avatar
John Koleszar committed
287

John Koleszar's avatar
John Koleszar committed
288 289
  if (cpi->RDMULT < 7)
    cpi->RDMULT = 7;
290

John Koleszar's avatar
John Koleszar committed
291 292
  cpi->mb.errorperbit = (cpi->RDMULT / 110);
  cpi->mb.errorperbit += (cpi->mb.errorperbit == 0);
293

294
  vp9_set_speed_features(cpi);
John Koleszar's avatar
John Koleszar committed
295

Dmitry Kovalev's avatar
Dmitry Kovalev committed
296 297
  q = (int)pow(vp9_dc_quant(qindex, 0) >> 2, 1.25);
  q <<= 2;
John Koleszar's avatar
John Koleszar committed
298
  cpi->RDMULT = cpi->RDMULT << 4;
Paul Wilkins's avatar
Paul Wilkins committed
299

John Koleszar's avatar
John Koleszar committed
300 301
  if (q < 8)
    q = 8;
302

John Koleszar's avatar
John Koleszar committed
303 304 305
  if (cpi->RDMULT > 1000) {
    cpi->RDDIV = 1;
    cpi->RDMULT /= 100;
John Koleszar's avatar
John Koleszar committed
306

John Koleszar's avatar
John Koleszar committed
307 308 309 310 311 312
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < INT_MAX) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q / 100;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
313

John Koleszar's avatar
John Koleszar committed
314
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
315
    }
John Koleszar's avatar
John Koleszar committed
316 317
  } else {
    cpi->RDDIV = 100;
John Koleszar's avatar
John Koleszar committed
318

John Koleszar's avatar
John Koleszar committed
319 320 321 322 323 324
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < (INT_MAX / q)) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
325

John Koleszar's avatar
John Koleszar committed
326
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
327
    }
John Koleszar's avatar
John Koleszar committed
328
  }
John Koleszar's avatar
John Koleszar committed
329

330
  fill_token_costs(cpi->mb.token_costs[TX_4X4],
331
                   cpi->common.fc.coef_probs_4x4, BLOCK_TYPES);
332
  fill_token_costs(cpi->mb.token_costs[TX_8X8],
333
                   cpi->common.fc.coef_probs_8x8, BLOCK_TYPES);
334
  fill_token_costs(cpi->mb.token_costs[TX_16X16],
335
                   cpi->common.fc.coef_probs_16x16, BLOCK_TYPES);
336
  fill_token_costs(cpi->mb.token_costs[TX_32X32],
337
                   cpi->common.fc.coef_probs_32x32, BLOCK_TYPES);
338 339 340 341 342 343
#if CONFIG_CODE_NONZEROCOUNT
  fill_nzc_costs(cpi, 4);
  fill_nzc_costs(cpi, 8);
  fill_nzc_costs(cpi, 16);
  fill_nzc_costs(cpi, 32);
#endif
344

John Koleszar's avatar
John Koleszar committed
345 346
  /*rough estimate for costing*/
  cpi->common.kf_ymode_probs_index = cpi->common.base_qindex >> 4;
347
  vp9_init_mode_costs(cpi);
John Koleszar's avatar
John Koleszar committed
348

349
  if (cpi->common.frame_type != KEY_FRAME) {
350
    vp9_build_nmv_cost_table(
351 352 353 354 355 356
        cpi->mb.nmvjointcost,
        cpi->mb.e_mbd.allow_high_precision_mv ?
        cpi->mb.nmvcost_hp : cpi->mb.nmvcost,
        &cpi->common.fc.nmvc,
        cpi->mb.e_mbd.allow_high_precision_mv, 1, 1);
  }
John Koleszar's avatar
John Koleszar committed
357 358
}

359
int vp9_block_error_c(int16_t *coeff, int16_t *dqcoeff, int block_size) {
360
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
361

362
  for (i = 0; i < block_size; i++) {
John Koleszar's avatar
John Koleszar committed
363 364 365
    int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }
John Koleszar's avatar
John Koleszar committed
366

John Koleszar's avatar
John Koleszar committed
367
  return error;
John Koleszar's avatar
John Koleszar committed
368 369
}

370
int vp9_mbblock_error_c(MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
371 372 373 374
  BLOCK  *be;
  BLOCKD *bd;
  int i, j;
  int berror, error = 0;
John Koleszar's avatar
John Koleszar committed
375

John Koleszar's avatar
John Koleszar committed
376 377 378 379
  for (i = 0; i < 16; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
    berror = 0;
380
    for (j = 0; j < 16; j++) {
John Koleszar's avatar
John Koleszar committed
381 382
      int this_diff = be->coeff[j] - bd->dqcoeff[j];
      berror += this_diff * this_diff;
John Koleszar's avatar
John Koleszar committed
383
    }
John Koleszar's avatar
John Koleszar committed
384 385 386
    error += berror;
  }
  return error;
John Koleszar's avatar
John Koleszar committed
387 388
}

389
int vp9_mbuverror_c(MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
390 391
  BLOCK  *be;
  BLOCKD *bd;
John Koleszar's avatar
John Koleszar committed
392

393
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
394

John Koleszar's avatar
John Koleszar committed
395 396 397
  for (i = 16; i < 24; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
John Koleszar's avatar
John Koleszar committed
398

399
    error += vp9_block_error_c(be->coeff, bd->dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
400
  }
John Koleszar's avatar
John Koleszar committed
401

John Koleszar's avatar
John Koleszar committed
402
  return error;
John Koleszar's avatar
John Koleszar committed
403 404
}

405
int vp9_uvsse(MACROBLOCK *x) {
406 407 408
  uint8_t *uptr, *vptr;
  uint8_t *upred_ptr = (*(x->block[16].base_src) + x->block[16].src);
  uint8_t *vpred_ptr = (*(x->block[20].base_src) + x->block[20].src);
John Koleszar's avatar
John Koleszar committed
409 410 411 412
  int uv_stride = x->block[16].src_stride;

  unsigned int sse1 = 0;
  unsigned int sse2 = 0;
413 414
  int mv_row = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.row;
  int mv_col = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.col;
John Koleszar's avatar
John Koleszar committed
415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435
  int offset;
  int pre_stride = x->e_mbd.block[16].pre_stride;

  if (mv_row < 0)
    mv_row -= 1;
  else
    mv_row += 1;

  if (mv_col < 0)
    mv_col -= 1;
  else
    mv_col += 1;

  mv_row /= 2;
  mv_col /= 2;

  offset = (mv_row >> 3) * pre_stride + (mv_col >> 3);
  uptr = x->e_mbd.pre.u_buffer + offset;
  vptr = x->e_mbd.pre.v_buffer + offset;

  if ((mv_row | mv_col) & 7) {
436
    vp9_sub_pixel_variance8x8(uptr, pre_stride, (mv_col & 7) << 1,
437
                              (mv_row & 7) << 1, upred_ptr, uv_stride, &sse2);
438
    vp9_sub_pixel_variance8x8(vptr, pre_stride, (mv_col & 7) << 1,
439
                              (mv_row & 7) << 1, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
440 441
    sse2 += sse1;
  } else {
442 443
    vp9_variance8x8(uptr, pre_stride, upred_ptr, uv_stride, &sse2);
    vp9_variance8x8(vptr, pre_stride, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
444 445 446
    sse2 += sse1;
  }
  return sse2;
John Koleszar's avatar
John Koleszar committed
447 448
}

449
static INLINE int cost_coeffs(VP9_COMMON *const cm, MACROBLOCK *mb,
450
                              int ib, PLANE_TYPE type,
451 452 453
                              ENTROPY_CONTEXT *a,
                              ENTROPY_CONTEXT *l,
                              TX_SIZE tx_size) {
454
  MACROBLOCKD *const xd = &mb->e_mbd;
455 456 457
  MB_MODE_INFO *mbmi = &xd->mode_info_context->mbmi;
  const BLOCK_SIZE_TYPE sb_type = mbmi->sb_type;
  int pt;
458
  const int eob = xd->eobs[ib];
459
  int c = 0;
460
  int cost = 0;
461
  const int *scan;
462 463 464 465 466
  const int16_t *qcoeff_ptr = xd->qcoeff + ib * 16;
  const int ref = mbmi->ref_frame != INTRA_FRAME;
  const TX_TYPE tx_type = (sb_type == BLOCK_SIZE_MB16X16 &&
                           type == PLANE_TYPE_Y_WITH_DC) ?
                          get_tx_type(xd, &xd->block[ib]) : DCT_DCT;
467
  unsigned int (*token_costs)[PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS] =
468
      mb->token_costs[tx_size][type][ref];
469
  ENTROPY_CONTEXT a_ec, l_ec;
470 471 472 473
  ENTROPY_CONTEXT *const a1 = a +
      sizeof(ENTROPY_CONTEXT_PLANES)/sizeof(ENTROPY_CONTEXT);
  ENTROPY_CONTEXT *const l1 = l +
      sizeof(ENTROPY_CONTEXT_PLANES)/sizeof(ENTROPY_CONTEXT);
474

475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490
#if CONFIG_CODE_NONZEROCOUNT
  int nzc_context = vp9_get_nzc_context(cm, xd, ib);
  unsigned int *nzc_cost;
#else
  int seg_eob;
  const int segment_id = xd->mode_info_context->mbmi.segment_id;
#endif

  // Check for consistency of tx_size with mode info
  if (type == PLANE_TYPE_Y_WITH_DC) {
    assert(xd->mode_info_context->mbmi.txfm_size == tx_size);
  } else {
    TX_SIZE tx_size_uv = get_uv_tx_size(xd);
    assert(tx_size == tx_size_uv);
  }

491
  switch (tx_size) {
Daniel Kang's avatar
Daniel Kang committed
492
    case TX_4X4:
493 494
      a_ec = *a;
      l_ec = *l;
495
      scan = vp9_default_zig_zag1d_4x4;
496 497 498
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_4x4[nzc_context][ref][type];
#else
499
      seg_eob = 16;
500
#endif
Deb Mukherjee's avatar
Deb Mukherjee committed
501
      if (type == PLANE_TYPE_Y_WITH_DC) {
502 503 504 505
        if (tx_type == ADST_DCT) {
          scan = vp9_row_scan_4x4;
        } else if (tx_type == DCT_ADST) {
          scan = vp9_col_scan_4x4;
506
        }
Daniel Kang's avatar
Daniel Kang committed
507 508 509
      }
      break;
    case TX_8X8:
510 511
      a_ec = (a[0] + a[1]) != 0;
      l_ec = (l[0] + l[1]) != 0;
512
      scan = vp9_default_zig_zag1d_8x8;
513 514 515
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_8x8[nzc_context][ref][type];
#else
516
      seg_eob = 64;
517
#endif
Daniel Kang's avatar
Daniel Kang committed
518 519
      break;
    case TX_16X16:
520
      scan = vp9_default_zig_zag1d_16x16;
521 522 523
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_16x16[nzc_context][ref][type];
#else
524
      seg_eob = 256;
525
#endif
526
      if (type == PLANE_TYPE_UV) {
527 528 529 530 531
        a_ec = (a[0] + a[1] + a1[0] + a1[1]) != 0;
        l_ec = (l[0] + l[1] + l1[0] + l1[1]) != 0;
      } else {
        a_ec = (a[0] + a[1] + a[2] + a[3]) != 0;
        l_ec = (l[0] + l[1] + l[2] + l[3]) != 0;
Deb Mukherjee's avatar
Deb Mukherjee committed
532
      }
Daniel Kang's avatar
Daniel Kang committed
533
      break;
534 535
    case TX_32X32:
      scan = vp9_default_zig_zag1d_32x32;
536 537 538
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_32x32[nzc_context][ref][type];
#else
539
      seg_eob = 1024;
540
#endif
541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556
      if (type == PLANE_TYPE_UV) {
        ENTROPY_CONTEXT *a2, *a3, *l2, *l3;
        a2 = a1 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        a3 = a2 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        l2 = l1 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        l3 = l2 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        a_ec = (a[0] + a[1] + a1[0] + a1[1] +
                a2[0] + a2[1] + a3[0] + a3[1]) != 0;
        l_ec = (l[0] + l[1] + l1[0] + l1[1] +
                l2[0] + l2[1] + l3[0] + l3[1]) != 0;
      } else {
        a_ec = (a[0] + a[1] + a[2] + a[3] +
                a1[0] + a1[1] + a1[2] + a1[3]) != 0;
        l_ec = (l[0] + l[1] + l[2] + l[3] +
                l1[0] + l1[1] + l1[2] + l1[3]) != 0;
      }
557
      break;
Daniel Kang's avatar
Daniel Kang committed
558
    default:
559
      abort();
Daniel Kang's avatar
Daniel Kang committed
560 561
      break;
  }
562

563 564
  VP9_COMBINEENTROPYCONTEXTS(pt, a_ec, l_ec);

565
#if CONFIG_CODE_NONZEROCOUNT == 0
Paul Wilkins's avatar
Paul Wilkins committed
566 567
  if (vp9_segfeature_active(xd, segment_id, SEG_LVL_SKIP))
    seg_eob = 0;
568
#endif
569

570
  {
571
    int recent_energy = 0;
572 573 574
#if CONFIG_CODE_NONZEROCOUNT
    int nzc = 0;
#endif
575 576
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
577
      int t = vp9_dct_value_tokens_ptr[v].Token;
578 579 580
#if CONFIG_CODE_NONZEROCOUNT
      nzc += (v != 0);
#endif
581
      cost += token_costs[get_coef_band(tx_size, c)][pt][t];
582
      cost += vp9_dct_value_cost_ptr[v];
583
      pt = vp9_get_coef_context(&recent_energy, t);
584
    }
585 586 587
#if CONFIG_CODE_NONZEROCOUNT
    cost += nzc_cost[nzc];
#else
588
    if (c < seg_eob)
589
      cost += mb->token_costs[tx_size][type][ref][get_coef_band(tx_size, c)]
590
          [pt][DCT_EOB_TOKEN];
591
#endif
592 593
  }

594
  // is eob first coefficient;
595
  pt = (c > 0);
596
  *a = *l = pt;
597 598 599 600 601 602 603 604 605 606 607 608 609 610
  if (tx_size >= TX_8X8) {
    a[1] = l[1] = pt;
    if (tx_size >= TX_16X16) {
      if (type == PLANE_TYPE_UV) {
        a1[0] = a1[1] = l1[0] = l1[1] = pt;
      } else {
        a[2] = a[3] = l[2] = l[3] = pt;
        if (tx_size >= TX_32X32) {
          a1[0] = a1[1] = a1[2] = a1[3] = pt;
          l1[0] = l1[1] = l1[2] = l1[3] = pt;
        }
      }
    }
  }
611 612 613
  return cost;
}

614
static int rdcost_mby_4x4(VP9_COMMON *const cm, MACROBLOCK *mb, int backup) {
John Koleszar's avatar
John Koleszar committed
615 616
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
617
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
618 619 620
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta;
  ENTROPY_CONTEXT *tl;
621

622 623 624
  if (backup) {
    vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
625

626 627 628 629 630 631
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)xd->above_context;
    tl = (ENTROPY_CONTEXT *)xd->left_context;
  }
John Koleszar's avatar
John Koleszar committed
632

John Koleszar's avatar
John Koleszar committed
633
  for (b = 0; b < 16; b++)
634
    cost += cost_coeffs(cm, mb, b, PLANE_TYPE_Y_WITH_DC,
Yaowu Xu's avatar
Yaowu Xu committed
635 636
                        ta + vp9_block2above[TX_4X4][b],
                        tl + vp9_block2left[TX_4X4][b],
Daniel Kang's avatar
Daniel Kang committed
637
                        TX_4X4);
John Koleszar's avatar
John Koleszar committed
638

John Koleszar's avatar
John Koleszar committed
639
  return cost;
John Koleszar's avatar
John Koleszar committed
640 641
}

642 643
static void macro_block_yrd_4x4(VP9_COMMON *const cm,
                                MACROBLOCK *mb,
644 645
                                int *Rate,
                                int *Distortion,
646
                                int *skippable, int backup) {
Paul Wilkins's avatar
Paul Wilkins committed
647
  MACROBLOCKD *const xd = &mb->e_mbd;
648

649 650 651
  xd->mode_info_context->mbmi.txfm_size = TX_4X4;
  vp9_transform_mby_4x4(mb);
  vp9_quantize_mby_4x4(mb);
652

653
  *Distortion = vp9_mbblock_error(mb) >> 2;
654
  *Rate = rdcost_mby_4x4(cm, mb, backup);
655
  *skippable = vp9_mby_is_skippable_4x4(xd);
656
}
John Koleszar's avatar
John Koleszar committed
657

658
static int rdcost_mby_8x8(VP9_COMMON *const cm, MACROBLOCK *mb, int backup) {
John Koleszar's avatar
John Koleszar committed
659 660
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
661
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
662 663 664 665
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta;
  ENTROPY_CONTEXT *tl;

Ronald S. Bultje's avatar
Ronald S. Bultje committed
666 667 668
  if (backup) {
    vpx_memcpy(&t_above,xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
669

Ronald S. Bultje's avatar
Ronald S. Bultje committed
670 671 672 673 674 675
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)mb->e_mbd.above_context;
    tl = (ENTROPY_CONTEXT *)mb->e_mbd.left_context;
  }
John Koleszar's avatar
John Koleszar committed
676 677

  for (b = 0; b < 16; b += 4)
678
    cost += cost_coeffs(cm, mb, b, PLANE_TYPE_Y_WITH_DC,
Yaowu Xu's avatar
Yaowu Xu committed
679 680
                        ta + vp9_block2above[TX_8X8][b],
                        tl + vp9_block2left[TX_8X8][b],
Daniel Kang's avatar
Daniel Kang committed
681
                        TX_8X8);
John Koleszar's avatar
John Koleszar committed
682 683

  return cost;
684 685
}

686 687
static void macro_block_yrd_8x8(VP9_COMMON *const cm,
                                MACROBLOCK *mb,
John Koleszar's avatar
John Koleszar committed
688 689
                                int *Rate,
                                int *Distortion,
690
                                int *skippable, int backup) {
Paul Wilkins's avatar
Paul Wilkins committed
691
  MACROBLOCKD *const xd = &mb->e_mbd;
692 693

  xd->mode_info_context->mbmi.txfm_size = TX_8X8;
694 695
  vp9_transform_mby_8x8(mb);
  vp9_quantize_mby_8x8(mb);
John Koleszar's avatar
John Koleszar committed
696

697
  *Distortion = vp9_mbblock_error(mb) >> 2;
698
  *Rate = rdcost_mby_8x8(cm, mb, backup);
699
  *skippable = vp9_mby_is_skippable_8x8(xd);
700
}
701

702
static int rdcost_mby_16x16(VP9_COMMON *const cm, MACROBLOCK *mb, int backup) {
Daniel Kang's avatar
Daniel Kang committed
703
  int cost;
Paul Wilkins's avatar
Paul Wilkins committed
704
  MACROBLOCKD *xd = &mb->e_mbd;
Daniel Kang's avatar
Daniel Kang committed
705 706 707
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta, *tl;

708 709 710
  if (backup) {
    vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
Daniel Kang's avatar
Daniel Kang committed
711

712 713 714 715 716 717
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)xd->above_context;
    tl = (ENTROPY_CONTEXT *)xd->left_context;
  }
Daniel Kang's avatar
Daniel Kang committed
718

719
  cost = cost_coeffs(cm, mb, 0, PLANE_TYPE_Y_WITH_DC, ta, tl, TX_16X16);
Daniel Kang's avatar
Daniel Kang committed
720 721
  return cost;
}
722

723 724
static void macro_block_yrd_16x16(VP9_COMMON *const cm, MACROBLOCK *mb,
                                  int *Rate, int *Distortion,
725
                                  int *skippable, int backup) {
Deb Mukherjee's avatar
Deb Mukherjee committed
726
  MACROBLOCKD *xd = &mb->e_mbd;
Daniel Kang's avatar
Daniel Kang committed
727

728 729
  xd->mode_info_context->mbmi.txfm_size = TX_16X16;
  vp9_transform_mby_16x16(mb);
730
  vp9_quantize_mby_16x16(mb);
731 732 733
  // TODO(jingning) is it possible to quickly determine whether to force
  //                trailing coefficients to be zero, instead of running trellis
  //                optimization in the rate-distortion optimization loop?
734 735
  if (mb->optimize &&
      xd->mode_info_context->mbmi.mode < I8X8_PRED)
736
    vp9_optimize_mby_16x16(cm, mb);
737

738
  *Distortion = vp9_mbblock_error(mb) >> 2;
739
  *Rate = rdcost_mby_16x16(cm, mb, backup);
740
  *skippable = vp9_mby_is_skippable_16x16(xd);
Daniel Kang's avatar
Daniel Kang committed
741 742
}

743
static void choose_txfm_size_from_rd(VP9_COMP *cpi, MACROBLOCK *x,
744 745 746 747 748
                                     int (*r)[2], int *rate,
                                     int *d, int *distortion,
                                     int *s, int *skip,
                                     int64_t txfm_cache[NB_TXFM_MODES],
                                     TX_SIZE max_txfm_size) {
749 750 751 752 753
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
  vp9_prob skip_prob = cm->mb_no_coeff_skip ?
                       vp9_get_pred_prob(cm, xd, PRED_MBSKIP) : 128;
754 755 756 757 758 759 760 761 762 763 764 765
  int64_t rd[TX_SIZE_MAX_SB][2];
  int n, m;

  for (n = TX_4X4; n <= max_txfm_size; n++) {
    r[n][1] = r[n][0];
    for (m = 0; m <= n - (n == max_txfm_size); m++) {
      if (m == n)
        r[n][1] += vp9_cost_zero(cm->prob_tx[m]);
      else
        r[n][1] += vp9_cost_one(cm->prob_tx[m]);
    }
  }
766 767 768 769 770 771 772 773

  if (cm->mb_no_coeff_skip) {
    int s0, s1;

    assert(skip_prob > 0);
    s0 = vp9_cost_bit(skip_prob, 0);
    s1 = vp9_cost_bit(skip_prob, 1);

774
    for (n = TX_4X4; n <= max_txfm_size; n++) {
775
      if (s[n]) {
776
        rd[n][0] = rd[n][1] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
777
      } else {
778 779
        rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0] + s0, d[n]);
        rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1] + s0, d[n]);
780
      }
781 782
    }
  } else {
783 784 785
    for (n = TX_4X4; n <= max_txfm_size; n++) {
      rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0], d[n]);
      rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1], d[n]);
786 787 788
    }
  }

789 790 791 792 793 794
  if (max_txfm_size == TX_32X32 &&
      (cm->txfm_mode == ALLOW_32X32 ||
       (cm->txfm_mode == TX_MODE_SELECT &&
        rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
        rd[TX_32X32][1] < rd[TX_4X4][1]))) {
    mbmi->txfm_size = TX_32X32;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
795 796 797 798 799
  } else if ( cm->txfm_mode == ALLOW_16X16 ||
             (max_txfm_size == TX_16X16 && cm->txfm_mode == ALLOW_32X32) ||
             (cm->txfm_mode == TX_MODE_SELECT &&
              rd[TX_16X16][1] < rd[TX_8X8][1] &&
              rd[TX_16X16][1] < rd[TX_4X4][1])) {
800
    mbmi->txfm_size = TX_16X16;
801
  } else if (cm->txfm_mode == ALLOW_8X8 ||
802
           (cm->txfm_mode == TX_MODE_SELECT && rd[TX_8X8][1] < rd[TX_4X4][1])) {
803 804
    mbmi->txfm_size = TX_8X8;
  } else {
805
    assert(cm->txfm_mode == ONLY_4X4 || cm->txfm_mode == TX_MODE_SELECT);
806 807 808
    mbmi->txfm_size = TX_4X4;
  }

809
  *distortion = d[mbmi->txfm_size];
810
  *rate       = r[mbmi->txfm_size][cm->txfm_mode == TX_MODE_SELECT];
811 812
  *skip       = s[mbmi->txfm_size];

813 814 815 816 817 818 819 820
  txfm_cache[ONLY_4X4] = rd[TX_4X4][0];
  txfm_cache[ALLOW_8X8] = rd[TX_8X8][0];
  txfm_cache[ALLOW_16X16] = rd[TX_16X16][0];
  txfm_cache[ALLOW_32X32] = rd[max_txfm_size][0];
  if (max_txfm_size == TX_32X32 &&
      rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
      rd[TX_32X32][1] < rd[TX_4X4][1])
    txfm_cache[TX_MODE_SELECT] = rd[TX_32X32][1];
Ronald S. Bultje's avatar
Ronald S. Bultje committed
821
  else if (rd[TX_16X16][1] < rd[TX_8X8][1] && rd[TX_16X16][1] < rd[TX_4X4][1])
822
    txfm_cache[TX_MODE_SELECT] = rd[TX_16X16][1];
823
  else
824 825
    txfm_cache[TX_MODE_SELECT] = rd[TX_4X4][1] < rd[TX_8X8][1] ?
                                 rd[TX_4X4][1] : rd[TX_8X8][1];
826 827 828 829 830
}

static void macro_block_yrd(VP9_COMP *cpi, MACROBLOCK *x, int *rate,
                            int *distortion, int *skippable,
                            int64_t txfm_cache[NB_TXFM_MODES]) {
831
  VP9_COMMON *const cm = &cpi->common;
832
  MACROBLOCKD *const xd = &x->e_mbd;
833
  int r[TX_SIZE_MAX_MB][2], d[TX_SIZE_MAX_MB], s[TX_SIZE_MAX_MB];
834 835 836

  vp9_subtract_mby(x->src_diff, *(x->block[0].base_src), xd->predictor,
                   x->block[0].src_stride);
837

838 839 840
  macro_block_yrd_16x16(cm, x, &r[TX_16X16][0], &d[TX_16X16], &s[TX_16X16], 1);
  macro_block_yrd_8x8(cm, x, &r[TX_8X8][0], &d[TX_8X8], &s[TX_8X8], 1);
  macro_block_yrd_4x4(cm, x, &r[TX_4X4][0], &d[TX_4X4], &s[TX_4X4], 1);
841 842

  choose_txfm_size_from_rd(cpi, x, r, rate, d, distortion, s, skippable,
843
                           txfm_cache, TX_16X16);
844 845
}

846
static void copy_predictor(uint8_t *dst, const uint8_t *predictor) {
John Koleszar's avatar
John Koleszar committed
847 848 849 850 851 852
  const unsigned int *p = (const unsigned int *)predictor;
  unsigned int *d = (unsigned int *)dst;
  d[0] = p[0];
  d[4] = p[4];
  d[8] = p[8];
  d[12] = p[12];
853
}
Yaowu Xu's avatar
Yaowu Xu committed