vp9_rdopt.c 203 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5
6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9
10
11
12
13
14
15
 */


#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
16
17
#include "vp9/common/vp9_pragmas.h"

18
19
20
21
22
#include "vp9/encoder/vp9_tokenize.h"
#include "vp9/encoder/vp9_treewriter.h"
#include "vp9/encoder/vp9_onyx_int.h"
#include "vp9/encoder/vp9_modecosts.h"
#include "vp9/encoder/vp9_encodeintra.h"
23
24
25
26
27
#include "vp9/common/vp9_entropymode.h"
#include "vp9/common/vp9_reconinter.h"
#include "vp9/common/vp9_reconintra.h"
#include "vp9/common/vp9_findnearmv.h"
#include "vp9/common/vp9_quant_common.h"
28
29
30
31
32
33
#include "vp9/encoder/vp9_encodemb.h"
#include "vp9/encoder/vp9_quantize.h"
#include "vp9/encoder/vp9_variance.h"
#include "vp9/encoder/vp9_mcomp.h"
#include "vp9/encoder/vp9_rdopt.h"
#include "vp9/encoder/vp9_ratectrl.h"
John Koleszar's avatar
John Koleszar committed
34
#include "vpx_mem/vpx_mem.h"
35
36
#include "vp9/common/vp9_systemdependent.h"
#include "vp9/encoder/vp9_encodemv.h"
John Koleszar's avatar
John Koleszar committed
37

38
39
40
#include "vp9/common/vp9_seg_common.h"
#include "vp9/common/vp9_pred_common.h"
#include "vp9/common/vp9_entropy.h"
41
#include "vp9_rtcd.h"
42
#include "vp9/common/vp9_mvref_common.h"
Ronald S. Bultje's avatar
Ronald S. Bultje committed
43
#include "vp9/common/vp9_common.h"
Paul Wilkins's avatar
Paul Wilkins committed
44

John Koleszar's avatar
John Koleszar committed
45
46
#define MAXF(a,b)            (((a) > (b)) ? (a) : (b))

47
48
#define INVALID_MV 0x80008000

49
50
51
/* Factor to weigh the rate for switchable interp filters */
#define SWITCHABLE_INTERP_RATE_FACTOR 1

John Koleszar's avatar
John Koleszar committed
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
static const int auto_speed_thresh[17] = {
  1000,
  200,
  150,
  130,
  150,
  125,
  120,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  105
John Koleszar's avatar
John Koleszar committed
70
71
};

72
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
73
74
  {ZEROMV,    LAST_FRAME,   NONE},
  {DC_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
75

76
77
  {NEARESTMV, LAST_FRAME,   NONE},
  {NEARMV,    LAST_FRAME,   NONE},
John Koleszar's avatar
John Koleszar committed
78

79
80
  {ZEROMV,    GOLDEN_FRAME, NONE},
  {NEARESTMV, GOLDEN_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
81

82
83
  {ZEROMV,    ALTREF_FRAME, NONE},
  {NEARESTMV, ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
84

85
86
  {NEARMV,    GOLDEN_FRAME, NONE},
  {NEARMV,    ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
87

88
89
90
91
92
93
94
95
  {V_PRED,    INTRA_FRAME,  NONE},
  {H_PRED,    INTRA_FRAME,  NONE},
  {D45_PRED,  INTRA_FRAME,  NONE},
  {D135_PRED, INTRA_FRAME,  NONE},
  {D117_PRED, INTRA_FRAME,  NONE},
  {D153_PRED, INTRA_FRAME,  NONE},
  {D27_PRED,  INTRA_FRAME,  NONE},
  {D63_PRED,  INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
96

97
  {TM_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
98

99
100
101
  {NEWMV,     LAST_FRAME,   NONE},
  {NEWMV,     GOLDEN_FRAME, NONE},
  {NEWMV,     ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
102

103
104
105
  {SPLITMV,   LAST_FRAME,   NONE},
  {SPLITMV,   GOLDEN_FRAME, NONE},
  {SPLITMV,   ALTREF_FRAME, NONE},
106

107
108
  {B_PRED,    INTRA_FRAME,  NONE},
  {I8X8_PRED, INTRA_FRAME,  NONE},
109

John Koleszar's avatar
John Koleszar committed
110
111
112
113
  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME},
114

John Koleszar's avatar
John Koleszar committed
115
116
117
  {ZEROMV,    ALTREF_FRAME, LAST_FRAME},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME},
118

John Koleszar's avatar
John Koleszar committed
119
120
121
  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME},
122

John Koleszar's avatar
John Koleszar committed
123
124
125
  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME  },
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME},
126

John Koleszar's avatar
John Koleszar committed
127
128
  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME  },
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME},

#if CONFIG_COMP_INTERINTRA_PRED
  /* compound inter-intra prediction */
  {ZEROMV,    LAST_FRAME,   INTRA_FRAME},
  {NEARESTMV, LAST_FRAME,   INTRA_FRAME},
  {NEARMV,    LAST_FRAME,   INTRA_FRAME},
  {NEWMV,     LAST_FRAME,   INTRA_FRAME},

  {ZEROMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEARESTMV, GOLDEN_FRAME,   INTRA_FRAME},
  {NEARMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEWMV,     GOLDEN_FRAME,   INTRA_FRAME},

  {ZEROMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEARESTMV, ALTREF_FRAME,   INTRA_FRAME},
  {NEARMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEWMV,     ALTREF_FRAME,   INTRA_FRAME},
#endif
John Koleszar's avatar
John Koleszar committed
148
149
};

150
151
152
static void fill_token_costs(vp9_coeff_count *c,
                             vp9_coeff_probs *p,
                             int block_type_counts) {
153
  int i, j, k, l;
John Koleszar's avatar
John Koleszar committed
154
155

  for (i = 0; i < block_type_counts; i++)
156
157
158
    for (j = 0; j < REF_TYPES; j++)
      for (k = 0; k < COEF_BANDS; k++)
        for (l = 0; l < PREV_COEF_CONTEXTS; l++) {
159
160
161
162
163
164
#if CONFIG_CODE_NONZEROCOUNT
          // All costs are without the EOB node
          vp9_cost_tokens_skip((int *)(c[i][j][k][l]),
                               p[i][j][k][l],
                               vp9_coef_tree);
#else
165
166
167
168
169
170
171
172
          if (l == 0 && k > 0)
            vp9_cost_tokens_skip((int *)(c[i][j][k][l]),
                                 p[i][j][k][l],
                                 vp9_coef_tree);
          else
            vp9_cost_tokens((int *)(c[i][j][k][l]),
                            p[i][j][k][l],
                            vp9_coef_tree);
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
#endif
        }
}

#if CONFIG_CODE_NONZEROCOUNT
static void fill_nzc_costs(VP9_COMP *cpi, int block_size) {
  int nzc_context, r, b, nzc, values;
  int cost[16];
  values = block_size * block_size + 1;

  for (nzc_context = 0; nzc_context < MAX_NZC_CONTEXTS; ++nzc_context) {
    for (r = 0; r < REF_TYPES; ++r) {
      for (b = 0; b < BLOCK_TYPES; ++b) {
        if (block_size == 4)
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_4x4[nzc_context][r][b],
                          vp9_nzc4x4_tree);
        else if (block_size == 8)
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_8x8[nzc_context][r][b],
                          vp9_nzc8x8_tree);
        else if (block_size == 16)
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_16x16[nzc_context][r][b],
                          vp9_nzc16x16_tree);
        else
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_32x32[nzc_context][r][b],
                          vp9_nzc32x32_tree);

        for (nzc = 0; nzc < values; ++nzc) {
          int e, c, totalcost = 0;
          c = codenzc(nzc);
          totalcost = cost[c];
          if ((e = extranzcbits(c))) {
            int x = nzc - basenzcvalue(c);
            while (e--) {
              if ((x >> e) & 1)
                totalcost += vp9_cost_one(Pcat_nzc[nzc_context][c - 3][e]);
              else
                totalcost += vp9_cost_zero(Pcat_nzc[nzc_context][c - 3][e]);
            }
          }
          if (block_size == 4)
            cpi->mb.nzc_costs_4x4[nzc_context][r][b][nzc] = totalcost;
          else if (block_size == 8)
            cpi->mb.nzc_costs_8x8[nzc_context][r][b][nzc] = totalcost;
          else if (block_size == 16)
            cpi->mb.nzc_costs_16x16[nzc_context][r][b][nzc] = totalcost;
          else
            cpi->mb.nzc_costs_32x32[nzc_context][r][b][nzc] = totalcost;
224
        }
225
226
227
      }
    }
  }
John Koleszar's avatar
John Koleszar committed
228
}
229
#endif
John Koleszar's avatar
John Koleszar committed
230

231

232
233
234
235
static int rd_iifactor[32] =  { 4, 4, 3, 2, 1, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0, };
John Koleszar's avatar
John Koleszar committed
236

237
// 3* dc_qlookup[Q]*dc_qlookup[Q];
238

239
/* values are now correlated to quantizer */
Paul Wilkins's avatar
Paul Wilkins committed
240
241
242
static int sad_per_bit16lut[QINDEX_RANGE];
static int sad_per_bit4lut[QINDEX_RANGE];

243
void vp9_init_me_luts() {
John Koleszar's avatar
John Koleszar committed
244
245
246
247
248
249
250
  int i;

  // Initialize the sad lut tables using a formulaic calculation for now
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < QINDEX_RANGE; i++) {
    sad_per_bit16lut[i] =
251
252
      (int)((0.0418 * vp9_convert_qindex_to_q(i)) + 2.4107);
    sad_per_bit4lut[i] = (int)((0.063 * vp9_convert_qindex_to_q(i)) + 2.742);
John Koleszar's avatar
John Koleszar committed
253
  }
Paul Wilkins's avatar
Paul Wilkins committed
254
}
John Koleszar's avatar
John Koleszar committed
255

256
static int compute_rd_mult(int qindex) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
257
  int q = vp9_dc_quant(qindex, 0);
John Koleszar's avatar
John Koleszar committed
258
  return (11 * q * q) >> 6;
259
260
}

Dmitry Kovalev's avatar
Dmitry Kovalev committed
261
262
263
void vp9_initialize_me_consts(VP9_COMP *cpi, int qindex) {
  cpi->mb.sadperbit16 = sad_per_bit16lut[qindex];
  cpi->mb.sadperbit4 = sad_per_bit4lut[qindex];
John Koleszar's avatar
John Koleszar committed
264
265
}

266

Dmitry Kovalev's avatar
Dmitry Kovalev committed
267
void vp9_initialize_rd_consts(VP9_COMP *cpi, int qindex) {
268
  int q, i;
John Koleszar's avatar
John Koleszar committed
269

270
  vp9_clear_system_state();  // __asm emms;
John Koleszar's avatar
John Koleszar committed
271

John Koleszar's avatar
John Koleszar committed
272
273
274
275
  // Further tests required to see if optimum is different
  // for key frames, golden frames and arf frames.
  // if (cpi->common.refresh_golden_frame ||
  //     cpi->common.refresh_alt_ref_frame)
Dmitry Kovalev's avatar
Dmitry Kovalev committed
276
  qindex = (qindex < 0) ? 0 : ((qindex > MAXQ) ? MAXQ : qindex);
277

Dmitry Kovalev's avatar
Dmitry Kovalev committed
278
  cpi->RDMULT = compute_rd_mult(qindex);
John Koleszar's avatar
John Koleszar committed
279

John Koleszar's avatar
John Koleszar committed
280
281
282
283
284
  if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    if (cpi->twopass.next_iiratio > 31)
      cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
    else
      cpi->RDMULT +=
Dmitry Kovalev's avatar
Dmitry Kovalev committed
285
          (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
John Koleszar's avatar
John Koleszar committed
286
  }
John Koleszar's avatar
John Koleszar committed
287

John Koleszar's avatar
John Koleszar committed
288
289
  if (cpi->RDMULT < 7)
    cpi->RDMULT = 7;
290

John Koleszar's avatar
John Koleszar committed
291
292
  cpi->mb.errorperbit = (cpi->RDMULT / 110);
  cpi->mb.errorperbit += (cpi->mb.errorperbit == 0);
293

294
  vp9_set_speed_features(cpi);
John Koleszar's avatar
John Koleszar committed
295

Dmitry Kovalev's avatar
Dmitry Kovalev committed
296
297
  q = (int)pow(vp9_dc_quant(qindex, 0) >> 2, 1.25);
  q <<= 2;
John Koleszar's avatar
John Koleszar committed
298
  cpi->RDMULT = cpi->RDMULT << 4;
Paul Wilkins's avatar
Paul Wilkins committed
299

John Koleszar's avatar
John Koleszar committed
300
301
  if (q < 8)
    q = 8;
302

John Koleszar's avatar
John Koleszar committed
303
304
305
  if (cpi->RDMULT > 1000) {
    cpi->RDDIV = 1;
    cpi->RDMULT /= 100;
John Koleszar's avatar
John Koleszar committed
306

John Koleszar's avatar
John Koleszar committed
307
308
309
310
311
312
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < INT_MAX) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q / 100;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
313

John Koleszar's avatar
John Koleszar committed
314
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
315
    }
John Koleszar's avatar
John Koleszar committed
316
317
  } else {
    cpi->RDDIV = 100;
John Koleszar's avatar
John Koleszar committed
318

John Koleszar's avatar
John Koleszar committed
319
320
321
322
323
324
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < (INT_MAX / q)) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
325

John Koleszar's avatar
John Koleszar committed
326
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
327
    }
John Koleszar's avatar
John Koleszar committed
328
  }
John Koleszar's avatar
John Koleszar committed
329

330
  fill_token_costs(cpi->mb.token_costs[TX_4X4],
331
                   cpi->common.fc.coef_probs_4x4, BLOCK_TYPES);
332
  fill_token_costs(cpi->mb.token_costs[TX_8X8],
333
                   cpi->common.fc.coef_probs_8x8, BLOCK_TYPES);
334
  fill_token_costs(cpi->mb.token_costs[TX_16X16],
335
                   cpi->common.fc.coef_probs_16x16, BLOCK_TYPES);
336
  fill_token_costs(cpi->mb.token_costs[TX_32X32],
337
                   cpi->common.fc.coef_probs_32x32, BLOCK_TYPES);
338
339
340
341
342
343
#if CONFIG_CODE_NONZEROCOUNT
  fill_nzc_costs(cpi, 4);
  fill_nzc_costs(cpi, 8);
  fill_nzc_costs(cpi, 16);
  fill_nzc_costs(cpi, 32);
#endif
344

John Koleszar's avatar
John Koleszar committed
345
346
  /*rough estimate for costing*/
  cpi->common.kf_ymode_probs_index = cpi->common.base_qindex >> 4;
347
  vp9_init_mode_costs(cpi);
John Koleszar's avatar
John Koleszar committed
348

349
  if (cpi->common.frame_type != KEY_FRAME) {
350
    vp9_build_nmv_cost_table(
351
352
353
354
355
356
        cpi->mb.nmvjointcost,
        cpi->mb.e_mbd.allow_high_precision_mv ?
        cpi->mb.nmvcost_hp : cpi->mb.nmvcost,
        &cpi->common.fc.nmvc,
        cpi->mb.e_mbd.allow_high_precision_mv, 1, 1);
  }
John Koleszar's avatar
John Koleszar committed
357
358
}

359
int vp9_block_error_c(int16_t *coeff, int16_t *dqcoeff, int block_size) {
360
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
361

362
  for (i = 0; i < block_size; i++) {
John Koleszar's avatar
John Koleszar committed
363
364
365
    int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }
John Koleszar's avatar
John Koleszar committed
366

John Koleszar's avatar
John Koleszar committed
367
  return error;
John Koleszar's avatar
John Koleszar committed
368
369
}

370
int vp9_mbblock_error_c(MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
371
372
373
374
  BLOCK  *be;
  BLOCKD *bd;
  int i, j;
  int berror, error = 0;
John Koleszar's avatar
John Koleszar committed
375

John Koleszar's avatar
John Koleszar committed
376
377
378
379
  for (i = 0; i < 16; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
    berror = 0;
380
    for (j = 0; j < 16; j++) {
John Koleszar's avatar
John Koleszar committed
381
382
      int this_diff = be->coeff[j] - bd->dqcoeff[j];
      berror += this_diff * this_diff;
John Koleszar's avatar
John Koleszar committed
383
    }
John Koleszar's avatar
John Koleszar committed
384
385
386
    error += berror;
  }
  return error;
John Koleszar's avatar
John Koleszar committed
387
388
}

389
int vp9_mbuverror_c(MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
390
391
  BLOCK  *be;
  BLOCKD *bd;
John Koleszar's avatar
John Koleszar committed
392

393
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
394

John Koleszar's avatar
John Koleszar committed
395
396
397
  for (i = 16; i < 24; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
John Koleszar's avatar
John Koleszar committed
398

399
    error += vp9_block_error_c(be->coeff, bd->dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
400
  }
John Koleszar's avatar
John Koleszar committed
401

John Koleszar's avatar
John Koleszar committed
402
  return error;
John Koleszar's avatar
John Koleszar committed
403
404
}

405
int vp9_uvsse(MACROBLOCK *x) {
406
407
408
  uint8_t *uptr, *vptr;
  uint8_t *upred_ptr = (*(x->block[16].base_src) + x->block[16].src);
  uint8_t *vpred_ptr = (*(x->block[20].base_src) + x->block[20].src);
John Koleszar's avatar
John Koleszar committed
409
410
411
412
  int uv_stride = x->block[16].src_stride;

  unsigned int sse1 = 0;
  unsigned int sse2 = 0;
413
414
  int mv_row = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.row;
  int mv_col = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.col;
John Koleszar's avatar
John Koleszar committed
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
  int offset;
  int pre_stride = x->e_mbd.block[16].pre_stride;

  if (mv_row < 0)
    mv_row -= 1;
  else
    mv_row += 1;

  if (mv_col < 0)
    mv_col -= 1;
  else
    mv_col += 1;

  mv_row /= 2;
  mv_col /= 2;

  offset = (mv_row >> 3) * pre_stride + (mv_col >> 3);
  uptr = x->e_mbd.pre.u_buffer + offset;
  vptr = x->e_mbd.pre.v_buffer + offset;

  if ((mv_row | mv_col) & 7) {
436
    vp9_sub_pixel_variance8x8(uptr, pre_stride, (mv_col & 7) << 1,
437
                              (mv_row & 7) << 1, upred_ptr, uv_stride, &sse2);
438
    vp9_sub_pixel_variance8x8(vptr, pre_stride, (mv_col & 7) << 1,
439
                              (mv_row & 7) << 1, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
440
441
    sse2 += sse1;
  } else {
442
443
    vp9_variance8x8(uptr, pre_stride, upred_ptr, uv_stride, &sse2);
    vp9_variance8x8(vptr, pre_stride, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
444
445
446
    sse2 += sse1;
  }
  return sse2;
John Koleszar's avatar
John Koleszar committed
447
448
}

449
static INLINE int cost_coeffs(VP9_COMMON *const cm, MACROBLOCK *mb,
450
                              int ib, PLANE_TYPE type,
451
452
453
                              ENTROPY_CONTEXT *a,
                              ENTROPY_CONTEXT *l,
                              TX_SIZE tx_size) {
454
  MACROBLOCKD *const xd = &mb->e_mbd;
455
456
457
  MB_MODE_INFO *mbmi = &xd->mode_info_context->mbmi;
  const BLOCK_SIZE_TYPE sb_type = mbmi->sb_type;
  int pt;
458
  const int eob = xd->eobs[ib];
459
  int c = 0;
460
  int cost = 0;
461
  const int *scan;
462
463
464
465
466
  const int16_t *qcoeff_ptr = xd->qcoeff + ib * 16;
  const int ref = mbmi->ref_frame != INTRA_FRAME;
  const TX_TYPE tx_type = (sb_type == BLOCK_SIZE_MB16X16 &&
                           type == PLANE_TYPE_Y_WITH_DC) ?
                          get_tx_type(xd, &xd->block[ib]) : DCT_DCT;
467
  unsigned int (*token_costs)[PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS] =
468
      mb->token_costs[tx_size][type][ref];
469
  ENTROPY_CONTEXT a_ec, l_ec;
470
471
472
473
  ENTROPY_CONTEXT *const a1 = a +
      sizeof(ENTROPY_CONTEXT_PLANES)/sizeof(ENTROPY_CONTEXT);
  ENTROPY_CONTEXT *const l1 = l +
      sizeof(ENTROPY_CONTEXT_PLANES)/sizeof(ENTROPY_CONTEXT);
474

475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
#if CONFIG_CODE_NONZEROCOUNT
  int nzc_context = vp9_get_nzc_context(cm, xd, ib);
  unsigned int *nzc_cost;
#else
  int seg_eob;
  const int segment_id = xd->mode_info_context->mbmi.segment_id;
#endif

  // Check for consistency of tx_size with mode info
  if (type == PLANE_TYPE_Y_WITH_DC) {
    assert(xd->mode_info_context->mbmi.txfm_size == tx_size);
  } else {
    TX_SIZE tx_size_uv = get_uv_tx_size(xd);
    assert(tx_size == tx_size_uv);
  }

491
  switch (tx_size) {
Daniel Kang's avatar
Daniel Kang committed
492
    case TX_4X4:
493
494
      a_ec = *a;
      l_ec = *l;
495
      scan = vp9_default_zig_zag1d_4x4;
496
497
498
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_4x4[nzc_context][ref][type];
#else
499
      seg_eob = 16;
500
#endif
Deb Mukherjee's avatar
Deb Mukherjee committed
501
      if (type == PLANE_TYPE_Y_WITH_DC) {
502
503
504
505
        if (tx_type == ADST_DCT) {
          scan = vp9_row_scan_4x4;
        } else if (tx_type == DCT_ADST) {
          scan = vp9_col_scan_4x4;
506
        }
Daniel Kang's avatar
Daniel Kang committed
507
508
509
      }
      break;
    case TX_8X8:
510
511
      a_ec = (a[0] + a[1]) != 0;
      l_ec = (l[0] + l[1]) != 0;
512
      scan = vp9_default_zig_zag1d_8x8;
513
514
515
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_8x8[nzc_context][ref][type];
#else
516
      seg_eob = 64;
517
#endif
Daniel Kang's avatar
Daniel Kang committed
518
519
      break;
    case TX_16X16:
520
      scan = vp9_default_zig_zag1d_16x16;
521
522
523
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_16x16[nzc_context][ref][type];
#else
524
      seg_eob = 256;
525
#endif
526
      if (type == PLANE_TYPE_UV) {
527
528
529
530
531
        a_ec = (a[0] + a[1] + a1[0] + a1[1]) != 0;
        l_ec = (l[0] + l[1] + l1[0] + l1[1]) != 0;
      } else {
        a_ec = (a[0] + a[1] + a[2] + a[3]) != 0;
        l_ec = (l[0] + l[1] + l[2] + l[3]) != 0;
Deb Mukherjee's avatar
Deb Mukherjee committed
532
      }
Daniel Kang's avatar
Daniel Kang committed
533
      break;
534
535
    case TX_32X32:
      scan = vp9_default_zig_zag1d_32x32;
536
537
538
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_32x32[nzc_context][ref][type];
#else
539
      seg_eob = 1024;
540
#endif
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
      if (type == PLANE_TYPE_UV) {
        ENTROPY_CONTEXT *a2, *a3, *l2, *l3;
        a2 = a1 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        a3 = a2 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        l2 = l1 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        l3 = l2 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        a_ec = (a[0] + a[1] + a1[0] + a1[1] +
                a2[0] + a2[1] + a3[0] + a3[1]) != 0;
        l_ec = (l[0] + l[1] + l1[0] + l1[1] +
                l2[0] + l2[1] + l3[0] + l3[1]) != 0;
      } else {
        a_ec = (a[0] + a[1] + a[2] + a[3] +
                a1[0] + a1[1] + a1[2] + a1[3]) != 0;
        l_ec = (l[0] + l[1] + l[2] + l[3] +
                l1[0] + l1[1] + l1[2] + l1[3]) != 0;
      }
557
      break;
Daniel Kang's avatar
Daniel Kang committed
558
    default:
559
      abort();
Daniel Kang's avatar
Daniel Kang committed
560
561
      break;
  }
562

563
564
  VP9_COMBINEENTROPYCONTEXTS(pt, a_ec, l_ec);

565
#if CONFIG_CODE_NONZEROCOUNT == 0
Paul Wilkins's avatar
Paul Wilkins committed
566
567
  if (vp9_segfeature_active(xd, segment_id, SEG_LVL_SKIP))
    seg_eob = 0;
568
#endif
569

570
  {
571
    int recent_energy = 0;
572
573
574
#if CONFIG_CODE_NONZEROCOUNT
    int nzc = 0;
#endif
575
576
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
577
      int t = vp9_dct_value_tokens_ptr[v].Token;
578
579
580
#if CONFIG_CODE_NONZEROCOUNT
      nzc += (v != 0);
#endif
581
      cost += token_costs[get_coef_band(tx_size, c)][pt][t];
582
      cost += vp9_dct_value_cost_ptr[v];
583
      pt = vp9_get_coef_context(&recent_energy, t);
584
    }
585
586
587
#if CONFIG_CODE_NONZEROCOUNT
    cost += nzc_cost[nzc];
#else
588
    if (c < seg_eob)
589
      cost += mb->token_costs[tx_size][type][ref][get_coef_band(tx_size, c)]
590
          [pt][DCT_EOB_TOKEN];
591
#endif
592
593
  }

594
  // is eob first coefficient;
595
  pt = (c > 0);
596
  *a = *l = pt;
597
598
599
600
601
602
603
604
605
606
607
608
609
610
  if (tx_size >= TX_8X8) {
    a[1] = l[1] = pt;
    if (tx_size >= TX_16X16) {
      if (type == PLANE_TYPE_UV) {
        a1[0] = a1[1] = l1[0] = l1[1] = pt;
      } else {
        a[2] = a[3] = l[2] = l[3] = pt;
        if (tx_size >= TX_32X32) {
          a1[0] = a1[1] = a1[2] = a1[3] = pt;
          l1[0] = l1[1] = l1[2] = l1[3] = pt;
        }
      }
    }
  }
611
612
613
  return cost;
}

614
static int rdcost_mby_4x4(VP9_COMMON *const cm, MACROBLOCK *mb, int backup) {
John Koleszar's avatar
John Koleszar committed
615
616
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
617
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
618
619
620
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta;
  ENTROPY_CONTEXT *tl;
621

622
623
624
  if (backup) {
    vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
625

626
627
628
629
630
631
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)xd->above_context;
    tl = (ENTROPY_CONTEXT *)xd->left_context;
  }
John Koleszar's avatar
John Koleszar committed
632

John Koleszar's avatar
John Koleszar committed
633
  for (b = 0; b < 16; b++)
634
    cost += cost_coeffs(cm, mb, b, PLANE_TYPE_Y_WITH_DC,
Yaowu Xu's avatar
Yaowu Xu committed
635
636
                        ta + vp9_block2above[TX_4X4][b],
                        tl + vp9_block2left[TX_4X4][b],
Daniel Kang's avatar
Daniel Kang committed
637
                        TX_4X4);
John Koleszar's avatar
John Koleszar committed
638

John Koleszar's avatar
John Koleszar committed
639
  return cost;
John Koleszar's avatar
John Koleszar committed
640
641
}

642
643
static void macro_block_yrd_4x4(VP9_COMMON *const cm,
                                MACROBLOCK *mb,
644
645
                                int *Rate,
                                int *Distortion,
646
                                int *skippable, int backup) {
Paul Wilkins's avatar
Paul Wilkins committed
647
  MACROBLOCKD *const xd = &mb->e_mbd;
648

649
650
651
  xd->mode_info_context->mbmi.txfm_size = TX_4X4;
  vp9_transform_mby_4x4(mb);
  vp9_quantize_mby_4x4(mb);
652

653
  *Distortion = vp9_mbblock_error(mb) >> 2;
654
  *Rate = rdcost_mby_4x4(cm, mb, backup);
655
  *skippable = vp9_mby_is_skippable_4x4(xd);
656
}
John Koleszar's avatar
John Koleszar committed
657

658
static int rdcost_mby_8x8(VP9_COMMON *const cm, MACROBLOCK *mb, int backup) {
John Koleszar's avatar
John Koleszar committed
659
660
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
661
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
662
663
664
665
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta;
  ENTROPY_CONTEXT *tl;

Ronald S. Bultje's avatar
Ronald S. Bultje committed
666
667
668
  if (backup) {
    vpx_memcpy(&t_above,xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
669

Ronald S. Bultje's avatar
Ronald S. Bultje committed
670
671
672
673
674
675
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)mb->e_mbd.above_context;
    tl = (ENTROPY_CONTEXT *)mb->e_mbd.left_context;
  }
John Koleszar's avatar
John Koleszar committed
676
677

  for (b = 0; b < 16; b += 4)
678
    cost += cost_coeffs(cm, mb, b, PLANE_TYPE_Y_WITH_DC,
Yaowu Xu's avatar
Yaowu Xu committed
679
680
                        ta + vp9_block2above[TX_8X8][b],
                        tl + vp9_block2left[TX_8X8][b],
Daniel Kang's avatar
Daniel Kang committed
681
                        TX_8X8);
John Koleszar's avatar
John Koleszar committed
682
683

  return cost;
684
685
}

686
687
static void macro_block_yrd_8x8(VP9_COMMON *const cm,
                                MACROBLOCK *mb,
John Koleszar's avatar
John Koleszar committed
688
689
                                int *Rate,
                                int *Distortion,
690
                                int *skippable, int backup) {
Paul Wilkins's avatar
Paul Wilkins committed
691
  MACROBLOCKD *const xd = &mb->e_mbd;
692
693

  xd->mode_info_context->mbmi.txfm_size = TX_8X8;
694
695
  vp9_transform_mby_8x8(mb);
  vp9_quantize_mby_8x8(mb);
John Koleszar's avatar
John Koleszar committed
696

697
  *Distortion = vp9_mbblock_error(mb) >> 2;
698
  *Rate = rdcost_mby_8x8(cm, mb, backup);
699
  *skippable = vp9_mby_is_skippable_8x8(xd);
700
}
701

702
static int rdcost_mby_16x16(VP9_COMMON *const cm, MACROBLOCK *mb, int backup) {
Daniel Kang's avatar
Daniel Kang committed
703
  int cost;
Paul Wilkins's avatar
Paul Wilkins committed
704
  MACROBLOCKD *xd = &mb->e_mbd;
Daniel Kang's avatar
Daniel Kang committed
705
706
707
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta, *tl;

708
709
710
  if (backup) {
    vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
Daniel Kang's avatar
Daniel Kang committed
711

712
713
714
715
716
717
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)xd->above_context;
    tl = (ENTROPY_CONTEXT *)xd->left_context;
  }
Daniel Kang's avatar
Daniel Kang committed
718

719
  cost = cost_coeffs(cm, mb, 0, PLANE_TYPE_Y_WITH_DC, ta, tl, TX_16X16);
Daniel Kang's avatar
Daniel Kang committed
720
721
  return cost;
}
722

723
724
static void macro_block_yrd_16x16(VP9_COMMON *const cm, MACROBLOCK *mb,
                                  int *Rate, int *Distortion,
725
                                  int *skippable, int backup) {
Deb Mukherjee's avatar
Deb Mukherjee committed
726
  MACROBLOCKD *xd = &mb->e_mbd;
Daniel Kang's avatar
Daniel Kang committed
727

728
729
  xd->mode_info_context->mbmi.txfm_size = TX_16X16;
  vp9_transform_mby_16x16(mb);
730
  vp9_quantize_mby_16x16(mb);
731
732
733
  // TODO(jingning) is it possible to quickly determine whether to force
  //                trailing coefficients to be zero, instead of running trellis
  //                optimization in the rate-distortion optimization loop?
734
735
  if (mb->optimize &&
      xd->mode_info_context->mbmi.mode < I8X8_PRED)
736
    vp9_optimize_mby_16x16(cm, mb);
737

738
  *Distortion = vp9_mbblock_error(mb) >> 2;
739
  *Rate = rdcost_mby_16x16(cm, mb, backup);
740
  *skippable = vp9_mby_is_skippable_16x16(xd);
Daniel Kang's avatar
Daniel Kang committed
741
742
}

743
static void choose_txfm_size_from_rd(VP9_COMP *cpi, MACROBLOCK *x,
744
745
746
747
748
                                     int (*r)[2], int *rate,
                                     int *d, int *distortion,
                                     int *s, int *skip,
                                     int64_t txfm_cache[NB_TXFM_MODES],
                                     TX_SIZE max_txfm_size) {
749
750
751
752
753
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
  vp9_prob skip_prob = cm->mb_no_coeff_skip ?
                       vp9_get_pred_prob(cm, xd, PRED_MBSKIP) : 128;
754
755
756
757
758
759
760
761
762
763
764
765
  int64_t rd[TX_SIZE_MAX_SB][2];
  int n, m;

  for (n = TX_4X4; n <= max_txfm_size; n++) {
    r[n][1] = r[n][0];
    for (m = 0; m <= n - (n == max_txfm_size); m++) {
      if (m == n)
        r[n][1] += vp9_cost_zero(cm->prob_tx[m]);
      else
        r[n][1] += vp9_cost_one(cm->prob_tx[m]);
    }
  }
766
767
768
769
770
771
772
773

  if (cm->mb_no_coeff_skip) {
    int s0, s1;

    assert(skip_prob > 0);
    s0 = vp9_cost_bit(skip_prob, 0);
    s1 = vp9_cost_bit(skip_prob, 1);

774
    for (n = TX_4X4; n <= max_txfm_size; n++) {
775
      if (s[n]) {
776
        rd[n][0] = rd[n][1] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
777
      } else {
778
779
        rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0] + s0, d[n]);
        rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1] + s0, d[n]);
780
      }
781
782
    }
  } else {
783
784
785
    for (n = TX_4X4; n <= max_txfm_size; n++) {
      rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0], d[n]);
      rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1], d[n]);
786
787
788
    }
  }

789
790
791
792
793
794
  if (max_txfm_size == TX_32X32 &&
      (cm->txfm_mode == ALLOW_32X32 ||
       (cm->txfm_mode == TX_MODE_SELECT &&
        rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
        rd[TX_32X32][1] < rd[TX_4X4][1]))) {
    mbmi->txfm_size = TX_32X32;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
795
796
797
798
799
  } else if ( cm->txfm_mode == ALLOW_16X16 ||
             (max_txfm_size == TX_16X16 && cm->txfm_mode == ALLOW_32X32) ||
             (cm->txfm_mode == TX_MODE_SELECT &&
              rd[TX_16X16][1] < rd[TX_8X8][1] &&
              rd[TX_16X16][1] < rd[TX_4X4][1])) {
800
    mbmi->txfm_size = TX_16X16;
801
  } else if (cm->txfm_mode == ALLOW_8X8 ||
802
           (cm->txfm_mode == TX_MODE_SELECT && rd[TX_8X8][1] < rd[TX_4X4][1])) {
803
804
    mbmi->txfm_size = TX_8X8;
  } else {
805
    assert(cm->txfm_mode == ONLY_4X4 || cm->txfm_mode == TX_MODE_SELECT);
806
807
808
    mbmi->txfm_size = TX_4X4;
  }

809
  *distortion = d[mbmi->txfm_size];
810
  *rate       = r[mbmi->txfm_size][cm->txfm_mode == TX_MODE_SELECT];
811
812
  *skip       = s[mbmi->txfm_size];

813
814
815
816
817
818
819
820
  txfm_cache[ONLY_4X4] = rd[TX_4X4][0];
  txfm_cache[ALLOW_8X8] = rd[TX_8X8][0];
  txfm_cache[ALLOW_16X16] = rd[TX_16X16][0];
  txfm_cache[ALLOW_32X32] = rd[max_txfm_size][0];
  if (max_txfm_size == TX_32X32 &&
      rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
      rd[TX_32X32][1] < rd[TX_4X4][1])
    txfm_cache[TX_MODE_SELECT] = rd[TX_32X32][1];
Ronald S. Bultje's avatar
Ronald S. Bultje committed
821
  else if (rd[TX_16X16][1] < rd[TX_8X8][1] && rd[TX_16X16][1] < rd[TX_4X4][1])
822
    txfm_cache[TX_MODE_SELECT] = rd[TX_16X16][1];
823
  else
824
825
    txfm_cache[TX_MODE_SELECT] = rd[TX_4X4][1] < rd[TX_8X8][1] ?
                                 rd[TX_4X4][1] : rd[TX_8X8][1];
826
827
828
829
830
}

static void macro_block_yrd(VP9_COMP *cpi, MACROBLOCK *x, int *rate,
                            int *distortion, int *skippable,
                            int64_t txfm_cache[NB_TXFM_MODES]) {
831
  VP9_COMMON *const cm = &cpi->common;
832
  MACROBLOCKD *const xd = &x->e_mbd;
833
  int r[TX_SIZE_MAX_MB][2], d[TX_SIZE_MAX_MB], s[TX_SIZE_MAX_MB];
834
835
836

  vp9_subtract_mby(x->src_diff, *(x->block[0].base_src), xd->predictor,
                   x->block[0].src_stride);
837

838
839
840
  macro_block_yrd_16x16(cm, x, &r[TX_16X16][0], &d[TX_16X16], &s[TX_16X16], 1);
  macro_block_yrd_8x8(cm, x, &r[TX_8X8][0], &d[TX_8X8], &s[TX_8X8], 1);
  macro_block_yrd_4x4(cm, x, &r[TX_4X4][0], &d[TX_4X4], &s[TX_4X4], 1);
841
842

  choose_txfm_size_from_rd(cpi, x, r, rate, d, distortion, s, skippable,
843
                           txfm_cache, TX_16X16);
844
845
}

846
static void copy_predictor(uint8_t *dst, const uint8_t *predictor) {
John Koleszar's avatar
John Koleszar committed
847
848
849
850
851
852
  const unsigned int *p = (const unsigned int *)predictor;
  unsigned int *d = (unsigned int *)dst;
  d[0] = p[0];
  d[4] = p[4];
  d[8] = p[8];
  d[12] = p[12];
853
}
Yaowu Xu's avatar
Yaowu Xu committed
854

855
856
static int rdcost_sby_32x32(VP9_COMMON *const cm, MACROBLOCK *x, int backup) {
  MACROBLOCKD * xd = &x->e_mbd;
857
  ENTROPY_CONTEXT_PLANES t_above[2], t_left[2];
Ronald S. Bultje's avatar
Ronald S. Bultje committed
858
  ENTROPY_CONTEXT *ta, *tl;
859

Ronald S. Bultje's avatar
Ronald S. Bultje committed
860
861
862
863
  if (backup) {
    ta = (ENTROPY_CONTEXT *) &t_above,
    tl = (ENTROPY_CONTEXT *) &t_left;

864
865
    vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES) * 2);
    vpx_memcpy(&t_left,  xd->left_context,  sizeof(ENTROPY_CONTEXT_PLANES) * 2);
Ronald S. Bultje's avatar
Ronald S. Bultje committed
866
867
868
869
  } else {
    ta = (ENTROPY_CONTEXT *) xd->above_context;
    tl = (ENTROPY_CONTEXT *) xd->left_context;
  }
870

871
  return cost_coeffs(cm, x, 0, PLANE_TYPE_Y_WITH_DC, ta, tl, TX_32X32);
872
873
}

874
875
static int vp9_sb_block_error_c(int16_t *coeff, int16_t *dqcoeff,
                                int block_size) {
876
877
878
879
880
881
882
883
  int i;
  int64_t error = 0;

  for (i = 0; i < block_size; i++) {
    unsigned int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }

Frank Galligan's avatar
Frank Galligan committed
884
  return error > INT_MAX ? INT_MAX : (int)error;
885
886
887
}

#define DEBUG_ERROR 0
888
static void super_block_yrd_32x32(VP9_COMMON *const cm, MACROBLOCK *x,
Ronald S. Bultje's avatar
Ronald S. Bultje committed
889
890
                                  int *rate, int *distortion, int *skippable,
                                  int backup) {
891
  MACROBLOCKD *const xd = &x->e_mbd;
892
#if DEBUG_ERROR
893
  int16_t out[1024];
894
#endif
895
  xd->mode_info_context->mbmi.txfm_size = TX_32X32;
896
897
898

  vp9_transform_sby_32x32(x);
  vp9_quantize_sby_32x32(x);
899
#if DEBUG_ERROR
900
  vp9_short_idct32x32(xd->dqcoeff, out, 64);
901
902
#endif

903
  *distortion = vp9_sb_block_error_c(x->coeff, xd->dqcoeff, 1024);
904

905
906
#if DEBUG_ERROR
  printf("IDCT/FDCT error 32x32: %d (d: %d)\n",
907
         vp9_block_error_c(x->src_diff, out, 1024), *distortion);
908
#endif
909
  *rate       = rdcost_sby_32x32(cm, x, backup);
910
  *skippable  = vp9_sby_is_skippable_32x32(xd);
911
912
}

913
914
static void super_block_yrd(VP9_COMP *cpi,
                            MACROBLOCK *x, int *rate, int *distortion,
915
                            int *skip,
916
                            int64_t txfm_cache[NB_TXFM_MODES]) {
Ronald S. Bultje's avatar
Ronald S. Bultje committed
917
  MACROBLOCKD *const xd = &x->e_mbd;
918
  int r[TX_SIZE_MAX_SB][2], d[TX_SIZE_MAX_SB], s[TX_SIZE_MAX_SB], n;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
919
920
  const uint8_t *src = x->src.y_buffer, *dst = xd->dst.y_buffer;
  int src_y_stride = x->src.y_stride, dst_y_stride = xd->dst.y_stride;
921
922
923
924
  ENTROPY_CONTEXT_PLANES t_above[TX_SIZE_MAX_MB][2],
                        *orig_above = xd->above_context;
  ENTROPY_CONTEXT_PLANES t_left[TX_SIZE_MAX_MB][2],
                        *orig_left = xd->left_context;
925

926
  for (n = TX_4X4; n < TX_SIZE_MAX_MB; n++) {
927
928
    vpx_memcpy(t_above[n], xd->above_context, sizeof(t_above[n]));
    vpx_memcpy(t_left[n], xd->left_context, sizeof(t_left[n]));
929
    r[n][0] = 0;