vp9_rdopt.c 174 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5
6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9
10
11
12
13
14
15
 */


#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
16
17
#include "vp9/common/vp9_pragmas.h"

18
19
20
21
22
#include "vp9/encoder/vp9_tokenize.h"
#include "vp9/encoder/vp9_treewriter.h"
#include "vp9/encoder/vp9_onyx_int.h"
#include "vp9/encoder/vp9_modecosts.h"
#include "vp9/encoder/vp9_encodeintra.h"
23
24
25
26
27
28
#include "vp9/common/vp9_entropymode.h"
#include "vp9/common/vp9_reconinter.h"
#include "vp9/common/vp9_reconintra.h"
#include "vp9/common/vp9_reconintra4x4.h"
#include "vp9/common/vp9_findnearmv.h"
#include "vp9/common/vp9_quant_common.h"
29
30
31
32
33
34
#include "vp9/encoder/vp9_encodemb.h"
#include "vp9/encoder/vp9_quantize.h"
#include "vp9/encoder/vp9_variance.h"
#include "vp9/encoder/vp9_mcomp.h"
#include "vp9/encoder/vp9_rdopt.h"
#include "vp9/encoder/vp9_ratectrl.h"
John Koleszar's avatar
John Koleszar committed
35
#include "vpx_mem/vpx_mem.h"
36
37
#include "vp9/common/vp9_systemdependent.h"
#include "vp9/encoder/vp9_encodemv.h"
John Koleszar's avatar
John Koleszar committed
38

39
40
41
#include "vp9/common/vp9_seg_common.h"
#include "vp9/common/vp9_pred_common.h"
#include "vp9/common/vp9_entropy.h"
42
#include "vp9_rtcd.h"
43
#include "vp9/common/vp9_mvref_common.h"
Ronald S. Bultje's avatar
Ronald S. Bultje committed
44
#include "vp9/common/vp9_common.h"
Paul Wilkins's avatar
Paul Wilkins committed
45

John Koleszar's avatar
John Koleszar committed
46
47
#define MAXF(a,b)            (((a) > (b)) ? (a) : (b))

48
49
#define INVALID_MV 0x80008000

50
51
52
/* Factor to weigh the rate for switchable interp filters */
#define SWITCHABLE_INTERP_RATE_FACTOR 1

John Koleszar's avatar
John Koleszar committed
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
static const int auto_speed_thresh[17] = {
  1000,
  200,
  150,
  130,
  150,
  125,
  120,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  105
John Koleszar's avatar
John Koleszar committed
71
72
};

73
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
74
75
  {ZEROMV,    LAST_FRAME,   NONE},
  {DC_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
76

77
78
  {NEARESTMV, LAST_FRAME,   NONE},
  {NEARMV,    LAST_FRAME,   NONE},
John Koleszar's avatar
John Koleszar committed
79

80
81
  {ZEROMV,    GOLDEN_FRAME, NONE},
  {NEARESTMV, GOLDEN_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
82

83
84
  {ZEROMV,    ALTREF_FRAME, NONE},
  {NEARESTMV, ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
85

86
87
  {NEARMV,    GOLDEN_FRAME, NONE},
  {NEARMV,    ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
88

89
90
91
92
93
94
95
96
  {V_PRED,    INTRA_FRAME,  NONE},
  {H_PRED,    INTRA_FRAME,  NONE},
  {D45_PRED,  INTRA_FRAME,  NONE},
  {D135_PRED, INTRA_FRAME,  NONE},
  {D117_PRED, INTRA_FRAME,  NONE},
  {D153_PRED, INTRA_FRAME,  NONE},
  {D27_PRED,  INTRA_FRAME,  NONE},
  {D63_PRED,  INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
97

98
  {TM_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
99

100
101
102
  {NEWMV,     LAST_FRAME,   NONE},
  {NEWMV,     GOLDEN_FRAME, NONE},
  {NEWMV,     ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
103

104
105
106
  {SPLITMV,   LAST_FRAME,   NONE},
  {SPLITMV,   GOLDEN_FRAME, NONE},
  {SPLITMV,   ALTREF_FRAME, NONE},
107

108
109
  {B_PRED,    INTRA_FRAME,  NONE},
  {I8X8_PRED, INTRA_FRAME,  NONE},
110

John Koleszar's avatar
John Koleszar committed
111
112
113
114
  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME},
115

John Koleszar's avatar
John Koleszar committed
116
117
118
  {ZEROMV,    ALTREF_FRAME, LAST_FRAME},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME},
119

John Koleszar's avatar
John Koleszar committed
120
121
122
  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME},
123

John Koleszar's avatar
John Koleszar committed
124
125
126
  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME  },
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME},
127

John Koleszar's avatar
John Koleszar committed
128
129
  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME  },
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME},

#if CONFIG_COMP_INTERINTRA_PRED
  /* compound inter-intra prediction */
  {ZEROMV,    LAST_FRAME,   INTRA_FRAME},
  {NEARESTMV, LAST_FRAME,   INTRA_FRAME},
  {NEARMV,    LAST_FRAME,   INTRA_FRAME},
  {NEWMV,     LAST_FRAME,   INTRA_FRAME},

  {ZEROMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEARESTMV, GOLDEN_FRAME,   INTRA_FRAME},
  {NEARMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEWMV,     GOLDEN_FRAME,   INTRA_FRAME},

  {ZEROMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEARESTMV, ALTREF_FRAME,   INTRA_FRAME},
  {NEARMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEWMV,     ALTREF_FRAME,   INTRA_FRAME},
#endif
John Koleszar's avatar
John Koleszar committed
149
150
};

151
152
153
static void fill_token_costs(vp9_coeff_count *c,
                             vp9_coeff_probs *p,
                             int block_type_counts) {
John Koleszar's avatar
John Koleszar committed
154
155
156
157
158
159
  int i, j, k;

  for (i = 0; i < block_type_counts; i++)
    for (j = 0; j < COEF_BANDS; j++)
      for (k = 0; k < PREV_COEF_CONTEXTS; k++) {
        if (k == 0 && ((j > 0 && i > 0) || (j > 1 && i == 0)))
160
161
          vp9_cost_tokens_skip((int *)(c[i][j][k]),
                               p[i][j][k],
162
                               vp9_coef_tree);
John Koleszar's avatar
John Koleszar committed
163
        else
164
165
          vp9_cost_tokens((int *)(c[i][j][k]),
                          p[i][j][k],
166
                          vp9_coef_tree);
John Koleszar's avatar
John Koleszar committed
167
      }
John Koleszar's avatar
John Koleszar committed
168
169
}

170

171
172
173
174
static int rd_iifactor[32] =  { 4, 4, 3, 2, 1, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0, };
John Koleszar's avatar
John Koleszar committed
175

176
// 3* dc_qlookup[Q]*dc_qlookup[Q];
177

178
/* values are now correlated to quantizer */
Paul Wilkins's avatar
Paul Wilkins committed
179
180
181
static int sad_per_bit16lut[QINDEX_RANGE];
static int sad_per_bit4lut[QINDEX_RANGE];

182
void vp9_init_me_luts() {
John Koleszar's avatar
John Koleszar committed
183
184
185
186
187
188
189
  int i;

  // Initialize the sad lut tables using a formulaic calculation for now
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < QINDEX_RANGE; i++) {
    sad_per_bit16lut[i] =
190
191
      (int)((0.0418 * vp9_convert_qindex_to_q(i)) + 2.4107);
    sad_per_bit4lut[i] = (int)((0.063 * vp9_convert_qindex_to_q(i)) + 2.742);
John Koleszar's avatar
John Koleszar committed
192
  }
Paul Wilkins's avatar
Paul Wilkins committed
193
}
John Koleszar's avatar
John Koleszar committed
194

195
static int compute_rd_mult(int qindex) {
John Koleszar's avatar
John Koleszar committed
196
  int q;
197

198
  q = vp9_dc_quant(qindex, 0);
John Koleszar's avatar
John Koleszar committed
199
  return (11 * q * q) >> 6;
200
201
}

202
void vp9_initialize_me_consts(VP9_COMP *cpi, int QIndex) {
John Koleszar's avatar
John Koleszar committed
203
204
  cpi->mb.sadperbit16 =  sad_per_bit16lut[QIndex];
  cpi->mb.sadperbit4  =  sad_per_bit4lut[QIndex];
John Koleszar's avatar
John Koleszar committed
205
206
}

207

208
void vp9_initialize_rd_consts(VP9_COMP *cpi, int QIndex) {
209
  int q, i;
John Koleszar's avatar
John Koleszar committed
210

211
  vp9_clear_system_state();  // __asm emms;
John Koleszar's avatar
John Koleszar committed
212

John Koleszar's avatar
John Koleszar committed
213
214
215
216
217
  // Further tests required to see if optimum is different
  // for key frames, golden frames and arf frames.
  // if (cpi->common.refresh_golden_frame ||
  //     cpi->common.refresh_alt_ref_frame)
  QIndex = (QIndex < 0) ? 0 : ((QIndex > MAXQ) ? MAXQ : QIndex);
218

John Koleszar's avatar
John Koleszar committed
219
  cpi->RDMULT = compute_rd_mult(QIndex);
John Koleszar's avatar
John Koleszar committed
220

John Koleszar's avatar
John Koleszar committed
221
222
223
  // Extend rate multiplier along side quantizer zbin increases
  if (cpi->zbin_over_quant  > 0) {
    double oq_factor;
224

John Koleszar's avatar
John Koleszar committed
225
226
227
228
229
    // Experimental code using the same basic equation as used for Q above
    // The units of cpi->zbin_over_quant are 1/128 of Q bin size
    oq_factor = 1.0 + ((double)0.0015625 * cpi->zbin_over_quant);
    cpi->RDMULT = (int)((double)cpi->RDMULT * oq_factor * oq_factor);
  }
John Koleszar's avatar
John Koleszar committed
230

John Koleszar's avatar
John Koleszar committed
231
232
233
234
235
236
237
  if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    if (cpi->twopass.next_iiratio > 31)
      cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
    else
      cpi->RDMULT +=
        (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
  }
John Koleszar's avatar
John Koleszar committed
238

John Koleszar's avatar
John Koleszar committed
239
240
  if (cpi->RDMULT < 7)
    cpi->RDMULT = 7;
241

John Koleszar's avatar
John Koleszar committed
242
243
  cpi->mb.errorperbit = (cpi->RDMULT / 110);
  cpi->mb.errorperbit += (cpi->mb.errorperbit == 0);
244

245
  vp9_set_speed_features(cpi);
John Koleszar's avatar
John Koleszar committed
246

247
  q = (int)pow(vp9_dc_quant(QIndex, 0) >> 2, 1.25);
John Koleszar's avatar
John Koleszar committed
248
249
  q = q << 2;
  cpi->RDMULT = cpi->RDMULT << 4;
Paul Wilkins's avatar
Paul Wilkins committed
250

John Koleszar's avatar
John Koleszar committed
251
252
  if (q < 8)
    q = 8;
253

John Koleszar's avatar
John Koleszar committed
254
255
256
  if (cpi->RDMULT > 1000) {
    cpi->RDDIV = 1;
    cpi->RDMULT /= 100;
John Koleszar's avatar
John Koleszar committed
257

John Koleszar's avatar
John Koleszar committed
258
259
260
261
262
263
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < INT_MAX) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q / 100;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
264

John Koleszar's avatar
John Koleszar committed
265
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
266
    }
John Koleszar's avatar
John Koleszar committed
267
268
  } else {
    cpi->RDDIV = 100;
John Koleszar's avatar
John Koleszar committed
269

John Koleszar's avatar
John Koleszar committed
270
271
272
273
274
275
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < (INT_MAX / q)) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
276

John Koleszar's avatar
John Koleszar committed
277
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
278
    }
John Koleszar's avatar
John Koleszar committed
279
  }
John Koleszar's avatar
John Koleszar committed
280

281
282
283
284
285
286
287
288
289
290
291
292
293
294
  fill_token_costs(cpi->mb.token_costs[TX_4X4],
                   cpi->common.fc.coef_probs_4x4, BLOCK_TYPES_4X4);
  fill_token_costs(cpi->mb.hybrid_token_costs[TX_4X4],
                   cpi->common.fc.hybrid_coef_probs_4x4, BLOCK_TYPES_4X4);

  fill_token_costs(cpi->mb.token_costs[TX_8X8],
                   cpi->common.fc.coef_probs_8x8, BLOCK_TYPES_8X8);
  fill_token_costs(cpi->mb.hybrid_token_costs[TX_8X8],
                   cpi->common.fc.hybrid_coef_probs_8x8, BLOCK_TYPES_8X8);

  fill_token_costs(cpi->mb.token_costs[TX_16X16],
                   cpi->common.fc.coef_probs_16x16, BLOCK_TYPES_16X16);
  fill_token_costs(cpi->mb.hybrid_token_costs[TX_16X16],
                   cpi->common.fc.hybrid_coef_probs_16x16, BLOCK_TYPES_16X16);
Daniel Kang's avatar
Daniel Kang committed
295

296
297
  fill_token_costs(cpi->mb.token_costs[TX_32X32],
                   cpi->common.fc.coef_probs_32x32, BLOCK_TYPES_32X32);
298

John Koleszar's avatar
John Koleszar committed
299
300
  /*rough estimate for costing*/
  cpi->common.kf_ymode_probs_index = cpi->common.base_qindex >> 4;
301
  vp9_init_mode_costs(cpi);
John Koleszar's avatar
John Koleszar committed
302

303
  if (cpi->common.frame_type != KEY_FRAME) {
304
    vp9_build_nmv_cost_table(
305
306
307
308
309
310
        cpi->mb.nmvjointcost,
        cpi->mb.e_mbd.allow_high_precision_mv ?
        cpi->mb.nmvcost_hp : cpi->mb.nmvcost,
        &cpi->common.fc.nmvc,
        cpi->mb.e_mbd.allow_high_precision_mv, 1, 1);
  }
John Koleszar's avatar
John Koleszar committed
311
312
}

313
int vp9_block_error_c(int16_t *coeff, int16_t *dqcoeff, int block_size) {
314
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
315

316
  for (i = 0; i < block_size; i++) {
John Koleszar's avatar
John Koleszar committed
317
318
319
    int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }
John Koleszar's avatar
John Koleszar committed
320

John Koleszar's avatar
John Koleszar committed
321
  return error;
John Koleszar's avatar
John Koleszar committed
322
323
}

324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
int vp9_mbblock_error_8x8_c(MACROBLOCK *mb, int dc) {
  BLOCK  *be;
  BLOCKD *bd;
  int i, j;
  int berror, error = 0;

  for (i = 0; i < 16; i+=4) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
    berror = 0;
    for (j = dc; j < 64; j++) {
      int this_diff = be->coeff[j] - bd->dqcoeff[j];
      berror += this_diff * this_diff;
    }
    error += berror;
  }
  return error;
}

343
int vp9_mbblock_error_c(MACROBLOCK *mb, int dc) {
John Koleszar's avatar
John Koleszar committed
344
345
346
347
  BLOCK  *be;
  BLOCKD *bd;
  int i, j;
  int berror, error = 0;
John Koleszar's avatar
John Koleszar committed
348

John Koleszar's avatar
John Koleszar committed
349
350
351
352
353
354
355
  for (i = 0; i < 16; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
    berror = 0;
    for (j = dc; j < 16; j++) {
      int this_diff = be->coeff[j] - bd->dqcoeff[j];
      berror += this_diff * this_diff;
John Koleszar's avatar
John Koleszar committed
356
    }
John Koleszar's avatar
John Koleszar committed
357
358
359
    error += berror;
  }
  return error;
John Koleszar's avatar
John Koleszar committed
360
361
}

362
int vp9_mbuverror_c(MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
363
364
  BLOCK  *be;
  BLOCKD *bd;
John Koleszar's avatar
John Koleszar committed
365

366
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
367

John Koleszar's avatar
John Koleszar committed
368
369
370
  for (i = 16; i < 24; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
John Koleszar's avatar
John Koleszar committed
371

372
    error += vp9_block_error_c(be->coeff, bd->dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
373
  }
John Koleszar's avatar
John Koleszar committed
374

John Koleszar's avatar
John Koleszar committed
375
  return error;
John Koleszar's avatar
John Koleszar committed
376
377
}

378
int vp9_uvsse(MACROBLOCK *x) {
379
380
381
  uint8_t *uptr, *vptr;
  uint8_t *upred_ptr = (*(x->block[16].base_src) + x->block[16].src);
  uint8_t *vpred_ptr = (*(x->block[20].base_src) + x->block[20].src);
John Koleszar's avatar
John Koleszar committed
382
383
384
385
  int uv_stride = x->block[16].src_stride;

  unsigned int sse1 = 0;
  unsigned int sse2 = 0;
386
387
  int mv_row = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.row;
  int mv_col = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.col;
John Koleszar's avatar
John Koleszar committed
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
  int offset;
  int pre_stride = x->e_mbd.block[16].pre_stride;

  if (mv_row < 0)
    mv_row -= 1;
  else
    mv_row += 1;

  if (mv_col < 0)
    mv_col -= 1;
  else
    mv_col += 1;

  mv_row /= 2;
  mv_col /= 2;

  offset = (mv_row >> 3) * pre_stride + (mv_col >> 3);
  uptr = x->e_mbd.pre.u_buffer + offset;
  vptr = x->e_mbd.pre.v_buffer + offset;

  if ((mv_row | mv_col) & 7) {
409
    vp9_sub_pixel_variance8x8(uptr, pre_stride, (mv_col & 7) << 1,
410
                              (mv_row & 7) << 1, upred_ptr, uv_stride, &sse2);
411
    vp9_sub_pixel_variance8x8(vptr, pre_stride, (mv_col & 7) << 1,
412
                              (mv_row & 7) << 1, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
413
414
    sse2 += sse1;
  } else {
415
416
    vp9_variance8x8(uptr, pre_stride, upred_ptr, uv_stride, &sse2);
    vp9_variance8x8(vptr, pre_stride, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
417
418
419
    sse2 += sse1;
  }
  return sse2;
John Koleszar's avatar
John Koleszar committed
420
421
422

}

423
424
425
426
427
#if CONFIG_NEWCOEFCONTEXT
#define PT pn
#else
#define PT pt
#endif
428
429
430
431
static int cost_coeffs(MACROBLOCK *mb,
                       BLOCKD *b, PLANE_TYPE type,
                       ENTROPY_CONTEXT *a,
                       ENTROPY_CONTEXT *l,
432
                       TX_SIZE tx_size) {
433
  int pt;
Daniel Kang's avatar
Daniel Kang committed
434
  const int eob = b->eob;
435
  MACROBLOCKD *xd = &mb->e_mbd;
436
437
438
439
440
441
442
443
  const int ib = (int)(b - xd->block);
  int c = (type == PLANE_TYPE_Y_NO_DC) ? 1 : 0;
  int cost = 0, seg_eob;
  const int segment_id = xd->mode_info_context->mbmi.segment_id;
  const int *scan, *band;
  int16_t *qcoeff_ptr = b->qcoeff;
  const TX_TYPE tx_type = (type == PLANE_TYPE_Y_WITH_DC) ?
                          get_tx_type(xd, b) : DCT_DCT;
444
445
446
447
#if CONFIG_NEWCOEFCONTEXT
  const int *neighbors;
  int pn;
#endif
448
449

  ENTROPY_CONTEXT a_ec = *a, l_ec = *l;
450

451
  switch (tx_size) {
Daniel Kang's avatar
Daniel Kang committed
452
    case TX_4X4:
453
454
455
      scan = vp9_default_zig_zag1d_4x4;
      band = vp9_coef_bands_4x4;
      seg_eob = 16;
Deb Mukherjee's avatar
Deb Mukherjee committed
456
      if (type == PLANE_TYPE_Y_WITH_DC) {
457
458
459
460
        if (tx_type == ADST_DCT) {
          scan = vp9_row_scan_4x4;
        } else if (tx_type == DCT_ADST) {
          scan = vp9_col_scan_4x4;
461
        }
Daniel Kang's avatar
Daniel Kang committed
462
463
464
      }
      break;
    case TX_8X8:
465
      if (type == PLANE_TYPE_Y2) {
466
467
        scan = vp9_default_zig_zag1d_4x4;
        band = vp9_coef_bands_4x4;
468
469
470
471
472
        seg_eob = 4;
      } else {
        scan = vp9_default_zig_zag1d_8x8;
        band = vp9_coef_bands_8x8;
        seg_eob = 64;
473
      }
Daniel Kang's avatar
Daniel Kang committed
474
475
      break;
    case TX_16X16:
476
477
      scan = vp9_default_zig_zag1d_16x16;
      band = vp9_coef_bands_16x16;
478
479
480
481
      seg_eob = 256;
      if (type == PLANE_TYPE_UV) {
        const int uv_idx = ib - 16;
        qcoeff_ptr = xd->sb_coeff_data.qcoeff + 1024 + 64 * uv_idx;
Deb Mukherjee's avatar
Deb Mukherjee committed
482
      }
Daniel Kang's avatar
Daniel Kang committed
483
      break;
484
485
486
    case TX_32X32:
      scan = vp9_default_zig_zag1d_32x32;
      band = vp9_coef_bands_32x32;
487
      seg_eob = 1024;
488
489
      qcoeff_ptr = xd->sb_coeff_data.qcoeff;
      break;
Daniel Kang's avatar
Daniel Kang committed
490
    default:
491
      abort();
Daniel Kang's avatar
Daniel Kang committed
492
493
      break;
  }
494

495
  VP9_COMBINEENTROPYCONTEXTS(pt, a_ec, l_ec);
496
497
498
499
#if CONFIG_NEWCOEFCONTEXT
  neighbors = vp9_get_coef_neighbors_handle(scan);
  pn = pt;
#endif
500

Paul Wilkins's avatar
Paul Wilkins committed
501
502
  if (vp9_segfeature_active(xd, segment_id, SEG_LVL_SKIP))
    seg_eob = 0;
503

504
505
506
  if (tx_type != DCT_DCT) {
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
507
      int t = vp9_dct_value_tokens_ptr[v].Token;
508
      cost += mb->hybrid_token_costs[tx_size][type][band[c]][PT][t];
509
510
      cost += vp9_dct_value_cost_ptr[v];
      pt = vp9_prev_token_class[t];
511
512
513
#if CONFIG_NEWCOEFCONTEXT
      if (c < seg_eob - 1 && NEWCOEFCONTEXT_BAND_COND(band[c + 1]))
        pn = vp9_get_coef_neighbor_context(
514
            qcoeff_ptr, (type == PLANE_TYPE_Y_NO_DC), neighbors, scan[c + 1]);
515
516
517
      else
        pn = pt;
#endif
518
519
520
    }
    if (c < seg_eob)
      cost += mb->hybrid_token_costs[tx_size][type][band[c]]
521
          [PT][DCT_EOB_TOKEN];
522
  } else {
523
524
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
525
      int t = vp9_dct_value_tokens_ptr[v].Token;
526
      cost += mb->token_costs[tx_size][type][band[c]][pt][t];
527
528
      cost += vp9_dct_value_cost_ptr[v];
      pt = vp9_prev_token_class[t];
529
530
531
#if CONFIG_NEWCOEFCONTEXT
      if (c < seg_eob - 1 && NEWCOEFCONTEXT_BAND_COND(band[c + 1]))
        pn = vp9_get_coef_neighbor_context(
532
            qcoeff_ptr, (type == PLANE_TYPE_Y_NO_DC), neighbors, scan[c + 1]);
533
534
535
      else
        pn = pt;
#endif
536
537
538
    }
    if (c < seg_eob)
      cost += mb->token_costs[tx_size][type][band[c]]
539
          [PT][DCT_EOB_TOKEN];
540
541
  }

542
543
  // is eob first coefficient;
  pt = (c > !type);
544
545
546
547
  *a = *l = pt;
  return cost;
}

548
static int rdcost_mby_4x4(MACROBLOCK *mb, int has_2nd_order, int backup) {
John Koleszar's avatar
John Koleszar committed
549
550
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
551
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
552
553
554
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta;
  ENTROPY_CONTEXT *tl;
555

556
557
558
  if (backup) {
    vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
559

560
561
562
563
564
565
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)xd->above_context;
    tl = (ENTROPY_CONTEXT *)xd->left_context;
  }
John Koleszar's avatar
John Koleszar committed
566

John Koleszar's avatar
John Koleszar committed
567
  for (b = 0; b < 16; b++)
568
569
570
    cost += cost_coeffs(mb, xd->block + b,
                        (has_2nd_order ?
                         PLANE_TYPE_Y_NO_DC : PLANE_TYPE_Y_WITH_DC),
Yaowu Xu's avatar
Yaowu Xu committed
571
572
                        ta + vp9_block2above[TX_4X4][b],
                        tl + vp9_block2left[TX_4X4][b],
Daniel Kang's avatar
Daniel Kang committed
573
                        TX_4X4);
John Koleszar's avatar
John Koleszar committed
574

575
576
  if (has_2nd_order)
    cost += cost_coeffs(mb, xd->block + 24, PLANE_TYPE_Y2,
Yaowu Xu's avatar
Yaowu Xu committed
577
578
                        ta + vp9_block2above[TX_4X4][24],
                        tl + vp9_block2left[TX_4X4][24],
579
                        TX_4X4);
John Koleszar's avatar
John Koleszar committed
580

John Koleszar's avatar
John Koleszar committed
581
  return cost;
John Koleszar's avatar
John Koleszar committed
582
583
}

584
585
586
static void macro_block_yrd_4x4(MACROBLOCK *mb,
                                int *Rate,
                                int *Distortion,
587
                                int *skippable, int backup) {
Paul Wilkins's avatar
Paul Wilkins committed
588
  MACROBLOCKD *const xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
589
  BLOCK   *const mb_y2 = mb->block + 24;
Paul Wilkins's avatar
Paul Wilkins committed
590
  BLOCKD *const x_y2  = xd->block + 24;
Jim Bankoski's avatar
Jim Bankoski committed
591
  int d, has_2nd_order;
592

593
594
  xd->mode_info_context->mbmi.txfm_size = TX_4X4;
  has_2nd_order = get_2nd_order_usage(xd);
John Koleszar's avatar
John Koleszar committed
595
  // Fdct and building the 2nd order block
596
597
598
599
600
  vp9_transform_mby_4x4(mb);
  vp9_quantize_mby_4x4(mb);
  d = vp9_mbblock_error(mb, has_2nd_order);
  if (has_2nd_order)
    d += vp9_block_error(mb_y2->coeff, x_y2->dqcoeff, 16);
601

John Koleszar's avatar
John Koleszar committed
602
603
  *Distortion = (d >> 2);
  // rate
604
605
  *Rate = rdcost_mby_4x4(mb, has_2nd_order, backup);
  *skippable = vp9_mby_is_skippable_4x4(&mb->e_mbd, has_2nd_order);
606
}
John Koleszar's avatar
John Koleszar committed
607

608
static int rdcost_mby_8x8(MACROBLOCK *mb, int has_2nd_order, int backup) {
John Koleszar's avatar
John Koleszar committed
609
610
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
611
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
612
613
614
615
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta;
  ENTROPY_CONTEXT *tl;

Ronald S. Bultje's avatar
Ronald S. Bultje committed
616
617
618
  if (backup) {
    vpx_memcpy(&t_above,xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
619

Ronald S. Bultje's avatar
Ronald S. Bultje committed
620
621
622
623
624
625
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)mb->e_mbd.above_context;
    tl = (ENTROPY_CONTEXT *)mb->e_mbd.left_context;
  }
John Koleszar's avatar
John Koleszar committed
626
627

  for (b = 0; b < 16; b += 4)
628
629
630
    cost += cost_coeffs(mb, xd->block + b,
                        (has_2nd_order ?
                         PLANE_TYPE_Y_NO_DC : PLANE_TYPE_Y_WITH_DC),
Yaowu Xu's avatar
Yaowu Xu committed
631
632
                        ta + vp9_block2above[TX_8X8][b],
                        tl + vp9_block2left[TX_8X8][b],
Daniel Kang's avatar
Daniel Kang committed
633
                        TX_8X8);
John Koleszar's avatar
John Koleszar committed
634

635
  if (has_2nd_order)
636
    cost += cost_coeffs(mb, xd->block + 24, PLANE_TYPE_Y2,
Yaowu Xu's avatar
Yaowu Xu committed
637
                            ta + vp9_block2above[TX_8X8][24],
638
639
                            tl + vp9_block2left[TX_8X8][24],
                            TX_8X8);
John Koleszar's avatar
John Koleszar committed
640
  return cost;
641
642
}

John Koleszar's avatar
John Koleszar committed
643
644
645
static void macro_block_yrd_8x8(MACROBLOCK *mb,
                                int *Rate,
                                int *Distortion,
646
                                int *skippable, int backup) {
Paul Wilkins's avatar
Paul Wilkins committed
647
  MACROBLOCKD *const xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
648
  BLOCK   *const mb_y2 = mb->block + 24;
Paul Wilkins's avatar
Paul Wilkins committed
649
  BLOCKD *const x_y2  = xd->block + 24;
650
651
652
  int d, has_2nd_order;

  xd->mode_info_context->mbmi.txfm_size = TX_8X8;
John Koleszar's avatar
John Koleszar committed
653

654
655
  vp9_transform_mby_8x8(mb);
  vp9_quantize_mby_8x8(mb);
656
657
658
659
  has_2nd_order = get_2nd_order_usage(xd);
  d = vp9_mbblock_error_8x8_c(mb, has_2nd_order);
  if (has_2nd_order)
    d += vp9_block_error(mb_y2->coeff, x_y2->dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
660
661
662

  *Distortion = (d >> 2);
  // rate
663
664
  *Rate = rdcost_mby_8x8(mb, has_2nd_order, backup);
  *skippable = vp9_mby_is_skippable_8x8(&mb->e_mbd, has_2nd_order);
665
}
666

667
static int rdcost_mby_16x16(MACROBLOCK *mb, int backup) {
Daniel Kang's avatar
Daniel Kang committed
668
  int cost;
Paul Wilkins's avatar
Paul Wilkins committed
669
  MACROBLOCKD *xd = &mb->e_mbd;
Daniel Kang's avatar
Daniel Kang committed
670
671
672
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta, *tl;

673
674
675
  if (backup) {
    vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
Daniel Kang's avatar
Daniel Kang committed
676

677
678
679
680
681
682
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)xd->above_context;
    tl = (ENTROPY_CONTEXT *)xd->left_context;
  }
Daniel Kang's avatar
Daniel Kang committed
683

Paul Wilkins's avatar
Paul Wilkins committed
684
  cost = cost_coeffs(mb, xd->block, PLANE_TYPE_Y_WITH_DC, ta, tl, TX_16X16);
Daniel Kang's avatar
Daniel Kang committed
685
686
  return cost;
}
687

Daniel Kang's avatar
Daniel Kang committed
688
static void macro_block_yrd_16x16(MACROBLOCK *mb, int *Rate, int *Distortion,
689
                                  int *skippable, int backup) {
Daniel Kang's avatar
Daniel Kang committed
690
  int d;
Deb Mukherjee's avatar
Deb Mukherjee committed
691
  MACROBLOCKD *xd = &mb->e_mbd;
Daniel Kang's avatar
Daniel Kang committed
692

693
694
  xd->mode_info_context->mbmi.txfm_size = TX_16X16;
  vp9_transform_mby_16x16(mb);
695
  vp9_quantize_mby_16x16(mb);
696
697
698
  // TODO(jingning) is it possible to quickly determine whether to force
  //                trailing coefficients to be zero, instead of running trellis
  //                optimization in the rate-distortion optimization loop?
699
  if (mb->e_mbd.mode_info_context->mbmi.mode < I8X8_PRED)
700
    vp9_optimize_mby_16x16(mb);
701

702
  d = vp9_mbblock_error(mb, 0);
Daniel Kang's avatar
Daniel Kang committed
703
704
705

  *Distortion = (d >> 2);
  // rate
706
  *Rate = rdcost_mby_16x16(mb, backup);
707
  *skippable = vp9_mby_is_skippable_16x16(&mb->e_mbd);
Daniel Kang's avatar
Daniel Kang committed
708
709
}

710
static void choose_txfm_size_from_rd(VP9_COMP *cpi, MACROBLOCK *x,
711
712
713
714
715
                                     int (*r)[2], int *rate,
                                     int *d, int *distortion,
                                     int *s, int *skip,
                                     int64_t txfm_cache[NB_TXFM_MODES],
                                     TX_SIZE max_txfm_size) {
716
717
718
719
720
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
  vp9_prob skip_prob = cm->mb_no_coeff_skip ?
                       vp9_get_pred_prob(cm, xd, PRED_MBSKIP) : 128;
721
722
723
724
725
726
727
728
729
730
731
732
  int64_t rd[TX_SIZE_MAX_SB][2];
  int n, m;

  for (n = TX_4X4; n <= max_txfm_size; n++) {
    r[n][1] = r[n][0];
    for (m = 0; m <= n - (n == max_txfm_size); m++) {
      if (m == n)
        r[n][1] += vp9_cost_zero(cm->prob_tx[m]);
      else
        r[n][1] += vp9_cost_one(cm->prob_tx[m]);
    }
  }
733
734
735
736
737
738
739
740

  if (cm->mb_no_coeff_skip) {
    int s0, s1;

    assert(skip_prob > 0);
    s0 = vp9_cost_bit(skip_prob, 0);
    s1 = vp9_cost_bit(skip_prob, 1);

741
    for (n = TX_4X4; n <= max_txfm_size; n++) {
742
      if (s[n]) {
743
        rd[n][0] = rd[n][1] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
744
      } else {
745
746
        rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0] + s0, d[n]);
        rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1] + s0, d[n]);
747
      }
748
749
    }
  } else {
750
751
752
    for (n = TX_4X4; n <= max_txfm_size; n++) {
      rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0], d[n]);
      rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1], d[n]);
753
754
755
    }
  }

756
757
758
759
760
761
  if (max_txfm_size == TX_32X32 &&
      (cm->txfm_mode == ALLOW_32X32 ||
       (cm->txfm_mode == TX_MODE_SELECT &&
        rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
        rd[TX_32X32][1] < rd[TX_4X4][1]))) {
    mbmi->txfm_size = TX_32X32;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
762
763
764
765
766
  } else if ( cm->txfm_mode == ALLOW_16X16 ||
             (max_txfm_size == TX_16X16 && cm->txfm_mode == ALLOW_32X32) ||
             (cm->txfm_mode == TX_MODE_SELECT &&
              rd[TX_16X16][1] < rd[TX_8X8][1] &&
              rd[TX_16X16][1] < rd[TX_4X4][1])) {
767
    mbmi->txfm_size = TX_16X16;
768
  } else if (cm->txfm_mode == ALLOW_8X8 ||
769
           (cm->txfm_mode == TX_MODE_SELECT && rd[TX_8X8][1] < rd[TX_4X4][1])) {
770
771
    mbmi->txfm_size = TX_8X8;
  } else {
772
    assert(cm->txfm_mode == ONLY_4X4 || cm->txfm_mode == TX_MODE_SELECT);
773
774
775
    mbmi->txfm_size = TX_4X4;
  }

776
  *distortion = d[mbmi->txfm_size];
777
  *rate       = r[mbmi->txfm_size][cm->txfm_mode == TX_MODE_SELECT];
778
779
  *skip       = s[mbmi->txfm_size];

780
781
782
783
784
785
786
787
  txfm_cache[ONLY_4X4] = rd[TX_4X4][0];
  txfm_cache[ALLOW_8X8] = rd[TX_8X8][0];
  txfm_cache[ALLOW_16X16] = rd[TX_16X16][0];
  txfm_cache[ALLOW_32X32] = rd[max_txfm_size][0];
  if (max_txfm_size == TX_32X32 &&
      rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
      rd[TX_32X32][1] < rd[TX_4X4][1])
    txfm_cache[TX_MODE_SELECT] = rd[TX_32X32][1];
Ronald S. Bultje's avatar
Ronald S. Bultje committed
788
  else if (rd[TX_16X16][1] < rd[TX_8X8][1] && rd[TX_16X16][1] < rd[TX_4X4][1])
789
    txfm_cache[TX_MODE_SELECT] = rd[TX_16X16][1];
790
  else
791
792
    txfm_cache[TX_MODE_SELECT] = rd[TX_4X4][1] < rd[TX_8X8][1] ?
                                 rd[TX_4X4][1] : rd[TX_8X8][1];
793
794
795
796
797
798
}

static void macro_block_yrd(VP9_COMP *cpi, MACROBLOCK *x, int *rate,
                            int *distortion, int *skippable,
                            int64_t txfm_cache[NB_TXFM_MODES]) {
  MACROBLOCKD *const xd = &x->e_mbd;
799
  int r[TX_SIZE_MAX_MB][2], d[TX_SIZE_MAX_MB], s[TX_SIZE_MAX_MB];
800
801
802

  vp9_subtract_mby(x->src_diff, *(x->block[0].base_src), xd->predictor,
                   x->block[0].src_stride);
803

804
805
806
  macro_block_yrd_16x16(x, &r[TX_16X16][0], &d[TX_16X16], &s[TX_16X16], 1);
  macro_block_yrd_8x8(x, &r[TX_8X8][0], &d[TX_8X8], &s[TX_8X8], 1);
  macro_block_yrd_4x4(x, &r[TX_4X4][0], &d[TX_4X4], &s[TX_4X4], 1);
807
808

  choose_txfm_size_from_rd(cpi, x, r, rate, d, distortion, s, skippable,
809
                           txfm_cache, TX_16X16);
810
811
}

812
static void copy_predictor(uint8_t *dst, const uint8_t *predictor) {
John Koleszar's avatar
John Koleszar committed
813
814
815
816
817
818
  const unsigned int *p = (const unsigned int *)predictor;
  unsigned int *d = (unsigned int *)dst;
  d[0] = p[0];
  d[4] = p[4];
  d[8] = p[8];
  d[12] = p[12];
819
}
Yaowu Xu's avatar
Yaowu Xu committed
820

Ronald S. Bultje's avatar
Ronald S. Bultje committed
821
static int rdcost_sby_32x32(MACROBLOCK *x, int backup) {
822
823
  MACROBLOCKD * const xd = &x->e_mbd;
  ENTROPY_CONTEXT_PLANES t_above, t_left;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
824
  ENTROPY_CONTEXT *ta, *tl;
825

Ronald S. Bultje's avatar
Ronald S. Bultje committed
826
827
828
829
830
831
832
833
834
835
  if (backup) {
    ta = (ENTROPY_CONTEXT *) &t_above,
    tl = (ENTROPY_CONTEXT *) &t_left;

    vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left,  xd->left_context,  sizeof(ENTROPY_CONTEXT_PLANES));
  } else {
    ta = (ENTROPY_CONTEXT *) xd->above_context;
    tl = (ENTROPY_CONTEXT *) xd->left_context;
  }
836
837
838
839

  return cost_coeffs(x, xd->block, PLANE_TYPE_Y_WITH_DC, ta, tl, TX_32X32);
}

840
841
static int vp9_sb_block_error_c(int16_t *coeff, int16_t *dqcoeff,
                                int block_size) {
842
843
844
845
846
847
848
849
850
851
852
853
854
  int i;
  int64_t error = 0;

  for (i = 0; i < block_size; i++) {
    unsigned int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }

  return error > INT_MAX ? INT_MAX : error;
}

#define DEBUG_ERROR 0
static void super_block_yrd_32x32(MACROBLOCK *x,
Ronald S. Bultje's avatar
Ronald S. Bultje committed
855
856
                                  int *rate, int *distortion, int *skippable,
                                  int backup) {
857
858
859
  SUPERBLOCK  * const x_sb = &x->sb_coeff_data;
  MACROBLOCKD * const xd = &x->e_mbd;
  SUPERBLOCKD * const xd_sb = &xd->sb_coeff_data;
860
#if DEBUG_ERROR || CONFIG_DWTDCTHYBRID
861
  int16_t out[1024];
862
863
864
865
#endif

  vp9_transform_sby_32x32(x);
  vp9_quantize_sby_32x32(x);
866
#if DEBUG_ERROR || CONFIG_DWTDCTHYBRID
867
868
869
  vp9_short_idct32x32(xd_sb->dqcoeff, out, 64);
#endif

870
#if !CONFIG_DWTDCTHYBRID
871
872
873
874
875
876
877
878
  *distortion = vp9_sb_block_error_c(x_sb->coeff, xd_sb->dqcoeff, 1024);
#else
  *distortion = vp9_block_error_c(x_sb->src_diff, out, 1024) << 4;
#endif
#if DEBUG_ERROR
  printf("IDCT/FDCT error 32x32: %d (d: %d)\n",
         vp9_block_error_c(x_sb->src_diff, out, 1024), *distortion);
#endif
Ronald S. Bultje's avatar
Ronald S. Bultje committed
879
  *rate       = rdcost_sby_32x32(x, backup);
880
881
882
  *skippable  = vp9_sby_is_skippable_32x32(&x->e_mbd);
}

883
884
static void super_block_yrd(VP9_COMP *cpi,
                            MACROBLOCK *x, int *rate, int *distortion,
885
                            int *skip,
886
                            int64_t txfm_cache[NB_TXFM_MODES]) {
Ronald S. Bultje's avatar
Ronald S. Bultje committed
887
  MACROBLOCKD *const xd = &x->e_mbd;
888
  int r[TX_SIZE_MAX_SB][2], d[TX_SIZE_MAX_SB], s[TX_SIZE_MAX_SB], n;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
889
890
  const uint8_t *src = x->src.y_buffer, *dst = xd->dst.y_buffer;
  int src_y_stride = x->src.y_stride, dst_y_stride = xd->dst.y_stride;
891
892
893
894
  ENTROPY_CONTEXT_PLANES t_above[TX_SIZE_MAX_MB][2],
                        *orig_above = xd->above_context;
  ENTROPY_CONTEXT_PLANES t_left[TX_SIZE_MAX_MB][2],
                        *orig_left = xd->left_context;
895

896
  for (n = TX_4X4; n < TX_SIZE_MAX_MB; n++) {
897
898
    vpx_memcpy(t_above[n], xd->above_context, sizeof(t_above[n]));
    vpx_memcpy(t_left[n], xd->left_context, sizeof(t_left[n]));
899
    r[n][0] = 0;
900
901
902
    d[n] = 0;
    s[n] = 1;
  }
Ronald S. Bultje's avatar
Ronald S. Bultje committed
903

904
905
  vp9_subtract_sby_s_c(x->sb_coeff_data.src_diff, src, src_y_stride,
                       dst, dst_y_stride);
Ronald S. Bultje's avatar
Ronald S. Bultje committed
906
  super_block_yrd_32x32(x, &r[TX_32X32][0], &d[TX_32X32], &s[TX_32X32], 1);
907
908
909
910

#if DEBUG_ERROR
  int err[3] = { 0, 0, 0 };
#endif
Ronald S. Bultje's avatar
Ronald S. Bultje committed
911
912
  for (n = 0; n < 4; n++) {
    int x_idx = n & 1, y_idx = n >> 1;
913
    int r_tmp, d_tmp, s_tmp;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
914

915
    vp9_subtract_mby_s_c(x->src_diff,
Ronald S. Bultje's avatar
Ronald S. Bultje committed
916
917
918
919
                         src + x_idx * 16 + y_idx * 16 * src_y_stride,
                         src_y_stride,
                         dst + x_idx * 16 + y_idx * 16 * dst_y_stride,
                         dst_y_stride);
920
921
922

    xd->above_context = &t_above[TX_16X16][x_idx];
    xd->left_context = &t_left[TX_16X16][y_idx];
923
    macro_block_yrd_16x16(x, &r_tmp, &d_tmp, &s_tmp, 0);
924
    d[TX_16X16] += d_tmp;
925
    r[TX_16X16][0] += r_tmp;
926
    s[TX_16X16] = s[TX_16X16] && s_tmp;
927
928
929
930
#if DEBUG_ERROR
    vp9_inverse_transform_mby_16x16(xd);
    err[2] += vp9_block_error_c(xd->diff, x->src_diff, 256);
#endif
931
932
933

    xd->above_context = &t_above[TX_4X4][x_idx];
    xd->left_context = &t_left[TX_4X4][y_idx];
934
    macro_block_yrd_4x4(x, &r_tmp, &d_tmp, &s_tmp, 0);
935
    d[TX_4X4] += d_tmp;
936
    r[TX_4X4][0] += r_tmp;
937
    s[TX_4X4] = s[TX_4X4] && s_tmp;
938
939
940
941
#if DEBUG_ERROR
    vp9_inverse_transform_mby_4x4(xd);
    err[0] += vp9_block_error_c(xd->diff, x->src_diff, 256);
#endif
942
943
944

    xd->above_context = &t_above[TX_8X8][x_idx];
    xd->left_context = &t_left[TX_8X8][y_idx];
945
    macro_block_yrd_8x8(x, &r_tmp, &d_tmp, &s_tmp, 0);
946
    d[TX_8X8] += d_tmp;
947
    r[TX_8X8][0] += r_tmp;
948
    s[TX_8X8] = s[TX_8X8] && s_tmp;
949
950
951
952
#if DEBUG_ERROR
    vp9_inverse_transform_mby_8x8(xd);
    err[1] += vp9_block_error_c(xd->diff, x->src_diff, 256);
#endif
Ronald S. Bultje's avatar
Ronald S. Bultje committed
953
  }
954
955
956
957
958
959
960
#if DEBUG_ERROR
  printf("IDCT/FDCT error 16x16: %d (d: %d)\n", err[2], d[2]);
  printf("IDCT/FDCT error 8x8: %d (d: %d)\n", err[1], d[1]);
  printf("IDCT/FDCT error 4x4: %d (d: %d)\n", err[0], d[0]);
#endif
  choose_txfm_size_from_rd(cpi, x, r, rate, d, distortion, s, skip, txfm_cache,
                           TX_SIZE_MAX_SB - 1);
961
962
963

  xd->above_context = orig_above;
  xd->left_context = orig_left;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
964
}
Ronald S. Bultje's avatar
Ronald S. Bultje committed
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
</