vp9_rdopt.c 206 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5
6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9
10
11
12
13
14
15
 */


#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
16
17
#include "vp9/common/vp9_pragmas.h"

18
19
20
21
22
#include "vp9/encoder/vp9_tokenize.h"
#include "vp9/encoder/vp9_treewriter.h"
#include "vp9/encoder/vp9_onyx_int.h"
#include "vp9/encoder/vp9_modecosts.h"
#include "vp9/encoder/vp9_encodeintra.h"
23
24
25
26
27
#include "vp9/common/vp9_entropymode.h"
#include "vp9/common/vp9_reconinter.h"
#include "vp9/common/vp9_reconintra.h"
#include "vp9/common/vp9_findnearmv.h"
#include "vp9/common/vp9_quant_common.h"
28
29
30
31
32
33
#include "vp9/encoder/vp9_encodemb.h"
#include "vp9/encoder/vp9_quantize.h"
#include "vp9/encoder/vp9_variance.h"
#include "vp9/encoder/vp9_mcomp.h"
#include "vp9/encoder/vp9_rdopt.h"
#include "vp9/encoder/vp9_ratectrl.h"
John Koleszar's avatar
John Koleszar committed
34
#include "vpx_mem/vpx_mem.h"
35
36
#include "vp9/common/vp9_systemdependent.h"
#include "vp9/encoder/vp9_encodemv.h"
John Koleszar's avatar
John Koleszar committed
37

38
39
40
#include "vp9/common/vp9_seg_common.h"
#include "vp9/common/vp9_pred_common.h"
#include "vp9/common/vp9_entropy.h"
41
#include "vp9_rtcd.h"
42
#include "vp9/common/vp9_mvref_common.h"
Ronald S. Bultje's avatar
Ronald S. Bultje committed
43
#include "vp9/common/vp9_common.h"
Paul Wilkins's avatar
Paul Wilkins committed
44

John Koleszar's avatar
John Koleszar committed
45
46
#define MAXF(a,b)            (((a) > (b)) ? (a) : (b))

47
48
#define INVALID_MV 0x80008000

49
50
51
/* Factor to weigh the rate for switchable interp filters */
#define SWITCHABLE_INTERP_RATE_FACTOR 1

John Koleszar's avatar
John Koleszar committed
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
static const int auto_speed_thresh[17] = {
  1000,
  200,
  150,
  130,
  150,
  125,
  120,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  105
John Koleszar's avatar
John Koleszar committed
70
71
};

72
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
73
74
  {ZEROMV,    LAST_FRAME,   NONE},
  {DC_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
75

76
77
  {NEARESTMV, LAST_FRAME,   NONE},
  {NEARMV,    LAST_FRAME,   NONE},
John Koleszar's avatar
John Koleszar committed
78

79
80
  {ZEROMV,    GOLDEN_FRAME, NONE},
  {NEARESTMV, GOLDEN_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
81

82
83
  {ZEROMV,    ALTREF_FRAME, NONE},
  {NEARESTMV, ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
84

85
86
  {NEARMV,    GOLDEN_FRAME, NONE},
  {NEARMV,    ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
87

88
89
90
91
92
93
94
95
  {V_PRED,    INTRA_FRAME,  NONE},
  {H_PRED,    INTRA_FRAME,  NONE},
  {D45_PRED,  INTRA_FRAME,  NONE},
  {D135_PRED, INTRA_FRAME,  NONE},
  {D117_PRED, INTRA_FRAME,  NONE},
  {D153_PRED, INTRA_FRAME,  NONE},
  {D27_PRED,  INTRA_FRAME,  NONE},
  {D63_PRED,  INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
96

97
  {TM_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
98

99
100
101
  {NEWMV,     LAST_FRAME,   NONE},
  {NEWMV,     GOLDEN_FRAME, NONE},
  {NEWMV,     ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
102

103
104
105
  {SPLITMV,   LAST_FRAME,   NONE},
  {SPLITMV,   GOLDEN_FRAME, NONE},
  {SPLITMV,   ALTREF_FRAME, NONE},
106

107
108
  {B_PRED,    INTRA_FRAME,  NONE},
  {I8X8_PRED, INTRA_FRAME,  NONE},
109

John Koleszar's avatar
John Koleszar committed
110
111
112
113
  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME},
114

John Koleszar's avatar
John Koleszar committed
115
116
117
  {ZEROMV,    ALTREF_FRAME, LAST_FRAME},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME},
118

John Koleszar's avatar
John Koleszar committed
119
120
121
  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME},
122

John Koleszar's avatar
John Koleszar committed
123
124
125
  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME  },
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME},
126

John Koleszar's avatar
John Koleszar committed
127
128
  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME  },
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME},

#if CONFIG_COMP_INTERINTRA_PRED
  /* compound inter-intra prediction */
  {ZEROMV,    LAST_FRAME,   INTRA_FRAME},
  {NEARESTMV, LAST_FRAME,   INTRA_FRAME},
  {NEARMV,    LAST_FRAME,   INTRA_FRAME},
  {NEWMV,     LAST_FRAME,   INTRA_FRAME},

  {ZEROMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEARESTMV, GOLDEN_FRAME,   INTRA_FRAME},
  {NEARMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEWMV,     GOLDEN_FRAME,   INTRA_FRAME},

  {ZEROMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEARESTMV, ALTREF_FRAME,   INTRA_FRAME},
  {NEARMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEWMV,     ALTREF_FRAME,   INTRA_FRAME},
#endif
John Koleszar's avatar
John Koleszar committed
148
149
};

150
151
152
static void fill_token_costs(vp9_coeff_count *c,
                             vp9_coeff_probs *p,
                             int block_type_counts) {
153
  int i, j, k, l;
John Koleszar's avatar
John Koleszar committed
154
155

  for (i = 0; i < block_type_counts; i++)
156
157
158
    for (j = 0; j < REF_TYPES; j++)
      for (k = 0; k < COEF_BANDS; k++)
        for (l = 0; l < PREV_COEF_CONTEXTS; l++) {
159
160
161
162
163
164
#if CONFIG_CODE_NONZEROCOUNT
          // All costs are without the EOB node
          vp9_cost_tokens_skip((int *)(c[i][j][k][l]),
                               p[i][j][k][l],
                               vp9_coef_tree);
#else
165
166
167
168
169
170
171
172
          if (l == 0 && k > 0)
            vp9_cost_tokens_skip((int *)(c[i][j][k][l]),
                                 p[i][j][k][l],
                                 vp9_coef_tree);
          else
            vp9_cost_tokens((int *)(c[i][j][k][l]),
                            p[i][j][k][l],
                            vp9_coef_tree);
173
174
175
176
177
178
179
180
181
182
183
184
185
#endif
        }
}

#if CONFIG_CODE_NONZEROCOUNT
static void fill_nzc_costs(VP9_COMP *cpi, int block_size) {
  int nzc_context, r, b, nzc, values;
  int cost[16];
  values = block_size * block_size + 1;

  for (nzc_context = 0; nzc_context < MAX_NZC_CONTEXTS; ++nzc_context) {
    for (r = 0; r < REF_TYPES; ++r) {
      for (b = 0; b < BLOCK_TYPES; ++b) {
186
187
        unsigned int *nzc_costs;
        if (block_size == 4) {
188
189
190
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_4x4[nzc_context][r][b],
                          vp9_nzc4x4_tree);
191
192
          nzc_costs = cpi->mb.nzc_costs_4x4[nzc_context][r][b];
        } else if (block_size == 8) {
193
194
195
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_8x8[nzc_context][r][b],
                          vp9_nzc8x8_tree);
196
197
          nzc_costs = cpi->mb.nzc_costs_8x8[nzc_context][r][b];
        } else if (block_size == 16) {
198
199
200
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_16x16[nzc_context][r][b],
                          vp9_nzc16x16_tree);
201
202
          nzc_costs = cpi->mb.nzc_costs_16x16[nzc_context][r][b];
        } else {
203
204
205
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_32x32[nzc_context][r][b],
                          vp9_nzc32x32_tree);
206
207
          nzc_costs = cpi->mb.nzc_costs_32x32[nzc_context][r][b];
        }
208
209
210
211
212

        for (nzc = 0; nzc < values; ++nzc) {
          int e, c, totalcost = 0;
          c = codenzc(nzc);
          totalcost = cost[c];
213
214
          if ((e = vp9_extranzcbits[c])) {
            int x = nzc - vp9_basenzcvalue[c];
215
            while (e--) {
216
217
218
219
              totalcost += vp9_cost_bit(
                  cpi->common.fc.nzc_pcat_probs[nzc_context]
                                               [c - NZC_TOKENS_NOEXTRA][e],
                  ((x >> e) & 1));
220
221
            }
          }
222
          nzc_costs[nzc] = totalcost;
223
        }
224
225
226
      }
    }
  }
John Koleszar's avatar
John Koleszar committed
227
}
228
#endif
John Koleszar's avatar
John Koleszar committed
229

230

231
232
233
234
static int rd_iifactor[32] =  { 4, 4, 3, 2, 1, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0, };
John Koleszar's avatar
John Koleszar committed
235

236
// 3* dc_qlookup[Q]*dc_qlookup[Q];
237

238
/* values are now correlated to quantizer */
Paul Wilkins's avatar
Paul Wilkins committed
239
240
241
static int sad_per_bit16lut[QINDEX_RANGE];
static int sad_per_bit4lut[QINDEX_RANGE];

242
void vp9_init_me_luts() {
John Koleszar's avatar
John Koleszar committed
243
244
245
246
247
248
249
  int i;

  // Initialize the sad lut tables using a formulaic calculation for now
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < QINDEX_RANGE; i++) {
    sad_per_bit16lut[i] =
250
251
      (int)((0.0418 * vp9_convert_qindex_to_q(i)) + 2.4107);
    sad_per_bit4lut[i] = (int)((0.063 * vp9_convert_qindex_to_q(i)) + 2.742);
John Koleszar's avatar
John Koleszar committed
252
  }
Paul Wilkins's avatar
Paul Wilkins committed
253
}
John Koleszar's avatar
John Koleszar committed
254

255
static int compute_rd_mult(int qindex) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
256
  int q = vp9_dc_quant(qindex, 0);
John Koleszar's avatar
John Koleszar committed
257
  return (11 * q * q) >> 6;
258
259
}

Dmitry Kovalev's avatar
Dmitry Kovalev committed
260
261
262
void vp9_initialize_me_consts(VP9_COMP *cpi, int qindex) {
  cpi->mb.sadperbit16 = sad_per_bit16lut[qindex];
  cpi->mb.sadperbit4 = sad_per_bit4lut[qindex];
John Koleszar's avatar
John Koleszar committed
263
264
}

265

Dmitry Kovalev's avatar
Dmitry Kovalev committed
266
void vp9_initialize_rd_consts(VP9_COMP *cpi, int qindex) {
267
  int q, i;
John Koleszar's avatar
John Koleszar committed
268

269
  vp9_clear_system_state();  // __asm emms;
John Koleszar's avatar
John Koleszar committed
270

John Koleszar's avatar
John Koleszar committed
271
272
273
274
  // Further tests required to see if optimum is different
  // for key frames, golden frames and arf frames.
  // if (cpi->common.refresh_golden_frame ||
  //     cpi->common.refresh_alt_ref_frame)
Dmitry Kovalev's avatar
Dmitry Kovalev committed
275
  qindex = (qindex < 0) ? 0 : ((qindex > MAXQ) ? MAXQ : qindex);
276

Dmitry Kovalev's avatar
Dmitry Kovalev committed
277
  cpi->RDMULT = compute_rd_mult(qindex);
John Koleszar's avatar
John Koleszar committed
278

John Koleszar's avatar
John Koleszar committed
279
280
281
282
283
  if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    if (cpi->twopass.next_iiratio > 31)
      cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
    else
      cpi->RDMULT +=
Dmitry Kovalev's avatar
Dmitry Kovalev committed
284
          (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
John Koleszar's avatar
John Koleszar committed
285
  }
John Koleszar's avatar
John Koleszar committed
286

John Koleszar's avatar
John Koleszar committed
287
288
  cpi->mb.errorperbit = (cpi->RDMULT / 110);
  cpi->mb.errorperbit += (cpi->mb.errorperbit == 0);
289

290
  vp9_set_speed_features(cpi);
John Koleszar's avatar
John Koleszar committed
291

Dmitry Kovalev's avatar
Dmitry Kovalev committed
292
293
  q = (int)pow(vp9_dc_quant(qindex, 0) >> 2, 1.25);
  q <<= 2;
John Koleszar's avatar
John Koleszar committed
294
  cpi->RDMULT = cpi->RDMULT << 4;
Paul Wilkins's avatar
Paul Wilkins committed
295

John Koleszar's avatar
John Koleszar committed
296
297
  if (q < 8)
    q = 8;
298

John Koleszar's avatar
John Koleszar committed
299
300
301
  if (cpi->RDMULT > 1000) {
    cpi->RDDIV = 1;
    cpi->RDMULT /= 100;
John Koleszar's avatar
John Koleszar committed
302

John Koleszar's avatar
John Koleszar committed
303
304
305
306
307
308
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < INT_MAX) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q / 100;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
309

John Koleszar's avatar
John Koleszar committed
310
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
311
    }
John Koleszar's avatar
John Koleszar committed
312
313
  } else {
    cpi->RDDIV = 100;
John Koleszar's avatar
John Koleszar committed
314

John Koleszar's avatar
John Koleszar committed
315
316
317
318
319
320
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < (INT_MAX / q)) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
321

John Koleszar's avatar
John Koleszar committed
322
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
323
    }
John Koleszar's avatar
John Koleszar committed
324
  }
John Koleszar's avatar
John Koleszar committed
325

326
  fill_token_costs(cpi->mb.token_costs[TX_4X4],
327
                   cpi->common.fc.coef_probs_4x4, BLOCK_TYPES);
328
  fill_token_costs(cpi->mb.token_costs[TX_8X8],
329
                   cpi->common.fc.coef_probs_8x8, BLOCK_TYPES);
330
  fill_token_costs(cpi->mb.token_costs[TX_16X16],
331
                   cpi->common.fc.coef_probs_16x16, BLOCK_TYPES);
332
  fill_token_costs(cpi->mb.token_costs[TX_32X32],
333
                   cpi->common.fc.coef_probs_32x32, BLOCK_TYPES);
334
335
336
337
338
339
#if CONFIG_CODE_NONZEROCOUNT
  fill_nzc_costs(cpi, 4);
  fill_nzc_costs(cpi, 8);
  fill_nzc_costs(cpi, 16);
  fill_nzc_costs(cpi, 32);
#endif
340

John Koleszar's avatar
John Koleszar committed
341
342
  /*rough estimate for costing*/
  cpi->common.kf_ymode_probs_index = cpi->common.base_qindex >> 4;
343
  vp9_init_mode_costs(cpi);
John Koleszar's avatar
John Koleszar committed
344

345
  if (cpi->common.frame_type != KEY_FRAME) {
346
    vp9_build_nmv_cost_table(
347
348
349
350
351
352
        cpi->mb.nmvjointcost,
        cpi->mb.e_mbd.allow_high_precision_mv ?
        cpi->mb.nmvcost_hp : cpi->mb.nmvcost,
        &cpi->common.fc.nmvc,
        cpi->mb.e_mbd.allow_high_precision_mv, 1, 1);
  }
John Koleszar's avatar
John Koleszar committed
353
354
}

355
int vp9_block_error_c(int16_t *coeff, int16_t *dqcoeff, int block_size) {
356
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
357

358
  for (i = 0; i < block_size; i++) {
John Koleszar's avatar
John Koleszar committed
359
360
361
    int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }
John Koleszar's avatar
John Koleszar committed
362

John Koleszar's avatar
John Koleszar committed
363
  return error;
John Koleszar's avatar
John Koleszar committed
364
365
}

366
int vp9_mbblock_error_c(MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
367
368
369
370
  BLOCK  *be;
  BLOCKD *bd;
  int i, j;
  int berror, error = 0;
John Koleszar's avatar
John Koleszar committed
371

John Koleszar's avatar
John Koleszar committed
372
373
374
375
  for (i = 0; i < 16; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
    berror = 0;
376
    for (j = 0; j < 16; j++) {
John Koleszar's avatar
John Koleszar committed
377
378
      int this_diff = be->coeff[j] - bd->dqcoeff[j];
      berror += this_diff * this_diff;
John Koleszar's avatar
John Koleszar committed
379
    }
John Koleszar's avatar
John Koleszar committed
380
381
382
    error += berror;
  }
  return error;
John Koleszar's avatar
John Koleszar committed
383
384
}

385
int vp9_mbuverror_c(MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
386
387
  BLOCK  *be;
  BLOCKD *bd;
John Koleszar's avatar
John Koleszar committed
388

389
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
390

John Koleszar's avatar
John Koleszar committed
391
392
393
  for (i = 16; i < 24; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
John Koleszar's avatar
John Koleszar committed
394

395
    error += vp9_block_error_c(be->coeff, bd->dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
396
  }
John Koleszar's avatar
John Koleszar committed
397

John Koleszar's avatar
John Koleszar committed
398
  return error;
John Koleszar's avatar
John Koleszar committed
399
400
}

401
int vp9_uvsse(MACROBLOCK *x) {
402
403
404
  uint8_t *uptr, *vptr;
  uint8_t *upred_ptr = (*(x->block[16].base_src) + x->block[16].src);
  uint8_t *vpred_ptr = (*(x->block[20].base_src) + x->block[20].src);
John Koleszar's avatar
John Koleszar committed
405
406
407
408
  int uv_stride = x->block[16].src_stride;

  unsigned int sse1 = 0;
  unsigned int sse2 = 0;
409
410
  int mv_row = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.row;
  int mv_col = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.col;
John Koleszar's avatar
John Koleszar committed
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
  int offset;
  int pre_stride = x->e_mbd.block[16].pre_stride;

  if (mv_row < 0)
    mv_row -= 1;
  else
    mv_row += 1;

  if (mv_col < 0)
    mv_col -= 1;
  else
    mv_col += 1;

  mv_row /= 2;
  mv_col /= 2;

  offset = (mv_row >> 3) * pre_stride + (mv_col >> 3);
  uptr = x->e_mbd.pre.u_buffer + offset;
  vptr = x->e_mbd.pre.v_buffer + offset;

  if ((mv_row | mv_col) & 7) {
432
    vp9_sub_pixel_variance8x8(uptr, pre_stride, (mv_col & 7) << 1,
433
                              (mv_row & 7) << 1, upred_ptr, uv_stride, &sse2);
434
    vp9_sub_pixel_variance8x8(vptr, pre_stride, (mv_col & 7) << 1,
435
                              (mv_row & 7) << 1, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
436
437
    sse2 += sse1;
  } else {
438
439
    vp9_variance8x8(uptr, pre_stride, upred_ptr, uv_stride, &sse2);
    vp9_variance8x8(vptr, pre_stride, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
440
441
442
    sse2 += sse1;
  }
  return sse2;
John Koleszar's avatar
John Koleszar committed
443
444
}

445
static INLINE int cost_coeffs(VP9_COMMON *const cm, MACROBLOCK *mb,
446
                              int ib, PLANE_TYPE type,
447
448
449
                              ENTROPY_CONTEXT *a,
                              ENTROPY_CONTEXT *l,
                              TX_SIZE tx_size) {
450
  MACROBLOCKD *const xd = &mb->e_mbd;
451
452
  MB_MODE_INFO *mbmi = &xd->mode_info_context->mbmi;
  int pt;
453
  const int eob = xd->eobs[ib];
454
  int c = 0;
455
  int cost = 0;
456
  const int *scan;
457
458
  const int16_t *qcoeff_ptr = xd->qcoeff + ib * 16;
  const int ref = mbmi->ref_frame != INTRA_FRAME;
459
  unsigned int (*token_costs)[PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS] =
460
      mb->token_costs[tx_size][type][ref];
461
  ENTROPY_CONTEXT a_ec, l_ec;
462
463
464
465
  ENTROPY_CONTEXT *const a1 = a +
      sizeof(ENTROPY_CONTEXT_PLANES)/sizeof(ENTROPY_CONTEXT);
  ENTROPY_CONTEXT *const l1 = l +
      sizeof(ENTROPY_CONTEXT_PLANES)/sizeof(ENTROPY_CONTEXT);
466

467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
#if CONFIG_CODE_NONZEROCOUNT
  int nzc_context = vp9_get_nzc_context(cm, xd, ib);
  unsigned int *nzc_cost;
#else
  int seg_eob;
  const int segment_id = xd->mode_info_context->mbmi.segment_id;
#endif

  // Check for consistency of tx_size with mode info
  if (type == PLANE_TYPE_Y_WITH_DC) {
    assert(xd->mode_info_context->mbmi.txfm_size == tx_size);
  } else {
    TX_SIZE tx_size_uv = get_uv_tx_size(xd);
    assert(tx_size == tx_size_uv);
  }

483
  switch (tx_size) {
484
485
486
    case TX_4X4: {
      const TX_TYPE tx_type = (type == PLANE_TYPE_Y_WITH_DC) ?
                              get_tx_type_4x4(xd, ib) : DCT_DCT;
487
488
      a_ec = *a;
      l_ec = *l;
489
490
491
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_4x4[nzc_context][ref][type];
#else
492
      seg_eob = 16;
493
#endif
494
495
496
497
498
499
      if (tx_type == ADST_DCT) {
        scan = vp9_row_scan_4x4;
      } else if (tx_type == DCT_ADST) {
        scan = vp9_col_scan_4x4;
      } else {
        scan = vp9_default_zig_zag1d_4x4;
Daniel Kang's avatar
Daniel Kang committed
500
501
      }
      break;
502
    }
Daniel Kang's avatar
Daniel Kang committed
503
    case TX_8X8:
504
505
      a_ec = (a[0] + a[1]) != 0;
      l_ec = (l[0] + l[1]) != 0;
506
      scan = vp9_default_zig_zag1d_8x8;
507
508
509
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_8x8[nzc_context][ref][type];
#else
510
      seg_eob = 64;
511
#endif
Daniel Kang's avatar
Daniel Kang committed
512
513
      break;
    case TX_16X16:
514
      scan = vp9_default_zig_zag1d_16x16;
515
516
517
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_16x16[nzc_context][ref][type];
#else
518
      seg_eob = 256;
519
#endif
520
      if (type == PLANE_TYPE_UV) {
521
522
523
524
525
        a_ec = (a[0] + a[1] + a1[0] + a1[1]) != 0;
        l_ec = (l[0] + l[1] + l1[0] + l1[1]) != 0;
      } else {
        a_ec = (a[0] + a[1] + a[2] + a[3]) != 0;
        l_ec = (l[0] + l[1] + l[2] + l[3]) != 0;
Deb Mukherjee's avatar
Deb Mukherjee committed
526
      }
Daniel Kang's avatar
Daniel Kang committed
527
      break;
528
529
    case TX_32X32:
      scan = vp9_default_zig_zag1d_32x32;
530
531
532
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_32x32[nzc_context][ref][type];
#else
533
      seg_eob = 1024;
534
#endif
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
      if (type == PLANE_TYPE_UV) {
        ENTROPY_CONTEXT *a2, *a3, *l2, *l3;
        a2 = a1 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        a3 = a2 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        l2 = l1 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        l3 = l2 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        a_ec = (a[0] + a[1] + a1[0] + a1[1] +
                a2[0] + a2[1] + a3[0] + a3[1]) != 0;
        l_ec = (l[0] + l[1] + l1[0] + l1[1] +
                l2[0] + l2[1] + l3[0] + l3[1]) != 0;
      } else {
        a_ec = (a[0] + a[1] + a[2] + a[3] +
                a1[0] + a1[1] + a1[2] + a1[3]) != 0;
        l_ec = (l[0] + l[1] + l[2] + l[3] +
                l1[0] + l1[1] + l1[2] + l1[3]) != 0;
      }
551
      break;
Daniel Kang's avatar
Daniel Kang committed
552
    default:
553
      abort();
Daniel Kang's avatar
Daniel Kang committed
554
555
      break;
  }
556

557
558
  VP9_COMBINEENTROPYCONTEXTS(pt, a_ec, l_ec);

559
#if CONFIG_CODE_NONZEROCOUNT == 0
Paul Wilkins's avatar
Paul Wilkins committed
560
561
  if (vp9_segfeature_active(xd, segment_id, SEG_LVL_SKIP))
    seg_eob = 0;
562
#endif
563

564
  {
565
    int recent_energy = 0;
566
567
568
#if CONFIG_CODE_NONZEROCOUNT
    int nzc = 0;
#endif
569
570
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
571
      int t = vp9_dct_value_tokens_ptr[v].Token;
572
573
574
#if CONFIG_CODE_NONZEROCOUNT
      nzc += (v != 0);
#endif
575
      cost += token_costs[get_coef_band(tx_size, c)][pt][t];
576
      cost += vp9_dct_value_cost_ptr[v];
577
      pt = vp9_get_coef_context(&recent_energy, t);
578
    }
579
580
581
#if CONFIG_CODE_NONZEROCOUNT
    cost += nzc_cost[nzc];
#else
582
    if (c < seg_eob)
583
      cost += mb->token_costs[tx_size][type][ref][get_coef_band(tx_size, c)]
584
          [pt][DCT_EOB_TOKEN];
585
#endif
586
587
  }

588
  // is eob first coefficient;
589
  pt = (c > 0);
590
  *a = *l = pt;
591
592
593
594
595
596
597
598
599
600
601
602
603
604
  if (tx_size >= TX_8X8) {
    a[1] = l[1] = pt;
    if (tx_size >= TX_16X16) {
      if (type == PLANE_TYPE_UV) {
        a1[0] = a1[1] = l1[0] = l1[1] = pt;
      } else {
        a[2] = a[3] = l[2] = l[3] = pt;
        if (tx_size >= TX_32X32) {
          a1[0] = a1[1] = a1[2] = a1[3] = pt;
          l1[0] = l1[1] = l1[2] = l1[3] = pt;
        }
      }
    }
  }
605
606
607
  return cost;
}

608
static int rdcost_mby_4x4(VP9_COMMON *const cm, MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
609
610
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
611
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
612
  ENTROPY_CONTEXT_PLANES t_above, t_left;
613
614
  ENTROPY_CONTEXT *ta = (ENTROPY_CONTEXT *)&t_above;
  ENTROPY_CONTEXT *tl = (ENTROPY_CONTEXT *)&t_left;
John Koleszar's avatar
John Koleszar committed
615

616
617
  vpx_memcpy(&t_above, xd->above_context, sizeof(t_above));
  vpx_memcpy(&t_left, xd->left_context, sizeof(t_left));
John Koleszar's avatar
John Koleszar committed
618

John Koleszar's avatar
John Koleszar committed
619
  for (b = 0; b < 16; b++)
620
    cost += cost_coeffs(cm, mb, b, PLANE_TYPE_Y_WITH_DC,
Yaowu Xu's avatar
Yaowu Xu committed
621
622
                        ta + vp9_block2above[TX_4X4][b],
                        tl + vp9_block2left[TX_4X4][b],
Daniel Kang's avatar
Daniel Kang committed
623
                        TX_4X4);
John Koleszar's avatar
John Koleszar committed
624

John Koleszar's avatar
John Koleszar committed
625
  return cost;
John Koleszar's avatar
John Koleszar committed
626
627
}

628
629
static void macro_block_yrd_4x4(VP9_COMMON *const cm,
                                MACROBLOCK *mb,
630
631
632
                                int *rate,
                                int *distortion,
                                int *skippable) {
Paul Wilkins's avatar
Paul Wilkins committed
633
  MACROBLOCKD *const xd = &mb->e_mbd;
634

635
636
637
  xd->mode_info_context->mbmi.txfm_size = TX_4X4;
  vp9_transform_mby_4x4(mb);
  vp9_quantize_mby_4x4(mb);
638

639
640
  *distortion = vp9_mbblock_error(mb) >> 2;
  *rate = rdcost_mby_4x4(cm, mb);
641
  *skippable = vp9_mby_is_skippable_4x4(xd);
642
}
John Koleszar's avatar
John Koleszar committed
643

644
static int rdcost_mby_8x8(VP9_COMMON *const cm, MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
645
646
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
647
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
648
  ENTROPY_CONTEXT_PLANES t_above, t_left;
649
650
  ENTROPY_CONTEXT *ta = (ENTROPY_CONTEXT *)&t_above;
  ENTROPY_CONTEXT *tl = (ENTROPY_CONTEXT *)&t_left;
John Koleszar's avatar
John Koleszar committed
651

652
653
  vpx_memcpy(&t_above, xd->above_context, sizeof(t_above));
  vpx_memcpy(&t_left,  xd->left_context, sizeof(t_left));
John Koleszar's avatar
John Koleszar committed
654
655

  for (b = 0; b < 16; b += 4)
656
    cost += cost_coeffs(cm, mb, b, PLANE_TYPE_Y_WITH_DC,
Yaowu Xu's avatar
Yaowu Xu committed
657
658
                        ta + vp9_block2above[TX_8X8][b],
                        tl + vp9_block2left[TX_8X8][b],
Daniel Kang's avatar
Daniel Kang committed
659
                        TX_8X8);
John Koleszar's avatar
John Koleszar committed
660
661

  return cost;
662
663
}

664
665
static void macro_block_yrd_8x8(VP9_COMMON *const cm,
                                MACROBLOCK *mb,
666
667
668
                                int *rate,
                                int *distortion,
                                int *skippable) {
Paul Wilkins's avatar
Paul Wilkins committed
669
  MACROBLOCKD *const xd = &mb->e_mbd;
670
671

  xd->mode_info_context->mbmi.txfm_size = TX_8X8;
672
673
  vp9_transform_mby_8x8(mb);
  vp9_quantize_mby_8x8(mb);
John Koleszar's avatar
John Koleszar committed
674

675
676
  *distortion = vp9_mbblock_error(mb) >> 2;
  *rate = rdcost_mby_8x8(cm, mb);
677
  *skippable = vp9_mby_is_skippable_8x8(xd);
678
}
679

680
681
static int rdcost_mby_16x16(VP9_COMMON *const cm, MACROBLOCK *mb) {
  MACROBLOCKD *const xd = &mb->e_mbd;
Daniel Kang's avatar
Daniel Kang committed
682
  ENTROPY_CONTEXT_PLANES t_above, t_left;
683
684
  ENTROPY_CONTEXT *ta = (ENTROPY_CONTEXT *)&t_above;
  ENTROPY_CONTEXT *tl = (ENTROPY_CONTEXT *)&t_left;
Daniel Kang's avatar
Daniel Kang committed
685

686
687
  vpx_memcpy(&t_above, xd->above_context, sizeof(t_above));
  vpx_memcpy(&t_left, xd->left_context, sizeof(t_left));
Daniel Kang's avatar
Daniel Kang committed
688

689
  return cost_coeffs(cm, mb, 0, PLANE_TYPE_Y_WITH_DC, ta, tl, TX_16X16);
Daniel Kang's avatar
Daniel Kang committed
690
}
691

692
static void macro_block_yrd_16x16(VP9_COMMON *const cm, MACROBLOCK *mb,
693
694
                                  int *rate, int *distortion, int *skippable) {
  MACROBLOCKD *const xd = &mb->e_mbd;
Daniel Kang's avatar
Daniel Kang committed
695

696
697
  xd->mode_info_context->mbmi.txfm_size = TX_16X16;
  vp9_transform_mby_16x16(mb);
698
  vp9_quantize_mby_16x16(mb);
699
700
701
  // TODO(jingning) is it possible to quickly determine whether to force
  //                trailing coefficients to be zero, instead of running trellis
  //                optimization in the rate-distortion optimization loop?
702
703
  if (mb->optimize &&
      xd->mode_info_context->mbmi.mode < I8X8_PRED)
704
    vp9_optimize_mby_16x16(cm, mb);
705

706
707
  *distortion = vp9_mbblock_error(mb) >> 2;
  *rate = rdcost_mby_16x16(cm, mb);
708
  *skippable = vp9_mby_is_skippable_16x16(xd);
Daniel Kang's avatar
Daniel Kang committed
709
710
}

711
static void choose_txfm_size_from_rd(VP9_COMP *cpi, MACROBLOCK *x,
712
713
714
715
716
                                     int (*r)[2], int *rate,
                                     int *d, int *distortion,
                                     int *s, int *skip,
                                     int64_t txfm_cache[NB_TXFM_MODES],
                                     TX_SIZE max_txfm_size) {
717
718
719
720
721
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
  vp9_prob skip_prob = cm->mb_no_coeff_skip ?
                       vp9_get_pred_prob(cm, xd, PRED_MBSKIP) : 128;
722
723
724
725
726
727
728
729
730
731
732
733
  int64_t rd[TX_SIZE_MAX_SB][2];
  int n, m;

  for (n = TX_4X4; n <= max_txfm_size; n++) {
    r[n][1] = r[n][0];
    for (m = 0; m <= n - (n == max_txfm_size); m++) {
      if (m == n)
        r[n][1] += vp9_cost_zero(cm->prob_tx[m]);
      else
        r[n][1] += vp9_cost_one(cm->prob_tx[m]);
    }
  }
734
735
736
737
738
739
740
741

  if (cm->mb_no_coeff_skip) {
    int s0, s1;

    assert(skip_prob > 0);
    s0 = vp9_cost_bit(skip_prob, 0);
    s1 = vp9_cost_bit(skip_prob, 1);

742
    for (n = TX_4X4; n <= max_txfm_size; n++) {
743
      if (s[n]) {
744
        rd[n][0] = rd[n][1] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
745
      } else {
746
747
        rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0] + s0, d[n]);
        rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1] + s0, d[n]);
748
      }
749
750
    }
  } else {
751
752
753
    for (n = TX_4X4; n <= max_txfm_size; n++) {
      rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0], d[n]);
      rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1], d[n]);
754
755
756
    }
  }

757
758
759
760
761
762
  if (max_txfm_size == TX_32X32 &&
      (cm->txfm_mode == ALLOW_32X32 ||
       (cm->txfm_mode == TX_MODE_SELECT &&
        rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
        rd[TX_32X32][1] < rd[TX_4X4][1]))) {
    mbmi->txfm_size = TX_32X32;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
763
764
765
766
767
  } else if ( cm->txfm_mode == ALLOW_16X16 ||
             (max_txfm_size == TX_16X16 && cm->txfm_mode == ALLOW_32X32) ||
             (cm->txfm_mode == TX_MODE_SELECT &&
              rd[TX_16X16][1] < rd[TX_8X8][1] &&
              rd[TX_16X16][1] < rd[TX_4X4][1])) {
768
    mbmi->txfm_size = TX_16X16;
769
  } else if (cm->txfm_mode == ALLOW_8X8 ||
770
           (cm->txfm_mode == TX_MODE_SELECT && rd[TX_8X8][1] < rd[TX_4X4][1])) {
771
772
    mbmi->txfm_size = TX_8X8;
  } else {
773
    assert(cm->txfm_mode == ONLY_4X4 || cm->txfm_mode == TX_MODE_SELECT);
774
775
776
    mbmi->txfm_size = TX_4X4;
  }

777
  *distortion = d[mbmi->txfm_size];
778
  *rate       = r[mbmi->txfm_size][cm->txfm_mode == TX_MODE_SELECT];
779
780
  *skip       = s[mbmi->txfm_size];

781
782
783
784
785
786
787
788
  txfm_cache[ONLY_4X4] = rd[TX_4X4][0];
  txfm_cache[ALLOW_8X8] = rd[TX_8X8][0];
  txfm_cache[ALLOW_16X16] = rd[TX_16X16][0];
  txfm_cache[ALLOW_32X32] = rd[max_txfm_size][0];
  if (max_txfm_size == TX_32X32 &&
      rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
      rd[TX_32X32][1] < rd[TX_4X4][1])
    txfm_cache[TX_MODE_SELECT] = rd[TX_32X32][1];
Ronald S. Bultje's avatar
Ronald S. Bultje committed
789
  else if (rd[TX_16X16][1] < rd[TX_8X8][1] && rd[TX_16X16][1] < rd[TX_4X4][1])
790
    txfm_cache[TX_MODE_SELECT] = rd[TX_16X16][1];
791
  else
792
793
    txfm_cache[TX_MODE_SELECT] = rd[TX_4X4][1] < rd[TX_8X8][1] ?
                                 rd[TX_4X4][1] : rd[TX_8X8][1];
794
795
796
797
798
}

static void macro_block_yrd(VP9_COMP *cpi, MACROBLOCK *x, int *rate,
                            int *distortion, int *skippable,
                            int64_t txfm_cache[NB_TXFM_MODES]) {
799
  VP9_COMMON *const cm = &cpi->common;
800
  MACROBLOCKD *const xd = &x->e_mbd;
801
  int r[TX_SIZE_MAX_MB][2], d[TX_SIZE_MAX_MB], s[TX_SIZE_MAX_MB];
802
803
804

  vp9_subtract_mby(x->src_diff, *(x->block[0].base_src), xd->predictor,
                   x->block[0].src_stride);
805

806
807
808
  macro_block_yrd_16x16(cm, x, &r[TX_16X16][0], &d[TX_16X16], &s[TX_16X16]);
  macro_block_yrd_8x8(cm, x, &r[TX_8X8][0], &d[TX_8X8], &s[TX_8X8]);
  macro_block_yrd_4x4(cm, x, &r[TX_4X4][0], &d[TX_4X4], &s[TX_4X4]);
809
810

  choose_txfm_size_from_rd(cpi, x, r, rate, d, distortion, s, skippable,
811
                           txfm_cache, TX_16X16);
812
813
}

814
static void copy_predictor(uint8_t *dst, const uint8_t *predictor) {
John Koleszar's avatar
John Koleszar committed
815
816
817
818
819
820
  const unsigned int *p = (const unsigned int *)predictor;
  unsigned int *d = (unsigned int *)dst;
  d[0] = p[0];
  d[4] = p[4];
  d[8] = p[8];
  d[12] = p[12];
821
}
Yaowu Xu's avatar
Yaowu Xu committed
822

823
static int vp9_sb_block_error_c(int16_t *coeff, int16_t *dqcoeff,
824
                                int block_size, int shift) {
825
826
827
828
829
830
831
  int i;
  int64_t error = 0;

  for (i = 0; i < block_size; i++) {
    unsigned int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }
832
  error >>= shift;
833

Frank Galligan's avatar
Frank Galligan committed
834
  return error > INT_MAX ? INT_MAX : (int)error;
835
836
}

837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
static int rdcost_sby_4x4(VP9_COMMON *const cm, MACROBLOCK *x) {
  int cost = 0, b;
  MACROBLOCKD *const xd = &x->e_mbd;
  ENTROPY_CONTEXT_PLANES t_above[2], t_left[2];
  ENTROPY_CONTEXT *ta = (ENTROPY_CONTEXT *) &t_above;
  ENTROPY_CONTEXT *tl = (ENTROPY_CONTEXT *) &t_left;

  vpx_memcpy(&t_above, xd->above_context, sizeof(t_above));
  vpx_memcpy(&t_left,  xd->left_context,  sizeof(t_left));

  for (b = 0; b < 64; b++)
    cost += cost_coeffs(cm, x, b, PLANE_TYPE_Y_WITH_DC,
                        ta + vp9_block2above_sb[TX_4X4][b],
                        tl + vp9_block2left_sb[TX_4X4][b], TX_4X4);

  return cost;
}

static void super_block_yrd_4x4(VP9_COMMON *const cm, MACROBLOCK *x,
                                int *rate, int *distortion, int *skippable) {
  MACROBLOCKD *const xd = &x->e_mbd;

  xd->mode_info_context->mbmi.txfm_size = TX_4X4;
  vp9_transform_sby_4x4(x);
  vp9_quantize_sby_4x4(x);

  *distortion = vp9_sb_block_error_c(x->coeff, xd->dqcoeff, 1024, 2);
  *rate       = rdcost_sby_4x4(cm, x);
  *skippable  = vp9_sby_is_skippable_4x4(xd);
}

static int rdcost_sby_8x8(VP9_COMMON *const cm, MACROBLOCK *x) {
  int cost = 0, b;
  MACROBLOCKD *const xd = &x->e_mbd;
  ENTROPY_CONTEXT_PLANES t_above[2], t_left[2];
  ENTROPY_CONTEXT *ta = (ENTROPY_CONTEXT *) &t_above;
  ENTROPY_CONTEXT *tl = (ENTROPY_CONTEXT *) &t_left;

  vpx_memcpy(&t_above, xd->above_context, sizeof(t_above));
  vpx_memcpy(&t_left,  xd->left_context,  sizeof(t_left));

  for (b = 0; b < 64; b += 4)
    cost += cost_coeffs(cm, x, b, PLANE_TYPE_Y_WITH_DC,
                        ta + vp9_block2above_sb[TX_8X8][b],
                        tl + vp9_block2left_sb[TX_8X8][b], TX_8X8);

  return cost;
}

static void super_block_yrd_8x8(VP9_COMMON *const cm, MACROBLOCK *x,
                                int *rate, int *distortion, int *skippable) {
  MACROBLOCKD *const xd = &x->e_mbd;

  xd->mode_info_context->mbmi.txfm_size = TX_8X8;
  vp9_transform_sby_8x8(x);
  vp9_quantize_sby_8x8(x);

  *distortion = vp9_sb_block_error_c(x->coeff, xd->dqcoeff, 1024, 2);
  *rate       = rdcost_sby_8x8(cm, x);
  *skippable  = vp9_sby_is_skippable_8x8(xd);
}

static int rdcost_sby_16x16(VP9_COMMON *const cm, MACROBLOCK *x) {
  int cost = 0, b;
  MACROBLOCKD *const xd = &x->e_mbd;
  ENTROPY_CONTEXT_PLANES t_above[2], t_left[2];
  ENTROPY_CONTEXT *ta = (ENTROPY_CONTEXT *) &t_above;
  ENTROPY_CONTEXT *tl = (ENTROPY_CONTEXT *) &t_left;

  vpx_memcpy(&t_above, xd->above_context, sizeof(t_above));
  vpx_memcpy(&t_left,  xd->left_context,  sizeof(t_left));

  for (b = 0; b < 64; b += 16)
    cost += cost_coeffs(cm, x, b, PLANE_TYPE_Y_WITH_DC,
                        ta + vp9_block2above_sb[TX_16X16][b],
                        tl + vp9_block2left_sb[TX_16X16][b], TX_16X16);

  return cost;
}

static void super_block_yrd_16x16(VP9_COMMON *const cm, MACROBLOCK *x,
                                  int *rate, int *distortion, int *skippable) {
  MACROBLOCKD *const xd = &x->e_mbd;

  xd->mode_info_context->mbmi.txfm_size = TX_16X16;
  vp9_transform_sby_16x16(x);
  vp9_quantize_sby_16x16(x);

  *distortion = vp9_sb_block_error_c(x->coeff, xd->dqcoeff, 1024, 2);
  *rate       = rdcost_sby_16x16(cm, x);
  *skippable  = vp9_sby_is_skippable_16x16(xd);
}

static int rdcost_sby_32x32(VP9_COMMON *const cm, MACROBLOCK *x) {
  MACROBLOCKD * const xd = &x->e_mbd;
  ENTROPY_CONTEXT_PLANES t_above[2], t_left[2];
  ENTROPY_CONTEXT *ta = (ENTROPY_CONTEXT *) &t_above;
  ENTROPY_CONTEXT *tl = (ENTROPY_CONTEXT *) &t_left;

  vpx_memcpy(&t_above, xd->above_context, sizeof(t_above));
  vpx_memcpy(&t_left,  xd->left_context,  sizeof(t_left));

  return cost_coeffs(cm, x, 0, PLANE_TYPE_Y_WITH_DC, ta, tl, TX_32X32);
}

942
static void super_block_yrd_32x32(VP9_COMMON *const cm, MACROBLOCK *x,
943
                                  int *rate, int *distortion, int *skippable) {
944
  MACROBLOCKD *const xd = &x->e_mbd;
945

946
  xd->mode_info_context->mbmi.txfm_size = TX_32X32;
947
948
  vp9_transform_sby_32x32(x);
  vp9_quantize_sby_32x32(x);
949

950
951
  *distortion = vp9_sb_block_error_c(x->coeff, xd->dqcoeff, 1024, 0);
  *rate       = rdcost_sby_32x32(cm, x);
952
  *skippable  = vp9_sby_is_skippable_32x32(xd);
953
954
}

955
956
static void super_block_yrd(VP9_COMP *cpi,
                            MACROBLOCK *x, int *rate, int *distortion,
957
                            int *skip,
958
                            int64_t txfm_cache[NB_TXFM_MODES]) {
959
  VP9_COMMON *const cm = &cpi->common;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
960
  MACROBLOCKD *const xd = &x->e_mbd;
961
  int r[TX_SIZE_MAX_SB][2], d[TX_SIZE_MAX_SB], s[TX_SIZE_MAX_SB];
Ronald S. Bultje's avatar
Ronald S. Bultje committed
962
963
964
  const uint8_t *src = x->src.y_buffer, *dst = xd->dst.y_buffer;
  int src_y_stride = x->src.y_stride, dst_y_stride = xd->dst.y_stride;

965
966
967
968
969
  vp9_subtract_sby_s_c(x->src_diff, src, src_y_stride, dst, dst_y_stride);
  super_block_yrd_32x32(cm, x, &r[TX_32X32][0], &d[TX_32X32], &s[TX_32X32]);
  super_block_yrd_16x16(cm, x, &r[TX_16X16][0], &d[TX_16X16], &s[TX_16X16]);
  super_block_yrd_8x8(cm, x,   &r[TX_8X8][0],   &d[TX_8X8],   &s[TX_8X8]);
  super_block_yrd_4x4(cm, x,   &r[TX_4X4][0],   &d[TX_4X4],   &s[TX_4X4]);
970
971
972