vp9_rdopt.c 206 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5
6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9
10
11
12
13
14
15
 */


#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
16
17
#include "vp9/common/vp9_pragmas.h"

18
19
20
21
22
#include "vp9/encoder/vp9_tokenize.h"
#include "vp9/encoder/vp9_treewriter.h"
#include "vp9/encoder/vp9_onyx_int.h"
#include "vp9/encoder/vp9_modecosts.h"
#include "vp9/encoder/vp9_encodeintra.h"
23
24
25
26
27
#include "vp9/common/vp9_entropymode.h"
#include "vp9/common/vp9_reconinter.h"
#include "vp9/common/vp9_reconintra.h"
#include "vp9/common/vp9_findnearmv.h"
#include "vp9/common/vp9_quant_common.h"
28
29
30
31
32
33
#include "vp9/encoder/vp9_encodemb.h"
#include "vp9/encoder/vp9_quantize.h"
#include "vp9/encoder/vp9_variance.h"
#include "vp9/encoder/vp9_mcomp.h"
#include "vp9/encoder/vp9_rdopt.h"
#include "vp9/encoder/vp9_ratectrl.h"
John Koleszar's avatar
John Koleszar committed
34
#include "vpx_mem/vpx_mem.h"
35
36
#include "vp9/common/vp9_systemdependent.h"
#include "vp9/encoder/vp9_encodemv.h"
John Koleszar's avatar
John Koleszar committed
37

38
39
40
#include "vp9/common/vp9_seg_common.h"
#include "vp9/common/vp9_pred_common.h"
#include "vp9/common/vp9_entropy.h"
41
#include "vp9_rtcd.h"
42
#include "vp9/common/vp9_mvref_common.h"
Ronald S. Bultje's avatar
Ronald S. Bultje committed
43
#include "vp9/common/vp9_common.h"
Paul Wilkins's avatar
Paul Wilkins committed
44

John Koleszar's avatar
John Koleszar committed
45
46
#define MAXF(a,b)            (((a) > (b)) ? (a) : (b))

47
48
#define INVALID_MV 0x80008000

49
50
51
/* Factor to weigh the rate for switchable interp filters */
#define SWITCHABLE_INTERP_RATE_FACTOR 1

John Koleszar's avatar
John Koleszar committed
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
static const int auto_speed_thresh[17] = {
  1000,
  200,
  150,
  130,
  150,
  125,
  120,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  105
John Koleszar's avatar
John Koleszar committed
70
71
};

72
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
73
74
  {ZEROMV,    LAST_FRAME,   NONE},
  {DC_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
75

76
77
  {NEARESTMV, LAST_FRAME,   NONE},
  {NEARMV,    LAST_FRAME,   NONE},
John Koleszar's avatar
John Koleszar committed
78

79
80
  {ZEROMV,    GOLDEN_FRAME, NONE},
  {NEARESTMV, GOLDEN_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
81

82
83
  {ZEROMV,    ALTREF_FRAME, NONE},
  {NEARESTMV, ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
84

85
86
  {NEARMV,    GOLDEN_FRAME, NONE},
  {NEARMV,    ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
87

88
89
90
91
92
93
94
95
  {V_PRED,    INTRA_FRAME,  NONE},
  {H_PRED,    INTRA_FRAME,  NONE},
  {D45_PRED,  INTRA_FRAME,  NONE},
  {D135_PRED, INTRA_FRAME,  NONE},
  {D117_PRED, INTRA_FRAME,  NONE},
  {D153_PRED, INTRA_FRAME,  NONE},
  {D27_PRED,  INTRA_FRAME,  NONE},
  {D63_PRED,  INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
96

97
  {TM_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
98

99
100
101
  {NEWMV,     LAST_FRAME,   NONE},
  {NEWMV,     GOLDEN_FRAME, NONE},
  {NEWMV,     ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
102

103
104
105
  {SPLITMV,   LAST_FRAME,   NONE},
  {SPLITMV,   GOLDEN_FRAME, NONE},
  {SPLITMV,   ALTREF_FRAME, NONE},
106

107
108
  {B_PRED,    INTRA_FRAME,  NONE},
  {I8X8_PRED, INTRA_FRAME,  NONE},
109

John Koleszar's avatar
John Koleszar committed
110
111
112
113
  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME},
114

John Koleszar's avatar
John Koleszar committed
115
116
117
  {ZEROMV,    ALTREF_FRAME, LAST_FRAME},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME},
118

John Koleszar's avatar
John Koleszar committed
119
120
121
  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME},
122

John Koleszar's avatar
John Koleszar committed
123
124
125
  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME  },
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME},
126

John Koleszar's avatar
John Koleszar committed
127
128
  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME  },
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME},

#if CONFIG_COMP_INTERINTRA_PRED
  /* compound inter-intra prediction */
  {ZEROMV,    LAST_FRAME,   INTRA_FRAME},
  {NEARESTMV, LAST_FRAME,   INTRA_FRAME},
  {NEARMV,    LAST_FRAME,   INTRA_FRAME},
  {NEWMV,     LAST_FRAME,   INTRA_FRAME},

  {ZEROMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEARESTMV, GOLDEN_FRAME,   INTRA_FRAME},
  {NEARMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEWMV,     GOLDEN_FRAME,   INTRA_FRAME},

  {ZEROMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEARESTMV, ALTREF_FRAME,   INTRA_FRAME},
  {NEARMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEWMV,     ALTREF_FRAME,   INTRA_FRAME},
#endif
John Koleszar's avatar
John Koleszar committed
148
149
};

150
151
152
static void fill_token_costs(vp9_coeff_count *c,
                             vp9_coeff_probs *p,
                             int block_type_counts) {
153
  int i, j, k, l;
John Koleszar's avatar
John Koleszar committed
154
155

  for (i = 0; i < block_type_counts; i++)
156
157
158
    for (j = 0; j < REF_TYPES; j++)
      for (k = 0; k < COEF_BANDS; k++)
        for (l = 0; l < PREV_COEF_CONTEXTS; l++) {
159
160
161
162
163
164
#if CONFIG_CODE_NONZEROCOUNT
          // All costs are without the EOB node
          vp9_cost_tokens_skip((int *)(c[i][j][k][l]),
                               p[i][j][k][l],
                               vp9_coef_tree);
#else
165
166
167
168
169
170
171
172
          if (l == 0 && k > 0)
            vp9_cost_tokens_skip((int *)(c[i][j][k][l]),
                                 p[i][j][k][l],
                                 vp9_coef_tree);
          else
            vp9_cost_tokens((int *)(c[i][j][k][l]),
                            p[i][j][k][l],
                            vp9_coef_tree);
173
174
175
176
177
178
179
180
181
182
183
184
185
#endif
        }
}

#if CONFIG_CODE_NONZEROCOUNT
static void fill_nzc_costs(VP9_COMP *cpi, int block_size) {
  int nzc_context, r, b, nzc, values;
  int cost[16];
  values = block_size * block_size + 1;

  for (nzc_context = 0; nzc_context < MAX_NZC_CONTEXTS; ++nzc_context) {
    for (r = 0; r < REF_TYPES; ++r) {
      for (b = 0; b < BLOCK_TYPES; ++b) {
186
187
        unsigned int *nzc_costs;
        if (block_size == 4) {
188
189
190
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_4x4[nzc_context][r][b],
                          vp9_nzc4x4_tree);
191
192
          nzc_costs = cpi->mb.nzc_costs_4x4[nzc_context][r][b];
        } else if (block_size == 8) {
193
194
195
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_8x8[nzc_context][r][b],
                          vp9_nzc8x8_tree);
196
197
          nzc_costs = cpi->mb.nzc_costs_8x8[nzc_context][r][b];
        } else if (block_size == 16) {
198
199
200
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_16x16[nzc_context][r][b],
                          vp9_nzc16x16_tree);
201
202
          nzc_costs = cpi->mb.nzc_costs_16x16[nzc_context][r][b];
        } else {
203
204
205
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_32x32[nzc_context][r][b],
                          vp9_nzc32x32_tree);
206
207
          nzc_costs = cpi->mb.nzc_costs_32x32[nzc_context][r][b];
        }
208
209
210
211
212

        for (nzc = 0; nzc < values; ++nzc) {
          int e, c, totalcost = 0;
          c = codenzc(nzc);
          totalcost = cost[c];
213
214
          if ((e = vp9_extranzcbits[c])) {
            int x = nzc - vp9_basenzcvalue[c];
215
            while (e--) {
216
217
218
219
              totalcost += vp9_cost_bit(
                  cpi->common.fc.nzc_pcat_probs[nzc_context]
                                               [c - NZC_TOKENS_NOEXTRA][e],
                  ((x >> e) & 1));
220
221
            }
          }
222
          nzc_costs[nzc] = totalcost;
223
        }
224
225
226
      }
    }
  }
John Koleszar's avatar
John Koleszar committed
227
}
228
#endif
John Koleszar's avatar
John Koleszar committed
229

230

231
232
233
234
static int rd_iifactor[32] =  { 4, 4, 3, 2, 1, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0, };
John Koleszar's avatar
John Koleszar committed
235

236
// 3* dc_qlookup[Q]*dc_qlookup[Q];
237

238
/* values are now correlated to quantizer */
Paul Wilkins's avatar
Paul Wilkins committed
239
240
241
static int sad_per_bit16lut[QINDEX_RANGE];
static int sad_per_bit4lut[QINDEX_RANGE];

242
void vp9_init_me_luts() {
John Koleszar's avatar
John Koleszar committed
243
244
245
246
247
248
249
  int i;

  // Initialize the sad lut tables using a formulaic calculation for now
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < QINDEX_RANGE; i++) {
    sad_per_bit16lut[i] =
250
251
      (int)((0.0418 * vp9_convert_qindex_to_q(i)) + 2.4107);
    sad_per_bit4lut[i] = (int)((0.063 * vp9_convert_qindex_to_q(i)) + 2.742);
John Koleszar's avatar
John Koleszar committed
252
  }
Paul Wilkins's avatar
Paul Wilkins committed
253
}
John Koleszar's avatar
John Koleszar committed
254

255
static int compute_rd_mult(int qindex) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
256
  int q = vp9_dc_quant(qindex, 0);
257
  return (11 * q * q) >> 2;
258
259
}

Dmitry Kovalev's avatar
Dmitry Kovalev committed
260
261
262
void vp9_initialize_me_consts(VP9_COMP *cpi, int qindex) {
  cpi->mb.sadperbit16 = sad_per_bit16lut[qindex];
  cpi->mb.sadperbit4 = sad_per_bit4lut[qindex];
John Koleszar's avatar
John Koleszar committed
263
264
}

265

Dmitry Kovalev's avatar
Dmitry Kovalev committed
266
void vp9_initialize_rd_consts(VP9_COMP *cpi, int qindex) {
267
  int q, i;
John Koleszar's avatar
John Koleszar committed
268

269
  vp9_clear_system_state();  // __asm emms;
John Koleszar's avatar
John Koleszar committed
270

John Koleszar's avatar
John Koleszar committed
271
272
273
274
  // Further tests required to see if optimum is different
  // for key frames, golden frames and arf frames.
  // if (cpi->common.refresh_golden_frame ||
  //     cpi->common.refresh_alt_ref_frame)
Dmitry Kovalev's avatar
Dmitry Kovalev committed
275
  qindex = (qindex < 0) ? 0 : ((qindex > MAXQ) ? MAXQ : qindex);
276

Dmitry Kovalev's avatar
Dmitry Kovalev committed
277
  cpi->RDMULT = compute_rd_mult(qindex);
John Koleszar's avatar
John Koleszar committed
278
279
280
281
282
  if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    if (cpi->twopass.next_iiratio > 31)
      cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
    else
      cpi->RDMULT +=
Dmitry Kovalev's avatar
Dmitry Kovalev committed
283
          (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
John Koleszar's avatar
John Koleszar committed
284
  }
285
  cpi->mb.errorperbit = cpi->RDMULT >> 6;
John Koleszar's avatar
John Koleszar committed
286
  cpi->mb.errorperbit += (cpi->mb.errorperbit == 0);
287

288
  vp9_set_speed_features(cpi);
John Koleszar's avatar
John Koleszar committed
289

Dmitry Kovalev's avatar
Dmitry Kovalev committed
290
291
  q = (int)pow(vp9_dc_quant(qindex, 0) >> 2, 1.25);
  q <<= 2;
John Koleszar's avatar
John Koleszar committed
292
293
  if (q < 8)
    q = 8;
294

John Koleszar's avatar
John Koleszar committed
295
296
297
  if (cpi->RDMULT > 1000) {
    cpi->RDDIV = 1;
    cpi->RDMULT /= 100;
John Koleszar's avatar
John Koleszar committed
298

John Koleszar's avatar
John Koleszar committed
299
300
301
302
303
304
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < INT_MAX) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q / 100;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
305

John Koleszar's avatar
John Koleszar committed
306
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
307
    }
John Koleszar's avatar
John Koleszar committed
308
309
  } else {
    cpi->RDDIV = 100;
John Koleszar's avatar
John Koleszar committed
310

John Koleszar's avatar
John Koleszar committed
311
312
313
314
315
316
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < (INT_MAX / q)) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
317

John Koleszar's avatar
John Koleszar committed
318
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
319
    }
John Koleszar's avatar
John Koleszar committed
320
  }
John Koleszar's avatar
John Koleszar committed
321

322
  fill_token_costs(cpi->mb.token_costs[TX_4X4],
323
                   cpi->common.fc.coef_probs_4x4, BLOCK_TYPES);
324
  fill_token_costs(cpi->mb.token_costs[TX_8X8],
325
                   cpi->common.fc.coef_probs_8x8, BLOCK_TYPES);
326
  fill_token_costs(cpi->mb.token_costs[TX_16X16],
327
                   cpi->common.fc.coef_probs_16x16, BLOCK_TYPES);
328
  fill_token_costs(cpi->mb.token_costs[TX_32X32],
329
                   cpi->common.fc.coef_probs_32x32, BLOCK_TYPES);
330
331
332
333
334
335
#if CONFIG_CODE_NONZEROCOUNT
  fill_nzc_costs(cpi, 4);
  fill_nzc_costs(cpi, 8);
  fill_nzc_costs(cpi, 16);
  fill_nzc_costs(cpi, 32);
#endif
336

John Koleszar's avatar
John Koleszar committed
337
338
  /*rough estimate for costing*/
  cpi->common.kf_ymode_probs_index = cpi->common.base_qindex >> 4;
339
  vp9_init_mode_costs(cpi);
John Koleszar's avatar
John Koleszar committed
340

341
  if (cpi->common.frame_type != KEY_FRAME) {
342
    vp9_build_nmv_cost_table(
343
344
345
346
347
348
        cpi->mb.nmvjointcost,
        cpi->mb.e_mbd.allow_high_precision_mv ?
        cpi->mb.nmvcost_hp : cpi->mb.nmvcost,
        &cpi->common.fc.nmvc,
        cpi->mb.e_mbd.allow_high_precision_mv, 1, 1);
  }
John Koleszar's avatar
John Koleszar committed
349
350
}

351
int vp9_block_error_c(int16_t *coeff, int16_t *dqcoeff, int block_size) {
352
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
353

354
  for (i = 0; i < block_size; i++) {
John Koleszar's avatar
John Koleszar committed
355
356
357
    int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }
John Koleszar's avatar
John Koleszar committed
358

John Koleszar's avatar
John Koleszar committed
359
  return error;
John Koleszar's avatar
John Koleszar committed
360
361
}

362
int vp9_mbblock_error_c(MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
363
364
365
366
  BLOCK  *be;
  BLOCKD *bd;
  int i, j;
  int berror, error = 0;
John Koleszar's avatar
John Koleszar committed
367

John Koleszar's avatar
John Koleszar committed
368
369
370
371
  for (i = 0; i < 16; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
    berror = 0;
372
    for (j = 0; j < 16; j++) {
John Koleszar's avatar
John Koleszar committed
373
374
      int this_diff = be->coeff[j] - bd->dqcoeff[j];
      berror += this_diff * this_diff;
John Koleszar's avatar
John Koleszar committed
375
    }
John Koleszar's avatar
John Koleszar committed
376
377
378
    error += berror;
  }
  return error;
John Koleszar's avatar
John Koleszar committed
379
380
}

381
int vp9_mbuverror_c(MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
382
383
  BLOCK  *be;
  BLOCKD *bd;
John Koleszar's avatar
John Koleszar committed
384

385
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
386

John Koleszar's avatar
John Koleszar committed
387
388
389
  for (i = 16; i < 24; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
John Koleszar's avatar
John Koleszar committed
390

391
    error += vp9_block_error_c(be->coeff, bd->dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
392
  }
John Koleszar's avatar
John Koleszar committed
393

John Koleszar's avatar
John Koleszar committed
394
  return error;
John Koleszar's avatar
John Koleszar committed
395
396
}

397
int vp9_uvsse(MACROBLOCK *x) {
398
399
400
  uint8_t *uptr, *vptr;
  uint8_t *upred_ptr = (*(x->block[16].base_src) + x->block[16].src);
  uint8_t *vpred_ptr = (*(x->block[20].base_src) + x->block[20].src);
John Koleszar's avatar
John Koleszar committed
401
402
403
404
  int uv_stride = x->block[16].src_stride;

  unsigned int sse1 = 0;
  unsigned int sse2 = 0;
405
406
  int mv_row = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.row;
  int mv_col = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.col;
John Koleszar's avatar
John Koleszar committed
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
  int offset;
  int pre_stride = x->e_mbd.block[16].pre_stride;

  if (mv_row < 0)
    mv_row -= 1;
  else
    mv_row += 1;

  if (mv_col < 0)
    mv_col -= 1;
  else
    mv_col += 1;

  mv_row /= 2;
  mv_col /= 2;

  offset = (mv_row >> 3) * pre_stride + (mv_col >> 3);
  uptr = x->e_mbd.pre.u_buffer + offset;
  vptr = x->e_mbd.pre.v_buffer + offset;

  if ((mv_row | mv_col) & 7) {
428
    vp9_sub_pixel_variance8x8(uptr, pre_stride, (mv_col & 7) << 1,
429
                              (mv_row & 7) << 1, upred_ptr, uv_stride, &sse2);
430
    vp9_sub_pixel_variance8x8(vptr, pre_stride, (mv_col & 7) << 1,
431
                              (mv_row & 7) << 1, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
432
433
    sse2 += sse1;
  } else {
434
435
    vp9_variance8x8(uptr, pre_stride, upred_ptr, uv_stride, &sse2);
    vp9_variance8x8(vptr, pre_stride, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
436
437
438
    sse2 += sse1;
  }
  return sse2;
John Koleszar's avatar
John Koleszar committed
439
440
}

441
static INLINE int cost_coeffs(VP9_COMMON *const cm, MACROBLOCK *mb,
442
                              int ib, PLANE_TYPE type,
443
444
445
                              ENTROPY_CONTEXT *a,
                              ENTROPY_CONTEXT *l,
                              TX_SIZE tx_size) {
446
  MACROBLOCKD *const xd = &mb->e_mbd;
447
448
  MB_MODE_INFO *mbmi = &xd->mode_info_context->mbmi;
  int pt;
449
  const int eob = xd->eobs[ib];
450
  int c = 0;
451
  int cost = 0;
452
  const int *scan;
453
454
  const int16_t *qcoeff_ptr = xd->qcoeff + ib * 16;
  const int ref = mbmi->ref_frame != INTRA_FRAME;
455
  unsigned int (*token_costs)[PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS] =
456
      mb->token_costs[tx_size][type][ref];
457
  ENTROPY_CONTEXT a_ec, l_ec;
458
459
460
461
  ENTROPY_CONTEXT *const a1 = a +
      sizeof(ENTROPY_CONTEXT_PLANES)/sizeof(ENTROPY_CONTEXT);
  ENTROPY_CONTEXT *const l1 = l +
      sizeof(ENTROPY_CONTEXT_PLANES)/sizeof(ENTROPY_CONTEXT);
462

463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
#if CONFIG_CODE_NONZEROCOUNT
  int nzc_context = vp9_get_nzc_context(cm, xd, ib);
  unsigned int *nzc_cost;
#else
  int seg_eob;
  const int segment_id = xd->mode_info_context->mbmi.segment_id;
#endif

  // Check for consistency of tx_size with mode info
  if (type == PLANE_TYPE_Y_WITH_DC) {
    assert(xd->mode_info_context->mbmi.txfm_size == tx_size);
  } else {
    TX_SIZE tx_size_uv = get_uv_tx_size(xd);
    assert(tx_size == tx_size_uv);
  }

479
  switch (tx_size) {
480
481
482
    case TX_4X4: {
      const TX_TYPE tx_type = (type == PLANE_TYPE_Y_WITH_DC) ?
                              get_tx_type_4x4(xd, ib) : DCT_DCT;
483
484
      a_ec = *a;
      l_ec = *l;
485
486
487
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_4x4[nzc_context][ref][type];
#else
488
      seg_eob = 16;
489
#endif
490
491
492
493
494
495
      if (tx_type == ADST_DCT) {
        scan = vp9_row_scan_4x4;
      } else if (tx_type == DCT_ADST) {
        scan = vp9_col_scan_4x4;
      } else {
        scan = vp9_default_zig_zag1d_4x4;
Daniel Kang's avatar
Daniel Kang committed
496
497
      }
      break;
498
    }
Daniel Kang's avatar
Daniel Kang committed
499
    case TX_8X8:
500
501
      a_ec = (a[0] + a[1]) != 0;
      l_ec = (l[0] + l[1]) != 0;
502
      scan = vp9_default_zig_zag1d_8x8;
503
504
505
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_8x8[nzc_context][ref][type];
#else
506
      seg_eob = 64;
507
#endif
Daniel Kang's avatar
Daniel Kang committed
508
509
      break;
    case TX_16X16:
510
      scan = vp9_default_zig_zag1d_16x16;
511
512
513
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_16x16[nzc_context][ref][type];
#else
514
      seg_eob = 256;
515
#endif
516
      if (type == PLANE_TYPE_UV) {
517
518
519
520
521
        a_ec = (a[0] + a[1] + a1[0] + a1[1]) != 0;
        l_ec = (l[0] + l[1] + l1[0] + l1[1]) != 0;
      } else {
        a_ec = (a[0] + a[1] + a[2] + a[3]) != 0;
        l_ec = (l[0] + l[1] + l[2] + l[3]) != 0;
Deb Mukherjee's avatar
Deb Mukherjee committed
522
      }
Daniel Kang's avatar
Daniel Kang committed
523
      break;
524
525
    case TX_32X32:
      scan = vp9_default_zig_zag1d_32x32;
526
527
528
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_32x32[nzc_context][ref][type];
#else
529
      seg_eob = 1024;
530
#endif
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
      if (type == PLANE_TYPE_UV) {
        ENTROPY_CONTEXT *a2, *a3, *l2, *l3;
        a2 = a1 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        a3 = a2 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        l2 = l1 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        l3 = l2 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        a_ec = (a[0] + a[1] + a1[0] + a1[1] +
                a2[0] + a2[1] + a3[0] + a3[1]) != 0;
        l_ec = (l[0] + l[1] + l1[0] + l1[1] +
                l2[0] + l2[1] + l3[0] + l3[1]) != 0;
      } else {
        a_ec = (a[0] + a[1] + a[2] + a[3] +
                a1[0] + a1[1] + a1[2] + a1[3]) != 0;
        l_ec = (l[0] + l[1] + l[2] + l[3] +
                l1[0] + l1[1] + l1[2] + l1[3]) != 0;
      }
547
      break;
Daniel Kang's avatar
Daniel Kang committed
548
    default:
549
      abort();
Daniel Kang's avatar
Daniel Kang committed
550
551
      break;
  }
552

553
554
  VP9_COMBINEENTROPYCONTEXTS(pt, a_ec, l_ec);

555
#if CONFIG_CODE_NONZEROCOUNT == 0
Paul Wilkins's avatar
Paul Wilkins committed
556
557
  if (vp9_segfeature_active(xd, segment_id, SEG_LVL_SKIP))
    seg_eob = 0;
558
#endif
559

560
  {
561
    int recent_energy = 0;
562
563
564
#if CONFIG_CODE_NONZEROCOUNT
    int nzc = 0;
#endif
565
566
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
567
      int t = vp9_dct_value_tokens_ptr[v].Token;
568
569
570
#if CONFIG_CODE_NONZEROCOUNT
      nzc += (v != 0);
#endif
571
      cost += token_costs[get_coef_band(tx_size, c)][pt][t];
572
      cost += vp9_dct_value_cost_ptr[v];
573
      pt = vp9_get_coef_context(&recent_energy, t);
574
    }
575
576
577
#if CONFIG_CODE_NONZEROCOUNT
    cost += nzc_cost[nzc];
#else
578
    if (c < seg_eob)
579
      cost += mb->token_costs[tx_size][type][ref][get_coef_band(tx_size, c)]
580
          [pt][DCT_EOB_TOKEN];
581
#endif
582
583
  }

584
  // is eob first coefficient;
585
  pt = (c > 0);
586
  *a = *l = pt;
587
588
589
590
591
592
593
594
595
596
597
598
599
600
  if (tx_size >= TX_8X8) {
    a[1] = l[1] = pt;
    if (tx_size >= TX_16X16) {
      if (type == PLANE_TYPE_UV) {
        a1[0] = a1[1] = l1[0] = l1[1] = pt;
      } else {
        a[2] = a[3] = l[2] = l[3] = pt;
        if (tx_size >= TX_32X32) {
          a1[0] = a1[1] = a1[2] = a1[3] = pt;
          l1[0] = l1[1] = l1[2] = l1[3] = pt;
        }
      }
    }
  }
601
602
603
  return cost;
}

604
static int rdcost_mby_4x4(VP9_COMMON *const cm, MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
605
606
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
607
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
608
  ENTROPY_CONTEXT_PLANES t_above, t_left;
609
610
  ENTROPY_CONTEXT *ta = (ENTROPY_CONTEXT *)&t_above;
  ENTROPY_CONTEXT *tl = (ENTROPY_CONTEXT *)&t_left;
John Koleszar's avatar
John Koleszar committed
611

612
613
  vpx_memcpy(&t_above, xd->above_context, sizeof(t_above));
  vpx_memcpy(&t_left, xd->left_context, sizeof(t_left));
John Koleszar's avatar
John Koleszar committed
614

John Koleszar's avatar
John Koleszar committed
615
  for (b = 0; b < 16; b++)
616
    cost += cost_coeffs(cm, mb, b, PLANE_TYPE_Y_WITH_DC,
Yaowu Xu's avatar
Yaowu Xu committed
617
618
                        ta + vp9_block2above[TX_4X4][b],
                        tl + vp9_block2left[TX_4X4][b],
Daniel Kang's avatar
Daniel Kang committed
619
                        TX_4X4);
John Koleszar's avatar
John Koleszar committed
620

John Koleszar's avatar
John Koleszar committed
621
  return cost;
John Koleszar's avatar
John Koleszar committed
622
623
}

624
625
static void macro_block_yrd_4x4(VP9_COMMON *const cm,
                                MACROBLOCK *mb,
626
627
628
                                int *rate,
                                int *distortion,
                                int *skippable) {
Paul Wilkins's avatar
Paul Wilkins committed
629
  MACROBLOCKD *const xd = &mb->e_mbd;
630

631
632
633
  xd->mode_info_context->mbmi.txfm_size = TX_4X4;
  vp9_transform_mby_4x4(mb);
  vp9_quantize_mby_4x4(mb);
634

635
636
  *distortion = vp9_mbblock_error(mb) >> 2;
  *rate = rdcost_mby_4x4(cm, mb);
637
  *skippable = vp9_mby_is_skippable_4x4(xd);
638
}
John Koleszar's avatar
John Koleszar committed
639

640
static int rdcost_mby_8x8(VP9_COMMON *const cm, MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
641
642
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
643
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
644
  ENTROPY_CONTEXT_PLANES t_above, t_left;
645
646
  ENTROPY_CONTEXT *ta = (ENTROPY_CONTEXT *)&t_above;
  ENTROPY_CONTEXT *tl = (ENTROPY_CONTEXT *)&t_left;
John Koleszar's avatar
John Koleszar committed
647

648
649
  vpx_memcpy(&t_above, xd->above_context, sizeof(t_above));
  vpx_memcpy(&t_left,  xd->left_context, sizeof(t_left));
John Koleszar's avatar
John Koleszar committed
650
651

  for (b = 0; b < 16; b += 4)
652
    cost += cost_coeffs(cm, mb, b, PLANE_TYPE_Y_WITH_DC,
Yaowu Xu's avatar
Yaowu Xu committed
653
654
                        ta + vp9_block2above[TX_8X8][b],
                        tl + vp9_block2left[TX_8X8][b],
Daniel Kang's avatar
Daniel Kang committed
655
                        TX_8X8);
John Koleszar's avatar
John Koleszar committed
656
657

  return cost;
658
659
}

660
661
static void macro_block_yrd_8x8(VP9_COMMON *const cm,
                                MACROBLOCK *mb,
662
663
664
                                int *rate,
                                int *distortion,
                                int *skippable) {
Paul Wilkins's avatar
Paul Wilkins committed
665
  MACROBLOCKD *const xd = &mb->e_mbd;
666
667

  xd->mode_info_context->mbmi.txfm_size = TX_8X8;
668
669
  vp9_transform_mby_8x8(mb);
  vp9_quantize_mby_8x8(mb);
John Koleszar's avatar
John Koleszar committed
670

671
672
  *distortion = vp9_mbblock_error(mb) >> 2;
  *rate = rdcost_mby_8x8(cm, mb);
673
  *skippable = vp9_mby_is_skippable_8x8(xd);
674
}
675

676
677
static int rdcost_mby_16x16(VP9_COMMON *const cm, MACROBLOCK *mb) {
  MACROBLOCKD *const xd = &mb->e_mbd;
Daniel Kang's avatar
Daniel Kang committed
678
  ENTROPY_CONTEXT_PLANES t_above, t_left;
679
680
  ENTROPY_CONTEXT *ta = (ENTROPY_CONTEXT *)&t_above;
  ENTROPY_CONTEXT *tl = (ENTROPY_CONTEXT *)&t_left;
Daniel Kang's avatar
Daniel Kang committed
681

682
683
  vpx_memcpy(&t_above, xd->above_context, sizeof(t_above));
  vpx_memcpy(&t_left, xd->left_context, sizeof(t_left));
Daniel Kang's avatar
Daniel Kang committed
684

685
  return cost_coeffs(cm, mb, 0, PLANE_TYPE_Y_WITH_DC, ta, tl, TX_16X16);
Daniel Kang's avatar
Daniel Kang committed
686
}
687

688
static void macro_block_yrd_16x16(VP9_COMMON *const cm, MACROBLOCK *mb,
689
690
                                  int *rate, int *distortion, int *skippable) {
  MACROBLOCKD *const xd = &mb->e_mbd;
Daniel Kang's avatar
Daniel Kang committed
691

692
693
  xd->mode_info_context->mbmi.txfm_size = TX_16X16;
  vp9_transform_mby_16x16(mb);
694
  vp9_quantize_mby_16x16(mb);
695
696
697
  // TODO(jingning) is it possible to quickly determine whether to force
  //                trailing coefficients to be zero, instead of running trellis
  //                optimization in the rate-distortion optimization loop?
698
699
  if (mb->optimize &&
      xd->mode_info_context->mbmi.mode < I8X8_PRED)
700
    vp9_optimize_mby_16x16(cm, mb);
701

702
703
  *distortion = vp9_mbblock_error(mb) >> 2;
  *rate = rdcost_mby_16x16(cm, mb);
704
  *skippable = vp9_mby_is_skippable_16x16(xd);
Daniel Kang's avatar
Daniel Kang committed
705
706
}

707
static void choose_txfm_size_from_rd(VP9_COMP *cpi, MACROBLOCK *x,
708
709
710
711
712
                                     int (*r)[2], int *rate,
                                     int *d, int *distortion,
                                     int *s, int *skip,
                                     int64_t txfm_cache[NB_TXFM_MODES],
                                     TX_SIZE max_txfm_size) {
713
714
715
716
717
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
  vp9_prob skip_prob = cm->mb_no_coeff_skip ?
                       vp9_get_pred_prob(cm, xd, PRED_MBSKIP) : 128;
718
719
720
721
722
723
724
725
726
727
728
729
  int64_t rd[TX_SIZE_MAX_SB][2];
  int n, m;

  for (n = TX_4X4; n <= max_txfm_size; n++) {
    r[n][1] = r[n][0];
    for (m = 0; m <= n - (n == max_txfm_size); m++) {
      if (m == n)
        r[n][1] += vp9_cost_zero(cm->prob_tx[m]);
      else
        r[n][1] += vp9_cost_one(cm->prob_tx[m]);
    }
  }
730
731
732
733
734
735
736
737

  if (cm->mb_no_coeff_skip) {
    int s0, s1;

    assert(skip_prob > 0);
    s0 = vp9_cost_bit(skip_prob, 0);
    s1 = vp9_cost_bit(skip_prob, 1);

738
    for (n = TX_4X4; n <= max_txfm_size; n++) {
739
      if (s[n]) {
740
        rd[n][0] = rd[n][1] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
741
      } else {
742
743
        rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0] + s0, d[n]);
        rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1] + s0, d[n]);
744
      }
745
746
    }
  } else {
747
748
749
    for (n = TX_4X4; n <= max_txfm_size; n++) {
      rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0], d[n]);
      rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1], d[n]);
750
751
752
    }
  }

753
754
755
756
757
758
  if (max_txfm_size == TX_32X32 &&
      (cm->txfm_mode == ALLOW_32X32 ||
       (cm->txfm_mode == TX_MODE_SELECT &&
        rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
        rd[TX_32X32][1] < rd[TX_4X4][1]))) {
    mbmi->txfm_size = TX_32X32;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
759
760
761
762
763
  } else if ( cm->txfm_mode == ALLOW_16X16 ||
             (max_txfm_size == TX_16X16 && cm->txfm_mode == ALLOW_32X32) ||
             (cm->txfm_mode == TX_MODE_SELECT &&
              rd[TX_16X16][1] < rd[TX_8X8][1] &&
              rd[TX_16X16][1] < rd[TX_4X4][1])) {
764
    mbmi->txfm_size = TX_16X16;
765
  } else if (cm->txfm_mode == ALLOW_8X8 ||
766
           (cm->txfm_mode == TX_MODE_SELECT && rd[TX_8X8][1] < rd[TX_4X4][1])) {
767
768
    mbmi->txfm_size = TX_8X8;
  } else {
769
    assert(cm->txfm_mode == ONLY_4X4 || cm->txfm_mode == TX_MODE_SELECT);
770
771
772
    mbmi->txfm_size = TX_4X4;
  }

773
  *distortion = d[mbmi->txfm_size];
774
  *rate       = r[mbmi->txfm_size][cm->txfm_mode == TX_MODE_SELECT];
775
776
  *skip       = s[mbmi->txfm_size];

777
778
779
780
781
782
783
784
  txfm_cache[ONLY_4X4] = rd[TX_4X4][0];
  txfm_cache[ALLOW_8X8] = rd[TX_8X8][0];
  txfm_cache[ALLOW_16X16] = rd[TX_16X16][0];
  txfm_cache[ALLOW_32X32] = rd[max_txfm_size][0];
  if (max_txfm_size == TX_32X32 &&
      rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
      rd[TX_32X32][1] < rd[TX_4X4][1])
    txfm_cache[TX_MODE_SELECT] = rd[TX_32X32][1];
Ronald S. Bultje's avatar
Ronald S. Bultje committed
785
  else if (rd[TX_16X16][1] < rd[TX_8X8][1] && rd[TX_16X16][1] < rd[TX_4X4][1])
786
    txfm_cache[TX_MODE_SELECT] = rd[TX_16X16][1];
787
  else
788
789
    txfm_cache[TX_MODE_SELECT] = rd[TX_4X4][1] < rd[TX_8X8][1] ?
                                 rd[TX_4X4][1] : rd[TX_8X8][1];
790
791
792
793
794
}

static void macro_block_yrd(VP9_COMP *cpi, MACROBLOCK *x, int *rate,
                            int *distortion, int *skippable,
                            int64_t txfm_cache[NB_TXFM_MODES]) {
795
  VP9_COMMON *const cm = &cpi->common;
796
  MACROBLOCKD *const xd = &x->e_mbd;
797
  int r[TX_SIZE_MAX_MB][2], d[TX_SIZE_MAX_MB], s[TX_SIZE_MAX_MB];
798
799
800

  vp9_subtract_mby(x->src_diff, *(x->block[0].base_src), xd->predictor,
                   x->block[0].src_stride);
801

802
803
804
  macro_block_yrd_16x16(cm, x, &r[TX_16X16][0], &d[TX_16X16], &s[TX_16X16]);
  macro_block_yrd_8x8(cm, x, &r[TX_8X8][0], &d[TX_8X8], &s[TX_8X8]);
  macro_block_yrd_4x4(cm, x, &r[TX_4X4][0], &d[TX_4X4], &s[TX_4X4]);
805
806

  choose_txfm_size_from_rd(cpi, x, r, rate, d, distortion, s, skippable,
807
                           txfm_cache, TX_16X16);
808
809
}

810
static void copy_predictor(uint8_t *dst, const uint8_t *predictor) {
John Koleszar's avatar
John Koleszar committed
811
812
813
814
815
816
  const unsigned int *p = (const unsigned int *)predictor;
  unsigned int *d = (unsigned int *)dst;
  d[0] = p[0];
  d[4] = p[4];
  d[8] = p[8];
  d[12] = p[12];
817
}
Yaowu Xu's avatar
Yaowu Xu committed
818

819
static int vp9_sb_block_error_c(int16_t *coeff, int16_t *dqcoeff,
820
                                int block_size, int shift) {
821
822
823
824
825
826
827
  int i;
  int64_t error = 0;

  for (i = 0; i < block_size; i++) {
    unsigned int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }
828
  error >>= shift;
829

Frank Galligan's avatar
Frank Galligan committed
830
  return error > INT_MAX ? INT_MAX : (int)error;
831
832
}

833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
static int rdcost_sby_4x4(VP9_COMMON *const cm, MACROBLOCK *x) {
  int cost = 0, b;
  MACROBLOCKD *const xd = &x->e_mbd;
  ENTROPY_CONTEXT_PLANES t_above[2], t_left[2];
  ENTROPY_CONTEXT *ta = (ENTROPY_CONTEXT *) &t_above;
  ENTROPY_CONTEXT *tl = (ENTROPY_CONTEXT *) &t_left;

  vpx_memcpy(&t_above, xd->above_context, sizeof(t_above));
  vpx_memcpy(&t_left,  xd->left_context,  sizeof(t_left));

  for (b = 0; b < 64; b++)
    cost += cost_coeffs(cm, x, b, PLANE_TYPE_Y_WITH_DC,
                        ta + vp9_block2above_sb[TX_4X4][b],
                        tl + vp9_block2left_sb[TX_4X4][b], TX_4X4);

  return cost;
}

static void super_block_yrd_4x4(VP9_COMMON *const cm, MACROBLOCK *x,
                                int *rate, int *distortion, int *skippable) {
  MACROBLOCKD *const xd = &x->e_mbd;

  xd->mode_info_context->mbmi.txfm_size = TX_4X4;
  vp9_transform_sby_4x4(x);
  vp9_quantize_sby_4x4(x);

  *distortion = vp9_sb_block_error_c(x->coeff, xd->dqcoeff, 1024, 2);
  *rate       = rdcost_sby_4x4(cm, x);
  *skippable  = vp9_sby_is_skippable_4x4(xd);
}

static int rdcost_sby_8x8(VP9_COMMON *const cm, MACROBLOCK *x) {
  int cost = 0, b;
  MACROBLOCKD *const xd = &x->e_mbd;
  ENTROPY_CONTEXT_PLANES t_above[2], t_left[2];
  ENTROPY_CONTEXT *ta = (ENTROPY_CONTEXT *) &t_above;
  ENTROPY_CONTEXT *tl = (ENTROPY_CONTEXT *) &t_left;

  vpx_memcpy(&t_above, xd->above_context, sizeof(t_above));
  vpx_memcpy(&t_left,  xd->left_context,  sizeof(t_left));

  for (b = 0; b < 64; b += 4)
    cost += cost_coeffs(cm, x, b, PLANE_TYPE_Y_WITH_DC,
                        ta + vp9_block2above_sb[TX_8X8][b],
                        tl + vp9_block2left_sb[TX_8X8][b], TX_8X8);

  return cost;
}

static void super_block_yrd_8x8(VP9_COMMON *const cm, MACROBLOCK *x,
                                int *rate, int *distortion, int *skippable) {
  MACROBLOCKD *const xd = &x->e_mbd;

  xd->mode_info_context->mbmi.txfm_size = TX_8X8;
  vp9_transform_sby_8x8(x);
  vp9_quantize_sby_8x8(x);

  *distortion = vp9_sb_block_error_c(x->coeff, xd->dqcoeff, 1024, 2);
  *rate       = rdcost_sby_8x8(cm, x);
  *skippable  = vp9_sby_is_skippable_8x8(xd);
}

static int rdcost_sby_16x16(VP9_COMMON *const cm, MACROBLOCK *x) {
  int cost = 0, b;
  MACROBLOCKD *const xd = &x->e_mbd;
  ENTROPY_CONTEXT_PLANES t_above[2], t_left[2];
  ENTROPY_CONTEXT *ta = (ENTROPY_CONTEXT *) &t_above;
  ENTROPY_CONTEXT *tl = (ENTROPY_CONTEXT *) &t_left;

  vpx_memcpy(&t_above, xd->above_context, sizeof(t_above));
  vpx_memcpy(&t_left,  xd->left_context,  sizeof(t_left));

  for (b = 0; b < 64; b += 16)
    cost += cost_coeffs(cm, x, b, PLANE_TYPE_Y_WITH_DC,
                        ta + vp9_block2above_sb[TX_16X16][b],
                        tl + vp9_block2left_sb[TX_16X16][b], TX_16X16);

  return cost;
}

static void super_block_yrd_16x16(VP9_COMMON *const cm, MACROBLOCK *x,
                                  int *rate, int *distortion, int *skippable) {
  MACROBLOCKD *const xd = &x->e_mbd;

  xd->mode_info_context->mbmi.txfm_size = TX_16X16;
  vp9_transform_sby_16x16(x);
  vp9_quantize_sby_16x16(x);

  *distortion = vp9_sb_block_error_c(x->coeff, xd->dqcoeff, 1024, 2);
  *rate       = rdcost_sby_16x16(cm, x);
  *skippable  = vp9_sby_is_skippable_16x16(xd);
}

static int rdcost_sby_32x32(VP9_COMMON *const cm, MACROBLOCK *x) {
  MACROBLOCKD * const xd = &x->e_mbd;
  ENTROPY_CONTEXT_PLANES t_above[2], t_left[2];
  ENTROPY_CONTEXT *ta = (ENTROPY_CONTEXT *) &t_above;
  ENTROPY_CONTEXT *tl = (ENTROPY_CONTEXT *) &t_left;

  vpx_memcpy(&t_above, xd->above_context, sizeof(t_above));
  vpx_memcpy(&t_left,  xd->left_context,  sizeof(t_left));

  return cost_coeffs(cm, x, 0, PLANE_TYPE_Y_WITH_DC, ta, tl, TX_32X32);
}

938
static void super_block_yrd_32x32(VP9_COMMON *const cm, MACROBLOCK *x,
939
                                  int *rate, int *distortion, int *skippable) {
940
  MACROBLOCKD *const xd = &x->e_mbd;
941

942
  xd->mode_info_context->mbmi.txfm_size = TX_32X32;
943
944
  vp9_transform_sby_32x32(x);
  vp9_quantize_sby_32x32(x);
945

946
947
  *distortion = vp9_sb_block_error_c(x->coeff, xd->dqcoeff, 1024, 0);
  *rate       = rdcost_sby_32x32(cm, x);
948
  *skippable  = vp9_sby_is_skippable_32x32(xd);
949
950
}

951
952
static void super_block_yrd(VP9_COMP *cpi,
                            MACROBLOCK *x, int *rate, int *distortion,
953
                            int *skip,
954
                            int64_t txfm_cache[NB_TXFM_MODES]) {
955
  VP9_COMMON *const cm = &cpi->common;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
956
  MACROBLOCKD *const xd = &x->e_mbd;
957
  int r[TX_SIZE_MAX_SB][2], d[TX_SIZE_MAX_SB], s[TX_SIZE_MAX_SB];
Ronald S. Bultje's avatar
Ronald S. Bultje committed
958
959
960
  const uint8_t *src = x->src.y_buffer, *dst = xd->dst.y_buffer;
  int src_y_stride = x->src.y_stride, dst_y_stride = xd->dst.y_stride;

961
962
963
964
965
  vp9_subtract_sby_s_c(x->src_diff, src, src_y_stride, dst, dst_y_stride);
  super_block_yrd_32x32(cm, x, &r[TX_32X32][0], &d[TX_32X32], &s[TX_32X32]);
  super_block_yrd_16x16(cm, x, &r[TX_16X16][0], &d[TX_16X16], &s[TX_16X16]);
  super_block_yrd_8x8(cm, x,   &r[TX_8X8][0],   &d[TX_8X8],   &s[TX_8X8]);
  super_block_yrd_4x4(cm, x,   &r[TX_4X4][0],   &d[TX_4X4],   &s[TX_4X4]);
966
967
968

  choose_txfm_size_from_rd(cpi, x, r, rate, d, distortion, s, skip, txfm_cache,
                           TX_SIZE_MAX_SB - 1);
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
}

static int rdcost_sb64y_4x4(VP9_COMMON *const cm, MACROBLOCK *x) {
  int cost = 0, b;
  MACROBLOCKD *const xd = &x->e_mbd;
  ENTROPY_CONTEXT_PLANES t_above[4], t_left[4];
  ENTROPY_CONTEXT *ta = (ENTROPY_CONTEXT *) &t_above;
  ENTROPY_CONTEXT *tl = (ENTROPY_CONTEXT *) &t_left;

  vpx_memcpy(&t_above, xd->above_context, sizeof(t_above));
  vpx_memcpy(&t_left,  xd->left_context,  sizeof(t_left));

  for (b = 0; b < 256; b++)
    cost += cost_coeffs(cm, x, b, PLANE_TYPE_Y_WITH_DC,
                        ta + vp9_block2above_sb64[TX_4X4][b],
                        tl + vp9_block2left_sb64[TX_4X4][b], TX_4X4);

  return cost;
}

static void super_block64_yrd_4x4(VP9_COMMON *const cm, MACROBLOCK *x,
                                  int *rate, int *distortion, int *skippable) {
  MACROBLOCKD *const xd = &x->e_mbd;

  xd->mode_info_context->mbmi.txfm_size = TX_4X4;
  vp9_transform_sb64y_4x4(x);
  vp9_quantize_sb64y_4x4(x);

  *distortion = vp9_sb_block_error_c(x->coeff, xd->dqcoeff, 4096, 2);
  *rate       = rdcost_sb64y_4x4(cm, x);
  *skippable  = vp9_sb64y_is_skippable_4x4(xd);
}

static int rdcost_sb64y_8x8(VP9_COMMON *const cm, MACROBLOCK *x) {
  int cost = 0, b;
  MACROBLOCKD *const xd = &x->e_mbd;
  ENTROPY_CONTEXT_PLANES t_above[4], t_left[4];
  ENTROPY_CONTEXT *ta = (ENTROPY_CONTEXT *) &t_above;
  ENTROPY_CONTEXT *tl = (ENTROPY_CONTEXT *) &t_left;

  vpx_memcpy(&t_above, xd->above_context, sizeof(t_above));
  vpx_memcpy(&t_left,  xd->left_context,  sizeof(t_left));

  for (b = 0; b < 256; b += 4)
    cost += cost_coeffs(cm, x, b, PLANE_TYPE_Y_WITH_DC,
                        ta + vp9_block2above_sb64[TX_8X8][b],
                        tl + vp9_block2left_sb64[TX_8X8][b], TX_8X8);

  return cost;
}

static void super_block64_yrd_8x8(VP9_COMMON *const cm, MACROBLOCK *x,
                                  int *rate, int *distortion, int *skippable) {
  MACROBLOCKD *const xd = &x->e_mbd;

  xd->mode_info_context->mbmi.txfm_size = TX_8X8;
  vp9_transform_sb64y_8x8(x);
  vp9_quantize_sb64y_8x8(x);

  *distortion = vp9_sb_block_error_c(x->coeff, xd->dqcoeff, 4096, 2);
  *rate       = rdcost_sb64y_8x8(cm, x);
  *skippable  = vp9_sb64y_is_skippable_8x8(xd);
}

static int rdcost_sb64y_16x16(VP9_COMMON *const cm, MACROBLOCK *x) {
  int cost = 0, b;
  MACROBLOCKD *const xd = &x->e_mbd;
  ENTROPY_CONTEXT_PLANES t_above[4], t_left[4];
  ENTROPY_CONTEXT *ta = (ENTROPY_CONTEXT *) &t_above;
  ENTROPY_CONTEXT *tl = (ENTROPY_CONTEXT *) &t_left;

  vpx_memcpy(&t_above, xd->above_context, sizeof(t_above));
  vpx_memcpy(&t_left,  xd->left_context,  sizeof(t_left));
1042