vp9_rdopt.c 207 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5
6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9
10
11
12
13
14
15
 */


#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
16
17
#include "vp9/common/vp9_pragmas.h"

18
19
20
21
22
#include "vp9/encoder/vp9_tokenize.h"
#include "vp9/encoder/vp9_treewriter.h"
#include "vp9/encoder/vp9_onyx_int.h"
#include "vp9/encoder/vp9_modecosts.h"
#include "vp9/encoder/vp9_encodeintra.h"
23
24
25
26
27
#include "vp9/common/vp9_entropymode.h"
#include "vp9/common/vp9_reconinter.h"
#include "vp9/common/vp9_reconintra.h"
#include "vp9/common/vp9_findnearmv.h"
#include "vp9/common/vp9_quant_common.h"
28
29
30
31
32
33
#include "vp9/encoder/vp9_encodemb.h"
#include "vp9/encoder/vp9_quantize.h"
#include "vp9/encoder/vp9_variance.h"
#include "vp9/encoder/vp9_mcomp.h"
#include "vp9/encoder/vp9_rdopt.h"
#include "vp9/encoder/vp9_ratectrl.h"
John Koleszar's avatar
John Koleszar committed
34
#include "vpx_mem/vpx_mem.h"
35
36
#include "vp9/common/vp9_systemdependent.h"
#include "vp9/encoder/vp9_encodemv.h"
John Koleszar's avatar
John Koleszar committed
37

38
39
40
#include "vp9/common/vp9_seg_common.h"
#include "vp9/common/vp9_pred_common.h"
#include "vp9/common/vp9_entropy.h"
41
#include "vp9_rtcd.h"
42
#include "vp9/common/vp9_mvref_common.h"
Ronald S. Bultje's avatar
Ronald S. Bultje committed
43
#include "vp9/common/vp9_common.h"
Paul Wilkins's avatar
Paul Wilkins committed
44

John Koleszar's avatar
John Koleszar committed
45
46
#define MAXF(a,b)            (((a) > (b)) ? (a) : (b))

47
48
#define INVALID_MV 0x80008000

49
50
51
/* Factor to weigh the rate for switchable interp filters */
#define SWITCHABLE_INTERP_RATE_FACTOR 1

John Koleszar's avatar
John Koleszar committed
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
static const int auto_speed_thresh[17] = {
  1000,
  200,
  150,
  130,
  150,
  125,
  120,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  105
John Koleszar's avatar
John Koleszar committed
70
71
};

72
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
73
74
  {ZEROMV,    LAST_FRAME,   NONE},
  {DC_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
75

76
77
  {NEARESTMV, LAST_FRAME,   NONE},
  {NEARMV,    LAST_FRAME,   NONE},
John Koleszar's avatar
John Koleszar committed
78

79
80
  {ZEROMV,    GOLDEN_FRAME, NONE},
  {NEARESTMV, GOLDEN_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
81

82
83
  {ZEROMV,    ALTREF_FRAME, NONE},
  {NEARESTMV, ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
84

85
86
  {NEARMV,    GOLDEN_FRAME, NONE},
  {NEARMV,    ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
87

88
89
90
91
92
93
94
95
  {V_PRED,    INTRA_FRAME,  NONE},
  {H_PRED,    INTRA_FRAME,  NONE},
  {D45_PRED,  INTRA_FRAME,  NONE},
  {D135_PRED, INTRA_FRAME,  NONE},
  {D117_PRED, INTRA_FRAME,  NONE},
  {D153_PRED, INTRA_FRAME,  NONE},
  {D27_PRED,  INTRA_FRAME,  NONE},
  {D63_PRED,  INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
96

97
  {TM_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
98

99
100
101
  {NEWMV,     LAST_FRAME,   NONE},
  {NEWMV,     GOLDEN_FRAME, NONE},
  {NEWMV,     ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
102

103
104
105
  {SPLITMV,   LAST_FRAME,   NONE},
  {SPLITMV,   GOLDEN_FRAME, NONE},
  {SPLITMV,   ALTREF_FRAME, NONE},
106

107
108
  {B_PRED,    INTRA_FRAME,  NONE},
  {I8X8_PRED, INTRA_FRAME,  NONE},
109

John Koleszar's avatar
John Koleszar committed
110
111
112
113
  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME},
114

John Koleszar's avatar
John Koleszar committed
115
116
117
  {ZEROMV,    ALTREF_FRAME, LAST_FRAME},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME},
118

John Koleszar's avatar
John Koleszar committed
119
120
121
  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME},
122

John Koleszar's avatar
John Koleszar committed
123
124
125
  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME  },
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME},
126

John Koleszar's avatar
John Koleszar committed
127
128
  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME  },
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME},

#if CONFIG_COMP_INTERINTRA_PRED
  /* compound inter-intra prediction */
  {ZEROMV,    LAST_FRAME,   INTRA_FRAME},
  {NEARESTMV, LAST_FRAME,   INTRA_FRAME},
  {NEARMV,    LAST_FRAME,   INTRA_FRAME},
  {NEWMV,     LAST_FRAME,   INTRA_FRAME},

  {ZEROMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEARESTMV, GOLDEN_FRAME,   INTRA_FRAME},
  {NEARMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEWMV,     GOLDEN_FRAME,   INTRA_FRAME},

  {ZEROMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEARESTMV, ALTREF_FRAME,   INTRA_FRAME},
  {NEARMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEWMV,     ALTREF_FRAME,   INTRA_FRAME},
#endif
John Koleszar's avatar
John Koleszar committed
148
149
};

150
151
152
static void fill_token_costs(vp9_coeff_count *c,
                             vp9_coeff_probs *p,
                             int block_type_counts) {
153
  int i, j, k, l;
John Koleszar's avatar
John Koleszar committed
154
155

  for (i = 0; i < block_type_counts; i++)
156
157
158
    for (j = 0; j < REF_TYPES; j++)
      for (k = 0; k < COEF_BANDS; k++)
        for (l = 0; l < PREV_COEF_CONTEXTS; l++) {
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
          vp9_cost_tokens_skip((int *)(c[i][j][k][l]),
                               p[i][j][k][l],
                               vp9_coef_tree);
        }
}

#if CONFIG_CODE_NONZEROCOUNT
static void fill_nzc_costs(VP9_COMP *cpi, int block_size) {
  int nzc_context, r, b, nzc, values;
  int cost[16];
  values = block_size * block_size + 1;

  for (nzc_context = 0; nzc_context < MAX_NZC_CONTEXTS; ++nzc_context) {
    for (r = 0; r < REF_TYPES; ++r) {
      for (b = 0; b < BLOCK_TYPES; ++b) {
174
175
        unsigned int *nzc_costs;
        if (block_size == 4) {
176
177
178
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_4x4[nzc_context][r][b],
                          vp9_nzc4x4_tree);
179
180
          nzc_costs = cpi->mb.nzc_costs_4x4[nzc_context][r][b];
        } else if (block_size == 8) {
181
182
183
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_8x8[nzc_context][r][b],
                          vp9_nzc8x8_tree);
184
185
          nzc_costs = cpi->mb.nzc_costs_8x8[nzc_context][r][b];
        } else if (block_size == 16) {
186
187
188
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_16x16[nzc_context][r][b],
                          vp9_nzc16x16_tree);
189
190
          nzc_costs = cpi->mb.nzc_costs_16x16[nzc_context][r][b];
        } else {
191
192
193
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_32x32[nzc_context][r][b],
                          vp9_nzc32x32_tree);
194
195
          nzc_costs = cpi->mb.nzc_costs_32x32[nzc_context][r][b];
        }
196
197
198
199
200

        for (nzc = 0; nzc < values; ++nzc) {
          int e, c, totalcost = 0;
          c = codenzc(nzc);
          totalcost = cost[c];
201
202
          if ((e = vp9_extranzcbits[c])) {
            int x = nzc - vp9_basenzcvalue[c];
203
            while (e--) {
204
205
206
207
              totalcost += vp9_cost_bit(
                  cpi->common.fc.nzc_pcat_probs[nzc_context]
                                               [c - NZC_TOKENS_NOEXTRA][e],
                  ((x >> e) & 1));
208
209
            }
          }
210
          nzc_costs[nzc] = totalcost;
211
        }
212
213
214
      }
    }
  }
John Koleszar's avatar
John Koleszar committed
215
}
216
#endif
John Koleszar's avatar
John Koleszar committed
217

218

219
220
221
222
static int rd_iifactor[32] =  { 4, 4, 3, 2, 1, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0, };
John Koleszar's avatar
John Koleszar committed
223

224
// 3* dc_qlookup[Q]*dc_qlookup[Q];
225

226
/* values are now correlated to quantizer */
Paul Wilkins's avatar
Paul Wilkins committed
227
228
229
static int sad_per_bit16lut[QINDEX_RANGE];
static int sad_per_bit4lut[QINDEX_RANGE];

230
void vp9_init_me_luts() {
John Koleszar's avatar
John Koleszar committed
231
232
233
234
235
236
237
  int i;

  // Initialize the sad lut tables using a formulaic calculation for now
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < QINDEX_RANGE; i++) {
    sad_per_bit16lut[i] =
238
239
      (int)((0.0418 * vp9_convert_qindex_to_q(i)) + 2.4107);
    sad_per_bit4lut[i] = (int)((0.063 * vp9_convert_qindex_to_q(i)) + 2.742);
John Koleszar's avatar
John Koleszar committed
240
  }
Paul Wilkins's avatar
Paul Wilkins committed
241
}
John Koleszar's avatar
John Koleszar committed
242

243
static int compute_rd_mult(int qindex) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
244
  int q = vp9_dc_quant(qindex, 0);
245
  return (11 * q * q) >> 2;
246
247
}

Dmitry Kovalev's avatar
Dmitry Kovalev committed
248
249
250
void vp9_initialize_me_consts(VP9_COMP *cpi, int qindex) {
  cpi->mb.sadperbit16 = sad_per_bit16lut[qindex];
  cpi->mb.sadperbit4 = sad_per_bit4lut[qindex];
John Koleszar's avatar
John Koleszar committed
251
252
}

253

Dmitry Kovalev's avatar
Dmitry Kovalev committed
254
void vp9_initialize_rd_consts(VP9_COMP *cpi, int qindex) {
255
  int q, i;
John Koleszar's avatar
John Koleszar committed
256

257
  vp9_clear_system_state();  // __asm emms;
John Koleszar's avatar
John Koleszar committed
258

John Koleszar's avatar
John Koleszar committed
259
260
261
262
  // Further tests required to see if optimum is different
  // for key frames, golden frames and arf frames.
  // if (cpi->common.refresh_golden_frame ||
  //     cpi->common.refresh_alt_ref_frame)
Dmitry Kovalev's avatar
Dmitry Kovalev committed
263
  qindex = (qindex < 0) ? 0 : ((qindex > MAXQ) ? MAXQ : qindex);
264

Dmitry Kovalev's avatar
Dmitry Kovalev committed
265
  cpi->RDMULT = compute_rd_mult(qindex);
John Koleszar's avatar
John Koleszar committed
266
267
268
269
270
  if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    if (cpi->twopass.next_iiratio > 31)
      cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
    else
      cpi->RDMULT +=
Dmitry Kovalev's avatar
Dmitry Kovalev committed
271
          (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
John Koleszar's avatar
John Koleszar committed
272
  }
273
  cpi->mb.errorperbit = cpi->RDMULT >> 6;
John Koleszar's avatar
John Koleszar committed
274
  cpi->mb.errorperbit += (cpi->mb.errorperbit == 0);
275

276
  vp9_set_speed_features(cpi);
John Koleszar's avatar
John Koleszar committed
277

Dmitry Kovalev's avatar
Dmitry Kovalev committed
278
279
  q = (int)pow(vp9_dc_quant(qindex, 0) >> 2, 1.25);
  q <<= 2;
John Koleszar's avatar
John Koleszar committed
280
281
  if (q < 8)
    q = 8;
282

John Koleszar's avatar
John Koleszar committed
283
284
285
  if (cpi->RDMULT > 1000) {
    cpi->RDDIV = 1;
    cpi->RDMULT /= 100;
John Koleszar's avatar
John Koleszar committed
286

John Koleszar's avatar
John Koleszar committed
287
288
289
290
291
292
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < INT_MAX) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q / 100;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
293

John Koleszar's avatar
John Koleszar committed
294
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
295
    }
John Koleszar's avatar
John Koleszar committed
296
297
  } else {
    cpi->RDDIV = 100;
John Koleszar's avatar
John Koleszar committed
298

John Koleszar's avatar
John Koleszar committed
299
300
301
302
303
304
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < (INT_MAX / q)) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
305

John Koleszar's avatar
John Koleszar committed
306
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
307
    }
John Koleszar's avatar
John Koleszar committed
308
  }
John Koleszar's avatar
John Koleszar committed
309

310
  fill_token_costs(cpi->mb.token_costs[TX_4X4],
311
                   cpi->common.fc.coef_probs_4x4, BLOCK_TYPES);
312
  fill_token_costs(cpi->mb.token_costs[TX_8X8],
313
                   cpi->common.fc.coef_probs_8x8, BLOCK_TYPES);
314
  fill_token_costs(cpi->mb.token_costs[TX_16X16],
315
                   cpi->common.fc.coef_probs_16x16, BLOCK_TYPES);
316
  fill_token_costs(cpi->mb.token_costs[TX_32X32],
317
                   cpi->common.fc.coef_probs_32x32, BLOCK_TYPES);
318
319
320
321
322
323
#if CONFIG_CODE_NONZEROCOUNT
  fill_nzc_costs(cpi, 4);
  fill_nzc_costs(cpi, 8);
  fill_nzc_costs(cpi, 16);
  fill_nzc_costs(cpi, 32);
#endif
324

John Koleszar's avatar
John Koleszar committed
325
326
  /*rough estimate for costing*/
  cpi->common.kf_ymode_probs_index = cpi->common.base_qindex >> 4;
327
  vp9_init_mode_costs(cpi);
John Koleszar's avatar
John Koleszar committed
328

329
  if (cpi->common.frame_type != KEY_FRAME) {
330
    vp9_build_nmv_cost_table(
331
332
333
334
335
336
        cpi->mb.nmvjointcost,
        cpi->mb.e_mbd.allow_high_precision_mv ?
        cpi->mb.nmvcost_hp : cpi->mb.nmvcost,
        &cpi->common.fc.nmvc,
        cpi->mb.e_mbd.allow_high_precision_mv, 1, 1);
  }
John Koleszar's avatar
John Koleszar committed
337
338
}

339
int vp9_block_error_c(int16_t *coeff, int16_t *dqcoeff, int block_size) {
340
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
341

342
  for (i = 0; i < block_size; i++) {
John Koleszar's avatar
John Koleszar committed
343
344
345
    int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }
John Koleszar's avatar
John Koleszar committed
346

John Koleszar's avatar
John Koleszar committed
347
  return error;
John Koleszar's avatar
John Koleszar committed
348
349
}

350
int vp9_mbblock_error_c(MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
351
352
353
354
  BLOCK  *be;
  BLOCKD *bd;
  int i, j;
  int berror, error = 0;
John Koleszar's avatar
John Koleszar committed
355

John Koleszar's avatar
John Koleszar committed
356
357
358
359
  for (i = 0; i < 16; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
    berror = 0;
360
    for (j = 0; j < 16; j++) {
John Koleszar's avatar
John Koleszar committed
361
362
      int this_diff = be->coeff[j] - bd->dqcoeff[j];
      berror += this_diff * this_diff;
John Koleszar's avatar
John Koleszar committed
363
    }
John Koleszar's avatar
John Koleszar committed
364
365
366
    error += berror;
  }
  return error;
John Koleszar's avatar
John Koleszar committed
367
368
}

369
int vp9_mbuverror_c(MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
370
371
  BLOCK  *be;
  BLOCKD *bd;
John Koleszar's avatar
John Koleszar committed
372

373
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
374

John Koleszar's avatar
John Koleszar committed
375
376
377
  for (i = 16; i < 24; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
John Koleszar's avatar
John Koleszar committed
378

379
    error += vp9_block_error_c(be->coeff, bd->dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
380
  }
John Koleszar's avatar
John Koleszar committed
381

John Koleszar's avatar
John Koleszar committed
382
  return error;
John Koleszar's avatar
John Koleszar committed
383
384
}

385
int vp9_uvsse(MACROBLOCK *x) {
386
387
388
  uint8_t *uptr, *vptr;
  uint8_t *upred_ptr = (*(x->block[16].base_src) + x->block[16].src);
  uint8_t *vpred_ptr = (*(x->block[20].base_src) + x->block[20].src);
John Koleszar's avatar
John Koleszar committed
389
390
391
392
  int uv_stride = x->block[16].src_stride;

  unsigned int sse1 = 0;
  unsigned int sse2 = 0;
393
394
  int mv_row = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.row;
  int mv_col = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.col;
John Koleszar's avatar
John Koleszar committed
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
  int offset;
  int pre_stride = x->e_mbd.block[16].pre_stride;

  if (mv_row < 0)
    mv_row -= 1;
  else
    mv_row += 1;

  if (mv_col < 0)
    mv_col -= 1;
  else
    mv_col += 1;

  mv_row /= 2;
  mv_col /= 2;

  offset = (mv_row >> 3) * pre_stride + (mv_col >> 3);
  uptr = x->e_mbd.pre.u_buffer + offset;
  vptr = x->e_mbd.pre.v_buffer + offset;

  if ((mv_row | mv_col) & 7) {
416
    vp9_sub_pixel_variance8x8(uptr, pre_stride, (mv_col & 7) << 1,
417
                              (mv_row & 7) << 1, upred_ptr, uv_stride, &sse2);
418
    vp9_sub_pixel_variance8x8(vptr, pre_stride, (mv_col & 7) << 1,
419
                              (mv_row & 7) << 1, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
420
421
    sse2 += sse1;
  } else {
422
423
    vp9_variance8x8(uptr, pre_stride, upred_ptr, uv_stride, &sse2);
    vp9_variance8x8(vptr, pre_stride, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
424
425
426
    sse2 += sse1;
  }
  return sse2;
John Koleszar's avatar
John Koleszar committed
427
428
}

429
static INLINE int cost_coeffs(VP9_COMMON *const cm, MACROBLOCK *mb,
430
                              int ib, PLANE_TYPE type,
431
432
433
                              ENTROPY_CONTEXT *a,
                              ENTROPY_CONTEXT *l,
                              TX_SIZE tx_size) {
434
  MACROBLOCKD *const xd = &mb->e_mbd;
435
436
  MB_MODE_INFO *mbmi = &xd->mode_info_context->mbmi;
  int pt;
437
  const int eob = xd->eobs[ib];
438
  int c = 0;
439
440
  int cost = 0, pad;
  const int *scan, *nb;
441
442
  const int16_t *qcoeff_ptr = xd->qcoeff + ib * 16;
  const int ref = mbmi->ref_frame != INTRA_FRAME;
443
  unsigned int (*token_costs)[PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS] =
444
      mb->token_costs[tx_size][type][ref];
445
  ENTROPY_CONTEXT a_ec, l_ec;
446
447
448
449
  ENTROPY_CONTEXT *const a1 = a +
      sizeof(ENTROPY_CONTEXT_PLANES)/sizeof(ENTROPY_CONTEXT);
  ENTROPY_CONTEXT *const l1 = l +
      sizeof(ENTROPY_CONTEXT_PLANES)/sizeof(ENTROPY_CONTEXT);
450

451
452
453
454
455
#if CONFIG_CODE_NONZEROCOUNT
  int nzc_context = vp9_get_nzc_context(cm, xd, ib);
  unsigned int *nzc_cost;
#else
  const int segment_id = xd->mode_info_context->mbmi.segment_id;
456
457
  vp9_prob (*coef_probs)[REF_TYPES][COEF_BANDS][PREV_COEF_CONTEXTS]
                        [ENTROPY_NODES];
458
#endif
459
460
  int seg_eob, default_eob;
  uint8_t token_cache[1024];
461
462
463
464
465
466
467
468
469

  // Check for consistency of tx_size with mode info
  if (type == PLANE_TYPE_Y_WITH_DC) {
    assert(xd->mode_info_context->mbmi.txfm_size == tx_size);
  } else {
    TX_SIZE tx_size_uv = get_uv_tx_size(xd);
    assert(tx_size == tx_size_uv);
  }

470
  switch (tx_size) {
471
472
473
    case TX_4X4: {
      const TX_TYPE tx_type = (type == PLANE_TYPE_Y_WITH_DC) ?
                              get_tx_type_4x4(xd, ib) : DCT_DCT;
474
475
      a_ec = *a;
      l_ec = *l;
476
477
478
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_4x4[nzc_context][ref][type];
#else
479
      coef_probs = cm->fc.coef_probs_4x4;
480
#endif
481
      seg_eob = 16;
482
483
484
485
486
487
      if (tx_type == ADST_DCT) {
        scan = vp9_row_scan_4x4;
      } else if (tx_type == DCT_ADST) {
        scan = vp9_col_scan_4x4;
      } else {
        scan = vp9_default_zig_zag1d_4x4;
Daniel Kang's avatar
Daniel Kang committed
488
489
      }
      break;
490
    }
491
492
493
494
495
    case TX_8X8: {
      const BLOCK_SIZE_TYPE sb_type = xd->mode_info_context->mbmi.sb_type;
      const int sz = 3 + sb_type, x = ib & ((1 << sz) - 1), y = ib - x;
      const TX_TYPE tx_type = (type == PLANE_TYPE_Y_WITH_DC) ?
                              get_tx_type_8x8(xd, y + (x >> 1)) : DCT_DCT;
496
497
      a_ec = (a[0] + a[1]) != 0;
      l_ec = (l[0] + l[1]) != 0;
498
499
500
501
502
503
504
      if (tx_type == ADST_DCT) {
        scan = vp9_row_scan_8x8;
      } else if (tx_type == DCT_ADST) {
        scan = vp9_col_scan_8x8;
      } else {
        scan = vp9_default_zig_zag1d_8x8;
      }
505
506
507
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_8x8[nzc_context][ref][type];
#else
508
      coef_probs = cm->fc.coef_probs_8x8;
509
#endif
510
      seg_eob = 64;
Daniel Kang's avatar
Daniel Kang committed
511
      break;
512
513
514
515
516
517
518
519
520
521
522
523
524
    }
    case TX_16X16: {
      const BLOCK_SIZE_TYPE sb_type = xd->mode_info_context->mbmi.sb_type;
      const int sz = 4 + sb_type, x = ib & ((1 << sz) - 1), y = ib - x;
      const TX_TYPE tx_type = (type == PLANE_TYPE_Y_WITH_DC) ?
                              get_tx_type_16x16(xd, y + (x >> 2)) : DCT_DCT;
      if (tx_type == ADST_DCT) {
        scan = vp9_row_scan_16x16;
      } else if (tx_type == DCT_ADST) {
        scan = vp9_col_scan_16x16;
      } else {
        scan = vp9_default_zig_zag1d_16x16;
      }
525
526
527
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_16x16[nzc_context][ref][type];
#else
528
      coef_probs = cm->fc.coef_probs_16x16;
529
#endif
530
      seg_eob = 256;
531
      if (type == PLANE_TYPE_UV) {
532
533
534
535
536
        a_ec = (a[0] + a[1] + a1[0] + a1[1]) != 0;
        l_ec = (l[0] + l[1] + l1[0] + l1[1]) != 0;
      } else {
        a_ec = (a[0] + a[1] + a[2] + a[3]) != 0;
        l_ec = (l[0] + l[1] + l[2] + l[3]) != 0;
Deb Mukherjee's avatar
Deb Mukherjee committed
537
      }
Daniel Kang's avatar
Daniel Kang committed
538
      break;
539
    }
540
541
    case TX_32X32:
      scan = vp9_default_zig_zag1d_32x32;
542
543
544
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_32x32[nzc_context][ref][type];
#else
545
      coef_probs = cm->fc.coef_probs_32x32;
546
#endif
547
      seg_eob = 1024;
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
      if (type == PLANE_TYPE_UV) {
        ENTROPY_CONTEXT *a2, *a3, *l2, *l3;
        a2 = a1 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        a3 = a2 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        l2 = l1 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        l3 = l2 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        a_ec = (a[0] + a[1] + a1[0] + a1[1] +
                a2[0] + a2[1] + a3[0] + a3[1]) != 0;
        l_ec = (l[0] + l[1] + l1[0] + l1[1] +
                l2[0] + l2[1] + l3[0] + l3[1]) != 0;
      } else {
        a_ec = (a[0] + a[1] + a[2] + a[3] +
                a1[0] + a1[1] + a1[2] + a1[3]) != 0;
        l_ec = (l[0] + l[1] + l[2] + l[3] +
                l1[0] + l1[1] + l1[2] + l1[3]) != 0;
      }
564
      break;
Daniel Kang's avatar
Daniel Kang committed
565
    default:
566
      abort();
Daniel Kang's avatar
Daniel Kang committed
567
568
      break;
  }
569

570
  VP9_COMBINEENTROPYCONTEXTS(pt, a_ec, l_ec);
571
572
  nb = vp9_get_coef_neighbors_handle(scan, &pad);
  default_eob = seg_eob;
573

574
#if CONFIG_CODE_NONZEROCOUNT == 0
Paul Wilkins's avatar
Paul Wilkins committed
575
576
  if (vp9_segfeature_active(xd, segment_id, SEG_LVL_SKIP))
    seg_eob = 0;
577
#endif
578

579
  {
580
581
582
#if CONFIG_CODE_NONZEROCOUNT
    int nzc = 0;
#endif
583
584
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
585
      int t = vp9_dct_value_tokens_ptr[v].Token;
586
587
588
#if CONFIG_CODE_NONZEROCOUNT
      nzc += (v != 0);
#endif
589
      token_cache[c] = t;
590
      cost += token_costs[get_coef_band(scan, tx_size, c)][pt][t];
591
      cost += vp9_dct_value_cost_ptr[v];
592
593
594
#if !CONFIG_CODE_NONZEROCOUNT
      if (!c || token_cache[c - 1])
        cost += vp9_cost_bit(coef_probs[type][ref]
595
                                       [get_coef_band(scan, tx_size, c)]
596
597
598
                                       [pt][0], 1);
#endif
      pt = vp9_get_coef_context(scan, nb, pad, token_cache, c, default_eob);
599
    }
600
601
602
#if CONFIG_CODE_NONZEROCOUNT
    cost += nzc_cost[nzc];
#else
603
    if (c < seg_eob)
604
605
606
      cost += mb->token_costs[tx_size][type][ref]
                             [get_coef_band(scan, tx_size, c)]
                             [pt][DCT_EOB_TOKEN];
607
#endif
608
609
  }

610
  // is eob first coefficient;
611
  pt = (c > 0);
612
  *a = *l = pt;
613
614
615
616
617
618
619
620
621
622
623
624
625
626
  if (tx_size >= TX_8X8) {
    a[1] = l[1] = pt;
    if (tx_size >= TX_16X16) {
      if (type == PLANE_TYPE_UV) {
        a1[0] = a1[1] = l1[0] = l1[1] = pt;
      } else {
        a[2] = a[3] = l[2] = l[3] = pt;
        if (tx_size >= TX_32X32) {
          a1[0] = a1[1] = a1[2] = a1[3] = pt;
          l1[0] = l1[1] = l1[2] = l1[3] = pt;
        }
      }
    }
  }
627
628
629
  return cost;
}

630
static int rdcost_mby_4x4(VP9_COMMON *const cm, MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
631
632
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
633
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
634
  ENTROPY_CONTEXT_PLANES t_above, t_left;
635
636
  ENTROPY_CONTEXT *ta = (ENTROPY_CONTEXT *)&t_above;
  ENTROPY_CONTEXT *tl = (ENTROPY_CONTEXT *)&t_left;
John Koleszar's avatar
John Koleszar committed
637

638
639
  vpx_memcpy(&t_above, xd->above_context, sizeof(t_above));
  vpx_memcpy(&t_left, xd->left_context, sizeof(t_left));
John Koleszar's avatar
John Koleszar committed
640

John Koleszar's avatar
John Koleszar committed
641
  for (b = 0; b < 16; b++)
642
    cost += cost_coeffs(cm, mb, b, PLANE_TYPE_Y_WITH_DC,
Yaowu Xu's avatar
Yaowu Xu committed
643
644
                        ta + vp9_block2above[TX_4X4][b],
                        tl + vp9_block2left[TX_4X4][b],
Daniel Kang's avatar
Daniel Kang committed
645
                        TX_4X4);
John Koleszar's avatar
John Koleszar committed
646

John Koleszar's avatar
John Koleszar committed
647
  return cost;
John Koleszar's avatar
John Koleszar committed
648
649
}

650
651
static void macro_block_yrd_4x4(VP9_COMMON *const cm,
                                MACROBLOCK *mb,
652
653
654
                                int *rate,
                                int *distortion,
                                int *skippable) {
Paul Wilkins's avatar
Paul Wilkins committed
655
  MACROBLOCKD *const xd = &mb->e_mbd;
656

657
658
659
  xd->mode_info_context->mbmi.txfm_size = TX_4X4;
  vp9_transform_mby_4x4(mb);
  vp9_quantize_mby_4x4(mb);
660

661
662
  *distortion = vp9_mbblock_error(mb) >> 2;
  *rate = rdcost_mby_4x4(cm, mb);
663
  *skippable = vp9_mby_is_skippable_4x4(xd);
664
}
John Koleszar's avatar
John Koleszar committed
665

666
static int rdcost_mby_8x8(VP9_COMMON *const cm, MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
667
668
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
669
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
670
  ENTROPY_CONTEXT_PLANES t_above, t_left;
671
672
  ENTROPY_CONTEXT *ta = (ENTROPY_CONTEXT *)&t_above;
  ENTROPY_CONTEXT *tl = (ENTROPY_CONTEXT *)&t_left;
John Koleszar's avatar
John Koleszar committed
673

674
675
  vpx_memcpy(&t_above, xd->above_context, sizeof(t_above));
  vpx_memcpy(&t_left,  xd->left_context, sizeof(t_left));
John Koleszar's avatar
John Koleszar committed
676
677

  for (b = 0; b < 16; b += 4)
678
    cost += cost_coeffs(cm, mb, b, PLANE_TYPE_Y_WITH_DC,
Yaowu Xu's avatar
Yaowu Xu committed
679
680
                        ta + vp9_block2above[TX_8X8][b],
                        tl + vp9_block2left[TX_8X8][b],
Daniel Kang's avatar
Daniel Kang committed
681
                        TX_8X8);
John Koleszar's avatar
John Koleszar committed
682
683

  return cost;
684
685
}

686
687
static void macro_block_yrd_8x8(VP9_COMMON *const cm,
                                MACROBLOCK *mb,
688
689
690
                                int *rate,
                                int *distortion,
                                int *skippable) {
Paul Wilkins's avatar
Paul Wilkins committed
691
  MACROBLOCKD *const xd = &mb->e_mbd;
692
693

  xd->mode_info_context->mbmi.txfm_size = TX_8X8;
694
695
  vp9_transform_mby_8x8(mb);
  vp9_quantize_mby_8x8(mb);
John Koleszar's avatar
John Koleszar committed
696

697
698
  *distortion = vp9_mbblock_error(mb) >> 2;
  *rate = rdcost_mby_8x8(cm, mb);
699
  *skippable = vp9_mby_is_skippable_8x8(xd);
700
}
701

702
703
static int rdcost_mby_16x16(VP9_COMMON *const cm, MACROBLOCK *mb) {
  MACROBLOCKD *const xd = &mb->e_mbd;
Daniel Kang's avatar
Daniel Kang committed
704
  ENTROPY_CONTEXT_PLANES t_above, t_left;
705
706
  ENTROPY_CONTEXT *ta = (ENTROPY_CONTEXT *)&t_above;
  ENTROPY_CONTEXT *tl = (ENTROPY_CONTEXT *)&t_left;
Daniel Kang's avatar
Daniel Kang committed
707

708
709
  vpx_memcpy(&t_above, xd->above_context, sizeof(t_above));
  vpx_memcpy(&t_left, xd->left_context, sizeof(t_left));
Daniel Kang's avatar
Daniel Kang committed
710

711
  return cost_coeffs(cm, mb, 0, PLANE_TYPE_Y_WITH_DC, ta, tl, TX_16X16);
Daniel Kang's avatar
Daniel Kang committed
712
}
713

714
static void macro_block_yrd_16x16(VP9_COMMON *const cm, MACROBLOCK *mb,
715
716
                                  int *rate, int *distortion, int *skippable) {
  MACROBLOCKD *const xd = &mb->e_mbd;
Daniel Kang's avatar
Daniel Kang committed
717

718
719
  xd->mode_info_context->mbmi.txfm_size = TX_16X16;
  vp9_transform_mby_16x16(mb);
720
  vp9_quantize_mby_16x16(mb);
721
722
723
  // TODO(jingning) is it possible to quickly determine whether to force
  //                trailing coefficients to be zero, instead of running trellis
  //                optimization in the rate-distortion optimization loop?
724
725
  if (mb->optimize &&
      xd->mode_info_context->mbmi.mode < I8X8_PRED)
726
    vp9_optimize_mby_16x16(cm, mb);
727

728
729
  *distortion = vp9_mbblock_error(mb) >> 2;
  *rate = rdcost_mby_16x16(cm, mb);
730
  *skippable = vp9_mby_is_skippable_16x16(xd);
Daniel Kang's avatar
Daniel Kang committed
731
732
}

733
static void choose_txfm_size_from_rd(VP9_COMP *cpi, MACROBLOCK *x,
734
735
736
737
738
                                     int (*r)[2], int *rate,
                                     int *d, int *distortion,
                                     int *s, int *skip,
                                     int64_t txfm_cache[NB_TXFM_MODES],
                                     TX_SIZE max_txfm_size) {
739
740
741
742
743
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
  vp9_prob skip_prob = cm->mb_no_coeff_skip ?
                       vp9_get_pred_prob(cm, xd, PRED_MBSKIP) : 128;
744
745
746
747
748
749
750
751
752
753
754
755
  int64_t rd[TX_SIZE_MAX_SB][2];
  int n, m;

  for (n = TX_4X4; n <= max_txfm_size; n++) {
    r[n][1] = r[n][0];
    for (m = 0; m <= n - (n == max_txfm_size); m++) {
      if (m == n)
        r[n][1] += vp9_cost_zero(cm->prob_tx[m]);
      else
        r[n][1] += vp9_cost_one(cm->prob_tx[m]);
    }
  }
756
757
758
759
760
761
762
763

  if (cm->mb_no_coeff_skip) {
    int s0, s1;

    assert(skip_prob > 0);
    s0 = vp9_cost_bit(skip_prob, 0);
    s1 = vp9_cost_bit(skip_prob, 1);

764
    for (n = TX_4X4; n <= max_txfm_size; n++) {
765
      if (s[n]) {
766
        rd[n][0] = rd[n][1] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
767
      } else {
768
769
        rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0] + s0, d[n]);
        rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1] + s0, d[n]);
770
      }
771
772
    }
  } else {
773
774
775
    for (n = TX_4X4; n <= max_txfm_size; n++) {
      rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0], d[n]);
      rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1], d[n]);
776
777
778
    }
  }

779
780
781
782
783
784
  if (max_txfm_size == TX_32X32 &&
      (cm->txfm_mode == ALLOW_32X32 ||
       (cm->txfm_mode == TX_MODE_SELECT &&
        rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
        rd[TX_32X32][1] < rd[TX_4X4][1]))) {
    mbmi->txfm_size = TX_32X32;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
785
786
787
788
789
  } else if ( cm->txfm_mode == ALLOW_16X16 ||
             (max_txfm_size == TX_16X16 && cm->txfm_mode == ALLOW_32X32) ||
             (cm->txfm_mode == TX_MODE_SELECT &&
              rd[TX_16X16][1] < rd[TX_8X8][1] &&
              rd[TX_16X16][1] < rd[TX_4X4][1])) {
790
    mbmi->txfm_size = TX_16X16;
791
  } else if (cm->txfm_mode == ALLOW_8X8 ||
792
           (cm->txfm_mode == TX_MODE_SELECT && rd[TX_8X8][1] < rd[TX_4X4][1])) {
793
794
    mbmi->txfm_size = TX_8X8;
  } else {
795
    assert(cm->txfm_mode == ONLY_4X4 || cm->txfm_mode == TX_MODE_SELECT);
796
797
798
    mbmi->txfm_size = TX_4X4;
  }

799
  *distortion = d[mbmi->txfm_size];
800
  *rate       = r[mbmi->txfm_size][cm->txfm_mode == TX_MODE_SELECT];
801
802
  *skip       = s[mbmi->txfm_size];

803
804
805
806
807
808
809
810
  txfm_cache[ONLY_4X4] = rd[TX_4X4][0];
  txfm_cache[ALLOW_8X8] = rd[TX_8X8][0];
  txfm_cache[ALLOW_16X16] = rd[TX_16X16][0];
  txfm_cache[ALLOW_32X32] = rd[max_txfm_size][0];
  if (max_txfm_size == TX_32X32 &&
      rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
      rd[TX_32X32][1] < rd[TX_4X4][1])
    txfm_cache[TX_MODE_SELECT] = rd[TX_32X32][1];
Ronald S. Bultje's avatar
Ronald S. Bultje committed
811
  else if (rd[TX_16X16][1] < rd[TX_8X8][1] && rd[TX_16X16][1] < rd[TX_4X4][1])
812
    txfm_cache[TX_MODE_SELECT] = rd[TX_16X16][1];
813
  else
814
815
    txfm_cache[TX_MODE_SELECT] = rd[TX_4X4][1] < rd[TX_8X8][1] ?
                                 rd[TX_4X4][1] : rd[TX_8X8][1];
816
817
818
819
820
}

static void macro_block_yrd(VP9_COMP *cpi, MACROBLOCK *x, int *rate,
                            int *distortion, int *skippable,
                            int64_t txfm_cache[NB_TXFM_MODES]) {
821
  VP9_COMMON *const cm = &cpi->common;
822
  MACROBLOCKD *const xd = &x->e_mbd;
823
  int r[TX_SIZE_MAX_MB][2], d[TX_SIZE_MAX_MB], s[TX_SIZE_MAX_MB];
824
825
826

  vp9_subtract_mby(x->src_diff, *(x->block[0].base_src), xd->predictor,
                   x->block[0].src_stride);
827

828
829
830
  macro_block_yrd_16x16(cm, x, &r[TX_16X16][0], &d[TX_16X16], &s[TX_16X16]);
  macro_block_yrd_8x8(cm, x, &r[TX_8X8][0], &d[TX_8X8], &s[TX_8X8]);
  macro_block_yrd_4x4(cm, x, &r[TX_4X4][0], &d[TX_4X4], &s[TX_4X4]);
831
832

  choose_txfm_size_from_rd(cpi, x, r, rate, d, distortion, s, skippable,
833
                           txfm_cache, TX_16X16);
834
835
}

836
static void copy_predictor(uint8_t *dst, const uint8_t *predictor) {
John Koleszar's avatar
John Koleszar committed
837
838
839
840
841
842
  const unsigned int *p = (const unsigned int *)predictor;
  unsigned int *d = (unsigned int *)dst;
  d[0] = p[0];
  d[4] = p[4];
  d[8] = p[8];
  d[12] = p[12];
843
}
Yaowu Xu's avatar
Yaowu Xu committed
844

845
static int vp9_sb_block_error_c(int16_t *coeff, int16_t *dqcoeff,
846
                                int block_size, int shift) {
847
848
849
850
851
852
853
  int i;
  int64_t error = 0;

  for (i = 0; i < block_size; i++) {
    unsigned int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }
854
  error >>= shift;
855

Frank Galligan's avatar
Frank Galligan committed
856
  return error > INT_MAX ? INT_MAX : (int)error;
857
858
}

859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
static int rdcost_sby_4x4(VP9_COMMON *const cm, MACROBLOCK *x) {
  int cost = 0, b;
  MACROBLOCKD *const xd = &x->e_mbd;
  ENTROPY_CONTEXT_PLANES t_above[2], t_left[2];
  ENTROPY_CONTEXT *ta = (ENTROPY_CONTEXT *) &t_above;
  ENTROPY_CONTEXT *tl = (ENTROPY_CONTEXT *) &t_left;

  vpx_memcpy(&t_above, xd->above_context, sizeof(t_above));
  vpx_memcpy(&t_left,  xd->left_context,  sizeof(t_left));

  for (b = 0; b < 64; b++)
    cost += cost_coeffs(cm, x, b, PLANE_TYPE_Y_WITH_DC,
                        ta + vp9_block2above_sb[TX_4X4][b],
                        tl + vp9_block2left_sb[TX_4X4][b], TX_4X4);

  return cost;
}

static void super_block_yrd_4x4(VP9_COMMON *const cm, MACROBLOCK *x,
                                int *rate, int *distortion, int *skippable) {
  MACROBLOCKD *const xd = &x->e_mbd;

  xd->mode_info_context->mbmi.txfm_size = TX_4X4;
  vp9_transform_sby_4x4(x);
  vp9_quantize_sby_4x4(x);

  *distortion = vp9_sb_block_error_c(x->coeff, xd->dqcoeff, 1024, 2);
  *rate       = rdcost_sby_4x4(cm, x);
  *skippable  = vp9_sby_is_skippable_4x4(xd);
}

static int rdcost_sby_8x8(VP9_COMMON *const cm, MACROBLOCK *x) {
  int cost = 0, b;
  MACROBLOCKD *const xd = &x->e_mbd;
  ENTROPY_CONTEXT_PLANES t_above[2], t_left[2];
  ENTROPY_CONTEXT *ta = (ENTROPY_CONTEXT *) &t_above;
  ENTROPY_CONTEXT *tl = (ENTROPY_CONTEXT *) &t_left;

  vpx_memcpy(&t_above, xd->above_context, sizeof(t_above));
  vpx_memcpy(&t_left,  xd->left_context,  sizeof(t_left));

  for (b = 0; b < 64; b += 4)
    cost += cost_coeffs(cm, x, b, PLANE_TYPE_Y_WITH_DC,
                        ta + vp9_block2above_sb[TX_8X8][b],
                        tl + vp9_block2left_sb[TX_8X8][b], TX_8X8);

  return cost;
}

static void super_block_yrd_8x8(VP9_COMMON *const cm, MACROBLOCK *x,
                                int *rate, int *distortion, int *skippable) {
  MACROBLOCKD *const xd = &x->e_mbd;

  xd->mode_info_context->mbmi.txfm_size = TX_8X8;
  vp9_transform_sby_8x8(x);
  vp9_quantize_sby_8x8(x);

  *distortion = vp9_sb_block_error_c(x->coeff, xd->dqcoeff, 1024, 2);
  *rate       = rdcost_sby_8x8(cm, x);
  *skippable  = vp9_sby_is_skippable_8x8(xd);
}

static int rdcost_sby_16x16(VP9_COMMON *const cm, MACROBLOCK *x) {
  int cost = 0, b;
  MACROBLOCKD *const xd = &x->e_mbd;
  ENTROPY_CONTEXT_PLANES t_above[2], t_left[2];
  ENTROPY_CONTEXT *ta = (ENTROPY_CONTEXT *) &t_above;
  ENTROPY_CONTEXT *tl = (ENTROPY_CONTEXT *) &t_left;

  vpx_memcpy(&t_above, xd->above_context, sizeof(t_above));
  vpx_memcpy(&t_left,  xd->left_context,  sizeof(t_left));

  for (b = 0; b < 64; b += 16)
    cost += cost_coeffs(cm, x, b, PLANE_TYPE_Y_WITH_DC,
                        ta + vp9_block2above_sb[TX_16X16][b],
                        tl + vp9_block2left_sb[TX_16X16][b], TX_16X16);

  return cost;
}

static void super_block_yrd_16x16(VP9_COMMON *const cm, MACROBLOCK *x,
                                  int *rate, int *distortion, int *skippable) {
  MACROBLOCKD *const xd = &x->e_mbd;

  xd->mode_info_context->mbmi.txfm_size = TX_16X16;
  vp9_transform_sby_16x16(x);
  vp9_quantize_sby_16x16(x);

  *distortion = vp9_sb_block_error_c(x->coeff, xd->dqcoeff, 1024, 2);
  *rate       = rdcost_sby_16x16(cm, x);
  *skippable  = vp9_sby_is_skippable_16x16(xd);
}

static int rdcost_sby_32x32(VP9_COMMON *const cm, MACROBLOCK *x) {
  MACROBLOCKD * const xd = &x->e_mbd;
  ENTROPY_CONTEXT_PLANES t_above[2], t_left[2];
  ENTROPY_CONTEXT *ta = (ENTROPY_CONTEXT *) &t_above;
  ENTROPY_CONTEXT *tl = (ENTROPY_CONTEXT *) &t_left;

  vpx_memcpy(&t_above, xd->above_context, sizeof(t_above));
  vpx_memcpy(&t_left,  xd->left_context,  sizeof(t_left));

  return cost_coeffs(cm, x, 0, PLANE_TYPE_Y_WITH_DC, ta, tl, TX_32X32);
}

964
static void super_block_yrd_32x32(VP9_COMMON *const cm, MACROBLOCK *x,
965
                                  int *rate, int *distortion, int *skippable) {
966
  MACROBLOCKD *const xd = &x->e_mbd;
967

968
  xd->mode_info_context->mbmi.txfm_size = TX_32X32;
969
970
  vp9_transform_sby_32x32(x);
  vp9_quantize_sby_32x32(x);
971

972
973
  *distortion = vp9_sb_block_error_c(x->coeff, xd->dqcoeff, 1024, 0);
  *rate       = rdcost_sby_32x32(cm, x);
974
  *skippable  = vp9_sby_is_skippable_32x32(xd);
975
976
}

977
978
static void super_block_yrd(VP9_COMP *cpi,
                            MACROBLOCK *x, int *rate, int *distortion,
979
                            int *skip,
980
                            int64_t txfm_cache[NB_TXFM_MODES]) {
981
  VP9_COMMON *const cm = &cpi->common;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
982
  MACROBLOCKD *const xd = &x->e_mbd;
983
  int r[TX_SIZE_MAX_SB][2], d[TX_SIZE_MAX_SB], s[TX_SIZE_MAX_SB];
Ronald S. Bultje's avatar
Ronald S. Bultje committed
984
985
986
  const uint8_t *src = x->src.y_buffer, *dst = xd->dst.y_buffer;
  int src_y_stride = x->src.y_stride, dst_y_stride = xd->dst.y_stride;

987
988
989
990
991
  vp9_subtract_sby_s_c(x->src_diff, src, src_y_stride, dst, dst_y_stride);
  super_block_yrd_32x32(cm, x, &r[TX_32X32][0], &d[TX_32X32], &s[TX_32X32]);
  super_block_yrd_16x16(cm, x, &r[TX_16X16][0], &d[TX_16X16], &s[TX_16X16]);
  super_block_yrd_8x8(cm, x,   &r[TX_8X8][0],   &d[TX_8X8],   &s[TX_8X8]);
  super_block_yrd_4x4(cm, x,   &r[TX_4X4][0],   &d[TX_4X4],   &s[TX_4X4]);
992
993
994

  choose_txfm_size_from_rd(cpi, x, r, rate, d, distortion, s, skip, txfm_cache,
                           TX_SIZE_MAX_SB - 1);
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032