vp9_rdopt.c 179 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5
6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9
10
11
12
13
14
15
 */


#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
16
17
#include "vp9/common/vp9_pragmas.h"

18
19
20
21
22
#include "vp9/encoder/vp9_tokenize.h"
#include "vp9/encoder/vp9_treewriter.h"
#include "vp9/encoder/vp9_onyx_int.h"
#include "vp9/encoder/vp9_modecosts.h"
#include "vp9/encoder/vp9_encodeintra.h"
23
24
25
26
27
#include "vp9/common/vp9_entropymode.h"
#include "vp9/common/vp9_reconinter.h"
#include "vp9/common/vp9_reconintra.h"
#include "vp9/common/vp9_findnearmv.h"
#include "vp9/common/vp9_quant_common.h"
28
29
30
31
32
33
#include "vp9/encoder/vp9_encodemb.h"
#include "vp9/encoder/vp9_quantize.h"
#include "vp9/encoder/vp9_variance.h"
#include "vp9/encoder/vp9_mcomp.h"
#include "vp9/encoder/vp9_rdopt.h"
#include "vp9/encoder/vp9_ratectrl.h"
John Koleszar's avatar
John Koleszar committed
34
#include "vpx_mem/vpx_mem.h"
35
36
#include "vp9/common/vp9_systemdependent.h"
#include "vp9/encoder/vp9_encodemv.h"
John Koleszar's avatar
John Koleszar committed
37

38
39
40
#include "vp9/common/vp9_seg_common.h"
#include "vp9/common/vp9_pred_common.h"
#include "vp9/common/vp9_entropy.h"
41
#include "vp9_rtcd.h"
42
#include "vp9/common/vp9_mvref_common.h"
Ronald S. Bultje's avatar
Ronald S. Bultje committed
43
#include "vp9/common/vp9_common.h"
Paul Wilkins's avatar
Paul Wilkins committed
44

John Koleszar's avatar
John Koleszar committed
45
46
#define MAXF(a,b)            (((a) > (b)) ? (a) : (b))

47
48
#define INVALID_MV 0x80008000

49
50
51
/* Factor to weigh the rate for switchable interp filters */
#define SWITCHABLE_INTERP_RATE_FACTOR 1

John Koleszar's avatar
John Koleszar committed
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
static const int auto_speed_thresh[17] = {
  1000,
  200,
  150,
  130,
  150,
  125,
  120,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  105
John Koleszar's avatar
John Koleszar committed
70
71
};

72
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
73
74
  {ZEROMV,    LAST_FRAME,   NONE},
  {DC_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
75

76
77
  {NEARESTMV, LAST_FRAME,   NONE},
  {NEARMV,    LAST_FRAME,   NONE},
John Koleszar's avatar
John Koleszar committed
78

79
80
  {ZEROMV,    GOLDEN_FRAME, NONE},
  {NEARESTMV, GOLDEN_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
81

82
83
  {ZEROMV,    ALTREF_FRAME, NONE},
  {NEARESTMV, ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
84

85
86
  {NEARMV,    GOLDEN_FRAME, NONE},
  {NEARMV,    ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
87

88
89
90
91
92
93
94
95
  {V_PRED,    INTRA_FRAME,  NONE},
  {H_PRED,    INTRA_FRAME,  NONE},
  {D45_PRED,  INTRA_FRAME,  NONE},
  {D135_PRED, INTRA_FRAME,  NONE},
  {D117_PRED, INTRA_FRAME,  NONE},
  {D153_PRED, INTRA_FRAME,  NONE},
  {D27_PRED,  INTRA_FRAME,  NONE},
  {D63_PRED,  INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
96

97
  {TM_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
98

99
100
101
  {NEWMV,     LAST_FRAME,   NONE},
  {NEWMV,     GOLDEN_FRAME, NONE},
  {NEWMV,     ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
102

103
104
105
  {SPLITMV,   LAST_FRAME,   NONE},
  {SPLITMV,   GOLDEN_FRAME, NONE},
  {SPLITMV,   ALTREF_FRAME, NONE},
106

Yaowu Xu's avatar
Yaowu Xu committed
107
  {I4X4_PRED,    INTRA_FRAME,  NONE},
108
  {I8X8_PRED, INTRA_FRAME,  NONE},
109

John Koleszar's avatar
John Koleszar committed
110
111
112
113
  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME},
114

John Koleszar's avatar
John Koleszar committed
115
116
117
  {ZEROMV,    ALTREF_FRAME, LAST_FRAME},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME},
118

John Koleszar's avatar
John Koleszar committed
119
120
121
  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME},
122

John Koleszar's avatar
John Koleszar committed
123
124
125
  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME  },
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME},
126

John Koleszar's avatar
John Koleszar committed
127
128
  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME  },
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME},

#if CONFIG_COMP_INTERINTRA_PRED
  /* compound inter-intra prediction */
  {ZEROMV,    LAST_FRAME,   INTRA_FRAME},
  {NEARESTMV, LAST_FRAME,   INTRA_FRAME},
  {NEARMV,    LAST_FRAME,   INTRA_FRAME},
  {NEWMV,     LAST_FRAME,   INTRA_FRAME},

  {ZEROMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEARESTMV, GOLDEN_FRAME,   INTRA_FRAME},
  {NEARMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEWMV,     GOLDEN_FRAME,   INTRA_FRAME},

  {ZEROMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEARESTMV, ALTREF_FRAME,   INTRA_FRAME},
  {NEARMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEWMV,     ALTREF_FRAME,   INTRA_FRAME},
#endif
John Koleszar's avatar
John Koleszar committed
148
149
};

150
151
static void fill_token_costs(vp9_coeff_count *c,
                             vp9_coeff_probs *p,
152
                             TX_SIZE tx_size) {
153
  int i, j, k, l;
John Koleszar's avatar
John Koleszar committed
154

155
  for (i = 0; i < BLOCK_TYPES; i++)
156
157
158
    for (j = 0; j < REF_TYPES; j++)
      for (k = 0; k < COEF_BANDS; k++)
        for (l = 0; l < PREV_COEF_CONTEXTS; l++) {
159
160
161
162
163
164
165
          vp9_cost_tokens_skip((int *)(c[i][j][k][l]),
                               p[i][j][k][l],
                               vp9_coef_tree);
        }
}

#if CONFIG_CODE_NONZEROCOUNT
166
static void fill_nzc_costs(VP9_COMP *cpi, TX_SIZE tx_size) {
167
168
  int nzc_context, r, b, nzc, values;
  int cost[16];
169
  values = (16 << (2 * tx_size)) + 1;
170
171
172
173

  for (nzc_context = 0; nzc_context < MAX_NZC_CONTEXTS; ++nzc_context) {
    for (r = 0; r < REF_TYPES; ++r) {
      for (b = 0; b < BLOCK_TYPES; ++b) {
174
        unsigned int *nzc_costs;
175
        if (tx_size == TX_4X4) {
176
177
178
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_4x4[nzc_context][r][b],
                          vp9_nzc4x4_tree);
179
          nzc_costs = cpi->mb.nzc_costs_4x4[nzc_context][r][b];
180
        } else if (tx_size == TX_8X8) {
181
182
183
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_8x8[nzc_context][r][b],
                          vp9_nzc8x8_tree);
184
          nzc_costs = cpi->mb.nzc_costs_8x8[nzc_context][r][b];
185
        } else if (tx_size == TX_16X16) {
186
187
188
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_16x16[nzc_context][r][b],
                          vp9_nzc16x16_tree);
189
190
          nzc_costs = cpi->mb.nzc_costs_16x16[nzc_context][r][b];
        } else {
191
192
193
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_32x32[nzc_context][r][b],
                          vp9_nzc32x32_tree);
194
195
          nzc_costs = cpi->mb.nzc_costs_32x32[nzc_context][r][b];
        }
196
197
198
199
200

        for (nzc = 0; nzc < values; ++nzc) {
          int e, c, totalcost = 0;
          c = codenzc(nzc);
          totalcost = cost[c];
201
202
          if ((e = vp9_extranzcbits[c])) {
            int x = nzc - vp9_basenzcvalue[c];
203
            while (e--) {
204
205
206
207
              totalcost += vp9_cost_bit(
                  cpi->common.fc.nzc_pcat_probs[nzc_context]
                                               [c - NZC_TOKENS_NOEXTRA][e],
                  ((x >> e) & 1));
208
209
            }
          }
210
          nzc_costs[nzc] = totalcost;
211
        }
212
213
214
      }
    }
  }
John Koleszar's avatar
John Koleszar committed
215
}
216
#endif
John Koleszar's avatar
John Koleszar committed
217

218

219
220
221
222
static int rd_iifactor[32] =  { 4, 4, 3, 2, 1, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0, };
John Koleszar's avatar
John Koleszar committed
223

224
// 3* dc_qlookup[Q]*dc_qlookup[Q];
225

226
/* values are now correlated to quantizer */
Paul Wilkins's avatar
Paul Wilkins committed
227
228
229
static int sad_per_bit16lut[QINDEX_RANGE];
static int sad_per_bit4lut[QINDEX_RANGE];

230
void vp9_init_me_luts() {
John Koleszar's avatar
John Koleszar committed
231
232
233
234
235
236
237
  int i;

  // Initialize the sad lut tables using a formulaic calculation for now
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < QINDEX_RANGE; i++) {
    sad_per_bit16lut[i] =
238
239
      (int)((0.0418 * vp9_convert_qindex_to_q(i)) + 2.4107);
    sad_per_bit4lut[i] = (int)((0.063 * vp9_convert_qindex_to_q(i)) + 2.742);
John Koleszar's avatar
John Koleszar committed
240
  }
Paul Wilkins's avatar
Paul Wilkins committed
241
}
John Koleszar's avatar
John Koleszar committed
242

243
static int compute_rd_mult(int qindex) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
244
  int q = vp9_dc_quant(qindex, 0);
245
  return (11 * q * q) >> 2;
246
247
}

Dmitry Kovalev's avatar
Dmitry Kovalev committed
248
249
250
void vp9_initialize_me_consts(VP9_COMP *cpi, int qindex) {
  cpi->mb.sadperbit16 = sad_per_bit16lut[qindex];
  cpi->mb.sadperbit4 = sad_per_bit4lut[qindex];
John Koleszar's avatar
John Koleszar committed
251
252
}

253

Dmitry Kovalev's avatar
Dmitry Kovalev committed
254
void vp9_initialize_rd_consts(VP9_COMP *cpi, int qindex) {
255
  int q, i;
John Koleszar's avatar
John Koleszar committed
256

257
  vp9_clear_system_state();  // __asm emms;
John Koleszar's avatar
John Koleszar committed
258

John Koleszar's avatar
John Koleszar committed
259
260
261
262
  // Further tests required to see if optimum is different
  // for key frames, golden frames and arf frames.
  // if (cpi->common.refresh_golden_frame ||
  //     cpi->common.refresh_alt_ref_frame)
Dmitry Kovalev's avatar
Dmitry Kovalev committed
263
  qindex = (qindex < 0) ? 0 : ((qindex > MAXQ) ? MAXQ : qindex);
264

Dmitry Kovalev's avatar
Dmitry Kovalev committed
265
  cpi->RDMULT = compute_rd_mult(qindex);
John Koleszar's avatar
John Koleszar committed
266
267
268
269
270
  if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    if (cpi->twopass.next_iiratio > 31)
      cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
    else
      cpi->RDMULT +=
Dmitry Kovalev's avatar
Dmitry Kovalev committed
271
          (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
John Koleszar's avatar
John Koleszar committed
272
  }
273
  cpi->mb.errorperbit = cpi->RDMULT >> 6;
John Koleszar's avatar
John Koleszar committed
274
  cpi->mb.errorperbit += (cpi->mb.errorperbit == 0);
275

276
  vp9_set_speed_features(cpi);
John Koleszar's avatar
John Koleszar committed
277

Dmitry Kovalev's avatar
Dmitry Kovalev committed
278
279
  q = (int)pow(vp9_dc_quant(qindex, 0) >> 2, 1.25);
  q <<= 2;
John Koleszar's avatar
John Koleszar committed
280
281
  if (q < 8)
    q = 8;
282

John Koleszar's avatar
John Koleszar committed
283
284
285
  if (cpi->RDMULT > 1000) {
    cpi->RDDIV = 1;
    cpi->RDMULT /= 100;
John Koleszar's avatar
John Koleszar committed
286

John Koleszar's avatar
John Koleszar committed
287
288
289
290
291
292
293
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < INT_MAX) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q / 100;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
294
    }
John Koleszar's avatar
John Koleszar committed
295
296
  } else {
    cpi->RDDIV = 100;
John Koleszar's avatar
John Koleszar committed
297

John Koleszar's avatar
John Koleszar committed
298
299
300
301
302
303
304
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < (INT_MAX / q)) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
305
    }
John Koleszar's avatar
John Koleszar committed
306
  }
John Koleszar's avatar
John Koleszar committed
307

308
  fill_token_costs(cpi->mb.token_costs[TX_4X4],
309
                   cpi->common.fc.coef_probs_4x4, TX_4X4);
310
  fill_token_costs(cpi->mb.token_costs[TX_8X8],
311
                   cpi->common.fc.coef_probs_8x8, TX_8X8);
312
  fill_token_costs(cpi->mb.token_costs[TX_16X16],
313
                   cpi->common.fc.coef_probs_16x16, TX_16X16);
314
  fill_token_costs(cpi->mb.token_costs[TX_32X32],
315
                   cpi->common.fc.coef_probs_32x32, TX_32X32);
316
#if CONFIG_CODE_NONZEROCOUNT
317
318
319
320
  fill_nzc_costs(cpi, TX_4X4);
  fill_nzc_costs(cpi, TX_8X8);
  fill_nzc_costs(cpi, TX_16X16);
  fill_nzc_costs(cpi, TX_32X32);
321
#endif
322

323
324
325
326
327
  for (i = 0; i < 2; i++)
    vp9_cost_tokens(cpi->mb.partition_cost[i],
                    cpi->common.fc.partition_prob[i],
                    vp9_partition_tree);

John Koleszar's avatar
John Koleszar committed
328
329
  /*rough estimate for costing*/
  cpi->common.kf_ymode_probs_index = cpi->common.base_qindex >> 4;
330
  vp9_init_mode_costs(cpi);
John Koleszar's avatar
John Koleszar committed
331

332
  if (cpi->common.frame_type != KEY_FRAME) {
333
    vp9_build_nmv_cost_table(
334
335
336
337
338
339
        cpi->mb.nmvjointcost,
        cpi->mb.e_mbd.allow_high_precision_mv ?
        cpi->mb.nmvcost_hp : cpi->mb.nmvcost,
        &cpi->common.fc.nmvc,
        cpi->mb.e_mbd.allow_high_precision_mv, 1, 1);
  }
John Koleszar's avatar
John Koleszar committed
340
341
}

342
int vp9_block_error_c(int16_t *coeff, int16_t *dqcoeff, int block_size) {
343
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
344

345
  for (i = 0; i < block_size; i++) {
John Koleszar's avatar
John Koleszar committed
346
347
348
    int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }
John Koleszar's avatar
John Koleszar committed
349

John Koleszar's avatar
John Koleszar committed
350
  return error;
John Koleszar's avatar
John Koleszar committed
351
352
}

353
static INLINE int cost_coeffs(VP9_COMMON *const cm, MACROBLOCK *mb,
354
                              int ib, PLANE_TYPE type,
355
356
                              ENTROPY_CONTEXT *a,
                              ENTROPY_CONTEXT *l,
John Koleszar's avatar
John Koleszar committed
357
358
                              TX_SIZE tx_size,
                              int y_blocks) {
359
  MACROBLOCKD *const xd = &mb->e_mbd;
360
361
  MB_MODE_INFO *mbmi = &xd->mode_info_context->mbmi;
  int pt;
362
  int c = 0;
363
364
  int cost = 0, pad;
  const int *scan, *nb;
John Koleszar's avatar
John Koleszar committed
365
366
  const struct plane_block_idx pb_idx = plane_block_idx(y_blocks, ib);
  const int eob = xd->plane[pb_idx.plane].eobs[pb_idx.block];
367
368
  const int16_t *qcoeff_ptr = BLOCK_OFFSET(xd->plane[pb_idx.plane].qcoeff,
                                           pb_idx.block, 16);
369
  const int ref = mbmi->ref_frame != INTRA_FRAME;
370
  unsigned int (*token_costs)[PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS] =
371
      mb->token_costs[tx_size][type][ref];
372
  ENTROPY_CONTEXT a_ec, l_ec;
373
374
375
376
  ENTROPY_CONTEXT *const a1 = a +
      sizeof(ENTROPY_CONTEXT_PLANES)/sizeof(ENTROPY_CONTEXT);
  ENTROPY_CONTEXT *const l1 = l +
      sizeof(ENTROPY_CONTEXT_PLANES)/sizeof(ENTROPY_CONTEXT);
377
  TX_TYPE tx_type = DCT_DCT;
378

379
#if CONFIG_CODE_NONZEROCOUNT
380
  const int nzc_used = get_nzc_used(tx_size);
381
382
  int nzc_context = vp9_get_nzc_context(cm, xd, ib);
  unsigned int *nzc_cost;
383
384
385
386
387
388
389
390
391
#endif
#if CONFIG_CODE_ZEROGROUP
  int last_nz_pos[3] = {-1, -1, -1};  // Encoder only
  int is_eoo_list[3] = {0, 0, 0};
  int is_eoo_negative[3] = {0, 0, 0};
  int is_last_zero[3] = {0, 0, 0};
  int o, rc, skip_coef_val;
  vp9_zpc_probs *zpc_probs;
  uint8_t token_cache_full[1024];
392
#endif
393
  const int segment_id = xd->mode_info_context->mbmi.segment_id;
394
395
396
397
  vp9_prob (*coef_probs)[REF_TYPES][COEF_BANDS][PREV_COEF_CONTEXTS]
                        [ENTROPY_NODES];
  int seg_eob, default_eob;
  uint8_t token_cache[1024];
398

399
400
401
402
#if CONFIG_CODE_ZEROGROUP
  vpx_memset(token_cache, UNKNOWN_TOKEN, sizeof(token_cache));
#endif

403
  // Check for consistency of tx_size with mode info
John Koleszar's avatar
John Koleszar committed
404
  assert((!type && !pb_idx.plane) || (type && pb_idx.plane));
405
406
407
408
409
410
411
  if (type == PLANE_TYPE_Y_WITH_DC) {
    assert(xd->mode_info_context->mbmi.txfm_size == tx_size);
  } else {
    TX_SIZE tx_size_uv = get_uv_tx_size(xd);
    assert(tx_size == tx_size_uv);
  }

412
  switch (tx_size) {
413
    case TX_4X4: {
414
415
      tx_type = (type == PLANE_TYPE_Y_WITH_DC) ?
          get_tx_type_4x4(xd, ib) : DCT_DCT;
416
417
      a_ec = *a;
      l_ec = *l;
418
419
420
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_4x4[nzc_context][ref][type];
#endif
421
      coef_probs = cm->fc.coef_probs_4x4;
422
      seg_eob = 16;
423
424
425
426
427
428
      if (tx_type == ADST_DCT) {
        scan = vp9_row_scan_4x4;
      } else if (tx_type == DCT_ADST) {
        scan = vp9_col_scan_4x4;
      } else {
        scan = vp9_default_zig_zag1d_4x4;
Daniel Kang's avatar
Daniel Kang committed
429
      }
430
431
432
#if CONFIG_CODE_ZEROGROUP
      zpc_probs = &cm->fc.zpc_probs_4x4;
#endif
Daniel Kang's avatar
Daniel Kang committed
433
      break;
434
    }
435
436
    case TX_8X8: {
      const BLOCK_SIZE_TYPE sb_type = xd->mode_info_context->mbmi.sb_type;
437
438
      const int sz = 3 + mb_width_log2(sb_type);
      const int x = ib & ((1 << sz) - 1), y = ib - x;
439
440
      TX_TYPE tx_type = (type == PLANE_TYPE_Y_WITH_DC) ?
          get_tx_type_8x8(xd, y + (x >> 1)) : DCT_DCT;
441
442
      a_ec = (a[0] + a[1]) != 0;
      l_ec = (l[0] + l[1]) != 0;
443
444
445
446
447
448
449
      if (tx_type == ADST_DCT) {
        scan = vp9_row_scan_8x8;
      } else if (tx_type == DCT_ADST) {
        scan = vp9_col_scan_8x8;
      } else {
        scan = vp9_default_zig_zag1d_8x8;
      }
450
451
452
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_8x8[nzc_context][ref][type];
#endif
453
      coef_probs = cm->fc.coef_probs_8x8;
454
      seg_eob = 64;
455
456
457
#if CONFIG_CODE_ZEROGROUP
      zpc_probs = &cm->fc.zpc_probs_8x8;
#endif
Daniel Kang's avatar
Daniel Kang committed
458
      break;
459
460
461
    }
    case TX_16X16: {
      const BLOCK_SIZE_TYPE sb_type = xd->mode_info_context->mbmi.sb_type;
462
463
      const int sz = 4 + mb_width_log2(sb_type);
      const int x = ib & ((1 << sz) - 1), y = ib - x;
464
465
      TX_TYPE tx_type = (type == PLANE_TYPE_Y_WITH_DC) ?
          get_tx_type_16x16(xd, y + (x >> 2)) : DCT_DCT;
466
467
468
469
470
471
472
      if (tx_type == ADST_DCT) {
        scan = vp9_row_scan_16x16;
      } else if (tx_type == DCT_ADST) {
        scan = vp9_col_scan_16x16;
      } else {
        scan = vp9_default_zig_zag1d_16x16;
      }
473
474
475
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_16x16[nzc_context][ref][type];
#endif
476
      coef_probs = cm->fc.coef_probs_16x16;
477
      seg_eob = 256;
478
      if (type == PLANE_TYPE_UV) {
479
480
481
482
483
        a_ec = (a[0] + a[1] + a1[0] + a1[1]) != 0;
        l_ec = (l[0] + l[1] + l1[0] + l1[1]) != 0;
      } else {
        a_ec = (a[0] + a[1] + a[2] + a[3]) != 0;
        l_ec = (l[0] + l[1] + l[2] + l[3]) != 0;
Deb Mukherjee's avatar
Deb Mukherjee committed
484
      }
485
486
487
#if CONFIG_CODE_ZEROGROUP
      zpc_probs = &cm->fc.zpc_probs_16x16;
#endif
Daniel Kang's avatar
Daniel Kang committed
488
      break;
489
    }
490
491
    case TX_32X32:
      scan = vp9_default_zig_zag1d_32x32;
492
493
494
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_32x32[nzc_context][ref][type];
#endif
495
      coef_probs = cm->fc.coef_probs_32x32;
496
      seg_eob = 1024;
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
      if (type == PLANE_TYPE_UV) {
        ENTROPY_CONTEXT *a2, *a3, *l2, *l3;
        a2 = a1 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        a3 = a2 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        l2 = l1 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        l3 = l2 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        a_ec = (a[0] + a[1] + a1[0] + a1[1] +
                a2[0] + a2[1] + a3[0] + a3[1]) != 0;
        l_ec = (l[0] + l[1] + l1[0] + l1[1] +
                l2[0] + l2[1] + l3[0] + l3[1]) != 0;
      } else {
        a_ec = (a[0] + a[1] + a[2] + a[3] +
                a1[0] + a1[1] + a1[2] + a1[3]) != 0;
        l_ec = (l[0] + l[1] + l[2] + l[3] +
                l1[0] + l1[1] + l1[2] + l1[3]) != 0;
      }
513
514
515
#if CONFIG_CODE_ZEROGROUP
      zpc_probs = &cm->fc.zpc_probs_32x32;
#endif
516
      break;
Daniel Kang's avatar
Daniel Kang committed
517
    default:
518
      abort();
Daniel Kang's avatar
Daniel Kang committed
519
520
      break;
  }
John Koleszar's avatar
John Koleszar committed
521
  assert(eob <= seg_eob);
522

523
  pt = combine_entropy_contexts(a_ec, l_ec);
524
525
  nb = vp9_get_coef_neighbors_handle(scan, &pad);
  default_eob = seg_eob;
526

527
528
#if CONFIG_CODE_NONZEROCOUNT
  if (!nzc_used)
529
#endif
530
531
    if (vp9_segfeature_active(xd, segment_id, SEG_LVL_SKIP))
      seg_eob = 0;
532

533
534
535
536
  /* sanity check to ensure that we do not have spurious non-zero q values */
  if (eob < seg_eob)
    assert(qcoeff_ptr[scan[eob]] == 0);

537
538
539
540
541
542
543
544
545
546
#if CONFIG_CODE_ZEROGROUP
  vpx_memset(token_cache_full, ZERO_TOKEN, sizeof(token_cache_full));
  for (c = 0; c < eob; ++c) {
    rc = scan[c];
    token_cache_full[rc] = vp9_dct_value_tokens_ptr[qcoeff_ptr[rc]].token;
    o = vp9_get_orientation(rc, tx_size);
    if (qcoeff_ptr[rc] != 0)
      last_nz_pos[o] = c;
  }
#endif
547
  {
548
549
550
#if CONFIG_CODE_NONZEROCOUNT
    int nzc = 0;
#endif
551
    for (c = 0; c < eob; c++) {
552
      int v = qcoeff_ptr[scan[c]];
553
      int t = vp9_dct_value_tokens_ptr[v].token;
554
      int band = get_coef_band(scan, tx_size, c);
555
556
557
#if CONFIG_CODE_NONZEROCOUNT
      nzc += (v != 0);
#endif
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
      if (c)
        pt = vp9_get_coef_context(scan, nb, pad, token_cache, c, default_eob);
#if CONFIG_CODE_ZEROGROUP
      rc = scan[c];
      o = vp9_get_orientation(rc, tx_size);
      skip_coef_val = (token_cache[rc] == ZERO_TOKEN || is_eoo_list[o]);
      if (!skip_coef_val) {
        cost += token_costs[band][pt][t] + vp9_dct_value_cost_ptr[v];
      } else {
        assert(v == 0);
      }
#else
      cost += token_costs[band][pt][t] + vp9_dct_value_cost_ptr[v];
#endif
#if CONFIG_CODE_NONZEROCOUNT
      if (!nzc_used)
#endif
        if (!c || token_cache[scan[c - 1]])
          cost += vp9_cost_bit(coef_probs[type][ref][band][pt][0], 1);
      token_cache[scan[c]] = t;
#if CONFIG_CODE_ZEROGROUP
      if (t == ZERO_TOKEN && !skip_coef_val) {
        int eoo = 0, use_eoo;
#if USE_ZPC_EOORIENT == 1
        use_eoo = vp9_use_eoo(c, seg_eob, scan, tx_size,
                              is_last_zero, is_eoo_list);
#else
        use_eoo = 0;
#endif
        if (use_eoo) {
          eoo = vp9_is_eoo(c, eob, scan, tx_size, qcoeff_ptr, last_nz_pos);
          if (eoo && is_eoo_negative[o]) eoo = 0;
          if (eoo) {
            int c_;
            int savings = 0;
            int zsaved = 0;
            savings = vp9_cost_bit((*zpc_probs)[ref]
                                   [coef_to_zpc_band(band)]
                                   [coef_to_zpc_ptok(pt)][0], 1) -
                      vp9_cost_bit((*zpc_probs)[ref]
                                   [coef_to_zpc_band(band)]
                                   [coef_to_zpc_ptok(pt)][0], 0);
            for (c_ = c + 1; c_ < eob; ++c_) {
              if (o == vp9_get_orientation(scan[c_], tx_size)) {
                int pt_ = vp9_get_coef_context(scan, nb, pad,
                                               token_cache_full, c_,
                                               default_eob);
                int band_ = get_coef_band(scan, tx_size, c_);
                assert(token_cache_full[scan[c_]] == ZERO_TOKEN);
                if (!c_ || token_cache_full[scan[c_ - 1]])
                  savings += vp9_cost_bit(
                      coef_probs[type][ref][band_][pt_][0], 1);
                savings += vp9_cost_bit(
                    coef_probs[type][ref][band_][pt_][1], 0);
                zsaved++;
              }
            }
            if (savings < 0) {
            // if (zsaved < ZPC_ZEROSSAVED_EOO) {
              eoo = 0;
              is_eoo_negative[o] = 1;
            }
          }
        }
        if (use_eoo) {
          cost += vp9_cost_bit((*zpc_probs)[ref]
                                           [coef_to_zpc_band(band)]
                                           [coef_to_zpc_ptok(pt)][0], !eoo);
          if (eoo) {
            assert(is_eoo_list[o] == 0);
            is_eoo_list[o] = 1;
          }
        }
      }
      is_last_zero[o] = (t == ZERO_TOKEN);
633
#endif
634
    }
635
#if CONFIG_CODE_NONZEROCOUNT
636
637
638
    if (nzc_used)
      cost += nzc_cost[nzc];
    else
639
#endif
640
641
642
      if (c < seg_eob) {
        if (c)
          pt = vp9_get_coef_context(scan, nb, pad, token_cache, c, default_eob);
643
644
645
        cost += mb->token_costs[tx_size][type][ref]
                               [get_coef_band(scan, tx_size, c)]
                               [pt][DCT_EOB_TOKEN];
646
      }
647
648
  }

649
  // is eob first coefficient;
650
  pt = (c > 0);
651
  *a = *l = pt;
652
653
654
655
656
657
658
659
660
661
662
663
664
665
  if (tx_size >= TX_8X8) {
    a[1] = l[1] = pt;
    if (tx_size >= TX_16X16) {
      if (type == PLANE_TYPE_UV) {
        a1[0] = a1[1] = l1[0] = l1[1] = pt;
      } else {
        a[2] = a[3] = l[2] = l[3] = pt;
        if (tx_size >= TX_32X32) {
          a1[0] = a1[1] = a1[2] = a1[3] = pt;
          l1[0] = l1[1] = l1[2] = l1[3] = pt;
        }
      }
    }
  }
666
667
668
  return cost;
}

669
static void choose_txfm_size_from_rd(VP9_COMP *cpi, MACROBLOCK *x,
670
671
672
673
674
                                     int (*r)[2], int *rate,
                                     int *d, int *distortion,
                                     int *s, int *skip,
                                     int64_t txfm_cache[NB_TXFM_MODES],
                                     TX_SIZE max_txfm_size) {
675
676
677
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
678
  vp9_prob skip_prob = vp9_get_pred_prob(cm, xd, PRED_MBSKIP);
679
680
  int64_t rd[TX_SIZE_MAX_SB][2];
  int n, m;
681
  int s0, s1;
682
683
684
685
686
687
688
689
690
691

  for (n = TX_4X4; n <= max_txfm_size; n++) {
    r[n][1] = r[n][0];
    for (m = 0; m <= n - (n == max_txfm_size); m++) {
      if (m == n)
        r[n][1] += vp9_cost_zero(cm->prob_tx[m]);
      else
        r[n][1] += vp9_cost_one(cm->prob_tx[m]);
    }
  }
692

693
694
695
  assert(skip_prob > 0);
  s0 = vp9_cost_bit(skip_prob, 0);
  s1 = vp9_cost_bit(skip_prob, 1);
696

697
698
699
700
701
702
  for (n = TX_4X4; n <= max_txfm_size; n++) {
    if (s[n]) {
      rd[n][0] = rd[n][1] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
    } else {
      rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0] + s0, d[n]);
      rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1] + s0, d[n]);
703
704
705
    }
  }

706
707
708
709
710
711
  if (max_txfm_size == TX_32X32 &&
      (cm->txfm_mode == ALLOW_32X32 ||
       (cm->txfm_mode == TX_MODE_SELECT &&
        rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
        rd[TX_32X32][1] < rd[TX_4X4][1]))) {
    mbmi->txfm_size = TX_32X32;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
712
713
714
715
716
  } else if ( cm->txfm_mode == ALLOW_16X16 ||
             (max_txfm_size == TX_16X16 && cm->txfm_mode == ALLOW_32X32) ||
             (cm->txfm_mode == TX_MODE_SELECT &&
              rd[TX_16X16][1] < rd[TX_8X8][1] &&
              rd[TX_16X16][1] < rd[TX_4X4][1])) {
717
    mbmi->txfm_size = TX_16X16;
718
  } else if (cm->txfm_mode == ALLOW_8X8 ||
719
           (cm->txfm_mode == TX_MODE_SELECT && rd[TX_8X8][1] < rd[TX_4X4][1])) {
720
721
    mbmi->txfm_size = TX_8X8;
  } else {
722
    assert(cm->txfm_mode == ONLY_4X4 || cm->txfm_mode == TX_MODE_SELECT);
723
724
725
    mbmi->txfm_size = TX_4X4;
  }

726
  *distortion = d[mbmi->txfm_size];
727
  *rate       = r[mbmi->txfm_size][cm->txfm_mode == TX_MODE_SELECT];
728
729
  *skip       = s[mbmi->txfm_size];

730
731
732
733
734
735
736
737
  txfm_cache[ONLY_4X4] = rd[TX_4X4][0];
  txfm_cache[ALLOW_8X8] = rd[TX_8X8][0];
  txfm_cache[ALLOW_16X16] = rd[TX_16X16][0];
  txfm_cache[ALLOW_32X32] = rd[max_txfm_size][0];
  if (max_txfm_size == TX_32X32 &&
      rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
      rd[TX_32X32][1] < rd[TX_4X4][1])
    txfm_cache[TX_MODE_SELECT] = rd[TX_32X32][1];
Ronald S. Bultje's avatar
Ronald S. Bultje committed
738
  else if (rd[TX_16X16][1] < rd[TX_8X8][1] && rd[TX_16X16][1] < rd[TX_4X4][1])
739
    txfm_cache[TX_MODE_SELECT] = rd[TX_16X16][1];
740
  else
741
742
    txfm_cache[TX_MODE_SELECT] = rd[TX_4X4][1] < rd[TX_8X8][1] ?
                                 rd[TX_4X4][1] : rd[TX_8X8][1];
743
744
}

745
static int vp9_sb_block_error_c(int16_t *coeff, int16_t *dqcoeff,
746
                                int block_size, int shift) {
747
748
749
750
751
752
753
  int i;
  int64_t error = 0;

  for (i = 0; i < block_size; i++) {
    unsigned int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }
754
  error >>= shift;
755

Frank Galligan's avatar
Frank Galligan committed
756
  return error > INT_MAX ? INT_MAX : (int)error;
757
758
}

759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
static int vp9_sb_uv_block_error_c(int16_t *coeff,
                                   int16_t *dqcoeff0, int16_t *dqcoeff1,
                                   int block_size, int shift) {
  int i;
  int64_t error = 0;

  for (i = 0; i < block_size / 2; i++) {
    unsigned int this_diff = coeff[i] - dqcoeff0[i];
    error += this_diff * this_diff;
  }
  coeff += block_size / 2;
  for (i = 0; i < block_size / 2; i++) {
    unsigned int this_diff = coeff[i] - dqcoeff1[i];
    error += this_diff * this_diff;
  }
  error >>= shift;

  return error > INT_MAX ? INT_MAX : (int)error;
}

779
780
781
782
static int rdcost_sby_4x4(VP9_COMMON *const cm, MACROBLOCK *x,
                          BLOCK_SIZE_TYPE bsize) {
  const int bwl = mb_width_log2(bsize) + 2, bw = 1 << bwl;
  const int bh = 1 << (mb_height_log2(bsize) + 2);
783
784
  int cost = 0, b;
  MACROBLOCKD *const xd = &x->e_mbd;
785
  ENTROPY_CONTEXT_PLANES t_above[4], t_left[4];
786

787
788
789
790
  vpx_memcpy(&t_above, xd->above_context,
             (sizeof(ENTROPY_CONTEXT_PLANES) * bw) >> 2);
  vpx_memcpy(&t_left,  xd->left_context,
             (sizeof(ENTROPY_CONTEXT_PLANES) * bh) >> 2);
791

792
793
  for (b = 0; b < bw * bh; b++) {
    const int x_idx = b & (bw - 1), y_idx = b >> bwl;
794
    cost += cost_coeffs(cm, x, b, PLANE_TYPE_Y_WITH_DC,
795
796
797
798
                ((ENTROPY_CONTEXT *) &t_above[x_idx >> 2]) + (x_idx & 3),
                ((ENTROPY_CONTEXT *) &t_left[y_idx >> 2]) + (y_idx & 3),
                TX_4X4, bw * bh);
  }
799
800
801
802
803

  return cost;
}

static void super_block_yrd_4x4(VP9_COMMON *const cm, MACROBLOCK *x,
804
805
806
                                int *rate, int *distortion, int *skippable,
                                BLOCK_SIZE_TYPE bsize) {
  const int bwl = mb_width_log2(bsize) + 2, bhl = mb_height_log2(bsize) + 2;
807
808
809
  MACROBLOCKD *const xd = &x->e_mbd;

  xd->mode_info_context->mbmi.txfm_size = TX_4X4;
810
811
  vp9_transform_sby_4x4(x, bsize);
  vp9_quantize_sby_4x4(x, bsize);
812

813
814
815
  *distortion = vp9_sb_block_error_c(x->coeff, xd->plane[0].dqcoeff,
                                     16 << (bwl + bhl), 2);
  *rate       = rdcost_sby_4x4(cm, x, bsize);
816
  *skippable  = vp9_sby_is_skippable(xd, bsize);
817
818
}

819
820
821
822
static int rdcost_sby_8x8(VP9_COMMON *const cm, MACROBLOCK *x,
                          BLOCK_SIZE_TYPE bsize) {
  const int bwl = mb_width_log2(bsize) + 1, bw = 1 << bwl;
  const int bh = 1 << (mb_height_log2(bsize) + 1);
823
824
  int cost = 0, b;
  MACROBLOCKD *const xd = &x->e_mbd;
825
  ENTROPY_CONTEXT_PLANES t_above[4], t_left[4];
826

827
828
829
830
831
832
833
834
835
836
837
838
  vpx_memcpy(&t_above, xd->above_context,
             (sizeof(ENTROPY_CONTEXT_PLANES) * bw) >> 1);
  vpx_memcpy(&t_left,  xd->left_context,
             (sizeof(ENTROPY_CONTEXT_PLANES) * bh) >> 1);

  for (b = 0; b < bw * bh; b++) {
    const int x_idx = b & (bw - 1), y_idx = b >> bwl;
    cost += cost_coeffs(cm, x, b * 4, PLANE_TYPE_Y_WITH_DC,
                ((ENTROPY_CONTEXT *) &t_above[x_idx >> 1]) + ((x_idx & 1) << 1),
                ((ENTROPY_CONTEXT *) &t_left[y_idx >> 1]) + ((y_idx & 1) << 1),
                TX_8X8, 4 * bw * bh);
  }
839
840
841
842
843

  return cost;
}

static void super_block_yrd_8x8(VP9_COMMON *const cm, MACROBLOCK *x,
844
845
846
                                int *rate, int *distortion, int *skippable,
                                BLOCK_SIZE_TYPE bsize) {
  const int bwl = mb_width_log2(bsize) + 1, bhl = mb_height_log2(bsize) + 1;
847
848
849
  MACROBLOCKD *const xd = &x->e_mbd;

  xd->mode_info_context->mbmi.txfm_size = TX_8X8;
850
851
  vp9_transform_sby_8x8(x, bsize);
  vp9_quantize_sby_8x8(x, bsize);
852

853
854
855
  *distortion = vp9_sb_block_error_c(x->coeff, xd->plane[0].dqcoeff,
                                     64 << (bhl + bwl), 2);
  *rate       = rdcost_sby_8x8(cm, x, bsize);
856
  *skippable  = vp9_sby_is_skippable(xd, bsize);
857
858
}

859
860
861
862
static int rdcost_sby_16x16(VP9_COMMON *const cm, MACROBLOCK *x,
                            BLOCK_SIZE_TYPE bsize) {
  const int bwl = mb_width_log2(bsize), bw = 1 << bwl;
  const int bh = 1 << mb_height_log2(bsize);
863
864
  int cost = 0, b;
  MACROBLOCKD *const xd = &x->e_mbd;
865
  ENTROPY_CONTEXT_PLANES t_above[4], t_left[4];
866

867
868
  vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES) * bw);
  vpx_memcpy(&t_left,  xd->left_context,  sizeof(ENTROPY_CONTEXT_PLANES) * bh);
869

870
871
872
873
874
875
876
  for (b = 0; b < bw * bh; b++) {
    const int x_idx = b & (bw - 1), y_idx = b >> bwl;
    cost += cost_coeffs(cm, x, b * 16, PLANE_TYPE_Y_WITH_DC,
                        (ENTROPY_CONTEXT *) &t_above[x_idx],
                        (ENTROPY_CONTEXT *) &t_left[y_idx],
                        TX_16X16, bw * bh * 16);
  }
877
878
879
880
881

  return cost;
}

static void super_block_yrd_16x16(VP9_COMMON *const cm, MACROBLOCK *x,
882
883
884
                                  int *rate, int *distortion, int *skippable,
                                  BLOCK_SIZE_TYPE bsize) {
  const int bwl = mb_width_log2(bsize), bhl = mb_height_log2(bsize);
885
886
887
  MACROBLOCKD *const xd = &x->e_mbd;

  xd->mode_info_context->mbmi.txfm_size = TX_16X16;
888
889
  vp9_transform_sby_16x16(x, bsize);
  vp9_quantize_sby_16x16(x, bsize);
890

891
892
893
  *distortion = vp9_sb_block_error_c(x->coeff, xd->plane[0].dqcoeff,
                                     256 << (bwl + bhl), 2);
  *rate       = rdcost_sby_16x16(cm, x, bsize);
894
  *skippable  = vp9_sby_is_skippable(xd, bsize);
895
896
}

897
898
899
900
901
static int rdcost_sby_32x32(VP9_COMMON *const cm, MACROBLOCK *x,
                            BLOCK_SIZE_TYPE bsize) {
  const int bwl = mb_width_log2(bsize) - 1, bw = 1 << bwl;
  const int bh = 1 << (mb_height_log2(bsize) - 1);
  int cost = 0, b;
902
  MACROBLOCKD * const xd = &x->e_mbd;
903
  ENTROPY_CONTEXT_PLANES t_above[4], t_left[4];
904

905
906
907
908
909
910
911
912
913
914
915
916
  vpx_memcpy(&t_above, xd->above_context,
             sizeof(ENTROPY_CONTEXT_PLANES) * bw * 2);
  vpx_memcpy(&t_left,  xd->left_context,
             sizeof(ENTROPY_CONTEXT_PLANES) * bh * 2);

  for (b = 0; b < bw * bh; b++) {
    const int x_idx = b & (bw - 1), y_idx = b >> bwl;
    cost += cost_coeffs(cm, x, b * 64, PLANE_TYPE_Y_WITH_DC,
                        (ENTROPY_CONTEXT *) &t_above[x_idx * 2],
                        (ENTROPY_CONTEXT *) &t_left[y_idx * 2],
                        TX_32X32, bw * bh * 64);
  }
917

918
  return cost;
919
920
}

921
static void super_block_yrd_32x32(VP9_COMMON *const cm, MACROBLOCK *x,
922
923
924
                                  int *rate, int *distortion, int *skippable,
                                  BLOCK_SIZE_TYPE bsize) {
  const int bwl = mb_width_log2(bsize) - 1, bhl = mb_height_log2(bsize) - 1;
925
  MACROBLOCKD *const xd = &x->e_mbd;
926

927
  xd->mode_info_context->mbmi.txfm_size = TX_32X32;
928
929
  vp9_transform_sby_32x32(x, bsize);
  vp9_quantize_sby_32x32(x, bsize);
930

931
932
933
  *distortion = vp9_sb_block_error_c(x->coeff, xd->plane[0].dqcoeff,
                                     1024 << (bwl + bhl), 0);
  *rate       = rdcost_sby_32x32(cm, x, bsize);
934
  *skippable  = vp9_sby_is_skippable(xd, bsize);
935
936
}

937
938
static void super_block_yrd(VP9_COMP *cpi,
                            MACROBLOCK *x, int *rate, int *distortion,
939
                            int *skip, BLOCK_SIZE_TYPE bs,
940
                            int64_t txfm_cache[NB_TXFM_MODES]) {
941
  VP9_COMMON *const cm = &cpi->common;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
942
  MACROBLOCKD *const xd = &x->e_mbd;
943
  int r[TX_SIZE_MAX_SB][2], d[TX_SIZE_MAX_SB], s[TX_SIZE_MAX_SB];
944
945
  uint8_t *src = x->src.y_buffer, *dst = xd->plane[0].dst.buf;
  int src_y_stride = x->src.y_stride, dst_y_stride = xd->plane[0].dst.stride;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
946

947
  vp9_subtract_sby_s_c(x->src_diff, src, src_y_stride, dst, dst_y_stride, bs);
Ronald S. Bultje's avatar
Ronald S. Bultje committed
948

949
950
951
952
953
954
  if (bs >= BLOCK_SIZE_SB32X32)
    super_block_yrd_32x32(cm, x, &r[TX_32X32][0], &d[TX_32X32], &s[TX_32X32],
                          bs);
  super_block_yrd_16x16(cm, x, &r[TX_16X16][0], &d[TX_16X16], &s[TX_16X16], bs);
  super_block_yrd_8x8(cm, x,   &r[TX_8X8][0],   &d[TX_8X8],   &s[TX_8X8],   bs);
  super_block_yrd_4x4(cm, x,   &r[TX_4X4][0],   &d[TX_4X4],   &s[TX_4X4],   bs);
Ronald S. Bultje's avatar
Ronald S. Bultje committed
955
956

  choose_txfm_size_from_rd(cpi, x, r, rate, d, distortion, s, skip, txfm_cache,
957
                           TX_32X32 - (bs < BLOCK_SIZE_SB32X32));
Ronald S. Bultje's avatar
Ronald S. Bultje committed
958
}
Ronald S. Bultje's avatar
Ronald S. Bultje committed
959

960
961
static int64_t rd_pick_intra4x4block(VP9_COMP *cpi, MACROBLOCK *x, int ib,
                                     B_PREDICTION_MODE *best_mode,
962
963
964
965
                                     int *bmode_costs,
                                     ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l,
                                     int *bestrate, int *bestratey,
                                     int *bestdistortion) {
John Koleszar's avatar
John Koleszar committed
966
  B_PREDICTION_MODE mode;
Deb Mukherjee's avatar
Deb Mukherjee committed
967
  MACROBLOCKD *xd = &x->e_mbd;
968
  int64_t best_rd = INT64_MAX;
John Koleszar's avatar
John Koleszar committed
969
970
  int rate = 0;
  int distortion;
971
  VP9_COMMON *const cm = &cpi->common;
972
973
  BLOCK *be = x->block + ib;
  BLOCKD *b = xd->block + ib;
John Koleszar's avatar
John Koleszar committed
974
975
976

  ENTROPY_CONTEXT ta = *a, tempa = *a;
  ENTROPY_CONTEXT tl = *l, templ = *l;
Deb Mukherjee's avatar
Deb Mukherjee committed
977
978
  TX_TYPE tx_type = DCT_DCT;
  TX_TYPE best_tx_type = DCT_DCT;
John Koleszar's avatar
John Koleszar committed
979
980