vp9_rdopt.c 175 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5
6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9
10
11
12
13
14
15
 */


#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
16
17
#include "vp9/common/vp9_pragmas.h"

18
19
20
21
22
#include "vp9/encoder/vp9_tokenize.h"
#include "vp9/encoder/vp9_treewriter.h"
#include "vp9/encoder/vp9_onyx_int.h"
#include "vp9/encoder/vp9_modecosts.h"
#include "vp9/encoder/vp9_encodeintra.h"
23
24
25
26
27
#include "vp9/common/vp9_entropymode.h"
#include "vp9/common/vp9_reconinter.h"
#include "vp9/common/vp9_reconintra.h"
#include "vp9/common/vp9_findnearmv.h"
#include "vp9/common/vp9_quant_common.h"
28
29
30
31
32
33
#include "vp9/encoder/vp9_encodemb.h"
#include "vp9/encoder/vp9_quantize.h"
#include "vp9/encoder/vp9_variance.h"
#include "vp9/encoder/vp9_mcomp.h"
#include "vp9/encoder/vp9_rdopt.h"
#include "vp9/encoder/vp9_ratectrl.h"
John Koleszar's avatar
John Koleszar committed
34
#include "vpx_mem/vpx_mem.h"
35
36
#include "vp9/common/vp9_systemdependent.h"
#include "vp9/encoder/vp9_encodemv.h"
John Koleszar's avatar
John Koleszar committed
37

38
39
40
#include "vp9/common/vp9_seg_common.h"
#include "vp9/common/vp9_pred_common.h"
#include "vp9/common/vp9_entropy.h"
41
#include "vp9_rtcd.h"
42
#include "vp9/common/vp9_mvref_common.h"
Ronald S. Bultje's avatar
Ronald S. Bultje committed
43
#include "vp9/common/vp9_common.h"
Paul Wilkins's avatar
Paul Wilkins committed
44

John Koleszar's avatar
John Koleszar committed
45
46
#define MAXF(a,b)            (((a) > (b)) ? (a) : (b))

47
48
#define INVALID_MV 0x80008000

49
50
51
/* Factor to weigh the rate for switchable interp filters */
#define SWITCHABLE_INTERP_RATE_FACTOR 1

John Koleszar's avatar
John Koleszar committed
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
static const int auto_speed_thresh[17] = {
  1000,
  200,
  150,
  130,
  150,
  125,
  120,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  105
John Koleszar's avatar
John Koleszar committed
70
71
};

72
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
73
74
  {ZEROMV,    LAST_FRAME,   NONE},
  {DC_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
75

76
77
  {NEARESTMV, LAST_FRAME,   NONE},
  {NEARMV,    LAST_FRAME,   NONE},
John Koleszar's avatar
John Koleszar committed
78

79
80
  {ZEROMV,    GOLDEN_FRAME, NONE},
  {NEARESTMV, GOLDEN_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
81

82
83
  {ZEROMV,    ALTREF_FRAME, NONE},
  {NEARESTMV, ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
84

85
86
  {NEARMV,    GOLDEN_FRAME, NONE},
  {NEARMV,    ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
87

88
89
90
91
92
93
94
95
  {V_PRED,    INTRA_FRAME,  NONE},
  {H_PRED,    INTRA_FRAME,  NONE},
  {D45_PRED,  INTRA_FRAME,  NONE},
  {D135_PRED, INTRA_FRAME,  NONE},
  {D117_PRED, INTRA_FRAME,  NONE},
  {D153_PRED, INTRA_FRAME,  NONE},
  {D27_PRED,  INTRA_FRAME,  NONE},
  {D63_PRED,  INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
96

97
  {TM_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
98

99
100
101
  {NEWMV,     LAST_FRAME,   NONE},
  {NEWMV,     GOLDEN_FRAME, NONE},
  {NEWMV,     ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
102

103
104
105
  {SPLITMV,   LAST_FRAME,   NONE},
  {SPLITMV,   GOLDEN_FRAME, NONE},
  {SPLITMV,   ALTREF_FRAME, NONE},
106

Yaowu Xu's avatar
Yaowu Xu committed
107
  {I4X4_PRED,    INTRA_FRAME,  NONE},
108
  {I8X8_PRED, INTRA_FRAME,  NONE},
109

John Koleszar's avatar
John Koleszar committed
110
111
112
113
  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME},
114

John Koleszar's avatar
John Koleszar committed
115
116
117
  {ZEROMV,    ALTREF_FRAME, LAST_FRAME},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME},
118

John Koleszar's avatar
John Koleszar committed
119
120
121
  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME},
122

John Koleszar's avatar
John Koleszar committed
123
124
125
  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME  },
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME},
126

John Koleszar's avatar
John Koleszar committed
127
128
  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME  },
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME},

#if CONFIG_COMP_INTERINTRA_PRED
  /* compound inter-intra prediction */
  {ZEROMV,    LAST_FRAME,   INTRA_FRAME},
  {NEARESTMV, LAST_FRAME,   INTRA_FRAME},
  {NEARMV,    LAST_FRAME,   INTRA_FRAME},
  {NEWMV,     LAST_FRAME,   INTRA_FRAME},

  {ZEROMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEARESTMV, GOLDEN_FRAME,   INTRA_FRAME},
  {NEARMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEWMV,     GOLDEN_FRAME,   INTRA_FRAME},

  {ZEROMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEARESTMV, ALTREF_FRAME,   INTRA_FRAME},
  {NEARMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEWMV,     ALTREF_FRAME,   INTRA_FRAME},
#endif
John Koleszar's avatar
John Koleszar committed
148
149
};

150
151
static void fill_token_costs(vp9_coeff_count *c,
                             vp9_coeff_probs *p,
152
                             TX_SIZE tx_size) {
153
  int i, j, k, l;
John Koleszar's avatar
John Koleszar committed
154

155
  for (i = 0; i < BLOCK_TYPES; i++)
156
157
158
    for (j = 0; j < REF_TYPES; j++)
      for (k = 0; k < COEF_BANDS; k++)
        for (l = 0; l < PREV_COEF_CONTEXTS; l++) {
159
160
161
162
163
164
165
          vp9_cost_tokens_skip((int *)(c[i][j][k][l]),
                               p[i][j][k][l],
                               vp9_coef_tree);
        }
}

#if CONFIG_CODE_NONZEROCOUNT
166
static void fill_nzc_costs(VP9_COMP *cpi, TX_SIZE tx_size) {
167
168
  int nzc_context, r, b, nzc, values;
  int cost[16];
169
  values = (16 << (2 * tx_size)) + 1;
170
171
172
173

  for (nzc_context = 0; nzc_context < MAX_NZC_CONTEXTS; ++nzc_context) {
    for (r = 0; r < REF_TYPES; ++r) {
      for (b = 0; b < BLOCK_TYPES; ++b) {
174
        unsigned int *nzc_costs;
175
        if (tx_size == TX_4X4) {
176
177
178
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_4x4[nzc_context][r][b],
                          vp9_nzc4x4_tree);
179
          nzc_costs = cpi->mb.nzc_costs_4x4[nzc_context][r][b];
180
        } else if (tx_size == TX_8X8) {
181
182
183
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_8x8[nzc_context][r][b],
                          vp9_nzc8x8_tree);
184
          nzc_costs = cpi->mb.nzc_costs_8x8[nzc_context][r][b];
185
        } else if (tx_size == TX_16X16) {
186
187
188
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_16x16[nzc_context][r][b],
                          vp9_nzc16x16_tree);
189
190
          nzc_costs = cpi->mb.nzc_costs_16x16[nzc_context][r][b];
        } else {
191
192
193
          vp9_cost_tokens(cost,
                          cpi->common.fc.nzc_probs_32x32[nzc_context][r][b],
                          vp9_nzc32x32_tree);
194
195
          nzc_costs = cpi->mb.nzc_costs_32x32[nzc_context][r][b];
        }
196
197
198
199
200

        for (nzc = 0; nzc < values; ++nzc) {
          int e, c, totalcost = 0;
          c = codenzc(nzc);
          totalcost = cost[c];
201
202
          if ((e = vp9_extranzcbits[c])) {
            int x = nzc - vp9_basenzcvalue[c];
203
            while (e--) {
204
205
206
207
              totalcost += vp9_cost_bit(
                  cpi->common.fc.nzc_pcat_probs[nzc_context]
                                               [c - NZC_TOKENS_NOEXTRA][e],
                  ((x >> e) & 1));
208
209
            }
          }
210
          nzc_costs[nzc] = totalcost;
211
        }
212
213
214
      }
    }
  }
John Koleszar's avatar
John Koleszar committed
215
}
216
#endif
John Koleszar's avatar
John Koleszar committed
217

218

219
220
221
222
static int rd_iifactor[32] =  { 4, 4, 3, 2, 1, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0, };
John Koleszar's avatar
John Koleszar committed
223

224
// 3* dc_qlookup[Q]*dc_qlookup[Q];
225

226
/* values are now correlated to quantizer */
Paul Wilkins's avatar
Paul Wilkins committed
227
228
229
static int sad_per_bit16lut[QINDEX_RANGE];
static int sad_per_bit4lut[QINDEX_RANGE];

230
void vp9_init_me_luts() {
John Koleszar's avatar
John Koleszar committed
231
232
233
234
235
236
237
  int i;

  // Initialize the sad lut tables using a formulaic calculation for now
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < QINDEX_RANGE; i++) {
    sad_per_bit16lut[i] =
238
239
      (int)((0.0418 * vp9_convert_qindex_to_q(i)) + 2.4107);
    sad_per_bit4lut[i] = (int)((0.063 * vp9_convert_qindex_to_q(i)) + 2.742);
John Koleszar's avatar
John Koleszar committed
240
  }
Paul Wilkins's avatar
Paul Wilkins committed
241
}
John Koleszar's avatar
John Koleszar committed
242

243
static int compute_rd_mult(int qindex) {
Dmitry Kovalev's avatar
Dmitry Kovalev committed
244
  int q = vp9_dc_quant(qindex, 0);
245
  return (11 * q * q) >> 2;
246
247
}

Dmitry Kovalev's avatar
Dmitry Kovalev committed
248
249
250
void vp9_initialize_me_consts(VP9_COMP *cpi, int qindex) {
  cpi->mb.sadperbit16 = sad_per_bit16lut[qindex];
  cpi->mb.sadperbit4 = sad_per_bit4lut[qindex];
John Koleszar's avatar
John Koleszar committed
251
252
}

253

Dmitry Kovalev's avatar
Dmitry Kovalev committed
254
void vp9_initialize_rd_consts(VP9_COMP *cpi, int qindex) {
255
  int q, i;
John Koleszar's avatar
John Koleszar committed
256

257
  vp9_clear_system_state();  // __asm emms;
John Koleszar's avatar
John Koleszar committed
258

John Koleszar's avatar
John Koleszar committed
259
260
261
262
  // Further tests required to see if optimum is different
  // for key frames, golden frames and arf frames.
  // if (cpi->common.refresh_golden_frame ||
  //     cpi->common.refresh_alt_ref_frame)
Dmitry Kovalev's avatar
Dmitry Kovalev committed
263
  qindex = (qindex < 0) ? 0 : ((qindex > MAXQ) ? MAXQ : qindex);
264

Dmitry Kovalev's avatar
Dmitry Kovalev committed
265
  cpi->RDMULT = compute_rd_mult(qindex);
John Koleszar's avatar
John Koleszar committed
266
267
268
269
270
  if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    if (cpi->twopass.next_iiratio > 31)
      cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
    else
      cpi->RDMULT +=
Dmitry Kovalev's avatar
Dmitry Kovalev committed
271
          (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
John Koleszar's avatar
John Koleszar committed
272
  }
273
  cpi->mb.errorperbit = cpi->RDMULT >> 6;
John Koleszar's avatar
John Koleszar committed
274
  cpi->mb.errorperbit += (cpi->mb.errorperbit == 0);
275

276
  vp9_set_speed_features(cpi);
John Koleszar's avatar
John Koleszar committed
277

Dmitry Kovalev's avatar
Dmitry Kovalev committed
278
279
  q = (int)pow(vp9_dc_quant(qindex, 0) >> 2, 1.25);
  q <<= 2;
John Koleszar's avatar
John Koleszar committed
280
281
  if (q < 8)
    q = 8;
282

John Koleszar's avatar
John Koleszar committed
283
284
285
  if (cpi->RDMULT > 1000) {
    cpi->RDDIV = 1;
    cpi->RDMULT /= 100;
John Koleszar's avatar
John Koleszar committed
286

John Koleszar's avatar
John Koleszar committed
287
288
289
290
291
292
293
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < INT_MAX) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q / 100;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
294
    }
John Koleszar's avatar
John Koleszar committed
295
296
  } else {
    cpi->RDDIV = 100;
John Koleszar's avatar
John Koleszar committed
297

John Koleszar's avatar
John Koleszar committed
298
299
300
301
302
303
304
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < (INT_MAX / q)) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
305
    }
John Koleszar's avatar
John Koleszar committed
306
  }
John Koleszar's avatar
John Koleszar committed
307

308
  fill_token_costs(cpi->mb.token_costs[TX_4X4],
309
                   cpi->common.fc.coef_probs_4x4, TX_4X4);
310
  fill_token_costs(cpi->mb.token_costs[TX_8X8],
311
                   cpi->common.fc.coef_probs_8x8, TX_8X8);
312
  fill_token_costs(cpi->mb.token_costs[TX_16X16],
313
                   cpi->common.fc.coef_probs_16x16, TX_16X16);
314
  fill_token_costs(cpi->mb.token_costs[TX_32X32],
315
                   cpi->common.fc.coef_probs_32x32, TX_32X32);
316
#if CONFIG_CODE_NONZEROCOUNT
317
318
319
320
  fill_nzc_costs(cpi, TX_4X4);
  fill_nzc_costs(cpi, TX_8X8);
  fill_nzc_costs(cpi, TX_16X16);
  fill_nzc_costs(cpi, TX_32X32);
321
#endif
322

323
324
325
326
327
  for (i = 0; i < 2; i++)
    vp9_cost_tokens(cpi->mb.partition_cost[i],
                    cpi->common.fc.partition_prob[i],
                    vp9_partition_tree);

John Koleszar's avatar
John Koleszar committed
328
329
  /*rough estimate for costing*/
  cpi->common.kf_ymode_probs_index = cpi->common.base_qindex >> 4;
330
  vp9_init_mode_costs(cpi);
John Koleszar's avatar
John Koleszar committed
331

332
  if (cpi->common.frame_type != KEY_FRAME) {
333
    vp9_build_nmv_cost_table(
334
335
336
337
338
339
        cpi->mb.nmvjointcost,
        cpi->mb.e_mbd.allow_high_precision_mv ?
        cpi->mb.nmvcost_hp : cpi->mb.nmvcost,
        &cpi->common.fc.nmvc,
        cpi->mb.e_mbd.allow_high_precision_mv, 1, 1);
  }
John Koleszar's avatar
John Koleszar committed
340
341
}

342
int vp9_block_error_c(int16_t *coeff, int16_t *dqcoeff, int block_size) {
343
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
344

345
  for (i = 0; i < block_size; i++) {
John Koleszar's avatar
John Koleszar committed
346
347
348
    int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }
John Koleszar's avatar
John Koleszar committed
349

John Koleszar's avatar
John Koleszar committed
350
  return error;
John Koleszar's avatar
John Koleszar committed
351
352
}

353
static INLINE int cost_coeffs(VP9_COMMON *const cm, MACROBLOCK *mb,
354
                              int ib, PLANE_TYPE type,
355
356
                              ENTROPY_CONTEXT *a,
                              ENTROPY_CONTEXT *l,
John Koleszar's avatar
John Koleszar committed
357
358
                              TX_SIZE tx_size,
                              int y_blocks) {
359
  MACROBLOCKD *const xd = &mb->e_mbd;
360
361
  MB_MODE_INFO *mbmi = &xd->mode_info_context->mbmi;
  int pt;
362
  int c = 0;
363
364
  int cost = 0, pad;
  const int *scan, *nb;
John Koleszar's avatar
John Koleszar committed
365
366
  const struct plane_block_idx pb_idx = plane_block_idx(y_blocks, ib);
  const int eob = xd->plane[pb_idx.plane].eobs[pb_idx.block];
367
368
  const int16_t *qcoeff_ptr = BLOCK_OFFSET(xd->plane[pb_idx.plane].qcoeff,
                                           pb_idx.block, 16);
369
  const int ref = mbmi->ref_frame != INTRA_FRAME;
370
  unsigned int (*token_costs)[PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS] =
371
      mb->token_costs[tx_size][type][ref];
372
  ENTROPY_CONTEXT a_ec, l_ec;
373
374
375
376
  ENTROPY_CONTEXT *const a1 = a +
      sizeof(ENTROPY_CONTEXT_PLANES)/sizeof(ENTROPY_CONTEXT);
  ENTROPY_CONTEXT *const l1 = l +
      sizeof(ENTROPY_CONTEXT_PLANES)/sizeof(ENTROPY_CONTEXT);
377

378
#if CONFIG_CODE_NONZEROCOUNT
379
  const int nzc_used = get_nzc_used(tx_size);
380
381
  int nzc_context = vp9_get_nzc_context(cm, xd, ib);
  unsigned int *nzc_cost;
382
#endif
383
  const int segment_id = xd->mode_info_context->mbmi.segment_id;
384
385
386
387
  vp9_prob (*coef_probs)[REF_TYPES][COEF_BANDS][PREV_COEF_CONTEXTS]
                        [ENTROPY_NODES];
  int seg_eob, default_eob;
  uint8_t token_cache[1024];
388
389

  // Check for consistency of tx_size with mode info
John Koleszar's avatar
John Koleszar committed
390
  assert((!type && !pb_idx.plane) || (type && pb_idx.plane));
391
392
393
394
395
396
397
  if (type == PLANE_TYPE_Y_WITH_DC) {
    assert(xd->mode_info_context->mbmi.txfm_size == tx_size);
  } else {
    TX_SIZE tx_size_uv = get_uv_tx_size(xd);
    assert(tx_size == tx_size_uv);
  }

398
  switch (tx_size) {
399
400
401
    case TX_4X4: {
      const TX_TYPE tx_type = (type == PLANE_TYPE_Y_WITH_DC) ?
                              get_tx_type_4x4(xd, ib) : DCT_DCT;
402
403
      a_ec = *a;
      l_ec = *l;
404
405
406
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_4x4[nzc_context][ref][type];
#endif
407
      coef_probs = cm->fc.coef_probs_4x4;
408
      seg_eob = 16;
409
410
411
412
413
414
      if (tx_type == ADST_DCT) {
        scan = vp9_row_scan_4x4;
      } else if (tx_type == DCT_ADST) {
        scan = vp9_col_scan_4x4;
      } else {
        scan = vp9_default_zig_zag1d_4x4;
Daniel Kang's avatar
Daniel Kang committed
415
416
      }
      break;
417
    }
418
419
    case TX_8X8: {
      const BLOCK_SIZE_TYPE sb_type = xd->mode_info_context->mbmi.sb_type;
420
421
      const int sz = 3 + mb_width_log2(sb_type);
      const int x = ib & ((1 << sz) - 1), y = ib - x;
422
423
      const TX_TYPE tx_type = (type == PLANE_TYPE_Y_WITH_DC) ?
                              get_tx_type_8x8(xd, y + (x >> 1)) : DCT_DCT;
424
425
      a_ec = (a[0] + a[1]) != 0;
      l_ec = (l[0] + l[1]) != 0;
426
427
428
429
430
431
432
      if (tx_type == ADST_DCT) {
        scan = vp9_row_scan_8x8;
      } else if (tx_type == DCT_ADST) {
        scan = vp9_col_scan_8x8;
      } else {
        scan = vp9_default_zig_zag1d_8x8;
      }
433
434
435
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_8x8[nzc_context][ref][type];
#endif
436
      coef_probs = cm->fc.coef_probs_8x8;
437
      seg_eob = 64;
Daniel Kang's avatar
Daniel Kang committed
438
      break;
439
440
441
    }
    case TX_16X16: {
      const BLOCK_SIZE_TYPE sb_type = xd->mode_info_context->mbmi.sb_type;
442
443
      const int sz = 4 + mb_width_log2(sb_type);
      const int x = ib & ((1 << sz) - 1), y = ib - x;
444
445
446
447
448
449
450
451
452
      const TX_TYPE tx_type = (type == PLANE_TYPE_Y_WITH_DC) ?
                              get_tx_type_16x16(xd, y + (x >> 2)) : DCT_DCT;
      if (tx_type == ADST_DCT) {
        scan = vp9_row_scan_16x16;
      } else if (tx_type == DCT_ADST) {
        scan = vp9_col_scan_16x16;
      } else {
        scan = vp9_default_zig_zag1d_16x16;
      }
453
454
455
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_16x16[nzc_context][ref][type];
#endif
456
      coef_probs = cm->fc.coef_probs_16x16;
457
      seg_eob = 256;
458
      if (type == PLANE_TYPE_UV) {
459
460
461
462
463
        a_ec = (a[0] + a[1] + a1[0] + a1[1]) != 0;
        l_ec = (l[0] + l[1] + l1[0] + l1[1]) != 0;
      } else {
        a_ec = (a[0] + a[1] + a[2] + a[3]) != 0;
        l_ec = (l[0] + l[1] + l[2] + l[3]) != 0;
Deb Mukherjee's avatar
Deb Mukherjee committed
464
      }
Daniel Kang's avatar
Daniel Kang committed
465
      break;
466
    }
467
468
    case TX_32X32:
      scan = vp9_default_zig_zag1d_32x32;
469
470
471
#if CONFIG_CODE_NONZEROCOUNT
      nzc_cost = mb->nzc_costs_32x32[nzc_context][ref][type];
#endif
472
      coef_probs = cm->fc.coef_probs_32x32;
473
      seg_eob = 1024;
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
      if (type == PLANE_TYPE_UV) {
        ENTROPY_CONTEXT *a2, *a3, *l2, *l3;
        a2 = a1 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        a3 = a2 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        l2 = l1 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        l3 = l2 + sizeof(ENTROPY_CONTEXT_PLANES) / sizeof(ENTROPY_CONTEXT);
        a_ec = (a[0] + a[1] + a1[0] + a1[1] +
                a2[0] + a2[1] + a3[0] + a3[1]) != 0;
        l_ec = (l[0] + l[1] + l1[0] + l1[1] +
                l2[0] + l2[1] + l3[0] + l3[1]) != 0;
      } else {
        a_ec = (a[0] + a[1] + a[2] + a[3] +
                a1[0] + a1[1] + a1[2] + a1[3]) != 0;
        l_ec = (l[0] + l[1] + l[2] + l[3] +
                l1[0] + l1[1] + l1[2] + l1[3]) != 0;
      }
490
      break;
Daniel Kang's avatar
Daniel Kang committed
491
    default:
492
      abort();
Daniel Kang's avatar
Daniel Kang committed
493
494
      break;
  }
John Koleszar's avatar
John Koleszar committed
495
  assert(eob <= seg_eob);
496

497
  pt = combine_entropy_contexts(a_ec, l_ec);
498
499
  nb = vp9_get_coef_neighbors_handle(scan, &pad);
  default_eob = seg_eob;
500

501
502
#if CONFIG_CODE_NONZEROCOUNT
  if (!nzc_used)
503
#endif
504
505
    if (vp9_segfeature_active(xd, segment_id, SEG_LVL_SKIP))
      seg_eob = 0;
506

507
508
509
510
  /* sanity check to ensure that we do not have spurious non-zero q values */
  if (eob < seg_eob)
    assert(qcoeff_ptr[scan[eob]] == 0);

511
  {
512
513
514
#if CONFIG_CODE_NONZEROCOUNT
    int nzc = 0;
#endif
515
516
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
517
      int t = vp9_dct_value_tokens_ptr[v].Token;
518
519
520
#if CONFIG_CODE_NONZEROCOUNT
      nzc += (v != 0);
#endif
521
      token_cache[c] = t;
522
      cost += token_costs[get_coef_band(scan, tx_size, c)][pt][t];
523
      cost += vp9_dct_value_cost_ptr[v];
524
525
526
#if !CONFIG_CODE_NONZEROCOUNT
      if (!c || token_cache[c - 1])
        cost += vp9_cost_bit(coef_probs[type][ref]
527
                                       [get_coef_band(scan, tx_size, c)]
528
529
                                       [pt][0], 1);
#endif
530
      pt = vp9_get_coef_context(scan, nb, pad, token_cache, c + 1, default_eob);
531
    }
532
#if CONFIG_CODE_NONZEROCOUNT
533
534
535
    if (nzc_used)
      cost += nzc_cost[nzc];
    else
536
#endif
537
538
539
540
      if (c < seg_eob)
        cost += mb->token_costs[tx_size][type][ref]
                               [get_coef_band(scan, tx_size, c)]
                               [pt][DCT_EOB_TOKEN];
541
542
  }

543
  // is eob first coefficient;
544
  pt = (c > 0);
545
  *a = *l = pt;
546
547
548
549
550
551
552
553
554
555
556
557
558
559
  if (tx_size >= TX_8X8) {
    a[1] = l[1] = pt;
    if (tx_size >= TX_16X16) {
      if (type == PLANE_TYPE_UV) {
        a1[0] = a1[1] = l1[0] = l1[1] = pt;
      } else {
        a[2] = a[3] = l[2] = l[3] = pt;
        if (tx_size >= TX_32X32) {
          a1[0] = a1[1] = a1[2] = a1[3] = pt;
          l1[0] = l1[1] = l1[2] = l1[3] = pt;
        }
      }
    }
  }
560
561
562
  return cost;
}

563
static void choose_txfm_size_from_rd(VP9_COMP *cpi, MACROBLOCK *x,
564
565
566
567
568
                                     int (*r)[2], int *rate,
                                     int *d, int *distortion,
                                     int *s, int *skip,
                                     int64_t txfm_cache[NB_TXFM_MODES],
                                     TX_SIZE max_txfm_size) {
569
570
571
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
572
  vp9_prob skip_prob = vp9_get_pred_prob(cm, xd, PRED_MBSKIP);
573
574
  int64_t rd[TX_SIZE_MAX_SB][2];
  int n, m;
575
  int s0, s1;
576
577
578
579
580
581
582
583
584
585

  for (n = TX_4X4; n <= max_txfm_size; n++) {
    r[n][1] = r[n][0];
    for (m = 0; m <= n - (n == max_txfm_size); m++) {
      if (m == n)
        r[n][1] += vp9_cost_zero(cm->prob_tx[m]);
      else
        r[n][1] += vp9_cost_one(cm->prob_tx[m]);
    }
  }
586

587
588
589
  assert(skip_prob > 0);
  s0 = vp9_cost_bit(skip_prob, 0);
  s1 = vp9_cost_bit(skip_prob, 1);
590

591
592
593
594
595
596
  for (n = TX_4X4; n <= max_txfm_size; n++) {
    if (s[n]) {
      rd[n][0] = rd[n][1] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
    } else {
      rd[n][0] = RDCOST(x->rdmult, x->rddiv, r[n][0] + s0, d[n]);
      rd[n][1] = RDCOST(x->rdmult, x->rddiv, r[n][1] + s0, d[n]);
597
598
599
    }
  }

600
601
602
603
604
605
  if (max_txfm_size == TX_32X32 &&
      (cm->txfm_mode == ALLOW_32X32 ||
       (cm->txfm_mode == TX_MODE_SELECT &&
        rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
        rd[TX_32X32][1] < rd[TX_4X4][1]))) {
    mbmi->txfm_size = TX_32X32;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
606
607
608
609
610
  } else if ( cm->txfm_mode == ALLOW_16X16 ||
             (max_txfm_size == TX_16X16 && cm->txfm_mode == ALLOW_32X32) ||
             (cm->txfm_mode == TX_MODE_SELECT &&
              rd[TX_16X16][1] < rd[TX_8X8][1] &&
              rd[TX_16X16][1] < rd[TX_4X4][1])) {
611
    mbmi->txfm_size = TX_16X16;
612
  } else if (cm->txfm_mode == ALLOW_8X8 ||
613
           (cm->txfm_mode == TX_MODE_SELECT && rd[TX_8X8][1] < rd[TX_4X4][1])) {
614
615
    mbmi->txfm_size = TX_8X8;
  } else {
616
    assert(cm->txfm_mode == ONLY_4X4 || cm->txfm_mode == TX_MODE_SELECT);
617
618
619
    mbmi->txfm_size = TX_4X4;
  }

620
  *distortion = d[mbmi->txfm_size];
621
  *rate       = r[mbmi->txfm_size][cm->txfm_mode == TX_MODE_SELECT];
622
623
  *skip       = s[mbmi->txfm_size];

624
625
626
627
628
629
630
631
  txfm_cache[ONLY_4X4] = rd[TX_4X4][0];
  txfm_cache[ALLOW_8X8] = rd[TX_8X8][0];
  txfm_cache[ALLOW_16X16] = rd[TX_16X16][0];
  txfm_cache[ALLOW_32X32] = rd[max_txfm_size][0];
  if (max_txfm_size == TX_32X32 &&
      rd[TX_32X32][1] < rd[TX_16X16][1] && rd[TX_32X32][1] < rd[TX_8X8][1] &&
      rd[TX_32X32][1] < rd[TX_4X4][1])
    txfm_cache[TX_MODE_SELECT] = rd[TX_32X32][1];
Ronald S. Bultje's avatar
Ronald S. Bultje committed
632
  else if (rd[TX_16X16][1] < rd[TX_8X8][1] && rd[TX_16X16][1] < rd[TX_4X4][1])
633
    txfm_cache[TX_MODE_SELECT] = rd[TX_16X16][1];
634
  else
635
636
    txfm_cache[TX_MODE_SELECT] = rd[TX_4X4][1] < rd[TX_8X8][1] ?
                                 rd[TX_4X4][1] : rd[TX_8X8][1];
637
638
}

639
static int vp9_sb_block_error_c(int16_t *coeff, int16_t *dqcoeff,
640
                                int block_size, int shift) {
641
642
643
644
645
646
647
  int i;
  int64_t error = 0;

  for (i = 0; i < block_size; i++) {
    unsigned int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }
648
  error >>= shift;
649

Frank Galligan's avatar
Frank Galligan committed
650
  return error > INT_MAX ? INT_MAX : (int)error;
651
652
}

653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
static int vp9_sb_uv_block_error_c(int16_t *coeff,
                                   int16_t *dqcoeff0, int16_t *dqcoeff1,
                                   int block_size, int shift) {
  int i;
  int64_t error = 0;

  for (i = 0; i < block_size / 2; i++) {
    unsigned int this_diff = coeff[i] - dqcoeff0[i];
    error += this_diff * this_diff;
  }
  coeff += block_size / 2;
  for (i = 0; i < block_size / 2; i++) {
    unsigned int this_diff = coeff[i] - dqcoeff1[i];
    error += this_diff * this_diff;
  }
  error >>= shift;

  return error > INT_MAX ? INT_MAX : (int)error;
}

673
674
675
676
static int rdcost_sby_4x4(VP9_COMMON *const cm, MACROBLOCK *x,
                          BLOCK_SIZE_TYPE bsize) {
  const int bwl = mb_width_log2(bsize) + 2, bw = 1 << bwl;
  const int bh = 1 << (mb_height_log2(bsize) + 2);
677
678
  int cost = 0, b;
  MACROBLOCKD *const xd = &x->e_mbd;
679
  ENTROPY_CONTEXT_PLANES t_above[4], t_left[4];
680

681
682
683
684
  vpx_memcpy(&t_above, xd->above_context,
             (sizeof(ENTROPY_CONTEXT_PLANES) * bw) >> 2);
  vpx_memcpy(&t_left,  xd->left_context,
             (sizeof(ENTROPY_CONTEXT_PLANES) * bh) >> 2);
685

686
687
  for (b = 0; b < bw * bh; b++) {
    const int x_idx = b & (bw - 1), y_idx = b >> bwl;
688
    cost += cost_coeffs(cm, x, b, PLANE_TYPE_Y_WITH_DC,
689
690
691
692
                ((ENTROPY_CONTEXT *) &t_above[x_idx >> 2]) + (x_idx & 3),
                ((ENTROPY_CONTEXT *) &t_left[y_idx >> 2]) + (y_idx & 3),
                TX_4X4, bw * bh);
  }
693
694
695
696
697

  return cost;
}

static void super_block_yrd_4x4(VP9_COMMON *const cm, MACROBLOCK *x,
698
699
700
                                int *rate, int *distortion, int *skippable,
                                BLOCK_SIZE_TYPE bsize) {
  const int bwl = mb_width_log2(bsize) + 2, bhl = mb_height_log2(bsize) + 2;
701
702
703
  MACROBLOCKD *const xd = &x->e_mbd;

  xd->mode_info_context->mbmi.txfm_size = TX_4X4;
704
705
  vp9_transform_sby_4x4(x, bsize);
  vp9_quantize_sby_4x4(x, bsize);
706

707
708
709
  *distortion = vp9_sb_block_error_c(x->coeff, xd->plane[0].dqcoeff,
                                     16 << (bwl + bhl), 2);
  *rate       = rdcost_sby_4x4(cm, x, bsize);
710
  *skippable  = vp9_sby_is_skippable(xd, bsize);
711
712
}

713
714
715
716
static int rdcost_sby_8x8(VP9_COMMON *const cm, MACROBLOCK *x,
                          BLOCK_SIZE_TYPE bsize) {
  const int bwl = mb_width_log2(bsize) + 1, bw = 1 << bwl;
  const int bh = 1 << (mb_height_log2(bsize) + 1);
717
718
  int cost = 0, b;
  MACROBLOCKD *const xd = &x->e_mbd;
719
  ENTROPY_CONTEXT_PLANES t_above[4], t_left[4];
720

721
722
723
724
725
726
727
728
729
730
731
732
  vpx_memcpy(&t_above, xd->above_context,
             (sizeof(ENTROPY_CONTEXT_PLANES) * bw) >> 1);
  vpx_memcpy(&t_left,  xd->left_context,
             (sizeof(ENTROPY_CONTEXT_PLANES) * bh) >> 1);

  for (b = 0; b < bw * bh; b++) {
    const int x_idx = b & (bw - 1), y_idx = b >> bwl;
    cost += cost_coeffs(cm, x, b * 4, PLANE_TYPE_Y_WITH_DC,
                ((ENTROPY_CONTEXT *) &t_above[x_idx >> 1]) + ((x_idx & 1) << 1),
                ((ENTROPY_CONTEXT *) &t_left[y_idx >> 1]) + ((y_idx & 1) << 1),
                TX_8X8, 4 * bw * bh);
  }
733
734
735
736
737

  return cost;
}

static void super_block_yrd_8x8(VP9_COMMON *const cm, MACROBLOCK *x,
738
739
740
                                int *rate, int *distortion, int *skippable,
                                BLOCK_SIZE_TYPE bsize) {
  const int bwl = mb_width_log2(bsize) + 1, bhl = mb_height_log2(bsize) + 1;
741
742
743
  MACROBLOCKD *const xd = &x->e_mbd;

  xd->mode_info_context->mbmi.txfm_size = TX_8X8;
744
745
  vp9_transform_sby_8x8(x, bsize);
  vp9_quantize_sby_8x8(x, bsize);
746

747
748
749
  *distortion = vp9_sb_block_error_c(x->coeff, xd->plane[0].dqcoeff,
                                     64 << (bhl + bwl), 2);
  *rate       = rdcost_sby_8x8(cm, x, bsize);
750
  *skippable  = vp9_sby_is_skippable(xd, bsize);
751
752
}

753
754
755
756
static int rdcost_sby_16x16(VP9_COMMON *const cm, MACROBLOCK *x,
                            BLOCK_SIZE_TYPE bsize) {
  const int bwl = mb_width_log2(bsize), bw = 1 << bwl;
  const int bh = 1 << mb_height_log2(bsize);
757
758
  int cost = 0, b;
  MACROBLOCKD *const xd = &x->e_mbd;
759
  ENTROPY_CONTEXT_PLANES t_above[4], t_left[4];
760

761
762
  vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES) * bw);
  vpx_memcpy(&t_left,  xd->left_context,  sizeof(ENTROPY_CONTEXT_PLANES) * bh);
763

764
765
766
767
768
769
770
  for (b = 0; b < bw * bh; b++) {
    const int x_idx = b & (bw - 1), y_idx = b >> bwl;
    cost += cost_coeffs(cm, x, b * 16, PLANE_TYPE_Y_WITH_DC,
                        (ENTROPY_CONTEXT *) &t_above[x_idx],
                        (ENTROPY_CONTEXT *) &t_left[y_idx],
                        TX_16X16, bw * bh * 16);
  }
771
772
773
774
775

  return cost;
}

static void super_block_yrd_16x16(VP9_COMMON *const cm, MACROBLOCK *x,
776
777
778
                                  int *rate, int *distortion, int *skippable,
                                  BLOCK_SIZE_TYPE bsize) {
  const int bwl = mb_width_log2(bsize), bhl = mb_height_log2(bsize);
779
780
781
  MACROBLOCKD *const xd = &x->e_mbd;

  xd->mode_info_context->mbmi.txfm_size = TX_16X16;
782
783
  vp9_transform_sby_16x16(x, bsize);
  vp9_quantize_sby_16x16(x, bsize);
784

785
786
787
  *distortion = vp9_sb_block_error_c(x->coeff, xd->plane[0].dqcoeff,
                                     256 << (bwl + bhl), 2);
  *rate       = rdcost_sby_16x16(cm, x, bsize);
788
  *skippable  = vp9_sby_is_skippable(xd, bsize);
789
790
}

791
792
793
794
795
static int rdcost_sby_32x32(VP9_COMMON *const cm, MACROBLOCK *x,
                            BLOCK_SIZE_TYPE bsize) {
  const int bwl = mb_width_log2(bsize) - 1, bw = 1 << bwl;
  const int bh = 1 << (mb_height_log2(bsize) - 1);
  int cost = 0, b;
796
  MACROBLOCKD * const xd = &x->e_mbd;
797
  ENTROPY_CONTEXT_PLANES t_above[4], t_left[4];
798

799
800
801
802
803
804
805
806
807
808
809
810
  vpx_memcpy(&t_above, xd->above_context,
             sizeof(ENTROPY_CONTEXT_PLANES) * bw * 2);
  vpx_memcpy(&t_left,  xd->left_context,
             sizeof(ENTROPY_CONTEXT_PLANES) * bh * 2);

  for (b = 0; b < bw * bh; b++) {
    const int x_idx = b & (bw - 1), y_idx = b >> bwl;
    cost += cost_coeffs(cm, x, b * 64, PLANE_TYPE_Y_WITH_DC,
                        (ENTROPY_CONTEXT *) &t_above[x_idx * 2],
                        (ENTROPY_CONTEXT *) &t_left[y_idx * 2],
                        TX_32X32, bw * bh * 64);
  }
811

812
  return cost;
813
814
}

815
static void super_block_yrd_32x32(VP9_COMMON *const cm, MACROBLOCK *x,
816
817
818
                                  int *rate, int *distortion, int *skippable,
                                  BLOCK_SIZE_TYPE bsize) {
  const int bwl = mb_width_log2(bsize) - 1, bhl = mb_height_log2(bsize) - 1;
819
  MACROBLOCKD *const xd = &x->e_mbd;
820

821
  xd->mode_info_context->mbmi.txfm_size = TX_32X32;
822
823
  vp9_transform_sby_32x32(x, bsize);
  vp9_quantize_sby_32x32(x, bsize);
824

825
826
827
  *distortion = vp9_sb_block_error_c(x->coeff, xd->plane[0].dqcoeff,
                                     1024 << (bwl + bhl), 0);
  *rate       = rdcost_sby_32x32(cm, x, bsize);
828
  *skippable  = vp9_sby_is_skippable(xd, bsize);
829
830
}

831
832
static void super_block_yrd(VP9_COMP *cpi,
                            MACROBLOCK *x, int *rate, int *distortion,
833
                            int *skip, BLOCK_SIZE_TYPE bs,
834
                            int64_t txfm_cache[NB_TXFM_MODES]) {
835
  VP9_COMMON *const cm = &cpi->common;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
836
  MACROBLOCKD *const xd = &x->e_mbd;
837
  int r[TX_SIZE_MAX_SB][2], d[TX_SIZE_MAX_SB], s[TX_SIZE_MAX_SB];
838
  uint8_t *src = x->src.y_buffer, *dst = xd->dst.y_buffer;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
839
840
  int src_y_stride = x->src.y_stride, dst_y_stride = xd->dst.y_stride;

841
  vp9_subtract_sby_s_c(x->src_diff, src, src_y_stride, dst, dst_y_stride, bs);
Ronald S. Bultje's avatar
Ronald S. Bultje committed
842

843
844
845
846
847
848
  if (bs >= BLOCK_SIZE_SB32X32)
    super_block_yrd_32x32(cm, x, &r[TX_32X32][0], &d[TX_32X32], &s[TX_32X32],
                          bs);
  super_block_yrd_16x16(cm, x, &r[TX_16X16][0], &d[TX_16X16], &s[TX_16X16], bs);
  super_block_yrd_8x8(cm, x,   &r[TX_8X8][0],   &d[TX_8X8],   &s[TX_8X8],   bs);
  super_block_yrd_4x4(cm, x,   &r[TX_4X4][0],   &d[TX_4X4],   &s[TX_4X4],   bs);
Ronald S. Bultje's avatar
Ronald S. Bultje committed
849
850

  choose_txfm_size_from_rd(cpi, x, r, rate, d, distortion, s, skip, txfm_cache,
851
                           TX_32X32 - (bs < BLOCK_SIZE_SB32X32));
Ronald S. Bultje's avatar
Ronald S. Bultje committed
852
}
Ronald S. Bultje's avatar
Ronald S. Bultje committed
853

854
855
static int64_t rd_pick_intra4x4block(VP9_COMP *cpi, MACROBLOCK *x, int ib,
                                     B_PREDICTION_MODE *best_mode,
856
857
858
859
                                     int *bmode_costs,
                                     ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l,
                                     int *bestrate, int *bestratey,
                                     int *bestdistortion) {
John Koleszar's avatar
John Koleszar committed
860
  B_PREDICTION_MODE mode;
Deb Mukherjee's avatar
Deb Mukherjee committed
861
  MACROBLOCKD *xd = &x->e_mbd;
862
  int64_t best_rd = INT64_MAX;
John Koleszar's avatar
John Koleszar committed
863
864
  int rate = 0;
  int distortion;
865
  VP9_COMMON *const cm = &cpi->common;
866
867
  BLOCK *be = x->block + ib;
  BLOCKD *b = xd->block + ib;
John Koleszar's avatar
John Koleszar committed
868
869
870

  ENTROPY_CONTEXT ta = *a, tempa = *a;
  ENTROPY_CONTEXT tl = *l, templ = *l;
Deb Mukherjee's avatar
Deb Mukherjee committed
871
872
  TX_TYPE tx_type = DCT_DCT;
  TX_TYPE best_tx_type = DCT_DCT;
John Koleszar's avatar
John Koleszar committed
873
874
875
876
877
  /*
   * The predictor buffer is a 2d buffer with a stride of 16.  Create
   * a temp buffer that meets the stride requirements, but we are only
   * interested in the left 4x4 block
   * */
878
  DECLARE_ALIGNED_ARRAY(16, int16_t, best_dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
879

880
  assert(ib < 16);
881
#if CONFIG_NEWBINTRAMODES
882
  b->bmi.as_mode.context = vp9_find_bpred_context(xd, b);
883
#endif
884
  xd->mode_info_context->mbmi.txfm_size = TX_4X4;
885
  for (mode = B_DC_PRED; mode < LEFT4X4; mode++) {
886
887
    int64_t this_rd;
    int ratey;
888
889

#if CONFIG_NEWBINTRAMODES
890
891
892
893
894
895
896
    if (xd->frame_type == KEY_FRAME) {
      if (mode == B_CONTEXT_PRED) continue;
    } else {
      if (mode >= B_CONTEXT_PRED - CONTEXT_PRED_REPLACEMENTS &&
          mode < B_CONTEXT_PRED)
        continue;
    }
897
#endif
898

899
    b->bmi.as_mode.first = mode;
900
#if CONFIG_NEWBINTRAMODES
901
902
    rate = bmode_costs[
        mode == B_CONTEXT_PRED ? mode - CONTEXT_PRED_REPLACEMENTS : mode];
903
#else
904
    rate = bmode_costs[mode];
905
#endif
John Koleszar's avatar
John Koleszar committed
906

907
    vp9_intra4x4_predict(xd, b, mode, *(b->base_dst) + b->dst, b->dst_stride);
908
    vp9_subtract_b(be, b, 16);
John Koleszar's avatar
John Koleszar committed
909

910
    b->bmi.as_mode.first = mode;
911
    tx_type = get_tx_type_4x4(xd, be - x->block);
912
    if (tx_type != DCT_DCT) {
913
      vp9_short_fht4x4(be->src_diff, be->coeff, 16, tx_type);
914
      vp9_ht_quantize_b_4x4(x, be - x->block, tx_type);
915
    } else {
Yaowu Xu's avatar
Yaowu Xu committed
916
      x->fwd_txm4x4(be->src_diff, be->coeff, 32);
John Koleszar's avatar
John Koleszar committed
917
      x->quantize_b_4x4(x, be - x->block, 16);
918
    }
John Koleszar's avatar
John Koleszar committed
919

920
921
    tempa = ta;
    templ = tl;
Jingning Han's avatar
Jingning Han committed
922

923
    ratey = cost_coeffs(cm, x, b - xd->block,
John Koleszar's avatar
John Koleszar committed
924
                        PLANE_TYPE_Y_WITH_DC, &tempa, &templ, TX_4X4, 16);
925
    rate += ratey;
926
927
928
    distortion = vp9_block_error(be->coeff,
                                 BLOCK_OFFSET(xd->plane[0].dqcoeff, ib, 16),
                                 16) >> 2;
Jingning Han's avatar
Jingning Han committed
929

930
    this_rd = RDCOST(x->rdmult, x->rddiv, rate, distortion);
Jingning Han's avatar
Jingning Han committed
931

932
933
934
935
936
937
938
939
940
    if (this_rd < best_rd) {
      *bestrate = rate;
      *bestratey = ratey;
      *bestdistortion = distortion;
      best_rd = this_rd;
      *best_mode = mode;
      best_tx_type = tx_type;
      *a = tempa;
      *l = templ;
941
      vpx_memcpy