vp9_rdopt.c 165 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5
6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9
10
11
12
13
14
15
 */


#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
#include "vp9/common/vp9_pragmas.h"

#include "vp9_tokenize.h"
#include "vp9_treewriter.h"
#include "vp9_onyx_int.h"
#include "vp9_modecosts.h"
#include "vp9_encodeintra.h"
#include "vp9/common/vp9_entropymode.h"
#include "vp9/common/vp9_reconinter.h"
#include "vp9/common/vp9_reconintra.h"
#include "vp9/common/vp9_reconintra4x4.h"
#include "vp9/common/vp9_findnearmv.h"
#include "vp9/common/vp9_quant_common.h"
#include "vp9_encodemb.h"
#include "vp9_quantize.h"
#include "vp9_variance.h"
#include "vp9_mcomp.h"
#include "vp9_rdopt.h"
#include "vp9_ratectrl.h"
John Koleszar's avatar
John Koleszar committed
35
#include "vpx_mem/vpx_mem.h"
36
37
#include "vp9/common/vp9_systemdependent.h"
#include "vp9/encoder/vp9_encodemv.h"
John Koleszar's avatar
John Koleszar committed
38

39
40
41
#include "vp9/common/vp9_seg_common.h"
#include "vp9/common/vp9_pred_common.h"
#include "vp9/common/vp9_entropy.h"
42
#include "vp9_rtcd.h"
43
#include "vp9/common/vp9_mvref_common.h"
Paul Wilkins's avatar
Paul Wilkins committed
44

John Koleszar's avatar
John Koleszar committed
45
46
47
48
49
50
51
52
#if CONFIG_RUNTIME_CPU_DETECT
#define IF_RTCD(x)  (x)
#else
#define IF_RTCD(x)  NULL
#endif

#define MAXF(a,b)            (((a) > (b)) ? (a) : (b))

53
54
#define INVALID_MV 0x80008000

55
56
57
/* Factor to weigh the rate for switchable interp filters */
#define SWITCHABLE_INTERP_RATE_FACTOR 1

John Koleszar's avatar
John Koleszar committed
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
static const int auto_speed_thresh[17] = {
  1000,
  200,
  150,
  130,
  150,
  125,
  120,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  105
John Koleszar's avatar
John Koleszar committed
76
77
};

78
#if CONFIG_PRED_FILTER
79
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
  {ZEROMV,    LAST_FRAME,   NONE,  0},
  {ZEROMV,    LAST_FRAME,   NONE,  1},
  {DC_PRED,   INTRA_FRAME,  NONE,  0},

  {NEARESTMV, LAST_FRAME,   NONE,  0},
  {NEARESTMV, LAST_FRAME,   NONE,  1},
  {NEARMV,    LAST_FRAME,   NONE,  0},
  {NEARMV,    LAST_FRAME,   NONE,  1},

  {ZEROMV,    GOLDEN_FRAME, NONE,  0},
  {ZEROMV,    GOLDEN_FRAME, NONE,  1},
  {NEARESTMV, GOLDEN_FRAME, NONE,  0},
  {NEARESTMV, GOLDEN_FRAME, NONE,  1},

  {ZEROMV,    ALTREF_FRAME, NONE,  0},
  {ZEROMV,    ALTREF_FRAME, NONE,  1},
  {NEARESTMV, ALTREF_FRAME, NONE,  0},
  {NEARESTMV, ALTREF_FRAME, NONE,  1},

  {NEARMV,    GOLDEN_FRAME, NONE,  0},
  {NEARMV,    GOLDEN_FRAME, NONE,  1},
  {NEARMV,    ALTREF_FRAME, NONE,  0},
  {NEARMV,    ALTREF_FRAME, NONE,  1},

  {V_PRED,    INTRA_FRAME,  NONE,  0},
  {H_PRED,    INTRA_FRAME,  NONE,  0},
  {D45_PRED,  INTRA_FRAME,  NONE,  0},
  {D135_PRED, INTRA_FRAME,  NONE,  0},
  {D117_PRED, INTRA_FRAME,  NONE,  0},
  {D153_PRED, INTRA_FRAME,  NONE,  0},
  {D27_PRED,  INTRA_FRAME,  NONE,  0},
  {D63_PRED,  INTRA_FRAME,  NONE,  0},

  {TM_PRED,   INTRA_FRAME,  NONE,  0},

  {NEWMV,     LAST_FRAME,   NONE,  0},
  {NEWMV,     LAST_FRAME,   NONE,  1},
  {NEWMV,     GOLDEN_FRAME, NONE,  0},
  {NEWMV,     GOLDEN_FRAME, NONE,  1},
  {NEWMV,     ALTREF_FRAME, NONE,  0},
  {NEWMV,     ALTREF_FRAME, NONE,  1},

  {SPLITMV,   LAST_FRAME,   NONE,  0},
  {SPLITMV,   GOLDEN_FRAME, NONE,  0},
  {SPLITMV,   ALTREF_FRAME, NONE,  0},

  {B_PRED,    INTRA_FRAME,  NONE,  0},
  {I8X8_PRED, INTRA_FRAME,  NONE,  0},
John Koleszar's avatar
John Koleszar committed
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147

  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME, 0},

  {ZEROMV,    ALTREF_FRAME, LAST_FRAME,   0},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME,   0},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME,   0},

  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME, 0},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME, 0},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME, 0},

  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME,   0},
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME, 0},

  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME, 0},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME,   0},
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME, 0},

#if CONFIG_COMP_INTERINTRA_PRED
  /* compound inter-intra prediction */
  {ZEROMV,    LAST_FRAME,   INTRA_FRAME, 0},
  {NEARESTMV, LAST_FRAME,   INTRA_FRAME, 0},
  {NEARMV,    LAST_FRAME,   INTRA_FRAME, 0},
  {NEWMV,     LAST_FRAME,   INTRA_FRAME, 0},

  {ZEROMV,    GOLDEN_FRAME,   INTRA_FRAME, 0},
  {NEARESTMV, GOLDEN_FRAME,   INTRA_FRAME, 0},
  {NEARMV,    GOLDEN_FRAME,   INTRA_FRAME, 0},
  {NEWMV,     GOLDEN_FRAME,   INTRA_FRAME, 0},

  {ZEROMV,    ALTREF_FRAME,   INTRA_FRAME, 0},
  {NEARESTMV, ALTREF_FRAME,   INTRA_FRAME, 0},
  {NEARMV,    ALTREF_FRAME,   INTRA_FRAME, 0},
  {NEWMV,     ALTREF_FRAME,   INTRA_FRAME, 0},
#endif
John Koleszar's avatar
John Koleszar committed
167
};
168
#else
169
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
170
171
  {ZEROMV,    LAST_FRAME,   NONE},
  {DC_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
172

173
174
  {NEARESTMV, LAST_FRAME,   NONE},
  {NEARMV,    LAST_FRAME,   NONE},
John Koleszar's avatar
John Koleszar committed
175

176
177
  {ZEROMV,    GOLDEN_FRAME, NONE},
  {NEARESTMV, GOLDEN_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
178

179
180
  {ZEROMV,    ALTREF_FRAME, NONE},
  {NEARESTMV, ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
181

182
183
  {NEARMV,    GOLDEN_FRAME, NONE},
  {NEARMV,    ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
184

185
186
187
188
189
190
191
192
  {V_PRED,    INTRA_FRAME,  NONE},
  {H_PRED,    INTRA_FRAME,  NONE},
  {D45_PRED,  INTRA_FRAME,  NONE},
  {D135_PRED, INTRA_FRAME,  NONE},
  {D117_PRED, INTRA_FRAME,  NONE},
  {D153_PRED, INTRA_FRAME,  NONE},
  {D27_PRED,  INTRA_FRAME,  NONE},
  {D63_PRED,  INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
193

194
  {TM_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
195

196
197
198
  {NEWMV,     LAST_FRAME,   NONE},
  {NEWMV,     GOLDEN_FRAME, NONE},
  {NEWMV,     ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
199

200
201
202
  {SPLITMV,   LAST_FRAME,   NONE},
  {SPLITMV,   GOLDEN_FRAME, NONE},
  {SPLITMV,   ALTREF_FRAME, NONE},
203

204
205
  {B_PRED,    INTRA_FRAME,  NONE},
  {I8X8_PRED, INTRA_FRAME,  NONE},
206

John Koleszar's avatar
John Koleszar committed
207
208
209
210
  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME},
211

John Koleszar's avatar
John Koleszar committed
212
213
214
  {ZEROMV,    ALTREF_FRAME, LAST_FRAME},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME},
215

John Koleszar's avatar
John Koleszar committed
216
217
218
  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME},
219

John Koleszar's avatar
John Koleszar committed
220
221
222
  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME  },
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME},
223

John Koleszar's avatar
John Koleszar committed
224
225
  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME  },
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME},

#if CONFIG_COMP_INTERINTRA_PRED
  /* compound inter-intra prediction */
  {ZEROMV,    LAST_FRAME,   INTRA_FRAME},
  {NEARESTMV, LAST_FRAME,   INTRA_FRAME},
  {NEARMV,    LAST_FRAME,   INTRA_FRAME},
  {NEWMV,     LAST_FRAME,   INTRA_FRAME},

  {ZEROMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEARESTMV, GOLDEN_FRAME,   INTRA_FRAME},
  {NEARMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEWMV,     GOLDEN_FRAME,   INTRA_FRAME},

  {ZEROMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEARESTMV, ALTREF_FRAME,   INTRA_FRAME},
  {NEARMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEWMV,     ALTREF_FRAME,   INTRA_FRAME},
#endif
John Koleszar's avatar
John Koleszar committed
245
};
246
#endif
John Koleszar's avatar
John Koleszar committed
247
248

static void fill_token_costs(
249
  unsigned int (*c)[COEF_BANDS][PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS],
250
  const vp9_prob(*p)[COEF_BANDS][PREV_COEF_CONTEXTS][ENTROPY_NODES],
John Koleszar's avatar
John Koleszar committed
251
252
253
254
255
256
257
  int block_type_counts) {
  int i, j, k;

  for (i = 0; i < block_type_counts; i++)
    for (j = 0; j < COEF_BANDS; j++)
      for (k = 0; k < PREV_COEF_CONTEXTS; k++) {
        if (k == 0 && ((j > 0 && i > 0) || (j > 1 && i == 0)))
258
259
          vp9_cost_tokens_skip((int *)(c[i][j][k]),
                               p[i][j][k],
260
                               vp9_coef_tree);
John Koleszar's avatar
John Koleszar committed
261
        else
262
263
          vp9_cost_tokens((int *)(c[i][j][k]),
                          p[i][j][k],
264
                          vp9_coef_tree);
John Koleszar's avatar
John Koleszar committed
265
      }
John Koleszar's avatar
John Koleszar committed
266
267
}

268

269
270
271
272
static int rd_iifactor[32] =  { 4, 4, 3, 2, 1, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0, };
John Koleszar's avatar
John Koleszar committed
273

274
// 3* dc_qlookup[Q]*dc_qlookup[Q];
275

276
/* values are now correlated to quantizer */
Paul Wilkins's avatar
Paul Wilkins committed
277
278
279
static int sad_per_bit16lut[QINDEX_RANGE];
static int sad_per_bit4lut[QINDEX_RANGE];

280
void vp9_init_me_luts() {
John Koleszar's avatar
John Koleszar committed
281
282
283
284
285
286
287
  int i;

  // Initialize the sad lut tables using a formulaic calculation for now
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < QINDEX_RANGE; i++) {
    sad_per_bit16lut[i] =
288
289
      (int)((0.0418 * vp9_convert_qindex_to_q(i)) + 2.4107);
    sad_per_bit4lut[i] = (int)((0.063 * vp9_convert_qindex_to_q(i)) + 2.742);
John Koleszar's avatar
John Koleszar committed
290
  }
Paul Wilkins's avatar
Paul Wilkins committed
291
}
John Koleszar's avatar
John Koleszar committed
292

293
static int compute_rd_mult(int qindex) {
John Koleszar's avatar
John Koleszar committed
294
  int q;
295

296
  q = vp9_dc_quant(qindex, 0);
John Koleszar's avatar
John Koleszar committed
297
  return (11 * q * q) >> 6;
298
299
}

300
void vp9_initialize_me_consts(VP9_COMP *cpi, int QIndex) {
John Koleszar's avatar
John Koleszar committed
301
302
  cpi->mb.sadperbit16 =  sad_per_bit16lut[QIndex];
  cpi->mb.sadperbit4  =  sad_per_bit4lut[QIndex];
John Koleszar's avatar
John Koleszar committed
303
304
}

305

306
void vp9_initialize_rd_consts(VP9_COMP *cpi, int QIndex) {
307
  int q, i;
John Koleszar's avatar
John Koleszar committed
308

309
  vp9_clear_system_state();  // __asm emms;
John Koleszar's avatar
John Koleszar committed
310

John Koleszar's avatar
John Koleszar committed
311
312
313
314
315
  // Further tests required to see if optimum is different
  // for key frames, golden frames and arf frames.
  // if (cpi->common.refresh_golden_frame ||
  //     cpi->common.refresh_alt_ref_frame)
  QIndex = (QIndex < 0) ? 0 : ((QIndex > MAXQ) ? MAXQ : QIndex);
316

John Koleszar's avatar
John Koleszar committed
317
  cpi->RDMULT = compute_rd_mult(QIndex);
John Koleszar's avatar
John Koleszar committed
318

John Koleszar's avatar
John Koleszar committed
319
320
321
  // Extend rate multiplier along side quantizer zbin increases
  if (cpi->zbin_over_quant  > 0) {
    double oq_factor;
322

John Koleszar's avatar
John Koleszar committed
323
324
325
326
327
    // Experimental code using the same basic equation as used for Q above
    // The units of cpi->zbin_over_quant are 1/128 of Q bin size
    oq_factor = 1.0 + ((double)0.0015625 * cpi->zbin_over_quant);
    cpi->RDMULT = (int)((double)cpi->RDMULT * oq_factor * oq_factor);
  }
John Koleszar's avatar
John Koleszar committed
328

John Koleszar's avatar
John Koleszar committed
329
330
331
332
333
334
335
  if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    if (cpi->twopass.next_iiratio > 31)
      cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
    else
      cpi->RDMULT +=
        (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
  }
John Koleszar's avatar
John Koleszar committed
336

John Koleszar's avatar
John Koleszar committed
337
338
  if (cpi->RDMULT < 7)
    cpi->RDMULT = 7;
339

John Koleszar's avatar
John Koleszar committed
340
341
  cpi->mb.errorperbit = (cpi->RDMULT / 110);
  cpi->mb.errorperbit += (cpi->mb.errorperbit == 0);
342

343
  vp9_set_speed_features(cpi);
John Koleszar's avatar
John Koleszar committed
344

345
  q = (int)pow(vp9_dc_quant(QIndex, 0) >> 2, 1.25);
John Koleszar's avatar
John Koleszar committed
346
347
  q = q << 2;
  cpi->RDMULT = cpi->RDMULT << 4;
Paul Wilkins's avatar
Paul Wilkins committed
348

John Koleszar's avatar
John Koleszar committed
349
350
  if (q < 8)
    q = 8;
351

John Koleszar's avatar
John Koleszar committed
352
353
354
  if (cpi->RDMULT > 1000) {
    cpi->RDDIV = 1;
    cpi->RDMULT /= 100;
John Koleszar's avatar
John Koleszar committed
355

John Koleszar's avatar
John Koleszar committed
356
357
358
359
360
361
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < INT_MAX) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q / 100;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
362

John Koleszar's avatar
John Koleszar committed
363
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
364
    }
John Koleszar's avatar
John Koleszar committed
365
366
  } else {
    cpi->RDDIV = 100;
John Koleszar's avatar
John Koleszar committed
367

John Koleszar's avatar
John Koleszar committed
368
369
370
371
372
373
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < (INT_MAX / q)) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
374

John Koleszar's avatar
John Koleszar committed
375
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
376
    }
John Koleszar's avatar
John Koleszar committed
377
  }
John Koleszar's avatar
John Koleszar committed
378

John Koleszar's avatar
John Koleszar committed
379
  fill_token_costs(
380
    cpi->mb.token_costs[TX_4X4],
381
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs,
John Koleszar's avatar
John Koleszar committed
382
    BLOCK_TYPES);
383
384
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_4X4],
385
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11])
386
387
    cpi->common.fc.hybrid_coef_probs,
    BLOCK_TYPES);
John Koleszar's avatar
John Koleszar committed
388

John Koleszar's avatar
John Koleszar committed
389
  fill_token_costs(
390
    cpi->mb.token_costs[TX_8X8],
391
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs_8x8,
John Koleszar's avatar
John Koleszar committed
392
    BLOCK_TYPES_8X8);
393
394
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_8X8],
395
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11])
396
397
    cpi->common.fc.hybrid_coef_probs_8x8,
    BLOCK_TYPES_8X8);
398

Daniel Kang's avatar
Daniel Kang committed
399
  fill_token_costs(
400
    cpi->mb.token_costs[TX_16X16],
401
    (const vp9_prob(*)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs_16x16,
Daniel Kang's avatar
Daniel Kang committed
402
    BLOCK_TYPES_16X16);
403
404
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_16X16],
405
    (const vp9_prob(*)[8][PREV_COEF_CONTEXTS][11])
406
407
    cpi->common.fc.hybrid_coef_probs_16x16,
    BLOCK_TYPES_16X16);
Daniel Kang's avatar
Daniel Kang committed
408

John Koleszar's avatar
John Koleszar committed
409
410
  /*rough estimate for costing*/
  cpi->common.kf_ymode_probs_index = cpi->common.base_qindex >> 4;
411
  vp9_init_mode_costs(cpi);
John Koleszar's avatar
John Koleszar committed
412

413
414
  if (cpi->common.frame_type != KEY_FRAME)
  {
415
    vp9_build_nmv_cost_table(
416
417
418
419
420
421
        cpi->mb.nmvjointcost,
        cpi->mb.e_mbd.allow_high_precision_mv ?
        cpi->mb.nmvcost_hp : cpi->mb.nmvcost,
        &cpi->common.fc.nmvc,
        cpi->mb.e_mbd.allow_high_precision_mv, 1, 1);
  }
John Koleszar's avatar
John Koleszar committed
422
423
}

424
int vp9_block_error_c(short *coeff, short *dqcoeff, int block_size) {
425
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
426

427
  for (i = 0; i < block_size; i++) {
John Koleszar's avatar
John Koleszar committed
428
429
430
    int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }
John Koleszar's avatar
John Koleszar committed
431

John Koleszar's avatar
John Koleszar committed
432
  return error;
John Koleszar's avatar
John Koleszar committed
433
434
}

435
int vp9_mbblock_error_c(MACROBLOCK *mb, int dc) {
John Koleszar's avatar
John Koleszar committed
436
437
438
439
  BLOCK  *be;
  BLOCKD *bd;
  int i, j;
  int berror, error = 0;
John Koleszar's avatar
John Koleszar committed
440

John Koleszar's avatar
John Koleszar committed
441
442
443
  for (i = 0; i < 16; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
John Koleszar's avatar
John Koleszar committed
444

John Koleszar's avatar
John Koleszar committed
445
    berror = 0;
John Koleszar's avatar
John Koleszar committed
446

John Koleszar's avatar
John Koleszar committed
447
448
449
    for (j = dc; j < 16; j++) {
      int this_diff = be->coeff[j] - bd->dqcoeff[j];
      berror += this_diff * this_diff;
John Koleszar's avatar
John Koleszar committed
450
451
    }

John Koleszar's avatar
John Koleszar committed
452
453
454
455
    error += berror;
  }

  return error;
John Koleszar's avatar
John Koleszar committed
456
457
}

458
int vp9_mbuverror_c(MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
459
460
  BLOCK  *be;
  BLOCKD *bd;
John Koleszar's avatar
John Koleszar committed
461

462
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
463

John Koleszar's avatar
John Koleszar committed
464
465
466
  for (i = 16; i < 24; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
John Koleszar's avatar
John Koleszar committed
467

468
    error += vp9_block_error_c(be->coeff, bd->dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
469
  }
John Koleszar's avatar
John Koleszar committed
470

John Koleszar's avatar
John Koleszar committed
471
  return error;
John Koleszar's avatar
John Koleszar committed
472
473
}

474
int vp9_uvsse(MACROBLOCK *x) {
John Koleszar's avatar
John Koleszar committed
475
476
477
478
479
480
481
  unsigned char *uptr, *vptr;
  unsigned char *upred_ptr = (*(x->block[16].base_src) + x->block[16].src);
  unsigned char *vpred_ptr = (*(x->block[20].base_src) + x->block[20].src);
  int uv_stride = x->block[16].src_stride;

  unsigned int sse1 = 0;
  unsigned int sse2 = 0;
482
483
  int mv_row = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.row;
  int mv_col = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.col;
John Koleszar's avatar
John Koleszar committed
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
  int offset;
  int pre_stride = x->e_mbd.block[16].pre_stride;

  if (mv_row < 0)
    mv_row -= 1;
  else
    mv_row += 1;

  if (mv_col < 0)
    mv_col -= 1;
  else
    mv_col += 1;

  mv_row /= 2;
  mv_col /= 2;

  offset = (mv_row >> 3) * pre_stride + (mv_col >> 3);
  uptr = x->e_mbd.pre.u_buffer + offset;
  vptr = x->e_mbd.pre.v_buffer + offset;

  if ((mv_row | mv_col) & 7) {
505
    vp9_sub_pixel_variance8x8(uptr, pre_stride, (mv_col & 7) << 1,
506
                              (mv_row & 7) << 1, upred_ptr, uv_stride, &sse2);
507
    vp9_sub_pixel_variance8x8(vptr, pre_stride, (mv_col & 7) << 1,
508
                              (mv_row & 7) << 1, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
509
510
    sse2 += sse1;
  } else {
511
512
    vp9_variance8x8(uptr, pre_stride, upred_ptr, uv_stride, &sse2);
    vp9_variance8x8(vptr, pre_stride, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
513
514
515
    sse2 += sse1;
  }
  return sse2;
John Koleszar's avatar
John Koleszar committed
516
517
518

}

519
static int cost_coeffs_2x2(MACROBLOCK *mb,
520
                           BLOCKD *b, PLANE_TYPE type,
521
                           ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l) {
522
  int c = (type == PLANE_TYPE_Y_NO_DC); /* start at coef 0, unless Y with Y2 */
523
524
525
526
527
  int eob = b->eob;
  int pt;    /* surrounding block/prev coef predictor */
  int cost = 0;
  short *qcoeff_ptr = b->qcoeff;

528
  VP9_COMBINEENTROPYCONTEXTS(pt, *a, *l);
529
530
531
  assert(eob <= 4);

  for (; c < eob; c++) {
532
533
534
535
536
    int v = qcoeff_ptr[vp9_default_zig_zag1d[c]];
    int t = vp9_dct_value_tokens_ptr[v].Token;
    cost += mb->token_costs[TX_8X8][type][vp9_coef_bands[c]][pt][t];
    cost += vp9_dct_value_cost_ptr[v];
    pt = vp9_prev_token_class[t];
537
538
539
  }

  if (c < 4)
540
    cost += mb->token_costs[TX_8X8][type][vp9_coef_bands[c]]
541
542
543
544
545
546
547
            [pt] [DCT_EOB_TOKEN];

  pt = (c != !type); // is eob first coefficient;
  *a = *l = pt;
  return cost;
}

548
static int cost_coeffs(MACROBLOCK *mb, BLOCKD *b, PLANE_TYPE type,
Daniel Kang's avatar
Daniel Kang committed
549
                       ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l,
550
                       int tx_size) {
Daniel Kang's avatar
Daniel Kang committed
551
  const int eob = b->eob;
552
  int c = (type == PLANE_TYPE_Y_NO_DC); /* start at coef 0, unless Y with Y2 */
553
  int cost = 0, default_eob, seg_eob;
Daniel Kang's avatar
Daniel Kang committed
554
555
  int pt;                     /* surrounding block/prev coef predictor */
  int const *scan, *band;
John Koleszar's avatar
John Koleszar committed
556
  short *qcoeff_ptr = b->qcoeff;
557
558
559
  MACROBLOCKD *xd = &mb->e_mbd;
  MB_MODE_INFO *mbmi = &mb->e_mbd.mode_info_context->mbmi;
  TX_TYPE tx_type = DCT_DCT;
Paul Wilkins's avatar
Paul Wilkins committed
560
  int segment_id = mbmi->segment_id;
561

562
  switch (tx_size) {
Daniel Kang's avatar
Daniel Kang committed
563
    case TX_4X4:
564
565
      scan = vp9_default_zig_zag1d;
      band = vp9_coef_bands;
Daniel Kang's avatar
Daniel Kang committed
566
      default_eob = 16;
Deb Mukherjee's avatar
Deb Mukherjee committed
567
568
569
570
571
      if (type == PLANE_TYPE_Y_WITH_DC) {
        tx_type = get_tx_type_4x4(xd, b);
        if (tx_type != DCT_DCT) {
          switch (tx_type) {
            case ADST_DCT:
572
              scan = vp9_row_scan;
Deb Mukherjee's avatar
Deb Mukherjee committed
573
574
575
              break;

            case DCT_ADST:
576
              scan = vp9_col_scan;
Deb Mukherjee's avatar
Deb Mukherjee committed
577
578
579
              break;

            default:
580
              scan = vp9_default_zig_zag1d;
Deb Mukherjee's avatar
Deb Mukherjee committed
581
582
              break;
          }
583
        }
Daniel Kang's avatar
Daniel Kang committed
584
      }
Deb Mukherjee's avatar
Deb Mukherjee committed
585

Daniel Kang's avatar
Daniel Kang committed
586
587
      break;
    case TX_8X8:
588
589
      scan = vp9_default_zig_zag1d_8x8;
      band = vp9_coef_bands_8x8;
Daniel Kang's avatar
Daniel Kang committed
590
      default_eob = 64;
Deb Mukherjee's avatar
Deb Mukherjee committed
591
      if (type == PLANE_TYPE_Y_WITH_DC) {
592
        BLOCKD *bb;
593
        int ib = (int)(b - xd->block);
594
595
596
        if (ib < 16) {
          ib = (ib & 8) + ((ib & 4) >> 1);
          bb = xd->block + ib;
Deb Mukherjee's avatar
Deb Mukherjee committed
597
          tx_type = get_tx_type_8x8(xd, bb);
598
        }
599
      }
Daniel Kang's avatar
Daniel Kang committed
600
601
      break;
    case TX_16X16:
602
603
      scan = vp9_default_zig_zag1d_16x16;
      band = vp9_coef_bands_16x16;
Daniel Kang's avatar
Daniel Kang committed
604
      default_eob = 256;
Deb Mukherjee's avatar
Deb Mukherjee committed
605
606
607
      if (type == PLANE_TYPE_Y_WITH_DC) {
        tx_type = get_tx_type_16x16(xd, b);
      }
Daniel Kang's avatar
Daniel Kang committed
608
609
610
611
      break;
    default:
      break;
  }
612
613
  if (vp9_segfeature_active(&mb->e_mbd, segment_id, SEG_LVL_EOB))
    seg_eob = vp9_get_segdata(&mb->e_mbd, segment_id, SEG_LVL_EOB);
614
615
616
  else
    seg_eob = default_eob;

617
  VP9_COMBINEENTROPYCONTEXTS(pt, *a, *l);
618

619
620
621
  if (tx_type != DCT_DCT) {
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
622
      int t = vp9_dct_value_tokens_ptr[v].Token;
623
      cost += mb->hybrid_token_costs[tx_size][type][band[c]][pt][t];
624
625
      cost += vp9_dct_value_cost_ptr[v];
      pt = vp9_prev_token_class[t];
626
627
628
629
    }
    if (c < seg_eob)
      cost += mb->hybrid_token_costs[tx_size][type][band[c]]
          [pt][DCT_EOB_TOKEN];
630
  } else {
631
632
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
633
      int t = vp9_dct_value_tokens_ptr[v].Token;
634
      cost += mb->token_costs[tx_size][type][band[c]][pt][t];
635
636
      cost += vp9_dct_value_cost_ptr[v];
      pt = vp9_prev_token_class[t];
637
638
639
640
    }
    if (c < seg_eob)
      cost += mb->token_costs[tx_size][type][band[c]]
          [pt][DCT_EOB_TOKEN];
641
642
643
644
645
646
647
  }

  pt = (c != !type); // is eob first coefficient;
  *a = *l = pt;
  return cost;
}

648
static int rdcost_mby_4x4(MACROBLOCK *mb, int backup) {
John Koleszar's avatar
John Koleszar committed
649
650
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
651
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
652
653
654
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta;
  ENTROPY_CONTEXT *tl;
655

656
657
658
  if (backup) {
    vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
659

660
661
662
663
664
665
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)xd->above_context;
    tl = (ENTROPY_CONTEXT *)xd->left_context;
  }
John Koleszar's avatar
John Koleszar committed
666

John Koleszar's avatar
John Koleszar committed
667
  for (b = 0; b < 16; b++)
Paul Wilkins's avatar
Paul Wilkins committed
668
    cost += cost_coeffs(mb, xd->block + b, PLANE_TYPE_Y_NO_DC,
669
                        ta + vp9_block2above[b], tl + vp9_block2left[b],
Daniel Kang's avatar
Daniel Kang committed
670
                        TX_4X4);
John Koleszar's avatar
John Koleszar committed
671

Paul Wilkins's avatar
Paul Wilkins committed
672
  cost += cost_coeffs(mb, xd->block + 24, PLANE_TYPE_Y2,
673
                      ta + vp9_block2above[24], tl + vp9_block2left[24],
Daniel Kang's avatar
Daniel Kang committed
674
                      TX_4X4);
John Koleszar's avatar
John Koleszar committed
675

John Koleszar's avatar
John Koleszar committed
676
  return cost;
John Koleszar's avatar
John Koleszar committed
677
678
}

679
680
681
static void macro_block_yrd_4x4(MACROBLOCK *mb,
                                int *Rate,
                                int *Distortion,
682
                                int *skippable, int backup) {
John Koleszar's avatar
John Koleszar committed
683
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
684
  MACROBLOCKD *const xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
685
  BLOCK   *const mb_y2 = mb->block + 24;
Paul Wilkins's avatar
Paul Wilkins committed
686
  BLOCKD *const x_y2  = xd->block + 24;
John Koleszar's avatar
John Koleszar committed
687
688
689
  short *Y2DCPtr = mb_y2->src_diff;
  BLOCK *beptr;
  int d;
690

John Koleszar's avatar
John Koleszar committed
691
692
  // Fdct and building the 2nd order block
  for (beptr = mb->block; beptr < mb->block + 16; beptr += 2) {
693
    mb->vp9_short_fdct8x4(beptr->src_diff, beptr->coeff, 32);
John Koleszar's avatar
John Koleszar committed
694
695
696
697
698
699
700
701
702
    *Y2DCPtr++ = beptr->coeff[0];
    *Y2DCPtr++ = beptr->coeff[16];
  }

  // 2nd order fdct
  mb->short_walsh4x4(mb_y2->src_diff, mb_y2->coeff, 8);

  // Quantization
  for (b = 0; b < 16; b++) {
703
    mb->quantize_b_4x4(&mb->block[b], &xd->block[b]);
John Koleszar's avatar
John Koleszar committed
704
  }
705

John Koleszar's avatar
John Koleszar committed
706
  // DC predication and Quantization of 2nd Order block
707
  mb->quantize_b_4x4(mb_y2, x_y2);
708

John Koleszar's avatar
John Koleszar committed
709
  // Distortion
710
  d = vp9_mbblock_error(mb, 1);
711

712
  d += vp9_block_error(mb_y2->coeff, x_y2->dqcoeff, 16);
713

John Koleszar's avatar
John Koleszar committed
714
715
  *Distortion = (d >> 2);
  // rate
716
  *Rate = rdcost_mby_4x4(mb, backup);
717
  *skippable = vp9_mby_is_skippable_4x4(&mb->e_mbd, 1);
718
}
John Koleszar's avatar
John Koleszar committed
719

720
static int rdcost_mby_8x8(MACROBLOCK *mb, int backup) {
John Koleszar's avatar
John Koleszar committed
721
722
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
723
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
724
725
726
727
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta;
  ENTROPY_CONTEXT *tl;

Ronald S. Bultje's avatar
Ronald S. Bultje committed
728
729
730
  if (backup) {
    vpx_memcpy(&t_above,xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
731

Ronald S. Bultje's avatar
Ronald S. Bultje committed
732
733
734
735
736
737
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)mb->e_mbd.above_context;
    tl = (ENTROPY_CONTEXT *)mb->e_mbd.left_context;
  }
John Koleszar's avatar
John Koleszar committed
738
739

  for (b = 0; b < 16; b += 4)
Paul Wilkins's avatar
Paul Wilkins committed
740
    cost += cost_coeffs(mb, xd->block + b, PLANE_TYPE_Y_NO_DC,
741
                        ta + vp9_block2above_8x8[b], tl + vp9_block2left_8x8[b],
Daniel Kang's avatar
Daniel Kang committed
742
                        TX_8X8);
John Koleszar's avatar
John Koleszar committed
743

Paul Wilkins's avatar
Paul Wilkins committed
744
  cost += cost_coeffs_2x2(mb, xd->block + 24, PLANE_TYPE_Y2,
745
                          ta + vp9_block2above[24], tl + vp9_block2left[24]);
John Koleszar's avatar
John Koleszar committed
746
  return cost;
747
748
}

John Koleszar's avatar
John Koleszar committed
749
750
751
static void macro_block_yrd_8x8(MACROBLOCK *mb,
                                int *Rate,
                                int *Distortion,
752
                                int *skippable, int backup) {
Paul Wilkins's avatar
Paul Wilkins committed
753
  MACROBLOCKD *const xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
754
  BLOCK   *const mb_y2 = mb->block + 24;
Paul Wilkins's avatar
Paul Wilkins committed
755
  BLOCKD *const x_y2  = xd->block + 24;
John Koleszar's avatar
John Koleszar committed
756
757
  int d;

758
759
  vp9_transform_mby_8x8(mb);
  vp9_quantize_mby_8x8(mb);
John Koleszar's avatar
John Koleszar committed
760
761
762
763
764
765

  /* remove 1st order dc to properly combine 1st/2nd order distortion */
  mb->coeff[0] = 0;
  mb->coeff[64] = 0;
  mb->coeff[128] = 0;
  mb->coeff[192] = 0;
Paul Wilkins's avatar
Paul Wilkins committed
766
767
768
769
  xd->dqcoeff[0] = 0;
  xd->dqcoeff[64] = 0;
  xd->dqcoeff[128] = 0;
  xd->dqcoeff[192] = 0;
John Koleszar's avatar
John Koleszar committed
770

771
772
  d = vp9_mbblock_error(mb, 0);
  d += vp9_block_error(mb_y2->coeff, x_y2->dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
773
774
775

  *Distortion = (d >> 2);
  // rate
776
  *Rate = rdcost_mby_8x8(mb, backup);
777
  *skippable = vp9_mby_is_skippable_8x8(&mb->e_mbd, 1);
778
}
779

780
static int rdcost_mby_16x16(MACROBLOCK *mb, int backup) {
Daniel Kang's avatar
Daniel Kang committed
781
  int cost;
Paul Wilkins's avatar
Paul Wilkins committed
782
  MACROBLOCKD *xd = &mb->e_mbd;
Daniel Kang's avatar
Daniel Kang committed
783
784
785
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta, *tl;

786
787
788
  if (backup) {
    vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
Daniel Kang's avatar
Daniel Kang committed
789

790
791
792
793
794
795
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)xd->above_context;
    tl = (ENTROPY_CONTEXT *)xd->left_context;
  }
Daniel Kang's avatar
Daniel Kang committed
796

Paul Wilkins's avatar
Paul Wilkins committed
797
  cost = cost_coeffs(mb, xd->block, PLANE_TYPE_Y_WITH_DC, ta, tl, TX_16X16);
Daniel Kang's avatar
Daniel Kang committed
798
799
  return cost;
}
800

Daniel Kang's avatar
Daniel Kang committed
801
static void macro_block_yrd_16x16(MACROBLOCK *mb, int *Rate, int *Distortion,
802
                                  int *skippable, int backup) {
Daniel Kang's avatar
Daniel Kang committed
803
  int d;
Deb Mukherjee's avatar
Deb Mukherjee committed
804
805
806
807
  MACROBLOCKD *xd = &mb->e_mbd;
  BLOCKD *b  = &mb->e_mbd.block[0];
  BLOCK  *be = &mb->block[0];
  TX_TYPE tx_type;
Daniel Kang's avatar
Daniel Kang committed
808

Deb Mukherjee's avatar
Deb Mukherjee committed
809
810
  tx_type = get_tx_type_16x16(xd, b);
  if (tx_type != DCT_DCT) {
811
    vp9_fht(be->src_diff, 32, be->coeff, tx_type, 16);
812
  } else
813
    vp9_transform_mby_16x16(mb);
814

815
  vp9_quantize_mby_16x16(mb);
816
817
818
  // TODO(jingning) is it possible to quickly determine whether to force
  //                trailing coefficients to be zero, instead of running trellis
  //                optimization in the rate-distortion optimization loop?
819
  if (mb->e_mbd.mode_info_context->mbmi.mode < I8X8_PRED)
820
    vp9_optimize_mby_16x16(mb);
821

822
  d = vp9_mbblock_error(mb, 0);
Daniel Kang's avatar
Daniel Kang committed
823
824
825

  *Distortion = (d >> 2);
  // rate
826
  *Rate = rdcost_mby_16x16(mb, backup);
827
  *skippable = vp9_mby_is_skippable_16x16(&mb->e_mbd);
Daniel Kang's avatar
Daniel Kang committed
828
829
}

830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
static void choose_txfm_size_from_rd(VP9_COMP *cpi, MACROBLOCK *x,
                                     int r[2][TX_SIZE_MAX], int *rate,
                                     int d[TX_SIZE_MAX], int *distortion,
                                     int s[TX_SIZE_MAX], int *skip,
                                     int64_t txfm_cache[NB_TXFM_MODES]) {
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
  vp9_prob skip_prob = cm->mb_no_coeff_skip ?
                       vp9_get_pred_prob(cm, xd, PRED_MBSKIP) : 128;
  int64_t rd[2][TX_SIZE_MAX];
  int n;

  r[1][TX_16X16] = r[0][TX_16X16] + vp9_cost_one(cm->prob_tx[0]) +
                   vp9_cost_one(cm->prob_tx[1]);
  r[1][TX_8X8]   = r[0][TX_8X8] + vp9_cost_one(cm->prob_tx[0]) +
                   vp9_cost_zero(cm->prob_tx[1]);
  r[1][TX_4X4]   = r[0][TX_4X4] + vp9_cost_zero(cm->prob_tx[0]);

  if (cm->mb_no_coeff_skip) {
    int s0, s1;

    assert(skip_prob > 0);
    s0 = vp9_cost_bit(skip_prob, 0);
    s1 = vp9_cost_bit(skip_prob, 1);

    for (n = TX_4X4; n <= TX_16X16; n++) {
      if (s[n]) {
        rd[0][n] = rd[1][n] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
      } else {
        rd[0][n] = RDCOST(x->rdmult, x->rddiv, r[0][n] + s0, d[n]);
        rd[1][n] = RDCOST(x->rdmult, x->rddiv, r[1][n] + s0, d[n]);
      }
863
864
    }
  } else {
865
866
867
    for (n = TX_4X4; n <= TX_16X16; n++) {
      rd[0][n] = RDCOST(x->rdmult, x->rddiv, r[0][n], d[n]);
      rd[1][n] = RDCOST(x->rdmult, x->rddiv, r[1][n], d[n]);
868
869
870
    }
  }

871
872
873
  if ( cm->txfm_mode == ALLOW_16X16 ||
      (cm->txfm_mode == TX_MODE_SELECT &&
       rd[1][TX_16X16] < rd[1][TX_8X8] && rd[1][TX_16X16] < rd[1][TX_4X4])) {
874
    mbmi->txfm_size = TX_16X16;
875
876
  } else if (cm->txfm_mode == ALLOW_8X8 ||
           (cm->txfm_mode == TX_MODE_SELECT && rd[1][TX_8X8] < rd[1][TX_4X4])) {
877
878
    mbmi->txfm_size = TX_8X8;
  } else {
879
880
    assert(cm->txfm_mode == ONLY_4X4 ||
          (cm->txfm_mode == TX_MODE_SELECT && rd[1][TX_4X4] <= rd[1][TX_8X8]));
881
882
883
    mbmi->txfm_size = TX_4X4;
  }

884
885
886
887
888
889
890
891
892
  *distortion = d[mbmi->txfm_size];
  *rate       = r[cm->txfm_mode == TX_MODE_SELECT][mbmi->txfm_size];
  *skip       = s[mbmi->txfm_size];

  txfm_cache[ONLY_4X4] = rd[0][TX_4X4];
  txfm_cache[ALLOW_8X8] = rd[0][TX_8X8];
  txfm_cache[ALLOW_16X16] = rd[0][TX_16X16];
  if (rd[1][TX_16X16] < rd[1][TX_8X8] && rd[1][TX_16X16] < rd[1][TX_4X4])
    txfm_cache[TX_MODE_SELECT] = rd[1][TX_16X16];
893
  else
894
895
896
897
898
899
900
901
902
903
904
905
    txfm_cache[TX_MODE_SELECT] = rd[1][TX_4X4] < rd[1][TX_8X8] ?
                                 rd[1][TX_4X4] : rd[1][TX_8X8];
}

static void macro_block_yrd(VP9_COMP *cpi, MACROBLOCK *x, int *rate,
                            int *distortion, int *skippable,
                            int64_t txfm_cache[NB_TXFM_MODES]) {
  MACROBLOCKD *const xd = &x->e_mbd;
  int r[2][TX_SIZE_MAX], d[TX_SIZE_MAX], s[TX_SIZE_MAX];

  vp9_subtract_mby(x->src_diff, *(x->block[0].base_src), xd->predictor,
                   x->block[0].src_stride);
906

907
  macro_block_yrd_16x16(x, &r[0][TX_16X16], &d[TX_16X16],
908
909
910
                        &s[TX_16X16], 1);
  macro_block_yrd_8x8(x, &r[0][TX_8X8], &d[TX_8X8], &s[TX_8X8], 1);
  macro_block_yrd_4x4(x, &r[0][TX_4X4], &d[TX_4X4], &s[TX_4X4], 1);
911
912
913

  choose_txfm_size_from_rd(cpi, x, r, rate, d, distortion, s, skippable,
                           txfm_cache);
914
915
}

John Koleszar's avatar
John Koleszar committed
916
917
918
919
920
921
922
static void copy_predictor(unsigned char *dst, const unsigned char *predictor) {
  const unsigned int *p = (const unsigned int *)predictor;
  unsigned int *d = (unsigned int *)dst;
  d[0] = p[0];
  d[4] = p[4];
  d[8] = p[8];
  d[12] = p[12];
923
}
Yaowu Xu's avatar
Yaowu Xu committed
924

Ronald S. Bultje's avatar
Ronald S. Bultje committed
925
#if CONFIG_SUPERBLOCKS
926
927
static void super_block_yrd(VP9_COMP *cpi,
                            MACROBLOCK *x, int *rate, int *distortion,
928
                            int *skip,
929
                            int64_t txfm_cache[NB_TXFM_MODES]) {
Ronald S. Bultje's avatar
Ronald S. Bultje committed
930
  MACROBLOCKD *const xd = &x->e_mbd;
931
  int r[2][TX_SIZE_MAX], d[TX_SIZE_MAX], s[TX_SIZE_MAX], n;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
932
933
  const uint8_t *src = x->src.y_buffer, *dst = xd->dst.y_buffer;
  int src_y_stride = x->src.y_stride, dst_y_stride = xd->dst.y_stride;
934
935
936
937
938
939
940
941
942
943
  ENTROPY_CONTEXT_PLANES t_above[3][2], *orig_above = xd->above_context;
  ENTROPY_CONTEXT_PLANES t_left[3][2], *orig_left = xd->left_context;

  for (n = TX_4X4; n <= TX_16X16; n++) {
    vpx_memcpy(t_above[n], xd->above_context, sizeof(t_above[n]));
    vpx_memcpy(t_left[n], xd->left_context, sizeof(t_left[n]));
    r[0][n] = 0;
    d[n] = 0;
    s[n] = 1;
  }
Ronald S. Bultje's avatar
Ronald S. Bultje committed
944
945
946

  for (n = 0; n < 4; n++) {
    int x_idx = n & 1, y_idx = n >> 1;
947
    int r_tmp, d_tmp, s_tmp;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
948

949
    vp9_subtract_mby_s_c(x->src_diff,
Ronald S. Bultje's avatar
Ronald S. Bultje committed
950
951
952
953
                         src + x_idx * 16 + y_idx * 16 * src_y_stride,
                         src_y_stride,
                         dst + x_idx * 16 + y_idx * 16 * dst_y_stride,
                         dst_y_stride);
954
955
956

    xd->above_context = &t_above[TX_16X16][x_idx];
    xd->left_context = &t_left[TX_16X16][y_idx];
957
    macro_block_yrd_16x16(x, &r_tmp, &d_tmp, &s_tmp, 0);
958
959
960
961
962
963
    d[TX_16X16] += d_tmp;
    r[0][TX_16X16] += r_tmp;
    s[TX_16X16] = s[TX_16X16] && s_tmp;

    xd->above_context = &t_above[TX_4X4][x_idx];
    xd->left_context = &t_left[TX_4X4][y_idx];
964
    macro_block_yrd_4x4(x, &r_tmp, &d_tmp, &s_tmp, 0);
965
966
967
968
969
970
    d[TX_4X4] += d_tmp;
    r[0][TX_4X4] += r_tmp;
    s[TX_4X4] = s[TX_4X4] && s_tmp;

    xd->above_context = &t_above[TX_8X8][x_idx];
    xd->left_context = &t_left[TX_8X8][y_idx];
971
    macro_block_yrd_8x8(x, &r_tmp, &d_tmp, &s_tmp, 0);
972
973
974
    d[TX_8X8] += d_tmp;
    r[0][TX_8X8] += r_tmp;
    s[TX_8X8] = s[TX_8X8] && s_tmp;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
975
976
  }

977
978
979
980
  choose_txfm_size_from_rd(cpi, x, r, rate, d, distortion, s, skip, txfm_cache);

  xd->above_context = orig_above;
  xd->left_context = orig_left;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
981
982
983
}
#endif

John Koleszar's avatar
John Koleszar committed
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
static void copy_predictor_8x8(unsigned char *dst, const unsigned char *predictor) {
  const unsigned int *p = (const unsigned int *)predictor;
  unsigned int *d = (unsigned int *)dst;
  d[0] = p[0];
  d[1] = p[1];
  d[4] = p[4];
  d[5] = p[5];
  d[8] = p[8];
  d[9] = p[9];
  d[12] = p[12];
  d[13] = p[13];
  d[16] = p[16];
  d[17] = p[17];
  d[20] = p[20];
  d[21] = p[21];
  d[24] = p[24];
  d[25] = p[25];
  d[28] = p[28];
  d[29] = p[29];
Yaowu Xu's avatar
Yaowu Xu committed
1003
1004
}

1005
static int64_t rd_pick_intra4x4block(VP9_COMP *cpi, MACROBLOCK *x, BLOCK *be,
1006
                                     BLOCKD *b, B_PREDICTION_MODE *best_mode,
1007
#if CONFIG_COMP_INTRA_PRED
1008
1009
1010
1011
1012
1013
1014
                                     B_PREDICTION_MODE *best_second_mode,
                                     int allow_comp,
#endif
                                     int *bmode_costs,