vp9_rdopt.c 166 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5
6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9
10
11
12
13
14
15
 */


#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
16
17
#include "vp9/common/vp9_pragmas.h"

18
19
20
21
22
#include "vp9/encoder/vp9_tokenize.h"
#include "vp9/encoder/vp9_treewriter.h"
#include "vp9/encoder/vp9_onyx_int.h"
#include "vp9/encoder/vp9_modecosts.h"
#include "vp9/encoder/vp9_encodeintra.h"
23
24
25
26
27
28
#include "vp9/common/vp9_entropymode.h"
#include "vp9/common/vp9_reconinter.h"
#include "vp9/common/vp9_reconintra.h"
#include "vp9/common/vp9_reconintra4x4.h"
#include "vp9/common/vp9_findnearmv.h"
#include "vp9/common/vp9_quant_common.h"
29
30
31
32
33
34
#include "vp9/encoder/vp9_encodemb.h"
#include "vp9/encoder/vp9_quantize.h"
#include "vp9/encoder/vp9_variance.h"
#include "vp9/encoder/vp9_mcomp.h"
#include "vp9/encoder/vp9_rdopt.h"
#include "vp9/encoder/vp9_ratectrl.h"
John Koleszar's avatar
John Koleszar committed
35
#include "vpx_mem/vpx_mem.h"
36
37
#include "vp9/common/vp9_systemdependent.h"
#include "vp9/encoder/vp9_encodemv.h"
John Koleszar's avatar
John Koleszar committed
38

39
40
41
#include "vp9/common/vp9_seg_common.h"
#include "vp9/common/vp9_pred_common.h"
#include "vp9/common/vp9_entropy.h"
42
#include "vp9_rtcd.h"
43
#include "vp9/common/vp9_mvref_common.h"
Paul Wilkins's avatar
Paul Wilkins committed
44

John Koleszar's avatar
John Koleszar committed
45
46
#define MAXF(a,b)            (((a) > (b)) ? (a) : (b))

47
48
#define INVALID_MV 0x80008000

49
50
51
/* Factor to weigh the rate for switchable interp filters */
#define SWITCHABLE_INTERP_RATE_FACTOR 1

John Koleszar's avatar
John Koleszar committed
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
static const int auto_speed_thresh[17] = {
  1000,
  200,
  150,
  130,
  150,
  125,
  120,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  105
John Koleszar's avatar
John Koleszar committed
70
71
};

72
#if CONFIG_PRED_FILTER
73
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
  {ZEROMV,    LAST_FRAME,   NONE,  0},
  {ZEROMV,    LAST_FRAME,   NONE,  1},
  {DC_PRED,   INTRA_FRAME,  NONE,  0},

  {NEARESTMV, LAST_FRAME,   NONE,  0},
  {NEARESTMV, LAST_FRAME,   NONE,  1},
  {NEARMV,    LAST_FRAME,   NONE,  0},
  {NEARMV,    LAST_FRAME,   NONE,  1},

  {ZEROMV,    GOLDEN_FRAME, NONE,  0},
  {ZEROMV,    GOLDEN_FRAME, NONE,  1},
  {NEARESTMV, GOLDEN_FRAME, NONE,  0},
  {NEARESTMV, GOLDEN_FRAME, NONE,  1},

  {ZEROMV,    ALTREF_FRAME, NONE,  0},
  {ZEROMV,    ALTREF_FRAME, NONE,  1},
  {NEARESTMV, ALTREF_FRAME, NONE,  0},
  {NEARESTMV, ALTREF_FRAME, NONE,  1},

  {NEARMV,    GOLDEN_FRAME, NONE,  0},
  {NEARMV,    GOLDEN_FRAME, NONE,  1},
  {NEARMV,    ALTREF_FRAME, NONE,  0},
  {NEARMV,    ALTREF_FRAME, NONE,  1},

  {V_PRED,    INTRA_FRAME,  NONE,  0},
  {H_PRED,    INTRA_FRAME,  NONE,  0},
  {D45_PRED,  INTRA_FRAME,  NONE,  0},
  {D135_PRED, INTRA_FRAME,  NONE,  0},
  {D117_PRED, INTRA_FRAME,  NONE,  0},
  {D153_PRED, INTRA_FRAME,  NONE,  0},
  {D27_PRED,  INTRA_FRAME,  NONE,  0},
  {D63_PRED,  INTRA_FRAME,  NONE,  0},

  {TM_PRED,   INTRA_FRAME,  NONE,  0},

  {NEWMV,     LAST_FRAME,   NONE,  0},
  {NEWMV,     LAST_FRAME,   NONE,  1},
  {NEWMV,     GOLDEN_FRAME, NONE,  0},
  {NEWMV,     GOLDEN_FRAME, NONE,  1},
  {NEWMV,     ALTREF_FRAME, NONE,  0},
  {NEWMV,     ALTREF_FRAME, NONE,  1},

  {SPLITMV,   LAST_FRAME,   NONE,  0},
  {SPLITMV,   GOLDEN_FRAME, NONE,  0},
  {SPLITMV,   ALTREF_FRAME, NONE,  0},

  {B_PRED,    INTRA_FRAME,  NONE,  0},
  {I8X8_PRED, INTRA_FRAME,  NONE,  0},
John Koleszar's avatar
John Koleszar committed
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141

  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME, 0},

  {ZEROMV,    ALTREF_FRAME, LAST_FRAME,   0},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME,   0},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME,   0},

  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME, 0},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME, 0},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME, 0},

  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME,   0},
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME, 0},

  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME, 0},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME,   0},
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME, 0},

#if CONFIG_COMP_INTERINTRA_PRED
  /* compound inter-intra prediction */
  {ZEROMV,    LAST_FRAME,   INTRA_FRAME, 0},
  {NEARESTMV, LAST_FRAME,   INTRA_FRAME, 0},
  {NEARMV,    LAST_FRAME,   INTRA_FRAME, 0},
  {NEWMV,     LAST_FRAME,   INTRA_FRAME, 0},

  {ZEROMV,    GOLDEN_FRAME,   INTRA_FRAME, 0},
  {NEARESTMV, GOLDEN_FRAME,   INTRA_FRAME, 0},
  {NEARMV,    GOLDEN_FRAME,   INTRA_FRAME, 0},
  {NEWMV,     GOLDEN_FRAME,   INTRA_FRAME, 0},

  {ZEROMV,    ALTREF_FRAME,   INTRA_FRAME, 0},
  {NEARESTMV, ALTREF_FRAME,   INTRA_FRAME, 0},
  {NEARMV,    ALTREF_FRAME,   INTRA_FRAME, 0},
  {NEWMV,     ALTREF_FRAME,   INTRA_FRAME, 0},
#endif
John Koleszar's avatar
John Koleszar committed
161
};
162
#else
163
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
164
165
  {ZEROMV,    LAST_FRAME,   NONE},
  {DC_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
166

167
168
  {NEARESTMV, LAST_FRAME,   NONE},
  {NEARMV,    LAST_FRAME,   NONE},
John Koleszar's avatar
John Koleszar committed
169

170
171
  {ZEROMV,    GOLDEN_FRAME, NONE},
  {NEARESTMV, GOLDEN_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
172

173
174
  {ZEROMV,    ALTREF_FRAME, NONE},
  {NEARESTMV, ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
175

176
177
  {NEARMV,    GOLDEN_FRAME, NONE},
  {NEARMV,    ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
178

179
180
181
182
183
184
185
186
  {V_PRED,    INTRA_FRAME,  NONE},
  {H_PRED,    INTRA_FRAME,  NONE},
  {D45_PRED,  INTRA_FRAME,  NONE},
  {D135_PRED, INTRA_FRAME,  NONE},
  {D117_PRED, INTRA_FRAME,  NONE},
  {D153_PRED, INTRA_FRAME,  NONE},
  {D27_PRED,  INTRA_FRAME,  NONE},
  {D63_PRED,  INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
187

188
  {TM_PRED,   INTRA_FRAME,  NONE},
John Koleszar's avatar
John Koleszar committed
189

190
191
192
  {NEWMV,     LAST_FRAME,   NONE},
  {NEWMV,     GOLDEN_FRAME, NONE},
  {NEWMV,     ALTREF_FRAME, NONE},
John Koleszar's avatar
John Koleszar committed
193

194
195
196
  {SPLITMV,   LAST_FRAME,   NONE},
  {SPLITMV,   GOLDEN_FRAME, NONE},
  {SPLITMV,   ALTREF_FRAME, NONE},
197

198
199
  {B_PRED,    INTRA_FRAME,  NONE},
  {I8X8_PRED, INTRA_FRAME,  NONE},
200

John Koleszar's avatar
John Koleszar committed
201
202
203
204
  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME},
205

John Koleszar's avatar
John Koleszar committed
206
207
208
  {ZEROMV,    ALTREF_FRAME, LAST_FRAME},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME},
209

John Koleszar's avatar
John Koleszar committed
210
211
212
  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME},
213

John Koleszar's avatar
John Koleszar committed
214
215
216
  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME  },
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME},
217

John Koleszar's avatar
John Koleszar committed
218
219
  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME  },
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME},

#if CONFIG_COMP_INTERINTRA_PRED
  /* compound inter-intra prediction */
  {ZEROMV,    LAST_FRAME,   INTRA_FRAME},
  {NEARESTMV, LAST_FRAME,   INTRA_FRAME},
  {NEARMV,    LAST_FRAME,   INTRA_FRAME},
  {NEWMV,     LAST_FRAME,   INTRA_FRAME},

  {ZEROMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEARESTMV, GOLDEN_FRAME,   INTRA_FRAME},
  {NEARMV,    GOLDEN_FRAME,   INTRA_FRAME},
  {NEWMV,     GOLDEN_FRAME,   INTRA_FRAME},

  {ZEROMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEARESTMV, ALTREF_FRAME,   INTRA_FRAME},
  {NEARMV,    ALTREF_FRAME,   INTRA_FRAME},
  {NEWMV,     ALTREF_FRAME,   INTRA_FRAME},
#endif
John Koleszar's avatar
John Koleszar committed
239
};
240
#endif
John Koleszar's avatar
John Koleszar committed
241
242

static void fill_token_costs(
243
  unsigned int (*c)[COEF_BANDS][PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS],
244
  const vp9_prob(*p)[COEF_BANDS][PREV_COEF_CONTEXTS][ENTROPY_NODES],
John Koleszar's avatar
John Koleszar committed
245
246
247
248
249
250
251
  int block_type_counts) {
  int i, j, k;

  for (i = 0; i < block_type_counts; i++)
    for (j = 0; j < COEF_BANDS; j++)
      for (k = 0; k < PREV_COEF_CONTEXTS; k++) {
        if (k == 0 && ((j > 0 && i > 0) || (j > 1 && i == 0)))
252
253
          vp9_cost_tokens_skip((int *)(c[i][j][k]),
                               p[i][j][k],
254
                               vp9_coef_tree);
John Koleszar's avatar
John Koleszar committed
255
        else
256
257
          vp9_cost_tokens((int *)(c[i][j][k]),
                          p[i][j][k],
258
                          vp9_coef_tree);
John Koleszar's avatar
John Koleszar committed
259
      }
John Koleszar's avatar
John Koleszar committed
260
261
}

262

263
264
265
266
static int rd_iifactor[32] =  { 4, 4, 3, 2, 1, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0, };
John Koleszar's avatar
John Koleszar committed
267

268
// 3* dc_qlookup[Q]*dc_qlookup[Q];
269

270
/* values are now correlated to quantizer */
Paul Wilkins's avatar
Paul Wilkins committed
271
272
273
static int sad_per_bit16lut[QINDEX_RANGE];
static int sad_per_bit4lut[QINDEX_RANGE];

274
void vp9_init_me_luts() {
John Koleszar's avatar
John Koleszar committed
275
276
277
278
279
280
281
  int i;

  // Initialize the sad lut tables using a formulaic calculation for now
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < QINDEX_RANGE; i++) {
    sad_per_bit16lut[i] =
282
283
      (int)((0.0418 * vp9_convert_qindex_to_q(i)) + 2.4107);
    sad_per_bit4lut[i] = (int)((0.063 * vp9_convert_qindex_to_q(i)) + 2.742);
John Koleszar's avatar
John Koleszar committed
284
  }
Paul Wilkins's avatar
Paul Wilkins committed
285
}
John Koleszar's avatar
John Koleszar committed
286

287
static int compute_rd_mult(int qindex) {
John Koleszar's avatar
John Koleszar committed
288
  int q;
289

290
  q = vp9_dc_quant(qindex, 0);
John Koleszar's avatar
John Koleszar committed
291
  return (11 * q * q) >> 6;
292
293
}

294
void vp9_initialize_me_consts(VP9_COMP *cpi, int QIndex) {
John Koleszar's avatar
John Koleszar committed
295
296
  cpi->mb.sadperbit16 =  sad_per_bit16lut[QIndex];
  cpi->mb.sadperbit4  =  sad_per_bit4lut[QIndex];
John Koleszar's avatar
John Koleszar committed
297
298
}

299

300
void vp9_initialize_rd_consts(VP9_COMP *cpi, int QIndex) {
301
  int q, i;
John Koleszar's avatar
John Koleszar committed
302

303
  vp9_clear_system_state();  // __asm emms;
John Koleszar's avatar
John Koleszar committed
304

John Koleszar's avatar
John Koleszar committed
305
306
307
308
309
  // Further tests required to see if optimum is different
  // for key frames, golden frames and arf frames.
  // if (cpi->common.refresh_golden_frame ||
  //     cpi->common.refresh_alt_ref_frame)
  QIndex = (QIndex < 0) ? 0 : ((QIndex > MAXQ) ? MAXQ : QIndex);
310

John Koleszar's avatar
John Koleszar committed
311
  cpi->RDMULT = compute_rd_mult(QIndex);
John Koleszar's avatar
John Koleszar committed
312

John Koleszar's avatar
John Koleszar committed
313
314
315
  // Extend rate multiplier along side quantizer zbin increases
  if (cpi->zbin_over_quant  > 0) {
    double oq_factor;
316

John Koleszar's avatar
John Koleszar committed
317
318
319
320
321
    // Experimental code using the same basic equation as used for Q above
    // The units of cpi->zbin_over_quant are 1/128 of Q bin size
    oq_factor = 1.0 + ((double)0.0015625 * cpi->zbin_over_quant);
    cpi->RDMULT = (int)((double)cpi->RDMULT * oq_factor * oq_factor);
  }
John Koleszar's avatar
John Koleszar committed
322

John Koleszar's avatar
John Koleszar committed
323
324
325
326
327
328
329
  if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    if (cpi->twopass.next_iiratio > 31)
      cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
    else
      cpi->RDMULT +=
        (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
  }
John Koleszar's avatar
John Koleszar committed
330

John Koleszar's avatar
John Koleszar committed
331
332
  if (cpi->RDMULT < 7)
    cpi->RDMULT = 7;
333

John Koleszar's avatar
John Koleszar committed
334
335
  cpi->mb.errorperbit = (cpi->RDMULT / 110);
  cpi->mb.errorperbit += (cpi->mb.errorperbit == 0);
336

337
  vp9_set_speed_features(cpi);
John Koleszar's avatar
John Koleszar committed
338

339
  q = (int)pow(vp9_dc_quant(QIndex, 0) >> 2, 1.25);
John Koleszar's avatar
John Koleszar committed
340
341
  q = q << 2;
  cpi->RDMULT = cpi->RDMULT << 4;
Paul Wilkins's avatar
Paul Wilkins committed
342

John Koleszar's avatar
John Koleszar committed
343
344
  if (q < 8)
    q = 8;
345

John Koleszar's avatar
John Koleszar committed
346
347
348
  if (cpi->RDMULT > 1000) {
    cpi->RDDIV = 1;
    cpi->RDMULT /= 100;
John Koleszar's avatar
John Koleszar committed
349

John Koleszar's avatar
John Koleszar committed
350
351
352
353
354
355
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < INT_MAX) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q / 100;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
356

John Koleszar's avatar
John Koleszar committed
357
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
358
    }
John Koleszar's avatar
John Koleszar committed
359
360
  } else {
    cpi->RDDIV = 100;
John Koleszar's avatar
John Koleszar committed
361

John Koleszar's avatar
John Koleszar committed
362
363
364
365
366
367
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < (INT_MAX / q)) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
368

John Koleszar's avatar
John Koleszar committed
369
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
370
    }
John Koleszar's avatar
John Koleszar committed
371
  }
John Koleszar's avatar
John Koleszar committed
372

John Koleszar's avatar
John Koleszar committed
373
  fill_token_costs(
374
    cpi->mb.token_costs[TX_4X4],
375
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs,
John Koleszar's avatar
John Koleszar committed
376
    BLOCK_TYPES);
377
378
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_4X4],
379
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11])
380
381
    cpi->common.fc.hybrid_coef_probs,
    BLOCK_TYPES);
John Koleszar's avatar
John Koleszar committed
382

John Koleszar's avatar
John Koleszar committed
383
  fill_token_costs(
384
    cpi->mb.token_costs[TX_8X8],
385
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs_8x8,
John Koleszar's avatar
John Koleszar committed
386
    BLOCK_TYPES_8X8);
387
388
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_8X8],
389
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11])
390
391
    cpi->common.fc.hybrid_coef_probs_8x8,
    BLOCK_TYPES_8X8);
392

Daniel Kang's avatar
Daniel Kang committed
393
  fill_token_costs(
394
    cpi->mb.token_costs[TX_16X16],
395
    (const vp9_prob(*)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs_16x16,
Daniel Kang's avatar
Daniel Kang committed
396
    BLOCK_TYPES_16X16);
397
398
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_16X16],
399
    (const vp9_prob(*)[8][PREV_COEF_CONTEXTS][11])
400
401
    cpi->common.fc.hybrid_coef_probs_16x16,
    BLOCK_TYPES_16X16);
Daniel Kang's avatar
Daniel Kang committed
402

John Koleszar's avatar
John Koleszar committed
403
404
  /*rough estimate for costing*/
  cpi->common.kf_ymode_probs_index = cpi->common.base_qindex >> 4;
405
  vp9_init_mode_costs(cpi);
John Koleszar's avatar
John Koleszar committed
406

407
408
  if (cpi->common.frame_type != KEY_FRAME)
  {
409
    vp9_build_nmv_cost_table(
410
411
412
413
414
415
        cpi->mb.nmvjointcost,
        cpi->mb.e_mbd.allow_high_precision_mv ?
        cpi->mb.nmvcost_hp : cpi->mb.nmvcost,
        &cpi->common.fc.nmvc,
        cpi->mb.e_mbd.allow_high_precision_mv, 1, 1);
  }
John Koleszar's avatar
John Koleszar committed
416
417
}

418
int vp9_block_error_c(short *coeff, short *dqcoeff, int block_size) {
419
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
420

421
  for (i = 0; i < block_size; i++) {
John Koleszar's avatar
John Koleszar committed
422
423
424
    int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }
John Koleszar's avatar
John Koleszar committed
425

John Koleszar's avatar
John Koleszar committed
426
  return error;
John Koleszar's avatar
John Koleszar committed
427
428
}

429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
int vp9_mbblock_error_8x8_c(MACROBLOCK *mb, int dc) {
  BLOCK  *be;
  BLOCKD *bd;
  int i, j;
  int berror, error = 0;

  for (i = 0; i < 16; i+=4) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
    berror = 0;
    for (j = dc; j < 64; j++) {
      int this_diff = be->coeff[j] - bd->dqcoeff[j];
      berror += this_diff * this_diff;
    }
    error += berror;
  }
  return error;
}

448
int vp9_mbblock_error_c(MACROBLOCK *mb, int dc) {
John Koleszar's avatar
John Koleszar committed
449
450
451
452
  BLOCK  *be;
  BLOCKD *bd;
  int i, j;
  int berror, error = 0;
John Koleszar's avatar
John Koleszar committed
453

John Koleszar's avatar
John Koleszar committed
454
455
456
457
458
459
460
  for (i = 0; i < 16; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
    berror = 0;
    for (j = dc; j < 16; j++) {
      int this_diff = be->coeff[j] - bd->dqcoeff[j];
      berror += this_diff * this_diff;
John Koleszar's avatar
John Koleszar committed
461
    }
John Koleszar's avatar
John Koleszar committed
462
463
464
    error += berror;
  }
  return error;
John Koleszar's avatar
John Koleszar committed
465
466
}

467
int vp9_mbuverror_c(MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
468
469
  BLOCK  *be;
  BLOCKD *bd;
John Koleszar's avatar
John Koleszar committed
470

471
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
472

John Koleszar's avatar
John Koleszar committed
473
474
475
  for (i = 16; i < 24; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
John Koleszar's avatar
John Koleszar committed
476

477
    error += vp9_block_error_c(be->coeff, bd->dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
478
  }
John Koleszar's avatar
John Koleszar committed
479

John Koleszar's avatar
John Koleszar committed
480
  return error;
John Koleszar's avatar
John Koleszar committed
481
482
}

483
int vp9_uvsse(MACROBLOCK *x) {
John Koleszar's avatar
John Koleszar committed
484
485
486
487
488
489
490
  unsigned char *uptr, *vptr;
  unsigned char *upred_ptr = (*(x->block[16].base_src) + x->block[16].src);
  unsigned char *vpred_ptr = (*(x->block[20].base_src) + x->block[20].src);
  int uv_stride = x->block[16].src_stride;

  unsigned int sse1 = 0;
  unsigned int sse2 = 0;
491
492
  int mv_row = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.row;
  int mv_col = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.col;
John Koleszar's avatar
John Koleszar committed
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
  int offset;
  int pre_stride = x->e_mbd.block[16].pre_stride;

  if (mv_row < 0)
    mv_row -= 1;
  else
    mv_row += 1;

  if (mv_col < 0)
    mv_col -= 1;
  else
    mv_col += 1;

  mv_row /= 2;
  mv_col /= 2;

  offset = (mv_row >> 3) * pre_stride + (mv_col >> 3);
  uptr = x->e_mbd.pre.u_buffer + offset;
  vptr = x->e_mbd.pre.v_buffer + offset;

  if ((mv_row | mv_col) & 7) {
514
    vp9_sub_pixel_variance8x8(uptr, pre_stride, (mv_col & 7) << 1,
515
                              (mv_row & 7) << 1, upred_ptr, uv_stride, &sse2);
516
    vp9_sub_pixel_variance8x8(vptr, pre_stride, (mv_col & 7) << 1,
517
                              (mv_row & 7) << 1, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
518
519
    sse2 += sse1;
  } else {
520
521
    vp9_variance8x8(uptr, pre_stride, upred_ptr, uv_stride, &sse2);
    vp9_variance8x8(vptr, pre_stride, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
522
523
524
    sse2 += sse1;
  }
  return sse2;
John Koleszar's avatar
John Koleszar committed
525
526
527

}

528
static int cost_coeffs_2x2(MACROBLOCK *mb,
529
                           BLOCKD *b, PLANE_TYPE type,
530
                           ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l) {
531
  int c = (type == PLANE_TYPE_Y_NO_DC); /* start at coef 0, unless Y with Y2 */
532
533
534
535
536
  int eob = b->eob;
  int pt;    /* surrounding block/prev coef predictor */
  int cost = 0;
  short *qcoeff_ptr = b->qcoeff;

537
  VP9_COMBINEENTROPYCONTEXTS(pt, *a, *l);
538
539
540
  assert(eob <= 4);

  for (; c < eob; c++) {
541
542
543
544
545
    int v = qcoeff_ptr[vp9_default_zig_zag1d[c]];
    int t = vp9_dct_value_tokens_ptr[v].Token;
    cost += mb->token_costs[TX_8X8][type][vp9_coef_bands[c]][pt][t];
    cost += vp9_dct_value_cost_ptr[v];
    pt = vp9_prev_token_class[t];
546
547
548
  }

  if (c < 4)
549
    cost += mb->token_costs[TX_8X8][type][vp9_coef_bands[c]]
550
            [pt] [DCT_EOB_TOKEN];
551
552
  // is eob first coefficient;
  pt = (c > !type);
553
554
555
556
  *a = *l = pt;
  return cost;
}

557
static int cost_coeffs(MACROBLOCK *mb, BLOCKD *b, PLANE_TYPE type,
Daniel Kang's avatar
Daniel Kang committed
558
                       ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l,
559
                       int tx_size) {
Daniel Kang's avatar
Daniel Kang committed
560
  const int eob = b->eob;
561
  int c = (type == PLANE_TYPE_Y_NO_DC); /* start at coef 0, unless Y with Y2 */
562
  int cost = 0, default_eob, seg_eob;
Daniel Kang's avatar
Daniel Kang committed
563
564
  int pt;                     /* surrounding block/prev coef predictor */
  int const *scan, *band;
John Koleszar's avatar
John Koleszar committed
565
  short *qcoeff_ptr = b->qcoeff;
566
567
568
  MACROBLOCKD *xd = &mb->e_mbd;
  MB_MODE_INFO *mbmi = &mb->e_mbd.mode_info_context->mbmi;
  TX_TYPE tx_type = DCT_DCT;
Paul Wilkins's avatar
Paul Wilkins committed
569
  int segment_id = mbmi->segment_id;
Jim Bankoski's avatar
Jim Bankoski committed
570
571
572
  scan = vp9_default_zig_zag1d;
  band = vp9_coef_bands;
  default_eob = 16;
573

574
  switch (tx_size) {
Daniel Kang's avatar
Daniel Kang committed
575
    case TX_4X4:
Deb Mukherjee's avatar
Deb Mukherjee committed
576
577
578
579
580
      if (type == PLANE_TYPE_Y_WITH_DC) {
        tx_type = get_tx_type_4x4(xd, b);
        if (tx_type != DCT_DCT) {
          switch (tx_type) {
            case ADST_DCT:
581
              scan = vp9_row_scan;
Deb Mukherjee's avatar
Deb Mukherjee committed
582
583
584
              break;

            case DCT_ADST:
585
              scan = vp9_col_scan;
Deb Mukherjee's avatar
Deb Mukherjee committed
586
587
588
              break;

            default:
589
              scan = vp9_default_zig_zag1d;
Deb Mukherjee's avatar
Deb Mukherjee committed
590
591
              break;
          }
592
        }
Daniel Kang's avatar
Daniel Kang committed
593
      }
Deb Mukherjee's avatar
Deb Mukherjee committed
594

Daniel Kang's avatar
Daniel Kang committed
595
596
      break;
    case TX_8X8:
597
598
      scan = vp9_default_zig_zag1d_8x8;
      band = vp9_coef_bands_8x8;
Daniel Kang's avatar
Daniel Kang committed
599
      default_eob = 64;
Deb Mukherjee's avatar
Deb Mukherjee committed
600
      if (type == PLANE_TYPE_Y_WITH_DC) {
601
        BLOCKD *bb;
602
        int ib = (int)(b - xd->block);
603
604
605
        if (ib < 16) {
          ib = (ib & 8) + ((ib & 4) >> 1);
          bb = xd->block + ib;
Deb Mukherjee's avatar
Deb Mukherjee committed
606
          tx_type = get_tx_type_8x8(xd, bb);
607
        }
608
      }
Daniel Kang's avatar
Daniel Kang committed
609
610
      break;
    case TX_16X16:
611
612
      scan = vp9_default_zig_zag1d_16x16;
      band = vp9_coef_bands_16x16;
Daniel Kang's avatar
Daniel Kang committed
613
      default_eob = 256;
Deb Mukherjee's avatar
Deb Mukherjee committed
614
615
616
      if (type == PLANE_TYPE_Y_WITH_DC) {
        tx_type = get_tx_type_16x16(xd, b);
      }
Daniel Kang's avatar
Daniel Kang committed
617
618
619
620
      break;
    default:
      break;
  }
621
622
  if (vp9_segfeature_active(&mb->e_mbd, segment_id, SEG_LVL_EOB))
    seg_eob = vp9_get_segdata(&mb->e_mbd, segment_id, SEG_LVL_EOB);
623
624
625
  else
    seg_eob = default_eob;

626
  VP9_COMBINEENTROPYCONTEXTS(pt, *a, *l);
627

628
629
630
  if (tx_type != DCT_DCT) {
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
631
      int t = vp9_dct_value_tokens_ptr[v].Token;
632
      cost += mb->hybrid_token_costs[tx_size][type][band[c]][pt][t];
633
634
      cost += vp9_dct_value_cost_ptr[v];
      pt = vp9_prev_token_class[t];
635
636
637
638
    }
    if (c < seg_eob)
      cost += mb->hybrid_token_costs[tx_size][type][band[c]]
          [pt][DCT_EOB_TOKEN];
639
  } else {
640
641
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
642
      int t = vp9_dct_value_tokens_ptr[v].Token;
643
      cost += mb->token_costs[tx_size][type][band[c]][pt][t];
644
645
      cost += vp9_dct_value_cost_ptr[v];
      pt = vp9_prev_token_class[t];
646
647
648
649
    }
    if (c < seg_eob)
      cost += mb->token_costs[tx_size][type][band[c]]
          [pt][DCT_EOB_TOKEN];
650
651
  }

652
653
  // is eob first coefficient;
  pt = (c > !type);
654
655
656
657
  *a = *l = pt;
  return cost;
}

658
static int rdcost_mby_4x4(MACROBLOCK *mb, int has_2nd_order, int backup) {
John Koleszar's avatar
John Koleszar committed
659
660
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
661
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
662
663
664
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta;
  ENTROPY_CONTEXT *tl;
665

666
667
668
  if (backup) {
    vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
669

670
671
672
673
674
675
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)xd->above_context;
    tl = (ENTROPY_CONTEXT *)xd->left_context;
  }
John Koleszar's avatar
John Koleszar committed
676

John Koleszar's avatar
John Koleszar committed
677
  for (b = 0; b < 16; b++)
678
679
680
    cost += cost_coeffs(mb, xd->block + b,
                        (has_2nd_order ?
                         PLANE_TYPE_Y_NO_DC : PLANE_TYPE_Y_WITH_DC),
681
                        ta + vp9_block2above[b], tl + vp9_block2left[b],
Daniel Kang's avatar
Daniel Kang committed
682
                        TX_4X4);
John Koleszar's avatar
John Koleszar committed
683

684
685
686
687
  if (has_2nd_order)
    cost += cost_coeffs(mb, xd->block + 24, PLANE_TYPE_Y2,
                        ta + vp9_block2above[24], tl + vp9_block2left[24],
                        TX_4X4);
John Koleszar's avatar
John Koleszar committed
688

John Koleszar's avatar
John Koleszar committed
689
  return cost;
John Koleszar's avatar
John Koleszar committed
690
691
}

692
693
694
static void macro_block_yrd_4x4(MACROBLOCK *mb,
                                int *Rate,
                                int *Distortion,
695
                                int *skippable, int backup) {
Paul Wilkins's avatar
Paul Wilkins committed
696
  MACROBLOCKD *const xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
697
  BLOCK   *const mb_y2 = mb->block + 24;
Paul Wilkins's avatar
Paul Wilkins committed
698
  BLOCKD *const x_y2  = xd->block + 24;
Jim Bankoski's avatar
Jim Bankoski committed
699
  int d, has_2nd_order;
700

701
702
  xd->mode_info_context->mbmi.txfm_size = TX_4X4;
  has_2nd_order = get_2nd_order_usage(xd);
John Koleszar's avatar
John Koleszar committed
703
  // Fdct and building the 2nd order block
704
705
706
707
708
  vp9_transform_mby_4x4(mb);
  vp9_quantize_mby_4x4(mb);
  d = vp9_mbblock_error(mb, has_2nd_order);
  if (has_2nd_order)
    d += vp9_block_error(mb_y2->coeff, x_y2->dqcoeff, 16);
709

John Koleszar's avatar
John Koleszar committed
710
711
  *Distortion = (d >> 2);
  // rate
712
713
  *Rate = rdcost_mby_4x4(mb, has_2nd_order, backup);
  *skippable = vp9_mby_is_skippable_4x4(&mb->e_mbd, has_2nd_order);
714
}
John Koleszar's avatar
John Koleszar committed
715

716
static int rdcost_mby_8x8(MACROBLOCK *mb, int has_2nd_order, int backup) {
John Koleszar's avatar
John Koleszar committed
717
718
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
719
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
720
721
722
723
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta;
  ENTROPY_CONTEXT *tl;

Ronald S. Bultje's avatar
Ronald S. Bultje committed
724
725
726
  if (backup) {
    vpx_memcpy(&t_above,xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
727

Ronald S. Bultje's avatar
Ronald S. Bultje committed
728
729
730
731
732
733
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)mb->e_mbd.above_context;
    tl = (ENTROPY_CONTEXT *)mb->e_mbd.left_context;
  }
John Koleszar's avatar
John Koleszar committed
734
735

  for (b = 0; b < 16; b += 4)
736
737
738
    cost += cost_coeffs(mb, xd->block + b,
                        (has_2nd_order ?
                         PLANE_TYPE_Y_NO_DC : PLANE_TYPE_Y_WITH_DC),
739
                        ta + vp9_block2above_8x8[b], tl + vp9_block2left_8x8[b],
Daniel Kang's avatar
Daniel Kang committed
740
                        TX_8X8);
John Koleszar's avatar
John Koleszar committed
741

742
743
744
  if (has_2nd_order)
    cost += cost_coeffs_2x2(mb, xd->block + 24, PLANE_TYPE_Y2,
                            ta + vp9_block2above[24], tl + vp9_block2left[24]);
John Koleszar's avatar
John Koleszar committed
745
  return cost;
746
747
}

John Koleszar's avatar
John Koleszar committed
748
749
750
static void macro_block_yrd_8x8(MACROBLOCK *mb,
                                int *Rate,
                                int *Distortion,
751
                                int *skippable, int backup) {
Paul Wilkins's avatar
Paul Wilkins committed
752
  MACROBLOCKD *const xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
753
  BLOCK   *const mb_y2 = mb->block + 24;
Paul Wilkins's avatar
Paul Wilkins committed
754
  BLOCKD *const x_y2  = xd->block + 24;
755
756
757
  int d, has_2nd_order;

  xd->mode_info_context->mbmi.txfm_size = TX_8X8;
John Koleszar's avatar
John Koleszar committed
758

759
760
  vp9_transform_mby_8x8(mb);
  vp9_quantize_mby_8x8(mb);
761
762
763
764
  has_2nd_order = get_2nd_order_usage(xd);
  d = vp9_mbblock_error_8x8_c(mb, has_2nd_order);
  if (has_2nd_order)
    d += vp9_block_error(mb_y2->coeff, x_y2->dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
765
766
767

  *Distortion = (d >> 2);
  // rate
768
769
  *Rate = rdcost_mby_8x8(mb, has_2nd_order, backup);
  *skippable = vp9_mby_is_skippable_8x8(&mb->e_mbd, has_2nd_order);
770
}
771

772
static int rdcost_mby_16x16(MACROBLOCK *mb, int backup) {
Daniel Kang's avatar
Daniel Kang committed
773
  int cost;
Paul Wilkins's avatar
Paul Wilkins committed
774
  MACROBLOCKD *xd = &mb->e_mbd;
Daniel Kang's avatar
Daniel Kang committed
775
776
777
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta, *tl;

778
779
780
  if (backup) {
    vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
Daniel Kang's avatar
Daniel Kang committed
781

782
783
784
785
786
787
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)xd->above_context;
    tl = (ENTROPY_CONTEXT *)xd->left_context;
  }
Daniel Kang's avatar
Daniel Kang committed
788

Paul Wilkins's avatar
Paul Wilkins committed
789
  cost = cost_coeffs(mb, xd->block, PLANE_TYPE_Y_WITH_DC, ta, tl, TX_16X16);
Daniel Kang's avatar
Daniel Kang committed
790
791
  return cost;
}
792

Daniel Kang's avatar
Daniel Kang committed
793
static void macro_block_yrd_16x16(MACROBLOCK *mb, int *Rate, int *Distortion,
794
                                  int *skippable, int backup) {
Daniel Kang's avatar
Daniel Kang committed
795
  int d;
Deb Mukherjee's avatar
Deb Mukherjee committed
796
  MACROBLOCKD *xd = &mb->e_mbd;
Daniel Kang's avatar
Daniel Kang committed
797

798
799
  xd->mode_info_context->mbmi.txfm_size = TX_16X16;
  vp9_transform_mby_16x16(mb);
800
  vp9_quantize_mby_16x16(mb);
801
802
803
  // TODO(jingning) is it possible to quickly determine whether to force
  //                trailing coefficients to be zero, instead of running trellis
  //                optimization in the rate-distortion optimization loop?
804
  if (mb->e_mbd.mode_info_context->mbmi.mode < I8X8_PRED)
805
    vp9_optimize_mby_16x16(mb);
806

807
  d = vp9_mbblock_error(mb, 0);
Daniel Kang's avatar
Daniel Kang committed
808
809
810

  *Distortion = (d >> 2);
  // rate
811
  *Rate = rdcost_mby_16x16(mb, backup);
812
  *skippable = vp9_mby_is_skippable_16x16(&mb->e_mbd);
Daniel Kang's avatar
Daniel Kang committed
813
814
}

815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
static void choose_txfm_size_from_rd(VP9_COMP *cpi, MACROBLOCK *x,
                                     int r[2][TX_SIZE_MAX], int *rate,
                                     int d[TX_SIZE_MAX], int *distortion,
                                     int s[TX_SIZE_MAX], int *skip,
                                     int64_t txfm_cache[NB_TXFM_MODES]) {
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
  vp9_prob skip_prob = cm->mb_no_coeff_skip ?
                       vp9_get_pred_prob(cm, xd, PRED_MBSKIP) : 128;
  int64_t rd[2][TX_SIZE_MAX];
  int n;

  r[1][TX_16X16] = r[0][TX_16X16] + vp9_cost_one(cm->prob_tx[0]) +
                   vp9_cost_one(cm->prob_tx[1]);
  r[1][TX_8X8]   = r[0][TX_8X8] + vp9_cost_one(cm->prob_tx[0]) +
                   vp9_cost_zero(cm->prob_tx[1]);
  r[1][TX_4X4]   = r[0][TX_4X4] + vp9_cost_zero(cm->prob_tx[0]);

  if (cm->mb_no_coeff_skip) {
    int s0, s1;

    assert(skip_prob > 0);
    s0 = vp9_cost_bit(skip_prob, 0);
    s1 = vp9_cost_bit(skip_prob, 1);

    for (n = TX_4X4; n <= TX_16X16; n++) {
      if (s[n]) {
        rd[0][n] = rd[1][n] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
      } else {
        rd[0][n] = RDCOST(x->rdmult, x->rddiv, r[0][n] + s0, d[n]);
        rd[1][n] = RDCOST(x->rdmult, x->rddiv, r[1][n] + s0, d[n]);
      }
848
849
    }
  } else {
850
851
852
    for (n = TX_4X4; n <= TX_16X16; n++) {
      rd[0][n] = RDCOST(x->rdmult, x->rddiv, r[0][n], d[n]);
      rd[1][n] = RDCOST(x->rdmult, x->rddiv, r[1][n], d[n]);
853
854
855
    }
  }

856
857
858
  if ( cm->txfm_mode == ALLOW_16X16 ||
      (cm->txfm_mode == TX_MODE_SELECT &&
       rd[1][TX_16X16] < rd[1][TX_8X8] && rd[1][TX_16X16] < rd[1][TX_4X4])) {
859
    mbmi->txfm_size = TX_16X16;
860
861
  } else if (cm->txfm_mode == ALLOW_8X8 ||
           (cm->txfm_mode == TX_MODE_SELECT && rd[1][TX_8X8] < rd[1][TX_4X4])) {
862
863
    mbmi->txfm_size = TX_8X8;
  } else {
864
865
    assert(cm->txfm_mode == ONLY_4X4 ||
          (cm->txfm_mode == TX_MODE_SELECT && rd[1][TX_4X4] <= rd[1][TX_8X8]));
866
867
868
    mbmi->txfm_size = TX_4X4;
  }

869
870
871
872
873
874
875
876
877
  *distortion = d[mbmi->txfm_size];
  *rate       = r[cm->txfm_mode == TX_MODE_SELECT][mbmi->txfm_size];
  *skip       = s[mbmi->txfm_size];

  txfm_cache[ONLY_4X4] = rd[0][TX_4X4];
  txfm_cache[ALLOW_8X8] = rd[0][TX_8X8];
  txfm_cache[ALLOW_16X16] = rd[0][TX_16X16];
  if (rd[1][TX_16X16] < rd[1][TX_8X8] && rd[1][TX_16X16] < rd[1][TX_4X4])
    txfm_cache[TX_MODE_SELECT] = rd[1][TX_16X16];
878
  else
879
880
881
882
883
884
885
886
887
888
889
890
    txfm_cache[TX_MODE_SELECT] = rd[1][TX_4X4] < rd[1][TX_8X8] ?
                                 rd[1][TX_4X4] : rd[1][TX_8X8];
}

static void macro_block_yrd(VP9_COMP *cpi, MACROBLOCK *x, int *rate,
                            int *distortion, int *skippable,
                            int64_t txfm_cache[NB_TXFM_MODES]) {
  MACROBLOCKD *const xd = &x->e_mbd;
  int r[2][TX_SIZE_MAX], d[TX_SIZE_MAX], s[TX_SIZE_MAX];

  vp9_subtract_mby(x->src_diff, *(x->block[0].base_src), xd->predictor,
                   x->block[0].src_stride);
891

892
  macro_block_yrd_16x16(x, &r[0][TX_16X16], &d[TX_16X16],
893
894
895
                        &s[TX_16X16], 1);
  macro_block_yrd_8x8(x, &r[0][TX_8X8], &d[TX_8X8], &s[TX_8X8], 1);
  macro_block_yrd_4x4(x, &r[0][TX_4X4], &d[TX_4X4], &s[TX_4X4], 1);
896
897
898

  choose_txfm_size_from_rd(cpi, x, r, rate, d, distortion, s, skippable,
                           txfm_cache);
899
900
}

John Koleszar's avatar
John Koleszar committed
901
902
903
904
905
906
907
static void copy_predictor(unsigned char *dst, const unsigned char *predictor) {
  const unsigned int *p = (const unsigned int *)predictor;
  unsigned int *d = (unsigned int *)dst;
  d[0] = p[0];
  d[4] = p[4];
  d[8] = p[8];
  d[12] = p[12];
908
}
Yaowu Xu's avatar
Yaowu Xu committed
909

Ronald S. Bultje's avatar
Ronald S. Bultje committed
910
#if CONFIG_SUPERBLOCKS
911
912
static void super_block_yrd(VP9_COMP *cpi,
                            MACROBLOCK *x, int *rate, int *distortion,
913
                            int *skip,
914
                            int64_t txfm_cache[NB_TXFM_MODES]) {
Ronald S. Bultje's avatar
Ronald S. Bultje committed
915
  MACROBLOCKD *const xd = &x->e_mbd;
916
  int r[2][TX_SIZE_MAX], d[TX_SIZE_MAX], s[TX_SIZE_MAX], n;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
917
918
  const uint8_t *src = x->src.y_buffer, *dst = xd->dst.y_buffer;
  int src_y_stride = x->src.y_stride, dst_y_stride = xd->dst.y_stride;
919
920
921
922
923
924
925
926
927
928
  ENTROPY_CONTEXT_PLANES t_above[3][2], *orig_above = xd->above_context;
  ENTROPY_CONTEXT_PLANES t_left[3][2], *orig_left = xd->left_context;

  for (n = TX_4X4; n <= TX_16X16; n++) {
    vpx_memcpy(t_above[n], xd->above_context, sizeof(t_above[n]));
    vpx_memcpy(t_left[n], xd->left_context, sizeof(t_left[n]));
    r[0][n] = 0;
    d[n] = 0;
    s[n] = 1;
  }
Ronald S. Bultje's avatar
Ronald S. Bultje committed
929
930
931

  for (n = 0; n < 4; n++) {
    int x_idx = n & 1, y_idx = n >> 1;
932
    int r_tmp, d_tmp, s_tmp;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
933

934
    vp9_subtract_mby_s_c(x->src_diff,
Ronald S. Bultje's avatar
Ronald S. Bultje committed
935
936
937
938
                         src + x_idx * 16 + y_idx * 16 * src_y_stride,
                         src_y_stride,
                         dst + x_idx * 16 + y_idx * 16 * dst_y_stride,
                         dst_y_stride);
939
940
941

    xd->above_context = &t_above[TX_16X16][x_idx];
    xd->left_context = &t_left[TX_16X16][y_idx];
942
    macro_block_yrd_16x16(x, &r_tmp, &d_tmp, &s_tmp, 0);
943
944
945
946
947
948
    d[TX_16X16] += d_tmp;
    r[0][TX_16X16] += r_tmp;
    s[TX_16X16] = s[TX_16X16] && s_tmp;

    xd->above_context = &t_above[TX_4X4][x_idx];
    xd->left_context = &t_left[TX_4X4][y_idx];
949
    macro_block_yrd_4x4(x, &r_tmp, &d_tmp, &s_tmp, 0);
950
951
952
953
954
955
    d[TX_4X4] += d_tmp;
    r[0][TX_4X4] += r_tmp;
    s[TX_4X4] = s[TX_4X4] && s_tmp;

    xd->above_context = &t_above[TX_8X8][x_idx];
    xd->left_context = &t_left[TX_8X8][y_idx];
956
    macro_block_yrd_8x8(x, &r_tmp, &d_tmp, &s_tmp, 0);
957
958
959
    d[TX_8X8] += d_tmp;
    r[0][TX_8X8] += r_tmp;
    s[TX_8X8] = s[TX_8X8] && s_tmp;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
960
961
  }

962
963
964
965
  choose_txfm_size_from_rd(cpi, x, r, rate, d, distortion, s, skip, txfm_cache);

  xd->above_context = orig_above;
  xd->left_context = orig_left;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
966
967
968
}
#endif

John Koleszar's avatar
John Koleszar committed
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
static void copy_predictor_8x8(unsigned char *dst, const unsigned char *predictor) {
  const unsigned int *p = (const unsigned int *)predictor;
  unsigned int *d = (unsigned int *)dst;
  d[0] = p[0];
  d[1] = p[1];
  d[4] = p[4];
  d[5] = p[5];
  d[8] = p[8];
  d[9] = p[9];
  d[12] = p[12];
  d[13] = p[13];
  d[16] = p[16];
  d[17] = p[17];
  d[20] = p[20];
  d[21] = p[21];
  d[24] = p[24];
  d[25] = p[25];
  d[28] = p[28];
  d[29] = p[29];
Yaowu Xu's avatar
Yaowu Xu committed
988
989
}

990
static int64_t rd_pick_intra4x4block(VP9_COMP *cpi, MACROBLOCK *x, BLOCK *be,
991
                                     BLOCKD *b, B_PREDICTION_MODE *best_mode,
992
#if CONFIG_COMP_INTRA_PRED
993
994
995
996
997
998
999
                                     B_PREDICTION_MODE *best_second_mode,
                                     int allow_comp,
#endif
                                     int *bmode_costs,
                                     ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l,
                                     int *bestrate, int *bestratey,
                                     int *bestdistortion