rdopt.c 166 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5
6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9
10
11
12
13
14
15
 */


#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
16
#include "vp9/common/pragmas.h"
John Koleszar's avatar
John Koleszar committed
17
18
19
20
21
22

#include "tokenize.h"
#include "treewriter.h"
#include "onyx_int.h"
#include "modecosts.h"
#include "encodeintra.h"
23
24
25
26
27
28
#include "vp9/common/entropymode.h"
#include "vp9/common/reconinter.h"
#include "vp9/common/reconintra.h"
#include "vp9/common/reconintra4x4.h"
#include "vp9/common/findnearmv.h"
#include "vp9/common/quant_common.h"
John Koleszar's avatar
John Koleszar committed
29
30
#include "encodemb.h"
#include "quantize.h"
31
#include "vp9/common/idct.h"
John Koleszar's avatar
John Koleszar committed
32
33
#include "variance.h"
#include "mcomp.h"
Yunqing Wang's avatar
Yunqing Wang committed
34
#include "rdopt.h"
Paul Wilkins's avatar
Paul Wilkins committed
35
#include "ratectrl.h"
John Koleszar's avatar
John Koleszar committed
36
#include "vpx_mem/vpx_mem.h"
37
38
#include "vp9/common/systemdependent.h"
#include "vp9/encoder/encodemv.h"
John Koleszar's avatar
John Koleszar committed
39

40
41
42
#include "vp9/common/seg_common.h"
#include "vp9/common/pred_common.h"
#include "vp9/common/entropy.h"
Jim Bankoski's avatar
Jim Bankoski committed
43
#include "vpx_rtcd.h"
44
#include "vp9/common/mvref_common.h"
Paul Wilkins's avatar
Paul Wilkins committed
45

John Koleszar's avatar
John Koleszar committed
46
47
48
49
50
51
52
53
#if CONFIG_RUNTIME_CPU_DETECT
#define IF_RTCD(x)  (x)
#else
#define IF_RTCD(x)  NULL
#endif

#define MAXF(a,b)            (((a) > (b)) ? (a) : (b))

54
55
#define INVALID_MV 0x80008000

56
57
58
/* Factor to weigh the rate for switchable interp filters */
#define SWITCHABLE_INTERP_RATE_FACTOR 1

John Koleszar's avatar
John Koleszar committed
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
static const int auto_speed_thresh[17] = {
  1000,
  200,
  150,
  130,
  150,
  125,
  120,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  105
John Koleszar's avatar
John Koleszar committed
77
78
};

79
#if CONFIG_PRED_FILTER
80
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
John Koleszar's avatar
John Koleszar committed
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
  {ZEROMV,    LAST_FRAME,   0,  0},
  {ZEROMV,    LAST_FRAME,   0,  1},
  {DC_PRED,   INTRA_FRAME,  0,  0},

  {NEARESTMV, LAST_FRAME,   0,  0},
  {NEARESTMV, LAST_FRAME,   0,  1},
  {NEARMV,    LAST_FRAME,   0,  0},
  {NEARMV,    LAST_FRAME,   0,  1},

  {ZEROMV,    GOLDEN_FRAME, 0,  0},
  {ZEROMV,    GOLDEN_FRAME, 0,  1},
  {NEARESTMV, GOLDEN_FRAME, 0,  0},
  {NEARESTMV, GOLDEN_FRAME, 0,  1},

  {ZEROMV,    ALTREF_FRAME, 0,  0},
  {ZEROMV,    ALTREF_FRAME, 0,  1},
  {NEARESTMV, ALTREF_FRAME, 0,  0},
  {NEARESTMV, ALTREF_FRAME, 0,  1},

  {NEARMV,    GOLDEN_FRAME, 0,  0},
  {NEARMV,    GOLDEN_FRAME, 0,  1},
  {NEARMV,    ALTREF_FRAME, 0,  0},
  {NEARMV,    ALTREF_FRAME, 0,  1},

  {V_PRED,    INTRA_FRAME,  0,  0},
  {H_PRED,    INTRA_FRAME,  0,  0},
  {D45_PRED,  INTRA_FRAME,  0,  0},
  {D135_PRED, INTRA_FRAME,  0,  0},
  {D117_PRED, INTRA_FRAME,  0,  0},
  {D153_PRED, INTRA_FRAME,  0,  0},
  {D27_PRED,  INTRA_FRAME,  0,  0},
  {D63_PRED,  INTRA_FRAME,  0,  0},

  {TM_PRED,   INTRA_FRAME,  0,  0},

  {NEWMV,     LAST_FRAME,   0,  0},
  {NEWMV,     LAST_FRAME,   0,  1},
  {NEWMV,     GOLDEN_FRAME, 0,  0},
  {NEWMV,     GOLDEN_FRAME, 0,  1},
  {NEWMV,     ALTREF_FRAME, 0,  0},
  {NEWMV,     ALTREF_FRAME, 0,  1},

  {SPLITMV,   LAST_FRAME,   0,  0},
  {SPLITMV,   GOLDEN_FRAME, 0,  0},
  {SPLITMV,   ALTREF_FRAME, 0,  0},

  {B_PRED,    INTRA_FRAME,  0,  0},
  {I8X8_PRED, INTRA_FRAME,  0,  0},

  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME, 0},

  {ZEROMV,    ALTREF_FRAME, LAST_FRAME,   0},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME,   0},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME,   0},

  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME, 0},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME, 0},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME, 0},

  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME,   0},
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME, 0},

  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME, 0},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME,   0},
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME, 0}
John Koleszar's avatar
John Koleszar committed
150
};
151
#else
152
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
John Koleszar's avatar
John Koleszar committed
153
154
  {ZEROMV,    LAST_FRAME,   0},
  {DC_PRED,   INTRA_FRAME,  0},
John Koleszar's avatar
John Koleszar committed
155

John Koleszar's avatar
John Koleszar committed
156
157
  {NEARESTMV, LAST_FRAME,   0},
  {NEARMV,    LAST_FRAME,   0},
John Koleszar's avatar
John Koleszar committed
158

John Koleszar's avatar
John Koleszar committed
159
160
  {ZEROMV,    GOLDEN_FRAME, 0},
  {NEARESTMV, GOLDEN_FRAME, 0},
John Koleszar's avatar
John Koleszar committed
161

John Koleszar's avatar
John Koleszar committed
162
163
  {ZEROMV,    ALTREF_FRAME, 0},
  {NEARESTMV, ALTREF_FRAME, 0},
John Koleszar's avatar
John Koleszar committed
164

John Koleszar's avatar
John Koleszar committed
165
166
  {NEARMV,    GOLDEN_FRAME, 0},
  {NEARMV,    ALTREF_FRAME, 0},
John Koleszar's avatar
John Koleszar committed
167

John Koleszar's avatar
John Koleszar committed
168
169
170
171
172
173
174
175
  {V_PRED,    INTRA_FRAME,  0},
  {H_PRED,    INTRA_FRAME,  0},
  {D45_PRED,  INTRA_FRAME,  0},
  {D135_PRED, INTRA_FRAME,  0},
  {D117_PRED, INTRA_FRAME,  0},
  {D153_PRED, INTRA_FRAME,  0},
  {D27_PRED,  INTRA_FRAME,  0},
  {D63_PRED,  INTRA_FRAME,  0},
John Koleszar's avatar
John Koleszar committed
176

John Koleszar's avatar
John Koleszar committed
177
  {TM_PRED,   INTRA_FRAME,  0},
John Koleszar's avatar
John Koleszar committed
178

John Koleszar's avatar
John Koleszar committed
179
180
181
  {NEWMV,     LAST_FRAME,   0},
  {NEWMV,     GOLDEN_FRAME, 0},
  {NEWMV,     ALTREF_FRAME, 0},
John Koleszar's avatar
John Koleszar committed
182

John Koleszar's avatar
John Koleszar committed
183
184
185
  {SPLITMV,   LAST_FRAME,   0},
  {SPLITMV,   GOLDEN_FRAME, 0},
  {SPLITMV,   ALTREF_FRAME, 0},
186

John Koleszar's avatar
John Koleszar committed
187
188
  {B_PRED,    INTRA_FRAME,  0},
  {I8X8_PRED, INTRA_FRAME,  0},
189

John Koleszar's avatar
John Koleszar committed
190
191
192
193
  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME},
194

John Koleszar's avatar
John Koleszar committed
195
196
197
  {ZEROMV,    ALTREF_FRAME, LAST_FRAME},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME},
198

John Koleszar's avatar
John Koleszar committed
199
200
201
  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME},
202

John Koleszar's avatar
John Koleszar committed
203
204
205
  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME  },
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME},
206

John Koleszar's avatar
John Koleszar committed
207
208
209
  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME  },
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME}
John Koleszar's avatar
John Koleszar committed
210
};
211
#endif
John Koleszar's avatar
John Koleszar committed
212
213

static void fill_token_costs(
214
  unsigned int (*c)[COEF_BANDS][PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS],
215
  const vp9_prob(*p)[COEF_BANDS][PREV_COEF_CONTEXTS][ENTROPY_NODES],
John Koleszar's avatar
John Koleszar committed
216
217
218
219
220
221
222
  int block_type_counts) {
  int i, j, k;

  for (i = 0; i < block_type_counts; i++)
    for (j = 0; j < COEF_BANDS; j++)
      for (k = 0; k < PREV_COEF_CONTEXTS; k++) {
        if (k == 0 && ((j > 0 && i > 0) || (j > 1 && i == 0)))
223
224
          vp9_cost_tokens_skip((int *)(c[i][j][k]),
                               p[i][j][k],
225
                               vp9_coef_tree);
John Koleszar's avatar
John Koleszar committed
226
        else
227
228
          vp9_cost_tokens((int *)(c[i][j][k]),
                          p[i][j][k],
229
                          vp9_coef_tree);
John Koleszar's avatar
John Koleszar committed
230
      }
John Koleszar's avatar
John Koleszar committed
231
232
}

233

234
235
236
237
static int rd_iifactor[32] =  { 4, 4, 3, 2, 1, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0, };
John Koleszar's avatar
John Koleszar committed
238

239
// 3* dc_qlookup[Q]*dc_qlookup[Q];
240

241
/* values are now correlated to quantizer */
Paul Wilkins's avatar
Paul Wilkins committed
242
243
244
static int sad_per_bit16lut[QINDEX_RANGE];
static int sad_per_bit4lut[QINDEX_RANGE];

245
void vp9_init_me_luts() {
John Koleszar's avatar
John Koleszar committed
246
247
248
249
250
251
252
  int i;

  // Initialize the sad lut tables using a formulaic calculation for now
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < QINDEX_RANGE; i++) {
    sad_per_bit16lut[i] =
253
254
      (int)((0.0418 * vp9_convert_qindex_to_q(i)) + 2.4107);
    sad_per_bit4lut[i] = (int)((0.063 * vp9_convert_qindex_to_q(i)) + 2.742);
John Koleszar's avatar
John Koleszar committed
255
  }
Paul Wilkins's avatar
Paul Wilkins committed
256
}
John Koleszar's avatar
John Koleszar committed
257

258
static int compute_rd_mult(int qindex) {
John Koleszar's avatar
John Koleszar committed
259
  int q;
260

261
  q = vp9_dc_quant(qindex, 0);
John Koleszar's avatar
John Koleszar committed
262
  return (11 * q * q) >> 6;
263
264
}

265
void vp9_initialize_me_consts(VP9_COMP *cpi, int QIndex) {
John Koleszar's avatar
John Koleszar committed
266
267
  cpi->mb.sadperbit16 =  sad_per_bit16lut[QIndex];
  cpi->mb.sadperbit4  =  sad_per_bit4lut[QIndex];
John Koleszar's avatar
John Koleszar committed
268
269
}

270

271
void vp9_initialize_rd_consts(VP9_COMP *cpi, int QIndex) {
272
  int q, i;
John Koleszar's avatar
John Koleszar committed
273

274
  vp9_clear_system_state();  // __asm emms;
John Koleszar's avatar
John Koleszar committed
275

John Koleszar's avatar
John Koleszar committed
276
277
278
279
280
  // Further tests required to see if optimum is different
  // for key frames, golden frames and arf frames.
  // if (cpi->common.refresh_golden_frame ||
  //     cpi->common.refresh_alt_ref_frame)
  QIndex = (QIndex < 0) ? 0 : ((QIndex > MAXQ) ? MAXQ : QIndex);
281

John Koleszar's avatar
John Koleszar committed
282
  cpi->RDMULT = compute_rd_mult(QIndex);
John Koleszar's avatar
John Koleszar committed
283

John Koleszar's avatar
John Koleszar committed
284
285
286
  // Extend rate multiplier along side quantizer zbin increases
  if (cpi->zbin_over_quant  > 0) {
    double oq_factor;
287

John Koleszar's avatar
John Koleszar committed
288
289
290
291
292
    // Experimental code using the same basic equation as used for Q above
    // The units of cpi->zbin_over_quant are 1/128 of Q bin size
    oq_factor = 1.0 + ((double)0.0015625 * cpi->zbin_over_quant);
    cpi->RDMULT = (int)((double)cpi->RDMULT * oq_factor * oq_factor);
  }
John Koleszar's avatar
John Koleszar committed
293

John Koleszar's avatar
John Koleszar committed
294
295
296
297
298
299
300
  if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    if (cpi->twopass.next_iiratio > 31)
      cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
    else
      cpi->RDMULT +=
        (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
  }
John Koleszar's avatar
John Koleszar committed
301

John Koleszar's avatar
John Koleszar committed
302
303
  if (cpi->RDMULT < 7)
    cpi->RDMULT = 7;
304

John Koleszar's avatar
John Koleszar committed
305
306
  cpi->mb.errorperbit = (cpi->RDMULT / 110);
  cpi->mb.errorperbit += (cpi->mb.errorperbit == 0);
307

308
  vp9_set_speed_features(cpi);
John Koleszar's avatar
John Koleszar committed
309

310
  q = (int)pow(vp9_dc_quant(QIndex, 0) >> 2, 1.25);
John Koleszar's avatar
John Koleszar committed
311
312
  q = q << 2;
  cpi->RDMULT = cpi->RDMULT << 4;
Paul Wilkins's avatar
Paul Wilkins committed
313

John Koleszar's avatar
John Koleszar committed
314
315
  if (q < 8)
    q = 8;
316

John Koleszar's avatar
John Koleszar committed
317
318
319
  if (cpi->RDMULT > 1000) {
    cpi->RDDIV = 1;
    cpi->RDMULT /= 100;
John Koleszar's avatar
John Koleszar committed
320

John Koleszar's avatar
John Koleszar committed
321
322
323
324
325
326
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < INT_MAX) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q / 100;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
327

John Koleszar's avatar
John Koleszar committed
328
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
329
    }
John Koleszar's avatar
John Koleszar committed
330
331
  } else {
    cpi->RDDIV = 100;
John Koleszar's avatar
John Koleszar committed
332

John Koleszar's avatar
John Koleszar committed
333
334
335
336
337
338
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < (INT_MAX / q)) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
339

John Koleszar's avatar
John Koleszar committed
340
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
341
    }
John Koleszar's avatar
John Koleszar committed
342
  }
John Koleszar's avatar
John Koleszar committed
343

John Koleszar's avatar
John Koleszar committed
344
  fill_token_costs(
345
    cpi->mb.token_costs[TX_4X4],
346
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs,
John Koleszar's avatar
John Koleszar committed
347
    BLOCK_TYPES);
348
349
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_4X4],
350
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11])
351
352
    cpi->common.fc.hybrid_coef_probs,
    BLOCK_TYPES);
John Koleszar's avatar
John Koleszar committed
353

John Koleszar's avatar
John Koleszar committed
354
  fill_token_costs(
355
    cpi->mb.token_costs[TX_8X8],
356
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs_8x8,
John Koleszar's avatar
John Koleszar committed
357
    BLOCK_TYPES_8X8);
358
359
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_8X8],
360
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11])
361
362
    cpi->common.fc.hybrid_coef_probs_8x8,
    BLOCK_TYPES_8X8);
363

Daniel Kang's avatar
Daniel Kang committed
364
  fill_token_costs(
365
    cpi->mb.token_costs[TX_16X16],
366
    (const vp9_prob(*)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs_16x16,
Daniel Kang's avatar
Daniel Kang committed
367
    BLOCK_TYPES_16X16);
368
369
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_16X16],
370
    (const vp9_prob(*)[8][PREV_COEF_CONTEXTS][11])
371
372
    cpi->common.fc.hybrid_coef_probs_16x16,
    BLOCK_TYPES_16X16);
Daniel Kang's avatar
Daniel Kang committed
373

John Koleszar's avatar
John Koleszar committed
374
375
  /*rough estimate for costing*/
  cpi->common.kf_ymode_probs_index = cpi->common.base_qindex >> 4;
376
  vp9_init_mode_costs(cpi);
John Koleszar's avatar
John Koleszar committed
377

378
379
  if (cpi->common.frame_type != KEY_FRAME)
  {
380
    vp9_build_nmv_cost_table(
381
382
383
384
385
386
        cpi->mb.nmvjointcost,
        cpi->mb.e_mbd.allow_high_precision_mv ?
        cpi->mb.nmvcost_hp : cpi->mb.nmvcost,
        &cpi->common.fc.nmvc,
        cpi->mb.e_mbd.allow_high_precision_mv, 1, 1);
  }
John Koleszar's avatar
John Koleszar committed
387
388
}

389
int vp9_block_error_c(short *coeff, short *dqcoeff, int block_size) {
390
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
391

392
  for (i = 0; i < block_size; i++) {
John Koleszar's avatar
John Koleszar committed
393
394
395
    int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }
John Koleszar's avatar
John Koleszar committed
396

John Koleszar's avatar
John Koleszar committed
397
  return error;
John Koleszar's avatar
John Koleszar committed
398
399
}

400
int vp9_mbblock_error_c(MACROBLOCK *mb, int dc) {
John Koleszar's avatar
John Koleszar committed
401
402
403
404
  BLOCK  *be;
  BLOCKD *bd;
  int i, j;
  int berror, error = 0;
John Koleszar's avatar
John Koleszar committed
405

John Koleszar's avatar
John Koleszar committed
406
407
408
  for (i = 0; i < 16; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
John Koleszar's avatar
John Koleszar committed
409

John Koleszar's avatar
John Koleszar committed
410
    berror = 0;
John Koleszar's avatar
John Koleszar committed
411

John Koleszar's avatar
John Koleszar committed
412
413
414
    for (j = dc; j < 16; j++) {
      int this_diff = be->coeff[j] - bd->dqcoeff[j];
      berror += this_diff * this_diff;
John Koleszar's avatar
John Koleszar committed
415
416
    }

John Koleszar's avatar
John Koleszar committed
417
418
419
420
    error += berror;
  }

  return error;
John Koleszar's avatar
John Koleszar committed
421
422
}

423
int vp9_mbuverror_c(MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
424
425
  BLOCK  *be;
  BLOCKD *bd;
John Koleszar's avatar
John Koleszar committed
426

427
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
428

John Koleszar's avatar
John Koleszar committed
429
430
431
  for (i = 16; i < 24; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
John Koleszar's avatar
John Koleszar committed
432

433
    error += vp9_block_error_c(be->coeff, bd->dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
434
  }
John Koleszar's avatar
John Koleszar committed
435

John Koleszar's avatar
John Koleszar committed
436
  return error;
John Koleszar's avatar
John Koleszar committed
437
438
}

439
int vp9_uvsse(MACROBLOCK *x) {
John Koleszar's avatar
John Koleszar committed
440
441
442
443
444
445
446
  unsigned char *uptr, *vptr;
  unsigned char *upred_ptr = (*(x->block[16].base_src) + x->block[16].src);
  unsigned char *vpred_ptr = (*(x->block[20].base_src) + x->block[20].src);
  int uv_stride = x->block[16].src_stride;

  unsigned int sse1 = 0;
  unsigned int sse2 = 0;
447
448
  int mv_row = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.row;
  int mv_col = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.col;
John Koleszar's avatar
John Koleszar committed
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
  int offset;
  int pre_stride = x->e_mbd.block[16].pre_stride;

  if (mv_row < 0)
    mv_row -= 1;
  else
    mv_row += 1;

  if (mv_col < 0)
    mv_col -= 1;
  else
    mv_col += 1;

  mv_row /= 2;
  mv_col /= 2;

  offset = (mv_row >> 3) * pre_stride + (mv_col >> 3);
  uptr = x->e_mbd.pre.u_buffer + offset;
  vptr = x->e_mbd.pre.v_buffer + offset;

  if ((mv_row | mv_col) & 7) {
470
    vp9_sub_pixel_variance8x8(uptr, pre_stride, (mv_col & 7) << 1,
471
                              (mv_row & 7) << 1, upred_ptr, uv_stride, &sse2);
472
    vp9_sub_pixel_variance8x8(vptr, pre_stride, (mv_col & 7) << 1,
473
                              (mv_row & 7) << 1, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
474
475
    sse2 += sse1;
  } else {
476
477
    vp9_variance8x8(uptr, pre_stride, upred_ptr, uv_stride, &sse2);
    vp9_variance8x8(vptr, pre_stride, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
478
479
480
    sse2 += sse1;
  }
  return sse2;
John Koleszar's avatar
John Koleszar committed
481
482
483

}

484
static int cost_coeffs_2x2(MACROBLOCK *mb,
485
                           BLOCKD *b, PLANE_TYPE type,
486
                           ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l) {
487
  int c = (type == PLANE_TYPE_Y_NO_DC); /* start at coef 0, unless Y with Y2 */
488
489
490
491
492
  int eob = b->eob;
  int pt;    /* surrounding block/prev coef predictor */
  int cost = 0;
  short *qcoeff_ptr = b->qcoeff;

493
  VP9_COMBINEENTROPYCONTEXTS(pt, *a, *l);
494
495
496
  assert(eob <= 4);

  for (; c < eob; c++) {
497
498
499
500
501
    int v = qcoeff_ptr[vp9_default_zig_zag1d[c]];
    int t = vp9_dct_value_tokens_ptr[v].Token;
    cost += mb->token_costs[TX_8X8][type][vp9_coef_bands[c]][pt][t];
    cost += vp9_dct_value_cost_ptr[v];
    pt = vp9_prev_token_class[t];
502
503
504
  }

  if (c < 4)
505
    cost += mb->token_costs[TX_8X8][type][vp9_coef_bands[c]]
506
507
508
509
510
511
512
            [pt] [DCT_EOB_TOKEN];

  pt = (c != !type); // is eob first coefficient;
  *a = *l = pt;
  return cost;
}

513
static int cost_coeffs(MACROBLOCK *mb, BLOCKD *b, PLANE_TYPE type,
Daniel Kang's avatar
Daniel Kang committed
514
                       ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l,
515
                       int tx_size) {
Daniel Kang's avatar
Daniel Kang committed
516
  const int eob = b->eob;
517
  int c = (type == PLANE_TYPE_Y_NO_DC); /* start at coef 0, unless Y with Y2 */
518
  int cost = 0, default_eob, seg_eob;
Daniel Kang's avatar
Daniel Kang committed
519
520
  int pt;                     /* surrounding block/prev coef predictor */
  int const *scan, *band;
John Koleszar's avatar
John Koleszar committed
521
  short *qcoeff_ptr = b->qcoeff;
522
523
524
  MACROBLOCKD *xd = &mb->e_mbd;
  MB_MODE_INFO *mbmi = &mb->e_mbd.mode_info_context->mbmi;
  TX_TYPE tx_type = DCT_DCT;
Paul Wilkins's avatar
Paul Wilkins committed
525
  int segment_id = mbmi->segment_id;
526

527
  switch (tx_size) {
Daniel Kang's avatar
Daniel Kang committed
528
    case TX_4X4:
529
530
      scan = vp9_default_zig_zag1d;
      band = vp9_coef_bands;
Daniel Kang's avatar
Daniel Kang committed
531
      default_eob = 16;
Deb Mukherjee's avatar
Deb Mukherjee committed
532
533
534
535
536
      if (type == PLANE_TYPE_Y_WITH_DC) {
        tx_type = get_tx_type_4x4(xd, b);
        if (tx_type != DCT_DCT) {
          switch (tx_type) {
            case ADST_DCT:
537
              scan = vp9_row_scan;
Deb Mukherjee's avatar
Deb Mukherjee committed
538
539
540
              break;

            case DCT_ADST:
541
              scan = vp9_col_scan;
Deb Mukherjee's avatar
Deb Mukherjee committed
542
543
544
              break;

            default:
545
              scan = vp9_default_zig_zag1d;
Deb Mukherjee's avatar
Deb Mukherjee committed
546
547
              break;
          }
548
        }
Daniel Kang's avatar
Daniel Kang committed
549
      }
Deb Mukherjee's avatar
Deb Mukherjee committed
550

Daniel Kang's avatar
Daniel Kang committed
551
552
      break;
    case TX_8X8:
553
554
      scan = vp9_default_zig_zag1d_8x8;
      band = vp9_coef_bands_8x8;
Daniel Kang's avatar
Daniel Kang committed
555
      default_eob = 64;
Deb Mukherjee's avatar
Deb Mukherjee committed
556
      if (type == PLANE_TYPE_Y_WITH_DC) {
557
        BLOCKD *bb;
558
        int ib = (int)(b - xd->block);
559
560
561
        if (ib < 16) {
          ib = (ib & 8) + ((ib & 4) >> 1);
          bb = xd->block + ib;
Deb Mukherjee's avatar
Deb Mukherjee committed
562
          tx_type = get_tx_type_8x8(xd, bb);
563
        }
564
      }
Daniel Kang's avatar
Daniel Kang committed
565
566
      break;
    case TX_16X16:
567
568
      scan = vp9_default_zig_zag1d_16x16;
      band = vp9_coef_bands_16x16;
Daniel Kang's avatar
Daniel Kang committed
569
      default_eob = 256;
Deb Mukherjee's avatar
Deb Mukherjee committed
570
571
572
      if (type == PLANE_TYPE_Y_WITH_DC) {
        tx_type = get_tx_type_16x16(xd, b);
      }
Daniel Kang's avatar
Daniel Kang committed
573
574
575
576
      break;
    default:
      break;
  }
577
578
  if (vp9_segfeature_active(&mb->e_mbd, segment_id, SEG_LVL_EOB))
    seg_eob = vp9_get_segdata(&mb->e_mbd, segment_id, SEG_LVL_EOB);
579
580
581
  else
    seg_eob = default_eob;

582
  VP9_COMBINEENTROPYCONTEXTS(pt, *a, *l);
583

584
585
586
  if (tx_type != DCT_DCT) {
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
587
      int t = vp9_dct_value_tokens_ptr[v].Token;
588
      cost += mb->hybrid_token_costs[tx_size][type][band[c]][pt][t];
589
590
      cost += vp9_dct_value_cost_ptr[v];
      pt = vp9_prev_token_class[t];
591
592
593
594
    }
    if (c < seg_eob)
      cost += mb->hybrid_token_costs[tx_size][type][band[c]]
          [pt][DCT_EOB_TOKEN];
595
  } else {
596
597
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
598
      int t = vp9_dct_value_tokens_ptr[v].Token;
599
      cost += mb->token_costs[tx_size][type][band[c]][pt][t];
600
601
      cost += vp9_dct_value_cost_ptr[v];
      pt = vp9_prev_token_class[t];
602
603
604
605
    }
    if (c < seg_eob)
      cost += mb->token_costs[tx_size][type][band[c]]
          [pt][DCT_EOB_TOKEN];
606
607
608
609
610
611
612
  }

  pt = (c != !type); // is eob first coefficient;
  *a = *l = pt;
  return cost;
}

613
static int rdcost_mby_4x4(MACROBLOCK *mb, int backup) {
John Koleszar's avatar
John Koleszar committed
614
615
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
616
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
617
618
619
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta;
  ENTROPY_CONTEXT *tl;
620

621
622
623
  if (backup) {
    vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
624

625
626
627
628
629
630
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)xd->above_context;
    tl = (ENTROPY_CONTEXT *)xd->left_context;
  }
John Koleszar's avatar
John Koleszar committed
631

John Koleszar's avatar
John Koleszar committed
632
  for (b = 0; b < 16; b++)
Paul Wilkins's avatar
Paul Wilkins committed
633
    cost += cost_coeffs(mb, xd->block + b, PLANE_TYPE_Y_NO_DC,
634
                        ta + vp9_block2above[b], tl + vp9_block2left[b],
Daniel Kang's avatar
Daniel Kang committed
635
                        TX_4X4);
John Koleszar's avatar
John Koleszar committed
636

Paul Wilkins's avatar
Paul Wilkins committed
637
  cost += cost_coeffs(mb, xd->block + 24, PLANE_TYPE_Y2,
638
                      ta + vp9_block2above[24], tl + vp9_block2left[24],
Daniel Kang's avatar
Daniel Kang committed
639
                      TX_4X4);
John Koleszar's avatar
John Koleszar committed
640

John Koleszar's avatar
John Koleszar committed
641
  return cost;
John Koleszar's avatar
John Koleszar committed
642
643
}

644
645
646
static void macro_block_yrd_4x4(MACROBLOCK *mb,
                                int *Rate,
                                int *Distortion,
647
                                const VP9_ENCODER_RTCD *rtcd,
648
                                int *skippable, int backup) {
John Koleszar's avatar
John Koleszar committed
649
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
650
  MACROBLOCKD *const xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
651
  BLOCK   *const mb_y2 = mb->block + 24;
Paul Wilkins's avatar
Paul Wilkins committed
652
  BLOCKD *const x_y2  = xd->block + 24;
John Koleszar's avatar
John Koleszar committed
653
654
655
  short *Y2DCPtr = mb_y2->src_diff;
  BLOCK *beptr;
  int d;
656

John Koleszar's avatar
John Koleszar committed
657
658
  // Fdct and building the 2nd order block
  for (beptr = mb->block; beptr < mb->block + 16; beptr += 2) {
659
    mb->vp9_short_fdct8x4(beptr->src_diff, beptr->coeff, 32);
John Koleszar's avatar
John Koleszar committed
660
661
662
663
664
665
666
667
668
    *Y2DCPtr++ = beptr->coeff[0];
    *Y2DCPtr++ = beptr->coeff[16];
  }

  // 2nd order fdct
  mb->short_walsh4x4(mb_y2->src_diff, mb_y2->coeff, 8);

  // Quantization
  for (b = 0; b < 16; b++) {
669
    mb->quantize_b_4x4(&mb->block[b], &xd->block[b]);
John Koleszar's avatar
John Koleszar committed
670
  }
671

John Koleszar's avatar
John Koleszar committed
672
  // DC predication and Quantization of 2nd Order block
673
  mb->quantize_b_4x4(mb_y2, x_y2);
674

John Koleszar's avatar
John Koleszar committed
675
  // Distortion
676
  d = vp9_mbblock_error(mb, 1);
677

678
  d += vp9_block_error(mb_y2->coeff, x_y2->dqcoeff, 16);
679

John Koleszar's avatar
John Koleszar committed
680
681
  *Distortion = (d >> 2);
  // rate
682
  *Rate = rdcost_mby_4x4(mb, backup);
683
  *skippable = vp9_mby_is_skippable_4x4(&mb->e_mbd, 1);
684
}
John Koleszar's avatar
John Koleszar committed
685

686
static int rdcost_mby_8x8(MACROBLOCK *mb, int backup) {
John Koleszar's avatar
John Koleszar committed
687
688
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
689
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
690
691
692
693
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta;
  ENTROPY_CONTEXT *tl;

Ronald S. Bultje's avatar
Ronald S. Bultje committed
694
695
696
  if (backup) {
    vpx_memcpy(&t_above,xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
697

Ronald S. Bultje's avatar
Ronald S. Bultje committed
698
699
700
701
702
703
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)mb->e_mbd.above_context;
    tl = (ENTROPY_CONTEXT *)mb->e_mbd.left_context;
  }
John Koleszar's avatar
John Koleszar committed
704
705

  for (b = 0; b < 16; b += 4)
Paul Wilkins's avatar
Paul Wilkins committed
706
    cost += cost_coeffs(mb, xd->block + b, PLANE_TYPE_Y_NO_DC,
707
                        ta + vp9_block2above_8x8[b], tl + vp9_block2left_8x8[b],
Daniel Kang's avatar
Daniel Kang committed
708
                        TX_8X8);
John Koleszar's avatar
John Koleszar committed
709

Paul Wilkins's avatar
Paul Wilkins committed
710
  cost += cost_coeffs_2x2(mb, xd->block + 24, PLANE_TYPE_Y2,
711
                          ta + vp9_block2above[24], tl + vp9_block2left[24]);
John Koleszar's avatar
John Koleszar committed
712
  return cost;
713
714
}

John Koleszar's avatar
John Koleszar committed
715
716
717
static void macro_block_yrd_8x8(MACROBLOCK *mb,
                                int *Rate,
                                int *Distortion,
718
                                const VP9_ENCODER_RTCD *rtcd,
719
                                int *skippable, int backup) {
Paul Wilkins's avatar
Paul Wilkins committed
720
  MACROBLOCKD *const xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
721
  BLOCK   *const mb_y2 = mb->block + 24;
Paul Wilkins's avatar
Paul Wilkins committed
722
  BLOCKD *const x_y2  = xd->block + 24;
John Koleszar's avatar
John Koleszar committed
723
724
  int d;

725
726
  vp9_transform_mby_8x8(mb);
  vp9_quantize_mby_8x8(mb);
John Koleszar's avatar
John Koleszar committed
727
728
729
730
731
732

  /* remove 1st order dc to properly combine 1st/2nd order distortion */
  mb->coeff[0] = 0;
  mb->coeff[64] = 0;
  mb->coeff[128] = 0;
  mb->coeff[192] = 0;
Paul Wilkins's avatar
Paul Wilkins committed
733
734
735
736
  xd->dqcoeff[0] = 0;
  xd->dqcoeff[64] = 0;
  xd->dqcoeff[128] = 0;
  xd->dqcoeff[192] = 0;
John Koleszar's avatar
John Koleszar committed
737

738
739
  d = vp9_mbblock_error(mb, 0);
  d += vp9_block_error(mb_y2->coeff, x_y2->dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
740
741
742

  *Distortion = (d >> 2);
  // rate
743
  *Rate = rdcost_mby_8x8(mb, backup);
744
  *skippable = vp9_mby_is_skippable_8x8(&mb->e_mbd, 1);
745
}
746

747
static int rdcost_mby_16x16(MACROBLOCK *mb, int backup) {
Daniel Kang's avatar
Daniel Kang committed
748
  int cost;
Paul Wilkins's avatar
Paul Wilkins committed
749
  MACROBLOCKD *xd = &mb->e_mbd;
Daniel Kang's avatar
Daniel Kang committed
750
751
752
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta, *tl;

753
754
755
  if (backup) {
    vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
Daniel Kang's avatar
Daniel Kang committed
756

757
758
759
760
761
762
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)xd->above_context;
    tl = (ENTROPY_CONTEXT *)xd->left_context;
  }
Daniel Kang's avatar
Daniel Kang committed
763

Paul Wilkins's avatar
Paul Wilkins committed
764
  cost = cost_coeffs(mb, xd->block, PLANE_TYPE_Y_WITH_DC, ta, tl, TX_16X16);
Daniel Kang's avatar
Daniel Kang committed
765
766
  return cost;
}
767

Daniel Kang's avatar
Daniel Kang committed
768
static void macro_block_yrd_16x16(MACROBLOCK *mb, int *Rate, int *Distortion,
769
770
                                  const VP9_ENCODER_RTCD *rtcd, int *skippable,
                                  int backup) {
Daniel Kang's avatar
Daniel Kang committed
771
  int d;
Deb Mukherjee's avatar
Deb Mukherjee committed
772
773
774
775
  MACROBLOCKD *xd = &mb->e_mbd;
  BLOCKD *b  = &mb->e_mbd.block[0];
  BLOCK  *be = &mb->block[0];
  TX_TYPE tx_type;
Daniel Kang's avatar
Daniel Kang committed
776

Deb Mukherjee's avatar
Deb Mukherjee committed
777
778
  tx_type = get_tx_type_16x16(xd, b);
  if (tx_type != DCT_DCT) {
779
    vp9_fht(be->src_diff, 32, be->coeff, tx_type, 16);
780
  } else
781
    vp9_transform_mby_16x16(mb);
782

783
  vp9_quantize_mby_16x16(mb);
784
785
786
  // TODO(jingning) is it possible to quickly determine whether to force
  //                trailing coefficients to be zero, instead of running trellis
  //                optimization in the rate-distortion optimization loop?
787
  if (mb->e_mbd.mode_info_context->mbmi.mode < I8X8_PRED)
788
    vp9_optimize_mby_16x16(mb, rtcd);
789

790
  d = vp9_mbblock_error(mb, 0);
Daniel Kang's avatar
Daniel Kang committed
791
792
793

  *Distortion = (d >> 2);
  // rate
794
  *Rate = rdcost_mby_16x16(mb, backup);
795
  *skippable = vp9_mby_is_skippable_16x16(&mb->e_mbd);
Daniel Kang's avatar
Daniel Kang committed
796
797
}

798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
static void choose_txfm_size_from_rd(VP9_COMP *cpi, MACROBLOCK *x,
                                     int r[2][TX_SIZE_MAX], int *rate,
                                     int d[TX_SIZE_MAX], int *distortion,
                                     int s[TX_SIZE_MAX], int *skip,
                                     int64_t txfm_cache[NB_TXFM_MODES]) {
  VP9_COMMON *const cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *const mbmi = &xd->mode_info_context->mbmi;
  vp9_prob skip_prob = cm->mb_no_coeff_skip ?
                       vp9_get_pred_prob(cm, xd, PRED_MBSKIP) : 128;
  int64_t rd[2][TX_SIZE_MAX];
  int n;

  r[1][TX_16X16] = r[0][TX_16X16] + vp9_cost_one(cm->prob_tx[0]) +
                   vp9_cost_one(cm->prob_tx[1]);
  r[1][TX_8X8]   = r[0][TX_8X8] + vp9_cost_one(cm->prob_tx[0]) +
                   vp9_cost_zero(cm->prob_tx[1]);
  r[1][TX_4X4]   = r[0][TX_4X4] + vp9_cost_zero(cm->prob_tx[0]);

  if (cm->mb_no_coeff_skip) {
    int s0, s1;

    assert(skip_prob > 0);
    s0 = vp9_cost_bit(skip_prob, 0);
    s1 = vp9_cost_bit(skip_prob, 1);

    for (n = TX_4X4; n <= TX_16X16; n++) {
      if (s[n]) {
        rd[0][n] = rd[1][n] = RDCOST(x->rdmult, x->rddiv, s1, d[n]);
      } else {
        rd[0][n] = RDCOST(x->rdmult, x->rddiv, r[0][n] + s0, d[n]);
        rd[1][n] = RDCOST(x->rdmult, x->rddiv, r[1][n] + s0, d[n]);
      }
831
832
    }
  } else {
833
834
835
    for (n = TX_4X4; n <= TX_16X16; n++) {
      rd[0][n] = RDCOST(x->rdmult, x->rddiv, r[0][n], d[n]);
      rd[1][n] = RDCOST(x->rdmult, x->rddiv, r[1][n], d[n]);
836
837
838
    }
  }

839
840
841
  if ( cm->txfm_mode == ALLOW_16X16 ||
      (cm->txfm_mode == TX_MODE_SELECT &&
       rd[1][TX_16X16] < rd[1][TX_8X8] && rd[1][TX_16X16] < rd[1][TX_4X4])) {
842
    mbmi->txfm_size = TX_16X16;
843
844
  } else if (cm->txfm_mode == ALLOW_8X8 ||
           (cm->txfm_mode == TX_MODE_SELECT && rd[1][TX_8X8] < rd[1][TX_4X4])) {
845
846
    mbmi->txfm_size = TX_8X8;
  } else {
847
848
    assert(cm->txfm_mode == ONLY_4X4 ||
          (cm->txfm_mode == TX_MODE_SELECT && rd[1][TX_4X4] <= rd[1][TX_8X8]));
849
850
851
    mbmi->txfm_size = TX_4X4;
  }

852
853
854
855
856
857
858
859
860
  *distortion = d[mbmi->txfm_size];
  *rate       = r[cm->txfm_mode == TX_MODE_SELECT][mbmi->txfm_size];
  *skip       = s[mbmi->txfm_size];

  txfm_cache[ONLY_4X4] = rd[0][TX_4X4];
  txfm_cache[ALLOW_8X8] = rd[0][TX_8X8];
  txfm_cache[ALLOW_16X16] = rd[0][TX_16X16];
  if (rd[1][TX_16X16] < rd[1][TX_8X8] && rd[1][TX_16X16] < rd[1][TX_4X4])
    txfm_cache[TX_MODE_SELECT] = rd[1][TX_16X16];
861
  else
862
863
864
865
866
867
868
869
870
871
872
873
874
    txfm_cache[TX_MODE_SELECT] = rd[1][TX_4X4] < rd[1][TX_8X8] ?
                                 rd[1][TX_4X4] : rd[1][TX_8X8];
}

static void macro_block_yrd(VP9_COMP *cpi, MACROBLOCK *x, int *rate,
                            int *distortion, int *skippable,
                            int64_t txfm_cache[NB_TXFM_MODES]) {
  VP9_COMMON *cm = &cpi->common;
  MACROBLOCKD *const xd = &x->e_mbd;
  int r[2][TX_SIZE_MAX], d[TX_SIZE_MAX], s[TX_SIZE_MAX];

  vp9_subtract_mby(x->src_diff, *(x->block[0].base_src), xd->predictor,
                   x->block[0].src_stride);
875

876
877
878
879
880
881
882
883
884
  macro_block_yrd_16x16(x, &r[0][TX_16X16], &d[TX_16X16],
                        IF_RTCD(&cpi->rtcd), &s[TX_16X16], 1);
  macro_block_yrd_8x8(x, &r[0][TX_8X8], &d[TX_8X8],
                      IF_RTCD(&cpi->rtcd), &s[TX_8X8], 1);
  macro_block_yrd_4x4(x, &r[0][TX_4X4], &d[TX_4X4],
                      IF_RTCD(&cpi->rtcd), &s[TX_4X4], 1);

  choose_txfm_size_from_rd(cpi, x, r, rate, d, distortion, s, skippable,
                           txfm_cache);
885
886
}

John Koleszar's avatar
John Koleszar committed
887
888
889
890
891
892
893
static void copy_predictor(unsigned char *dst, const unsigned char *predictor) {
  const unsigned int *p = (const unsigned int *)predictor;
  unsigned int *d = (unsigned int *)dst;
  d[0] = p[0];
  d[4] = p[4];
  d[8] = p[8];
  d[12] = p[12];
894
}
Yaowu Xu's avatar
Yaowu Xu committed
895

Ronald S. Bultje's avatar
Ronald S. Bultje committed
896
#if CONFIG_SUPERBLOCKS
897
898
899
900
static void super_block_yrd(VP9_COMP *cpi,
                            MACROBLOCK *x, int *rate, int *distortion,
                            const VP9_ENCODER_RTCD *rtcd, int *skip,
                            int64_t txfm_cache[NB_TXFM_MODES]) {
Ronald S. Bultje's avatar
Ronald S. Bultje committed
901
  MACROBLOCKD *const xd = &x->e_mbd;
902
  int r[2][TX_SIZE_MAX], d[TX_SIZE_MAX], s[TX_SIZE_MAX], n;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
903
904
  const uint8_t *src = x->src.y_buffer, *dst = xd->dst.y_buffer;
  int src_y_stride = x->src.y_stride, dst_y_stride = xd->dst.y_stride;
905
906
907
908
909
910
911
912
913
914
  ENTROPY_CONTEXT_PLANES t_above[3][2], *orig_above = xd->above_context;
  ENTROPY_CONTEXT_PLANES t_left[3][2], *orig_left = xd->left_context;

  for (n = TX_4X4; n <= TX_16X16; n++) {
    vpx_memcpy(t_above[n], xd->above_context, sizeof(t_above[n]));
    vpx_memcpy(t_left[n], xd->left_context, sizeof(t_left[n]));
    r[0][n] = 0;
    d[n] = 0;
    s[n] = 1;
  }
Ronald S. Bultje's avatar
Ronald S. Bultje committed
915
916
917

  for (n = 0; n < 4; n++) {
    int x_idx = n & 1, y_idx = n >> 1;
918
    int r_tmp, d_tmp, s_tmp;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
919

920
    vp9_subtract_mby_s_c(x->src_diff,
Ronald S. Bultje's avatar
Ronald S. Bultje committed
921
922
923
924
                         src + x_idx * 16 + y_idx * 16 * src_y_stride,
                         src_y_stride,
                         dst + x_idx * 16 + y_idx * 16 * dst_y_stride,
                         dst_y_stride);
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945

    xd->above_context = &t_above[TX_16X16][x_idx];
    xd->left_context = &t_left[TX_16X16][y_idx];
    macro_block_yrd_16x16(x, &r_tmp, &d_tmp, IF_RTCD(&cpi->rtcd), &s_tmp, 0);
    d[TX_16X16] += d_tmp;
    r[0][TX_16X16] += r_tmp;
    s[TX_16X16] = s[TX_16X16] && s_tmp;

    xd->above_context = &t_above[TX_4X4][x_idx];
    xd->left_context = &t_left[TX_4X4][y_idx];
    macro_block_yrd_4x4(x, &r_tmp, &d_tmp, IF_RTCD(&cpi->rtcd), &s_tmp, 0);
    d[TX_4X4] += d_tmp;
    r[0][TX_4X4] += r_tmp;
    s[TX_4X4] = s[TX_4X4] && s_tmp;

    xd->above_context = &t_above[TX_8X8][x_idx];
    xd->left_context = &t_left[TX_8X8][y_idx];
    macro_block_yrd_8x8(x, &r_tmp, &d_tmp, IF_RTCD(&cpi->rtcd), &s_tmp, 0);
    d[TX_8X8] += d_tmp;
    r[0][TX_8X8] += r_tmp;
    s[TX_8X8] = s[TX_8X8] && s_tmp;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
946
947
  }

948
949
950
951
  choose_txfm_size_from_rd(cpi, x, r, rate, d, distortion, s, skip, txfm_cache);

  xd->above_context = orig_above;
  xd->left_context = orig_left;
Ronald S. Bultje's avatar
Ronald S. Bultje committed
952
953
954
}
#endif

John Koleszar's avatar
John Koleszar committed
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
static void copy_predictor_8x8(unsigned char *dst, const unsigned char *predictor) {
  const unsigned int *p = (const unsigned int *)predictor;
  unsigned int *d = (unsigned int *)dst;
  d[0] = p[0];
  d[1] = p[1];
  d[4] = p[4];
  d[5] = p[5];
  d[8] = p[8];
  d[9] = p[9];
  d[12] = p