rdopt.c 164 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5
6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9
10
11
12
13
14
15
 */


#include <stdio.h>
#include <math.h>
#include <limits.h>
#include <assert.h>
16
#include "vp9/common/pragmas.h"
John Koleszar's avatar
John Koleszar committed
17
18
19
20
21
22

#include "tokenize.h"
#include "treewriter.h"
#include "onyx_int.h"
#include "modecosts.h"
#include "encodeintra.h"
23
24
25
26
27
28
#include "vp9/common/entropymode.h"
#include "vp9/common/reconinter.h"
#include "vp9/common/reconintra.h"
#include "vp9/common/reconintra4x4.h"
#include "vp9/common/findnearmv.h"
#include "vp9/common/quant_common.h"
John Koleszar's avatar
John Koleszar committed
29
30
#include "encodemb.h"
#include "quantize.h"
31
#include "vp9/common/idct.h"
John Koleszar's avatar
John Koleszar committed
32
33
#include "variance.h"
#include "mcomp.h"
Yunqing Wang's avatar
Yunqing Wang committed
34
#include "rdopt.h"
Paul Wilkins's avatar
Paul Wilkins committed
35
#include "ratectrl.h"
John Koleszar's avatar
John Koleszar committed
36
#include "vpx_mem/vpx_mem.h"
37
38
#include "vp9/common/systemdependent.h"
#include "vp9/encoder/encodemv.h"
John Koleszar's avatar
John Koleszar committed
39

40
41
42
#include "vp9/common/seg_common.h"
#include "vp9/common/pred_common.h"
#include "vp9/common/entropy.h"
Jim Bankoski's avatar
Jim Bankoski committed
43
#include "vpx_rtcd.h"
44
#include "vp9/common/mvref_common.h"
Paul Wilkins's avatar
Paul Wilkins committed
45

John Koleszar's avatar
John Koleszar committed
46
47
48
49
50
51
52
53
#if CONFIG_RUNTIME_CPU_DETECT
#define IF_RTCD(x)  (x)
#else
#define IF_RTCD(x)  NULL
#endif

#define MAXF(a,b)            (((a) > (b)) ? (a) : (b))

54
55
#define INVALID_MV 0x80008000

56
57
58
/* Factor to weigh the rate for switchable interp filters */
#define SWITCHABLE_INTERP_RATE_FACTOR 1

John Koleszar's avatar
John Koleszar committed
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
static const int auto_speed_thresh[17] = {
  1000,
  200,
  150,
  130,
  150,
  125,
  120,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  115,
  105
John Koleszar's avatar
John Koleszar committed
77
78
};

79
#if CONFIG_PRED_FILTER
80
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
John Koleszar's avatar
John Koleszar committed
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
  {ZEROMV,    LAST_FRAME,   0,  0},
  {ZEROMV,    LAST_FRAME,   0,  1},
  {DC_PRED,   INTRA_FRAME,  0,  0},

  {NEARESTMV, LAST_FRAME,   0,  0},
  {NEARESTMV, LAST_FRAME,   0,  1},
  {NEARMV,    LAST_FRAME,   0,  0},
  {NEARMV,    LAST_FRAME,   0,  1},

  {ZEROMV,    GOLDEN_FRAME, 0,  0},
  {ZEROMV,    GOLDEN_FRAME, 0,  1},
  {NEARESTMV, GOLDEN_FRAME, 0,  0},
  {NEARESTMV, GOLDEN_FRAME, 0,  1},

  {ZEROMV,    ALTREF_FRAME, 0,  0},
  {ZEROMV,    ALTREF_FRAME, 0,  1},
  {NEARESTMV, ALTREF_FRAME, 0,  0},
  {NEARESTMV, ALTREF_FRAME, 0,  1},

  {NEARMV,    GOLDEN_FRAME, 0,  0},
  {NEARMV,    GOLDEN_FRAME, 0,  1},
  {NEARMV,    ALTREF_FRAME, 0,  0},
  {NEARMV,    ALTREF_FRAME, 0,  1},

  {V_PRED,    INTRA_FRAME,  0,  0},
  {H_PRED,    INTRA_FRAME,  0,  0},
  {D45_PRED,  INTRA_FRAME,  0,  0},
  {D135_PRED, INTRA_FRAME,  0,  0},
  {D117_PRED, INTRA_FRAME,  0,  0},
  {D153_PRED, INTRA_FRAME,  0,  0},
  {D27_PRED,  INTRA_FRAME,  0,  0},
  {D63_PRED,  INTRA_FRAME,  0,  0},

  {TM_PRED,   INTRA_FRAME,  0,  0},

  {NEWMV,     LAST_FRAME,   0,  0},
  {NEWMV,     LAST_FRAME,   0,  1},
  {NEWMV,     GOLDEN_FRAME, 0,  0},
  {NEWMV,     GOLDEN_FRAME, 0,  1},
  {NEWMV,     ALTREF_FRAME, 0,  0},
  {NEWMV,     ALTREF_FRAME, 0,  1},

  {SPLITMV,   LAST_FRAME,   0,  0},
  {SPLITMV,   GOLDEN_FRAME, 0,  0},
  {SPLITMV,   ALTREF_FRAME, 0,  0},

  {B_PRED,    INTRA_FRAME,  0,  0},
  {I8X8_PRED, INTRA_FRAME,  0,  0},

  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME, 0},

  {ZEROMV,    ALTREF_FRAME, LAST_FRAME,   0},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME,   0},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME,   0},

  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME, 0},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME, 0},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME, 0},

  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME, 0},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME,   0},
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME, 0},

  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME, 0},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME,   0},
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME, 0}
John Koleszar's avatar
John Koleszar committed
150
};
151
#else
152
const MODE_DEFINITION vp9_mode_order[MAX_MODES] = {
John Koleszar's avatar
John Koleszar committed
153
154
  {ZEROMV,    LAST_FRAME,   0},
  {DC_PRED,   INTRA_FRAME,  0},
John Koleszar's avatar
John Koleszar committed
155

John Koleszar's avatar
John Koleszar committed
156
157
  {NEARESTMV, LAST_FRAME,   0},
  {NEARMV,    LAST_FRAME,   0},
John Koleszar's avatar
John Koleszar committed
158

John Koleszar's avatar
John Koleszar committed
159
160
  {ZEROMV,    GOLDEN_FRAME, 0},
  {NEARESTMV, GOLDEN_FRAME, 0},
John Koleszar's avatar
John Koleszar committed
161

John Koleszar's avatar
John Koleszar committed
162
163
  {ZEROMV,    ALTREF_FRAME, 0},
  {NEARESTMV, ALTREF_FRAME, 0},
John Koleszar's avatar
John Koleszar committed
164

John Koleszar's avatar
John Koleszar committed
165
166
  {NEARMV,    GOLDEN_FRAME, 0},
  {NEARMV,    ALTREF_FRAME, 0},
John Koleszar's avatar
John Koleszar committed
167

John Koleszar's avatar
John Koleszar committed
168
169
170
171
172
173
174
175
  {V_PRED,    INTRA_FRAME,  0},
  {H_PRED,    INTRA_FRAME,  0},
  {D45_PRED,  INTRA_FRAME,  0},
  {D135_PRED, INTRA_FRAME,  0},
  {D117_PRED, INTRA_FRAME,  0},
  {D153_PRED, INTRA_FRAME,  0},
  {D27_PRED,  INTRA_FRAME,  0},
  {D63_PRED,  INTRA_FRAME,  0},
John Koleszar's avatar
John Koleszar committed
176

John Koleszar's avatar
John Koleszar committed
177
  {TM_PRED,   INTRA_FRAME,  0},
John Koleszar's avatar
John Koleszar committed
178

John Koleszar's avatar
John Koleszar committed
179
180
181
  {NEWMV,     LAST_FRAME,   0},
  {NEWMV,     GOLDEN_FRAME, 0},
  {NEWMV,     ALTREF_FRAME, 0},
John Koleszar's avatar
John Koleszar committed
182

John Koleszar's avatar
John Koleszar committed
183
184
185
  {SPLITMV,   LAST_FRAME,   0},
  {SPLITMV,   GOLDEN_FRAME, 0},
  {SPLITMV,   ALTREF_FRAME, 0},
186

John Koleszar's avatar
John Koleszar committed
187
188
  {B_PRED,    INTRA_FRAME,  0},
  {I8X8_PRED, INTRA_FRAME,  0},
189

John Koleszar's avatar
John Koleszar committed
190
191
192
193
  /* compound prediction modes */
  {ZEROMV,    LAST_FRAME,   GOLDEN_FRAME},
  {NEARESTMV, LAST_FRAME,   GOLDEN_FRAME},
  {NEARMV,    LAST_FRAME,   GOLDEN_FRAME},
194

John Koleszar's avatar
John Koleszar committed
195
196
197
  {ZEROMV,    ALTREF_FRAME, LAST_FRAME},
  {NEARESTMV, ALTREF_FRAME, LAST_FRAME},
  {NEARMV,    ALTREF_FRAME, LAST_FRAME},
198

John Koleszar's avatar
John Koleszar committed
199
200
201
  {ZEROMV,    GOLDEN_FRAME, ALTREF_FRAME},
  {NEARESTMV, GOLDEN_FRAME, ALTREF_FRAME},
  {NEARMV,    GOLDEN_FRAME, ALTREF_FRAME},
202

John Koleszar's avatar
John Koleszar committed
203
204
205
  {NEWMV,     LAST_FRAME,   GOLDEN_FRAME},
  {NEWMV,     ALTREF_FRAME, LAST_FRAME  },
  {NEWMV,     GOLDEN_FRAME, ALTREF_FRAME},
206

John Koleszar's avatar
John Koleszar committed
207
208
209
  {SPLITMV,   LAST_FRAME,   GOLDEN_FRAME},
  {SPLITMV,   ALTREF_FRAME, LAST_FRAME  },
  {SPLITMV,   GOLDEN_FRAME, ALTREF_FRAME}
John Koleszar's avatar
John Koleszar committed
210
};
211
#endif
John Koleszar's avatar
John Koleszar committed
212
213

static void fill_token_costs(
214
  unsigned int (*c)[COEF_BANDS][PREV_COEF_CONTEXTS][MAX_ENTROPY_TOKENS],
215
  const vp9_prob(*p)[COEF_BANDS][PREV_COEF_CONTEXTS][ENTROPY_NODES],
John Koleszar's avatar
John Koleszar committed
216
217
218
219
220
221
222
  int block_type_counts) {
  int i, j, k;

  for (i = 0; i < block_type_counts; i++)
    for (j = 0; j < COEF_BANDS; j++)
      for (k = 0; k < PREV_COEF_CONTEXTS; k++) {
        if (k == 0 && ((j > 0 && i > 0) || (j > 1 && i == 0)))
223
224
          vp9_cost_tokens_skip((int *)(c[i][j][k]),
                               p[i][j][k],
225
                               vp9_coef_tree);
John Koleszar's avatar
John Koleszar committed
226
        else
227
228
          vp9_cost_tokens((int *)(c[i][j][k]),
                          p[i][j][k],
229
                          vp9_coef_tree);
John Koleszar's avatar
John Koleszar committed
230
      }
John Koleszar's avatar
John Koleszar committed
231
232
}

233

234
235
236
237
static int rd_iifactor[32] =  { 4, 4, 3, 2, 1, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0,
                                0, 0, 0, 0, 0, 0, 0, 0, };
John Koleszar's avatar
John Koleszar committed
238

239
// 3* dc_qlookup[Q]*dc_qlookup[Q];
240

241
/* values are now correlated to quantizer */
Paul Wilkins's avatar
Paul Wilkins committed
242
243
244
static int sad_per_bit16lut[QINDEX_RANGE];
static int sad_per_bit4lut[QINDEX_RANGE];

245
void vp9_init_me_luts() {
John Koleszar's avatar
John Koleszar committed
246
247
248
249
250
251
252
  int i;

  // Initialize the sad lut tables using a formulaic calculation for now
  // This is to make it easier to resolve the impact of experimental changes
  // to the quantizer tables.
  for (i = 0; i < QINDEX_RANGE; i++) {
    sad_per_bit16lut[i] =
253
254
      (int)((0.0418 * vp9_convert_qindex_to_q(i)) + 2.4107);
    sad_per_bit4lut[i] = (int)((0.063 * vp9_convert_qindex_to_q(i)) + 2.742);
John Koleszar's avatar
John Koleszar committed
255
  }
Paul Wilkins's avatar
Paul Wilkins committed
256
}
John Koleszar's avatar
John Koleszar committed
257

258
static int compute_rd_mult(int qindex) {
John Koleszar's avatar
John Koleszar committed
259
  int q;
260

261
  q = vp9_dc_quant(qindex, 0);
John Koleszar's avatar
John Koleszar committed
262
  return (11 * q * q) >> 6;
263
264
}

265
void vp9_initialize_me_consts(VP9_COMP *cpi, int QIndex) {
John Koleszar's avatar
John Koleszar committed
266
267
  cpi->mb.sadperbit16 =  sad_per_bit16lut[QIndex];
  cpi->mb.sadperbit4  =  sad_per_bit4lut[QIndex];
John Koleszar's avatar
John Koleszar committed
268
269
}

270

271
void vp9_initialize_rd_consts(VP9_COMP *cpi, int QIndex) {
272
  int q, i;
John Koleszar's avatar
John Koleszar committed
273

274
  vp9_clear_system_state();  // __asm emms;
John Koleszar's avatar
John Koleszar committed
275

John Koleszar's avatar
John Koleszar committed
276
277
278
279
280
  // Further tests required to see if optimum is different
  // for key frames, golden frames and arf frames.
  // if (cpi->common.refresh_golden_frame ||
  //     cpi->common.refresh_alt_ref_frame)
  QIndex = (QIndex < 0) ? 0 : ((QIndex > MAXQ) ? MAXQ : QIndex);
281

John Koleszar's avatar
John Koleszar committed
282
  cpi->RDMULT = compute_rd_mult(QIndex);
John Koleszar's avatar
John Koleszar committed
283

John Koleszar's avatar
John Koleszar committed
284
285
286
  // Extend rate multiplier along side quantizer zbin increases
  if (cpi->zbin_over_quant  > 0) {
    double oq_factor;
287

John Koleszar's avatar
John Koleszar committed
288
289
290
291
292
    // Experimental code using the same basic equation as used for Q above
    // The units of cpi->zbin_over_quant are 1/128 of Q bin size
    oq_factor = 1.0 + ((double)0.0015625 * cpi->zbin_over_quant);
    cpi->RDMULT = (int)((double)cpi->RDMULT * oq_factor * oq_factor);
  }
John Koleszar's avatar
John Koleszar committed
293

John Koleszar's avatar
John Koleszar committed
294
295
296
297
298
299
300
  if (cpi->pass == 2 && (cpi->common.frame_type != KEY_FRAME)) {
    if (cpi->twopass.next_iiratio > 31)
      cpi->RDMULT += (cpi->RDMULT * rd_iifactor[31]) >> 4;
    else
      cpi->RDMULT +=
        (cpi->RDMULT * rd_iifactor[cpi->twopass.next_iiratio]) >> 4;
  }
John Koleszar's avatar
John Koleszar committed
301

John Koleszar's avatar
John Koleszar committed
302
303
  if (cpi->RDMULT < 7)
    cpi->RDMULT = 7;
304

John Koleszar's avatar
John Koleszar committed
305
306
  cpi->mb.errorperbit = (cpi->RDMULT / 110);
  cpi->mb.errorperbit += (cpi->mb.errorperbit == 0);
307

308
  vp9_set_speed_features(cpi);
John Koleszar's avatar
John Koleszar committed
309

310
  q = (int)pow(vp9_dc_quant(QIndex, 0) >> 2, 1.25);
John Koleszar's avatar
John Koleszar committed
311
312
  q = q << 2;
  cpi->RDMULT = cpi->RDMULT << 4;
Paul Wilkins's avatar
Paul Wilkins committed
313

John Koleszar's avatar
John Koleszar committed
314
315
  if (q < 8)
    q = 8;
316

John Koleszar's avatar
John Koleszar committed
317
318
319
  if (cpi->RDMULT > 1000) {
    cpi->RDDIV = 1;
    cpi->RDMULT /= 100;
John Koleszar's avatar
John Koleszar committed
320

John Koleszar's avatar
John Koleszar committed
321
322
323
324
325
326
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < INT_MAX) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q / 100;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
327

John Koleszar's avatar
John Koleszar committed
328
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
329
    }
John Koleszar's avatar
John Koleszar committed
330
331
  } else {
    cpi->RDDIV = 100;
John Koleszar's avatar
John Koleszar committed
332

John Koleszar's avatar
John Koleszar committed
333
334
335
336
337
338
    for (i = 0; i < MAX_MODES; i++) {
      if (cpi->sf.thresh_mult[i] < (INT_MAX / q)) {
        cpi->rd_threshes[i] = cpi->sf.thresh_mult[i] * q;
      } else {
        cpi->rd_threshes[i] = INT_MAX;
      }
John Koleszar's avatar
John Koleszar committed
339

John Koleszar's avatar
John Koleszar committed
340
      cpi->rd_baseline_thresh[i] = cpi->rd_threshes[i];
John Koleszar's avatar
John Koleszar committed
341
    }
John Koleszar's avatar
John Koleszar committed
342
  }
John Koleszar's avatar
John Koleszar committed
343

John Koleszar's avatar
John Koleszar committed
344
  fill_token_costs(
345
    cpi->mb.token_costs[TX_4X4],
346
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs,
John Koleszar's avatar
John Koleszar committed
347
    BLOCK_TYPES);
348
349
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_4X4],
350
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11])
351
352
    cpi->common.fc.hybrid_coef_probs,
    BLOCK_TYPES);
John Koleszar's avatar
John Koleszar committed
353

John Koleszar's avatar
John Koleszar committed
354
  fill_token_costs(
355
    cpi->mb.token_costs[TX_8X8],
356
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs_8x8,
John Koleszar's avatar
John Koleszar committed
357
    BLOCK_TYPES_8X8);
358
359
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_8X8],
360
    (const vp9_prob( *)[8][PREV_COEF_CONTEXTS][11])
361
362
    cpi->common.fc.hybrid_coef_probs_8x8,
    BLOCK_TYPES_8X8);
363

Daniel Kang's avatar
Daniel Kang committed
364
  fill_token_costs(
365
    cpi->mb.token_costs[TX_16X16],
366
    (const vp9_prob(*)[8][PREV_COEF_CONTEXTS][11]) cpi->common.fc.coef_probs_16x16,
Daniel Kang's avatar
Daniel Kang committed
367
    BLOCK_TYPES_16X16);
368
369
  fill_token_costs(
    cpi->mb.hybrid_token_costs[TX_16X16],
370
    (const vp9_prob(*)[8][PREV_COEF_CONTEXTS][11])
371
372
    cpi->common.fc.hybrid_coef_probs_16x16,
    BLOCK_TYPES_16X16);
Daniel Kang's avatar
Daniel Kang committed
373

John Koleszar's avatar
John Koleszar committed
374
375
  /*rough estimate for costing*/
  cpi->common.kf_ymode_probs_index = cpi->common.base_qindex >> 4;
376
  vp9_init_mode_costs(cpi);
John Koleszar's avatar
John Koleszar committed
377

378
379
  if (cpi->common.frame_type != KEY_FRAME)
  {
380
    vp9_build_nmv_cost_table(
381
382
383
384
385
386
        cpi->mb.nmvjointcost,
        cpi->mb.e_mbd.allow_high_precision_mv ?
        cpi->mb.nmvcost_hp : cpi->mb.nmvcost,
        &cpi->common.fc.nmvc,
        cpi->mb.e_mbd.allow_high_precision_mv, 1, 1);
  }
John Koleszar's avatar
John Koleszar committed
387
388
}

389
int vp9_block_error_c(short *coeff, short *dqcoeff, int block_size) {
390
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
391

392
  for (i = 0; i < block_size; i++) {
John Koleszar's avatar
John Koleszar committed
393
394
395
    int this_diff = coeff[i] - dqcoeff[i];
    error += this_diff * this_diff;
  }
John Koleszar's avatar
John Koleszar committed
396

John Koleszar's avatar
John Koleszar committed
397
  return error;
John Koleszar's avatar
John Koleszar committed
398
399
}

400
int vp9_mbblock_error_c(MACROBLOCK *mb, int dc) {
John Koleszar's avatar
John Koleszar committed
401
402
403
404
  BLOCK  *be;
  BLOCKD *bd;
  int i, j;
  int berror, error = 0;
John Koleszar's avatar
John Koleszar committed
405

John Koleszar's avatar
John Koleszar committed
406
407
408
  for (i = 0; i < 16; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
John Koleszar's avatar
John Koleszar committed
409

John Koleszar's avatar
John Koleszar committed
410
    berror = 0;
John Koleszar's avatar
John Koleszar committed
411

John Koleszar's avatar
John Koleszar committed
412
413
414
    for (j = dc; j < 16; j++) {
      int this_diff = be->coeff[j] - bd->dqcoeff[j];
      berror += this_diff * this_diff;
John Koleszar's avatar
John Koleszar committed
415
416
    }

John Koleszar's avatar
John Koleszar committed
417
418
419
420
    error += berror;
  }

  return error;
John Koleszar's avatar
John Koleszar committed
421
422
}

423
int vp9_mbuverror_c(MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
424
425
  BLOCK  *be;
  BLOCKD *bd;
John Koleszar's avatar
John Koleszar committed
426

427
  int i, error = 0;
John Koleszar's avatar
John Koleszar committed
428

John Koleszar's avatar
John Koleszar committed
429
430
431
  for (i = 16; i < 24; i++) {
    be = &mb->block[i];
    bd = &mb->e_mbd.block[i];
John Koleszar's avatar
John Koleszar committed
432

433
    error += vp9_block_error_c(be->coeff, bd->dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
434
  }
John Koleszar's avatar
John Koleszar committed
435

John Koleszar's avatar
John Koleszar committed
436
  return error;
John Koleszar's avatar
John Koleszar committed
437
438
}

439
int vp9_uvsse(MACROBLOCK *x) {
John Koleszar's avatar
John Koleszar committed
440
441
442
443
444
445
446
  unsigned char *uptr, *vptr;
  unsigned char *upred_ptr = (*(x->block[16].base_src) + x->block[16].src);
  unsigned char *vpred_ptr = (*(x->block[20].base_src) + x->block[20].src);
  int uv_stride = x->block[16].src_stride;

  unsigned int sse1 = 0;
  unsigned int sse2 = 0;
447
448
  int mv_row = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.row;
  int mv_col = x->e_mbd.mode_info_context->mbmi.mv[0].as_mv.col;
John Koleszar's avatar
John Koleszar committed
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
  int offset;
  int pre_stride = x->e_mbd.block[16].pre_stride;

  if (mv_row < 0)
    mv_row -= 1;
  else
    mv_row += 1;

  if (mv_col < 0)
    mv_col -= 1;
  else
    mv_col += 1;

  mv_row /= 2;
  mv_col /= 2;

  offset = (mv_row >> 3) * pre_stride + (mv_col >> 3);
  uptr = x->e_mbd.pre.u_buffer + offset;
  vptr = x->e_mbd.pre.v_buffer + offset;

  if ((mv_row | mv_col) & 7) {
470
    vp9_sub_pixel_variance8x8(uptr, pre_stride, (mv_col & 7) << 1,
471
                              (mv_row & 7) << 1, upred_ptr, uv_stride, &sse2);
472
    vp9_sub_pixel_variance8x8(vptr, pre_stride, (mv_col & 7) << 1,
473
                              (mv_row & 7) << 1, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
474
475
    sse2 += sse1;
  } else {
476
477
    vp9_variance8x8(uptr, pre_stride, upred_ptr, uv_stride, &sse2);
    vp9_variance8x8(vptr, pre_stride, vpred_ptr, uv_stride, &sse1);
John Koleszar's avatar
John Koleszar committed
478
479
480
    sse2 += sse1;
  }
  return sse2;
John Koleszar's avatar
John Koleszar committed
481
482
483

}

484
static int cost_coeffs_2x2(MACROBLOCK *mb,
485
                           BLOCKD *b, PLANE_TYPE type,
486
                           ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l) {
487
  int c = (type == PLANE_TYPE_Y_NO_DC); /* start at coef 0, unless Y with Y2 */
488
489
490
491
492
  int eob = b->eob;
  int pt;    /* surrounding block/prev coef predictor */
  int cost = 0;
  short *qcoeff_ptr = b->qcoeff;

493
  VP9_COMBINEENTROPYCONTEXTS(pt, *a, *l);
494
495
496
  assert(eob <= 4);

  for (; c < eob; c++) {
497
498
499
500
501
    int v = qcoeff_ptr[vp9_default_zig_zag1d[c]];
    int t = vp9_dct_value_tokens_ptr[v].Token;
    cost += mb->token_costs[TX_8X8][type][vp9_coef_bands[c]][pt][t];
    cost += vp9_dct_value_cost_ptr[v];
    pt = vp9_prev_token_class[t];
502
503
504
  }

  if (c < 4)
505
    cost += mb->token_costs[TX_8X8][type][vp9_coef_bands[c]]
506
507
508
509
510
511
512
            [pt] [DCT_EOB_TOKEN];

  pt = (c != !type); // is eob first coefficient;
  *a = *l = pt;
  return cost;
}

513
static int cost_coeffs(MACROBLOCK *mb, BLOCKD *b, PLANE_TYPE type,
Daniel Kang's avatar
Daniel Kang committed
514
                       ENTROPY_CONTEXT *a, ENTROPY_CONTEXT *l,
515
                       int tx_size) {
Daniel Kang's avatar
Daniel Kang committed
516
  const int eob = b->eob;
517
  int c = (type == PLANE_TYPE_Y_NO_DC); /* start at coef 0, unless Y with Y2 */
518
  int cost = 0, default_eob, seg_eob;
Daniel Kang's avatar
Daniel Kang committed
519
520
  int pt;                     /* surrounding block/prev coef predictor */
  int const *scan, *band;
John Koleszar's avatar
John Koleszar committed
521
  short *qcoeff_ptr = b->qcoeff;
522
523
524
  MACROBLOCKD *xd = &mb->e_mbd;
  MB_MODE_INFO *mbmi = &mb->e_mbd.mode_info_context->mbmi;
  TX_TYPE tx_type = DCT_DCT;
Paul Wilkins's avatar
Paul Wilkins committed
525
  int segment_id = mbmi->segment_id;
526

527
  switch (tx_size) {
Daniel Kang's avatar
Daniel Kang committed
528
    case TX_4X4:
529
530
      scan = vp9_default_zig_zag1d;
      band = vp9_coef_bands;
Daniel Kang's avatar
Daniel Kang committed
531
      default_eob = 16;
Deb Mukherjee's avatar
Deb Mukherjee committed
532
533
534
535
536
      if (type == PLANE_TYPE_Y_WITH_DC) {
        tx_type = get_tx_type_4x4(xd, b);
        if (tx_type != DCT_DCT) {
          switch (tx_type) {
            case ADST_DCT:
537
              scan = vp9_row_scan;
Deb Mukherjee's avatar
Deb Mukherjee committed
538
539
540
              break;

            case DCT_ADST:
541
              scan = vp9_col_scan;
Deb Mukherjee's avatar
Deb Mukherjee committed
542
543
544
              break;

            default:
545
              scan = vp9_default_zig_zag1d;
Deb Mukherjee's avatar
Deb Mukherjee committed
546
547
              break;
          }
548
        }
Daniel Kang's avatar
Daniel Kang committed
549
      }
Deb Mukherjee's avatar
Deb Mukherjee committed
550

Daniel Kang's avatar
Daniel Kang committed
551
552
      break;
    case TX_8X8:
553
554
      scan = vp9_default_zig_zag1d_8x8;
      band = vp9_coef_bands_8x8;
Daniel Kang's avatar
Daniel Kang committed
555
      default_eob = 64;
Deb Mukherjee's avatar
Deb Mukherjee committed
556
      if (type == PLANE_TYPE_Y_WITH_DC) {
557
        BLOCKD *bb;
558
        int ib = (int)(b - xd->block);
559
560
561
        if (ib < 16) {
          ib = (ib & 8) + ((ib & 4) >> 1);
          bb = xd->block + ib;
Deb Mukherjee's avatar
Deb Mukherjee committed
562
          tx_type = get_tx_type_8x8(xd, bb);
563
        }
564
      }
Daniel Kang's avatar
Daniel Kang committed
565
566
      break;
    case TX_16X16:
567
568
      scan = vp9_default_zig_zag1d_16x16;
      band = vp9_coef_bands_16x16;
Daniel Kang's avatar
Daniel Kang committed
569
      default_eob = 256;
Deb Mukherjee's avatar
Deb Mukherjee committed
570
571
572
      if (type == PLANE_TYPE_Y_WITH_DC) {
        tx_type = get_tx_type_16x16(xd, b);
      }
Daniel Kang's avatar
Daniel Kang committed
573
574
575
576
      break;
    default:
      break;
  }
577
578
  if (vp9_segfeature_active(&mb->e_mbd, segment_id, SEG_LVL_EOB))
    seg_eob = vp9_get_segdata(&mb->e_mbd, segment_id, SEG_LVL_EOB);
579
580
581
  else
    seg_eob = default_eob;

582
  VP9_COMBINEENTROPYCONTEXTS(pt, *a, *l);
583

584
585
586
  if (tx_type != DCT_DCT) {
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
587
      int t = vp9_dct_value_tokens_ptr[v].Token;
588
      cost += mb->hybrid_token_costs[tx_size][type][band[c]][pt][t];
589
590
      cost += vp9_dct_value_cost_ptr[v];
      pt = vp9_prev_token_class[t];
591
592
593
594
    }
    if (c < seg_eob)
      cost += mb->hybrid_token_costs[tx_size][type][band[c]]
          [pt][DCT_EOB_TOKEN];
595
  } else {
596
597
    for (; c < eob; c++) {
      int v = qcoeff_ptr[scan[c]];
598
      int t = vp9_dct_value_tokens_ptr[v].Token;
599
      cost += mb->token_costs[tx_size][type][band[c]][pt][t];
600
601
      cost += vp9_dct_value_cost_ptr[v];
      pt = vp9_prev_token_class[t];
602
603
604
605
    }
    if (c < seg_eob)
      cost += mb->token_costs[tx_size][type][band[c]]
          [pt][DCT_EOB_TOKEN];
606
607
608
609
610
611
612
  }

  pt = (c != !type); // is eob first coefficient;
  *a = *l = pt;
  return cost;
}

613
static int rdcost_mby_4x4(MACROBLOCK *mb) {
John Koleszar's avatar
John Koleszar committed
614
615
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
616
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
617
618
619
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta;
  ENTROPY_CONTEXT *tl;
620

Paul Wilkins's avatar
Paul Wilkins committed
621
622
  vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
  vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
623

John Koleszar's avatar
John Koleszar committed
624
625
  ta = (ENTROPY_CONTEXT *)&t_above;
  tl = (ENTROPY_CONTEXT *)&t_left;
John Koleszar's avatar
John Koleszar committed
626

John Koleszar's avatar
John Koleszar committed
627
  for (b = 0; b < 16; b++)
Paul Wilkins's avatar
Paul Wilkins committed
628
    cost += cost_coeffs(mb, xd->block + b, PLANE_TYPE_Y_NO_DC,
629
                        ta + vp9_block2above[b], tl + vp9_block2left[b],
Daniel Kang's avatar
Daniel Kang committed
630
                        TX_4X4);
John Koleszar's avatar
John Koleszar committed
631

Paul Wilkins's avatar
Paul Wilkins committed
632
  cost += cost_coeffs(mb, xd->block + 24, PLANE_TYPE_Y2,
633
                      ta + vp9_block2above[24], tl + vp9_block2left[24],
Daniel Kang's avatar
Daniel Kang committed
634
                      TX_4X4);
John Koleszar's avatar
John Koleszar committed
635

John Koleszar's avatar
John Koleszar committed
636
  return cost;
John Koleszar's avatar
John Koleszar committed
637
638
}

639
640
641
static void macro_block_yrd_4x4(MACROBLOCK *mb,
                                int *Rate,
                                int *Distortion,
642
                                const VP9_ENCODER_RTCD *rtcd,
643
                                int *skippable) {
John Koleszar's avatar
John Koleszar committed
644
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
645
  MACROBLOCKD *const xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
646
  BLOCK   *const mb_y2 = mb->block + 24;
Paul Wilkins's avatar
Paul Wilkins committed
647
  BLOCKD *const x_y2  = xd->block + 24;
John Koleszar's avatar
John Koleszar committed
648
649
650
  short *Y2DCPtr = mb_y2->src_diff;
  BLOCK *beptr;
  int d;
651

John Koleszar's avatar
John Koleszar committed
652
653
  // Fdct and building the 2nd order block
  for (beptr = mb->block; beptr < mb->block + 16; beptr += 2) {
654
    mb->vp9_short_fdct8x4(beptr->src_diff, beptr->coeff, 32);
John Koleszar's avatar
John Koleszar committed
655
656
657
658
659
660
661
662
663
    *Y2DCPtr++ = beptr->coeff[0];
    *Y2DCPtr++ = beptr->coeff[16];
  }

  // 2nd order fdct
  mb->short_walsh4x4(mb_y2->src_diff, mb_y2->coeff, 8);

  // Quantization
  for (b = 0; b < 16; b++) {
664
    mb->quantize_b_4x4(&mb->block[b], &xd->block[b]);
John Koleszar's avatar
John Koleszar committed
665
  }
666

John Koleszar's avatar
John Koleszar committed
667
  // DC predication and Quantization of 2nd Order block
668
  mb->quantize_b_4x4(mb_y2, x_y2);
669

John Koleszar's avatar
John Koleszar committed
670
  // Distortion
671
  d = vp9_mbblock_error(mb, 1);
672

673
  d += vp9_block_error(mb_y2->coeff, x_y2->dqcoeff, 16);
674

John Koleszar's avatar
John Koleszar committed
675
676
  *Distortion = (d >> 2);
  // rate
677
  *Rate = rdcost_mby_4x4(mb);
678
  *skippable = vp9_mby_is_skippable_4x4(&mb->e_mbd, 1);
679
}
John Koleszar's avatar
John Koleszar committed
680

681
static int rdcost_mby_8x8(MACROBLOCK *mb, int backup) {
John Koleszar's avatar
John Koleszar committed
682
683
  int cost = 0;
  int b;
Paul Wilkins's avatar
Paul Wilkins committed
684
  MACROBLOCKD *xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
685
686
687
688
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta;
  ENTROPY_CONTEXT *tl;

Ronald S. Bultje's avatar
Ronald S. Bultje committed
689
690
691
  if (backup) {
    vpx_memcpy(&t_above,xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
    vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
John Koleszar's avatar
John Koleszar committed
692

Ronald S. Bultje's avatar
Ronald S. Bultje committed
693
694
695
696
697
698
    ta = (ENTROPY_CONTEXT *)&t_above;
    tl = (ENTROPY_CONTEXT *)&t_left;
  } else {
    ta = (ENTROPY_CONTEXT *)mb->e_mbd.above_context;
    tl = (ENTROPY_CONTEXT *)mb->e_mbd.left_context;
  }
John Koleszar's avatar
John Koleszar committed
699
700

  for (b = 0; b < 16; b += 4)
Paul Wilkins's avatar
Paul Wilkins committed
701
    cost += cost_coeffs(mb, xd->block + b, PLANE_TYPE_Y_NO_DC,
702
                        ta + vp9_block2above_8x8[b], tl + vp9_block2left_8x8[b],
Daniel Kang's avatar
Daniel Kang committed
703
                        TX_8X8);
John Koleszar's avatar
John Koleszar committed
704

Paul Wilkins's avatar
Paul Wilkins committed
705
  cost += cost_coeffs_2x2(mb, xd->block + 24, PLANE_TYPE_Y2,
706
                          ta + vp9_block2above[24], tl + vp9_block2left[24]);
John Koleszar's avatar
John Koleszar committed
707
  return cost;
708
709
}

John Koleszar's avatar
John Koleszar committed
710
711
712
static void macro_block_yrd_8x8(MACROBLOCK *mb,
                                int *Rate,
                                int *Distortion,
713
                                const VP9_ENCODER_RTCD *rtcd,
714
                                int *skippable) {
Paul Wilkins's avatar
Paul Wilkins committed
715
  MACROBLOCKD *const xd = &mb->e_mbd;
John Koleszar's avatar
John Koleszar committed
716
  BLOCK   *const mb_y2 = mb->block + 24;
Paul Wilkins's avatar
Paul Wilkins committed
717
  BLOCKD *const x_y2  = xd->block + 24;
John Koleszar's avatar
John Koleszar committed
718
719
  int d;

720
721
  vp9_transform_mby_8x8(mb);
  vp9_quantize_mby_8x8(mb);
John Koleszar's avatar
John Koleszar committed
722
723
724
725
726
727

  /* remove 1st order dc to properly combine 1st/2nd order distortion */
  mb->coeff[0] = 0;
  mb->coeff[64] = 0;
  mb->coeff[128] = 0;
  mb->coeff[192] = 0;
Paul Wilkins's avatar
Paul Wilkins committed
728
729
730
731
  xd->dqcoeff[0] = 0;
  xd->dqcoeff[64] = 0;
  xd->dqcoeff[128] = 0;
  xd->dqcoeff[192] = 0;
John Koleszar's avatar
John Koleszar committed
732

733
734
  d = vp9_mbblock_error(mb, 0);
  d += vp9_block_error(mb_y2->coeff, x_y2->dqcoeff, 16);
John Koleszar's avatar
John Koleszar committed
735
736
737

  *Distortion = (d >> 2);
  // rate
738
  *Rate = rdcost_mby_8x8(mb, 1);
739
  *skippable = vp9_mby_is_skippable_8x8(&mb->e_mbd, 1);
740
}
741

742
static int rdcost_mby_16x16(MACROBLOCK *mb) {
Daniel Kang's avatar
Daniel Kang committed
743
  int cost;
Paul Wilkins's avatar
Paul Wilkins committed
744
  MACROBLOCKD *xd = &mb->e_mbd;
Daniel Kang's avatar
Daniel Kang committed
745
746
747
  ENTROPY_CONTEXT_PLANES t_above, t_left;
  ENTROPY_CONTEXT *ta, *tl;

Paul Wilkins's avatar
Paul Wilkins committed
748
749
  vpx_memcpy(&t_above, xd->above_context, sizeof(ENTROPY_CONTEXT_PLANES));
  vpx_memcpy(&t_left, xd->left_context, sizeof(ENTROPY_CONTEXT_PLANES));
Daniel Kang's avatar
Daniel Kang committed
750
751
752
753

  ta = (ENTROPY_CONTEXT *)&t_above;
  tl = (ENTROPY_CONTEXT *)&t_left;

Paul Wilkins's avatar
Paul Wilkins committed
754
  cost = cost_coeffs(mb, xd->block, PLANE_TYPE_Y_WITH_DC, ta, tl, TX_16X16);
Daniel Kang's avatar
Daniel Kang committed
755
756
  return cost;
}
757

Daniel Kang's avatar
Daniel Kang committed
758
static void macro_block_yrd_16x16(MACROBLOCK *mb, int *Rate, int *Distortion,
759
                                  const VP9_ENCODER_RTCD *rtcd, int *skippable) {
Daniel Kang's avatar
Daniel Kang committed
760
  int d;
Deb Mukherjee's avatar
Deb Mukherjee committed
761
762
763
764
  MACROBLOCKD *xd = &mb->e_mbd;
  BLOCKD *b  = &mb->e_mbd.block[0];
  BLOCK  *be = &mb->block[0];
  TX_TYPE tx_type;
Daniel Kang's avatar
Daniel Kang committed
765

Deb Mukherjee's avatar
Deb Mukherjee committed
766
767
  tx_type = get_tx_type_16x16(xd, b);
  if (tx_type != DCT_DCT) {
768
    vp9_fht(be->src_diff, 32, be->coeff, tx_type, 16);
769
  } else
770
    vp9_transform_mby_16x16(mb);
771

772
  vp9_quantize_mby_16x16(mb);
773
774
775
  // TODO(jingning) is it possible to quickly determine whether to force
  //                trailing coefficients to be zero, instead of running trellis
  //                optimization in the rate-distortion optimization loop?
776
  if (mb->e_mbd.mode_info_context->mbmi.mode < I8X8_PRED)
777
    vp9_optimize_mby_16x16(mb, rtcd);
778

779
  d = vp9_mbblock_error(mb, 0);
Daniel Kang's avatar
Daniel Kang committed
780
781
782

  *Distortion = (d >> 2);
  // rate
783
  *Rate = rdcost_mby_16x16(mb);
784
  *skippable = vp9_mby_is_skippable_16x16(&mb->e_mbd);
Daniel Kang's avatar
Daniel Kang committed
785
786
}

787
static void macro_block_yrd(VP9_COMP *cpi, MACROBLOCK *x, int *rate,
788
789
                            int *distortion, int *skippable,
                            int64_t txfm_cache[NB_TXFM_MODES]) {
790
  VP9_COMMON *cm = &cpi->common;
791
792
793
794
  MB_MODE_INFO *mbmi = &x->e_mbd.mode_info_context->mbmi;

  MACROBLOCKD *xd = &x->e_mbd;
  int can_skip = cm->mb_no_coeff_skip;
795
  vp9_prob skip_prob = can_skip ? vp9_get_pred_prob(cm, xd, PRED_MBSKIP) : 128;
796
797
798
799
800
801
  int s0, s1;
  int r4x4, r4x4s, r8x8, r8x8s, d4x4, d8x8, s4x4, s8x8;
  int64_t rd4x4, rd8x8, rd4x4s, rd8x8s;
  int d16x16, r16x16, r16x16s, s16x16;
  int64_t rd16x16, rd16x16s;

802
803
804
  vp9_subtract_mby(x->src_diff, *(x->block[0].base_src), xd->predictor,
                   x->block[0].src_stride);

805
806
  if (skip_prob == 0)
    skip_prob = 1;
807
808
  s0 = vp9_cost_bit(skip_prob, 0);
  s1 = vp9_cost_bit(skip_prob, 1);
809
810
811
812
813
814
815
816
817
818
  macro_block_yrd_16x16(x, &r16x16, &d16x16, IF_RTCD(&cpi->rtcd), &s16x16);
  if (can_skip) {
    if (s16x16) {
      rd16x16 = RDCOST(x->rdmult, x->rddiv, s1, d16x16);
    } else {
      rd16x16 = RDCOST(x->rdmult, x->rddiv, r16x16 + s0, d16x16);
    }
  } else {
    rd16x16 = RDCOST(x->rdmult, x->rddiv, r16x16, d16x16);
  }
819
  r16x16s = r16x16 + vp9_cost_one(cm->prob_tx[0]) + vp9_cost_one(cm->prob_tx[1]);
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
  if (can_skip) {
    if (s16x16) {
      rd16x16s = RDCOST(x->rdmult, x->rddiv, s1, d16x16);
    } else {
      rd16x16s = RDCOST(x->rdmult, x->rddiv, r16x16s + s0, d16x16);
    }
  } else {
    rd16x16s = RDCOST(x->rdmult, x->rddiv, r16x16s, d16x16);
  }
  macro_block_yrd_8x8(x, &r8x8, &d8x8, IF_RTCD(&cpi->rtcd), &s8x8);
  if (can_skip) {
    if (s8x8) {
      rd8x8 = RDCOST(x->rdmult, x->rddiv, s1, d8x8);
    } else {
      rd8x8 = RDCOST(x->rdmult, x->rddiv, r8x8 + s0, d8x8);
    }
  } else {
    rd8x8 = RDCOST(x->rdmult, x->rddiv, r8x8, d8x8);
  }
839
840
  r8x8s = r8x8 + vp9_cost_one(cm->prob_tx[0]);
  r8x8s += vp9_cost_zero(cm->prob_tx[1]);
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
  if (can_skip) {
    if (s8x8) {
      rd8x8s = RDCOST(x->rdmult, x->rddiv, s1, d8x8);
    } else {
      rd8x8s = RDCOST(x->rdmult, x->rddiv, r8x8s + s0, d8x8);
    }
  } else {
    rd8x8s = RDCOST(x->rdmult, x->rddiv, r8x8s, d8x8);
  }
  macro_block_yrd_4x4(x, &r4x4, &d4x4, IF_RTCD(&cpi->rtcd), &s4x4);
  if (can_skip) {
    if (s4x4) {
      rd4x4 = RDCOST(x->rdmult, x->rddiv, s1, d4x4);
    } else {
      rd4x4 = RDCOST(x->rdmult, x->rddiv, r4x4 + s0, d4x4);
    }
  } else {
    rd4x4 = RDCOST(x->rdmult, x->rddiv, r4x4, d4x4);
  }
860
  r4x4s = r4x4 + vp9_cost_zero(cm->prob_tx[0]);
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
  if (can_skip) {
    if (s4x4) {
      rd4x4s = RDCOST(x->rdmult, x->rddiv, s1, d4x4);
    } else {
      rd4x4s = RDCOST(x->rdmult, x->rddiv, r4x4s + s0, d4x4);
    }
  } else {
    rd4x4s = RDCOST(x->rdmult, x->rddiv, r4x4s, d4x4);
  }

  if ( cpi->common.txfm_mode == ALLOW_16X16 ||
      (cpi->common.txfm_mode == TX_MODE_SELECT &&
       rd16x16s < rd8x8s && rd16x16s < rd4x4s)) {
    mbmi->txfm_size = TX_16X16;
    *skippable = s16x16;
    *distortion = d16x16;
    *rate = (cpi->common.txfm_mode == ALLOW_16X16) ? r16x16 : r16x16s;
  } else
  if ( cpi->common.txfm_mode == ALLOW_8X8 ||
      (cpi->common.txfm_mode == TX_MODE_SELECT && rd8x8s < rd4x4s)) {
    mbmi->txfm_size = TX_8X8;
    *skippable = s8x8;
    *distortion = d8x8;
    *rate = (cpi->common.txfm_mode == ALLOW_8X8) ? r8x8 : r8x8s;
  } else {
    assert(cpi->common.txfm_mode == ONLY_4X4 ||
           (cpi->common.txfm_mode == TX_MODE_SELECT && rd4x4s <= rd8x8s));
    mbmi->txfm_size = TX_4X4;
    *skippable = s4x4;
    *distortion = d4x4;
    *rate = (cpi->common.txfm_mode == ONLY_4X4) ? r4x4 : r4x4s;
  }

  txfm_cache[ONLY_4X4] = rd4x4;
  txfm_cache[ALLOW_8X8] = rd8x8;
  txfm_cache[ALLOW_16X16] = rd16x16;
  if (rd16x16s < rd8x8s && rd16x16s < rd4x4s)
    txfm_cache[TX_MODE_SELECT] = rd16x16s;
  else
    txfm_cache[TX_MODE_SELECT] = rd4x4s < rd8x8s ? rd4x4s : rd8x8s;

}

John Koleszar's avatar
John Koleszar committed
904
905
906
907
908
909
910
static void copy_predictor(unsigned char *dst, const unsigned char *predictor) {
  const unsigned int *p = (const unsigned int *)predictor;
  unsigned int *d = (unsigned int *)dst;
  d[0] = p[0];
  d[4] = p[4];
  d[8] = p[8];
  d[12] = p[12];
911
}
Yaowu Xu's avatar
Yaowu Xu committed
912

Ronald S. Bultje's avatar
Ronald S. Bultje committed
913
914
915
916
#if CONFIG_SUPERBLOCKS
static void super_block_yrd_8x8(MACROBLOCK *x,
                                int *rate,
                                int *distortion,
917
                                const VP9_ENCODER_RTCD *rtcd, int *skip)
Ronald S. Bultje's avatar
Ronald S. Bultje committed
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
{
  MACROBLOCKD *const xd = &x->e_mbd;
  BLOCK *const by2 = x->block + 24;
  BLOCKD *const bdy2  = xd->block + 24;
  int d = 0, r = 0, n;
  const uint8_t *src = x->src.y_buffer, *dst = xd->dst.y_buffer;
  int src_y_stride = x->src.y_stride, dst_y_stride = xd->dst.y_stride;
  ENTROPY_CONTEXT_PLANES *ta = xd->above_context;
  ENTROPY_CONTEXT_PLANES *tl = xd->left_context;
  ENTROPY_CONTEXT_PLANES t_above[2];
  ENTROPY_CONTEXT_PLANES t_left[2];
  int skippable = 1;

  vpx_memcpy(t_above, xd->above_context, sizeof(t_above));
  vpx_memcpy(t_left, xd->left_context, sizeof(t_left));

  for (n = 0; n < 4; n++) {
    int x_idx = n & 1, y_idx = n >> 1;

937
    vp9_subtract_mby_s_c(x->src_diff,
Ronald S. Bultje's avatar
Ronald S. Bultje committed
938
939
940
941
                         src + x_idx * 16 + y_idx * 16 * src_y_stride,
                         src_y_stride,
                         dst + x_idx * 16 + y_idx * 16 * dst_y_stride,
                         dst_y_stride);
942
943
    vp9_transform_mby_8x8(x);
    vp9_quantize_mby_8x8(x);
Ronald S. Bultje's avatar
Ronald S. Bultje committed
944
945
946
947
948
949
950
951
952
953
954

    /* remove 1st order dc to properly combine 1st/2nd order distortion */
    x->coeff[  0] = 0;
    x->coeff[ 64] = 0;
    x->coeff[128] = 0;
    x->coeff[192] = 0;
    xd->dqcoeff[  0] = 0;
    xd->dqcoeff[ 64] = 0;
    xd->dqcoeff[128] = 0;
    xd->dqcoeff[192] = 0;

955
956
    d += vp9_mbblock_error(x, 0);
    d += vp9_block_error(by2->coeff, bdy2->dqcoeff, 16);
Ronald S. Bultje's avatar
Ronald S. Bultje committed
957
958
    xd->above_context = ta + x_idx;
    xd->left_context = tl + y_idx;
959
    r += rdcost_mby_8x8(x, 0);
960
    skippable = skippable && vp9_mby_is_skippable_8x8(xd, 1);
Ronald S. Bultje's avatar
Ronald S. Bultje committed
961
962
963
964
965
966
967
968
969
970
971
972
  }

  *distortion = (d >> 2);
  *rate       = r;
  if (skip) *skip = skippable;
  xd->above_context = ta;
  xd->left_context = tl;
  vpx_memcpy(xd->above_context, &t_above, sizeof(t_above));
  vpx_memcpy(xd->left_context, &t_left, sizeof(t_left));
}
#endif

John Koleszar's avatar
John Koleszar committed
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
static void copy_predictor_8x8(unsigned char *dst, const unsigned char *predictor) {
  const unsigned int *p = (const unsigned int *)predictor;
  unsigned int *d = (unsigned int *)dst;
  d[0] = p[0];
  d[1] = p[1];
  d[4] = p[4];
  d[5] = p[5];
  d[8] = p[8];
  d[9] = p[9];
  d[12] = p[12];
  d[13] = p[13];
  d[16] = p[16];
  d[17] = p[17];
  d[20] = p[20];
  d[21] = p[21];
  d[24] = p[24];
  d[25] = p[25];
  d[28] = p[28];
  d[29] = p[29];
Yaowu Xu's avatar
Yaowu Xu committed
992
993
}

994
static int64_t rd_pick_intra4x4block(VP9_COMP *cpi, MACROBLOCK *x, BLOCK *be,