vp9_pickmode.c 27.3 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
/*
 *  Copyright (c) 2014 The WebM project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include <assert.h>
12
13
14
#include <limits.h>
#include <math.h>
#include <stdio.h>
15

16
17
18
19
#include "./vp9_rtcd.h"

#include "vpx_mem/vpx_mem.h"

20
#include "vp9/common/vp9_blockd.h"
21
22
#include "vp9/common/vp9_common.h"
#include "vp9/common/vp9_mvref_common.h"
23
24
#include "vp9/common/vp9_reconinter.h"
#include "vp9/common/vp9_reconintra.h"
25

Dmitry Kovalev's avatar
Dmitry Kovalev committed
26
#include "vp9/encoder/vp9_encoder.h"
27
#include "vp9/encoder/vp9_pickmode.h"
28
#include "vp9/encoder/vp9_ratectrl.h"
29
#include "vp9/encoder/vp9_rd.h"
30

31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
static int mv_refs_rt(const VP9_COMMON *cm, const MACROBLOCKD *xd,
                       const TileInfo *const tile,
                       MODE_INFO *mi, MV_REFERENCE_FRAME ref_frame,
                       int_mv *mv_ref_list,
                       int mi_row, int mi_col) {
  const int *ref_sign_bias = cm->ref_frame_sign_bias;
  int i, refmv_count = 0;

  const POSITION *const mv_ref_search = mv_ref_blocks[mi->mbmi.sb_type];

  int different_ref_found = 0;
  int context_counter = 0;
  int const_motion = 0;

  // Blank the reference vector list
  vpx_memset(mv_ref_list, 0, sizeof(*mv_ref_list) * MAX_MV_REF_CANDIDATES);

  // The nearest 2 blocks are treated differently
  // if the size < 8x8 we get the mv from the bmi substructure,
  // and we also need to keep a mode count.
  for (i = 0; i < 2; ++i) {
    const POSITION *const mv_ref = &mv_ref_search[i];
    if (is_inside(tile, mi_col, mi_row, cm->mi_rows, mv_ref)) {
      const MODE_INFO *const candidate_mi = xd->mi[mv_ref->col + mv_ref->row *
                                                   xd->mi_stride];
      const MB_MODE_INFO *const candidate = &candidate_mi->mbmi;
      // Keep counts for entropy encoding.
      context_counter += mode_2_counter[candidate->mode];
      different_ref_found = 1;

      if (candidate->ref_frame[0] == ref_frame)
        ADD_MV_REF_LIST(get_sub_block_mv(candidate_mi, 0, mv_ref->col, -1));
    }
  }

  const_motion = 1;

  // Check the rest of the neighbors in much the same way
  // as before except we don't need to keep track of sub blocks or
  // mode counts.
  for (; i < MVREF_NEIGHBOURS && !refmv_count; ++i) {
    const POSITION *const mv_ref = &mv_ref_search[i];
    if (is_inside(tile, mi_col, mi_row, cm->mi_rows, mv_ref)) {
      const MB_MODE_INFO *const candidate = &xd->mi[mv_ref->col + mv_ref->row *
                                                    xd->mi_stride]->mbmi;
      different_ref_found = 1;

      if (candidate->ref_frame[0] == ref_frame)
        ADD_MV_REF_LIST(candidate->mv[0]);
    }
  }

  // Since we couldn't find 2 mvs from the same reference frame
  // go back through the neighbors and find motion vectors from
  // different reference frames.
  if (different_ref_found && !refmv_count) {
    for (i = 0; i < MVREF_NEIGHBOURS; ++i) {
      const POSITION *mv_ref = &mv_ref_search[i];
      if (is_inside(tile, mi_col, mi_row, cm->mi_rows, mv_ref)) {
        const MB_MODE_INFO *const candidate = &xd->mi[mv_ref->col + mv_ref->row
                                              * xd->mi_stride]->mbmi;

        // If the candidate is INTRA we don't want to consider its mv.
        IF_DIFF_REF_FRAME_ADD_MV(candidate);
      }
    }
  }

 Done:

  mi->mbmi.mode_context[ref_frame] = counter_to_context[context_counter];

  // Clamp vectors
  for (i = 0; i < MAX_MV_REF_CANDIDATES; ++i)
    clamp_mv_ref(&mv_ref_list[i].as_mv, xd);

  return const_motion;
}

110
111
112
113
static int combined_motion_search(VP9_COMP *cpi, MACROBLOCK *x,
                                  BLOCK_SIZE bsize, int mi_row, int mi_col,
                                  int_mv *tmp_mv, int *rate_mv,
                                  int64_t best_rd_sofar) {
114
  MACROBLOCKD *xd = &x->e_mbd;
115
  MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
116
  struct buf_2d backup_yv12[MAX_MB_PLANE] = {{0, 0}};
117
118
  const int step_param = cpi->sf.mv.fullpel_search_step_param;
  const int sadpb = x->sadperbit16;
119
  MV mvp_full;
120
  const int ref = mbmi->ref_frame[0];
121
  const MV ref_mv = mbmi->ref_mvs[ref][0].as_mv;
122
123
124
125
126
127
128
  int dis;
  int rate_mode;
  const int tmp_col_min = x->mv_col_min;
  const int tmp_col_max = x->mv_col_max;
  const int tmp_row_min = x->mv_row_min;
  const int tmp_row_max = x->mv_row_max;
  int rv = 0;
129
  int sad_list[5];
130
131
  const YV12_BUFFER_CONFIG *scaled_ref_frame = vp9_get_scaled_ref_frame(cpi,
                                                                        ref);
Yaowu Xu's avatar
Yaowu Xu committed
132
133
134
135
  if (cpi->common.show_frame &&
      (x->pred_mv_sad[ref] >> 3) > x->pred_mv_sad[LAST_FRAME])
    return rv;

136
137
138
139
140
141
142
  if (scaled_ref_frame) {
    int i;
    // Swap out the reference frame for a version that's been scaled to
    // match the resolution of the current frame, allowing the existing
    // motion search code to be used without additional modifications.
    for (i = 0; i < MAX_MB_PLANE; i++)
      backup_yv12[i] = xd->plane[i].pre[0];
143
    vp9_setup_pre_planes(xd, 0, scaled_ref_frame, mi_row, mi_col, NULL);
144
  }
145
  vp9_set_mv_search_range(x, &ref_mv);
146

Yaowu Xu's avatar
Yaowu Xu committed
147
148
149
150
  assert(x->mv_best_ref_index[ref] <= 2);
  if (x->mv_best_ref_index[ref] < 2)
    mvp_full = mbmi->ref_mvs[ref][x->mv_best_ref_index[ref]].as_mv;
  else
Dmitry Kovalev's avatar
Dmitry Kovalev committed
151
    mvp_full = x->pred_mv[ref];
152
153
154
155

  mvp_full.col >>= 3;
  mvp_full.row >>= 3;

156
157
158
  vp9_full_pixel_search(cpi, x, bsize, &mvp_full, step_param, sadpb,
                        cond_sad_list(cpi, sad_list),
                        &ref_mv, &tmp_mv->as_mv, INT_MAX, 0);
159

160
161
162
163
164
  x->mv_col_min = tmp_col_min;
  x->mv_col_max = tmp_col_max;
  x->mv_row_min = tmp_row_min;
  x->mv_row_max = tmp_row_max;

165
166
167
  // calculate the bit cost on motion vector
  mvp_full.row = tmp_mv->as_mv.row * 8;
  mvp_full.col = tmp_mv->as_mv.col * 8;
168

169
170
  *rate_mv = vp9_mv_bit_cost(&mvp_full, &ref_mv,
                             x->nmvjointcost, x->mvcost, MV_COST_WEIGHT);
171

172
173
174
175
176
177
178
179
180
181
182
183
  rate_mode = cpi->inter_mode_cost[mbmi->mode_context[ref]]
                                  [INTER_OFFSET(NEWMV)];
  rv = !(RDCOST(x->rdmult, x->rddiv, (*rate_mv + rate_mode), 0) >
         best_rd_sofar);

  if (rv) {
    cpi->find_fractional_mv_step(x, &tmp_mv->as_mv, &ref_mv,
                                 cpi->common.allow_high_precision_mv,
                                 x->errorperbit,
                                 &cpi->fn_ptr[bsize],
                                 cpi->sf.mv.subpel_force_stop,
                                 cpi->sf.mv.subpel_iters_per_step,
184
                                 cond_sad_list(cpi, sad_list),
185
                                 x->nmvjointcost, x->mvcost,
186
                                 &dis, &x->pred_sse[ref], NULL, 0, 0);
187
    x->pred_mv[ref] = tmp_mv->as_mv;
188
189
190
191
192
193
194
  }

  if (scaled_ref_frame) {
    int i;
    for (i = 0; i < MAX_MB_PLANE; i++)
      xd->plane[i].pre[0] = backup_yv12[i];
  }
195
  return rv;
196
197
}

198

199
200
static void model_rd_for_sb_y(VP9_COMP *cpi, BLOCK_SIZE bsize,
                              MACROBLOCK *x, MACROBLOCKD *xd,
201
202
                              int *out_rate_sum, int64_t *out_dist_sum,
                              unsigned int *var_y, unsigned int *sse_y) {
203
204
205
206
  // Note our transform coeffs are 8 times an orthogonal transform.
  // Hence quantizer step is also 8 times. To get effective quantizer
  // we need to divide by 8 before sending to modeling function.
  unsigned int sse;
207
208
  int rate;
  int64_t dist;
209
210
  struct macroblock_plane *const p = &x->plane[0];
  struct macroblockd_plane *const pd = &xd->plane[0];
211
212
  const uint32_t dc_quant = pd->dequant[0];
  const uint32_t ac_quant = pd->dequant[1];
213
214
  unsigned int var = cpi->fn_ptr[bsize].vf(p->src.buf, p->src.stride,
                                           pd->dst.buf, pd->dst.stride, &sse);
215
216
217
  *var_y = var;
  *sse_y = sse;

218
  if (sse < dc_quant * dc_quant >> 6)
219
    x->skip_txfm[0] = 1;
220
  else if (var < ac_quant * ac_quant >> 6)
221
    x->skip_txfm[0] = 2;
222
  else
223
    x->skip_txfm[0] = 0;
224

225
226
227
228
229
230
231
232
233
234
235
  if (cpi->common.tx_mode == TX_MODE_SELECT) {
    if (sse > (var << 2))
      xd->mi[0]->mbmi.tx_size = MIN(max_txsize_lookup[bsize],
                          tx_mode_to_biggest_tx_size[cpi->common.tx_mode]);
    else
      xd->mi[0]->mbmi.tx_size = TX_8X8;
  } else {
    xd->mi[0]->mbmi.tx_size = MIN(max_txsize_lookup[bsize],
                         tx_mode_to_biggest_tx_size[cpi->common.tx_mode]);
  }

236
237
238
  vp9_model_rd_from_var_lapndz(sse - var, 1 << num_pels_log2_lookup[bsize],
                               dc_quant >> 3, &rate, &dist);
  *out_rate_sum = rate >> 1;
239
  *out_dist_sum = dist << 3;
240
241
242
243
244

  vp9_model_rd_from_var_lapndz(var, 1 << num_pels_log2_lookup[bsize],
                               ac_quant >> 3, &rate, &dist);
  *out_rate_sum += rate;
  *out_dist_sum += dist << 4;
245
246
}

247
248
249
250
251
252
253
254
255
256
257
258
259
static int get_pred_buffer(PRED_BUFFER *p, int len) {
  int i;

  for (i = 0; i < len; i++) {
    if (!p[i].in_use) {
      p[i].in_use = 1;
      return i;
    }
  }
  return -1;
}

static void free_pred_buffer(PRED_BUFFER *p) {
260
261
  if (p != NULL)
    p->in_use = 0;
262
263
}

264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
static void encode_breakout_test(VP9_COMP *cpi, MACROBLOCK *x,
                                 BLOCK_SIZE bsize, int mi_row, int mi_col,
                                 MV_REFERENCE_FRAME ref_frame,
                                 PREDICTION_MODE this_mode,
                                 unsigned int var_y, unsigned int sse_y,
                                 struct buf_2d yv12_mb[][MAX_MB_PLANE],
                                 int *rate, int64_t *dist) {
  MACROBLOCKD *xd = &x->e_mbd;
  MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;

  const BLOCK_SIZE uv_size = get_plane_block_size(bsize, &xd->plane[1]);
  unsigned int var = var_y, sse = sse_y;
  // Skipping threshold for ac.
  unsigned int thresh_ac;
  // Skipping threshold for dc.
  unsigned int thresh_dc;
  if (x->encode_breakout > 0) {
    // Set a maximum for threshold to avoid big PSNR loss in low bit rate
    // case. Use extreme low threshold for static frames to limit
    // skipping.
    const unsigned int max_thresh = 36000;
    // The encode_breakout input
    const unsigned int min_thresh =
        MIN(((unsigned int)x->encode_breakout << 4), max_thresh);

    // Calculate threshold according to dequant value.
    thresh_ac = (xd->plane[0].dequant[1] * xd->plane[0].dequant[1]) / 9;
    thresh_ac = clamp(thresh_ac, min_thresh, max_thresh);

    // Adjust ac threshold according to partition size.
    thresh_ac >>=
295
        8 - (b_width_log2(bsize) + b_height_log2(bsize));
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350

    thresh_dc = (xd->plane[0].dequant[0] * xd->plane[0].dequant[0] >> 6);
  } else {
    thresh_ac = 0;
    thresh_dc = 0;
  }

  // Y skipping condition checking for ac and dc.
  if (var <= thresh_ac && (sse - var) <= thresh_dc) {
    unsigned int sse_u, sse_v;
    unsigned int var_u, var_v;

    // Skip UV prediction unless breakout is zero (lossless) to save
    // computation with low impact on the result
    if (x->encode_breakout == 0) {
      xd->plane[1].pre[0] = yv12_mb[ref_frame][1];
      xd->plane[2].pre[0] = yv12_mb[ref_frame][2];
      vp9_build_inter_predictors_sbuv(xd, mi_row, mi_col, bsize);
    }

    var_u = cpi->fn_ptr[uv_size].vf(x->plane[1].src.buf,
                                    x->plane[1].src.stride,
                                    xd->plane[1].dst.buf,
                                    xd->plane[1].dst.stride, &sse_u);

    // U skipping condition checking
    if ((var_u * 4 <= thresh_ac) && (sse_u - var_u <= thresh_dc)) {
      var_v = cpi->fn_ptr[uv_size].vf(x->plane[2].src.buf,
                                      x->plane[2].src.stride,
                                      xd->plane[2].dst.buf,
                                      xd->plane[2].dst.stride, &sse_v);

      // V skipping condition checking
      if ((var_v * 4 <= thresh_ac) && (sse_v - var_v <= thresh_dc)) {
        x->skip = 1;

        // The cost of skip bit needs to be added.
        *rate = cpi->inter_mode_cost[mbmi->mode_context[ref_frame]]
                                     [INTER_OFFSET(this_mode)];

        // More on this part of rate
        // rate += vp9_cost_bit(vp9_get_skip_prob(cm, xd), 1);

        // Scaling factor for SSE from spatial domain to frequency
        // domain is 16. Adjust distortion accordingly.
        // TODO(yunqingwang): In this function, only y-plane dist is
        // calculated.
        *dist = (sse << 4);  // + ((sse_u + sse_v) << 4);

        // *disable_skip = 1;
      }
    }
  }
}

351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
struct estimate_block_intra_args {
  VP9_COMP *cpi;
  MACROBLOCK *x;
  PREDICTION_MODE mode;
  int rate;
  int64_t dist;
};

static void estimate_block_intra(int plane, int block, BLOCK_SIZE plane_bsize,
                                 TX_SIZE tx_size, void *arg) {
  struct estimate_block_intra_args* const args = arg;
  VP9_COMP *const cpi = args->cpi;
  MACROBLOCK *const x = args->x;
  MACROBLOCKD *const xd = &x->e_mbd;
  struct macroblock_plane *const p = &x->plane[0];
  struct macroblockd_plane *const pd = &xd->plane[0];
  const BLOCK_SIZE bsize_tx = txsize_to_bsize[tx_size];
  uint8_t *const src_buf_base = p->src.buf;
  uint8_t *const dst_buf_base = pd->dst.buf;
  const int src_stride = p->src.stride;
  const int dst_stride = pd->dst.stride;
  int i, j;
  int rate;
  int64_t dist;
  unsigned int var_y, sse_y;
  txfrm_block_to_raster_xy(plane_bsize, tx_size, block, &i, &j);
  assert(plane == 0);
  (void) plane;

  p->src.buf = &src_buf_base[4 * (j * src_stride + i)];
  pd->dst.buf = &dst_buf_base[4 * (j * dst_stride + i)];
  // Use source buffer as an approximation for the fully reconstructed buffer.
  vp9_predict_intra_block(xd, block >> (2 * tx_size),
                          b_width_log2(plane_bsize),
                          tx_size, args->mode,
                          p->src.buf, src_stride,
                          pd->dst.buf, dst_stride,
                          i, j, 0);
  // This procedure assumes zero offset from p->src.buf and pd->dst.buf.
  model_rd_for_sb_y(cpi, bsize_tx, x, xd, &rate, &dist, &var_y, &sse_y);
  p->src.buf = src_buf_base;
  pd->dst.buf = dst_buf_base;
  args->rate += rate;
  args->dist += dist;
}

397
398
399
400
401
402
static const THR_MODES mode_idx[MAX_REF_FRAMES - 1][4] = {
  {THR_NEARESTMV, THR_NEARMV, THR_ZEROMV, THR_NEWMV},
  {THR_NEARESTG, THR_NEARG, THR_ZEROG, THR_NEWG},
  {THR_NEARESTA, THR_NEARA, THR_ZEROA, THR_NEWA},
};

403
404
405
406
407
408
409
// TODO(jingning) placeholder for inter-frame non-RD mode decision.
// this needs various further optimizations. to be continued..
int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
                            const TileInfo *const tile,
                            int mi_row, int mi_col,
                            int *returnrate,
                            int64_t *returndistortion,
410
411
                            BLOCK_SIZE bsize,
                            PICK_MODE_CONTEXT *ctx) {
412
  MACROBLOCKD *xd = &x->e_mbd;
413
  MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
414
  struct macroblockd_plane *const pd = &xd->plane[0];
415
  PREDICTION_MODE this_mode, best_mode = ZEROMV;
416
  MV_REFERENCE_FRAME ref_frame, best_ref_frame = LAST_FRAME;
417
418
  TX_SIZE best_tx_size = MIN(max_txsize_lookup[bsize],
                             tx_mode_to_biggest_tx_size[cpi->common.tx_mode]);
419
  INTERP_FILTER best_pred_filter = EIGHTTAP;
420
421
422
423
424
  int_mv frame_mv[MB_MODE_COUNT][MAX_REF_FRAMES];
  struct buf_2d yv12_mb[4][MAX_MB_PLANE];
  static const int flag_list[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG,
                                    VP9_ALT_FLAG };
  int64_t best_rd = INT64_MAX;
425
  int64_t this_rd = INT64_MAX;
426
  int skip_txfm = 0;
427
428
  int rate = INT_MAX;
  int64_t dist = INT64_MAX;
429
430
431
  // var_y and sse_y are saved to be used in skipping checking
  unsigned int var_y = UINT_MAX;
  unsigned int sse_y = UINT_MAX;
432

433
434
435
436
437
438
439
  VP9_COMMON *cm = &cpi->common;
  int intra_cost_penalty = 20 * vp9_dc_quant(cm->base_qindex, cm->y_dc_delta_q);

  const int64_t inter_mode_thresh = RDCOST(x->rdmult, x->rddiv,
                                           intra_cost_penalty, 0);
  const int64_t intra_mode_cost = 50;

440
  unsigned char segment_id = mbmi->segment_id;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
441
442
  const int *const rd_threshes = cpi->rd.threshes[segment_id][bsize];
  const int *const rd_thresh_freq_fact = cpi->rd.thresh_freq_fact[bsize];
443
  // Mode index conversion form THR_MODES to PREDICTION_MODE for a ref frame.
444
  INTERP_FILTER filter_ref = cm->interp_filter;
445
  int bsl = mi_width_log2(bsize);
446
  const int pred_filter_search = cm->interp_filter == SWITCHABLE ?
447
448
      (((mi_row + mi_col) >> bsl) +
       get_chessboard_index(cm->current_video_frame)) & 0x1 : 0;
449
  int const_motion[MAX_REF_FRAMES] = { 0 };
450
451
452
  const int bh = num_4x4_blocks_high_lookup[bsize] << 2;
  const int bw = num_4x4_blocks_wide_lookup[bsize] << 2;
  const int pixels_in_block = bh * bw;
453
454
  // For speed 6, the result of interp filter is reused later in actual encoding
  // process.
455
456
457
458
459
460
461
462
  // tmp[3] points to dst buffer, and the other 3 point to allocated buffers.
  PRED_BUFFER tmp[4];
  DECLARE_ALIGNED_ARRAY(16, uint8_t, pred_buf, 3 * 64 * 64);
  struct buf_2d orig_dst = pd->dst;
  PRED_BUFFER *best_pred = NULL;
  PRED_BUFFER *this_mode_pred = NULL;
  int i;

463
464
465
466
  // CTX is used by the temporal denoiser which is currently being developed.
  // TODO(jbb): when temporal denoiser is finished and in the default build
  // remove the following line;
  (void) ctx;
467
468
469
470
471
472
473
474
475
476
477
  if (cpi->sf.reuse_inter_pred_sby) {
    for (i = 0; i < 3; i++) {
      tmp[i].data = &pred_buf[pixels_in_block * i];
      tmp[i].stride = bw;
      tmp[i].in_use = 0;
    }
    tmp[3].data = pd->dst.buf;
    tmp[3].stride = pd->dst.stride;
    tmp[3].in_use = 0;
  }

478
479
  x->skip_encode = cpi->sf.skip_encode_frame && x->q_index < QIDX_SKIP_THRESH;
  x->skip = 0;
480

481
482
  // initialize mode decisions
  *returnrate = INT_MAX;
483
  *returndistortion = INT64_MAX;
484
485
486
487
488
  vpx_memset(mbmi, 0, sizeof(MB_MODE_INFO));
  mbmi->sb_type = bsize;
  mbmi->ref_frame[0] = NONE;
  mbmi->ref_frame[1] = NONE;
  mbmi->tx_size = MIN(max_txsize_lookup[bsize],
489
490
491
                      tx_mode_to_biggest_tx_size[cm->tx_mode]);
  mbmi->interp_filter = cm->interp_filter == SWITCHABLE ?
                        EIGHTTAP : cm->interp_filter;
492
  mbmi->segment_id = segment_id;
493

494
  for (ref_frame = LAST_FRAME; ref_frame <= ALTREF_FRAME; ++ref_frame) {
495
    x->pred_mv_sad[ref_frame] = INT_MAX;
496
497
498
499
500
501
502
503
    frame_mv[NEWMV][ref_frame].as_int = INVALID_MV;
    frame_mv[ZEROMV][ref_frame].as_int = 0;

    if (xd->up_available)
      filter_ref = xd->mi[-xd->mi_stride]->mbmi.interp_filter;
    else if (xd->left_available)
      filter_ref = xd->mi[-1]->mbmi.interp_filter;

504
    if (cpi->ref_frame_flags & flag_list[ref_frame]) {
505
506
507
508
509
510
      const YV12_BUFFER_CONFIG *yv12 = get_ref_frame_buffer(cpi, ref_frame);
      int_mv *const candidates = mbmi->ref_mvs[ref_frame];
      const struct scale_factors *const sf = &cm->frame_refs[ref_frame - 1].sf;
      vp9_setup_pred_block(xd, yv12_mb[ref_frame], yv12, mi_row, mi_col,
                           sf, sf);

511
      if (!cm->error_resilient_mode)
512
513
514
515
516
517
518
519
520
521
522
523
524
525
        vp9_find_mv_refs(cm, xd, tile, xd->mi[0], ref_frame,
                         candidates, mi_row, mi_col);
      else
        const_motion[ref_frame] = mv_refs_rt(cm, xd, tile, xd->mi[0],
                                             ref_frame, candidates,
                                             mi_row, mi_col);

      vp9_find_best_ref_mvs(xd, cm->allow_high_precision_mv, candidates,
                            &frame_mv[NEARESTMV][ref_frame],
                            &frame_mv[NEARMV][ref_frame]);

      if (!vp9_is_scaled(sf) && bsize >= BLOCK_8X8)
        vp9_mv_pred(cpi, x, yv12_mb[ref_frame][0].buf, yv12->y_stride,
                    ref_frame, bsize);
526
    } else {
527
      continue;
528
    }
529
530
531
532
533
534
535

    // Select prediction reference frames.
    xd->plane[0].pre[0] = yv12_mb[ref_frame][0];

    clamp_mv2(&frame_mv[NEARESTMV][ref_frame].as_mv, xd);
    clamp_mv2(&frame_mv[NEARMV][ref_frame].as_mv, xd);

536
537
    mbmi->ref_frame[0] = ref_frame;

538
    for (this_mode = NEARESTMV; this_mode <= NEWMV; ++this_mode) {
539
      int rate_mv = 0;
540
      int mode_rd_thresh;
541

542
543
544
545
      if (const_motion[ref_frame] &&
          (this_mode == NEARMV || this_mode == ZEROMV))
        continue;

546
      if (!(cpi->sf.inter_mode_mask[bsize] & (1 << this_mode)))
547
        continue;
Jim Bankoski's avatar
Jim Bankoski committed
548

549
550
551
      mode_rd_thresh =  rd_threshes[mode_idx[ref_frame - LAST_FRAME]
                                            [this_mode - NEARESTMV]];
      if (rd_less_than_thresh(best_rd, mode_rd_thresh,
552
                              rd_thresh_freq_fact[this_mode]))
553
554
        continue;

Jim Bankoski's avatar
Jim Bankoski committed
555
      if (this_mode == NEWMV) {
556
        if (this_rd < (int64_t)(1 << num_pels_log2_lookup[bsize]))
557
          continue;
558
559
560
        if (!combined_motion_search(cpi, x, bsize, mi_row, mi_col,
                                    &frame_mv[NEWMV][ref_frame],
                                    &rate_mv, best_rd))
Jim Bankoski's avatar
Jim Bankoski committed
561
562
563
          continue;
      }

564
565
566
      if (this_mode != NEARESTMV &&
          frame_mv[this_mode][ref_frame].as_int ==
              frame_mv[NEARESTMV][ref_frame].as_int)
567
568
          continue;

569
570
      mbmi->mode = this_mode;
      mbmi->mv[0].as_int = frame_mv[this_mode][ref_frame].as_int;
571

572
573
574
      // Search for the best prediction filter type, when the resulting
      // motion vector is at sub-pixel accuracy level for luma component, i.e.,
      // the last three bits are all zeros.
575
576
577
578
579
580
581
582
583
584
      if (cpi->sf.reuse_inter_pred_sby) {
        if (this_mode == NEARESTMV) {
          this_mode_pred = &tmp[3];
        } else {
          this_mode_pred = &tmp[get_pred_buffer(tmp, 3)];
          pd->dst.buf = this_mode_pred->data;
          pd->dst.stride = bw;
        }
      }

585
      if ((this_mode == NEWMV || filter_ref == SWITCHABLE) &&
586
          pred_filter_search &&
587
588
589
590
          ((mbmi->mv[0].as_mv.row & 0x07) != 0 ||
           (mbmi->mv[0].as_mv.col & 0x07) != 0)) {
        int pf_rate[3];
        int64_t pf_dist[3];
591
592
        unsigned int pf_var[3];
        unsigned int pf_sse[3];
593
        TX_SIZE pf_tx_size[3];
594
595
        int64_t best_cost = INT64_MAX;
        INTERP_FILTER best_filter = SWITCHABLE, filter;
596
        PRED_BUFFER *current_pred = this_mode_pred;
597
598
599
600
601
602

        for (filter = EIGHTTAP; filter <= EIGHTTAP_SHARP; ++filter) {
          int64_t cost;
          mbmi->interp_filter = filter;
          vp9_build_inter_predictors_sby(xd, mi_row, mi_col, bsize);
          model_rd_for_sb_y(cpi, bsize, x, xd, &pf_rate[filter],
603
                            &pf_dist[filter], &pf_var[filter], &pf_sse[filter]);
604
605
606
          cost = RDCOST(x->rdmult, x->rddiv,
                        vp9_get_switchable_rate(cpi) + pf_rate[filter],
                        pf_dist[filter]);
607
          pf_tx_size[filter] = mbmi->tx_size;
608
          if (cost < best_cost) {
609
610
            best_filter = filter;
            best_cost = cost;
611
            skip_txfm = x->skip_txfm[0];
612
613
614
615
616
617
618
619
620
621
622
623
624

            if (cpi->sf.reuse_inter_pred_sby) {
              if (this_mode_pred != current_pred) {
                free_pred_buffer(this_mode_pred);
                this_mode_pred = current_pred;
              }

              if (filter < EIGHTTAP_SHARP) {
                current_pred = &tmp[get_pred_buffer(tmp, 3)];
                pd->dst.buf = current_pred->data;
                pd->dst.stride = bw;
              }
            }
625
          }
626
627
        }

628
629
630
        if (cpi->sf.reuse_inter_pred_sby && this_mode_pred != current_pred)
          free_pred_buffer(current_pred);

631
        mbmi->interp_filter = best_filter;
632
        mbmi->tx_size = pf_tx_size[mbmi->interp_filter];
633
634
        rate = pf_rate[mbmi->interp_filter];
        dist = pf_dist[mbmi->interp_filter];
635
636
        var_y = pf_var[mbmi->interp_filter];
        sse_y = pf_sse[mbmi->interp_filter];
637
        x->skip_txfm[0] = skip_txfm;
638
639
640
      } else {
        mbmi->interp_filter = (filter_ref == SWITCHABLE) ? EIGHTTAP: filter_ref;
        vp9_build_inter_predictors_sby(xd, mi_row, mi_col, bsize);
641
        model_rd_for_sb_y(cpi, bsize, x, xd, &rate, &dist, &var_y, &sse_y);
642
643
      }

644
      rate += rate_mv;
645
      rate += cpi->inter_mode_cost[mbmi->mode_context[ref_frame]]
646
647
                                [INTER_OFFSET(this_mode)];
      this_rd = RDCOST(x->rdmult, x->rddiv, rate, dist);
648

649
650
      // Skipping checking: test to see if this block can be reconstructed by
      // prediction only.
651
      if (cpi->allow_encode_breakout) {
652
653
654
655
656
        encode_breakout_test(cpi, x, bsize, mi_row, mi_col, ref_frame,
                             this_mode, var_y, sse_y, yv12_mb, &rate, &dist);
        if (x->skip) {
          rate += rate_mv;
          this_rd = RDCOST(x->rdmult, x->rddiv, rate, dist);
657
658
659
        }
      }

660
#if CONFIG_VP9_TEMPORAL_DENOISING
661
      if (cpi->oxcf.noise_sensitivity > 0) {
662
        vp9_denoiser_update_frame_stats(mbmi, sse_y, this_mode, ctx);
663
      }
Tim Kopp's avatar
Tim Kopp committed
664
665
#endif

666
      if (this_rd < best_rd || x->skip) {
667
        best_rd = this_rd;
668
669
        *returnrate = rate;
        *returndistortion = dist;
670
        best_mode = this_mode;
671
        best_pred_filter = mbmi->interp_filter;
672
        best_tx_size = mbmi->tx_size;
673
        best_ref_frame = ref_frame;
674
        skip_txfm = x->skip_txfm[0];
675
676

        if (cpi->sf.reuse_inter_pred_sby) {
677
          free_pred_buffer(best_pred);
678
679
680
681
682
683

          best_pred = this_mode_pred;
        }
      } else {
        if (cpi->sf.reuse_inter_pred_sby)
          free_pred_buffer(this_mode_pred);
684
      }
685
686
687

      if (x->skip)
        break;
688
    }
689
690
691
692
    // If the current reference frame is valid and we found a usable mode,
    // we are done.
    if (best_rd < INT64_MAX)
      break;
693
694
  }

695
696
  // If best prediction is not in dst buf, then copy the prediction block from
  // temp buf to dst buf.
697
698
  if (best_pred != NULL && cpi->sf.reuse_inter_pred_sby &&
      best_pred->data != orig_dst.buf) {
699
700
701
702
703
704
705
706
707
708
    uint8_t *copy_from, *copy_to;

    pd->dst = orig_dst;
    copy_to = pd->dst.buf;

    copy_from = best_pred->data;

    vp9_convolve_copy(copy_from, bw, copy_to, pd->dst.stride, NULL, 0, NULL, 0,
                      bw, bh);
  }
Tim Kopp's avatar
Tim Kopp committed
709

710
  mbmi->mode          = best_mode;
711
  mbmi->interp_filter = best_pred_filter;
712
713
714
  mbmi->tx_size       = best_tx_size;
  mbmi->ref_frame[0]  = best_ref_frame;
  mbmi->mv[0].as_int  = frame_mv[best_mode][best_ref_frame].as_int;
715
  xd->mi[0]->bmi[0].as_mv[0].as_int = mbmi->mv[0].as_int;
716
  x->skip_txfm[0] = skip_txfm;
717

718
719
  // Perform intra prediction search, if the best SAD is above a certain
  // threshold.
720
  if (!x->skip && best_rd > inter_mode_thresh &&
721
      bsize <= cpi->sf.max_intra_bsize) {
722
    struct estimate_block_intra_args args = { cpi, x, DC_PRED, 0, 0 };
723

724
725
726
    const TX_SIZE intra_tx_size =
        MIN(max_txsize_lookup[bsize],
            tx_mode_to_biggest_tx_size[cpi->common.tx_mode]);
727

728
729
730
731
    if (cpi->sf.reuse_inter_pred_sby) {
      pd->dst.buf = tmp[0].data;
      pd->dst.stride = bw;
    }
732

733
    for (this_mode = DC_PRED; this_mode <= DC_PRED; ++this_mode) {
734
735
736
737
738
739
740
741
742
743
      const TX_SIZE saved_tx_size = mbmi->tx_size;
      args.mode = this_mode;
      args.rate = 0;
      args.dist = 0;
      mbmi->tx_size = intra_tx_size;
      vp9_foreach_transformed_block_in_plane(xd, bsize, 0,
                                             estimate_block_intra, &args);
      mbmi->tx_size = saved_tx_size;
      rate = args.rate;
      dist = args.dist;
744
      rate += cpi->mbmode_cost[this_mode];
745
      rate += intra_cost_penalty;
746
      this_rd = RDCOST(x->rdmult, x->rddiv, rate, dist);
747
748
749

      if (this_rd + intra_mode_cost < best_rd) {
        best_rd = this_rd;
750
751
        *returnrate = rate;
        *returndistortion = dist;
752
        mbmi->mode = this_mode;
753
        mbmi->tx_size = intra_tx_size;
754
755
        mbmi->ref_frame[0] = INTRA_FRAME;
        mbmi->uv_mode = this_mode;
756
        mbmi->mv[0].as_int = INVALID_MV;
757
      } else {
758
        x->skip_txfm[0] = skip_txfm;
759
760
      }
    }
761
762
    if (cpi->sf.reuse_inter_pred_sby)
      pd->dst = orig_dst;
763
  }
764

765
766
  return INT64_MAX;
}