vp9_pickmode.c 34 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
/*
 *  Copyright (c) 2014 The WebM project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include <assert.h>
12
13
14
#include <limits.h>
#include <math.h>
#include <stdio.h>
15

16
17
18
19
#include "./vp9_rtcd.h"

#include "vpx_mem/vpx_mem.h"

20
#include "vp9/common/vp9_blockd.h"
21
22
#include "vp9/common/vp9_common.h"
#include "vp9/common/vp9_mvref_common.h"
23
24
#include "vp9/common/vp9_reconinter.h"
#include "vp9/common/vp9_reconintra.h"
25

Dmitry Kovalev's avatar
Dmitry Kovalev committed
26
#include "vp9/encoder/vp9_encoder.h"
27
#include "vp9/encoder/vp9_pickmode.h"
28
#include "vp9/encoder/vp9_ratectrl.h"
29
#include "vp9/encoder/vp9_rd.h"
30

31
32
33
34
35
36
typedef struct {
  uint8_t *data;
  int stride;
  int in_use;
} PRED_BUFFER;

37
static int mv_refs_rt(const VP9_COMMON *cm, const MACROBLOCKD *xd,
38
39
40
41
                      const TileInfo *const tile,
                      MODE_INFO *mi, MV_REFERENCE_FRAME ref_frame,
                      int_mv *mv_ref_list,
                      int mi_row, int mi_col) {
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
  const int *ref_sign_bias = cm->ref_frame_sign_bias;
  int i, refmv_count = 0;

  const POSITION *const mv_ref_search = mv_ref_blocks[mi->mbmi.sb_type];

  int different_ref_found = 0;
  int context_counter = 0;
  int const_motion = 0;

  // Blank the reference vector list
  vpx_memset(mv_ref_list, 0, sizeof(*mv_ref_list) * MAX_MV_REF_CANDIDATES);

  // The nearest 2 blocks are treated differently
  // if the size < 8x8 we get the mv from the bmi substructure,
  // and we also need to keep a mode count.
  for (i = 0; i < 2; ++i) {
    const POSITION *const mv_ref = &mv_ref_search[i];
    if (is_inside(tile, mi_col, mi_row, cm->mi_rows, mv_ref)) {
      const MODE_INFO *const candidate_mi = xd->mi[mv_ref->col + mv_ref->row *
hkuang's avatar
hkuang committed
61
                                                   xd->mi_stride].src_mi;
62
63
64
65
66
67
      const MB_MODE_INFO *const candidate = &candidate_mi->mbmi;
      // Keep counts for entropy encoding.
      context_counter += mode_2_counter[candidate->mode];
      different_ref_found = 1;

      if (candidate->ref_frame[0] == ref_frame)
68
69
        ADD_MV_REF_LIST(get_sub_block_mv(candidate_mi, 0, mv_ref->col, -1),
                        refmv_count, mv_ref_list, Done);
70
71
72
73
74
75
76
77
78
79
80
81
    }
  }

  const_motion = 1;

  // Check the rest of the neighbors in much the same way
  // as before except we don't need to keep track of sub blocks or
  // mode counts.
  for (; i < MVREF_NEIGHBOURS && !refmv_count; ++i) {
    const POSITION *const mv_ref = &mv_ref_search[i];
    if (is_inside(tile, mi_col, mi_row, cm->mi_rows, mv_ref)) {
      const MB_MODE_INFO *const candidate = &xd->mi[mv_ref->col + mv_ref->row *
hkuang's avatar
hkuang committed
82
                                                    xd->mi_stride].src_mi->mbmi;
83
84
85
      different_ref_found = 1;

      if (candidate->ref_frame[0] == ref_frame)
86
        ADD_MV_REF_LIST(candidate->mv[0], refmv_count, mv_ref_list, Done);
87
88
89
90
91
92
93
94
95
96
97
    }
  }

  // Since we couldn't find 2 mvs from the same reference frame
  // go back through the neighbors and find motion vectors from
  // different reference frames.
  if (different_ref_found && !refmv_count) {
    for (i = 0; i < MVREF_NEIGHBOURS; ++i) {
      const POSITION *mv_ref = &mv_ref_search[i];
      if (is_inside(tile, mi_col, mi_row, cm->mi_rows, mv_ref)) {
        const MB_MODE_INFO *const candidate = &xd->mi[mv_ref->col + mv_ref->row
hkuang's avatar
hkuang committed
98
                                              * xd->mi_stride].src_mi->mbmi;
99
100

        // If the candidate is INTRA we don't want to consider its mv.
101
102
        IF_DIFF_REF_FRAME_ADD_MV(candidate, ref_frame, ref_sign_bias,
                                 refmv_count, mv_ref_list, Done);
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
      }
    }
  }

 Done:

  mi->mbmi.mode_context[ref_frame] = counter_to_context[context_counter];

  // Clamp vectors
  for (i = 0; i < MAX_MV_REF_CANDIDATES; ++i)
    clamp_mv_ref(&mv_ref_list[i].as_mv, xd);

  return const_motion;
}

118
119
120
121
static int combined_motion_search(VP9_COMP *cpi, MACROBLOCK *x,
                                  BLOCK_SIZE bsize, int mi_row, int mi_col,
                                  int_mv *tmp_mv, int *rate_mv,
                                  int64_t best_rd_sofar) {
122
  MACROBLOCKD *xd = &x->e_mbd;
hkuang's avatar
hkuang committed
123
  MB_MODE_INFO *mbmi = &xd->mi[0].src_mi->mbmi;
124
  struct buf_2d backup_yv12[MAX_MB_PLANE] = {{0, 0}};
125
126
  const int step_param = cpi->sf.mv.fullpel_search_step_param;
  const int sadpb = x->sadperbit16;
127
  MV mvp_full;
128
  const int ref = mbmi->ref_frame[0];
129
  const MV ref_mv = mbmi->ref_mvs[ref][0].as_mv;
130
131
132
133
134
135
136
  int dis;
  int rate_mode;
  const int tmp_col_min = x->mv_col_min;
  const int tmp_col_max = x->mv_col_max;
  const int tmp_row_min = x->mv_row_min;
  const int tmp_row_max = x->mv_row_max;
  int rv = 0;
137
  int cost_list[5];
138
139
  const YV12_BUFFER_CONFIG *scaled_ref_frame = vp9_get_scaled_ref_frame(cpi,
                                                                        ref);
Yaowu Xu's avatar
Yaowu Xu committed
140
141
142
143
  if (cpi->common.show_frame &&
      (x->pred_mv_sad[ref] >> 3) > x->pred_mv_sad[LAST_FRAME])
    return rv;

144
145
146
147
148
149
150
  if (scaled_ref_frame) {
    int i;
    // Swap out the reference frame for a version that's been scaled to
    // match the resolution of the current frame, allowing the existing
    // motion search code to be used without additional modifications.
    for (i = 0; i < MAX_MB_PLANE; i++)
      backup_yv12[i] = xd->plane[i].pre[0];
151
    vp9_setup_pre_planes(xd, 0, scaled_ref_frame, mi_row, mi_col, NULL);
152
  }
153
  vp9_set_mv_search_range(x, &ref_mv);
154

Yaowu Xu's avatar
Yaowu Xu committed
155
156
157
158
  assert(x->mv_best_ref_index[ref] <= 2);
  if (x->mv_best_ref_index[ref] < 2)
    mvp_full = mbmi->ref_mvs[ref][x->mv_best_ref_index[ref]].as_mv;
  else
Dmitry Kovalev's avatar
Dmitry Kovalev committed
159
    mvp_full = x->pred_mv[ref];
160
161
162
163

  mvp_full.col >>= 3;
  mvp_full.row >>= 3;

164
  vp9_full_pixel_search(cpi, x, bsize, &mvp_full, step_param, sadpb,
165
                        cond_cost_list(cpi, cost_list),
166
                        &ref_mv, &tmp_mv->as_mv, INT_MAX, 0);
167

168
169
170
171
172
  x->mv_col_min = tmp_col_min;
  x->mv_col_max = tmp_col_max;
  x->mv_row_min = tmp_row_min;
  x->mv_row_max = tmp_row_max;

173
174
175
  // calculate the bit cost on motion vector
  mvp_full.row = tmp_mv->as_mv.row * 8;
  mvp_full.col = tmp_mv->as_mv.col * 8;
176

177
178
  *rate_mv = vp9_mv_bit_cost(&mvp_full, &ref_mv,
                             x->nmvjointcost, x->mvcost, MV_COST_WEIGHT);
179

180
181
182
183
184
185
186
187
188
189
190
191
  rate_mode = cpi->inter_mode_cost[mbmi->mode_context[ref]]
                                  [INTER_OFFSET(NEWMV)];
  rv = !(RDCOST(x->rdmult, x->rddiv, (*rate_mv + rate_mode), 0) >
         best_rd_sofar);

  if (rv) {
    cpi->find_fractional_mv_step(x, &tmp_mv->as_mv, &ref_mv,
                                 cpi->common.allow_high_precision_mv,
                                 x->errorperbit,
                                 &cpi->fn_ptr[bsize],
                                 cpi->sf.mv.subpel_force_stop,
                                 cpi->sf.mv.subpel_iters_per_step,
192
                                 cond_cost_list(cpi, cost_list),
193
                                 x->nmvjointcost, x->mvcost,
194
                                 &dis, &x->pred_sse[ref], NULL, 0, 0);
195
    x->pred_mv[ref] = tmp_mv->as_mv;
196
197
198
199
200
201
202
  }

  if (scaled_ref_frame) {
    int i;
    for (i = 0; i < MAX_MB_PLANE; i++)
      xd->plane[i].pre[0] = backup_yv12[i];
  }
203
  return rv;
204
205
}

206

207
208
static void model_rd_for_sb_y(VP9_COMP *cpi, BLOCK_SIZE bsize,
                              MACROBLOCK *x, MACROBLOCKD *xd,
209
210
                              int *out_rate_sum, int64_t *out_dist_sum,
                              unsigned int *var_y, unsigned int *sse_y) {
211
212
213
214
  // Note our transform coeffs are 8 times an orthogonal transform.
  // Hence quantizer step is also 8 times. To get effective quantizer
  // we need to divide by 8 before sending to modeling function.
  unsigned int sse;
215
216
  int rate;
  int64_t dist;
217
218
  struct macroblock_plane *const p = &x->plane[0];
  struct macroblockd_plane *const pd = &xd->plane[0];
219
220
  const uint32_t dc_quant = pd->dequant[0];
  const uint32_t ac_quant = pd->dequant[1];
221
222
  unsigned int var = cpi->fn_ptr[bsize].vf(p->src.buf, p->src.stride,
                                           pd->dst.buf, pd->dst.stride, &sse);
223
224
225
  *var_y = var;
  *sse_y = sse;

226
  if (sse < dc_quant * dc_quant >> 6)
227
    x->skip_txfm[0] = 1;
228
  else if (var < ac_quant * ac_quant >> 6)
229
    x->skip_txfm[0] = 2;
230
  else
231
    x->skip_txfm[0] = 0;
232

233
234
  if (cpi->common.tx_mode == TX_MODE_SELECT) {
    if (sse > (var << 2))
hkuang's avatar
hkuang committed
235
      xd->mi[0].src_mi->mbmi.tx_size =
236
237
          MIN(max_txsize_lookup[bsize],
              tx_mode_to_biggest_tx_size[cpi->common.tx_mode]);
238
    else
hkuang's avatar
hkuang committed
239
      xd->mi[0].src_mi->mbmi.tx_size = TX_8X8;
240
241
242
243

    if (cpi->sf.partition_search_type == VAR_BASED_PARTITION &&
        xd->mi[0].src_mi->mbmi.tx_size > TX_16X16)
      xd->mi[0].src_mi->mbmi.tx_size = TX_16X16;
244
  } else {
hkuang's avatar
hkuang committed
245
    xd->mi[0].src_mi->mbmi.tx_size =
246
247
        MIN(max_txsize_lookup[bsize],
            tx_mode_to_biggest_tx_size[cpi->common.tx_mode]);
248
249
  }

250
251
252
253
254
255
256
257
258
#if CONFIG_VP9_HIGHBITDEPTH
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
    vp9_model_rd_from_var_lapndz(sse - var, 1 << num_pels_log2_lookup[bsize],
                                 dc_quant >> (xd->bd - 5), &rate, &dist);
  } else {
    vp9_model_rd_from_var_lapndz(sse - var, 1 << num_pels_log2_lookup[bsize],
                                 dc_quant >> 3, &rate, &dist);
  }
#else
259
260
  vp9_model_rd_from_var_lapndz(sse - var, 1 << num_pels_log2_lookup[bsize],
                               dc_quant >> 3, &rate, &dist);
261
262
#endif  // CONFIG_VP9_HIGHBITDEPTH

263
  *out_rate_sum = rate >> 1;
264
  *out_dist_sum = dist << 3;
265

266
267
#if CONFIG_VP9_HIGHBITDEPTH
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
268
269
    vp9_model_rd_from_var_lapndz(var, 1 << num_pels_log2_lookup[bsize],
                                 ac_quant >> (xd->bd - 5), &rate, &dist);
270
  } else {
271
272
    vp9_model_rd_from_var_lapndz(var, 1 << num_pels_log2_lookup[bsize],
                                 ac_quant >> 3, &rate, &dist);
273
274
  }
#else
275
276
  vp9_model_rd_from_var_lapndz(var, 1 << num_pels_log2_lookup[bsize],
                               ac_quant >> 3, &rate, &dist);
277
278
#endif  // CONFIG_VP9_HIGHBITDEPTH

279
280
  *out_rate_sum += rate;
  *out_dist_sum += dist << 4;
281
282
283

  if (*out_rate_sum == 0)
    x->skip_txfm[0] = 1;
284
285
}

286
287
288
289
290
291
292
293
294
295
296
297
298
static int get_pred_buffer(PRED_BUFFER *p, int len) {
  int i;

  for (i = 0; i < len; i++) {
    if (!p[i].in_use) {
      p[i].in_use = 1;
      return i;
    }
  }
  return -1;
}

static void free_pred_buffer(PRED_BUFFER *p) {
299
300
  if (p != NULL)
    p->in_use = 0;
301
302
}

303
304
305
306
307
308
309
310
static void encode_breakout_test(VP9_COMP *cpi, MACROBLOCK *x,
                                 BLOCK_SIZE bsize, int mi_row, int mi_col,
                                 MV_REFERENCE_FRAME ref_frame,
                                 PREDICTION_MODE this_mode,
                                 unsigned int var_y, unsigned int sse_y,
                                 struct buf_2d yv12_mb[][MAX_MB_PLANE],
                                 int *rate, int64_t *dist) {
  MACROBLOCKD *xd = &x->e_mbd;
hkuang's avatar
hkuang committed
311
  MB_MODE_INFO *mbmi = &xd->mi[0].src_mi->mbmi;
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326

  const BLOCK_SIZE uv_size = get_plane_block_size(bsize, &xd->plane[1]);
  unsigned int var = var_y, sse = sse_y;
  // Skipping threshold for ac.
  unsigned int thresh_ac;
  // Skipping threshold for dc.
  unsigned int thresh_dc;
  if (x->encode_breakout > 0) {
    // Set a maximum for threshold to avoid big PSNR loss in low bit rate
    // case. Use extreme low threshold for static frames to limit
    // skipping.
    const unsigned int max_thresh = 36000;
    // The encode_breakout input
    const unsigned int min_thresh =
        MIN(((unsigned int)x->encode_breakout << 4), max_thresh);
327
328
329
#if CONFIG_VP9_HIGHBITDEPTH
    const int shift = 2 * xd->bd - 16;
#endif
330
331
332

    // Calculate threshold according to dequant value.
    thresh_ac = (xd->plane[0].dequant[1] * xd->plane[0].dequant[1]) / 9;
333
334
335
336
337
#if CONFIG_VP9_HIGHBITDEPTH
    if ((xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) && shift > 0) {
      thresh_ac = ROUND_POWER_OF_TWO(thresh_ac, shift);
    }
#endif  // CONFIG_VP9_HIGHBITDEPTH
338
339
340
341
    thresh_ac = clamp(thresh_ac, min_thresh, max_thresh);

    // Adjust ac threshold according to partition size.
    thresh_ac >>=
342
        8 - (b_width_log2_lookup[bsize] + b_height_log2_lookup[bsize]);
343
344

    thresh_dc = (xd->plane[0].dequant[0] * xd->plane[0].dequant[0] >> 6);
345
346
347
348
349
#if CONFIG_VP9_HIGHBITDEPTH
    if ((xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) && shift > 0) {
      thresh_dc = ROUND_POWER_OF_TWO(thresh_dc, shift);
    }
#endif  // CONFIG_VP9_HIGHBITDEPTH
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
  } else {
    thresh_ac = 0;
    thresh_dc = 0;
  }

  // Y skipping condition checking for ac and dc.
  if (var <= thresh_ac && (sse - var) <= thresh_dc) {
    unsigned int sse_u, sse_v;
    unsigned int var_u, var_v;

    // Skip UV prediction unless breakout is zero (lossless) to save
    // computation with low impact on the result
    if (x->encode_breakout == 0) {
      xd->plane[1].pre[0] = yv12_mb[ref_frame][1];
      xd->plane[2].pre[0] = yv12_mb[ref_frame][2];
      vp9_build_inter_predictors_sbuv(xd, mi_row, mi_col, bsize);
    }

    var_u = cpi->fn_ptr[uv_size].vf(x->plane[1].src.buf,
                                    x->plane[1].src.stride,
                                    xd->plane[1].dst.buf,
                                    xd->plane[1].dst.stride, &sse_u);

    // U skipping condition checking
    if ((var_u * 4 <= thresh_ac) && (sse_u - var_u <= thresh_dc)) {
      var_v = cpi->fn_ptr[uv_size].vf(x->plane[2].src.buf,
                                      x->plane[2].src.stride,
                                      xd->plane[2].dst.buf,
                                      xd->plane[2].dst.stride, &sse_v);

      // V skipping condition checking
      if ((var_v * 4 <= thresh_ac) && (sse_v - var_v <= thresh_dc)) {
        x->skip = 1;

        // The cost of skip bit needs to be added.
        *rate = cpi->inter_mode_cost[mbmi->mode_context[ref_frame]]
386
                                    [INTER_OFFSET(this_mode)];
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402

        // More on this part of rate
        // rate += vp9_cost_bit(vp9_get_skip_prob(cm, xd), 1);

        // Scaling factor for SSE from spatial domain to frequency
        // domain is 16. Adjust distortion accordingly.
        // TODO(yunqingwang): In this function, only y-plane dist is
        // calculated.
        *dist = (sse << 4);  // + ((sse_u + sse_v) << 4);

        // *disable_skip = 1;
      }
    }
  }
}

403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
struct estimate_block_intra_args {
  VP9_COMP *cpi;
  MACROBLOCK *x;
  PREDICTION_MODE mode;
  int rate;
  int64_t dist;
};

static void estimate_block_intra(int plane, int block, BLOCK_SIZE plane_bsize,
                                 TX_SIZE tx_size, void *arg) {
  struct estimate_block_intra_args* const args = arg;
  VP9_COMP *const cpi = args->cpi;
  MACROBLOCK *const x = args->x;
  MACROBLOCKD *const xd = &x->e_mbd;
  struct macroblock_plane *const p = &x->plane[0];
  struct macroblockd_plane *const pd = &xd->plane[0];
  const BLOCK_SIZE bsize_tx = txsize_to_bsize[tx_size];
  uint8_t *const src_buf_base = p->src.buf;
  uint8_t *const dst_buf_base = pd->dst.buf;
  const int src_stride = p->src.stride;
  const int dst_stride = pd->dst.stride;
  int i, j;
  int rate;
  int64_t dist;
  unsigned int var_y, sse_y;
  txfrm_block_to_raster_xy(plane_bsize, tx_size, block, &i, &j);
  assert(plane == 0);
  (void) plane;

  p->src.buf = &src_buf_base[4 * (j * src_stride + i)];
  pd->dst.buf = &dst_buf_base[4 * (j * dst_stride + i)];
  // Use source buffer as an approximation for the fully reconstructed buffer.
  vp9_predict_intra_block(xd, block >> (2 * tx_size),
436
                          b_width_log2_lookup[plane_bsize],
437
                          tx_size, args->mode,
438
439
                          x->skip_encode ? p->src.buf : pd->dst.buf,
                          x->skip_encode ? src_stride : dst_stride,
440
441
442
443
444
445
446
447
448
449
                          pd->dst.buf, dst_stride,
                          i, j, 0);
  // This procedure assumes zero offset from p->src.buf and pd->dst.buf.
  model_rd_for_sb_y(cpi, bsize_tx, x, xd, &rate, &dist, &var_y, &sse_y);
  p->src.buf = src_buf_base;
  pd->dst.buf = dst_buf_base;
  args->rate += rate;
  args->dist += dist;
}

450
static const THR_MODES mode_idx[MAX_REF_FRAMES][4] = {
451
  {THR_DC, THR_H_PRED, THR_V_PRED, THR_TM},
452
453
454
455
456
  {THR_NEARESTMV, THR_NEARMV, THR_ZEROMV, THR_NEWMV},
  {THR_NEARESTG, THR_NEARG, THR_ZEROG, THR_NEWG},
  {THR_NEARESTA, THR_NEARA, THR_ZEROA, THR_NEWA},
};

457
458
459
460
static const PREDICTION_MODE intra_mode_list[] = {
  DC_PRED, V_PRED, H_PRED, TM_PRED
};

461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
void vp9_pick_intra_mode(VP9_COMP *cpi, MACROBLOCK *x, RD_COST *rd_cost,
                         BLOCK_SIZE bsize, PICK_MODE_CONTEXT *ctx) {
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *const mbmi = &xd->mi[0].src_mi->mbmi;
  RD_COST this_rdc, best_rdc;
  PREDICTION_MODE this_mode;
  struct estimate_block_intra_args args = { cpi, x, DC_PRED, 0, 0 };
  const TX_SIZE intra_tx_size =
      MIN(max_txsize_lookup[bsize],
          tx_mode_to_biggest_tx_size[cpi->common.tx_mode]);
  MODE_INFO *const mic = xd->mi[0].src_mi;
  int *bmode_costs;
  const MODE_INFO *above_mi = xd->mi[-xd->mi_stride].src_mi;
  const MODE_INFO *left_mi = xd->left_available ? xd->mi[-1].src_mi : NULL;
  const PREDICTION_MODE A = vp9_above_block_mode(mic, above_mi, 0);
  const PREDICTION_MODE L = vp9_left_block_mode(mic, left_mi, 0);
  bmode_costs = cpi->y_mode_costs[A][L];

  (void) ctx;
  vp9_rd_cost_reset(&best_rdc);
  vp9_rd_cost_reset(&this_rdc);

  mbmi->ref_frame[0] = INTRA_FRAME;
  mbmi->mv[0].as_int = INVALID_MV;
  mbmi->uv_mode = DC_PRED;
  vpx_memset(x->skip_txfm, 0, sizeof(x->skip_txfm));

  // Change the limit of this loop to add other intra prediction
  // mode tests.
  for (this_mode = DC_PRED; this_mode <= H_PRED; ++this_mode) {
    args.mode = this_mode;
    args.rate = 0;
    args.dist = 0;
    mbmi->tx_size = intra_tx_size;
    vp9_foreach_transformed_block_in_plane(xd, bsize, 0,
                                           estimate_block_intra, &args);
    this_rdc.rate = args.rate;
    this_rdc.dist = args.dist;
    this_rdc.rate += bmode_costs[this_mode];
    this_rdc.rdcost = RDCOST(x->rdmult, x->rddiv,
                             this_rdc.rate, this_rdc.dist);

    if (this_rdc.rdcost < best_rdc.rdcost) {
      best_rdc = this_rdc;
      mbmi->mode = this_mode;
    }
  }

  *rd_cost = best_rdc;
}

512
513
// TODO(jingning) placeholder for inter-frame non-RD mode decision.
// this needs various further optimizations. to be continued..
514
void vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
515
                         TileDataEnc *tile_data,
516
517
                         int mi_row, int mi_col, RD_COST *rd_cost,
                         BLOCK_SIZE bsize, PICK_MODE_CONTEXT *ctx) {
518
  VP9_COMMON *const cm = &cpi->common;
519
  TileInfo *const tile_info = &tile_data->tile_info;
520
  MACROBLOCKD *const xd = &x->e_mbd;
hkuang's avatar
hkuang committed
521
  MB_MODE_INFO *const mbmi = &xd->mi[0].src_mi->mbmi;
522
  struct macroblockd_plane *const pd = &xd->plane[0];
523
  PREDICTION_MODE best_mode = ZEROMV;
524
  MV_REFERENCE_FRAME ref_frame, best_ref_frame = LAST_FRAME;
525
  TX_SIZE best_tx_size = MIN(max_txsize_lookup[bsize],
526
                             tx_mode_to_biggest_tx_size[cm->tx_mode]);
527
  INTERP_FILTER best_pred_filter = EIGHTTAP;
528
529
530
531
  int_mv frame_mv[MB_MODE_COUNT][MAX_REF_FRAMES];
  struct buf_2d yv12_mb[4][MAX_MB_PLANE];
  static const int flag_list[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG,
                                    VP9_ALT_FLAG };
532
  RD_COST this_rdc, best_rdc;
533
  uint8_t skip_txfm = 0, best_mode_skip_txfm = 0;
534
535
536
  // var_y and sse_y are saved to be used in skipping checking
  unsigned int var_y = UINT_MAX;
  unsigned int sse_y = UINT_MAX;
537
538
539
540
  // Reduce the intra cost penalty for small blocks (<=16x16).
  const int reduction_fac =
      (cpi->sf.partition_search_type == VAR_BASED_PARTITION &&
       bsize <= BLOCK_16X16) ? 4 : 1;
541
  const int intra_cost_penalty = vp9_get_intra_cost_penalty(
542
      cm->base_qindex, cm->y_dc_delta_q, cm->bit_depth) / reduction_fac;
543
544
  const int64_t inter_mode_thresh = RDCOST(x->rdmult, x->rddiv,
                                           intra_cost_penalty, 0);
545
  const int8_t segment_id = mbmi->segment_id;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
546
  const int *const rd_threshes = cpi->rd.threshes[segment_id][bsize];
547
  const int *const rd_thresh_freq_fact = tile_data->thresh_freq_fact[bsize];
Yaowu Xu's avatar
Yaowu Xu committed
548
  INTERP_FILTER filter_ref;
549
  const int bsl = mi_width_log2_lookup[bsize];
550
  const int pred_filter_search = cm->interp_filter == SWITCHABLE ?
551
552
      (((mi_row + mi_col) >> bsl) +
       get_chessboard_index(cm->current_video_frame)) & 0x1 : 0;
553
  int const_motion[MAX_REF_FRAMES] = { 0 };
554
555
  const int bh = num_4x4_blocks_high_lookup[bsize] << 2;
  const int bw = num_4x4_blocks_wide_lookup[bsize] << 2;
556
557
  // For speed 6, the result of interp filter is reused later in actual encoding
  // process.
558
559
560
  // tmp[3] points to dst buffer, and the other 3 point to allocated buffers.
  PRED_BUFFER tmp[4];
  DECLARE_ALIGNED_ARRAY(16, uint8_t, pred_buf, 3 * 64 * 64);
561
562
563
#if CONFIG_VP9_HIGHBITDEPTH
  DECLARE_ALIGNED_ARRAY(16, uint16_t, pred_buf_16, 3 * 64 * 64);
#endif
564
565
566
  struct buf_2d orig_dst = pd->dst;
  PRED_BUFFER *best_pred = NULL;
  PRED_BUFFER *this_mode_pred = NULL;
567
  const int pixels_in_block = bh * bw;
568
  int reuse_inter_pred = cpi->sf.reuse_inter_pred_sby && ctx->pred_pixel_ready;
569
  int ref_frame_skip_mask = 0;
570

571
  if (reuse_inter_pred) {
572
    int i;
573
    for (i = 0; i < 3; i++) {
574
575
576
577
578
579
580
581
#if CONFIG_VP9_HIGHBITDEPTH
      if (cm->use_highbitdepth)
        tmp[i].data = CONVERT_TO_BYTEPTR(&pred_buf_16[pixels_in_block * i]);
      else
        tmp[i].data = &pred_buf[pixels_in_block * i];
#else
      tmp[i].data = &pred_buf[pixels_in_block * i];
#endif  // CONFIG_VP9_HIGHBITDEPTH
582
583
584
585
586
587
588
589
      tmp[i].stride = bw;
      tmp[i].in_use = 0;
    }
    tmp[3].data = pd->dst.buf;
    tmp[3].stride = pd->dst.stride;
    tmp[3].in_use = 0;
  }

590
591
  x->skip_encode = cpi->sf.skip_encode_frame && x->q_index < QIDX_SKIP_THRESH;
  x->skip = 0;
592

Yaowu Xu's avatar
Yaowu Xu committed
593
594
595
596
597
598
599
  if (xd->up_available)
    filter_ref = xd->mi[-xd->mi_stride].src_mi->mbmi.interp_filter;
  else if (xd->left_available)
    filter_ref = xd->mi[-1].src_mi->mbmi.interp_filter;
  else
    filter_ref = cm->interp_filter;

600
  // initialize mode decisions
601
602
  vp9_rd_cost_reset(&best_rdc);
  vp9_rd_cost_reset(rd_cost);
603
604
605
606
  mbmi->sb_type = bsize;
  mbmi->ref_frame[0] = NONE;
  mbmi->ref_frame[1] = NONE;
  mbmi->tx_size = MIN(max_txsize_lookup[bsize],
607
608
609
                      tx_mode_to_biggest_tx_size[cm->tx_mode]);
  mbmi->interp_filter = cm->interp_filter == SWITCHABLE ?
                        EIGHTTAP : cm->interp_filter;
610
  mbmi->segment_id = segment_id;
611

612
  for (ref_frame = LAST_FRAME; ref_frame <= GOLDEN_FRAME; ++ref_frame) {
613
    x->pred_mv_sad[ref_frame] = INT_MAX;
614
615
616
    frame_mv[NEWMV][ref_frame].as_int = INVALID_MV;
    frame_mv[ZEROMV][ref_frame].as_int = 0;

617
    if (cpi->ref_frame_flags & flag_list[ref_frame]) {
618
619
620
      const YV12_BUFFER_CONFIG *yv12 = get_ref_frame_buffer(cpi, ref_frame);
      int_mv *const candidates = mbmi->ref_mvs[ref_frame];
      const struct scale_factors *const sf = &cm->frame_refs[ref_frame - 1].sf;
621

622
623
624
      vp9_setup_pred_block(xd, yv12_mb[ref_frame], yv12, mi_row, mi_col,
                           sf, sf);

625
      if (!cm->error_resilient_mode)
626
        vp9_find_mv_refs(cm, xd, tile_info, xd->mi[0].src_mi, ref_frame,
627
628
                         candidates, mi_row, mi_col);
      else
629
630
        const_motion[ref_frame] = mv_refs_rt(cm, xd, tile_info,
                                             xd->mi[0].src_mi,
631
632
633
634
635
636
637
638
639
640
                                             ref_frame, candidates,
                                             mi_row, mi_col);

      vp9_find_best_ref_mvs(xd, cm->allow_high_precision_mv, candidates,
                            &frame_mv[NEARESTMV][ref_frame],
                            &frame_mv[NEARMV][ref_frame]);

      if (!vp9_is_scaled(sf) && bsize >= BLOCK_8X8)
        vp9_mv_pred(cpi, x, yv12_mb[ref_frame][0].buf, yv12->y_stride,
                    ref_frame, bsize);
641
    } else {
642
      ref_frame_skip_mask |= (1 << ref_frame);
643
    }
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
  }

  for (ref_frame = LAST_FRAME; ref_frame <= GOLDEN_FRAME; ++ref_frame) {
    PREDICTION_MODE this_mode;
    int i = (ref_frame == LAST_FRAME) ? GOLDEN_FRAME : LAST_FRAME;

    if (!(cpi->ref_frame_flags & flag_list[ref_frame]))
      continue;

    if (cpi->ref_frame_flags & flag_list[i])
      if (x->pred_mv_sad[ref_frame] > (x->pred_mv_sad[i] << 1))
        ref_frame_skip_mask |= (1 << ref_frame);

    if (ref_frame_skip_mask & (1 << ref_frame))
      continue;
659
660
661
662
663
664
665

    // Select prediction reference frames.
    xd->plane[0].pre[0] = yv12_mb[ref_frame][0];

    clamp_mv2(&frame_mv[NEARESTMV][ref_frame].as_mv, xd);
    clamp_mv2(&frame_mv[NEARMV][ref_frame].as_mv, xd);

666
667
    mbmi->ref_frame[0] = ref_frame;

668
    for (this_mode = NEARESTMV; this_mode <= NEWMV; ++this_mode) {
669
      int rate_mv = 0;
670
      int mode_rd_thresh;
671
      int mode_index = mode_idx[ref_frame][INTER_OFFSET(this_mode)];
672

673
      if (const_motion[ref_frame] && this_mode == NEARMV)
674
675
        continue;

676
      if (!(cpi->sf.inter_mode_mask[bsize] & (1 << this_mode)))
677
        continue;
Jim Bankoski's avatar
Jim Bankoski committed
678

679
680
      mode_rd_thresh = best_mode_skip_txfm ? rd_threshes[mode_index] << 1 :
                                             rd_threshes[mode_index];
681
      if (rd_less_than_thresh(best_rdc.rdcost, mode_rd_thresh,
682
                              rd_thresh_freq_fact[mode_index]))
683
684
        continue;

Jim Bankoski's avatar
Jim Bankoski committed
685
      if (this_mode == NEWMV) {
686
687
        if (ref_frame > LAST_FRAME)
          continue;
688
        if (cpi->sf.partition_search_type != VAR_BASED_PARTITION &&
689
            best_rdc.rdcost < (int64_t)(1 << num_pels_log2_lookup[bsize]))
690
          continue;
691
692
        if (!combined_motion_search(cpi, x, bsize, mi_row, mi_col,
                                    &frame_mv[NEWMV][ref_frame],
693
                                    &rate_mv, best_rdc.rdcost))
Jim Bankoski's avatar
Jim Bankoski committed
694
695
696
          continue;
      }

697
698
699
      if (this_mode != NEARESTMV &&
          frame_mv[this_mode][ref_frame].as_int ==
              frame_mv[NEARESTMV][ref_frame].as_int)
700
        continue;
701

702
703
      mbmi->mode = this_mode;
      mbmi->mv[0].as_int = frame_mv[this_mode][ref_frame].as_int;
704

705
706
707
      // Search for the best prediction filter type, when the resulting
      // motion vector is at sub-pixel accuracy level for luma component, i.e.,
      // the last three bits are all zeros.
708
      if (reuse_inter_pred) {
709
        if (!this_mode_pred) {
710
711
712
713
714
715
716
717
          this_mode_pred = &tmp[3];
        } else {
          this_mode_pred = &tmp[get_pred_buffer(tmp, 3)];
          pd->dst.buf = this_mode_pred->data;
          pd->dst.stride = bw;
        }
      }

718
      if ((this_mode == NEWMV || filter_ref == SWITCHABLE) &&
719
          pred_filter_search &&
720
721
722
723
          ((mbmi->mv[0].as_mv.row & 0x07) != 0 ||
           (mbmi->mv[0].as_mv.col & 0x07) != 0)) {
        int pf_rate[3];
        int64_t pf_dist[3];
724
725
        unsigned int pf_var[3];
        unsigned int pf_sse[3];
726
        TX_SIZE pf_tx_size[3];
727
728
        int64_t best_cost = INT64_MAX;
        INTERP_FILTER best_filter = SWITCHABLE, filter;
729
        PRED_BUFFER *current_pred = this_mode_pred;
730
731
732
733
734
735

        for (filter = EIGHTTAP; filter <= EIGHTTAP_SHARP; ++filter) {
          int64_t cost;
          mbmi->interp_filter = filter;
          vp9_build_inter_predictors_sby(xd, mi_row, mi_col, bsize);
          model_rd_for_sb_y(cpi, bsize, x, xd, &pf_rate[filter],
736
                            &pf_dist[filter], &pf_var[filter], &pf_sse[filter]);
737
          cost = RDCOST(x->rdmult, x->rddiv,
738
                        vp9_get_switchable_rate(cpi, xd) + pf_rate[filter],
739
                        pf_dist[filter]);
740
          pf_tx_size[filter] = mbmi->tx_size;
741
          if (cost < best_cost) {
742
743
            best_filter = filter;
            best_cost = cost;
744
            skip_txfm = x->skip_txfm[0];
745

746
            if (reuse_inter_pred) {
747
748
749
750
751
752
753
754
755
756
757
              if (this_mode_pred != current_pred) {
                free_pred_buffer(this_mode_pred);
                this_mode_pred = current_pred;
              }

              if (filter < EIGHTTAP_SHARP) {
                current_pred = &tmp[get_pred_buffer(tmp, 3)];
                pd->dst.buf = current_pred->data;
                pd->dst.stride = bw;
              }
            }
758
          }
759
760
        }

761
        if (reuse_inter_pred && this_mode_pred != current_pred)
762
763
          free_pred_buffer(current_pred);

764
        mbmi->interp_filter = best_filter;
765
        mbmi->tx_size = pf_tx_size[mbmi->interp_filter];
766
767
        this_rdc.rate = pf_rate[mbmi->interp_filter];
        this_rdc.dist = pf_dist[mbmi->interp_filter];
768
769
        var_y = pf_var[mbmi->interp_filter];
        sse_y = pf_sse[mbmi->interp_filter];
770
        x->skip_txfm[0] = skip_txfm;
771
772
773
      } else {
        mbmi->interp_filter = (filter_ref == SWITCHABLE) ? EIGHTTAP: filter_ref;
        vp9_build_inter_predictors_sby(xd, mi_row, mi_col, bsize);
774
775
        model_rd_for_sb_y(cpi, bsize, x, xd, &this_rdc.rate, &this_rdc.dist,
                          &var_y, &sse_y);
776
777
      }

778
779
      this_rdc.rate += rate_mv;
      this_rdc.rate += cpi->inter_mode_cost[mbmi->mode_context[ref_frame]]
780
                                  [INTER_OFFSET(this_mode)];
781
782
      this_rdc.rdcost = RDCOST(x->rdmult, x->rddiv,
                               this_rdc.rate, this_rdc.dist);
783

784
785
      // Skipping checking: test to see if this block can be reconstructed by
      // prediction only.
786
      if (cpi->allow_encode_breakout) {
787
        encode_breakout_test(cpi, x, bsize, mi_row, mi_col, ref_frame,
788
789
                             this_mode, var_y, sse_y, yv12_mb,
                             &this_rdc.rate, &this_rdc.dist);
790
        if (x->skip) {
791
792
793
          this_rdc.rate += rate_mv;
          this_rdc.rdcost = RDCOST(x->rdmult, x->rddiv,
                                   this_rdc.rate, this_rdc.dist);
794
795
796
        }
      }

797
#if CONFIG_VP9_TEMPORAL_DENOISING
798
      if (cpi->oxcf.noise_sensitivity > 0)
799
        vp9_denoiser_update_frame_stats(mbmi, sse_y, this_mode, ctx);
800
801
#else
      (void)ctx;
Tim Kopp's avatar
Tim Kopp committed
802
803
#endif

804
805
      if (this_rdc.rdcost < best_rdc.rdcost || x->skip) {
        best_rdc = this_rdc;
806
        best_mode = this_mode;
807
        best_pred_filter = mbmi->interp_filter;
808
        best_tx_size = mbmi->tx_size;
809
        best_ref_frame = ref_frame;
810
        best_mode_skip_txfm = x->skip_txfm[0];
811

812
        if (reuse_inter_pred) {
813
          free_pred_buffer(best_pred);
814
815
816
          best_pred = this_mode_pred;
        }
      } else {
817
        if (reuse_inter_pred)
818
          free_pred_buffer(this_mode_pred);
819
      }
820
821
822

      if (x->skip)
        break;
823
    }
824
825
826

    // Check that a prediction mode has been selected.
    assert(best_rdc.rdcost < INT64_MAX);
827
828
829

    if (x->skip)
      break;
830
831
  }

832
  mbmi->mode          = best_mode;
833
  mbmi->interp_filter = best_pred_filter;
834
835
836
  mbmi->tx_size       = best_tx_size;
  mbmi->ref_frame[0]  = best_ref_frame;
  mbmi->mv[0].as_int  = frame_mv[best_mode][best_ref_frame].as_int;
hkuang's avatar
hkuang committed
837
  xd->mi[0].src_mi->bmi[0].as_mv[0].as_int = mbmi->mv[0].as_int;
838
  x->skip_txfm[0] = best_mode_skip_txfm;
839

840
841
  // Perform intra prediction search, if the best SAD is above a certain
  // threshold.
842
  if (!x->skip && best_rdc.rdcost > inter_mode_thresh &&
843
      bsize <= cpi->sf.max_intra_bsize) {
844
845
846
847
    struct estimate_block_intra_args args = { cpi, x, DC_PRED, 0, 0 };
    const TX_SIZE intra_tx_size =
        MIN(max_txsize_lookup[bsize],
            tx_mode_to_biggest_tx_size[cpi->common.tx_mode]);
848
    int i;
849

850
851
852
    if (reuse_inter_pred && best_pred != NULL) {
      if (best_pred->data == orig_dst.buf) {
        this_mode_pred = &tmp[get_pred_buffer(tmp, 3)];
853
854
855
856
857
858
859
860
861
862
#if CONFIG_VP9_HIGHBITDEPTH
        if (cm->use_highbitdepth)
          vp9_highbd_convolve_copy(best_pred->data, best_pred->stride,
                                   this_mode_pred->data, this_mode_pred->stride,
                                   NULL, 0, NULL, 0, bw, bh, xd->bd);
        else
          vp9_convolve_copy(best_pred->data, best_pred->stride,
                          this_mode_pred->data, this_mode_pred->stride,
                          NULL, 0, NULL, 0, bw, bh);
#else
863
864
865
        vp9_convolve_copy(best_pred->data, best_pred->stride,
                          this_mode_pred->data, this_mode_pred->stride,
                          NULL, 0, NULL, 0, bw, bh);
866
#endif  // CONFIG_VP9_HIGHBITDEPTH
867
868
        best_pred = this_mode_pred;
      }
869
    }
870
    pd->dst = orig_dst;
871

872
    for (i = 0; i < 4; ++i) {
873
      const TX_SIZE saved_tx_size = mbmi->tx_size;
874
875
876
877
      const PREDICTION_MODE this_mode = intra_mode_list[i];
      if (!((1 << this_mode) & cpi->sf.intra_y_mode_mask[intra_tx_size]))
        continue;
      skip_txfm = x->skip_txfm[0];
878
879
880
881
882
883
      args.mode = this_mode;
      args.rate = 0;
      args.dist = 0;
      mbmi->tx_size = intra_tx_size;
      vp9_foreach_transformed_block_in_plane(xd, bsize, 0,
                                             estimate_block_intra, &args);
884
885
886
887
888
889
890
      this_rdc.rate = args.rate;
      this_rdc.dist = args.dist;
      this_rdc.rate += cpi->mbmode_cost[this_mode];
      this_rdc.rate += intra_cost_penalty;
      this_rdc.rdcost = RDCOST(x->rdmult, x->rddiv,
                               this_rdc.rate, this_rdc.dist);

891
      if (this_rdc.rdcost < best_rdc.rdcost) {
892
        best_rdc = this_rdc;
893
        mbmi->mode = this_mode;
894
        mbmi->tx_size = intra_tx_size;
895
896
        mbmi->ref_frame[0] = INTRA_FRAME;
        mbmi->uv_mode = this_mode;
897
        mbmi->mv[0].as_int = INVALID_MV;
898
      } else {
899
        x->skip_txfm[0] = best_mode_skip_txfm;
900
        mbmi->tx_size = saved_tx_size;
901
902
      }
    }
903
904
905
  }

  pd->dst = orig_dst;
906
907
908

  if (reuse_inter_pred && best_pred != NULL) {
    if (best_pred->data != orig_dst.buf && is_inter_mode(mbmi->mode)) {
909
#if CONFIG_VP9_HIGHBITDEPTH
910
911
912
913
914
915
916
917
918
      if (cm->use_highbitdepth)
        vp9_highbd_convolve_copy(best_pred->data, best_pred->stride,
                                 pd->dst.buf, pd->dst.stride, NULL, 0,
                                 NULL, 0, bw, bh, xd->bd);
      else
        vp9_convolve_copy(best_pred->data, best_pred->stride,
                          pd->dst.buf, pd->dst.stride, NULL, 0,
                          NULL, 0, bw, bh);
#else
919
920
921
      vp9_convolve_copy(best_pred->data, best_pred->stride,
                        pd->dst.buf, pd->dst.stride, NULL, 0,
                        NULL, 0, bw, bh);
922
#endif  // CONFIG_VP9_HIGHBITDEPTH
923
    }
924
  }
925

926
  if (is_inter_block(mbmi))
927
    vp9_update_rd_thresh_fact(tile_data->thresh_freq_fact,
928
929
                            cpi->sf.adaptive_rd_thresh, bsize,
                            mode_idx[best_ref_frame][INTER_OFFSET(mbmi->mode)]);
930
  else
931
932
    vp9_update_rd_thresh_fact(tile_data->thresh_freq_fact,
                              cpi->sf.adaptive_rd_thresh, bsize,
933
                              mode_idx[INTRA_FRAME][mbmi->mode]);
934

935
  *rd_cost = best_rdc;
936
}