vp9_pickmode.c 16.2 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
/*
 *  Copyright (c) 2014 The WebM project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include <assert.h>
12
13
14
#include <limits.h>
#include <math.h>
#include <stdio.h>
15

16
17
18
19
20
21
#include "./vp9_rtcd.h"

#include "vpx_mem/vpx_mem.h"

#include "vp9/common/vp9_common.h"
#include "vp9/common/vp9_mvref_common.h"
22
23
#include "vp9/common/vp9_reconinter.h"
#include "vp9/common/vp9_reconintra.h"
24

Dmitry Kovalev's avatar
Dmitry Kovalev committed
25
#include "vp9/encoder/vp9_encoder.h"
26
#include "vp9/encoder/vp9_ratectrl.h"
27
#include "vp9/encoder/vp9_rdopt.h"
28

Yunqing Wang's avatar
Yunqing Wang committed
29
static void full_pixel_motion_search(VP9_COMP *cpi, MACROBLOCK *x,
30
31
                                    const TileInfo *const tile,
                                    BLOCK_SIZE bsize, int mi_row, int mi_col,
32
                                    int_mv *tmp_mv, int *rate_mv) {
33
  MACROBLOCKD *xd = &x->e_mbd;
34
  MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
35
  struct buf_2d backup_yv12[MAX_MB_PLANE] = {{0}};
Deb Mukherjee's avatar
Deb Mukherjee committed
36
  int step_param;
37
38
39
  int sadpb = x->sadperbit16;
  MV mvp_full;
  int ref = mbmi->ref_frame[0];
40
  const MV ref_mv = mbmi->ref_mvs[ref][0].as_mv;
41
42
43
44
45
46
47
  int i;

  int tmp_col_min = x->mv_col_min;
  int tmp_col_max = x->mv_col_max;
  int tmp_row_min = x->mv_row_min;
  int tmp_row_max = x->mv_row_max;

48
49
  const YV12_BUFFER_CONFIG *scaled_ref_frame = vp9_get_scaled_ref_frame(cpi,
                                                                        ref);
50
51
52
53
54
55
56
57
  if (scaled_ref_frame) {
    int i;
    // Swap out the reference frame for a version that's been scaled to
    // match the resolution of the current frame, allowing the existing
    // motion search code to be used without additional modifications.
    for (i = 0; i < MAX_MB_PLANE; i++)
      backup_yv12[i] = xd->plane[i].pre[0];

58
    vp9_setup_pre_planes(xd, 0, scaled_ref_frame, mi_row, mi_col, NULL);
59
60
  }

61
  vp9_set_mv_search_range(x, &ref_mv);
62
63
64
65
66

  // TODO(jingning) exploiting adaptive motion search control in non-RD
  // mode decision too.
  step_param = 6;

Jim Bankoski's avatar
Jim Bankoski committed
67
  for (i = LAST_FRAME; i <= LAST_FRAME && cpi->common.show_frame; ++i) {
68
69
70
71
72
73
74
75
    if ((x->pred_mv_sad[ref] >> 3) > x->pred_mv_sad[i]) {
      tmp_mv->as_int = INVALID_MV;

      if (scaled_ref_frame) {
        int i;
        for (i = 0; i < MAX_MB_PLANE; i++)
          xd->plane[i].pre[0] = backup_yv12[i];
      }
Yunqing Wang's avatar
Yunqing Wang committed
76
      return;
77
78
    }
  }
Yaowu Xu's avatar
Yaowu Xu committed
79
80
81
82
83
  assert(x->mv_best_ref_index[ref] <= 2);
  if (x->mv_best_ref_index[ref] < 2)
    mvp_full = mbmi->ref_mvs[ref][x->mv_best_ref_index[ref]].as_mv;
  else
    mvp_full = x->pred_mv[ref].as_mv;
84
85
86
87

  mvp_full.col >>= 3;
  mvp_full.row >>= 3;

88
89
90
91
92
93
  if (cpi->sf.search_method == FAST_DIAMOND) {
    // NOTE: this returns SAD
    vp9_fast_dia_search(x, &mvp_full, step_param, sadpb, 0,
                        &cpi->fn_ptr[bsize], 1,
                        &ref_mv, &tmp_mv->as_mv);
  } else if (cpi->sf.search_method == FAST_HEX) {
Deb Mukherjee's avatar
Deb Mukherjee committed
94
    // NOTE: this returns SAD
Deb Mukherjee's avatar
Deb Mukherjee committed
95
96
97
    vp9_fast_hex_search(x, &mvp_full, step_param, sadpb, 0,
                        &cpi->fn_ptr[bsize], 1,
                        &ref_mv, &tmp_mv->as_mv);
98
  } else if (cpi->sf.search_method == HEX) {
Deb Mukherjee's avatar
Deb Mukherjee committed
99
    // NOTE: this returns SAD
Deb Mukherjee's avatar
Deb Mukherjee committed
100
101
102
    vp9_hex_search(x, &mvp_full, step_param, sadpb, 1,
                   &cpi->fn_ptr[bsize], 1,
                   &ref_mv, &tmp_mv->as_mv);
103
  } else if (cpi->sf.search_method == SQUARE) {
Deb Mukherjee's avatar
Deb Mukherjee committed
104
    // NOTE: this returns SAD
Deb Mukherjee's avatar
Deb Mukherjee committed
105
106
107
    vp9_square_search(x, &mvp_full, step_param, sadpb, 1,
                      &cpi->fn_ptr[bsize], 1,
                      &ref_mv, &tmp_mv->as_mv);
108
  } else if (cpi->sf.search_method == BIGDIA) {
Deb Mukherjee's avatar
Deb Mukherjee committed
109
    // NOTE: this returns SAD
Deb Mukherjee's avatar
Deb Mukherjee committed
110
111
112
    vp9_bigdia_search(x, &mvp_full, step_param, sadpb, 1,
                      &cpi->fn_ptr[bsize], 1,
                      &ref_mv, &tmp_mv->as_mv);
113
  } else {
Deb Mukherjee's avatar
Deb Mukherjee committed
114
115
    int further_steps = (cpi->sf.max_step_search_steps - 1) - step_param;
    // NOTE: this returns variance
Deb Mukherjee's avatar
Deb Mukherjee committed
116
117
118
119
    vp9_full_pixel_diamond(cpi, x, &mvp_full, step_param,
                           sadpb, further_steps, 1,
                           &cpi->fn_ptr[bsize],
                           &ref_mv, &tmp_mv->as_mv);
120
  }
121
122
123
124
125
126
127
128
129
130
  x->mv_col_min = tmp_col_min;
  x->mv_col_max = tmp_col_max;
  x->mv_row_min = tmp_row_min;
  x->mv_row_max = tmp_row_max;

  if (scaled_ref_frame) {
    int i;
    for (i = 0; i < MAX_MB_PLANE; i++)
      xd->plane[i].pre[0] = backup_yv12[i];
  }
131
132
133
134
135
136

  // calculate the bit cost on motion vector
  mvp_full.row = tmp_mv->as_mv.row * 8;
  mvp_full.col = tmp_mv->as_mv.col * 8;
  *rate_mv = vp9_mv_bit_cost(&mvp_full, &ref_mv,
                             x->nmvjointcost, x->mvcost, MV_COST_WEIGHT);
137
}
138

139
140
141
static void sub_pixel_motion_search(VP9_COMP *cpi, MACROBLOCK *x,
                                    const TileInfo *const tile,
                                    BLOCK_SIZE bsize, int mi_row, int mi_col,
142
                                    MV *tmp_mv) {
143
  MACROBLOCKD *xd = &x->e_mbd;
144
  MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
145
146
  struct buf_2d backup_yv12[MAX_MB_PLANE] = {{0}};
  int ref = mbmi->ref_frame[0];
Dmitry Kovalev's avatar
Dmitry Kovalev committed
147
  MV ref_mv = mbmi->ref_mvs[ref][0].as_mv;
148
  int dis;
149

150
151
152
153
154
155
156
157
158
159
  const YV12_BUFFER_CONFIG *scaled_ref_frame = vp9_get_scaled_ref_frame(cpi,
                                                                        ref);
  if (scaled_ref_frame) {
    int i;
    // Swap out the reference frame for a version that's been scaled to
    // match the resolution of the current frame, allowing the existing
    // motion search code to be used without additional modifications.
    for (i = 0; i < MAX_MB_PLANE; i++)
      backup_yv12[i] = xd->plane[i].pre[0];

160
    vp9_setup_pre_planes(xd, 0, scaled_ref_frame, mi_row, mi_col, NULL);
161
162
  }

Dmitry Kovalev's avatar
Dmitry Kovalev committed
163
  cpi->find_fractional_mv_step(x, tmp_mv, &ref_mv,
164
165
166
167
168
169
170
171
172
173
174
175
176
                               cpi->common.allow_high_precision_mv,
                               x->errorperbit,
                               &cpi->fn_ptr[bsize],
                               cpi->sf.subpel_force_stop,
                               cpi->sf.subpel_iters_per_step,
                               x->nmvjointcost, x->mvcost,
                               &dis, &x->pred_sse[ref]);

  if (scaled_ref_frame) {
    int i;
    for (i = 0; i < MAX_MB_PLANE; i++)
      xd->plane[i].pre[0] = backup_yv12[i];
  }
177
178

  x->pred_mv[ref].as_mv = *tmp_mv;
179
180
}

181
182
183
184
185
186
187
static void model_rd_for_sb_y(VP9_COMP *cpi, BLOCK_SIZE bsize,
                              MACROBLOCK *x, MACROBLOCKD *xd,
                              int *out_rate_sum, int64_t *out_dist_sum) {
  // Note our transform coeffs are 8 times an orthogonal transform.
  // Hence quantizer step is also 8 times. To get effective quantizer
  // we need to divide by 8 before sending to modeling function.
  unsigned int sse;
188
189
  int rate;
  int64_t dist;
190
191
192
193

  struct macroblock_plane *const p = &x->plane[0];
  struct macroblockd_plane *const pd = &xd->plane[0];

194
195
  int var = cpi->fn_ptr[bsize].vf(p->src.buf, p->src.stride,
                                  pd->dst.buf, pd->dst.stride, &sse);
196

197
  vp9_model_rd_from_var_lapndz(sse + var, 1 << num_pels_log2_lookup[bsize],
198
199
                               pd->dequant[1] >> 3, &rate, &dist);
  *out_rate_sum = rate;
200
  *out_dist_sum = dist << 3;
201
202
}

203
204
205
206
207
208
209
// TODO(jingning) placeholder for inter-frame non-RD mode decision.
// this needs various further optimizations. to be continued..
int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
                            const TileInfo *const tile,
                            int mi_row, int mi_col,
                            int *returnrate,
                            int64_t *returndistortion,
Jim Bankoski's avatar
Jim Bankoski committed
210
                            BLOCK_SIZE bsize) {
211
  MACROBLOCKD *xd = &x->e_mbd;
212
  MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
213
214
  struct macroblock_plane *const p = &x->plane[0];
  struct macroblockd_plane *const pd = &xd->plane[0];
215
  PREDICTION_MODE this_mode, best_mode = ZEROMV;
216
  MV_REFERENCE_FRAME ref_frame, best_ref_frame = LAST_FRAME;
217
  INTERP_FILTER best_pred_filter = EIGHTTAP;
218
219
220
221
222
  int_mv frame_mv[MB_MODE_COUNT][MAX_REF_FRAMES];
  struct buf_2d yv12_mb[4][MAX_MB_PLANE];
  static const int flag_list[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG,
                                    VP9_ALT_FLAG };
  int64_t best_rd = INT64_MAX;
223
  int64_t this_rd = INT64_MAX;
224

225
226
227
  int rate = INT_MAX;
  int64_t dist = INT64_MAX;

228
229
230
231
232
233
234
  VP9_COMMON *cm = &cpi->common;
  int intra_cost_penalty = 20 * vp9_dc_quant(cm->base_qindex, cm->y_dc_delta_q);

  const int64_t inter_mode_thresh = RDCOST(x->rdmult, x->rddiv,
                                           intra_cost_penalty, 0);
  const int64_t intra_mode_cost = 50;

235
  unsigned char segment_id = mbmi->segment_id;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
236
237
  const int *const rd_threshes = cpi->rd.threshes[segment_id][bsize];
  const int *const rd_thresh_freq_fact = cpi->rd.thresh_freq_fact[bsize];
238
  // Mode index conversion form THR_MODES to PREDICTION_MODE for a ref frame.
239
  int mode_idx[MB_MODE_COUNT] = {0};
240
  INTERP_FILTER filter_ref = SWITCHABLE;
241

242
243
244
  x->skip_encode = cpi->sf.skip_encode_frame && x->q_index < QIDX_SKIP_THRESH;

  x->skip = 0;
245
  if (!x->in_active_map)
246
247
248
    x->skip = 1;
  // initialize mode decisions
  *returnrate = INT_MAX;
249
  *returndistortion = INT64_MAX;
250
251
252
253
254
255
  vpx_memset(mbmi, 0, sizeof(MB_MODE_INFO));
  mbmi->sb_type = bsize;
  mbmi->ref_frame[0] = NONE;
  mbmi->ref_frame[1] = NONE;
  mbmi->tx_size = MIN(max_txsize_lookup[bsize],
                      tx_mode_to_biggest_tx_size[cpi->common.tx_mode]);
256
257
  mbmi->interp_filter = cpi->common.interp_filter == SWITCHABLE ?
                        EIGHTTAP : cpi->common.interp_filter;
258
  mbmi->skip = 0;
259
  mbmi->segment_id = segment_id;
260

Jim Bankoski's avatar
Jim Bankoski committed
261
  for (ref_frame = LAST_FRAME; ref_frame <= LAST_FRAME ; ++ref_frame) {
262
263
    x->pred_mv_sad[ref_frame] = INT_MAX;
    if (cpi->ref_frame_flags & flag_list[ref_frame]) {
264
      vp9_setup_buffer_inter(cpi, x, tile,
265
                             ref_frame, bsize, mi_row, mi_col,
266
267
268
269
270
271
                             frame_mv[NEARESTMV], frame_mv[NEARMV], yv12_mb);
    }
    frame_mv[NEWMV][ref_frame].as_int = INVALID_MV;
    frame_mv[ZEROMV][ref_frame].as_int = 0;
  }

272
273
274
275
276
  if (xd->up_available)
    filter_ref = xd->mi[-xd->mi_stride]->mbmi.interp_filter;
  else if (xd->left_available)
    filter_ref = xd->mi[-1]->mbmi.interp_filter;

Jim Bankoski's avatar
Jim Bankoski committed
277
  for (ref_frame = LAST_FRAME; ref_frame <= LAST_FRAME ; ++ref_frame) {
278
279
280
281
282
283
284
285
286
    if (!(cpi->ref_frame_flags & flag_list[ref_frame]))
      continue;

    // Select prediction reference frames.
    xd->plane[0].pre[0] = yv12_mb[ref_frame][0];

    clamp_mv2(&frame_mv[NEARESTMV][ref_frame].as_mv, xd);
    clamp_mv2(&frame_mv[NEARMV][ref_frame].as_mv, xd);

287
288
    mbmi->ref_frame[0] = ref_frame;

289
290
291
292
293
294
295
296
    // Set conversion index for LAST_FRAME.
    if (ref_frame == LAST_FRAME) {
      mode_idx[NEARESTMV] = THR_NEARESTMV;   // LAST_FRAME, NEARESTMV
      mode_idx[NEARMV] = THR_NEARMV;         // LAST_FRAME, NEARMV
      mode_idx[ZEROMV] = THR_ZEROMV;         // LAST_FRAME, ZEROMV
      mode_idx[NEWMV] = THR_NEWMV;           // LAST_FRAME, NEWMV
    }

297
    for (this_mode = NEARESTMV; this_mode <= NEWMV; ++this_mode) {
298
299
      int rate_mv = 0;

300
301
302
      if (cpi->sf.disable_inter_mode_mask[bsize] &
          (1 << INTER_OFFSET(this_mode)))
        continue;
Jim Bankoski's avatar
Jim Bankoski committed
303

304
305
306
307
308
      if (best_rd < ((int64_t)rd_threshes[mode_idx[this_mode]] *
          rd_thresh_freq_fact[this_mode] >> 5) ||
          rd_threshes[mode_idx[this_mode]] == INT_MAX)
        continue;

Jim Bankoski's avatar
Jim Bankoski committed
309
      if (this_mode == NEWMV) {
310
        int rate_mode = 0;
311
        if (this_rd < (int64_t)(1 << num_pels_log2_lookup[bsize]))
312
313
          continue;

Yunqing Wang's avatar
Yunqing Wang committed
314
        full_pixel_motion_search(cpi, x, tile, bsize, mi_row, mi_col,
315
                                 &frame_mv[NEWMV][ref_frame], &rate_mv);
Jim Bankoski's avatar
Jim Bankoski committed
316
317
318

        if (frame_mv[NEWMV][ref_frame].as_int == INVALID_MV)
          continue;
319

320
321
322
323
324
        rate_mode = x->inter_mode_cost[mbmi->mode_context[ref_frame]]
                                      [INTER_OFFSET(this_mode)];
        if (RDCOST(x->rdmult, x->rddiv, rate_mv + rate_mode, 0) > best_rd)
          continue;

325
        sub_pixel_motion_search(cpi, x, tile, bsize, mi_row, mi_col,
326
                                &frame_mv[NEWMV][ref_frame].as_mv);
Jim Bankoski's avatar
Jim Bankoski committed
327
328
      }

329
330
331
332
333
      if (this_mode != NEARESTMV)
        if (frame_mv[this_mode][ref_frame].as_int ==
            frame_mv[NEARESTMV][ref_frame].as_int)
          continue;

334
335
      mbmi->mode = this_mode;
      mbmi->mv[0].as_int = frame_mv[this_mode][ref_frame].as_int;
336

337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
      // Search for the best prediction filter type, when the resulting
      // motion vector is at sub-pixel accuracy level for luma component, i.e.,
      // the last three bits are all zeros.
      if ((this_mode == NEWMV || filter_ref == SWITCHABLE) &&
          ((mbmi->mv[0].as_mv.row & 0x07) != 0 ||
           (mbmi->mv[0].as_mv.col & 0x07) != 0)) {
        int64_t tmp_rdcost1 = INT64_MAX;
        int64_t tmp_rdcost2 = INT64_MAX;
        int64_t tmp_rdcost3 = INT64_MAX;
        int pf_rate[3];
        int64_t pf_dist[3];

        mbmi->interp_filter = EIGHTTAP;
        vp9_build_inter_predictors_sby(xd, mi_row, mi_col, bsize);
        model_rd_for_sb_y(cpi, bsize, x, xd, &pf_rate[EIGHTTAP],
                          &pf_dist[EIGHTTAP]);
        tmp_rdcost1 = RDCOST(x->rdmult, x->rddiv,
                             vp9_get_switchable_rate(x) + pf_rate[EIGHTTAP],
                             pf_dist[EIGHTTAP]);

        mbmi->interp_filter = EIGHTTAP_SHARP;
        vp9_build_inter_predictors_sby(xd, mi_row, mi_col, bsize);
        model_rd_for_sb_y(cpi, bsize, x, xd, &pf_rate[EIGHTTAP_SHARP],
                          &pf_dist[EIGHTTAP_SHARP]);
        tmp_rdcost2 = RDCOST(x->rdmult, x->rddiv,
                          vp9_get_switchable_rate(x) + pf_rate[EIGHTTAP_SHARP],
                          pf_dist[EIGHTTAP_SHARP]);

        mbmi->interp_filter = EIGHTTAP_SMOOTH;
        vp9_build_inter_predictors_sby(xd, mi_row, mi_col, bsize);
        model_rd_for_sb_y(cpi, bsize, x, xd, &pf_rate[EIGHTTAP_SMOOTH],
                          &pf_dist[EIGHTTAP_SMOOTH]);
        tmp_rdcost3 = RDCOST(x->rdmult, x->rddiv,
                          vp9_get_switchable_rate(x) + pf_rate[EIGHTTAP_SMOOTH],
                          pf_dist[EIGHTTAP_SMOOTH]);

        if (tmp_rdcost2 < tmp_rdcost1) {
          if (tmp_rdcost2 < tmp_rdcost3)
            mbmi->interp_filter = EIGHTTAP_SHARP;
          else
            mbmi->interp_filter = EIGHTTAP_SMOOTH;
        } else {
          if (tmp_rdcost1 < tmp_rdcost3)
            mbmi->interp_filter = EIGHTTAP;
          else
            mbmi->interp_filter = EIGHTTAP_SMOOTH;
        }

        rate = pf_rate[mbmi->interp_filter];
        dist = pf_dist[mbmi->interp_filter];
      } else {
        mbmi->interp_filter = (filter_ref == SWITCHABLE) ? EIGHTTAP: filter_ref;
        vp9_build_inter_predictors_sby(xd, mi_row, mi_col, bsize);
        model_rd_for_sb_y(cpi, bsize, x, xd, &rate, &dist);
      }

393
394
395
396
      rate += rate_mv;
      rate += x->inter_mode_cost[mbmi->mode_context[ref_frame]]
                                [INTER_OFFSET(this_mode)];
      this_rd = RDCOST(x->rdmult, x->rddiv, rate, dist);
397
398
399

      if (this_rd < best_rd) {
        best_rd = this_rd;
400
401
        *returnrate = rate;
        *returndistortion = dist;
402
        best_mode = this_mode;
403
        best_pred_filter = mbmi->interp_filter;
404
        best_ref_frame = ref_frame;
405
406
407
408
      }
    }
  }

409
  mbmi->mode = best_mode;
410
  mbmi->interp_filter = best_pred_filter;
411
412
  mbmi->ref_frame[0] = best_ref_frame;
  mbmi->mv[0].as_int = frame_mv[best_mode][best_ref_frame].as_int;
413
  xd->mi[0]->bmi[0].as_mv[0].as_int = mbmi->mv[0].as_int;
414

415
416
417
  // Perform intra prediction search, if the best SAD is above a certain
  // threshold.
  if (best_rd > inter_mode_thresh) {
418
    for (this_mode = DC_PRED; this_mode <= DC_PRED; ++this_mode) {
419
420
421
422
423
      vp9_predict_intra_block(xd, 0, b_width_log2(bsize),
                              mbmi->tx_size, this_mode,
                              &p->src.buf[0], p->src.stride,
                              &pd->dst.buf[0], pd->dst.stride, 0, 0, 0);

424
425
      model_rd_for_sb_y(cpi, bsize, x, xd, &rate, &dist);
      rate += x->mbmode_cost[this_mode];
426
      rate += intra_cost_penalty;
427
      this_rd = RDCOST(x->rdmult, x->rddiv, rate, dist);
428
429
430

      if (this_rd + intra_mode_cost < best_rd) {
        best_rd = this_rd;
431
432
        *returnrate = rate;
        *returndistortion = dist;
433
434
435
        mbmi->mode = this_mode;
        mbmi->ref_frame[0] = INTRA_FRAME;
        mbmi->uv_mode = this_mode;
436
        mbmi->mv[0].as_int = INVALID_MV;
437
438
439
      }
    }
  }
440

441
442
  return INT64_MAX;
}