vp9_pickmode.c 16.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11
/*
 *  Copyright (c) 2014 The WebM project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include <assert.h>
12 13 14
#include <limits.h>
#include <math.h>
#include <stdio.h>
15

16 17 18 19 20 21
#include "./vp9_rtcd.h"

#include "vpx_mem/vpx_mem.h"

#include "vp9/common/vp9_common.h"
#include "vp9/common/vp9_mvref_common.h"
22 23
#include "vp9/common/vp9_reconinter.h"
#include "vp9/common/vp9_reconintra.h"
24

Dmitry Kovalev's avatar
Dmitry Kovalev committed
25
#include "vp9/encoder/vp9_encoder.h"
26
#include "vp9/encoder/vp9_ratectrl.h"
27
#include "vp9/encoder/vp9_rdopt.h"
28

Yunqing Wang's avatar
Yunqing Wang committed
29
static void full_pixel_motion_search(VP9_COMP *cpi, MACROBLOCK *x,
30 31
                                    const TileInfo *const tile,
                                    BLOCK_SIZE bsize, int mi_row, int mi_col,
32
                                    int_mv *tmp_mv, int *rate_mv) {
33
  MACROBLOCKD *xd = &x->e_mbd;
34
  MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
35
  struct buf_2d backup_yv12[MAX_MB_PLANE] = {{0}};
Deb Mukherjee's avatar
Deb Mukherjee committed
36
  int step_param;
37 38 39
  int sadpb = x->sadperbit16;
  MV mvp_full;
  int ref = mbmi->ref_frame[0];
40
  const MV ref_mv = mbmi->ref_mvs[ref][0].as_mv;
41 42 43 44 45 46 47
  int i;

  int tmp_col_min = x->mv_col_min;
  int tmp_col_max = x->mv_col_max;
  int tmp_row_min = x->mv_row_min;
  int tmp_row_max = x->mv_row_max;

48 49
  const YV12_BUFFER_CONFIG *scaled_ref_frame = vp9_get_scaled_ref_frame(cpi,
                                                                        ref);
50 51 52 53 54 55 56 57
  if (scaled_ref_frame) {
    int i;
    // Swap out the reference frame for a version that's been scaled to
    // match the resolution of the current frame, allowing the existing
    // motion search code to be used without additional modifications.
    for (i = 0; i < MAX_MB_PLANE; i++)
      backup_yv12[i] = xd->plane[i].pre[0];

58
    vp9_setup_pre_planes(xd, 0, scaled_ref_frame, mi_row, mi_col, NULL);
59 60
  }

61
  vp9_set_mv_search_range(x, &ref_mv);
62 63 64 65 66

  // TODO(jingning) exploiting adaptive motion search control in non-RD
  // mode decision too.
  step_param = 6;

Jim Bankoski's avatar
Jim Bankoski committed
67
  for (i = LAST_FRAME; i <= LAST_FRAME && cpi->common.show_frame; ++i) {
68 69 70 71 72 73 74 75
    if ((x->pred_mv_sad[ref] >> 3) > x->pred_mv_sad[i]) {
      tmp_mv->as_int = INVALID_MV;

      if (scaled_ref_frame) {
        int i;
        for (i = 0; i < MAX_MB_PLANE; i++)
          xd->plane[i].pre[0] = backup_yv12[i];
      }
Yunqing Wang's avatar
Yunqing Wang committed
76
      return;
77 78
    }
  }
Yaowu Xu's avatar
Yaowu Xu committed
79 80 81 82 83
  assert(x->mv_best_ref_index[ref] <= 2);
  if (x->mv_best_ref_index[ref] < 2)
    mvp_full = mbmi->ref_mvs[ref][x->mv_best_ref_index[ref]].as_mv;
  else
    mvp_full = x->pred_mv[ref].as_mv;
84 85 86 87

  mvp_full.col >>= 3;
  mvp_full.row >>= 3;

88 89 90 91 92 93
  if (cpi->sf.search_method == FAST_DIAMOND) {
    // NOTE: this returns SAD
    vp9_fast_dia_search(x, &mvp_full, step_param, sadpb, 0,
                        &cpi->fn_ptr[bsize], 1,
                        &ref_mv, &tmp_mv->as_mv);
  } else if (cpi->sf.search_method == FAST_HEX) {
Deb Mukherjee's avatar
Deb Mukherjee committed
94
    // NOTE: this returns SAD
Deb Mukherjee's avatar
Deb Mukherjee committed
95 96 97
    vp9_fast_hex_search(x, &mvp_full, step_param, sadpb, 0,
                        &cpi->fn_ptr[bsize], 1,
                        &ref_mv, &tmp_mv->as_mv);
98
  } else if (cpi->sf.search_method == HEX) {
Deb Mukherjee's avatar
Deb Mukherjee committed
99
    // NOTE: this returns SAD
Deb Mukherjee's avatar
Deb Mukherjee committed
100 101 102
    vp9_hex_search(x, &mvp_full, step_param, sadpb, 1,
                   &cpi->fn_ptr[bsize], 1,
                   &ref_mv, &tmp_mv->as_mv);
103
  } else if (cpi->sf.search_method == SQUARE) {
Deb Mukherjee's avatar
Deb Mukherjee committed
104
    // NOTE: this returns SAD
Deb Mukherjee's avatar
Deb Mukherjee committed
105 106 107
    vp9_square_search(x, &mvp_full, step_param, sadpb, 1,
                      &cpi->fn_ptr[bsize], 1,
                      &ref_mv, &tmp_mv->as_mv);
108
  } else if (cpi->sf.search_method == BIGDIA) {
Deb Mukherjee's avatar
Deb Mukherjee committed
109
    // NOTE: this returns SAD
Deb Mukherjee's avatar
Deb Mukherjee committed
110 111 112
    vp9_bigdia_search(x, &mvp_full, step_param, sadpb, 1,
                      &cpi->fn_ptr[bsize], 1,
                      &ref_mv, &tmp_mv->as_mv);
113
  } else {
Deb Mukherjee's avatar
Deb Mukherjee committed
114 115
    int further_steps = (cpi->sf.max_step_search_steps - 1) - step_param;
    // NOTE: this returns variance
Deb Mukherjee's avatar
Deb Mukherjee committed
116 117 118 119
    vp9_full_pixel_diamond(cpi, x, &mvp_full, step_param,
                           sadpb, further_steps, 1,
                           &cpi->fn_ptr[bsize],
                           &ref_mv, &tmp_mv->as_mv);
120
  }
121 122 123 124 125 126 127 128 129 130
  x->mv_col_min = tmp_col_min;
  x->mv_col_max = tmp_col_max;
  x->mv_row_min = tmp_row_min;
  x->mv_row_max = tmp_row_max;

  if (scaled_ref_frame) {
    int i;
    for (i = 0; i < MAX_MB_PLANE; i++)
      xd->plane[i].pre[0] = backup_yv12[i];
  }
131 132 133 134 135 136

  // calculate the bit cost on motion vector
  mvp_full.row = tmp_mv->as_mv.row * 8;
  mvp_full.col = tmp_mv->as_mv.col * 8;
  *rate_mv = vp9_mv_bit_cost(&mvp_full, &ref_mv,
                             x->nmvjointcost, x->mvcost, MV_COST_WEIGHT);
137
}
138

139 140 141
static void sub_pixel_motion_search(VP9_COMP *cpi, MACROBLOCK *x,
                                    const TileInfo *const tile,
                                    BLOCK_SIZE bsize, int mi_row, int mi_col,
142
                                    MV *tmp_mv) {
143
  MACROBLOCKD *xd = &x->e_mbd;
144
  MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
145 146
  struct buf_2d backup_yv12[MAX_MB_PLANE] = {{0}};
  int ref = mbmi->ref_frame[0];
Dmitry Kovalev's avatar
Dmitry Kovalev committed
147
  MV ref_mv = mbmi->ref_mvs[ref][0].as_mv;
148
  int dis;
149

150 151 152 153 154 155 156 157 158 159
  const YV12_BUFFER_CONFIG *scaled_ref_frame = vp9_get_scaled_ref_frame(cpi,
                                                                        ref);
  if (scaled_ref_frame) {
    int i;
    // Swap out the reference frame for a version that's been scaled to
    // match the resolution of the current frame, allowing the existing
    // motion search code to be used without additional modifications.
    for (i = 0; i < MAX_MB_PLANE; i++)
      backup_yv12[i] = xd->plane[i].pre[0];

160
    vp9_setup_pre_planes(xd, 0, scaled_ref_frame, mi_row, mi_col, NULL);
161 162
  }

Dmitry Kovalev's avatar
Dmitry Kovalev committed
163
  cpi->find_fractional_mv_step(x, tmp_mv, &ref_mv,
164 165 166 167 168 169 170 171 172 173 174 175 176
                               cpi->common.allow_high_precision_mv,
                               x->errorperbit,
                               &cpi->fn_ptr[bsize],
                               cpi->sf.subpel_force_stop,
                               cpi->sf.subpel_iters_per_step,
                               x->nmvjointcost, x->mvcost,
                               &dis, &x->pred_sse[ref]);

  if (scaled_ref_frame) {
    int i;
    for (i = 0; i < MAX_MB_PLANE; i++)
      xd->plane[i].pre[0] = backup_yv12[i];
  }
177 178

  x->pred_mv[ref].as_mv = *tmp_mv;
179 180
}

181 182 183 184 185 186 187
static void model_rd_for_sb_y(VP9_COMP *cpi, BLOCK_SIZE bsize,
                              MACROBLOCK *x, MACROBLOCKD *xd,
                              int *out_rate_sum, int64_t *out_dist_sum) {
  // Note our transform coeffs are 8 times an orthogonal transform.
  // Hence quantizer step is also 8 times. To get effective quantizer
  // we need to divide by 8 before sending to modeling function.
  unsigned int sse;
188 189
  int rate;
  int64_t dist;
190 191 192 193

  struct macroblock_plane *const p = &x->plane[0];
  struct macroblockd_plane *const pd = &xd->plane[0];

194 195
  int var = cpi->fn_ptr[bsize].vf(p->src.buf, p->src.stride,
                                  pd->dst.buf, pd->dst.stride, &sse);
196

197
  vp9_model_rd_from_var_lapndz(sse + var, 1 << num_pels_log2_lookup[bsize],
198 199
                               pd->dequant[1] >> 3, &rate, &dist);
  *out_rate_sum = rate;
200
  *out_dist_sum = dist << 3;
201 202
}

203 204 205 206 207 208 209
// TODO(jingning) placeholder for inter-frame non-RD mode decision.
// this needs various further optimizations. to be continued..
int64_t vp9_pick_inter_mode(VP9_COMP *cpi, MACROBLOCK *x,
                            const TileInfo *const tile,
                            int mi_row, int mi_col,
                            int *returnrate,
                            int64_t *returndistortion,
Jim Bankoski's avatar
Jim Bankoski committed
210
                            BLOCK_SIZE bsize) {
211
  MACROBLOCKD *xd = &x->e_mbd;
212
  MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
213 214
  struct macroblock_plane *const p = &x->plane[0];
  struct macroblockd_plane *const pd = &xd->plane[0];
215
  PREDICTION_MODE this_mode, best_mode = ZEROMV;
216
  MV_REFERENCE_FRAME ref_frame, best_ref_frame = LAST_FRAME;
217
  INTERP_FILTER best_pred_filter = EIGHTTAP;
218 219 220 221 222
  int_mv frame_mv[MB_MODE_COUNT][MAX_REF_FRAMES];
  struct buf_2d yv12_mb[4][MAX_MB_PLANE];
  static const int flag_list[4] = { 0, VP9_LAST_FLAG, VP9_GOLD_FLAG,
                                    VP9_ALT_FLAG };
  int64_t best_rd = INT64_MAX;
223
  int64_t this_rd = INT64_MAX;
224

225 226 227
  int rate = INT_MAX;
  int64_t dist = INT64_MAX;

228 229 230 231 232 233 234
  VP9_COMMON *cm = &cpi->common;
  int intra_cost_penalty = 20 * vp9_dc_quant(cm->base_qindex, cm->y_dc_delta_q);

  const int64_t inter_mode_thresh = RDCOST(x->rdmult, x->rddiv,
                                           intra_cost_penalty, 0);
  const int64_t intra_mode_cost = 50;

235
  unsigned char segment_id = mbmi->segment_id;
Dmitry Kovalev's avatar
Dmitry Kovalev committed
236 237
  const int *const rd_threshes = cpi->rd.threshes[segment_id][bsize];
  const int *const rd_thresh_freq_fact = cpi->rd.thresh_freq_fact[bsize];
238
  // Mode index conversion form THR_MODES to PREDICTION_MODE for a ref frame.
239
  int mode_idx[MB_MODE_COUNT] = {0};
240
  INTERP_FILTER filter_ref = SWITCHABLE;
241 242 243
  int bsl = mi_width_log2_lookup[bsize];
  int pred_filter_search = (((mi_row + mi_col) >> bsl) +
                            cpi->sf.chessboard_index) & 0x01;
244

245 246 247
  x->skip_encode = cpi->sf.skip_encode_frame && x->q_index < QIDX_SKIP_THRESH;

  x->skip = 0;
248
  if (!x->in_active_map)
249 250 251
    x->skip = 1;
  // initialize mode decisions
  *returnrate = INT_MAX;
252
  *returndistortion = INT64_MAX;
253 254 255 256 257 258
  vpx_memset(mbmi, 0, sizeof(MB_MODE_INFO));
  mbmi->sb_type = bsize;
  mbmi->ref_frame[0] = NONE;
  mbmi->ref_frame[1] = NONE;
  mbmi->tx_size = MIN(max_txsize_lookup[bsize],
                      tx_mode_to_biggest_tx_size[cpi->common.tx_mode]);
259 260
  mbmi->interp_filter = cpi->common.interp_filter == SWITCHABLE ?
                        EIGHTTAP : cpi->common.interp_filter;
261
  mbmi->skip = 0;
262
  mbmi->segment_id = segment_id;
263

Jim Bankoski's avatar
Jim Bankoski committed
264
  for (ref_frame = LAST_FRAME; ref_frame <= LAST_FRAME ; ++ref_frame) {
265 266
    x->pred_mv_sad[ref_frame] = INT_MAX;
    if (cpi->ref_frame_flags & flag_list[ref_frame]) {
267
      vp9_setup_buffer_inter(cpi, x, tile,
268
                             ref_frame, bsize, mi_row, mi_col,
269 270 271 272 273 274
                             frame_mv[NEARESTMV], frame_mv[NEARMV], yv12_mb);
    }
    frame_mv[NEWMV][ref_frame].as_int = INVALID_MV;
    frame_mv[ZEROMV][ref_frame].as_int = 0;
  }

275 276 277 278 279
  if (xd->up_available)
    filter_ref = xd->mi[-xd->mi_stride]->mbmi.interp_filter;
  else if (xd->left_available)
    filter_ref = xd->mi[-1]->mbmi.interp_filter;

Jim Bankoski's avatar
Jim Bankoski committed
280
  for (ref_frame = LAST_FRAME; ref_frame <= LAST_FRAME ; ++ref_frame) {
281 282 283 284 285 286 287 288 289
    if (!(cpi->ref_frame_flags & flag_list[ref_frame]))
      continue;

    // Select prediction reference frames.
    xd->plane[0].pre[0] = yv12_mb[ref_frame][0];

    clamp_mv2(&frame_mv[NEARESTMV][ref_frame].as_mv, xd);
    clamp_mv2(&frame_mv[NEARMV][ref_frame].as_mv, xd);

290 291
    mbmi->ref_frame[0] = ref_frame;

292 293 294 295 296 297 298 299
    // Set conversion index for LAST_FRAME.
    if (ref_frame == LAST_FRAME) {
      mode_idx[NEARESTMV] = THR_NEARESTMV;   // LAST_FRAME, NEARESTMV
      mode_idx[NEARMV] = THR_NEARMV;         // LAST_FRAME, NEARMV
      mode_idx[ZEROMV] = THR_ZEROMV;         // LAST_FRAME, ZEROMV
      mode_idx[NEWMV] = THR_NEWMV;           // LAST_FRAME, NEWMV
    }

300
    for (this_mode = NEARESTMV; this_mode <= NEWMV; ++this_mode) {
301 302
      int rate_mv = 0;

303 304 305
      if (cpi->sf.disable_inter_mode_mask[bsize] &
          (1 << INTER_OFFSET(this_mode)))
        continue;
Jim Bankoski's avatar
Jim Bankoski committed
306

307 308 309 310 311
      if (best_rd < ((int64_t)rd_threshes[mode_idx[this_mode]] *
          rd_thresh_freq_fact[this_mode] >> 5) ||
          rd_threshes[mode_idx[this_mode]] == INT_MAX)
        continue;

Jim Bankoski's avatar
Jim Bankoski committed
312
      if (this_mode == NEWMV) {
313
        int rate_mode = 0;
314
        if (this_rd < (int64_t)(1 << num_pels_log2_lookup[bsize]))
315 316
          continue;

Yunqing Wang's avatar
Yunqing Wang committed
317
        full_pixel_motion_search(cpi, x, tile, bsize, mi_row, mi_col,
318
                                 &frame_mv[NEWMV][ref_frame], &rate_mv);
Jim Bankoski's avatar
Jim Bankoski committed
319 320 321

        if (frame_mv[NEWMV][ref_frame].as_int == INVALID_MV)
          continue;
322

323 324 325 326 327
        rate_mode = x->inter_mode_cost[mbmi->mode_context[ref_frame]]
                                      [INTER_OFFSET(this_mode)];
        if (RDCOST(x->rdmult, x->rddiv, rate_mv + rate_mode, 0) > best_rd)
          continue;

328
        sub_pixel_motion_search(cpi, x, tile, bsize, mi_row, mi_col,
329
                                &frame_mv[NEWMV][ref_frame].as_mv);
Jim Bankoski's avatar
Jim Bankoski committed
330 331
      }

332 333 334 335 336
      if (this_mode != NEARESTMV)
        if (frame_mv[this_mode][ref_frame].as_int ==
            frame_mv[NEARESTMV][ref_frame].as_int)
          continue;

337 338
      mbmi->mode = this_mode;
      mbmi->mv[0].as_int = frame_mv[this_mode][ref_frame].as_int;
339

340 341 342 343
      // Search for the best prediction filter type, when the resulting
      // motion vector is at sub-pixel accuracy level for luma component, i.e.,
      // the last three bits are all zeros.
      if ((this_mode == NEWMV || filter_ref == SWITCHABLE) &&
344
          pred_filter_search &&
345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396
          ((mbmi->mv[0].as_mv.row & 0x07) != 0 ||
           (mbmi->mv[0].as_mv.col & 0x07) != 0)) {
        int64_t tmp_rdcost1 = INT64_MAX;
        int64_t tmp_rdcost2 = INT64_MAX;
        int64_t tmp_rdcost3 = INT64_MAX;
        int pf_rate[3];
        int64_t pf_dist[3];

        mbmi->interp_filter = EIGHTTAP;
        vp9_build_inter_predictors_sby(xd, mi_row, mi_col, bsize);
        model_rd_for_sb_y(cpi, bsize, x, xd, &pf_rate[EIGHTTAP],
                          &pf_dist[EIGHTTAP]);
        tmp_rdcost1 = RDCOST(x->rdmult, x->rddiv,
                             vp9_get_switchable_rate(x) + pf_rate[EIGHTTAP],
                             pf_dist[EIGHTTAP]);

        mbmi->interp_filter = EIGHTTAP_SHARP;
        vp9_build_inter_predictors_sby(xd, mi_row, mi_col, bsize);
        model_rd_for_sb_y(cpi, bsize, x, xd, &pf_rate[EIGHTTAP_SHARP],
                          &pf_dist[EIGHTTAP_SHARP]);
        tmp_rdcost2 = RDCOST(x->rdmult, x->rddiv,
                          vp9_get_switchable_rate(x) + pf_rate[EIGHTTAP_SHARP],
                          pf_dist[EIGHTTAP_SHARP]);

        mbmi->interp_filter = EIGHTTAP_SMOOTH;
        vp9_build_inter_predictors_sby(xd, mi_row, mi_col, bsize);
        model_rd_for_sb_y(cpi, bsize, x, xd, &pf_rate[EIGHTTAP_SMOOTH],
                          &pf_dist[EIGHTTAP_SMOOTH]);
        tmp_rdcost3 = RDCOST(x->rdmult, x->rddiv,
                          vp9_get_switchable_rate(x) + pf_rate[EIGHTTAP_SMOOTH],
                          pf_dist[EIGHTTAP_SMOOTH]);

        if (tmp_rdcost2 < tmp_rdcost1) {
          if (tmp_rdcost2 < tmp_rdcost3)
            mbmi->interp_filter = EIGHTTAP_SHARP;
          else
            mbmi->interp_filter = EIGHTTAP_SMOOTH;
        } else {
          if (tmp_rdcost1 < tmp_rdcost3)
            mbmi->interp_filter = EIGHTTAP;
          else
            mbmi->interp_filter = EIGHTTAP_SMOOTH;
        }

        rate = pf_rate[mbmi->interp_filter];
        dist = pf_dist[mbmi->interp_filter];
      } else {
        mbmi->interp_filter = (filter_ref == SWITCHABLE) ? EIGHTTAP: filter_ref;
        vp9_build_inter_predictors_sby(xd, mi_row, mi_col, bsize);
        model_rd_for_sb_y(cpi, bsize, x, xd, &rate, &dist);
      }

397 398 399 400
      rate += rate_mv;
      rate += x->inter_mode_cost[mbmi->mode_context[ref_frame]]
                                [INTER_OFFSET(this_mode)];
      this_rd = RDCOST(x->rdmult, x->rddiv, rate, dist);
401 402 403

      if (this_rd < best_rd) {
        best_rd = this_rd;
404 405
        *returnrate = rate;
        *returndistortion = dist;
406
        best_mode = this_mode;
407
        best_pred_filter = mbmi->interp_filter;
408
        best_ref_frame = ref_frame;
409 410 411 412
      }
    }
  }

413
  mbmi->mode = best_mode;
414
  mbmi->interp_filter = best_pred_filter;
415 416
  mbmi->ref_frame[0] = best_ref_frame;
  mbmi->mv[0].as_int = frame_mv[best_mode][best_ref_frame].as_int;
417
  xd->mi[0]->bmi[0].as_mv[0].as_int = mbmi->mv[0].as_int;
418

419 420
  // Perform intra prediction search, if the best SAD is above a certain
  // threshold.
421
  if (best_rd > inter_mode_thresh && bsize < cpi->sf.max_intra_bsize) {
422
    for (this_mode = DC_PRED; this_mode <= DC_PRED; ++this_mode) {
423 424 425 426 427
      vp9_predict_intra_block(xd, 0, b_width_log2(bsize),
                              mbmi->tx_size, this_mode,
                              &p->src.buf[0], p->src.stride,
                              &pd->dst.buf[0], pd->dst.stride, 0, 0, 0);

428 429
      model_rd_for_sb_y(cpi, bsize, x, xd, &rate, &dist);
      rate += x->mbmode_cost[this_mode];
430
      rate += intra_cost_penalty;
431
      this_rd = RDCOST(x->rdmult, x->rddiv, rate, dist);
432 433 434

      if (this_rd + intra_mode_cost < best_rd) {
        best_rd = this_rd;
435 436
        *returnrate = rate;
        *returndistortion = dist;
437 438 439
        mbmi->mode = this_mode;
        mbmi->ref_frame[0] = INTRA_FRAME;
        mbmi->uv_mode = this_mode;
440
        mbmi->mv[0].as_int = INVALID_MV;
441 442 443
      }
    }
  }
444

445 446
  return INT64_MAX;
}