decodeframe.c 151 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4 5 6 7 8 9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10 11 12 13 14
 */

#include <assert.h>
#include <stdlib.h>  // qsort()

Yaowu Xu's avatar
Yaowu Xu committed
15 16 17
#include "./aom_config.h"
#include "./aom_dsp_rtcd.h"
#include "./aom_scale_rtcd.h"
Jingning Han's avatar
Jingning Han committed
18
#include "./av1_rtcd.h"
Jingning Han's avatar
Jingning Han committed
19

20
#include "aom/aom_codec.h"
Yaowu Xu's avatar
Yaowu Xu committed
21
#include "aom_dsp/aom_dsp_common.h"
Jingning Han's avatar
Jingning Han committed
22 23
#include "aom_dsp/bitreader.h"
#include "aom_dsp/bitreader_buffer.h"
Yaowu Xu's avatar
Yaowu Xu committed
24
#include "aom_mem/aom_mem.h"
25 26
#include "aom_ports/mem.h"
#include "aom_ports/mem_ops.h"
Yaowu Xu's avatar
Yaowu Xu committed
27 28
#include "aom_scale/aom_scale.h"
#include "aom_util/aom_thread.h"
Jingning Han's avatar
Jingning Han committed
29

30
#include "av1/common/alloccommon.h"
31
#if CONFIG_CLPF
32
#include "av1/common/clpf.h"
33
#endif
34
#include "av1/common/common.h"
Yaowu Xu's avatar
Yaowu Xu committed
35
#if CONFIG_DERING
36
#include "av1/common/dering.h"
Yaowu Xu's avatar
Yaowu Xu committed
37
#endif  // CONFIG_DERING
38 39 40 41 42 43
#include "av1/common/entropy.h"
#include "av1/common/entropymode.h"
#include "av1/common/idct.h"
#include "av1/common/pred_common.h"
#include "av1/common/quant_common.h"
#include "av1/common/reconinter.h"
Jingning Han's avatar
Jingning Han committed
44
#include "av1/common/reconintra.h"
45
#include "av1/common/seg_common.h"
Jingning Han's avatar
Jingning Han committed
46
#include "av1/common/thread_common.h"
47
#include "av1/common/tile_common.h"
48

49 50 51
#include "av1/decoder/decodeframe.h"
#include "av1/decoder/decodemv.h"
#include "av1/decoder/decoder.h"
Jingning Han's avatar
Jingning Han committed
52
#include "av1/decoder/detokenize.h"
53
#include "av1/decoder/dsubexp.h"
Jingning Han's avatar
Jingning Han committed
54

Yaowu Xu's avatar
Yaowu Xu committed
55
#define MAX_AV1_HEADER_SIZE 80
Jingning Han's avatar
Jingning Han committed
56

Yaowu Xu's avatar
Yaowu Xu committed
57
static int is_compound_reference_allowed(const AV1_COMMON *cm) {
Jingning Han's avatar
Jingning Han committed
58
  int i;
59
  if (frame_is_intra_only(cm)) return 0;
60
  for (i = 1; i < INTER_REFS_PER_FRAME; ++i)
61
    if (cm->ref_frame_sign_bias[i + 1] != cm->ref_frame_sign_bias[1]) return 1;
Jingning Han's avatar
Jingning Han committed
62 63 64 65

  return 0;
}

Yaowu Xu's avatar
Yaowu Xu committed
66
static void setup_compound_reference_mode(AV1_COMMON *cm) {
67
#if CONFIG_EXT_REFS
68
  cm->comp_fwd_ref[0] = LAST_FRAME;
69 70 71 72
  cm->comp_fwd_ref[1] = LAST2_FRAME;
  cm->comp_fwd_ref[2] = LAST3_FRAME;
  cm->comp_fwd_ref[3] = GOLDEN_FRAME;

73 74
  cm->comp_bwd_ref[0] = BWDREF_FRAME;
  cm->comp_bwd_ref[1] = ALTREF_FRAME;
75
#else
Jingning Han's avatar
Jingning Han committed
76
  if (cm->ref_frame_sign_bias[LAST_FRAME] ==
77
      cm->ref_frame_sign_bias[GOLDEN_FRAME]) {
Jingning Han's avatar
Jingning Han committed
78 79 80 81
    cm->comp_fixed_ref = ALTREF_FRAME;
    cm->comp_var_ref[0] = LAST_FRAME;
    cm->comp_var_ref[1] = GOLDEN_FRAME;
  } else if (cm->ref_frame_sign_bias[LAST_FRAME] ==
82
             cm->ref_frame_sign_bias[ALTREF_FRAME]) {
Jingning Han's avatar
Jingning Han committed
83 84 85 86 87 88 89 90
    cm->comp_fixed_ref = GOLDEN_FRAME;
    cm->comp_var_ref[0] = LAST_FRAME;
    cm->comp_var_ref[1] = ALTREF_FRAME;
  } else {
    cm->comp_fixed_ref = LAST_FRAME;
    cm->comp_var_ref[0] = GOLDEN_FRAME;
    cm->comp_var_ref[1] = ALTREF_FRAME;
  }
91
#endif  // CONFIG_EXT_REFS
Jingning Han's avatar
Jingning Han committed
92 93 94 95 96 97
}

static int read_is_valid(const uint8_t *start, size_t len, const uint8_t *end) {
  return len != 0 && len <= (size_t)(end - start);
}

Yaowu Xu's avatar
Yaowu Xu committed
98 99
static int decode_unsigned_max(struct aom_read_bit_buffer *rb, int max) {
  const int data = aom_rb_read_literal(rb, get_unsigned_bits(max));
Jingning Han's avatar
Jingning Han committed
100 101 102
  return data > max ? max : data;
}

Yaowu Xu's avatar
Yaowu Xu committed
103 104
static TX_MODE read_tx_mode(struct aom_read_bit_buffer *rb) {
  return aom_rb_read_bit(rb) ? TX_MODE_SELECT : aom_rb_read_literal(rb, 2);
105
}
Jingning Han's avatar
Jingning Han committed
106

Yaowu Xu's avatar
Yaowu Xu committed
107
static void read_switchable_interp_probs(FRAME_CONTEXT *fc, aom_reader *r) {
Jingning Han's avatar
Jingning Han committed
108 109 110
  int i, j;
  for (j = 0; j < SWITCHABLE_FILTER_CONTEXTS; ++j)
    for (i = 0; i < SWITCHABLE_FILTERS - 1; ++i)
Yaowu Xu's avatar
Yaowu Xu committed
111
      av1_diff_update_prob(r, &fc->switchable_interp_prob[j][i]);
Jingning Han's avatar
Jingning Han committed
112 113
}

Yaowu Xu's avatar
Yaowu Xu committed
114
static void read_inter_mode_probs(FRAME_CONTEXT *fc, aom_reader *r) {
115 116 117
  int i;
#if CONFIG_REF_MV
  for (i = 0; i < NEWMV_MODE_CONTEXTS; ++i)
Yaowu Xu's avatar
Yaowu Xu committed
118
    av1_diff_update_prob(r, &fc->newmv_prob[i]);
119
  for (i = 0; i < ZEROMV_MODE_CONTEXTS; ++i)
Yaowu Xu's avatar
Yaowu Xu committed
120
    av1_diff_update_prob(r, &fc->zeromv_prob[i]);
121
  for (i = 0; i < REFMV_MODE_CONTEXTS; ++i)
Yaowu Xu's avatar
Yaowu Xu committed
122
    av1_diff_update_prob(r, &fc->refmv_prob[i]);
123
  for (i = 0; i < DRL_MODE_CONTEXTS; ++i)
Yaowu Xu's avatar
Yaowu Xu committed
124
    av1_diff_update_prob(r, &fc->drl_prob[i]);
Yue Chen's avatar
Yue Chen committed
125
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
126
  av1_diff_update_prob(r, &fc->new2mv_prob);
Yue Chen's avatar
Yue Chen committed
127
#endif  // CONFIG_EXT_INTER
128 129
#else
  int j;
Jingning Han's avatar
Jingning Han committed
130 131
  for (i = 0; i < INTER_MODE_CONTEXTS; ++i)
    for (j = 0; j < INTER_MODES - 1; ++j)
Yaowu Xu's avatar
Yaowu Xu committed
132
      av1_diff_update_prob(r, &fc->inter_mode_probs[i][j]);
133
#endif
Jingning Han's avatar
Jingning Han committed
134 135
}

136
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
137
static void read_inter_compound_mode_probs(FRAME_CONTEXT *fc, aom_reader *r) {
138
  int i, j;
Yaowu Xu's avatar
Yaowu Xu committed
139
  if (aom_read(r, GROUP_DIFF_UPDATE_PROB)) {
140 141
    for (j = 0; j < INTER_MODE_CONTEXTS; ++j) {
      for (i = 0; i < INTER_COMPOUND_MODES - 1; ++i) {
Yaowu Xu's avatar
Yaowu Xu committed
142
        av1_diff_update_prob(r, &fc->inter_compound_mode_probs[j][i]);
143 144 145 146 147 148
      }
    }
  }
}
#endif  // CONFIG_EXT_INTER

149
static REFERENCE_MODE read_frame_reference_mode(
Yaowu Xu's avatar
Yaowu Xu committed
150
    const AV1_COMMON *cm, struct aom_read_bit_buffer *rb) {
151
  if (is_compound_reference_allowed(cm)) {
Yaowu Xu's avatar
Yaowu Xu committed
152
    return aom_rb_read_bit(rb)
153
               ? REFERENCE_MODE_SELECT
Yaowu Xu's avatar
Yaowu Xu committed
154
               : (aom_rb_read_bit(rb) ? COMPOUND_REFERENCE : SINGLE_REFERENCE);
155 156 157 158
  } else {
    return SINGLE_REFERENCE;
  }
}
Jingning Han's avatar
Jingning Han committed
159

Yaowu Xu's avatar
Yaowu Xu committed
160
static void read_frame_reference_mode_probs(AV1_COMMON *cm, aom_reader *r) {
Jingning Han's avatar
Jingning Han committed
161
  FRAME_CONTEXT *const fc = cm->fc;
162
  int i, j;
Jingning Han's avatar
Jingning Han committed
163 164 165

  if (cm->reference_mode == REFERENCE_MODE_SELECT)
    for (i = 0; i < COMP_INTER_CONTEXTS; ++i)
Yaowu Xu's avatar
Yaowu Xu committed
166
      av1_diff_update_prob(r, &fc->comp_inter_prob[i]);
Jingning Han's avatar
Jingning Han committed
167

168
  if (cm->reference_mode != COMPOUND_REFERENCE) {
Jingning Han's avatar
Jingning Han committed
169
    for (i = 0; i < REF_CONTEXTS; ++i) {
170
      for (j = 0; j < (SINGLE_REFS - 1); ++j) {
Yaowu Xu's avatar
Yaowu Xu committed
171
        av1_diff_update_prob(r, &fc->single_ref_prob[i][j]);
172
      }
Jingning Han's avatar
Jingning Han committed
173
    }
174
  }
Jingning Han's avatar
Jingning Han committed
175

176 177
  if (cm->reference_mode != SINGLE_REFERENCE) {
    for (i = 0; i < REF_CONTEXTS; ++i) {
178
#if CONFIG_EXT_REFS
179
      for (j = 0; j < (FWD_REFS - 1); ++j)
Yaowu Xu's avatar
Yaowu Xu committed
180
        av1_diff_update_prob(r, &fc->comp_ref_prob[i][j]);
181
      for (j = 0; j < (BWD_REFS - 1); ++j)
Yaowu Xu's avatar
Yaowu Xu committed
182
        av1_diff_update_prob(r, &fc->comp_bwdref_prob[i][j]);
183 184
#else
      for (j = 0; j < (COMP_REFS - 1); ++j)
Yaowu Xu's avatar
Yaowu Xu committed
185
        av1_diff_update_prob(r, &fc->comp_ref_prob[i][j]);
186
#endif  // CONFIG_EXT_REFS
187 188
    }
  }
Jingning Han's avatar
Jingning Han committed
189 190
}

Yaowu Xu's avatar
Yaowu Xu committed
191
static void update_mv_probs(aom_prob *p, int n, aom_reader *r) {
Jingning Han's avatar
Jingning Han committed
192
  int i;
Yaowu Xu's avatar
Yaowu Xu committed
193
  for (i = 0; i < n; ++i) av1_diff_update_prob(r, &p[i]);
Jingning Han's avatar
Jingning Han committed
194 195
}

Yaowu Xu's avatar
Yaowu Xu committed
196
static void read_mv_probs(nmv_context *ctx, int allow_hp, aom_reader *r) {
Jingning Han's avatar
Jingning Han committed
197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224
  int i, j;

  update_mv_probs(ctx->joints, MV_JOINTS - 1, r);

  for (i = 0; i < 2; ++i) {
    nmv_component *const comp_ctx = &ctx->comps[i];
    update_mv_probs(&comp_ctx->sign, 1, r);
    update_mv_probs(comp_ctx->classes, MV_CLASSES - 1, r);
    update_mv_probs(comp_ctx->class0, CLASS0_SIZE - 1, r);
    update_mv_probs(comp_ctx->bits, MV_OFFSET_BITS, r);
  }

  for (i = 0; i < 2; ++i) {
    nmv_component *const comp_ctx = &ctx->comps[i];
    for (j = 0; j < CLASS0_SIZE; ++j)
      update_mv_probs(comp_ctx->class0_fp[j], MV_FP_SIZE - 1, r);
    update_mv_probs(comp_ctx->fp, 3, r);
  }

  if (allow_hp) {
    for (i = 0; i < 2; ++i) {
      nmv_component *const comp_ctx = &ctx->comps[i];
      update_mv_probs(&comp_ctx->class0_hp, 1, r);
      update_mv_probs(&comp_ctx->hp, 1, r);
    }
  }
}

225
static void inverse_transform_block(MACROBLOCKD *xd, int plane,
226
                                    const TX_TYPE tx_type,
227 228
                                    const TX_SIZE tx_size, uint8_t *dst,
                                    int stride, int eob) {
Jingning Han's avatar
Jingning Han committed
229 230 231
  struct macroblockd_plane *const pd = &xd->plane[plane];
  if (eob > 0) {
    tran_low_t *const dqcoeff = pd->dqcoeff;
Angie Chiang's avatar
Angie Chiang committed
232 233 234 235
    INV_TXFM_PARAM inv_txfm_param;
    inv_txfm_param.tx_type = tx_type;
    inv_txfm_param.tx_size = tx_size;
    inv_txfm_param.eob = eob;
236
    inv_txfm_param.lossless = xd->lossless[xd->mi[0]->mbmi.segment_id];
Angie Chiang's avatar
Angie Chiang committed
237

Yaowu Xu's avatar
Yaowu Xu committed
238
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
239
    if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
Angie Chiang's avatar
Angie Chiang committed
240 241
      inv_txfm_param.bd = xd->bd;
      highbd_inv_txfm_add(dqcoeff, dst, stride, &inv_txfm_param);
Jingning Han's avatar
Jingning Han committed
242
    } else {
Yaowu Xu's avatar
Yaowu Xu committed
243
#endif  // CONFIG_AOM_HIGHBITDEPTH
Angie Chiang's avatar
Angie Chiang committed
244
      inv_txfm_add(dqcoeff, dst, stride, &inv_txfm_param);
Yaowu Xu's avatar
Yaowu Xu committed
245
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
246
    }
Yaowu Xu's avatar
Yaowu Xu committed
247
#endif  // CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
248 249 250 251

    if (eob == 1) {
      dqcoeff[0] = 0;
    } else {
252
      if (tx_type == DCT_DCT && tx_size <= TX_16X16 && eob <= 10)
253
        memset(dqcoeff, 0, 4 * 4 * num_4x4_blocks_wide_txsize_lookup[tx_size] *
254
                               sizeof(dqcoeff[0]));
255 256
#if CONFIG_EXT_TX
      else
257
        memset(dqcoeff, 0, get_tx2d_size(tx_size) * sizeof(dqcoeff[0]));
258
#else
Jingning Han's avatar
Jingning Han committed
259 260 261
      else if (tx_size == TX_32X32 && eob <= 34)
        memset(dqcoeff, 0, 256 * sizeof(dqcoeff[0]));
      else
262
        memset(dqcoeff, 0, get_tx2d_size(tx_size) * sizeof(dqcoeff[0]));
263
#endif
Jingning Han's avatar
Jingning Han committed
264 265 266 267 268
    }
  }
}

static void predict_and_reconstruct_intra_block(MACROBLOCKD *const xd,
Alex Converse's avatar
Alex Converse committed
269 270 271
#if CONFIG_ANS
                                                struct AnsDecoder *const r,
#else
Yaowu Xu's avatar
Yaowu Xu committed
272
                                                aom_reader *r,
Alex Converse's avatar
Alex Converse committed
273
#endif  // CONFIG_ANS
Jingning Han's avatar
Jingning Han committed
274
                                                MB_MODE_INFO *const mbmi,
275
                                                int plane, int row, int col,
Jingning Han's avatar
Jingning Han committed
276 277 278
                                                TX_SIZE tx_size) {
  struct macroblockd_plane *const pd = &xd->plane[plane];
  PREDICTION_MODE mode = (plane == 0) ? mbmi->mode : mbmi->uv_mode;
hui su's avatar
hui su committed
279
  PLANE_TYPE plane_type = (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
Jingning Han's avatar
Jingning Han committed
280
  uint8_t *dst;
hui su's avatar
hui su committed
281
  int block_idx = (row << 1) + col;
Jingning Han's avatar
Jingning Han committed
282 283 284
  dst = &pd->dst.buf[4 * row * pd->dst.stride + 4 * col];

  if (mbmi->sb_type < BLOCK_8X8)
285
    if (plane == 0) mode = xd->mi[0]->bmi[(row << 1) + col].as_mode;
Jingning Han's avatar
Jingning Han committed
286

Yaowu Xu's avatar
Yaowu Xu committed
287 288
  av1_predict_intra_block(xd, pd->n4_wl, pd->n4_hl, tx_size, mode, dst,
                          pd->dst.stride, dst, pd->dst.stride, col, row, plane);
Jingning Han's avatar
Jingning Han committed
289 290

  if (!mbmi->skip) {
hui su's avatar
hui su committed
291
    TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, tx_size);
292
    const scan_order *sc = get_scan(tx_size, tx_type, 0);
Yaowu Xu's avatar
Yaowu Xu committed
293 294
    const int eob = av1_decode_block_tokens(xd, plane, sc, col, row, tx_size,
                                            tx_type, r, mbmi->segment_id);
295 296
    inverse_transform_block(xd, plane, tx_type, tx_size, dst, pd->dst.stride,
                            eob);
Jingning Han's avatar
Jingning Han committed
297 298 299
  }
}

300
#if CONFIG_VAR_TX
Yaowu Xu's avatar
Yaowu Xu committed
301
static void decode_reconstruct_tx(MACROBLOCKD *const xd, aom_reader *r,
302 303 304 305
                                  MB_MODE_INFO *const mbmi, int plane,
                                  BLOCK_SIZE plane_bsize, int block,
                                  int blk_row, int blk_col, TX_SIZE tx_size,
                                  int *eob_total) {
306
  const struct macroblockd_plane *const pd = &xd->plane[plane];
307
  const BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
308 309
  const int tx_row = blk_row >> (1 - pd->subsampling_y);
  const int tx_col = blk_col >> (1 - pd->subsampling_x);
310
  const TX_SIZE plane_tx_size =
311
      plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0]
312
            : mbmi->inter_tx_size[tx_row][tx_col];
313 314 315 316 317 318 319 320
  int max_blocks_high = num_4x4_blocks_high_lookup[plane_bsize];
  int max_blocks_wide = num_4x4_blocks_wide_lookup[plane_bsize];

  if (xd->mb_to_bottom_edge < 0)
    max_blocks_high += xd->mb_to_bottom_edge >> (5 + pd->subsampling_y);
  if (xd->mb_to_right_edge < 0)
    max_blocks_wide += xd->mb_to_right_edge >> (5 + pd->subsampling_x);

321
  if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
322

323
  if (tx_size == plane_tx_size) {
324
    PLANE_TYPE plane_type = (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
325 326
    TX_TYPE tx_type = get_tx_type(plane_type, xd, block, plane_tx_size);
    const scan_order *sc = get_scan(plane_tx_size, tx_type, 1);
327
    const int eob =
Yaowu Xu's avatar
Yaowu Xu committed
328 329
        av1_decode_block_tokens(xd, plane, sc, blk_col, blk_row, plane_tx_size,
                                tx_type, r, mbmi->segment_id);
330 331
    inverse_transform_block(
        xd, plane, tx_type, plane_tx_size,
332
        &pd->dst.buf[4 * blk_row * pd->dst.stride + 4 * blk_col],
333
        pd->dst.stride, eob);
334 335 336 337 338 339 340 341 342 343 344
    *eob_total += eob;
  } else {
    int bsl = b_width_log2_lookup[bsize];
    int i;

    assert(bsl > 0);
    --bsl;

    for (i = 0; i < 4; ++i) {
      const int offsetr = blk_row + ((i >> 1) << bsl);
      const int offsetc = blk_col + ((i & 0x01) << bsl);
345
      int step = num_4x4_blocks_txsize_lookup[tx_size - 1];
346

347
      if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;
348 349 350 351 352 353

      decode_reconstruct_tx(xd, r, mbmi, plane, plane_bsize, block + i * step,
                            offsetr, offsetc, tx_size - 1, eob_total);
    }
  }
}
354
#endif  // CONFIG_VAR_TX
355

356
#if !CONFIG_VAR_TX || CONFIG_SUPERTX || (CONFIG_EXT_TX && CONFIG_RECT_TX)
Alex Converse's avatar
Alex Converse committed
357 358 359 360
static int reconstruct_inter_block(MACROBLOCKD *const xd,
#if CONFIG_ANS
                                   struct AnsDecoder *const r,
#else
Yaowu Xu's avatar
Yaowu Xu committed
361
                                   aom_reader *r,
Alex Converse's avatar
Alex Converse committed
362
#endif
363 364
                                   int segment_id, int plane, int row, int col,
                                   TX_SIZE tx_size) {
Jingning Han's avatar
Jingning Han committed
365
  struct macroblockd_plane *const pd = &xd->plane[plane];
hui su's avatar
hui su committed
366 367
  PLANE_TYPE plane_type = (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
  int block_idx = (row << 1) + col;
hui su's avatar
hui su committed
368
  TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, tx_size);
369
  const scan_order *sc = get_scan(tx_size, tx_type, 1);
Yaowu Xu's avatar
Yaowu Xu committed
370 371
  const int eob = av1_decode_block_tokens(xd, plane, sc, col, row, tx_size,
                                          tx_type, r, segment_id);
Jingning Han's avatar
Jingning Han committed
372

373 374 375
  inverse_transform_block(xd, plane, tx_type, tx_size,
                          &pd->dst.buf[4 * row * pd->dst.stride + 4 * col],
                          pd->dst.stride, eob);
Jingning Han's avatar
Jingning Han committed
376 377
  return eob;
}
378
#endif  // !CONFIG_VAR_TX || CONFIG_SUPER_TX
Jingning Han's avatar
Jingning Han committed
379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399

static INLINE void dec_reset_skip_context(MACROBLOCKD *xd) {
  int i;
  for (i = 0; i < MAX_MB_PLANE; i++) {
    struct macroblockd_plane *const pd = &xd->plane[i];
    memset(pd->above_context, 0, sizeof(ENTROPY_CONTEXT) * pd->n4_w);
    memset(pd->left_context, 0, sizeof(ENTROPY_CONTEXT) * pd->n4_h);
  }
}

static void set_plane_n4(MACROBLOCKD *const xd, int bw, int bh, int bwl,
                         int bhl) {
  int i;
  for (i = 0; i < MAX_MB_PLANE; i++) {
    xd->plane[i].n4_w = (bw << 1) >> xd->plane[i].subsampling_x;
    xd->plane[i].n4_h = (bh << 1) >> xd->plane[i].subsampling_y;
    xd->plane[i].n4_wl = bwl - xd->plane[i].subsampling_x;
    xd->plane[i].n4_hl = bhl - xd->plane[i].subsampling_y;
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
400
static MB_MODE_INFO *set_offsets(AV1_COMMON *const cm, MACROBLOCKD *const xd,
Jingning Han's avatar
Jingning Han committed
401
                                 BLOCK_SIZE bsize, int mi_row, int mi_col,
402 403
                                 int bw, int bh, int x_mis, int y_mis, int bwl,
                                 int bhl) {
Jingning Han's avatar
Jingning Han committed
404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421
  const int offset = mi_row * cm->mi_stride + mi_col;
  int x, y;
  const TileInfo *const tile = &xd->tile;

  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = &cm->mi[offset];
  // TODO(slavarnway): Generate sb_type based on bwl and bhl, instead of
  // passing bsize from decode_partition().
  xd->mi[0]->mbmi.sb_type = bsize;
  for (y = 0; y < y_mis; ++y)
    for (x = !y; x < x_mis; ++x) {
      xd->mi[y * cm->mi_stride + x] = xd->mi[0];
    }

  set_plane_n4(xd, bw, bh, bwl, bhl);

  set_skip_context(xd, mi_row, mi_col);

422 423 424 425
#if CONFIG_VAR_TX
  xd->max_tx_size = max_txsize_lookup[bsize];
#endif

Jingning Han's avatar
Jingning Han committed
426 427 428 429
  // Distance of Mb to the various image edges. These are specified to 8th pel
  // as they are always compared to values that are in 1/8th pel units
  set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, cm->mi_rows, cm->mi_cols);

Yaowu Xu's avatar
Yaowu Xu committed
430
  av1_setup_dst_planes(xd->plane, get_frame_new_buffer(cm), mi_row, mi_col);
Jingning Han's avatar
Jingning Han committed
431 432 433
  return &xd->mi[0]->mbmi;
}

434
#if CONFIG_SUPERTX
Yaowu Xu's avatar
Yaowu Xu committed
435
static MB_MODE_INFO *set_offsets_extend(AV1_COMMON *const cm,
436 437
                                        MACROBLOCKD *const xd,
                                        const TileInfo *const tile,
438 439 440
                                        BLOCK_SIZE bsize_pred, int mi_row_pred,
                                        int mi_col_pred, int mi_row_ori,
                                        int mi_col_ori) {
441 442 443 444 445 446 447 448 449 450
  // Used in supertx
  // (mi_row_ori, mi_col_ori): location for mv
  // (mi_row_pred, mi_col_pred, bsize_pred): region to predict
  const int bw = num_8x8_blocks_wide_lookup[bsize_pred];
  const int bh = num_8x8_blocks_high_lookup[bsize_pred];
  const int offset = mi_row_ori * cm->mi_stride + mi_col_ori;
  const int bwl = b_width_log2_lookup[bsize_pred];
  const int bhl = b_height_log2_lookup[bsize_pred];
  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = cm->mi + offset;
451 452
  set_mi_row_col(xd, tile, mi_row_pred, bh, mi_col_pred, bw, cm->mi_rows,
                 cm->mi_cols);
453

454 455
  xd->up_available = (mi_row_ori > tile->mi_row_start);
  xd->left_available = (mi_col_ori > tile->mi_col_start);
456 457 458 459 460 461

  set_plane_n4(xd, bw, bh, bwl, bhl);

  return &xd->mi[0]->mbmi;
}

Yaowu Xu's avatar
Yaowu Xu committed
462 463 464
static MB_MODE_INFO *set_mb_offsets(AV1_COMMON *const cm, MACROBLOCKD *const xd,
                                    BLOCK_SIZE bsize, int mi_row, int mi_col,
                                    int bw, int bh, int x_mis, int y_mis) {
465 466 467 468 469 470 471 472
  const int offset = mi_row * cm->mi_stride + mi_col;
  const TileInfo *const tile = &xd->tile;
  int x, y;

  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = cm->mi + offset;
  xd->mi[0]->mbmi.sb_type = bsize;
  for (y = 0; y < y_mis; ++y)
473
    for (x = !y; x < x_mis; ++x) xd->mi[y * cm->mi_stride + x] = xd->mi[0];
474 475 476 477 478

  set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, cm->mi_rows, cm->mi_cols);
  return &xd->mi[0]->mbmi;
}

Yaowu Xu's avatar
Yaowu Xu committed
479
static void set_offsets_topblock(AV1_COMMON *const cm, MACROBLOCKD *const xd,
480 481
                                 const TileInfo *const tile, BLOCK_SIZE bsize,
                                 int mi_row, int mi_col) {
482 483 484 485 486 487 488 489 490 491 492 493 494
  const int bw = num_8x8_blocks_wide_lookup[bsize];
  const int bh = num_8x8_blocks_high_lookup[bsize];
  const int offset = mi_row * cm->mi_stride + mi_col;
  const int bwl = b_width_log2_lookup[bsize];
  const int bhl = b_height_log2_lookup[bsize];

  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = cm->mi + offset;

  set_plane_n4(xd, bw, bh, bwl, bhl);

  set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, cm->mi_rows, cm->mi_cols);

Yaowu Xu's avatar
Yaowu Xu committed
495
  av1_setup_dst_planes(xd->plane, get_frame_new_buffer(cm), mi_row, mi_col);
496 497
}

Yaowu Xu's avatar
Yaowu Xu committed
498
static void set_param_topblock(AV1_COMMON *const cm, MACROBLOCKD *const xd,
499
                               BLOCK_SIZE bsize, int mi_row, int mi_col,
500
                               int txfm, int skip) {
501 502
  const int bw = num_8x8_blocks_wide_lookup[bsize];
  const int bh = num_8x8_blocks_high_lookup[bsize];
Yaowu Xu's avatar
Yaowu Xu committed
503 504
  const int x_mis = AOMMIN(bw, cm->mi_cols - mi_col);
  const int y_mis = AOMMIN(bh, cm->mi_rows - mi_row);
505 506 507 508 509 510 511 512 513 514 515
  const int offset = mi_row * cm->mi_stride + mi_col;
  int x, y;

  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = cm->mi + offset;

  for (y = 0; y < y_mis; ++y)
    for (x = 0; x < x_mis; ++x) {
      xd->mi[y * cm->mi_stride + x]->mbmi.skip = skip;
      xd->mi[y * cm->mi_stride + x]->mbmi.tx_type = txfm;
    }
516 517
#if CONFIG_VAR_TX
  xd->above_txfm_context = cm->above_txfm_context + mi_col;
518
  xd->left_txfm_context =
519
      xd->left_txfm_context_buffer + (mi_row & MAX_MIB_MASK);
520
  set_txfm_ctxs(xd->mi[0]->mbmi.tx_size, bw, bh, xd);
521
#endif
522 523
}

Yaowu Xu's avatar
Yaowu Xu committed
524
static void set_ref(AV1_COMMON *const cm, MACROBLOCKD *const xd, int idx,
525
                    int mi_row, int mi_col) {
526 527 528
  MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
  RefBuffer *ref_buffer = &cm->frame_refs[mbmi->ref_frame[idx] - LAST_FRAME];
  xd->block_refs[idx] = ref_buffer;
Yaowu Xu's avatar
Yaowu Xu committed
529 530
  if (!av1_is_valid_scale(&ref_buffer->sf))
    aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM,
531
                       "Invalid scale factors");
Yaowu Xu's avatar
Yaowu Xu committed
532 533
  av1_setup_pre_planes(xd, idx, ref_buffer->buf, mi_row, mi_col,
                       &ref_buffer->sf);
534 535 536 537
  xd->corrupted |= ref_buffer->buf->corrupted;
}

static void dec_predict_b_extend(
Yaowu Xu's avatar
Yaowu Xu committed
538
    AV1Decoder *const pbi, MACROBLOCKD *const xd, const TileInfo *const tile,
539 540 541
    int block, int mi_row_ori, int mi_col_ori, int mi_row_pred, int mi_col_pred,
    int mi_row_top, int mi_col_top, uint8_t *dst_buf[3], int dst_stride[3],
    BLOCK_SIZE bsize_top, BLOCK_SIZE bsize_pred, int b_sub8x8, int bextend) {
542 543 544 545 546 547 548 549 550 551 552 553
  // Used in supertx
  // (mi_row_ori, mi_col_ori): location for mv
  // (mi_row_pred, mi_col_pred, bsize_pred): region to predict
  // (mi_row_top, mi_col_top, bsize_top): region of the top partition size
  // block: sub location of sub8x8 blocks
  // b_sub8x8: 1: ori is sub8x8; 0: ori is not sub8x8
  // bextend: 1: region to predict is an extension of ori; 0: not
  int r = (mi_row_pred - mi_row_top) * MI_SIZE;
  int c = (mi_col_pred - mi_col_top) * MI_SIZE;
  const int mi_width_top = num_8x8_blocks_wide_lookup[bsize_top];
  const int mi_height_top = num_8x8_blocks_high_lookup[bsize_top];
  MB_MODE_INFO *mbmi;
Yaowu Xu's avatar
Yaowu Xu committed
554
  AV1_COMMON *const cm = &pbi->common;
555 556 557

  if (mi_row_pred < mi_row_top || mi_col_pred < mi_col_top ||
      mi_row_pred >= mi_row_top + mi_height_top ||
558 559
      mi_col_pred >= mi_col_top + mi_width_top || mi_row_pred >= cm->mi_rows ||
      mi_col_pred >= cm->mi_cols)
560 561
    return;

562
  mbmi = set_offsets_extend(cm, xd, tile, bsize_pred, mi_row_pred, mi_col_pred,
563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585
                            mi_row_ori, mi_col_ori);
  set_ref(cm, xd, 0, mi_row_pred, mi_col_pred);
  if (has_second_ref(&xd->mi[0]->mbmi))
    set_ref(cm, xd, 1, mi_row_pred, mi_col_pred);

  if (!bextend) {
    mbmi->tx_size = b_width_log2_lookup[bsize_top];
  }

  xd->plane[0].dst.stride = dst_stride[0];
  xd->plane[1].dst.stride = dst_stride[1];
  xd->plane[2].dst.stride = dst_stride[2];
  xd->plane[0].dst.buf = dst_buf[0] +
                         (r >> xd->plane[0].subsampling_y) * dst_stride[0] +
                         (c >> xd->plane[0].subsampling_x);
  xd->plane[1].dst.buf = dst_buf[1] +
                         (r >> xd->plane[1].subsampling_y) * dst_stride[1] +
                         (c >> xd->plane[1].subsampling_x);
  xd->plane[2].dst.buf = dst_buf[2] +
                         (r >> xd->plane[2].subsampling_y) * dst_stride[2] +
                         (c >> xd->plane[2].subsampling_x);

  if (!b_sub8x8)
Yaowu Xu's avatar
Yaowu Xu committed
586
    av1_build_inter_predictors_sb_extend(xd,
587
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
588
                                         mi_row_ori, mi_col_ori,
589
#endif  // CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
590
                                         mi_row_pred, mi_col_pred, bsize_pred);
591
  else
Yaowu Xu's avatar
Yaowu Xu committed
592
    av1_build_inter_predictors_sb_sub8x8_extend(xd,
593
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
594
                                                mi_row_ori, mi_col_ori,
595
#endif  // CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
596 597
                                                mi_row_pred, mi_col_pred,
                                                bsize_pred, block);
598 599
}

Yaowu Xu's avatar
Yaowu Xu committed
600
static void dec_extend_dir(AV1Decoder *const pbi, MACROBLOCKD *const xd,
601
                           const TileInfo *const tile, int block,
602 603 604
                           BLOCK_SIZE bsize, BLOCK_SIZE top_bsize, int mi_row,
                           int mi_col, int mi_row_top, int mi_col_top,
                           uint8_t *dst_buf[3], int dst_stride[3], int dir) {
605 606 607 608 609 610 611 612 613 614 615
  // dir: 0-lower, 1-upper, 2-left, 3-right
  //      4-lowerleft, 5-upperleft, 6-lowerright, 7-upperright
  const int mi_width = num_8x8_blocks_wide_lookup[bsize];
  const int mi_height = num_8x8_blocks_high_lookup[bsize];
  int xss = xd->plane[1].subsampling_x;
  int yss = xd->plane[1].subsampling_y;
  int b_sub8x8 = (bsize < BLOCK_8X8) ? 1 : 0;
  BLOCK_SIZE extend_bsize;
  int unit, mi_row_pred, mi_col_pred;

  if (dir == 0 || dir == 1) {
616 617 618
    extend_bsize = (mi_width == 1 || bsize < BLOCK_8X8 || xss < yss)
                       ? BLOCK_8X8
                       : BLOCK_16X8;
619 620 621 622
    unit = num_8x8_blocks_wide_lookup[extend_bsize];
    mi_row_pred = mi_row + ((dir == 0) ? mi_height : -1);
    mi_col_pred = mi_col;

623 624 625
    dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col, mi_row_pred,
                         mi_col_pred, mi_row_top, mi_col_top, dst_buf,
                         dst_stride, top_bsize, extend_bsize, b_sub8x8, 1);
626 627 628 629

    if (mi_width > unit) {
      int i;
      assert(!b_sub8x8);
630
      for (i = 0; i < mi_width / unit - 1; i++) {
631
        mi_col_pred += unit;
632 633 634
        dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col, mi_row_pred,
                             mi_col_pred, mi_row_top, mi_col_top, dst_buf,
                             dst_stride, top_bsize, extend_bsize, b_sub8x8, 1);
635 636 637
      }
    }
  } else if (dir == 2 || dir == 3) {
638 639 640
    extend_bsize = (mi_height == 1 || bsize < BLOCK_8X8 || yss < xss)
                       ? BLOCK_8X8
                       : BLOCK_8X16;
641 642 643 644
    unit = num_8x8_blocks_high_lookup[extend_bsize];
    mi_row_pred = mi_row;
    mi_col_pred = mi_col + ((dir == 3) ? mi_width : -1);

645 646 647
    dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col, mi_row_pred,
                         mi_col_pred, mi_row_top, mi_col_top, dst_buf,
                         dst_stride, top_bsize, extend_bsize, b_sub8x8, 1);
648 649 650

    if (mi_height > unit) {
      int i;
651
      for (i = 0; i < mi_height / unit - 1; i++) {
652
        mi_row_pred += unit;
653 654 655
        dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col, mi_row_pred,
                             mi_col_pred, mi_row_top, mi_col_top, dst_buf,
                             dst_stride, top_bsize, extend_bsize, b_sub8x8, 1);
656 657 658 659 660 661
      }
    }
  } else {
    extend_bsize = BLOCK_8X8;
    mi_row_pred = mi_row + ((dir == 4 || dir == 6) ? mi_height : -1);
    mi_col_pred = mi_col + ((dir == 6 || dir == 7) ? mi_width : -1);
662 663 664
    dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col, mi_row_pred,
                         mi_col_pred, mi_row_top, mi_col_top, dst_buf,
                         dst_stride, top_bsize, extend_bsize, b_sub8x8, 1);
665 666 667
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
668
static void dec_extend_all(AV1Decoder *const pbi, MACROBLOCKD *const xd,
669
                           const TileInfo *const tile, int block,
670 671 672
                           BLOCK_SIZE bsize, BLOCK_SIZE top_bsize, int mi_row,
                           int mi_col, int mi_row_top, int mi_col_top,
                           uint8_t *dst_buf[3], int dst_stride[3]) {
673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 0);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 1);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 2);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 3);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 4);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 5);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 6);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 7);
}

Yaowu Xu's avatar
Yaowu Xu committed
691
static void dec_predict_sb_complex(AV1Decoder *const pbi, MACROBLOCKD *const xd,
692 693
                                   const TileInfo *const tile, int mi_row,
                                   int mi_col, int mi_row_top, int mi_col_top,
694 695
                                   BLOCK_SIZE bsize, BLOCK_SIZE top_bsize,
                                   uint8_t *dst_buf[3], int dst_stride[3]) {
Yaowu Xu's avatar
Yaowu Xu committed
696
  const AV1_COMMON *const cm = &pbi->common;
697 698 699
  const int hbs = num_8x8_blocks_wide_lookup[bsize] / 2;
  const PARTITION_TYPE partition = get_partition(cm, mi_row, mi_col, bsize);
  const BLOCK_SIZE subsize = get_subsize(bsize, partition);
700
#if CONFIG_EXT_PARTITION_TYPES
701
  const BLOCK_SIZE bsize2 = get_subsize(bsize, PARTITION_SPLIT);
702
#endif
703 704
  int i;
  const int mi_offset = mi_row * cm->mi_stride + mi_col;
705 706
  uint8_t *dst_buf1[3], *dst_buf2[3], *dst_buf3[3];

707 708 709 710 711 712
  DECLARE_ALIGNED(16, uint8_t, tmp_buf1[MAX_MB_PLANE * MAX_TX_SQUARE * 2]);
  DECLARE_ALIGNED(16, uint8_t, tmp_buf2[MAX_MB_PLANE * MAX_TX_SQUARE * 2]);
  DECLARE_ALIGNED(16, uint8_t, tmp_buf3[MAX_MB_PLANE * MAX_TX_SQUARE * 2]);
  int dst_stride1[3] = { MAX_TX_SIZE, MAX_TX_SIZE, MAX_TX_SIZE };
  int dst_stride2[3] = { MAX_TX_SIZE, MAX_TX_SIZE, MAX_TX_SIZE };
  int dst_stride3[3] = { MAX_TX_SIZE, MAX_TX_SIZE, MAX_TX_SIZE };
713

Yaowu Xu's avatar
Yaowu Xu committed
714
#if CONFIG_AOM_HIGHBITDEPTH
715 716 717
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
    int len = sizeof(uint16_t);
    dst_buf1[0] = CONVERT_TO_BYTEPTR(tmp_buf1);
718 719
    dst_buf1[1] = CONVERT_TO_BYTEPTR(tmp_buf1 + MAX_TX_SQUARE * len);
    dst_buf1[2] = CONVERT_TO_BYTEPTR(tmp_buf1 + 2 * MAX_TX_SQUARE * len);
720
    dst_buf2[0] = CONVERT_TO_BYTEPTR(tmp_buf2);
721 722
    dst_buf2[1] = CONVERT_TO_BYTEPTR(tmp_buf2 + MAX_TX_SQUARE * len);
    dst_buf2[2] = CONVERT_TO_BYTEPTR(tmp_buf2 + 2 * MAX_TX_SQUARE * len);
723
    dst_buf3[0] = CONVERT_TO_BYTEPTR(tmp_buf3);
724 725
    dst_buf3[1] = CONVERT_TO_BYTEPTR(tmp_buf3 + MAX_TX_SQUARE * len);
    dst_buf3[2] = CONVERT_TO_BYTEPTR(tmp_buf3 + 2 * MAX_TX_SQUARE * len);
726 727 728
  } else {
#endif
    dst_buf1[0] = tmp_buf1;
729 730
    dst_buf1[1] = tmp_buf1 + MAX_TX_SQUARE;
    dst_buf1[2] = tmp_buf1 + 2 * MAX_TX_SQUARE;
731
    dst_buf2[0] = tmp_buf2;
732 733
    dst_buf2[1] = tmp_buf2 + MAX_TX_SQUARE;
    dst_buf2[2] = tmp_buf2 + 2 * MAX_TX_SQUARE;
734
    dst_buf3[0] = tmp_buf3;
735 736
    dst_buf3[1] = tmp_buf3 + MAX_TX_SQUARE;
    dst_buf3[2] = tmp_buf3 + 2 * MAX_TX_SQUARE;
Yaowu Xu's avatar
Yaowu Xu committed
737
#if CONFIG_AOM_HIGHBITDEPTH
738 739 740
  }
#endif

741
  if (mi_row >= cm->mi_rows || mi_col >= cm->mi_cols) return;
742

743 744
  xd->mi = cm->mi_grid_visible + mi_offset;
  xd->mi[0] = cm->mi + mi_offset;
745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781

  for (i = 0; i < MAX_MB_PLANE; i++) {
    xd->plane[i].dst.buf = dst_buf[i];
    xd->plane[i].dst.stride = dst_stride[i];
  }

  switch (partition) {
    case PARTITION_NONE:
      assert(bsize < top_bsize);
      dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col,
                           mi_row_top, mi_col_top, dst_buf, dst_stride,
                           top_bsize, bsize, 0, 0);
      dec_extend_all(pbi, xd, tile, 0, bsize, top_bsize, mi_row, mi_col,
                     mi_row_top, mi_col_top, dst_buf, dst_stride);
      break;
    case PARTITION_HORZ:
      if (bsize == BLOCK_8X8) {
        // For sub8x8, predict in 8x8 unit
        // First half
        dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf, dst_stride,
                             top_bsize, BLOCK_8X8, 1, 0);
        if (bsize < top_bsize)
          dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf, dst_stride);

        // Second half
        dec_predict_b_extend(pbi, xd, tile, 2, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf1, dst_stride1,
                             top_bsize, BLOCK_8X8, 1, 1);
        if (bsize < top_bsize)
          dec_extend_all(pbi, xd, tile, 2, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf1, dst_stride1);

        // weighted average to smooth the boundary
        xd->plane[0].dst.buf = dst_buf[0];
        xd->plane[0].dst.stride = dst_stride[0];
Yaowu Xu's avatar
Yaowu Xu committed
782
        av1_build_masked_inter_predictor_complex(
783 784 785
            xd, dst_buf[0], dst_stride[0], dst_buf1[0], dst_stride1[0], mi_row,
            mi_col, mi_row_top, mi_col_top, bsize, top_bsize, PARTITION_HORZ,
            0);
786 787 788 789 790 791 792 793 794 795 796 797 798 799 800
      } else {
        // First half
        dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf, dst_stride,
                             top_bsize, subsize, 0, 0);
        if (bsize < top_bsize)
          dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf, dst_stride);
        else
          dec_extend_dir(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf, dst_stride, 0);

        if (mi_row + hbs < cm->mi_rows) {
          // Second half
          dec_predict_b_extend(pbi, xd, tile, 0, mi_row + hbs, mi_col,
801 802
                               mi_row + hbs, mi_col, mi_row_top, mi_col_top,
                               dst_buf1, dst_stride1, top_bsize, subsize, 0, 0);
803
          if (bsize < top_bsize)
804 805 806
            dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row + hbs,
                           mi_col, mi_row_top, mi_col_top, dst_buf1,
                           dst_stride1);
807
          else
808 809 810
            dec_extend_dir(pbi, xd, tile, 0, subsize, top_bsize, mi_row + hbs,
                           mi_col, mi_row_top, mi_col_top, dst_buf1,
                           dst_stride1, 1);
811 812 813 814 815

          // weighted average to smooth the boundary
          for (i = 0; i < MAX_MB_PLANE; i++) {
            xd->plane[i].dst.buf = dst_buf[i];
            xd->plane[i].dst.stride = dst_stride[i];
Yaowu Xu's avatar
Yaowu Xu committed
816
            av1_build_masked_inter_predictor_complex(
817
                xd, dst_buf[i], dst_stride[i], dst_buf1[i], dst_stride1[i],
818 819
                mi_row, mi_col, mi_row_top, mi_col_top, bsize, top_bsize,
                PARTITION_HORZ, i);
820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844
          }
        }
      }
      break;
    case PARTITION_VERT:
      if (bsize == BLOCK_8X8) {
        // First half
        dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf, dst_stride,
                             top_bsize, BLOCK_8X8, 1, 0);
        if (bsize < top_bsize)
          dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf, dst_stride);

        // Second half
        dec_predict_b_extend(pbi, xd, tile, 1, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf1, dst_stride1,
                             top_bsize, BLOCK_8X8, 1, 1);
        if (bsize < top_bsize)
          dec_extend_all(pbi, xd, tile, 1, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf1, dst_stride1);

        // Smooth
        xd->plane[0].dst.buf = dst_buf[0];
        xd->plane[0].dst.stride = dst_stride[0];
Yaowu Xu's avatar
Yaowu Xu committed
845
        av1_build_masked_inter_predictor_complex(
846 847 848
            xd, dst_buf[0], dst_stride[0], dst_buf1[0], dst_stride1[0], mi_row,
            mi_col, mi_row_top, mi_col_top, bsize, top_bsize, PARTITION_VERT,
            0);
849 850 851 852 853 854 855 856 857 858 859 860 861 862
      } else {
        // First half
        dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf, dst_stride,
                             top_bsize, subsize, 0, 0);
        if (bsize < top_bsize)
          dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf, dst_stride);
        else
          dec_extend_dir(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf, dst_stride, 3);

        // Second half
        if (mi_col + hbs < cm->mi_cols) {
863 864 865
          dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col + hbs, mi_row,
                               mi_col + hbs, mi_row_top, mi_col_top, dst_buf1,
                               dst_stride1, top_bsize, subsize, 0, 0);
866
          if (bsize < top_bsize)
867 868 869
            dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row,
                           mi_col + hbs, mi_row_top, mi_col_top, dst_buf1,
                           dst_stride1);
870
          else
871 872 873
            dec_extend_dir(pbi, xd, tile, 0, subsize, top_bsize, mi_row,
                           mi_col + hbs, mi_row_top, mi_col_top, dst_buf1,
                           dst_stride1, 2);
874 875 876 877 878

          // Smooth
          for (i = 0; i < MAX_MB_PLANE; i++) {
            xd->plane[i].dst.buf = dst_buf[i];
            xd->plane[i].dst.stride = dst_stride[i];
Yaowu Xu's avatar
Yaowu Xu committed
879
            av1_build_masked_inter_predictor_complex(
880
                xd, dst_buf[i], dst_stride[i], dst_buf1[i], dst_stride1[i],
881 882
                mi_row, mi_col, mi_row_top, mi_col_top, bsize, top_bsize,
                PARTITION_VERT, i);
883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911
          }
        }
      }
      break;
    case PARTITION_SPLIT:
      if (bsize == BLOCK_8X8) {
        dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf, dst_stride,
                             top_bsize, BLOCK_8X8, 1, 0);
        dec_predict_b_extend(pbi, xd, tile, 1, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf1, dst_stride1,
                             top_bsize, BLOCK_8X8, 1, 1);
        dec_predict_b_extend(pbi, xd, tile, 2, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf2, dst_stride2,
                             top_bsize, BLOCK_8X8, 1, 1);
        dec_predict_b_extend(pbi, xd, tile, 3, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf3, dst_stride3,
                             top_bsize, BLOCK_8X8, 1, 1);
        if (bsize < top_bsize) {
          dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf, dst_stride);
          dec_extend_all(pbi, xd, tile, 1, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf1, dst_stride1);
          dec_extend_all(pbi, xd, tile, 2, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf2, dst_stride2);
          dec_extend_all(pbi, xd, tile, 3, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf3, dst_stride3);
        }
      } else {
912 913 914
        dec_predict_sb_complex(pbi, xd, tile, mi_row, mi_col, mi_row_top,
                               mi_col_top, subsize, top_bsize, dst_buf,
                               dst_stride);
915 916 917 918 919 920 921 922 923 924 925 926 927
        if (mi_row < cm->mi_rows && mi_col + hbs < cm->mi_cols)
          dec_predict_sb_complex(pbi, xd, tile, mi_row, mi_col + hbs,
                                 mi_row_top, mi_col_top, subsize, top_bsize,
                                 dst_buf1, dst_stride1);
        if (mi_row + hbs < cm->mi_rows && mi_col < cm->mi_cols)
          dec_predict_sb_complex(pbi, xd, tile, mi_row + hbs, mi_col,
                                 mi_row_top, mi_col_top, subsize, top_bsize,
                                 dst_buf2, dst_stride2);
        if (mi_row + hbs < cm->mi_rows && mi_col + hbs < cm->mi_cols)
          dec_predict_sb_complex(pbi, xd, tile, mi_row + hbs, mi_col + hbs,
                                 mi_row_top, mi_col_top, subsize, top_bsize,
                                 dst_buf3, dst_stride3);
      }
928 929 930 931
      for (i = 0; i < MAX_MB_PLANE; i++) {
        if (bsize == BLOCK_8X8 && i != 0)
          continue;  // Skip <4x4 chroma smoothing
        if (mi_row < cm->mi_rows && mi_col + hbs < cm->mi_cols) {
Yaowu Xu's avatar
Yaowu Xu committed
932
          av1_build_masked_inter_predictor_complex(