decodeframe.c 161 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4 5 6 7 8 9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10 11 12 13 14
 */

#include <assert.h>
#include <stdlib.h>  // qsort()

Yaowu Xu's avatar
Yaowu Xu committed
15 16 17
#include "./aom_config.h"
#include "./aom_dsp_rtcd.h"
#include "./aom_scale_rtcd.h"
Jingning Han's avatar
Jingning Han committed
18
#include "./av1_rtcd.h"
Jingning Han's avatar
Jingning Han committed
19

20
#include "aom/aom_codec.h"
Yaowu Xu's avatar
Yaowu Xu committed
21
#include "aom_dsp/aom_dsp_common.h"
Jingning Han's avatar
Jingning Han committed
22 23
#include "aom_dsp/bitreader.h"
#include "aom_dsp/bitreader_buffer.h"
Yaowu Xu's avatar
Yaowu Xu committed
24
#include "aom_mem/aom_mem.h"
25 26
#include "aom_ports/mem.h"
#include "aom_ports/mem_ops.h"
Yaowu Xu's avatar
Yaowu Xu committed
27 28
#include "aom_scale/aom_scale.h"
#include "aom_util/aom_thread.h"
Jingning Han's avatar
Jingning Han committed
29

30
#include "av1/common/alloccommon.h"
31
#if CONFIG_CLPF
Steinar Midtskogen's avatar
Steinar Midtskogen committed
32
#include "aom/aom_image.h"
33
#include "av1/common/clpf.h"
34
#endif
35
#include "av1/common/common.h"
Yaowu Xu's avatar
Yaowu Xu committed
36
#if CONFIG_DERING
37
#include "av1/common/dering.h"
Yaowu Xu's avatar
Yaowu Xu committed
38
#endif  // CONFIG_DERING
39 40 41 42 43 44
#include "av1/common/entropy.h"
#include "av1/common/entropymode.h"
#include "av1/common/idct.h"
#include "av1/common/pred_common.h"
#include "av1/common/quant_common.h"
#include "av1/common/reconinter.h"
Jingning Han's avatar
Jingning Han committed
45
#include "av1/common/reconintra.h"
46
#include "av1/common/seg_common.h"
Jingning Han's avatar
Jingning Han committed
47
#include "av1/common/thread_common.h"
48
#include "av1/common/tile_common.h"
49

50 51 52
#include "av1/decoder/decodeframe.h"
#include "av1/decoder/decodemv.h"
#include "av1/decoder/decoder.h"
Jingning Han's avatar
Jingning Han committed
53
#include "av1/decoder/detokenize.h"
54
#include "av1/decoder/dsubexp.h"
Jingning Han's avatar
Jingning Han committed
55

Yaowu Xu's avatar
Yaowu Xu committed
56
#define MAX_AV1_HEADER_SIZE 80
Michael Bebenita's avatar
Michael Bebenita committed
57
#define ACCT_STR __func__
Jingning Han's avatar
Jingning Han committed
58

59 60 61 62 63 64 65 66
static struct aom_read_bit_buffer *init_read_bit_buffer(
    AV1Decoder *pbi, struct aom_read_bit_buffer *rb, const uint8_t *data,
    const uint8_t *data_end, uint8_t clear_data[MAX_AV1_HEADER_SIZE]);
static int read_compressed_header(AV1Decoder *pbi, const uint8_t *data,
                                  size_t partition_size);
static size_t read_uncompressed_header(AV1Decoder *pbi,
                                       struct aom_read_bit_buffer *rb);

Yaowu Xu's avatar
Yaowu Xu committed
67
static int is_compound_reference_allowed(const AV1_COMMON *cm) {
Jingning Han's avatar
Jingning Han committed
68
  int i;
69
  if (frame_is_intra_only(cm)) return 0;
70
  for (i = 1; i < INTER_REFS_PER_FRAME; ++i)
71
    if (cm->ref_frame_sign_bias[i + 1] != cm->ref_frame_sign_bias[1]) return 1;
Jingning Han's avatar
Jingning Han committed
72 73 74 75

  return 0;
}

Yaowu Xu's avatar
Yaowu Xu committed
76
static void setup_compound_reference_mode(AV1_COMMON *cm) {
77
#if CONFIG_EXT_REFS
78
  cm->comp_fwd_ref[0] = LAST_FRAME;
79 80 81 82
  cm->comp_fwd_ref[1] = LAST2_FRAME;
  cm->comp_fwd_ref[2] = LAST3_FRAME;
  cm->comp_fwd_ref[3] = GOLDEN_FRAME;

83 84
  cm->comp_bwd_ref[0] = BWDREF_FRAME;
  cm->comp_bwd_ref[1] = ALTREF_FRAME;
85
#else
Jingning Han's avatar
Jingning Han committed
86
  if (cm->ref_frame_sign_bias[LAST_FRAME] ==
87
      cm->ref_frame_sign_bias[GOLDEN_FRAME]) {
Jingning Han's avatar
Jingning Han committed
88 89 90 91
    cm->comp_fixed_ref = ALTREF_FRAME;
    cm->comp_var_ref[0] = LAST_FRAME;
    cm->comp_var_ref[1] = GOLDEN_FRAME;
  } else if (cm->ref_frame_sign_bias[LAST_FRAME] ==
92
             cm->ref_frame_sign_bias[ALTREF_FRAME]) {
Jingning Han's avatar
Jingning Han committed
93 94 95 96 97 98 99 100
    cm->comp_fixed_ref = GOLDEN_FRAME;
    cm->comp_var_ref[0] = LAST_FRAME;
    cm->comp_var_ref[1] = ALTREF_FRAME;
  } else {
    cm->comp_fixed_ref = LAST_FRAME;
    cm->comp_var_ref[0] = GOLDEN_FRAME;
    cm->comp_var_ref[1] = ALTREF_FRAME;
  }
101
#endif  // CONFIG_EXT_REFS
Jingning Han's avatar
Jingning Han committed
102 103 104 105 106 107
}

static int read_is_valid(const uint8_t *start, size_t len, const uint8_t *end) {
  return len != 0 && len <= (size_t)(end - start);
}

Yaowu Xu's avatar
Yaowu Xu committed
108 109
static int decode_unsigned_max(struct aom_read_bit_buffer *rb, int max) {
  const int data = aom_rb_read_literal(rb, get_unsigned_bits(max));
Jingning Han's avatar
Jingning Han committed
110 111 112
  return data > max ? max : data;
}

Yaowu Xu's avatar
Yaowu Xu committed
113 114
static TX_MODE read_tx_mode(struct aom_read_bit_buffer *rb) {
  return aom_rb_read_bit(rb) ? TX_MODE_SELECT : aom_rb_read_literal(rb, 2);
115
}
Jingning Han's avatar
Jingning Han committed
116

Yaowu Xu's avatar
Yaowu Xu committed
117
static void read_switchable_interp_probs(FRAME_CONTEXT *fc, aom_reader *r) {
Jingning Han's avatar
Jingning Han committed
118
  int i, j;
119
  for (j = 0; j < SWITCHABLE_FILTER_CONTEXTS; ++j) {
Jingning Han's avatar
Jingning Han committed
120
    for (i = 0; i < SWITCHABLE_FILTERS - 1; ++i)
Michael Bebenita's avatar
Michael Bebenita committed
121
      av1_diff_update_prob(r, &fc->switchable_interp_prob[j][i], ACCT_STR);
122 123 124 125 126
#if CONFIG_DAALA_EC
    av1_tree_to_cdf(av1_switchable_interp_tree, fc->switchable_interp_prob[j],
                    fc->switchable_interp_cdf[j]);
#endif
  }
Jingning Han's avatar
Jingning Han committed
127 128
}

Yaowu Xu's avatar
Yaowu Xu committed
129
static void read_inter_mode_probs(FRAME_CONTEXT *fc, aom_reader *r) {
130 131 132
  int i;
#if CONFIG_REF_MV
  for (i = 0; i < NEWMV_MODE_CONTEXTS; ++i)
Michael Bebenita's avatar
Michael Bebenita committed
133
    av1_diff_update_prob(r, &fc->newmv_prob[i], ACCT_STR);
134
  for (i = 0; i < ZEROMV_MODE_CONTEXTS; ++i)
Michael Bebenita's avatar
Michael Bebenita committed
135
    av1_diff_update_prob(r, &fc->zeromv_prob[i], ACCT_STR);
136
  for (i = 0; i < REFMV_MODE_CONTEXTS; ++i)
Michael Bebenita's avatar
Michael Bebenita committed
137
    av1_diff_update_prob(r, &fc->refmv_prob[i], ACCT_STR);
138
  for (i = 0; i < DRL_MODE_CONTEXTS; ++i)
Michael Bebenita's avatar
Michael Bebenita committed
139
    av1_diff_update_prob(r, &fc->drl_prob[i], ACCT_STR);
Yue Chen's avatar
Yue Chen committed
140
#if CONFIG_EXT_INTER
Michael Bebenita's avatar
Michael Bebenita committed
141
  av1_diff_update_prob(r, &fc->new2mv_prob, ACCT_STR);
Yue Chen's avatar
Yue Chen committed
142
#endif  // CONFIG_EXT_INTER
143 144
#else
  int j;
145
  for (i = 0; i < INTER_MODE_CONTEXTS; ++i) {
Jingning Han's avatar
Jingning Han committed
146
    for (j = 0; j < INTER_MODES - 1; ++j)
Michael Bebenita's avatar
Michael Bebenita committed
147
      av1_diff_update_prob(r, &fc->inter_mode_probs[i][j], ACCT_STR);
148 149 150 151 152
#if CONFIG_DAALA_EC
    av1_tree_to_cdf(av1_inter_mode_tree, fc->inter_mode_probs[i],
                    fc->inter_mode_cdf[i]);
#endif
  }
153
#endif
Jingning Han's avatar
Jingning Han committed
154 155
}

156
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
157
static void read_inter_compound_mode_probs(FRAME_CONTEXT *fc, aom_reader *r) {
158
  int i, j;
Michael Bebenita's avatar
Michael Bebenita committed
159
  if (aom_read(r, GROUP_DIFF_UPDATE_PROB, ACCT_STR)) {
160 161
    for (j = 0; j < INTER_MODE_CONTEXTS; ++j) {
      for (i = 0; i < INTER_COMPOUND_MODES - 1; ++i) {
Michael Bebenita's avatar
Michael Bebenita committed
162
        av1_diff_update_prob(r, &fc->inter_compound_mode_probs[j][i], ACCT_STR);
163 164 165 166 167 168
      }
    }
  }
}
#endif  // CONFIG_EXT_INTER

169
static REFERENCE_MODE read_frame_reference_mode(
Yaowu Xu's avatar
Yaowu Xu committed
170
    const AV1_COMMON *cm, struct aom_read_bit_buffer *rb) {
171
  if (is_compound_reference_allowed(cm)) {
Yaowu Xu's avatar
Yaowu Xu committed
172
    return aom_rb_read_bit(rb)
173
               ? REFERENCE_MODE_SELECT
Yaowu Xu's avatar
Yaowu Xu committed
174
               : (aom_rb_read_bit(rb) ? COMPOUND_REFERENCE : SINGLE_REFERENCE);
175 176 177 178
  } else {
    return SINGLE_REFERENCE;
  }
}
Jingning Han's avatar
Jingning Han committed
179

Yaowu Xu's avatar
Yaowu Xu committed
180
static void read_frame_reference_mode_probs(AV1_COMMON *cm, aom_reader *r) {
Jingning Han's avatar
Jingning Han committed
181
  FRAME_CONTEXT *const fc = cm->fc;
182
  int i, j;
Jingning Han's avatar
Jingning Han committed
183 184 185

  if (cm->reference_mode == REFERENCE_MODE_SELECT)
    for (i = 0; i < COMP_INTER_CONTEXTS; ++i)
Michael Bebenita's avatar
Michael Bebenita committed
186
      av1_diff_update_prob(r, &fc->comp_inter_prob[i], ACCT_STR);
Jingning Han's avatar
Jingning Han committed
187

188
  if (cm->reference_mode != COMPOUND_REFERENCE) {
Jingning Han's avatar
Jingning Han committed
189
    for (i = 0; i < REF_CONTEXTS; ++i) {
190
      for (j = 0; j < (SINGLE_REFS - 1); ++j) {
Michael Bebenita's avatar
Michael Bebenita committed
191
        av1_diff_update_prob(r, &fc->single_ref_prob[i][j], ACCT_STR);
192
      }
Jingning Han's avatar
Jingning Han committed
193
    }
194
  }
Jingning Han's avatar
Jingning Han committed
195

196 197
  if (cm->reference_mode != SINGLE_REFERENCE) {
    for (i = 0; i < REF_CONTEXTS; ++i) {
198
#if CONFIG_EXT_REFS
199
      for (j = 0; j < (FWD_REFS - 1); ++j)
Michael Bebenita's avatar
Michael Bebenita committed
200
        av1_diff_update_prob(r, &fc->comp_ref_prob[i][j], ACCT_STR);
201
      for (j = 0; j < (BWD_REFS - 1); ++j)
Michael Bebenita's avatar
Michael Bebenita committed
202
        av1_diff_update_prob(r, &fc->comp_bwdref_prob[i][j], ACCT_STR);
203 204
#else
      for (j = 0; j < (COMP_REFS - 1); ++j)
Michael Bebenita's avatar
Michael Bebenita committed
205
        av1_diff_update_prob(r, &fc->comp_ref_prob[i][j], ACCT_STR);
206
#endif  // CONFIG_EXT_REFS
207 208
    }
  }
Jingning Han's avatar
Jingning Han committed
209 210
}

Yaowu Xu's avatar
Yaowu Xu committed
211
static void update_mv_probs(aom_prob *p, int n, aom_reader *r) {
Jingning Han's avatar
Jingning Han committed
212
  int i;
Michael Bebenita's avatar
Michael Bebenita committed
213
  for (i = 0; i < n; ++i) av1_diff_update_prob(r, &p[i], ACCT_STR);
Jingning Han's avatar
Jingning Han committed
214 215
}

Yaowu Xu's avatar
Yaowu Xu committed
216
static void read_mv_probs(nmv_context *ctx, int allow_hp, aom_reader *r) {
Jingning Han's avatar
Jingning Han committed
217 218 219
  int i, j;

  update_mv_probs(ctx->joints, MV_JOINTS - 1, r);
220
#if CONFIG_DAALA_EC || CONFIG_RANS
221 222
  av1_tree_to_cdf(av1_mv_joint_tree, ctx->joints, ctx->joint_cdf);
#endif
Jingning Han's avatar
Jingning Han committed
223 224 225 226 227 228 229

  for (i = 0; i < 2; ++i) {
    nmv_component *const comp_ctx = &ctx->comps[i];
    update_mv_probs(&comp_ctx->sign, 1, r);
    update_mv_probs(comp_ctx->classes, MV_CLASSES - 1, r);
    update_mv_probs(comp_ctx->class0, CLASS0_SIZE - 1, r);
    update_mv_probs(comp_ctx->bits, MV_OFFSET_BITS, r);
230
#if CONFIG_DAALA_EC || CONFIG_RANS
231 232
    av1_tree_to_cdf(av1_mv_class_tree, comp_ctx->classes, comp_ctx->class_cdf);
#endif
Jingning Han's avatar
Jingning Han committed
233 234 235 236
  }

  for (i = 0; i < 2; ++i) {
    nmv_component *const comp_ctx = &ctx->comps[i];
237
    for (j = 0; j < CLASS0_SIZE; ++j) {
Jingning Han's avatar
Jingning Han committed
238
      update_mv_probs(comp_ctx->class0_fp[j], MV_FP_SIZE - 1, r);
239
#if CONFIG_DAALA_EC || CONFIG_RANS
240 241 242 243
      av1_tree_to_cdf(av1_mv_fp_tree, comp_ctx->class0_fp[j],
                      comp_ctx->class0_fp_cdf[j]);
#endif
    }
244
    update_mv_probs(comp_ctx->fp, MV_FP_SIZE - 1, r);
245
#if CONFIG_DAALA_EC || CONFIG_RANS
246 247
    av1_tree_to_cdf(av1_mv_fp_tree, comp_ctx->fp, comp_ctx->fp_cdf);
#endif
Jingning Han's avatar
Jingning Han committed
248 249 250 251 252 253 254 255 256 257 258
  }

  if (allow_hp) {
    for (i = 0; i < 2; ++i) {
      nmv_component *const comp_ctx = &ctx->comps[i];
      update_mv_probs(&comp_ctx->class0_hp, 1, r);
      update_mv_probs(&comp_ctx->hp, 1, r);
    }
  }
}

259
static void inverse_transform_block(MACROBLOCKD *xd, int plane,
260
                                    const TX_TYPE tx_type,
261
                                    const TX_SIZE tx_size, uint8_t *dst,
262
                                    int stride, int16_t scan_line, int eob) {
Jingning Han's avatar
Jingning Han committed
263
  struct macroblockd_plane *const pd = &xd->plane[plane];
264 265 266 267 268 269
  tran_low_t *const dqcoeff = pd->dqcoeff;
  INV_TXFM_PARAM inv_txfm_param;
  inv_txfm_param.tx_type = tx_type;
  inv_txfm_param.tx_size = tx_size;
  inv_txfm_param.eob = eob;
  inv_txfm_param.lossless = xd->lossless[xd->mi[0]->mbmi.segment_id];
Angie Chiang's avatar
Angie Chiang committed
270

Yaowu Xu's avatar
Yaowu Xu committed
271
#if CONFIG_AOM_HIGHBITDEPTH
272 273 274 275
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
    inv_txfm_param.bd = xd->bd;
    highbd_inv_txfm_add(dqcoeff, dst, stride, &inv_txfm_param);
  } else {
Yaowu Xu's avatar
Yaowu Xu committed
276
#endif  // CONFIG_AOM_HIGHBITDEPTH
277
    inv_txfm_add(dqcoeff, dst, stride, &inv_txfm_param);
Yaowu Xu's avatar
Yaowu Xu committed
278
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
279
  }
280 281
#endif  // CONFIG_AOM_HIGHBITDEPTH
  memset(dqcoeff, 0, (scan_line + 1) * sizeof(dqcoeff[0]));
Jingning Han's avatar
Jingning Han committed
282 283
}

Angie Chiang's avatar
Angie Chiang committed
284 285
static void predict_and_reconstruct_intra_block(AV1_COMMON *cm,
                                                MACROBLOCKD *const xd,
Alex Converse's avatar
Alex Converse committed
286 287 288
#if CONFIG_ANS
                                                struct AnsDecoder *const r,
#else
Yaowu Xu's avatar
Yaowu Xu committed
289
                                                aom_reader *r,
Alex Converse's avatar
Alex Converse committed
290
#endif  // CONFIG_ANS
Jingning Han's avatar
Jingning Han committed
291
                                                MB_MODE_INFO *const mbmi,
292
                                                int plane, int row, int col,
Jingning Han's avatar
Jingning Han committed
293 294 295
                                                TX_SIZE tx_size) {
  struct macroblockd_plane *const pd = &xd->plane[plane];
  PREDICTION_MODE mode = (plane == 0) ? mbmi->mode : mbmi->uv_mode;
hui su's avatar
hui su committed
296
  PLANE_TYPE plane_type = (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
Jingning Han's avatar
Jingning Han committed
297
  uint8_t *dst;
hui su's avatar
hui su committed
298
  int block_idx = (row << 1) + col;
Jingning Han's avatar
Jingning Han committed
299 300 301
  dst = &pd->dst.buf[4 * row * pd->dst.stride + 4 * col];

  if (mbmi->sb_type < BLOCK_8X8)
302
    if (plane == 0) mode = xd->mi[0]->bmi[(row << 1) + col].as_mode;
Jingning Han's avatar
Jingning Han committed
303

304
  av1_predict_intra_block(xd, pd->width, pd->height, tx_size, mode, dst,
Yaowu Xu's avatar
Yaowu Xu committed
305
                          pd->dst.stride, dst, pd->dst.stride, col, row, plane);
Jingning Han's avatar
Jingning Han committed
306 307

  if (!mbmi->skip) {
hui su's avatar
hui su committed
308
    TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, tx_size);
Angie Chiang's avatar
Angie Chiang committed
309
    const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, 0);
310 311 312 313
    int16_t max_scan_line = 0;
    const int eob =
        av1_decode_block_tokens(xd, plane, scan_order, col, row, tx_size,
                                tx_type, &max_scan_line, r, mbmi->segment_id);
Angie Chiang's avatar
Angie Chiang committed
314 315 316
#if CONFIG_ADAPT_SCAN
    av1_update_scan_count_facade(cm, tx_size, tx_type, pd->dqcoeff, eob);
#endif
317 318 319
    if (eob)
      inverse_transform_block(xd, plane, tx_type, tx_size, dst, pd->dst.stride,
                              max_scan_line, eob);
Jingning Han's avatar
Jingning Han committed
320 321 322
  }
}

323
#if CONFIG_VAR_TX
Angie Chiang's avatar
Angie Chiang committed
324 325
static void decode_reconstruct_tx(AV1_COMMON *cm, MACROBLOCKD *const xd,
                                  aom_reader *r, MB_MODE_INFO *const mbmi,
326
                                  int plane, BLOCK_SIZE plane_bsize,
327 328
                                  int blk_row, int blk_col, TX_SIZE tx_size,
                                  int *eob_total) {
329
  const struct macroblockd_plane *const pd = &xd->plane[plane];
330
  const BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
331 332
  const int tx_row = blk_row >> (1 - pd->subsampling_y);
  const int tx_col = blk_col >> (1 - pd->subsampling_x);
333
  const TX_SIZE plane_tx_size =
334
      plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0]
335
            : mbmi->inter_tx_size[tx_row][tx_col];
336 337 338
  // Scale to match transform block unit.
  int max_blocks_high = block_size_high[plane_bsize];
  int max_blocks_wide = block_size_wide[plane_bsize];
339 340

  if (xd->mb_to_bottom_edge < 0)
341
    max_blocks_high += xd->mb_to_bottom_edge >> (3 + pd->subsampling_y);
342
  if (xd->mb_to_right_edge < 0)
343 344 345 346
    max_blocks_wide += xd->mb_to_right_edge >> (3 + pd->subsampling_x);

  max_blocks_high >>= tx_size_wide_log2[0];
  max_blocks_wide >>= tx_size_wide_log2[0];
347

348
  if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
349

350
  if (tx_size == plane_tx_size) {
351
    PLANE_TYPE plane_type = (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
352 353
    int block_idx = (blk_row << 1) + blk_col;
    TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, plane_tx_size);
Angie Chiang's avatar
Angie Chiang committed
354
    const SCAN_ORDER *sc = get_scan(cm, plane_tx_size, tx_type, 1);
355
    int16_t max_scan_line = 0;
356
    const int eob =
Yaowu Xu's avatar
Yaowu Xu committed
357
        av1_decode_block_tokens(xd, plane, sc, blk_col, blk_row, plane_tx_size,
358
                                tx_type, &max_scan_line, r, mbmi->segment_id);
359 360
    inverse_transform_block(
        xd, plane, tx_type, plane_tx_size,
361
        &pd->dst.buf[4 * blk_row * pd->dst.stride + 4 * blk_col],
362
        pd->dst.stride, max_scan_line, eob);
363 364
    *eob_total += eob;
  } else {
365
    int bsl = block_size_wide[bsize] >> (tx_size_wide_log2[0] + 1);
366 367 368 369 370
    int i;

    assert(bsl > 0);

    for (i = 0; i < 4; ++i) {
371 372
      const int offsetr = blk_row + (i >> 1) * bsl;
      const int offsetc = blk_col + (i & 0x01) * bsl;
373

374
      if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;
375

376 377
      decode_reconstruct_tx(cm, xd, r, mbmi, plane, plane_bsize, offsetr,
                            offsetc, tx_size - 1, eob_total);
378 379 380
    }
  }
}
381
#endif  // CONFIG_VAR_TX
382

383
#if !CONFIG_VAR_TX || CONFIG_SUPERTX || (CONFIG_EXT_TX && CONFIG_RECT_TX)
Angie Chiang's avatar
Angie Chiang committed
384
static int reconstruct_inter_block(AV1_COMMON *cm, MACROBLOCKD *const xd,
Alex Converse's avatar
Alex Converse committed
385 386 387
#if CONFIG_ANS
                                   struct AnsDecoder *const r,
#else
Yaowu Xu's avatar
Yaowu Xu committed
388
                                   aom_reader *r,
Alex Converse's avatar
Alex Converse committed
389
#endif
390 391
                                   int segment_id, int plane, int row, int col,
                                   TX_SIZE tx_size) {
Jingning Han's avatar
Jingning Han committed
392
  struct macroblockd_plane *const pd = &xd->plane[plane];
hui su's avatar
hui su committed
393 394
  PLANE_TYPE plane_type = (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
  int block_idx = (row << 1) + col;
hui su's avatar
hui su committed
395
  TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, tx_size);
Angie Chiang's avatar
Angie Chiang committed
396
  const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, 1);
397 398 399 400
  int16_t max_scan_line = 0;
  const int eob =
      av1_decode_block_tokens(xd, plane, scan_order, col, row, tx_size, tx_type,
                              &max_scan_line, r, segment_id);
Angie Chiang's avatar
Angie Chiang committed
401 402 403
#if CONFIG_ADAPT_SCAN
  av1_update_scan_count_facade(cm, tx_size, tx_type, pd->dqcoeff, eob);
#endif
404 405 406 407
  if (eob)
    inverse_transform_block(xd, plane, tx_type, tx_size,
                            &pd->dst.buf[4 * row * pd->dst.stride + 4 * col],
                            pd->dst.stride, max_scan_line, eob);
Jingning Han's avatar
Jingning Han committed
408 409
  return eob;
}
410
#endif  // !CONFIG_VAR_TX || CONFIG_SUPER_TX
Jingning Han's avatar
Jingning Han committed
411 412 413 414 415 416 417 418 419 420

static INLINE void dec_reset_skip_context(MACROBLOCKD *xd) {
  int i;
  for (i = 0; i < MAX_MB_PLANE; i++) {
    struct macroblockd_plane *const pd = &xd->plane[i];
    memset(pd->above_context, 0, sizeof(ENTROPY_CONTEXT) * pd->n4_w);
    memset(pd->left_context, 0, sizeof(ENTROPY_CONTEXT) * pd->n4_h);
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
421
static MB_MODE_INFO *set_offsets(AV1_COMMON *const cm, MACROBLOCKD *const xd,
Jingning Han's avatar
Jingning Han committed
422
                                 BLOCK_SIZE bsize, int mi_row, int mi_col,
423 424
                                 int bw, int bh, int x_mis, int y_mis, int bwl,
                                 int bhl) {
Jingning Han's avatar
Jingning Han committed
425 426 427 428 429 430 431 432 433 434
  const int offset = mi_row * cm->mi_stride + mi_col;
  int x, y;
  const TileInfo *const tile = &xd->tile;

  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = &cm->mi[offset];
  // TODO(slavarnway): Generate sb_type based on bwl and bhl, instead of
  // passing bsize from decode_partition().
  xd->mi[0]->mbmi.sb_type = bsize;
  for (y = 0; y < y_mis; ++y)
Jingning Han's avatar
Jingning Han committed
435
    for (x = !y; x < x_mis; ++x) xd->mi[y * cm->mi_stride + x] = xd->mi[0];
Jingning Han's avatar
Jingning Han committed
436 437 438 439

  set_plane_n4(xd, bw, bh, bwl, bhl);
  set_skip_context(xd, mi_row, mi_col);

440 441 442 443
#if CONFIG_VAR_TX
  xd->max_tx_size = max_txsize_lookup[bsize];
#endif

Jingning Han's avatar
Jingning Han committed
444 445 446 447
  // Distance of Mb to the various image edges. These are specified to 8th pel
  // as they are always compared to values that are in 1/8th pel units
  set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, cm->mi_rows, cm->mi_cols);

Yaowu Xu's avatar
Yaowu Xu committed
448
  av1_setup_dst_planes(xd->plane, get_frame_new_buffer(cm), mi_row, mi_col);
Jingning Han's avatar
Jingning Han committed
449 450 451
  return &xd->mi[0]->mbmi;
}

452
#if CONFIG_SUPERTX
Yaowu Xu's avatar
Yaowu Xu committed
453
static MB_MODE_INFO *set_offsets_extend(AV1_COMMON *const cm,
454 455
                                        MACROBLOCKD *const xd,
                                        const TileInfo *const tile,
456 457 458
                                        BLOCK_SIZE bsize_pred, int mi_row_pred,
                                        int mi_col_pred, int mi_row_ori,
                                        int mi_col_ori) {
459 460 461 462 463 464 465 466 467 468
  // Used in supertx
  // (mi_row_ori, mi_col_ori): location for mv
  // (mi_row_pred, mi_col_pred, bsize_pred): region to predict
  const int bw = num_8x8_blocks_wide_lookup[bsize_pred];
  const int bh = num_8x8_blocks_high_lookup[bsize_pred];
  const int offset = mi_row_ori * cm->mi_stride + mi_col_ori;
  const int bwl = b_width_log2_lookup[bsize_pred];
  const int bhl = b_height_log2_lookup[bsize_pred];
  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = cm->mi + offset;
469 470
  set_mi_row_col(xd, tile, mi_row_pred, bh, mi_col_pred, bw, cm->mi_rows,
                 cm->mi_cols);
471

472 473
  xd->up_available = (mi_row_ori > tile->mi_row_start);
  xd->left_available = (mi_col_ori > tile->mi_col_start);
474 475 476 477 478 479

  set_plane_n4(xd, bw, bh, bwl, bhl);

  return &xd->mi[0]->mbmi;
}

Yaowu Xu's avatar
Yaowu Xu committed
480 481 482
static MB_MODE_INFO *set_mb_offsets(AV1_COMMON *const cm, MACROBLOCKD *const xd,
                                    BLOCK_SIZE bsize, int mi_row, int mi_col,
                                    int bw, int bh, int x_mis, int y_mis) {
483 484 485 486 487 488 489 490
  const int offset = mi_row * cm->mi_stride + mi_col;
  const TileInfo *const tile = &xd->tile;
  int x, y;

  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = cm->mi + offset;
  xd->mi[0]->mbmi.sb_type = bsize;
  for (y = 0; y < y_mis; ++y)
491
    for (x = !y; x < x_mis; ++x) xd->mi[y * cm->mi_stride + x] = xd->mi[0];
492 493 494 495 496

  set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, cm->mi_rows, cm->mi_cols);
  return &xd->mi[0]->mbmi;
}

Yaowu Xu's avatar
Yaowu Xu committed
497
static void set_offsets_topblock(AV1_COMMON *const cm, MACROBLOCKD *const xd,
498 499
                                 const TileInfo *const tile, BLOCK_SIZE bsize,
                                 int mi_row, int mi_col) {
500 501 502 503 504 505 506 507 508 509 510 511 512
  const int bw = num_8x8_blocks_wide_lookup[bsize];
  const int bh = num_8x8_blocks_high_lookup[bsize];
  const int offset = mi_row * cm->mi_stride + mi_col;
  const int bwl = b_width_log2_lookup[bsize];
  const int bhl = b_height_log2_lookup[bsize];

  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = cm->mi + offset;

  set_plane_n4(xd, bw, bh, bwl, bhl);

  set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, cm->mi_rows, cm->mi_cols);

Yaowu Xu's avatar
Yaowu Xu committed
513
  av1_setup_dst_planes(xd->plane, get_frame_new_buffer(cm), mi_row, mi_col);
514 515
}

Yaowu Xu's avatar
Yaowu Xu committed
516
static void set_param_topblock(AV1_COMMON *const cm, MACROBLOCKD *const xd,
517
                               BLOCK_SIZE bsize, int mi_row, int mi_col,
518
                               int txfm, int skip) {
519 520
  const int bw = num_8x8_blocks_wide_lookup[bsize];
  const int bh = num_8x8_blocks_high_lookup[bsize];
Yaowu Xu's avatar
Yaowu Xu committed
521 522
  const int x_mis = AOMMIN(bw, cm->mi_cols - mi_col);
  const int y_mis = AOMMIN(bh, cm->mi_rows - mi_row);
523 524 525 526 527 528 529 530 531 532 533
  const int offset = mi_row * cm->mi_stride + mi_col;
  int x, y;

  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = cm->mi + offset;

  for (y = 0; y < y_mis; ++y)
    for (x = 0; x < x_mis; ++x) {
      xd->mi[y * cm->mi_stride + x]->mbmi.skip = skip;
      xd->mi[y * cm->mi_stride + x]->mbmi.tx_type = txfm;
    }
534 535
#if CONFIG_VAR_TX
  xd->above_txfm_context = cm->above_txfm_context + mi_col;
536
  xd->left_txfm_context =
537
      xd->left_txfm_context_buffer + (mi_row & MAX_MIB_MASK);
538
  set_txfm_ctxs(xd->mi[0]->mbmi.tx_size, bw, bh, xd);
539
#endif
540 541
}

Yaowu Xu's avatar
Yaowu Xu committed
542
static void set_ref(AV1_COMMON *const cm, MACROBLOCKD *const xd, int idx,
543
                    int mi_row, int mi_col) {
544 545 546
  MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
  RefBuffer *ref_buffer = &cm->frame_refs[mbmi->ref_frame[idx] - LAST_FRAME];
  xd->block_refs[idx] = ref_buffer;
Yaowu Xu's avatar
Yaowu Xu committed
547 548
  if (!av1_is_valid_scale(&ref_buffer->sf))
    aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM,
549
                       "Invalid scale factors");
Yaowu Xu's avatar
Yaowu Xu committed
550 551
  av1_setup_pre_planes(xd, idx, ref_buffer->buf, mi_row, mi_col,
                       &ref_buffer->sf);
552 553 554 555
  xd->corrupted |= ref_buffer->buf->corrupted;
}

static void dec_predict_b_extend(
Yaowu Xu's avatar
Yaowu Xu committed
556
    AV1Decoder *const pbi, MACROBLOCKD *const xd, const TileInfo *const tile,
557 558 559
    int block, int mi_row_ori, int mi_col_ori, int mi_row_pred, int mi_col_pred,
    int mi_row_top, int mi_col_top, uint8_t *dst_buf[3], int dst_stride[3],
    BLOCK_SIZE bsize_top, BLOCK_SIZE bsize_pred, int b_sub8x8, int bextend) {
560 561 562 563 564 565 566 567 568 569 570 571
  // Used in supertx
  // (mi_row_ori, mi_col_ori): location for mv
  // (mi_row_pred, mi_col_pred, bsize_pred): region to predict
  // (mi_row_top, mi_col_top, bsize_top): region of the top partition size
  // block: sub location of sub8x8 blocks
  // b_sub8x8: 1: ori is sub8x8; 0: ori is not sub8x8
  // bextend: 1: region to predict is an extension of ori; 0: not
  int r = (mi_row_pred - mi_row_top) * MI_SIZE;
  int c = (mi_col_pred - mi_col_top) * MI_SIZE;
  const int mi_width_top = num_8x8_blocks_wide_lookup[bsize_top];
  const int mi_height_top = num_8x8_blocks_high_lookup[bsize_top];
  MB_MODE_INFO *mbmi;
Yaowu Xu's avatar
Yaowu Xu committed
572
  AV1_COMMON *const cm = &pbi->common;
573 574 575

  if (mi_row_pred < mi_row_top || mi_col_pred < mi_col_top ||
      mi_row_pred >= mi_row_top + mi_height_top ||
576 577
      mi_col_pred >= mi_col_top + mi_width_top || mi_row_pred >= cm->mi_rows ||
      mi_col_pred >= cm->mi_cols)
578 579
    return;

580
  mbmi = set_offsets_extend(cm, xd, tile, bsize_pred, mi_row_pred, mi_col_pred,
581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603
                            mi_row_ori, mi_col_ori);
  set_ref(cm, xd, 0, mi_row_pred, mi_col_pred);
  if (has_second_ref(&xd->mi[0]->mbmi))
    set_ref(cm, xd, 1, mi_row_pred, mi_col_pred);

  if (!bextend) {
    mbmi->tx_size = b_width_log2_lookup[bsize_top];
  }

  xd->plane[0].dst.stride = dst_stride[0];
  xd->plane[1].dst.stride = dst_stride[1];
  xd->plane[2].dst.stride = dst_stride[2];
  xd->plane[0].dst.buf = dst_buf[0] +
                         (r >> xd->plane[0].subsampling_y) * dst_stride[0] +
                         (c >> xd->plane[0].subsampling_x);
  xd->plane[1].dst.buf = dst_buf[1] +
                         (r >> xd->plane[1].subsampling_y) * dst_stride[1] +
                         (c >> xd->plane[1].subsampling_x);
  xd->plane[2].dst.buf = dst_buf[2] +
                         (r >> xd->plane[2].subsampling_y) * dst_stride[2] +
                         (c >> xd->plane[2].subsampling_x);

  if (!b_sub8x8)
Yaowu Xu's avatar
Yaowu Xu committed
604
    av1_build_inter_predictors_sb_extend(xd,
605
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
606
                                         mi_row_ori, mi_col_ori,
607
#endif  // CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
608
                                         mi_row_pred, mi_col_pred, bsize_pred);
609
  else
Yaowu Xu's avatar
Yaowu Xu committed
610
    av1_build_inter_predictors_sb_sub8x8_extend(xd,
611
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
612
                                                mi_row_ori, mi_col_ori,
613
#endif  // CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
614 615
                                                mi_row_pred, mi_col_pred,
                                                bsize_pred, block);
616 617
}

Yaowu Xu's avatar
Yaowu Xu committed
618
static void dec_extend_dir(AV1Decoder *const pbi, MACROBLOCKD *const xd,
619
                           const TileInfo *const tile, int block,
620 621 622
                           BLOCK_SIZE bsize, BLOCK_SIZE top_bsize, int mi_row,
                           int mi_col, int mi_row_top, int mi_col_top,
                           uint8_t *dst_buf[3], int dst_stride[3], int dir) {
623 624 625 626 627 628 629 630 631 632 633
  // dir: 0-lower, 1-upper, 2-left, 3-right
  //      4-lowerleft, 5-upperleft, 6-lowerright, 7-upperright
  const int mi_width = num_8x8_blocks_wide_lookup[bsize];
  const int mi_height = num_8x8_blocks_high_lookup[bsize];
  int xss = xd->plane[1].subsampling_x;
  int yss = xd->plane[1].subsampling_y;
  int b_sub8x8 = (bsize < BLOCK_8X8) ? 1 : 0;
  BLOCK_SIZE extend_bsize;
  int unit, mi_row_pred, mi_col_pred;

  if (dir == 0 || dir == 1) {
634 635 636
    extend_bsize = (mi_width == 1 || bsize < BLOCK_8X8 || xss < yss)
                       ? BLOCK_8X8
                       : BLOCK_16X8;
637 638 639 640
    unit = num_8x8_blocks_wide_lookup[extend_bsize];
    mi_row_pred = mi_row + ((dir == 0) ? mi_height : -1);
    mi_col_pred = mi_col;

641 642 643
    dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col, mi_row_pred,
                         mi_col_pred, mi_row_top, mi_col_top, dst_buf,
                         dst_stride, top_bsize, extend_bsize, b_sub8x8, 1);
644 645 646 647

    if (mi_width > unit) {
      int i;
      assert(!b_sub8x8);
648
      for (i = 0; i < mi_width / unit - 1; i++) {
649
        mi_col_pred += unit;
650 651 652
        dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col, mi_row_pred,
                             mi_col_pred, mi_row_top, mi_col_top, dst_buf,
                             dst_stride, top_bsize, extend_bsize, b_sub8x8, 1);
653 654 655
      }
    }
  } else if (dir == 2 || dir == 3) {
656 657 658
    extend_bsize = (mi_height == 1 || bsize < BLOCK_8X8 || yss < xss)
                       ? BLOCK_8X8
                       : BLOCK_8X16;
659 660 661 662
    unit = num_8x8_blocks_high_lookup[extend_bsize];
    mi_row_pred = mi_row;
    mi_col_pred = mi_col + ((dir == 3) ? mi_width : -1);

663 664 665
    dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col, mi_row_pred,
                         mi_col_pred, mi_row_top, mi_col_top, dst_buf,
                         dst_stride, top_bsize, extend_bsize, b_sub8x8, 1);
666 667 668

    if (mi_height > unit) {
      int i;
669
      for (i = 0; i < mi_height / unit - 1; i++) {
670
        mi_row_pred += unit;
671 672 673
        dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col, mi_row_pred,
                             mi_col_pred, mi_row_top, mi_col_top, dst_buf,
                             dst_stride, top_bsize, extend_bsize, b_sub8x8, 1);
674 675 676 677 678 679
      }
    }
  } else {
    extend_bsize = BLOCK_8X8;
    mi_row_pred = mi_row + ((dir == 4 || dir == 6) ? mi_height : -1);
    mi_col_pred = mi_col + ((dir == 6 || dir == 7) ? mi_width : -1);
680 681 682
    dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col, mi_row_pred,
                         mi_col_pred, mi_row_top, mi_col_top, dst_buf,
                         dst_stride, top_bsize, extend_bsize, b_sub8x8, 1);
683 684 685
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
686
static void dec_extend_all(AV1Decoder *const pbi, MACROBLOCKD *const xd,
687
                           const TileInfo *const tile, int block,
688 689 690
                           BLOCK_SIZE bsize, BLOCK_SIZE top_bsize, int mi_row,
                           int mi_col, int mi_row_top, int mi_col_top,
                           uint8_t *dst_buf[3], int dst_stride[3]) {
691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 0);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 1);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 2);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 3);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 4);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 5);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 6);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 7);
}

Yaowu Xu's avatar
Yaowu Xu committed
709
static void dec_predict_sb_complex(AV1Decoder *const pbi, MACROBLOCKD *const xd,
710 711
                                   const TileInfo *const tile, int mi_row,
                                   int mi_col, int mi_row_top, int mi_col_top,
712 713
                                   BLOCK_SIZE bsize, BLOCK_SIZE top_bsize,
                                   uint8_t *dst_buf[3], int dst_stride[3]) {
Yaowu Xu's avatar
Yaowu Xu committed
714
  const AV1_COMMON *const cm = &pbi->common;
715 716 717
  const int hbs = num_8x8_blocks_wide_lookup[bsize] / 2;
  const PARTITION_TYPE partition = get_partition(cm, mi_row, mi_col, bsize);
  const BLOCK_SIZE subsize = get_subsize(bsize, partition);
718
#if CONFIG_EXT_PARTITION_TYPES
719
  const BLOCK_SIZE bsize2 = get_subsize(bsize, PARTITION_SPLIT);
720
#endif
721 722
  int i;
  const int mi_offset = mi_row * cm->mi_stride + mi_col;
723 724
  uint8_t *dst_buf1[3], *dst_buf2[3], *dst_buf3[3];

725 726 727 728 729 730
  DECLARE_ALIGNED(16, uint8_t, tmp_buf1[MAX_MB_PLANE * MAX_TX_SQUARE * 2]);
  DECLARE_ALIGNED(16, uint8_t, tmp_buf2[MAX_MB_PLANE * MAX_TX_SQUARE * 2]);
  DECLARE_ALIGNED(16, uint8_t, tmp_buf3[MAX_MB_PLANE * MAX_TX_SQUARE * 2]);
  int dst_stride1[3] = { MAX_TX_SIZE, MAX_TX_SIZE, MAX_TX_SIZE };
  int dst_stride2[3] = { MAX_TX_SIZE, MAX_TX_SIZE, MAX_TX_SIZE };
  int dst_stride3[3] = { MAX_TX_SIZE, MAX_TX_SIZE, MAX_TX_SIZE };
731

Yaowu Xu's avatar
Yaowu Xu committed
732
#if CONFIG_AOM_HIGHBITDEPTH
733 734 735
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
    int len = sizeof(uint16_t);
    dst_buf1[0] = CONVERT_TO_BYTEPTR(tmp_buf1);
736 737
    dst_buf1[1] = CONVERT_TO_BYTEPTR(tmp_buf1 + MAX_TX_SQUARE * len);
    dst_buf1[2] = CONVERT_TO_BYTEPTR(tmp_buf1 + 2 * MAX_TX_SQUARE * len);
738
    dst_buf2[0] = CONVERT_TO_BYTEPTR(tmp_buf2);
739 740
    dst_buf2[1] = CONVERT_TO_BYTEPTR(tmp_buf2 + MAX_TX_SQUARE * len);
    dst_buf2[2] = CONVERT_TO_BYTEPTR(tmp_buf2 + 2 * MAX_TX_SQUARE * len);
741
    dst_buf3[0] = CONVERT_TO_BYTEPTR(tmp_buf3);
742 743
    dst_buf3[1] = CONVERT_TO_BYTEPTR(tmp_buf3 + MAX_TX_SQUARE * len);
    dst_buf3[2] = CONVERT_TO_BYTEPTR(tmp_buf3 + 2 * MAX_TX_SQUARE * len);
744 745 746
  } else {
#endif
    dst_buf1[0] = tmp_buf1;
747 748
    dst_buf1[1] = tmp_buf1 + MAX_TX_SQUARE;
    dst_buf1[2] = tmp_buf1 + 2 * MAX_TX_SQUARE;
749
    dst_buf2[0] = tmp_buf2;
750 751
    dst_buf2[1] = tmp_buf2 + MAX_TX_SQUARE;
    dst_buf2[2] = tmp_buf2 + 2 * MAX_TX_SQUARE;
752
    dst_buf3[0] = tmp_buf3;
753 754
    dst_buf3[1] = tmp_buf3 + MAX_TX_SQUARE;
    dst_buf3[2] = tmp_buf3 + 2 * MAX_TX_SQUARE;
Yaowu Xu's avatar
Yaowu Xu committed
755
#if CONFIG_AOM_HIGHBITDEPTH
756 757 758
  }
#endif

759
  if (mi_row >= cm->mi_rows || mi_col >= cm->mi_cols) return;
760

761 762
  xd->mi = cm->mi_grid_visible + mi_offset;
  xd->mi[0] = cm->mi + mi_offset;
763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799

  for (i = 0; i < MAX_MB_PLANE; i++) {
    xd->plane[i].dst.buf = dst_buf[i];
    xd->plane[i].dst.stride = dst_stride[i];
  }

  switch (partition) {
    case PARTITION_NONE:
      assert(bsize < top_bsize);
      dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col,
                           mi_row_top, mi_col_top, dst_buf, dst_stride,
                           top_bsize, bsize, 0, 0);
      dec_extend_all(pbi, xd, tile, 0, bsize, top_bsize, mi_row, mi_col,
                     mi_row_top, mi_col_top, dst_buf, dst_stride);
      break;
    case PARTITION_HORZ:
      if (bsize == BLOCK_8X8) {
        // For sub8x8, predict in 8x8 unit
        // First half
        dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf, dst_stride,
                             top_bsize, BLOCK_8X8, 1, 0);
        if (bsize < top_bsize)
          dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf, dst_stride);

        // Second half
        dec_predict_b_extend(pbi, xd, tile, 2, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf1, dst_stride1,
                             top_bsize, BLOCK_8X8, 1, 1);
        if (bsize < top_bsize)
          dec_extend_all(pbi, xd, tile, 2, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf1, dst_stride1);

        // weighted average to smooth the boundary
        xd->plane[0].dst.buf = dst_buf[0];
        xd->plane[0].dst.stride = dst_stride[0];
Yaowu Xu's avatar
Yaowu Xu committed
800
        av1_build_masked_inter_predictor_complex(
801 802 803
            xd, dst_buf[0], dst_stride[0], dst_buf1[0], dst_stride1[0], mi_row,
            mi_col, mi_row_top, mi_col_top, bsize, top_bsize, PARTITION_HORZ,
            0);
804 805 806 807 808 809 810 811 812 813 814 815 816 817 818
      } else {
        // First half
        dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf, dst_stride,
                             top_bsize, subsize, 0, 0);
        if (bsize < top_bsize)
          dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf, dst_stride);
        else
          dec_extend_dir(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf, dst_stride, 0);

        if (mi_row + hbs < cm->mi_rows) {
          // Second half
          dec_predict_b_extend(pbi, xd, tile, 0, mi_row + hbs, mi_col,
819 820
                               mi_row + hbs, mi_col, mi_row_top, mi_col_top,
                               dst_buf1, dst_stride1, top_bsize, subsize, 0, 0);
821
          if (bsize < top_bsize)
822 823 824
            dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row + hbs,
                           mi_col, mi_row_top, mi_col_top, dst_buf1,
                           dst_stride1);
825
          else
826 827 828
            dec_extend_dir(pbi, xd, tile, 0, subsize, top_bsize, mi_row + hbs,
                           mi_col, mi_row_top, mi_col_top, dst_buf1,
                           dst_stride1, 1);
829 830 831 832 833

          // weighted average to smooth the boundary
          for (i = 0; i < MAX_MB_PLANE; i++) {
            xd->plane[i].dst.buf = dst_buf[i];
            xd->plane[i].dst.stride = dst_stride[i];
Yaowu Xu's avatar
Yaowu Xu committed
834
            av1_build_masked_inter_predictor_complex(
835
                xd, dst_buf[i], dst_stride[i], dst_buf1[i], dst_stride1[i],
836 837
                mi_row, mi_col, mi_row_top, mi_col_top, bsize, top_bsize,
                PARTITION_HORZ, i);
838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862
          }
        }
      }
      break;
    case PARTITION_VERT:
      if (bsize == BLOCK_8X8) {
        // First half
        dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf, dst_stride,
                             top_bsize, BLOCK_8X8, 1, 0);
        if (bsize < top_bsize)
          dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf, dst_stride);

        // Second half
        dec_predict_b_extend(pbi, xd, tile, 1, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf1, dst_stride1,
                             top_bsize, BLOCK_8X8, 1, 1);
        if (bsize < top_bsize)
          dec_extend_all(pbi, xd, tile, 1, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf1, dst_stride1);

        // Smooth
        xd->plane[0].dst.buf = dst_buf[0];
        xd->plane[0].dst.stride = dst_stride[0];
Yaowu Xu's avatar
Yaowu Xu committed
863
        av1_build_masked_inter_predictor_complex(
864 865 866
            xd, dst_buf[0], dst_stride[0], dst_buf1[0], dst_stride1[0], mi_row,
            mi_col, mi_row_top, mi_col_top, bsize, top_bsize, PARTITION_VERT,
            0);
867 868 869 870 871 872 873 874 875 876 877 878 879 880
      } else {
        // First half
        dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf, dst_stride,
                             top_bsize, subsize, 0, 0);
        if (bsize < top_bsize)
          dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf, dst_stride);
        else
          dec_extend_dir(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf, dst_stride, 3);

        // Second half
        if (mi_col + hbs < cm->mi_cols) {
881 882 883
          dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col + hbs, mi_row,
                               mi_col + hbs, mi_row_top, mi_col_top, dst_buf1,
                               dst_stride1, top_bsize, subsize, 0, 0);
884
          if (bsize < top_bsize)
885 886 887
            dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row,
                           mi_col + hbs, mi_row_top, mi_col_top, dst_buf1,
                           dst_stride1);