decodeframe.c 160 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4 5 6 7 8 9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10 11 12 13 14
 */

#include <assert.h>
#include <stdlib.h>  // qsort()

Yaowu Xu's avatar
Yaowu Xu committed
15 16 17
#include "./aom_config.h"
#include "./aom_dsp_rtcd.h"
#include "./aom_scale_rtcd.h"
Jingning Han's avatar
Jingning Han committed
18
#include "./av1_rtcd.h"
Jingning Han's avatar
Jingning Han committed
19

20
#include "aom/aom_codec.h"
Yaowu Xu's avatar
Yaowu Xu committed
21
#include "aom_dsp/aom_dsp_common.h"
Jingning Han's avatar
Jingning Han committed
22 23
#include "aom_dsp/bitreader.h"
#include "aom_dsp/bitreader_buffer.h"
Yaowu Xu's avatar
Yaowu Xu committed
24
#include "aom_mem/aom_mem.h"
25 26
#include "aom_ports/mem.h"
#include "aom_ports/mem_ops.h"
Yaowu Xu's avatar
Yaowu Xu committed
27 28
#include "aom_scale/aom_scale.h"
#include "aom_util/aom_thread.h"
Jingning Han's avatar
Jingning Han committed
29

30
#include "av1/common/alloccommon.h"
31
#if CONFIG_CLPF
Steinar Midtskogen's avatar
Steinar Midtskogen committed
32
#include "aom/aom_image.h"
33
#include "av1/common/clpf.h"
34
#endif
35
#include "av1/common/common.h"
Yaowu Xu's avatar
Yaowu Xu committed
36
#if CONFIG_DERING
37
#include "av1/common/dering.h"
Yaowu Xu's avatar
Yaowu Xu committed
38
#endif  // CONFIG_DERING
39 40
#include "av1/common/entropy.h"
#include "av1/common/entropymode.h"
41
#include "av1/common/entropymv.h"
42 43 44 45
#include "av1/common/idct.h"
#include "av1/common/pred_common.h"
#include "av1/common/quant_common.h"
#include "av1/common/reconinter.h"
Jingning Han's avatar
Jingning Han committed
46
#include "av1/common/reconintra.h"
47
#include "av1/common/seg_common.h"
Jingning Han's avatar
Jingning Han committed
48
#include "av1/common/thread_common.h"
49
#include "av1/common/tile_common.h"
50

51 52 53
#include "av1/decoder/decodeframe.h"
#include "av1/decoder/decodemv.h"
#include "av1/decoder/decoder.h"
Jingning Han's avatar
Jingning Han committed
54
#include "av1/decoder/detokenize.h"
55
#include "av1/decoder/dsubexp.h"
Jingning Han's avatar
Jingning Han committed
56

Yaowu Xu's avatar
Yaowu Xu committed
57
#define MAX_AV1_HEADER_SIZE 80
Michael Bebenita's avatar
Michael Bebenita committed
58
#define ACCT_STR __func__
Jingning Han's avatar
Jingning Han committed
59

60 61 62 63 64 65 66 67
static struct aom_read_bit_buffer *init_read_bit_buffer(
    AV1Decoder *pbi, struct aom_read_bit_buffer *rb, const uint8_t *data,
    const uint8_t *data_end, uint8_t clear_data[MAX_AV1_HEADER_SIZE]);
static int read_compressed_header(AV1Decoder *pbi, const uint8_t *data,
                                  size_t partition_size);
static size_t read_uncompressed_header(AV1Decoder *pbi,
                                       struct aom_read_bit_buffer *rb);

Yaowu Xu's avatar
Yaowu Xu committed
68
static int is_compound_reference_allowed(const AV1_COMMON *cm) {
Jingning Han's avatar
Jingning Han committed
69
  int i;
70
  if (frame_is_intra_only(cm)) return 0;
71
  for (i = 1; i < INTER_REFS_PER_FRAME; ++i)
72
    if (cm->ref_frame_sign_bias[i + 1] != cm->ref_frame_sign_bias[1]) return 1;
Jingning Han's avatar
Jingning Han committed
73 74 75 76

  return 0;
}

Yaowu Xu's avatar
Yaowu Xu committed
77
static void setup_compound_reference_mode(AV1_COMMON *cm) {
78
#if CONFIG_EXT_REFS
79
  cm->comp_fwd_ref[0] = LAST_FRAME;
80 81 82 83
  cm->comp_fwd_ref[1] = LAST2_FRAME;
  cm->comp_fwd_ref[2] = LAST3_FRAME;
  cm->comp_fwd_ref[3] = GOLDEN_FRAME;

84 85
  cm->comp_bwd_ref[0] = BWDREF_FRAME;
  cm->comp_bwd_ref[1] = ALTREF_FRAME;
86
#else
Jingning Han's avatar
Jingning Han committed
87
  if (cm->ref_frame_sign_bias[LAST_FRAME] ==
88
      cm->ref_frame_sign_bias[GOLDEN_FRAME]) {
Jingning Han's avatar
Jingning Han committed
89 90 91 92
    cm->comp_fixed_ref = ALTREF_FRAME;
    cm->comp_var_ref[0] = LAST_FRAME;
    cm->comp_var_ref[1] = GOLDEN_FRAME;
  } else if (cm->ref_frame_sign_bias[LAST_FRAME] ==
93
             cm->ref_frame_sign_bias[ALTREF_FRAME]) {
Jingning Han's avatar
Jingning Han committed
94 95 96 97 98 99 100 101
    cm->comp_fixed_ref = GOLDEN_FRAME;
    cm->comp_var_ref[0] = LAST_FRAME;
    cm->comp_var_ref[1] = ALTREF_FRAME;
  } else {
    cm->comp_fixed_ref = LAST_FRAME;
    cm->comp_var_ref[0] = GOLDEN_FRAME;
    cm->comp_var_ref[1] = ALTREF_FRAME;
  }
102
#endif  // CONFIG_EXT_REFS
Jingning Han's avatar
Jingning Han committed
103 104 105 106 107 108
}

static int read_is_valid(const uint8_t *start, size_t len, const uint8_t *end) {
  return len != 0 && len <= (size_t)(end - start);
}

Yaowu Xu's avatar
Yaowu Xu committed
109 110
static int decode_unsigned_max(struct aom_read_bit_buffer *rb, int max) {
  const int data = aom_rb_read_literal(rb, get_unsigned_bits(max));
Jingning Han's avatar
Jingning Han committed
111 112 113
  return data > max ? max : data;
}

Yaowu Xu's avatar
Yaowu Xu committed
114 115
static TX_MODE read_tx_mode(struct aom_read_bit_buffer *rb) {
  return aom_rb_read_bit(rb) ? TX_MODE_SELECT : aom_rb_read_literal(rb, 2);
116
}
Jingning Han's avatar
Jingning Han committed
117

118
#if !CONFIG_EC_ADAPT || !CONFIG_DAALA_EC
Yaowu Xu's avatar
Yaowu Xu committed
119
static void read_switchable_interp_probs(FRAME_CONTEXT *fc, aom_reader *r) {
Jingning Han's avatar
Jingning Han committed
120
  int i, j;
121
  for (j = 0; j < SWITCHABLE_FILTER_CONTEXTS; ++j) {
Jingning Han's avatar
Jingning Han committed
122
    for (i = 0; i < SWITCHABLE_FILTERS - 1; ++i)
Michael Bebenita's avatar
Michael Bebenita committed
123
      av1_diff_update_prob(r, &fc->switchable_interp_prob[j][i], ACCT_STR);
124
  }
Jingning Han's avatar
Jingning Han committed
125
}
126
#endif
Jingning Han's avatar
Jingning Han committed
127

Yaowu Xu's avatar
Yaowu Xu committed
128
static void read_inter_mode_probs(FRAME_CONTEXT *fc, aom_reader *r) {
129 130 131
  int i;
#if CONFIG_REF_MV
  for (i = 0; i < NEWMV_MODE_CONTEXTS; ++i)
Michael Bebenita's avatar
Michael Bebenita committed
132
    av1_diff_update_prob(r, &fc->newmv_prob[i], ACCT_STR);
133
  for (i = 0; i < ZEROMV_MODE_CONTEXTS; ++i)
Michael Bebenita's avatar
Michael Bebenita committed
134
    av1_diff_update_prob(r, &fc->zeromv_prob[i], ACCT_STR);
135
  for (i = 0; i < REFMV_MODE_CONTEXTS; ++i)
Michael Bebenita's avatar
Michael Bebenita committed
136
    av1_diff_update_prob(r, &fc->refmv_prob[i], ACCT_STR);
137
  for (i = 0; i < DRL_MODE_CONTEXTS; ++i)
Michael Bebenita's avatar
Michael Bebenita committed
138
    av1_diff_update_prob(r, &fc->drl_prob[i], ACCT_STR);
Yue Chen's avatar
Yue Chen committed
139
#if CONFIG_EXT_INTER
Michael Bebenita's avatar
Michael Bebenita committed
140
  av1_diff_update_prob(r, &fc->new2mv_prob, ACCT_STR);
Yue Chen's avatar
Yue Chen committed
141
#endif  // CONFIG_EXT_INTER
142 143
#else
  int j;
144
#if !CONFIG_EC_ADAPT || !CONFIG_DAALA_EC
145
  for (i = 0; i < INTER_MODE_CONTEXTS; ++i) {
Jingning Han's avatar
Jingning Han committed
146
    for (j = 0; j < INTER_MODES - 1; ++j)
Michael Bebenita's avatar
Michael Bebenita committed
147
      av1_diff_update_prob(r, &fc->inter_mode_probs[i][j], ACCT_STR);
148
  }
149
#endif
150
#endif
Jingning Han's avatar
Jingning Han committed
151 152
}

153
#if !CONFIG_EC_ADAPT || !CONFIG_DAALA_EC
154
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
155
static void read_inter_compound_mode_probs(FRAME_CONTEXT *fc, aom_reader *r) {
156
  int i, j;
Michael Bebenita's avatar
Michael Bebenita committed
157
  if (aom_read(r, GROUP_DIFF_UPDATE_PROB, ACCT_STR)) {
158 159
    for (j = 0; j < INTER_MODE_CONTEXTS; ++j) {
      for (i = 0; i < INTER_COMPOUND_MODES - 1; ++i) {
Michael Bebenita's avatar
Michael Bebenita committed
160
        av1_diff_update_prob(r, &fc->inter_compound_mode_probs[j][i], ACCT_STR);
161 162 163 164 165
      }
    }
  }
}
#endif  // CONFIG_EXT_INTER
166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185
#if !CONFIG_EXT_TX
static void read_ext_tx_probs(FRAME_CONTEXT *fc, aom_reader *r) {
  int i, j, k;
  if (aom_read(r, GROUP_DIFF_UPDATE_PROB, ACCT_STR)) {
    for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
      for (j = 0; j < TX_TYPES; ++j) {
        for (k = 0; k < TX_TYPES - 1; ++k)
          av1_diff_update_prob(r, &fc->intra_ext_tx_prob[i][j][k], ACCT_STR);
      }
    }
  }
  if (aom_read(r, GROUP_DIFF_UPDATE_PROB, ACCT_STR)) {
    for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
      for (k = 0; k < TX_TYPES - 1; ++k)
        av1_diff_update_prob(r, &fc->inter_ext_tx_prob[i][k], ACCT_STR);
    }
  }
}
#endif
#endif
186

187
static REFERENCE_MODE read_frame_reference_mode(
Yaowu Xu's avatar
Yaowu Xu committed
188
    const AV1_COMMON *cm, struct aom_read_bit_buffer *rb) {
189
  if (is_compound_reference_allowed(cm)) {
Yaowu Xu's avatar
Yaowu Xu committed
190
    return aom_rb_read_bit(rb)
191
               ? REFERENCE_MODE_SELECT
Yaowu Xu's avatar
Yaowu Xu committed
192
               : (aom_rb_read_bit(rb) ? COMPOUND_REFERENCE : SINGLE_REFERENCE);
193 194 195 196
  } else {
    return SINGLE_REFERENCE;
  }
}
Jingning Han's avatar
Jingning Han committed
197

Yaowu Xu's avatar
Yaowu Xu committed
198
static void read_frame_reference_mode_probs(AV1_COMMON *cm, aom_reader *r) {
Jingning Han's avatar
Jingning Han committed
199
  FRAME_CONTEXT *const fc = cm->fc;
200
  int i, j;
Jingning Han's avatar
Jingning Han committed
201 202 203

  if (cm->reference_mode == REFERENCE_MODE_SELECT)
    for (i = 0; i < COMP_INTER_CONTEXTS; ++i)
Michael Bebenita's avatar
Michael Bebenita committed
204
      av1_diff_update_prob(r, &fc->comp_inter_prob[i], ACCT_STR);
Jingning Han's avatar
Jingning Han committed
205

206
  if (cm->reference_mode != COMPOUND_REFERENCE) {
Jingning Han's avatar
Jingning Han committed
207
    for (i = 0; i < REF_CONTEXTS; ++i) {
208
      for (j = 0; j < (SINGLE_REFS - 1); ++j) {
Michael Bebenita's avatar
Michael Bebenita committed
209
        av1_diff_update_prob(r, &fc->single_ref_prob[i][j], ACCT_STR);
210
      }
Jingning Han's avatar
Jingning Han committed
211
    }
212
  }
Jingning Han's avatar
Jingning Han committed
213

214 215
  if (cm->reference_mode != SINGLE_REFERENCE) {
    for (i = 0; i < REF_CONTEXTS; ++i) {
216
#if CONFIG_EXT_REFS
217
      for (j = 0; j < (FWD_REFS - 1); ++j)
Michael Bebenita's avatar
Michael Bebenita committed
218
        av1_diff_update_prob(r, &fc->comp_ref_prob[i][j], ACCT_STR);
219
      for (j = 0; j < (BWD_REFS - 1); ++j)
Michael Bebenita's avatar
Michael Bebenita committed
220
        av1_diff_update_prob(r, &fc->comp_bwdref_prob[i][j], ACCT_STR);
221 222
#else
      for (j = 0; j < (COMP_REFS - 1); ++j)
Michael Bebenita's avatar
Michael Bebenita committed
223
        av1_diff_update_prob(r, &fc->comp_ref_prob[i][j], ACCT_STR);
224
#endif  // CONFIG_EXT_REFS
225 226
    }
  }
Jingning Han's avatar
Jingning Han committed
227 228
}

Yaowu Xu's avatar
Yaowu Xu committed
229
static void update_mv_probs(aom_prob *p, int n, aom_reader *r) {
Jingning Han's avatar
Jingning Han committed
230
  int i;
Michael Bebenita's avatar
Michael Bebenita committed
231
  for (i = 0; i < n; ++i) av1_diff_update_prob(r, &p[i], ACCT_STR);
Jingning Han's avatar
Jingning Han committed
232 233
}

Yaowu Xu's avatar
Yaowu Xu committed
234
static void read_mv_probs(nmv_context *ctx, int allow_hp, aom_reader *r) {
235
  int i;
Jingning Han's avatar
Jingning Han committed
236

237
#if !CONFIG_EC_ADAPT || !CONFIG_EC_MULTISYMBOL
238
  int j;
Jingning Han's avatar
Jingning Han committed
239 240 241 242 243 244 245 246 247 248 249
  update_mv_probs(ctx->joints, MV_JOINTS - 1, r);

  for (i = 0; i < 2; ++i) {
    nmv_component *const comp_ctx = &ctx->comps[i];
    update_mv_probs(&comp_ctx->sign, 1, r);
    update_mv_probs(comp_ctx->classes, MV_CLASSES - 1, r);
    update_mv_probs(comp_ctx->class0, CLASS0_SIZE - 1, r);
    update_mv_probs(comp_ctx->bits, MV_OFFSET_BITS, r);
  }
  for (i = 0; i < 2; ++i) {
    nmv_component *const comp_ctx = &ctx->comps[i];
250
    for (j = 0; j < CLASS0_SIZE; ++j) {
Jingning Han's avatar
Jingning Han committed
251
      update_mv_probs(comp_ctx->class0_fp[j], MV_FP_SIZE - 1, r);
252
    }
253
    update_mv_probs(comp_ctx->fp, MV_FP_SIZE - 1, r);
Jingning Han's avatar
Jingning Han committed
254
  }
255
#endif  // !CONFIG_EC_ADAPT
Jingning Han's avatar
Jingning Han committed
256 257 258 259 260 261 262 263 264 265

  if (allow_hp) {
    for (i = 0; i < 2; ++i) {
      nmv_component *const comp_ctx = &ctx->comps[i];
      update_mv_probs(&comp_ctx->class0_hp, 1, r);
      update_mv_probs(&comp_ctx->hp, 1, r);
    }
  }
}

266
static void inverse_transform_block(MACROBLOCKD *xd, int plane,
267
                                    const TX_TYPE tx_type,
268
                                    const TX_SIZE tx_size, uint8_t *dst,
269
                                    int stride, int16_t scan_line, int eob) {
Jingning Han's avatar
Jingning Han committed
270
  struct macroblockd_plane *const pd = &xd->plane[plane];
271 272 273 274 275 276
  tran_low_t *const dqcoeff = pd->dqcoeff;
  INV_TXFM_PARAM inv_txfm_param;
  inv_txfm_param.tx_type = tx_type;
  inv_txfm_param.tx_size = tx_size;
  inv_txfm_param.eob = eob;
  inv_txfm_param.lossless = xd->lossless[xd->mi[0]->mbmi.segment_id];
Angie Chiang's avatar
Angie Chiang committed
277

Yaowu Xu's avatar
Yaowu Xu committed
278
#if CONFIG_AOM_HIGHBITDEPTH
279 280 281 282
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
    inv_txfm_param.bd = xd->bd;
    highbd_inv_txfm_add(dqcoeff, dst, stride, &inv_txfm_param);
  } else {
Yaowu Xu's avatar
Yaowu Xu committed
283
#endif  // CONFIG_AOM_HIGHBITDEPTH
284
    inv_txfm_add(dqcoeff, dst, stride, &inv_txfm_param);
Yaowu Xu's avatar
Yaowu Xu committed
285
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
286
  }
287 288
#endif  // CONFIG_AOM_HIGHBITDEPTH
  memset(dqcoeff, 0, (scan_line + 1) * sizeof(dqcoeff[0]));
Jingning Han's avatar
Jingning Han committed
289 290
}

Angie Chiang's avatar
Angie Chiang committed
291 292
static void predict_and_reconstruct_intra_block(AV1_COMMON *cm,
                                                MACROBLOCKD *const xd,
Alex Converse's avatar
Alex Converse committed
293 294 295
#if CONFIG_ANS
                                                struct AnsDecoder *const r,
#else
Yaowu Xu's avatar
Yaowu Xu committed
296
                                                aom_reader *r,
Alex Converse's avatar
Alex Converse committed
297
#endif  // CONFIG_ANS
Jingning Han's avatar
Jingning Han committed
298
                                                MB_MODE_INFO *const mbmi,
299
                                                int plane, int row, int col,
Jingning Han's avatar
Jingning Han committed
300 301 302
                                                TX_SIZE tx_size) {
  struct macroblockd_plane *const pd = &xd->plane[plane];
  PREDICTION_MODE mode = (plane == 0) ? mbmi->mode : mbmi->uv_mode;
hui su's avatar
hui su committed
303
  PLANE_TYPE plane_type = (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
Jingning Han's avatar
Jingning Han committed
304
  uint8_t *dst;
hui su's avatar
hui su committed
305
  int block_idx = (row << 1) + col;
Jingning Han's avatar
Jingning Han committed
306 307 308
  dst = &pd->dst.buf[4 * row * pd->dst.stride + 4 * col];

  if (mbmi->sb_type < BLOCK_8X8)
309
    if (plane == 0) mode = xd->mi[0]->bmi[(row << 1) + col].as_mode;
Jingning Han's avatar
Jingning Han committed
310

311
  av1_predict_intra_block(xd, pd->width, pd->height, tx_size, mode, dst,
Yaowu Xu's avatar
Yaowu Xu committed
312
                          pd->dst.stride, dst, pd->dst.stride, col, row, plane);
Jingning Han's avatar
Jingning Han committed
313 314

  if (!mbmi->skip) {
hui su's avatar
hui su committed
315
    TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, tx_size);
Angie Chiang's avatar
Angie Chiang committed
316
    const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, 0);
317 318 319 320
    int16_t max_scan_line = 0;
    const int eob =
        av1_decode_block_tokens(xd, plane, scan_order, col, row, tx_size,
                                tx_type, &max_scan_line, r, mbmi->segment_id);
Angie Chiang's avatar
Angie Chiang committed
321 322 323
#if CONFIG_ADAPT_SCAN
    av1_update_scan_count_facade(cm, tx_size, tx_type, pd->dqcoeff, eob);
#endif
324 325 326
    if (eob)
      inverse_transform_block(xd, plane, tx_type, tx_size, dst, pd->dst.stride,
                              max_scan_line, eob);
Jingning Han's avatar
Jingning Han committed
327 328 329
  }
}

330
#if CONFIG_VAR_TX
Angie Chiang's avatar
Angie Chiang committed
331 332
static void decode_reconstruct_tx(AV1_COMMON *cm, MACROBLOCKD *const xd,
                                  aom_reader *r, MB_MODE_INFO *const mbmi,
333
                                  int plane, BLOCK_SIZE plane_bsize,
334 335
                                  int blk_row, int blk_col, TX_SIZE tx_size,
                                  int *eob_total) {
336
  const struct macroblockd_plane *const pd = &xd->plane[plane];
337
  const BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
338 339
  const int tx_row = blk_row >> (1 - pd->subsampling_y);
  const int tx_col = blk_col >> (1 - pd->subsampling_x);
340
  const TX_SIZE plane_tx_size =
341
      plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0]
342
            : mbmi->inter_tx_size[tx_row][tx_col];
343 344 345
  // Scale to match transform block unit.
  int max_blocks_high = block_size_high[plane_bsize];
  int max_blocks_wide = block_size_wide[plane_bsize];
346 347

  if (xd->mb_to_bottom_edge < 0)
348
    max_blocks_high += xd->mb_to_bottom_edge >> (3 + pd->subsampling_y);
349
  if (xd->mb_to_right_edge < 0)
350 351 352 353
    max_blocks_wide += xd->mb_to_right_edge >> (3 + pd->subsampling_x);

  max_blocks_high >>= tx_size_wide_log2[0];
  max_blocks_wide >>= tx_size_wide_log2[0];
354

355
  if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
356

357
  if (tx_size == plane_tx_size) {
358
    PLANE_TYPE plane_type = (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
359 360
    int block_idx = (blk_row << 1) + blk_col;
    TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, plane_tx_size);
Angie Chiang's avatar
Angie Chiang committed
361
    const SCAN_ORDER *sc = get_scan(cm, plane_tx_size, tx_type, 1);
362
    int16_t max_scan_line = 0;
363
    const int eob =
Yaowu Xu's avatar
Yaowu Xu committed
364
        av1_decode_block_tokens(xd, plane, sc, blk_col, blk_row, plane_tx_size,
365
                                tx_type, &max_scan_line, r, mbmi->segment_id);
366 367
    inverse_transform_block(
        xd, plane, tx_type, plane_tx_size,
368
        &pd->dst.buf[4 * blk_row * pd->dst.stride + 4 * blk_col],
369
        pd->dst.stride, max_scan_line, eob);
370 371
    *eob_total += eob;
  } else {
372
    int bsl = block_size_wide[bsize] >> (tx_size_wide_log2[0] + 1);
373 374 375 376 377
    int i;

    assert(bsl > 0);

    for (i = 0; i < 4; ++i) {
378 379
      const int offsetr = blk_row + (i >> 1) * bsl;
      const int offsetc = blk_col + (i & 0x01) * bsl;
380

381
      if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;
382

383 384
      decode_reconstruct_tx(cm, xd, r, mbmi, plane, plane_bsize, offsetr,
                            offsetc, tx_size - 1, eob_total);
385 386 387
    }
  }
}
388
#endif  // CONFIG_VAR_TX
389

390
#if !CONFIG_VAR_TX || CONFIG_SUPERTX || (CONFIG_EXT_TX && CONFIG_RECT_TX)
Angie Chiang's avatar
Angie Chiang committed
391
static int reconstruct_inter_block(AV1_COMMON *cm, MACROBLOCKD *const xd,
Alex Converse's avatar
Alex Converse committed
392 393 394
#if CONFIG_ANS
                                   struct AnsDecoder *const r,
#else
Yaowu Xu's avatar
Yaowu Xu committed
395
                                   aom_reader *r,
Alex Converse's avatar
Alex Converse committed
396
#endif
397 398
                                   int segment_id, int plane, int row, int col,
                                   TX_SIZE tx_size) {
Jingning Han's avatar
Jingning Han committed
399
  struct macroblockd_plane *const pd = &xd->plane[plane];
hui su's avatar
hui su committed
400 401
  PLANE_TYPE plane_type = (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
  int block_idx = (row << 1) + col;
hui su's avatar
hui su committed
402
  TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, tx_size);
Angie Chiang's avatar
Angie Chiang committed
403
  const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, 1);
404 405 406 407
  int16_t max_scan_line = 0;
  const int eob =
      av1_decode_block_tokens(xd, plane, scan_order, col, row, tx_size, tx_type,
                              &max_scan_line, r, segment_id);
Angie Chiang's avatar
Angie Chiang committed
408 409 410
#if CONFIG_ADAPT_SCAN
  av1_update_scan_count_facade(cm, tx_size, tx_type, pd->dqcoeff, eob);
#endif
411 412 413 414
  if (eob)
    inverse_transform_block(xd, plane, tx_type, tx_size,
                            &pd->dst.buf[4 * row * pd->dst.stride + 4 * col],
                            pd->dst.stride, max_scan_line, eob);
Jingning Han's avatar
Jingning Han committed
415 416
  return eob;
}
417
#endif  // !CONFIG_VAR_TX || CONFIG_SUPER_TX
Jingning Han's avatar
Jingning Han committed
418 419 420 421 422 423 424 425 426 427

static INLINE void dec_reset_skip_context(MACROBLOCKD *xd) {
  int i;
  for (i = 0; i < MAX_MB_PLANE; i++) {
    struct macroblockd_plane *const pd = &xd->plane[i];
    memset(pd->above_context, 0, sizeof(ENTROPY_CONTEXT) * pd->n4_w);
    memset(pd->left_context, 0, sizeof(ENTROPY_CONTEXT) * pd->n4_h);
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
428
static MB_MODE_INFO *set_offsets(AV1_COMMON *const cm, MACROBLOCKD *const xd,
Jingning Han's avatar
Jingning Han committed
429
                                 BLOCK_SIZE bsize, int mi_row, int mi_col,
430 431
                                 int bw, int bh, int x_mis, int y_mis, int bwl,
                                 int bhl) {
Jingning Han's avatar
Jingning Han committed
432 433 434 435 436 437 438 439 440 441
  const int offset = mi_row * cm->mi_stride + mi_col;
  int x, y;
  const TileInfo *const tile = &xd->tile;

  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = &cm->mi[offset];
  // TODO(slavarnway): Generate sb_type based on bwl and bhl, instead of
  // passing bsize from decode_partition().
  xd->mi[0]->mbmi.sb_type = bsize;
  for (y = 0; y < y_mis; ++y)
Jingning Han's avatar
Jingning Han committed
442
    for (x = !y; x < x_mis; ++x) xd->mi[y * cm->mi_stride + x] = xd->mi[0];
Jingning Han's avatar
Jingning Han committed
443 444 445 446

  set_plane_n4(xd, bw, bh, bwl, bhl);
  set_skip_context(xd, mi_row, mi_col);

447 448 449 450
#if CONFIG_VAR_TX
  xd->max_tx_size = max_txsize_lookup[bsize];
#endif

Jingning Han's avatar
Jingning Han committed
451 452 453 454
  // Distance of Mb to the various image edges. These are specified to 8th pel
  // as they are always compared to values that are in 1/8th pel units
  set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, cm->mi_rows, cm->mi_cols);

Yaowu Xu's avatar
Yaowu Xu committed
455
  av1_setup_dst_planes(xd->plane, get_frame_new_buffer(cm), mi_row, mi_col);
Jingning Han's avatar
Jingning Han committed
456 457 458
  return &xd->mi[0]->mbmi;
}

459
#if CONFIG_SUPERTX
Yaowu Xu's avatar
Yaowu Xu committed
460
static MB_MODE_INFO *set_offsets_extend(AV1_COMMON *const cm,
461 462
                                        MACROBLOCKD *const xd,
                                        const TileInfo *const tile,
463 464 465
                                        BLOCK_SIZE bsize_pred, int mi_row_pred,
                                        int mi_col_pred, int mi_row_ori,
                                        int mi_col_ori) {
466 467 468 469 470 471 472 473 474 475
  // Used in supertx
  // (mi_row_ori, mi_col_ori): location for mv
  // (mi_row_pred, mi_col_pred, bsize_pred): region to predict
  const int bw = num_8x8_blocks_wide_lookup[bsize_pred];
  const int bh = num_8x8_blocks_high_lookup[bsize_pred];
  const int offset = mi_row_ori * cm->mi_stride + mi_col_ori;
  const int bwl = b_width_log2_lookup[bsize_pred];
  const int bhl = b_height_log2_lookup[bsize_pred];
  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = cm->mi + offset;
476 477
  set_mi_row_col(xd, tile, mi_row_pred, bh, mi_col_pred, bw, cm->mi_rows,
                 cm->mi_cols);
478

479 480
  xd->up_available = (mi_row_ori > tile->mi_row_start);
  xd->left_available = (mi_col_ori > tile->mi_col_start);
481 482 483 484 485 486

  set_plane_n4(xd, bw, bh, bwl, bhl);

  return &xd->mi[0]->mbmi;
}

Yaowu Xu's avatar
Yaowu Xu committed
487 488 489
static MB_MODE_INFO *set_mb_offsets(AV1_COMMON *const cm, MACROBLOCKD *const xd,
                                    BLOCK_SIZE bsize, int mi_row, int mi_col,
                                    int bw, int bh, int x_mis, int y_mis) {
490 491 492 493 494 495 496 497
  const int offset = mi_row * cm->mi_stride + mi_col;
  const TileInfo *const tile = &xd->tile;
  int x, y;

  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = cm->mi + offset;
  xd->mi[0]->mbmi.sb_type = bsize;
  for (y = 0; y < y_mis; ++y)
498
    for (x = !y; x < x_mis; ++x) xd->mi[y * cm->mi_stride + x] = xd->mi[0];
499 500 501 502 503

  set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, cm->mi_rows, cm->mi_cols);
  return &xd->mi[0]->mbmi;
}

Yaowu Xu's avatar
Yaowu Xu committed
504
static void set_offsets_topblock(AV1_COMMON *const cm, MACROBLOCKD *const xd,
505 506
                                 const TileInfo *const tile, BLOCK_SIZE bsize,
                                 int mi_row, int mi_col) {
507 508 509 510 511 512 513 514 515 516 517 518 519
  const int bw = num_8x8_blocks_wide_lookup[bsize];
  const int bh = num_8x8_blocks_high_lookup[bsize];
  const int offset = mi_row * cm->mi_stride + mi_col;
  const int bwl = b_width_log2_lookup[bsize];
  const int bhl = b_height_log2_lookup[bsize];

  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = cm->mi + offset;

  set_plane_n4(xd, bw, bh, bwl, bhl);

  set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, cm->mi_rows, cm->mi_cols);

Yaowu Xu's avatar
Yaowu Xu committed
520
  av1_setup_dst_planes(xd->plane, get_frame_new_buffer(cm), mi_row, mi_col);
521 522
}

Yaowu Xu's avatar
Yaowu Xu committed
523
static void set_param_topblock(AV1_COMMON *const cm, MACROBLOCKD *const xd,
524
                               BLOCK_SIZE bsize, int mi_row, int mi_col,
525
                               int txfm, int skip) {
526 527
  const int bw = num_8x8_blocks_wide_lookup[bsize];
  const int bh = num_8x8_blocks_high_lookup[bsize];
Yaowu Xu's avatar
Yaowu Xu committed
528 529
  const int x_mis = AOMMIN(bw, cm->mi_cols - mi_col);
  const int y_mis = AOMMIN(bh, cm->mi_rows - mi_row);
530 531 532 533 534 535 536 537 538 539 540
  const int offset = mi_row * cm->mi_stride + mi_col;
  int x, y;

  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = cm->mi + offset;

  for (y = 0; y < y_mis; ++y)
    for (x = 0; x < x_mis; ++x) {
      xd->mi[y * cm->mi_stride + x]->mbmi.skip = skip;
      xd->mi[y * cm->mi_stride + x]->mbmi.tx_type = txfm;
    }
541 542
#if CONFIG_VAR_TX
  xd->above_txfm_context = cm->above_txfm_context + mi_col;
543
  xd->left_txfm_context =
544
      xd->left_txfm_context_buffer + (mi_row & MAX_MIB_MASK);
545
  set_txfm_ctxs(xd->mi[0]->mbmi.tx_size, bw, bh, xd);
546
#endif
547 548
}

Yaowu Xu's avatar
Yaowu Xu committed
549
static void set_ref(AV1_COMMON *const cm, MACROBLOCKD *const xd, int idx,
550
                    int mi_row, int mi_col) {
551 552 553
  MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
  RefBuffer *ref_buffer = &cm->frame_refs[mbmi->ref_frame[idx] - LAST_FRAME];
  xd->block_refs[idx] = ref_buffer;
Yaowu Xu's avatar
Yaowu Xu committed
554 555
  if (!av1_is_valid_scale(&ref_buffer->sf))
    aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM,
556
                       "Invalid scale factors");
Yaowu Xu's avatar
Yaowu Xu committed
557 558
  av1_setup_pre_planes(xd, idx, ref_buffer->buf, mi_row, mi_col,
                       &ref_buffer->sf);
559 560 561 562
  xd->corrupted |= ref_buffer->buf->corrupted;
}

static void dec_predict_b_extend(
Yaowu Xu's avatar
Yaowu Xu committed
563
    AV1Decoder *const pbi, MACROBLOCKD *const xd, const TileInfo *const tile,
564 565 566
    int block, int mi_row_ori, int mi_col_ori, int mi_row_pred, int mi_col_pred,
    int mi_row_top, int mi_col_top, uint8_t *dst_buf[3], int dst_stride[3],
    BLOCK_SIZE bsize_top, BLOCK_SIZE bsize_pred, int b_sub8x8, int bextend) {
567 568 569 570 571 572 573 574 575 576 577 578
  // Used in supertx
  // (mi_row_ori, mi_col_ori): location for mv
  // (mi_row_pred, mi_col_pred, bsize_pred): region to predict
  // (mi_row_top, mi_col_top, bsize_top): region of the top partition size
  // block: sub location of sub8x8 blocks
  // b_sub8x8: 1: ori is sub8x8; 0: ori is not sub8x8
  // bextend: 1: region to predict is an extension of ori; 0: not
  int r = (mi_row_pred - mi_row_top) * MI_SIZE;
  int c = (mi_col_pred - mi_col_top) * MI_SIZE;
  const int mi_width_top = num_8x8_blocks_wide_lookup[bsize_top];
  const int mi_height_top = num_8x8_blocks_high_lookup[bsize_top];
  MB_MODE_INFO *mbmi;
Yaowu Xu's avatar
Yaowu Xu committed
579
  AV1_COMMON *const cm = &pbi->common;
580 581 582

  if (mi_row_pred < mi_row_top || mi_col_pred < mi_col_top ||
      mi_row_pred >= mi_row_top + mi_height_top ||
583 584
      mi_col_pred >= mi_col_top + mi_width_top || mi_row_pred >= cm->mi_rows ||
      mi_col_pred >= cm->mi_cols)
585 586
    return;

587
  mbmi = set_offsets_extend(cm, xd, tile, bsize_pred, mi_row_pred, mi_col_pred,
588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610
                            mi_row_ori, mi_col_ori);
  set_ref(cm, xd, 0, mi_row_pred, mi_col_pred);
  if (has_second_ref(&xd->mi[0]->mbmi))
    set_ref(cm, xd, 1, mi_row_pred, mi_col_pred);

  if (!bextend) {
    mbmi->tx_size = b_width_log2_lookup[bsize_top];
  }

  xd->plane[0].dst.stride = dst_stride[0];
  xd->plane[1].dst.stride = dst_stride[1];
  xd->plane[2].dst.stride = dst_stride[2];
  xd->plane[0].dst.buf = dst_buf[0] +
                         (r >> xd->plane[0].subsampling_y) * dst_stride[0] +
                         (c >> xd->plane[0].subsampling_x);
  xd->plane[1].dst.buf = dst_buf[1] +
                         (r >> xd->plane[1].subsampling_y) * dst_stride[1] +
                         (c >> xd->plane[1].subsampling_x);
  xd->plane[2].dst.buf = dst_buf[2] +
                         (r >> xd->plane[2].subsampling_y) * dst_stride[2] +
                         (c >> xd->plane[2].subsampling_x);

  if (!b_sub8x8)
Yaowu Xu's avatar
Yaowu Xu committed
611
    av1_build_inter_predictors_sb_extend(xd,
612
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
613
                                         mi_row_ori, mi_col_ori,
614
#endif  // CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
615
                                         mi_row_pred, mi_col_pred, bsize_pred);
616
  else
Yaowu Xu's avatar
Yaowu Xu committed
617
    av1_build_inter_predictors_sb_sub8x8_extend(xd,
618
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
619
                                                mi_row_ori, mi_col_ori,
620
#endif  // CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
621 622
                                                mi_row_pred, mi_col_pred,
                                                bsize_pred, block);
623 624
}

Yaowu Xu's avatar
Yaowu Xu committed
625
static void dec_extend_dir(AV1Decoder *const pbi, MACROBLOCKD *const xd,
626
                           const TileInfo *const tile, int block,
627 628 629
                           BLOCK_SIZE bsize, BLOCK_SIZE top_bsize, int mi_row,
                           int mi_col, int mi_row_top, int mi_col_top,
                           uint8_t *dst_buf[3], int dst_stride[3], int dir) {
630 631 632 633 634 635 636 637 638 639 640
  // dir: 0-lower, 1-upper, 2-left, 3-right
  //      4-lowerleft, 5-upperleft, 6-lowerright, 7-upperright
  const int mi_width = num_8x8_blocks_wide_lookup[bsize];
  const int mi_height = num_8x8_blocks_high_lookup[bsize];
  int xss = xd->plane[1].subsampling_x;
  int yss = xd->plane[1].subsampling_y;
  int b_sub8x8 = (bsize < BLOCK_8X8) ? 1 : 0;
  BLOCK_SIZE extend_bsize;
  int unit, mi_row_pred, mi_col_pred;

  if (dir == 0 || dir == 1) {
641 642 643
    extend_bsize = (mi_width == 1 || bsize < BLOCK_8X8 || xss < yss)
                       ? BLOCK_8X8
                       : BLOCK_16X8;
644 645 646 647
    unit = num_8x8_blocks_wide_lookup[extend_bsize];
    mi_row_pred = mi_row + ((dir == 0) ? mi_height : -1);
    mi_col_pred = mi_col;

648 649 650
    dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col, mi_row_pred,
                         mi_col_pred, mi_row_top, mi_col_top, dst_buf,
                         dst_stride, top_bsize, extend_bsize, b_sub8x8, 1);
651 652 653 654

    if (mi_width > unit) {
      int i;
      assert(!b_sub8x8);
655
      for (i = 0; i < mi_width / unit - 1; i++) {
656
        mi_col_pred += unit;
657 658 659
        dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col, mi_row_pred,
                             mi_col_pred, mi_row_top, mi_col_top, dst_buf,
                             dst_stride, top_bsize, extend_bsize, b_sub8x8, 1);
660 661 662
      }
    }
  } else if (dir == 2 || dir == 3) {
663 664 665
    extend_bsize = (mi_height == 1 || bsize < BLOCK_8X8 || yss < xss)
                       ? BLOCK_8X8
                       : BLOCK_8X16;
666 667 668 669
    unit = num_8x8_blocks_high_lookup[extend_bsize];
    mi_row_pred = mi_row;
    mi_col_pred = mi_col + ((dir == 3) ? mi_width : -1);

670 671 672
    dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col, mi_row_pred,
                         mi_col_pred, mi_row_top, mi_col_top, dst_buf,
                         dst_stride, top_bsize, extend_bsize, b_sub8x8, 1);
673 674 675

    if (mi_height > unit) {
      int i;
676
      for (i = 0; i < mi_height / unit - 1; i++) {
677
        mi_row_pred += unit;
678 679 680
        dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col, mi_row_pred,
                             mi_col_pred, mi_row_top, mi_col_top, dst_buf,
                             dst_stride, top_bsize, extend_bsize, b_sub8x8, 1);
681 682 683 684 685 686
      }
    }
  } else {
    extend_bsize = BLOCK_8X8;
    mi_row_pred = mi_row + ((dir == 4 || dir == 6) ? mi_height : -1);
    mi_col_pred = mi_col + ((dir == 6 || dir == 7) ? mi_width : -1);
687 688 689
    dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col, mi_row_pred,
                         mi_col_pred, mi_row_top, mi_col_top, dst_buf,
                         dst_stride, top_bsize, extend_bsize, b_sub8x8, 1);
690 691 692
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
693
static void dec_extend_all(AV1Decoder *const pbi, MACROBLOCKD *const xd,
694
                           const TileInfo *const tile, int block,
695 696 697
                           BLOCK_SIZE bsize, BLOCK_SIZE top_bsize, int mi_row,
                           int mi_col, int mi_row_top, int mi_col_top,
                           uint8_t *dst_buf[3], int dst_stride[3]) {
698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 0);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 1);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 2);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 3);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 4);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 5);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 6);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 7);
}

Yaowu Xu's avatar
Yaowu Xu committed
716
static void dec_predict_sb_complex(AV1Decoder *const pbi, MACROBLOCKD *const xd,
717 718
                                   const TileInfo *const tile, int mi_row,
                                   int mi_col, int mi_row_top, int mi_col_top,
719 720
                                   BLOCK_SIZE bsize, BLOCK_SIZE top_bsize,
                                   uint8_t *dst_buf[3], int dst_stride[3]) {
Yaowu Xu's avatar
Yaowu Xu committed
721
  const AV1_COMMON *const cm = &pbi->common;
722 723 724
  const int hbs = num_8x8_blocks_wide_lookup[bsize] / 2;
  const PARTITION_TYPE partition = get_partition(cm, mi_row, mi_col, bsize);
  const BLOCK_SIZE subsize = get_subsize(bsize, partition);
725
#if CONFIG_EXT_PARTITION_TYPES
726
  const BLOCK_SIZE bsize2 = get_subsize(bsize, PARTITION_SPLIT);
727
#endif
728 729
  int i;
  const int mi_offset = mi_row * cm->mi_stride + mi_col;
730 731
  uint8_t *dst_buf1[3], *dst_buf2[3], *dst_buf3[3];

732 733 734 735 736 737
  DECLARE_ALIGNED(16, uint8_t, tmp_buf1[MAX_MB_PLANE * MAX_TX_SQUARE * 2]);
  DECLARE_ALIGNED(16, uint8_t, tmp_buf2[MAX_MB_PLANE * MAX_TX_SQUARE * 2]);
  DECLARE_ALIGNED(16, uint8_t, tmp_buf3[MAX_MB_PLANE * MAX_TX_SQUARE * 2]);
  int dst_stride1[3] = { MAX_TX_SIZE, MAX_TX_SIZE, MAX_TX_SIZE };
  int dst_stride2[3] = { MAX_TX_SIZE, MAX_TX_SIZE, MAX_TX_SIZE };
  int dst_stride3[3] = { MAX_TX_SIZE, MAX_TX_SIZE, MAX_TX_SIZE };
738

Yaowu Xu's avatar
Yaowu Xu committed
739
#if CONFIG_AOM_HIGHBITDEPTH
740 741 742
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
    int len = sizeof(uint16_t);
    dst_buf1[0] = CONVERT_TO_BYTEPTR(tmp_buf1);
743 744
    dst_buf1[1] = CONVERT_TO_BYTEPTR(tmp_buf1 + MAX_TX_SQUARE * len);
    dst_buf1[2] = CONVERT_TO_BYTEPTR(tmp_buf1 + 2 * MAX_TX_SQUARE * len);
745
    dst_buf2[0] = CONVERT_TO_BYTEPTR(tmp_buf2);
746 747
    dst_buf2[1] = CONVERT_TO_BYTEPTR(tmp_buf2 + MAX_TX_SQUARE * len);
    dst_buf2[2] = CONVERT_TO_BYTEPTR(tmp_buf2 + 2 * MAX_TX_SQUARE * len);
748
    dst_buf3[0] = CONVERT_TO_BYTEPTR(tmp_buf3);
749 750
    dst_buf3[1] = CONVERT_TO_BYTEPTR(tmp_buf3 + MAX_TX_SQUARE * len);
    dst_buf3[2] = CONVERT_TO_BYTEPTR(tmp_buf3 + 2 * MAX_TX_SQUARE * len);
751 752 753
  } else {
#endif
    dst_buf1[0] = tmp_buf1;
754 755
    dst_buf1[1] = tmp_buf1 + MAX_TX_SQUARE;
    dst_buf1[2] = tmp_buf1 + 2 * MAX_TX_SQUARE;
756
    dst_buf2[0] = tmp_buf2;
757 758
    dst_buf2[1] = tmp_buf2 + MAX_TX_SQUARE;
    dst_buf2[2] = tmp_buf2 + 2 * MAX_TX_SQUARE;
759
    dst_buf3[0] = tmp_buf3;
760 761
    dst_buf3[1] = tmp_buf3 + MAX_TX_SQUARE;
    dst_buf3[2] = tmp_buf3 + 2 * MAX_TX_SQUARE;
Yaowu Xu's avatar
Yaowu Xu committed
762
#if CONFIG_AOM_HIGHBITDEPTH
763 764 765
  }
#endif

766
  if (mi_row >= cm->mi_rows || mi_col >= cm->mi_cols) return;
767

768 769
  xd->mi = cm->mi_grid_visible + mi_offset;
  xd->mi[0] = cm->mi + mi_offset;
770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806

  for (i = 0; i < MAX_MB_PLANE; i++) {
    xd->plane[i].dst.buf = dst_buf[i];
    xd->plane[i].dst.stride = dst_stride[i];
  }

  switch (partition) {
    case PARTITION_NONE:
      assert(bsize < top_bsize);
      dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col,
                           mi_row_top, mi_col_top, dst_buf, dst_stride,
                           top_bsize, bsize, 0, 0);
      dec_extend_all(pbi, xd, tile, 0, bsize, top_bsize, mi_row, mi_col,
                     mi_row_top, mi_col_top, dst_buf, dst_stride);
      break;
    case PARTITION_HORZ:
      if (bsize == BLOCK_8X8) {
        // For sub8x8, predict in 8x8 unit
        // First half
        dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf, dst_stride,
                             top_bsize, BLOCK_8X8, 1, 0);
        if (bsize < top_bsize)
          dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf, dst_stride);

        // Second half
        dec_predict_b_extend(pbi, xd, tile, 2, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf1, dst_stride1,
                             top_bsize, BLOCK_8X8, 1, 1);
        if (bsize < top_bsize)
          dec_extend_all(pbi, xd, tile, 2, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf1, dst_stride1);

        // weighted average to smooth the boundary
        xd->plane[0].dst.buf = dst_buf[0];
        xd->plane[0].dst.stride = dst_stride[0];
Yaowu Xu's avatar
Yaowu Xu committed
807
        av1_build_masked_inter_predictor_complex(
808 809 810
            xd, dst_buf[0], dst_stride[0], dst_buf1[0], dst_stride1[0], mi_row,
            mi_col, mi_row_top, mi_col_top, bsize, top_bsize, PARTITION_HORZ,
            0);
811 812 813 814 815 816 817 818 819 820 821 822 823 824 825
      } else {
        // First half
        dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf, dst_stride,
                             top_bsize, subsize, 0, 0);
        if (bsize < top_bsize)
          dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf, dst_stride);
        else
          dec_extend_dir(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf, dst_stride, 0);

        if (mi_row + hbs < cm->mi_rows) {
          // Second half
          dec_predict_b_extend(pbi, xd, tile, 0, mi_row + hbs, mi_col,
826 827
                               mi_row + hbs, mi_col, mi_row_top, mi_col_top,
                               dst_buf1, dst_stride1, top_bsize, subsize, 0, 0);
828
          if (bsize < top_bsize)
829 830 831
            dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row + hbs,
                           mi_col, mi_row_top, mi_col_top, dst_buf1,
                           dst_stride1);
832
          else
833 834 835
            dec_extend_dir(pbi, xd, tile, 0, subsize, top_bsize, mi_row + hbs,
                           mi_col, mi_row_top, mi_col_top, dst_buf1,
                           dst_stride1, 1);
836 837 838 839 840

          // weighted average to smooth the boundary
          for (i = 0; i < MAX_MB_PLANE; i++) {
            xd->plane[i].dst.buf = dst_buf[i];
            xd->plane[i].dst.stride = dst_stride[i];
Yaowu Xu's avatar
Yaowu Xu committed
841
            av1_build_masked_inter_predictor_complex(
842
                xd, dst_buf[i], dst_stride[i], dst_buf1[i], dst_stride1[i],
843 844
                mi_row, mi_col, mi_row_top, mi_col_top, bsize, top_bsize,
                PARTITION_HORZ, i);
845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869
          }
        }
      }
      break;
    case PARTITION_VERT:
      if (bsize == BLOCK_8X8) {
        // First half
        dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf, dst_stride,
                             top_bsize, BLOCK_8X8, 1, 0);
        if (bsize < top_bsize)
          dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf, dst_stride);

        // Second half
        dec_predict_b_extend(pbi, xd, tile, 1, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf1, dst_stride1,
                             top_bsize, BLOCK_8X8, 1, 1);
        if (bsize < top_bsize)
          dec_extend_all(pbi, xd, tile, 1, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf1, dst_stride1);

        // Smooth
        xd->plane[0].dst.buf = dst_buf[0];
        xd->plane[0].dst.stride = dst_stride[0];
Yaowu Xu's avatar
Yaowu Xu committed
870
        av1_build_masked_inter_predictor_complex(
871 872 873
            xd, dst_buf[0], dst_stride[0], dst_buf1[0], dst_stride1[0], mi_row,
            mi_col, mi_row_top, mi_col_top, bsize, top_bsize, PARTITION_VERT,
            0);
874 875 876 877 878 879 880 881 882 883 884 885