decodeframe.c 161 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4
5
6
7
8
9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10
11
12
13
14
 */

#include <assert.h>
#include <stdlib.h>  // qsort()

Yaowu Xu's avatar
Yaowu Xu committed
15
16
17
#include "./aom_config.h"
#include "./aom_dsp_rtcd.h"
#include "./aom_scale_rtcd.h"
Jingning Han's avatar
Jingning Han committed
18
#include "./av1_rtcd.h"
Jingning Han's avatar
Jingning Han committed
19

20
#include "aom/aom_codec.h"
Yaowu Xu's avatar
Yaowu Xu committed
21
#include "aom_dsp/aom_dsp_common.h"
Jingning Han's avatar
Jingning Han committed
22
23
#include "aom_dsp/bitreader.h"
#include "aom_dsp/bitreader_buffer.h"
Yaowu Xu's avatar
Yaowu Xu committed
24
#include "aom_mem/aom_mem.h"
25
26
#include "aom_ports/mem.h"
#include "aom_ports/mem_ops.h"
Yaowu Xu's avatar
Yaowu Xu committed
27
28
#include "aom_scale/aom_scale.h"
#include "aom_util/aom_thread.h"
Jingning Han's avatar
Jingning Han committed
29

30
#include "av1/common/alloccommon.h"
31
#if CONFIG_CLPF
Steinar Midtskogen's avatar
Steinar Midtskogen committed
32
#include "aom/aom_image.h"
33
#include "av1/common/clpf.h"
34
#endif
35
#include "av1/common/common.h"
Yaowu Xu's avatar
Yaowu Xu committed
36
#if CONFIG_DERING
37
#include "av1/common/dering.h"
Yaowu Xu's avatar
Yaowu Xu committed
38
#endif  // CONFIG_DERING
39
40
41
42
43
44
#include "av1/common/entropy.h"
#include "av1/common/entropymode.h"
#include "av1/common/idct.h"
#include "av1/common/pred_common.h"
#include "av1/common/quant_common.h"
#include "av1/common/reconinter.h"
Jingning Han's avatar
Jingning Han committed
45
#include "av1/common/reconintra.h"
46
#include "av1/common/seg_common.h"
Jingning Han's avatar
Jingning Han committed
47
#include "av1/common/thread_common.h"
48
#include "av1/common/tile_common.h"
49

50
51
52
#include "av1/decoder/decodeframe.h"
#include "av1/decoder/decodemv.h"
#include "av1/decoder/decoder.h"
Jingning Han's avatar
Jingning Han committed
53
#include "av1/decoder/detokenize.h"
54
#include "av1/decoder/dsubexp.h"
Jingning Han's avatar
Jingning Han committed
55

Yaowu Xu's avatar
Yaowu Xu committed
56
#define MAX_AV1_HEADER_SIZE 80
Michael Bebenita's avatar
Michael Bebenita committed
57
#define ACCT_STR __func__
Jingning Han's avatar
Jingning Han committed
58

59
60
61
62
63
64
65
66
static struct aom_read_bit_buffer *init_read_bit_buffer(
    AV1Decoder *pbi, struct aom_read_bit_buffer *rb, const uint8_t *data,
    const uint8_t *data_end, uint8_t clear_data[MAX_AV1_HEADER_SIZE]);
static int read_compressed_header(AV1Decoder *pbi, const uint8_t *data,
                                  size_t partition_size);
static size_t read_uncompressed_header(AV1Decoder *pbi,
                                       struct aom_read_bit_buffer *rb);

Yaowu Xu's avatar
Yaowu Xu committed
67
static int is_compound_reference_allowed(const AV1_COMMON *cm) {
Jingning Han's avatar
Jingning Han committed
68
  int i;
69
  if (frame_is_intra_only(cm)) return 0;
70
  for (i = 1; i < INTER_REFS_PER_FRAME; ++i)
71
    if (cm->ref_frame_sign_bias[i + 1] != cm->ref_frame_sign_bias[1]) return 1;
Jingning Han's avatar
Jingning Han committed
72
73
74
75

  return 0;
}

Yaowu Xu's avatar
Yaowu Xu committed
76
static void setup_compound_reference_mode(AV1_COMMON *cm) {
77
#if CONFIG_EXT_REFS
78
  cm->comp_fwd_ref[0] = LAST_FRAME;
79
80
81
82
  cm->comp_fwd_ref[1] = LAST2_FRAME;
  cm->comp_fwd_ref[2] = LAST3_FRAME;
  cm->comp_fwd_ref[3] = GOLDEN_FRAME;

83
84
  cm->comp_bwd_ref[0] = BWDREF_FRAME;
  cm->comp_bwd_ref[1] = ALTREF_FRAME;
85
#else
Jingning Han's avatar
Jingning Han committed
86
  if (cm->ref_frame_sign_bias[LAST_FRAME] ==
87
      cm->ref_frame_sign_bias[GOLDEN_FRAME]) {
Jingning Han's avatar
Jingning Han committed
88
89
90
91
    cm->comp_fixed_ref = ALTREF_FRAME;
    cm->comp_var_ref[0] = LAST_FRAME;
    cm->comp_var_ref[1] = GOLDEN_FRAME;
  } else if (cm->ref_frame_sign_bias[LAST_FRAME] ==
92
             cm->ref_frame_sign_bias[ALTREF_FRAME]) {
Jingning Han's avatar
Jingning Han committed
93
94
95
96
97
98
99
100
    cm->comp_fixed_ref = GOLDEN_FRAME;
    cm->comp_var_ref[0] = LAST_FRAME;
    cm->comp_var_ref[1] = ALTREF_FRAME;
  } else {
    cm->comp_fixed_ref = LAST_FRAME;
    cm->comp_var_ref[0] = GOLDEN_FRAME;
    cm->comp_var_ref[1] = ALTREF_FRAME;
  }
101
#endif  // CONFIG_EXT_REFS
Jingning Han's avatar
Jingning Han committed
102
103
104
105
106
107
}

static int read_is_valid(const uint8_t *start, size_t len, const uint8_t *end) {
  return len != 0 && len <= (size_t)(end - start);
}

Yaowu Xu's avatar
Yaowu Xu committed
108
109
static int decode_unsigned_max(struct aom_read_bit_buffer *rb, int max) {
  const int data = aom_rb_read_literal(rb, get_unsigned_bits(max));
Jingning Han's avatar
Jingning Han committed
110
111
112
  return data > max ? max : data;
}

Yaowu Xu's avatar
Yaowu Xu committed
113
114
static TX_MODE read_tx_mode(struct aom_read_bit_buffer *rb) {
  return aom_rb_read_bit(rb) ? TX_MODE_SELECT : aom_rb_read_literal(rb, 2);
115
}
Jingning Han's avatar
Jingning Han committed
116

Yaowu Xu's avatar
Yaowu Xu committed
117
static void read_switchable_interp_probs(FRAME_CONTEXT *fc, aom_reader *r) {
Jingning Han's avatar
Jingning Han committed
118
  int i, j;
119
  for (j = 0; j < SWITCHABLE_FILTER_CONTEXTS; ++j) {
Jingning Han's avatar
Jingning Han committed
120
    for (i = 0; i < SWITCHABLE_FILTERS - 1; ++i)
Michael Bebenita's avatar
Michael Bebenita committed
121
      av1_diff_update_prob(r, &fc->switchable_interp_prob[j][i], ACCT_STR);
122
123
124
125
126
#if CONFIG_DAALA_EC
    av1_tree_to_cdf(av1_switchable_interp_tree, fc->switchable_interp_prob[j],
                    fc->switchable_interp_cdf[j]);
#endif
  }
Jingning Han's avatar
Jingning Han committed
127
128
}

Yaowu Xu's avatar
Yaowu Xu committed
129
static void read_inter_mode_probs(FRAME_CONTEXT *fc, aom_reader *r) {
130
131
132
  int i;
#if CONFIG_REF_MV
  for (i = 0; i < NEWMV_MODE_CONTEXTS; ++i)
Michael Bebenita's avatar
Michael Bebenita committed
133
    av1_diff_update_prob(r, &fc->newmv_prob[i], ACCT_STR);
134
  for (i = 0; i < ZEROMV_MODE_CONTEXTS; ++i)
Michael Bebenita's avatar
Michael Bebenita committed
135
    av1_diff_update_prob(r, &fc->zeromv_prob[i], ACCT_STR);
136
  for (i = 0; i < REFMV_MODE_CONTEXTS; ++i)
Michael Bebenita's avatar
Michael Bebenita committed
137
    av1_diff_update_prob(r, &fc->refmv_prob[i], ACCT_STR);
138
  for (i = 0; i < DRL_MODE_CONTEXTS; ++i)
Michael Bebenita's avatar
Michael Bebenita committed
139
    av1_diff_update_prob(r, &fc->drl_prob[i], ACCT_STR);
Yue Chen's avatar
Yue Chen committed
140
#if CONFIG_EXT_INTER
Michael Bebenita's avatar
Michael Bebenita committed
141
  av1_diff_update_prob(r, &fc->new2mv_prob, ACCT_STR);
Yue Chen's avatar
Yue Chen committed
142
#endif  // CONFIG_EXT_INTER
143
144
#else
  int j;
145
  for (i = 0; i < INTER_MODE_CONTEXTS; ++i) {
Jingning Han's avatar
Jingning Han committed
146
    for (j = 0; j < INTER_MODES - 1; ++j)
Michael Bebenita's avatar
Michael Bebenita committed
147
      av1_diff_update_prob(r, &fc->inter_mode_probs[i][j], ACCT_STR);
148
149
150
151
152
#if CONFIG_DAALA_EC
    av1_tree_to_cdf(av1_inter_mode_tree, fc->inter_mode_probs[i],
                    fc->inter_mode_cdf[i]);
#endif
  }
153
#endif
Jingning Han's avatar
Jingning Han committed
154
155
}

156
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
157
static void read_inter_compound_mode_probs(FRAME_CONTEXT *fc, aom_reader *r) {
158
  int i, j;
Michael Bebenita's avatar
Michael Bebenita committed
159
  if (aom_read(r, GROUP_DIFF_UPDATE_PROB, ACCT_STR)) {
160
161
    for (j = 0; j < INTER_MODE_CONTEXTS; ++j) {
      for (i = 0; i < INTER_COMPOUND_MODES - 1; ++i) {
Michael Bebenita's avatar
Michael Bebenita committed
162
        av1_diff_update_prob(r, &fc->inter_compound_mode_probs[j][i], ACCT_STR);
163
164
165
166
167
168
      }
    }
  }
}
#endif  // CONFIG_EXT_INTER

169
static REFERENCE_MODE read_frame_reference_mode(
Yaowu Xu's avatar
Yaowu Xu committed
170
    const AV1_COMMON *cm, struct aom_read_bit_buffer *rb) {
171
  if (is_compound_reference_allowed(cm)) {
Yaowu Xu's avatar
Yaowu Xu committed
172
    return aom_rb_read_bit(rb)
173
               ? REFERENCE_MODE_SELECT
Yaowu Xu's avatar
Yaowu Xu committed
174
               : (aom_rb_read_bit(rb) ? COMPOUND_REFERENCE : SINGLE_REFERENCE);
175
176
177
178
  } else {
    return SINGLE_REFERENCE;
  }
}
Jingning Han's avatar
Jingning Han committed
179

Yaowu Xu's avatar
Yaowu Xu committed
180
static void read_frame_reference_mode_probs(AV1_COMMON *cm, aom_reader *r) {
Jingning Han's avatar
Jingning Han committed
181
  FRAME_CONTEXT *const fc = cm->fc;
182
  int i, j;
Jingning Han's avatar
Jingning Han committed
183
184
185

  if (cm->reference_mode == REFERENCE_MODE_SELECT)
    for (i = 0; i < COMP_INTER_CONTEXTS; ++i)
Michael Bebenita's avatar
Michael Bebenita committed
186
      av1_diff_update_prob(r, &fc->comp_inter_prob[i], ACCT_STR);
Jingning Han's avatar
Jingning Han committed
187

188
  if (cm->reference_mode != COMPOUND_REFERENCE) {
Jingning Han's avatar
Jingning Han committed
189
    for (i = 0; i < REF_CONTEXTS; ++i) {
190
      for (j = 0; j < (SINGLE_REFS - 1); ++j) {
Michael Bebenita's avatar
Michael Bebenita committed
191
        av1_diff_update_prob(r, &fc->single_ref_prob[i][j], ACCT_STR);
192
      }
Jingning Han's avatar
Jingning Han committed
193
    }
194
  }
Jingning Han's avatar
Jingning Han committed
195

196
197
  if (cm->reference_mode != SINGLE_REFERENCE) {
    for (i = 0; i < REF_CONTEXTS; ++i) {
198
#if CONFIG_EXT_REFS
199
      for (j = 0; j < (FWD_REFS - 1); ++j)
Michael Bebenita's avatar
Michael Bebenita committed
200
        av1_diff_update_prob(r, &fc->comp_ref_prob[i][j], ACCT_STR);
201
      for (j = 0; j < (BWD_REFS - 1); ++j)
Michael Bebenita's avatar
Michael Bebenita committed
202
        av1_diff_update_prob(r, &fc->comp_bwdref_prob[i][j], ACCT_STR);
203
204
#else
      for (j = 0; j < (COMP_REFS - 1); ++j)
Michael Bebenita's avatar
Michael Bebenita committed
205
        av1_diff_update_prob(r, &fc->comp_ref_prob[i][j], ACCT_STR);
206
#endif  // CONFIG_EXT_REFS
207
208
    }
  }
Jingning Han's avatar
Jingning Han committed
209
210
}

Yaowu Xu's avatar
Yaowu Xu committed
211
static void update_mv_probs(aom_prob *p, int n, aom_reader *r) {
Jingning Han's avatar
Jingning Han committed
212
  int i;
Michael Bebenita's avatar
Michael Bebenita committed
213
  for (i = 0; i < n; ++i) av1_diff_update_prob(r, &p[i], ACCT_STR);
Jingning Han's avatar
Jingning Han committed
214
215
}

Yaowu Xu's avatar
Yaowu Xu committed
216
static void read_mv_probs(nmv_context *ctx, int allow_hp, aom_reader *r) {
Jingning Han's avatar
Jingning Han committed
217
218
219
  int i, j;

  update_mv_probs(ctx->joints, MV_JOINTS - 1, r);
220
#if CONFIG_DAALA_EC || CONFIG_RANS
221
222
  av1_tree_to_cdf(av1_mv_joint_tree, ctx->joints, ctx->joint_cdf);
#endif
Jingning Han's avatar
Jingning Han committed
223
224
225
226
227
228
229

  for (i = 0; i < 2; ++i) {
    nmv_component *const comp_ctx = &ctx->comps[i];
    update_mv_probs(&comp_ctx->sign, 1, r);
    update_mv_probs(comp_ctx->classes, MV_CLASSES - 1, r);
    update_mv_probs(comp_ctx->class0, CLASS0_SIZE - 1, r);
    update_mv_probs(comp_ctx->bits, MV_OFFSET_BITS, r);
230
#if CONFIG_DAALA_EC || CONFIG_RANS
231
232
    av1_tree_to_cdf(av1_mv_class_tree, comp_ctx->classes, comp_ctx->class_cdf);
#endif
Jingning Han's avatar
Jingning Han committed
233
234
235
236
  }

  for (i = 0; i < 2; ++i) {
    nmv_component *const comp_ctx = &ctx->comps[i];
237
    for (j = 0; j < CLASS0_SIZE; ++j) {
Jingning Han's avatar
Jingning Han committed
238
      update_mv_probs(comp_ctx->class0_fp[j], MV_FP_SIZE - 1, r);
239
#if CONFIG_DAALA_EC || CONFIG_RANS
240
241
242
243
      av1_tree_to_cdf(av1_mv_fp_tree, comp_ctx->class0_fp[j],
                      comp_ctx->class0_fp_cdf[j]);
#endif
    }
244
    update_mv_probs(comp_ctx->fp, MV_FP_SIZE - 1, r);
245
#if CONFIG_DAALA_EC || CONFIG_RANS
246
247
    av1_tree_to_cdf(av1_mv_fp_tree, comp_ctx->fp, comp_ctx->fp_cdf);
#endif
Jingning Han's avatar
Jingning Han committed
248
249
250
251
252
253
254
255
256
257
258
  }

  if (allow_hp) {
    for (i = 0; i < 2; ++i) {
      nmv_component *const comp_ctx = &ctx->comps[i];
      update_mv_probs(&comp_ctx->class0_hp, 1, r);
      update_mv_probs(&comp_ctx->hp, 1, r);
    }
  }
}

259
static void inverse_transform_block(MACROBLOCKD *xd, int plane,
260
                                    const TX_TYPE tx_type,
261
                                    const TX_SIZE tx_size, uint8_t *dst,
262
                                    int stride, int16_t scan_line, int eob) {
Jingning Han's avatar
Jingning Han committed
263
  struct macroblockd_plane *const pd = &xd->plane[plane];
264
265
266
267
268
269
  tran_low_t *const dqcoeff = pd->dqcoeff;
  INV_TXFM_PARAM inv_txfm_param;
  inv_txfm_param.tx_type = tx_type;
  inv_txfm_param.tx_size = tx_size;
  inv_txfm_param.eob = eob;
  inv_txfm_param.lossless = xd->lossless[xd->mi[0]->mbmi.segment_id];
Angie Chiang's avatar
Angie Chiang committed
270

Yaowu Xu's avatar
Yaowu Xu committed
271
#if CONFIG_AOM_HIGHBITDEPTH
272
273
274
275
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
    inv_txfm_param.bd = xd->bd;
    highbd_inv_txfm_add(dqcoeff, dst, stride, &inv_txfm_param);
  } else {
Yaowu Xu's avatar
Yaowu Xu committed
276
#endif  // CONFIG_AOM_HIGHBITDEPTH
277
    inv_txfm_add(dqcoeff, dst, stride, &inv_txfm_param);
Yaowu Xu's avatar
Yaowu Xu committed
278
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
279
  }
280
281
#endif  // CONFIG_AOM_HIGHBITDEPTH
  memset(dqcoeff, 0, (scan_line + 1) * sizeof(dqcoeff[0]));
Jingning Han's avatar
Jingning Han committed
282
283
}

Angie Chiang's avatar
Angie Chiang committed
284
285
static void predict_and_reconstruct_intra_block(AV1_COMMON *cm,
                                                MACROBLOCKD *const xd,
Alex Converse's avatar
Alex Converse committed
286
287
288
#if CONFIG_ANS
                                                struct AnsDecoder *const r,
#else
Yaowu Xu's avatar
Yaowu Xu committed
289
                                                aom_reader *r,
Alex Converse's avatar
Alex Converse committed
290
#endif  // CONFIG_ANS
Jingning Han's avatar
Jingning Han committed
291
                                                MB_MODE_INFO *const mbmi,
292
                                                int plane, int row, int col,
Jingning Han's avatar
Jingning Han committed
293
294
295
                                                TX_SIZE tx_size) {
  struct macroblockd_plane *const pd = &xd->plane[plane];
  PREDICTION_MODE mode = (plane == 0) ? mbmi->mode : mbmi->uv_mode;
hui su's avatar
hui su committed
296
  PLANE_TYPE plane_type = (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
Jingning Han's avatar
Jingning Han committed
297
  uint8_t *dst;
hui su's avatar
hui su committed
298
  int block_idx = (row << 1) + col;
Jingning Han's avatar
Jingning Han committed
299
300
301
  dst = &pd->dst.buf[4 * row * pd->dst.stride + 4 * col];

  if (mbmi->sb_type < BLOCK_8X8)
302
    if (plane == 0) mode = xd->mi[0]->bmi[(row << 1) + col].as_mode;
Jingning Han's avatar
Jingning Han committed
303

304
  av1_predict_intra_block(xd, pd->width, pd->height, tx_size, mode, dst,
Yaowu Xu's avatar
Yaowu Xu committed
305
                          pd->dst.stride, dst, pd->dst.stride, col, row, plane);
Jingning Han's avatar
Jingning Han committed
306
307

  if (!mbmi->skip) {
hui su's avatar
hui su committed
308
    TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, tx_size);
Angie Chiang's avatar
Angie Chiang committed
309
    const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, 0);
310
311
312
313
    int16_t max_scan_line = 0;
    const int eob =
        av1_decode_block_tokens(xd, plane, scan_order, col, row, tx_size,
                                tx_type, &max_scan_line, r, mbmi->segment_id);
Angie Chiang's avatar
Angie Chiang committed
314
315
316
#if CONFIG_ADAPT_SCAN
    av1_update_scan_count_facade(cm, tx_size, tx_type, pd->dqcoeff, eob);
#endif
317
318
319
    if (eob)
      inverse_transform_block(xd, plane, tx_type, tx_size, dst, pd->dst.stride,
                              max_scan_line, eob);
Jingning Han's avatar
Jingning Han committed
320
321
322
  }
}

323
#if CONFIG_VAR_TX
Angie Chiang's avatar
Angie Chiang committed
324
325
static void decode_reconstruct_tx(AV1_COMMON *cm, MACROBLOCKD *const xd,
                                  aom_reader *r, MB_MODE_INFO *const mbmi,
326
                                  int plane, BLOCK_SIZE plane_bsize,
327
328
                                  int blk_row, int blk_col, TX_SIZE tx_size,
                                  int *eob_total) {
329
  const struct macroblockd_plane *const pd = &xd->plane[plane];
330
  const BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
331
332
  const int tx_row = blk_row >> (1 - pd->subsampling_y);
  const int tx_col = blk_col >> (1 - pd->subsampling_x);
333
  const TX_SIZE plane_tx_size =
334
      plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0]
335
            : mbmi->inter_tx_size[tx_row][tx_col];
336
337
338
  // Scale to match transform block unit.
  int max_blocks_high = block_size_high[plane_bsize];
  int max_blocks_wide = block_size_wide[plane_bsize];
339
340

  if (xd->mb_to_bottom_edge < 0)
341
    max_blocks_high += xd->mb_to_bottom_edge >> (3 + pd->subsampling_y);
342
  if (xd->mb_to_right_edge < 0)
343
344
345
346
    max_blocks_wide += xd->mb_to_right_edge >> (3 + pd->subsampling_x);

  max_blocks_high >>= tx_size_wide_log2[0];
  max_blocks_wide >>= tx_size_wide_log2[0];
347

348
  if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
349

350
  if (tx_size == plane_tx_size) {
351
    PLANE_TYPE plane_type = (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
352
353
    int block_idx = (blk_row << 1) + blk_col;
    TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, plane_tx_size);
Angie Chiang's avatar
Angie Chiang committed
354
    const SCAN_ORDER *sc = get_scan(cm, plane_tx_size, tx_type, 1);
355
    int16_t max_scan_line = 0;
356
    const int eob =
Yaowu Xu's avatar
Yaowu Xu committed
357
        av1_decode_block_tokens(xd, plane, sc, blk_col, blk_row, plane_tx_size,
358
                                tx_type, &max_scan_line, r, mbmi->segment_id);
359
360
    inverse_transform_block(
        xd, plane, tx_type, plane_tx_size,
361
        &pd->dst.buf[4 * blk_row * pd->dst.stride + 4 * blk_col],
362
        pd->dst.stride, max_scan_line, eob);
363
364
    *eob_total += eob;
  } else {
365
    int bsl = block_size_wide[bsize] >> (tx_size_wide_log2[0] + 1);
366
367
368
369
370
    int i;

    assert(bsl > 0);

    for (i = 0; i < 4; ++i) {
371
372
      const int offsetr = blk_row + (i >> 1) * bsl;
      const int offsetc = blk_col + (i & 0x01) * bsl;
373

374
      if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;
375

376
377
      decode_reconstruct_tx(cm, xd, r, mbmi, plane, plane_bsize, offsetr,
                            offsetc, tx_size - 1, eob_total);
378
379
380
    }
  }
}
381
#endif  // CONFIG_VAR_TX
382

383
#if !CONFIG_VAR_TX || CONFIG_SUPERTX || (CONFIG_EXT_TX && CONFIG_RECT_TX)
Angie Chiang's avatar
Angie Chiang committed
384
static int reconstruct_inter_block(AV1_COMMON *cm, MACROBLOCKD *const xd,
Alex Converse's avatar
Alex Converse committed
385
386
387
#if CONFIG_ANS
                                   struct AnsDecoder *const r,
#else
Yaowu Xu's avatar
Yaowu Xu committed
388
                                   aom_reader *r,
Alex Converse's avatar
Alex Converse committed
389
#endif
390
391
                                   int segment_id, int plane, int row, int col,
                                   TX_SIZE tx_size) {
Jingning Han's avatar
Jingning Han committed
392
  struct macroblockd_plane *const pd = &xd->plane[plane];
hui su's avatar
hui su committed
393
394
  PLANE_TYPE plane_type = (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
  int block_idx = (row << 1) + col;
hui su's avatar
hui su committed
395
  TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, tx_size);
Angie Chiang's avatar
Angie Chiang committed
396
  const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, 1);
397
398
399
400
  int16_t max_scan_line = 0;
  const int eob =
      av1_decode_block_tokens(xd, plane, scan_order, col, row, tx_size, tx_type,
                              &max_scan_line, r, segment_id);
Angie Chiang's avatar
Angie Chiang committed
401
402
403
#if CONFIG_ADAPT_SCAN
  av1_update_scan_count_facade(cm, tx_size, tx_type, pd->dqcoeff, eob);
#endif
404
405
406
407
  if (eob)
    inverse_transform_block(xd, plane, tx_type, tx_size,
                            &pd->dst.buf[4 * row * pd->dst.stride + 4 * col],
                            pd->dst.stride, max_scan_line, eob);
Jingning Han's avatar
Jingning Han committed
408
409
  return eob;
}
410
#endif  // !CONFIG_VAR_TX || CONFIG_SUPER_TX
Jingning Han's avatar
Jingning Han committed
411
412
413
414
415
416
417
418
419
420

static INLINE void dec_reset_skip_context(MACROBLOCKD *xd) {
  int i;
  for (i = 0; i < MAX_MB_PLANE; i++) {
    struct macroblockd_plane *const pd = &xd->plane[i];
    memset(pd->above_context, 0, sizeof(ENTROPY_CONTEXT) * pd->n4_w);
    memset(pd->left_context, 0, sizeof(ENTROPY_CONTEXT) * pd->n4_h);
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
421
static MB_MODE_INFO *set_offsets(AV1_COMMON *const cm, MACROBLOCKD *const xd,
Jingning Han's avatar
Jingning Han committed
422
                                 BLOCK_SIZE bsize, int mi_row, int mi_col,
423
424
                                 int bw, int bh, int x_mis, int y_mis, int bwl,
                                 int bhl) {
Jingning Han's avatar
Jingning Han committed
425
426
427
428
429
430
431
432
433
434
  const int offset = mi_row * cm->mi_stride + mi_col;
  int x, y;
  const TileInfo *const tile = &xd->tile;

  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = &cm->mi[offset];
  // TODO(slavarnway): Generate sb_type based on bwl and bhl, instead of
  // passing bsize from decode_partition().
  xd->mi[0]->mbmi.sb_type = bsize;
  for (y = 0; y < y_mis; ++y)
Jingning Han's avatar
Jingning Han committed
435
    for (x = !y; x < x_mis; ++x) xd->mi[y * cm->mi_stride + x] = xd->mi[0];
Jingning Han's avatar
Jingning Han committed
436
437
438
439

  set_plane_n4(xd, bw, bh, bwl, bhl);
  set_skip_context(xd, mi_row, mi_col);

440
441
442
443
#if CONFIG_VAR_TX
  xd->max_tx_size = max_txsize_lookup[bsize];
#endif

Jingning Han's avatar
Jingning Han committed
444
445
446
447
  // Distance of Mb to the various image edges. These are specified to 8th pel
  // as they are always compared to values that are in 1/8th pel units
  set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, cm->mi_rows, cm->mi_cols);

Yaowu Xu's avatar
Yaowu Xu committed
448
  av1_setup_dst_planes(xd->plane, get_frame_new_buffer(cm), mi_row, mi_col);
Jingning Han's avatar
Jingning Han committed
449
450
451
  return &xd->mi[0]->mbmi;
}

452
#if CONFIG_SUPERTX
Yaowu Xu's avatar
Yaowu Xu committed
453
static MB_MODE_INFO *set_offsets_extend(AV1_COMMON *const cm,
454
455
                                        MACROBLOCKD *const xd,
                                        const TileInfo *const tile,
456
457
458
                                        BLOCK_SIZE bsize_pred, int mi_row_pred,
                                        int mi_col_pred, int mi_row_ori,
                                        int mi_col_ori) {
459
460
461
462
463
464
465
466
467
468
  // Used in supertx
  // (mi_row_ori, mi_col_ori): location for mv
  // (mi_row_pred, mi_col_pred, bsize_pred): region to predict
  const int bw = num_8x8_blocks_wide_lookup[bsize_pred];
  const int bh = num_8x8_blocks_high_lookup[bsize_pred];
  const int offset = mi_row_ori * cm->mi_stride + mi_col_ori;
  const int bwl = b_width_log2_lookup[bsize_pred];
  const int bhl = b_height_log2_lookup[bsize_pred];
  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = cm->mi + offset;
469
470
  set_mi_row_col(xd, tile, mi_row_pred, bh, mi_col_pred, bw, cm->mi_rows,
                 cm->mi_cols);
471

472
473
  xd->up_available = (mi_row_ori > tile->mi_row_start);
  xd->left_available = (mi_col_ori > tile->mi_col_start);
474
475
476
477
478
479

  set_plane_n4(xd, bw, bh, bwl, bhl);

  return &xd->mi[0]->mbmi;
}

Yaowu Xu's avatar
Yaowu Xu committed
480
481
482
static MB_MODE_INFO *set_mb_offsets(AV1_COMMON *const cm, MACROBLOCKD *const xd,
                                    BLOCK_SIZE bsize, int mi_row, int mi_col,
                                    int bw, int bh, int x_mis, int y_mis) {
483
484
485
486
487
488
489
490
  const int offset = mi_row * cm->mi_stride + mi_col;
  const TileInfo *const tile = &xd->tile;
  int x, y;

  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = cm->mi + offset;
  xd->mi[0]->mbmi.sb_type = bsize;
  for (y = 0; y < y_mis; ++y)
491
    for (x = !y; x < x_mis; ++x) xd->mi[y * cm->mi_stride + x] = xd->mi[0];
492
493
494
495
496

  set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, cm->mi_rows, cm->mi_cols);
  return &xd->mi[0]->mbmi;
}

Yaowu Xu's avatar
Yaowu Xu committed
497
static void set_offsets_topblock(AV1_COMMON *const cm, MACROBLOCKD *const xd,
498
499
                                 const TileInfo *const tile, BLOCK_SIZE bsize,
                                 int mi_row, int mi_col) {
500
501
502
503
504
505
506
507
508
509
510
511
512
  const int bw = num_8x8_blocks_wide_lookup[bsize];
  const int bh = num_8x8_blocks_high_lookup[bsize];
  const int offset = mi_row * cm->mi_stride + mi_col;
  const int bwl = b_width_log2_lookup[bsize];
  const int bhl = b_height_log2_lookup[bsize];

  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = cm->mi + offset;

  set_plane_n4(xd, bw, bh, bwl, bhl);

  set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, cm->mi_rows, cm->mi_cols);

Yaowu Xu's avatar
Yaowu Xu committed
513
  av1_setup_dst_planes(xd->plane, get_frame_new_buffer(cm), mi_row, mi_col);
514
515
}

Yaowu Xu's avatar
Yaowu Xu committed
516
static void set_param_topblock(AV1_COMMON *const cm, MACROBLOCKD *const xd,
517
                               BLOCK_SIZE bsize, int mi_row, int mi_col,
518
                               int txfm, int skip) {
519
520
  const int bw = num_8x8_blocks_wide_lookup[bsize];
  const int bh = num_8x8_blocks_high_lookup[bsize];
Yaowu Xu's avatar
Yaowu Xu committed
521
522
  const int x_mis = AOMMIN(bw, cm->mi_cols - mi_col);
  const int y_mis = AOMMIN(bh, cm->mi_rows - mi_row);
523
524
525
526
527
528
529
530
531
532
533
  const int offset = mi_row * cm->mi_stride + mi_col;
  int x, y;

  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = cm->mi + offset;

  for (y = 0; y < y_mis; ++y)
    for (x = 0; x < x_mis; ++x) {
      xd->mi[y * cm->mi_stride + x]->mbmi.skip = skip;
      xd->mi[y * cm->mi_stride + x]->mbmi.tx_type = txfm;
    }
534
535
#if CONFIG_VAR_TX
  xd->above_txfm_context = cm->above_txfm_context + mi_col;
536
  xd->left_txfm_context =
537
      xd->left_txfm_context_buffer + (mi_row & MAX_MIB_MASK);
538
  set_txfm_ctxs(xd->mi[0]->mbmi.tx_size, bw, bh, xd);
539
#endif
540
541
}

Yaowu Xu's avatar
Yaowu Xu committed
542
static void set_ref(AV1_COMMON *const cm, MACROBLOCKD *const xd, int idx,
543
                    int mi_row, int mi_col) {
544
545
546
  MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
  RefBuffer *ref_buffer = &cm->frame_refs[mbmi->ref_frame[idx] - LAST_FRAME];
  xd->block_refs[idx] = ref_buffer;
Yaowu Xu's avatar
Yaowu Xu committed
547
548
  if (!av1_is_valid_scale(&ref_buffer->sf))
    aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM,
549
                       "Invalid scale factors");
Yaowu Xu's avatar
Yaowu Xu committed
550
551
  av1_setup_pre_planes(xd, idx, ref_buffer->buf, mi_row, mi_col,
                       &ref_buffer->sf);
552
553
554
555
  xd->corrupted |= ref_buffer->buf->corrupted;
}

static void dec_predict_b_extend(
Yaowu Xu's avatar
Yaowu Xu committed
556
    AV1Decoder *const pbi, MACROBLOCKD *const xd, const TileInfo *const tile,
557
558
559
    int block, int mi_row_ori, int mi_col_ori, int mi_row_pred, int mi_col_pred,
    int mi_row_top, int mi_col_top, uint8_t *dst_buf[3], int dst_stride[3],
    BLOCK_SIZE bsize_top, BLOCK_SIZE bsize_pred, int b_sub8x8, int bextend) {
560
561
562
563
564
565
566
567
568
569
570
571
  // Used in supertx
  // (mi_row_ori, mi_col_ori): location for mv
  // (mi_row_pred, mi_col_pred, bsize_pred): region to predict
  // (mi_row_top, mi_col_top, bsize_top): region of the top partition size
  // block: sub location of sub8x8 blocks
  // b_sub8x8: 1: ori is sub8x8; 0: ori is not sub8x8
  // bextend: 1: region to predict is an extension of ori; 0: not
  int r = (mi_row_pred - mi_row_top) * MI_SIZE;
  int c = (mi_col_pred - mi_col_top) * MI_SIZE;
  const int mi_width_top = num_8x8_blocks_wide_lookup[bsize_top];
  const int mi_height_top = num_8x8_blocks_high_lookup[bsize_top];
  MB_MODE_INFO *mbmi;
Yaowu Xu's avatar
Yaowu Xu committed
572
  AV1_COMMON *const cm = &pbi->common;
573
574
575

  if (mi_row_pred < mi_row_top || mi_col_pred < mi_col_top ||
      mi_row_pred >= mi_row_top + mi_height_top ||
576
577
      mi_col_pred >= mi_col_top + mi_width_top || mi_row_pred >= cm->mi_rows ||
      mi_col_pred >= cm->mi_cols)
578
579
    return;

580
  mbmi = set_offsets_extend(cm, xd, tile, bsize_pred, mi_row_pred, mi_col_pred,
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
                            mi_row_ori, mi_col_ori);
  set_ref(cm, xd, 0, mi_row_pred, mi_col_pred);
  if (has_second_ref(&xd->mi[0]->mbmi))
    set_ref(cm, xd, 1, mi_row_pred, mi_col_pred);

  if (!bextend) {
    mbmi->tx_size = b_width_log2_lookup[bsize_top];
  }

  xd->plane[0].dst.stride = dst_stride[0];
  xd->plane[1].dst.stride = dst_stride[1];
  xd->plane[2].dst.stride = dst_stride[2];
  xd->plane[0].dst.buf = dst_buf[0] +
                         (r >> xd->plane[0].subsampling_y) * dst_stride[0] +
                         (c >> xd->plane[0].subsampling_x);
  xd->plane[1].dst.buf = dst_buf[1] +
                         (r >> xd->plane[1].subsampling_y) * dst_stride[1] +
                         (c >> xd->plane[1].subsampling_x);
  xd->plane[2].dst.buf = dst_buf[2] +
                         (r >> xd->plane[2].subsampling_y) * dst_stride[2] +
                         (c >> xd->plane[2].subsampling_x);

  if (!b_sub8x8)
Yaowu Xu's avatar
Yaowu Xu committed
604
    av1_build_inter_predictors_sb_extend(xd,
605
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
606
                                         mi_row_ori, mi_col_ori,
607
#endif  // CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
608
                                         mi_row_pred, mi_col_pred, bsize_pred);
609
  else
Yaowu Xu's avatar
Yaowu Xu committed
610
    av1_build_inter_predictors_sb_sub8x8_extend(xd,
611
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
612
                                                mi_row_ori, mi_col_ori,
613
#endif  // CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
614
615
                                                mi_row_pred, mi_col_pred,
                                                bsize_pred, block);
616
617
}

Yaowu Xu's avatar
Yaowu Xu committed
618
static void dec_extend_dir(AV1Decoder *const pbi, MACROBLOCKD *const xd,
619
                           const TileInfo *const tile, int block,
620
621
622
                           BLOCK_SIZE bsize, BLOCK_SIZE top_bsize, int mi_row,
                           int mi_col, int mi_row_top, int mi_col_top,
                           uint8_t *dst_buf[3], int dst_stride[3], int dir) {
623
624
625
626
627
628
629
630
631
632
633
  // dir: 0-lower, 1-upper, 2-left, 3-right
  //      4-lowerleft, 5-upperleft, 6-lowerright, 7-upperright
  const int mi_width = num_8x8_blocks_wide_lookup[bsize];
  const int mi_height = num_8x8_blocks_high_lookup[bsize];
  int xss = xd->plane[1].subsampling_x;
  int yss = xd->plane[1].subsampling_y;
  int b_sub8x8 = (bsize < BLOCK_8X8) ? 1 : 0;
  BLOCK_SIZE extend_bsize;
  int unit, mi_row_pred, mi_col_pred;

  if (dir == 0 || dir == 1) {
634
635
636
    extend_bsize = (mi_width == 1 || bsize < BLOCK_8X8 || xss < yss)
                       ? BLOCK_8X8
                       : BLOCK_16X8;
637
638
639
640
    unit = num_8x8_blocks_wide_lookup[extend_bsize];
    mi_row_pred = mi_row + ((dir == 0) ? mi_height : -1);
    mi_col_pred = mi_col;

641
642
643
    dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col, mi_row_pred,
                         mi_col_pred, mi_row_top, mi_col_top, dst_buf,
                         dst_stride, top_bsize, extend_bsize, b_sub8x8, 1);
644
645
646
647

    if (mi_width > unit) {
      int i;
      assert(!b_sub8x8);
648
      for (i = 0; i < mi_width / unit - 1; i++) {
649
        mi_col_pred += unit;
650
651
652
        dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col, mi_row_pred,
                             mi_col_pred, mi_row_top, mi_col_top, dst_buf,
                             dst_stride, top_bsize, extend_bsize, b_sub8x8, 1);
653
654
655
      }
    }
  } else if (dir == 2 || dir == 3) {
656
657
658
    extend_bsize = (mi_height == 1 || bsize < BLOCK_8X8 || yss < xss)
                       ? BLOCK_8X8
                       : BLOCK_8X16;
659
660
661
662
    unit = num_8x8_blocks_high_lookup[extend_bsize];
    mi_row_pred = mi_row;
    mi_col_pred = mi_col + ((dir == 3) ? mi_width : -1);

663
664
665
    dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col, mi_row_pred,
                         mi_col_pred, mi_row_top, mi_col_top, dst_buf,
                         dst_stride, top_bsize, extend_bsize, b_sub8x8, 1);
666
667
668

    if (mi_height > unit) {
      int i;
669
      for (i = 0; i < mi_height / unit - 1; i++) {
670
        mi_row_pred += unit;
671
672
673
        dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col, mi_row_pred,
                             mi_col_pred, mi_row_top, mi_col_top, dst_buf,
                             dst_stride, top_bsize, extend_bsize, b_sub8x8, 1);
674
675
676
677
678
679
      }
    }
  } else {
    extend_bsize = BLOCK_8X8;
    mi_row_pred = mi_row + ((dir == 4 || dir == 6) ? mi_height : -1);
    mi_col_pred = mi_col + ((dir == 6 || dir == 7) ? mi_width : -1);
680
681
682
    dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col, mi_row_pred,
                         mi_col_pred, mi_row_top, mi_col_top, dst_buf,
                         dst_stride, top_bsize, extend_bsize, b_sub8x8, 1);
683
684
685
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
686
static void dec_extend_all(AV1Decoder *const pbi, MACROBLOCKD *const xd,
687
                           const TileInfo *const tile, int block,
688
689
690
                           BLOCK_SIZE bsize, BLOCK_SIZE top_bsize, int mi_row,
                           int mi_col, int mi_row_top, int mi_col_top,
                           uint8_t *dst_buf[3], int dst_stride[3]) {
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 0);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 1);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 2);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 3);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 4);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 5);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 6);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 7);
}

Yaowu Xu's avatar
Yaowu Xu committed
709
static void dec_predict_sb_complex(AV1Decoder *const pbi, MACROBLOCKD *const xd,
710
711
                                   const TileInfo *const tile, int mi_row,
                                   int mi_col, int mi_row_top, int mi_col_top,
712
713
                                   BLOCK_SIZE bsize, BLOCK_SIZE top_bsize,
                                   uint8_t *dst_buf[3], int dst_stride[3]) {
Yaowu Xu's avatar
Yaowu Xu committed
714
  const AV1_COMMON *const cm = &pbi->common;
715
716
717
  const int hbs = num_8x8_blocks_wide_lookup[bsize] / 2;
  const PARTITION_TYPE partition = get_partition(cm, mi_row, mi_col, bsize);
  const BLOCK_SIZE subsize = get_subsize(bsize, partition);
718
#if CONFIG_EXT_PARTITION_TYPES
719
  const BLOCK_SIZE bsize2 = get_subsize(bsize, PARTITION_SPLIT);
720
#endif
721
722
  int i;
  const int mi_offset = mi_row * cm->mi_stride + mi_col;
723
724
  uint8_t *dst_buf1[3], *dst_buf2[3], *dst_buf3[3];

725
726
727
728
729
730
  DECLARE_ALIGNED(16, uint8_t, tmp_buf1[MAX_MB_PLANE * MAX_TX_SQUARE * 2]);
  DECLARE_ALIGNED(16, uint8_t, tmp_buf2[MAX_MB_PLANE * MAX_TX_SQUARE * 2]);
  DECLARE_ALIGNED(16, uint8_t, tmp_buf3[MAX_MB_PLANE * MAX_TX_SQUARE * 2]);
  int dst_stride1[3] = { MAX_TX_SIZE, MAX_TX_SIZE, MAX_TX_SIZE };
  int dst_stride2[3] = { MAX_TX_SIZE, MAX_TX_SIZE, MAX_TX_SIZE };
  int dst_stride3[3] = { MAX_TX_SIZE, MAX_TX_SIZE, MAX_TX_SIZE };
731

Yaowu Xu's avatar
Yaowu Xu committed
732
#if CONFIG_AOM_HIGHBITDEPTH
733
734
735
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
    int len = sizeof(uint16_t);
    dst_buf1[0] = CONVERT_TO_BYTEPTR(tmp_buf1);
736
737
    dst_buf1[1] = CONVERT_TO_BYTEPTR(tmp_buf1 + MAX_TX_SQUARE * len);
    dst_buf1[2] = CONVERT_TO_BYTEPTR(tmp_buf1 + 2 * MAX_TX_SQUARE * len);
738
    dst_buf2[0] = CONVERT_TO_BYTEPTR(tmp_buf2);
739
740
    dst_buf2[1] = CONVERT_TO_BYTEPTR(tmp_buf2 + MAX_TX_SQUARE * len);
    dst_buf2[2] = CONVERT_TO_BYTEPTR(tmp_buf2 + 2 * MAX_TX_SQUARE * len);
741
    dst_buf3[0] = CONVERT_TO_BYTEPTR(tmp_buf3);
742
743
    dst_buf3[1] = CONVERT_TO_BYTEPTR(tmp_buf3 + MAX_TX_SQUARE * len);
    dst_buf3[2] = CONVERT_TO_BYTEPTR(tmp_buf3 + 2 * MAX_TX_SQUARE * len);
744
745
746
  } else {
#endif
    dst_buf1[0] = tmp_buf1;
747
748
    dst_buf1[1] = tmp_buf1 + MAX_TX_SQUARE;
    dst_buf1[2] = tmp_buf1 + 2 * MAX_TX_SQUARE;
749
    dst_buf2[0] = tmp_buf2;
750
751
    dst_buf2[1] = tmp_buf2 + MAX_TX_SQUARE;
    dst_buf2[2] = tmp_buf2 + 2 * MAX_TX_SQUARE;
752
    dst_buf3[0] = tmp_buf3;
753
754
    dst_buf3[1] = tmp_buf3 + MAX_TX_SQUARE;
    dst_buf3[2] = tmp_buf3 + 2 * MAX_TX_SQUARE;
Yaowu Xu's avatar
Yaowu Xu committed
755
#if CONFIG_AOM_HIGHBITDEPTH
756
757
758
  }
#endif

759
  if (mi_row >= cm->mi_rows || mi_col >= cm->mi_cols) return;
760

761
762
  xd->mi = cm->mi_grid_visible + mi_offset;
  xd->mi[0] = cm->mi + mi_offset;
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799

  for (i = 0; i < MAX_MB_PLANE; i++) {
    xd->plane[i].dst.buf = dst_buf[i];
    xd->plane[i].dst.stride = dst_stride[i];
  }

  switch (partition) {
    case PARTITION_NONE:
      assert(bsize < top_bsize);
      dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col,
                           mi_row_top, mi_col_top, dst_buf, dst_stride,
                           top_bsize, bsize, 0, 0);
      dec_extend_all(pbi, xd, tile, 0, bsize, top_bsize, mi_row, mi_col,
                     mi_row_top, mi_col_top, dst_buf, dst_stride);
      break;
    case PARTITION_HORZ:
      if (bsize == BLOCK_8X8) {
        // For sub8x8, predict in 8x8 unit
        // First half
        dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf, dst_stride,
                             top_bsize, BLOCK_8X8, 1, 0);
        if (bsize < top_bsize)
          dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf, dst_stride);

        // Second half
        dec_predict_b_extend(pbi, xd, tile, 2, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf1, dst_stride1,
                             top_bsize, BLOCK_8X8, 1, 1);
        if (bsize < top_bsize)
          dec_extend_all(pbi, xd, tile, 2, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf1, dst_stride1);

        // weighted average to smooth the boundary
        xd->plane[0].dst.buf = dst_buf[0];
        xd->plane[0].dst.stride = dst_stride[0];
Yaowu Xu's avatar
Yaowu Xu committed
800
        av1_build_masked_inter_predictor_complex(
801
802
803
            xd, dst_buf[0], dst_stride[0], dst_buf1[0], dst_stride1[0], mi_row,
            mi_col, mi_row_top, mi_col_top, bsize, top_bsize, PARTITION_HORZ,
            0);
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
      } else {
        // First half
        dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf, dst_stride,
                             top_bsize, subsize, 0, 0);
        if (bsize < top_bsize)
          dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf, dst_stride);
        else
          dec_extend_dir(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf, dst_stride, 0);

        if (mi_row + hbs < cm->mi_rows) {
          // Second half
          dec_predict_b_extend(pbi, xd, tile, 0, mi_row + hbs, mi_col,
819
820
                               mi_row + hbs, mi_col, mi_row_top, mi_col_top,
                               dst_buf1, dst_stride1, top_bsize, subsize, 0, 0);
821
          if (bsize < top_bsize)
822
823
824
            dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row + hbs,
                           mi_col, mi_row_top, mi_col_top, dst_buf1,
                           dst_stride1);
825
          else
826
827
828
            dec_extend_dir(pbi, xd, tile, 0, subsize, top_bsize, mi_row + hbs,
                           mi_col, mi_row_top, mi_col_top, dst_buf1,
                           dst_stride1, 1);
829
830
831
832
833

          // weighted average to smooth the boundary
          for (i = 0; i < MAX_MB_PLANE; i++) {
            xd->plane[i].dst.buf = dst_buf[i];
            xd->plane[i].dst.stride = dst_stride[i];
Yaowu Xu's avatar
Yaowu Xu committed
834
            av1_build_masked_inter_predictor_complex(
835
                xd, dst_buf[i], dst_stride[i], dst_buf1[i], dst_stride1[i],
836
837
                mi_row, mi_col, mi_row_top, mi_col_top, bsize, top_bsize,
                PARTITION_HORZ, i);
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
          }
        }
      }
      break;
    case PARTITION_VERT:
      if (bsize == BLOCK_8X8) {
        // First half
        dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf, dst_stride,
                             top_bsize, BLOCK_8X8, 1, 0);
        if (bsize < top_bsize)
          dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf, dst_stride);

        // Second half
        dec_predict_b_extend(pbi, xd, tile, 1, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf1, dst_stride1,
                             top_bsize, BLOCK_8X8, 1, 1);
        if (bsize < top_bsize)
          dec_extend_all(pbi, xd, tile, 1, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf1, dst_stride1);

        // Smooth
        xd->plane[0].dst.buf = dst_buf[0];
        xd->plane[0].dst.stride = dst_stride[0];
Yaowu Xu's avatar
Yaowu Xu committed
863
        av1_build_masked_inter_predictor_complex(
864
865
866
            xd, dst_buf[0], dst_stride[0], dst_buf1[0], dst_stride1[0], mi_row,
            mi_col, mi_row_top, mi_col_top, bsize, top_bsize, PARTITION_VERT,
            0);
867
868
869
870
871
872
873
874
875
876
877
878
879
880
      } else {
        // First half
        dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf, dst_stride,
                             top_bsize, subsize, 0, 0);
        if (bsize < top_bsize)
          dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf, dst_stride);
        else
          dec_extend_dir(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf, dst_stride, 3);

        // Second half
        if (mi_col + hbs < cm->mi_cols) {
881
882
883
          dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col + hbs, mi_row,
                               mi_col + hbs, mi_row_top, mi_col_top, dst_buf1,
                               dst_stride1, top_bsize, subsize, 0, 0);
884
          if (bsize < top_bsize)
885
886
887
            dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row,
                           mi_col + hbs, mi_row_top, mi_col_top, dst_buf1,
                           dst_stride1);
888
          else
889
890
891
            dec_extend_dir(pbi, xd, tile, 0, subsize, top_bsize, mi_row,
                           mi_col + hbs, mi_row_top, mi_col_top, dst_buf1,
                           dst_stride1, 2);
892
893
894
895
896

          // Smooth
          for (i = 0; i < MAX_MB_PLANE; i++) {
            xd->plane[i].dst.buf = dst_buf[i];
            xd->plane[i].dst.stride = dst_stride[i];
Yaowu Xu's avatar
Yaowu Xu committed
897
            av1_build_masked_inter_predictor_complex(
898
                xd, dst_buf[i], dst_stride[i], dst_buf1[i], dst_stride1[i],
899
900
                mi_row, mi_col, mi_row_top, mi_col_top, bsize, top_bsize,
                PARTITION_VERT, i);
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
          }
        }
      }
      break;
    case PARTITION_SPLIT:
      if (bsize == BLOCK_8X8) {
        dec_predict_b_extend(pbi, xd, tile, 0, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf, dst_stride,
                             top_bsize, BLOCK_8X8, 1, 0);
        dec_predict_b_extend(pbi, xd, tile, 1, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf1, dst_stride1,
                             top_bsize, BLOCK_8X8, 1, 1);
        dec_predict_b_extend(pbi, xd, tile, 2, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf2, dst_stride2,
                             top_bsize, BLOCK_8X8, 1, 1);
        dec_predict_b_extend(pbi, xd, tile, 3, mi_row, mi_col, mi_row, mi_col,
                             mi_row_top, mi_col_top, dst_buf3, dst_stride3,
                             top_bsize, BLOCK_8X8, 1, 1);
        if (bsize < top_bsize) {
          dec_extend_all(pbi, xd, tile, 0, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf, dst_stride);
          dec_extend_all(pbi, xd, tile, 1, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf1, dst_stride1);
          dec_extend_all(pbi, xd, tile, 2, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf2, dst_stride2);
          dec_extend_all(pbi, xd, tile, 3, subsize, top_bsize, mi_row, mi_col,
                         mi_row_top, mi_col_top, dst_buf3, dst_stride3);
        }
      } else {
930
931
932
        dec_predict_sb_complex(pbi, xd, tile, mi_row, mi_col, mi_row_top,
                               mi_col_top, subsize, top_bsize, dst_buf,
                               dst_stride);
933
934
935
936
937
938
939
940
941
942
943
944
945
        if (mi_row < cm->mi_rows && mi_col + hbs < cm->mi_cols)
          dec_predict_sb_complex(pbi, xd, tile, mi_row, mi_col + hbs,
                                 mi_row_top, mi_col_top, subsize, top_bsize,
                                 dst_buf1, dst_stride1);
        if (mi_row + hbs < cm->mi_rows && mi_col < cm->mi_cols)
          dec_predict_sb_complex(pbi, xd, tile, mi_row + hbs, mi_col,
                                 mi_row_top, mi_col_top, subsize, top_bsize,
                                 dst_buf2, dst_stride2);
        if (mi_row + hbs < cm->mi_rows && mi_col + hbs < cm->mi_cols)
          dec_predict_sb_complex(pbi, xd, tile, mi_row + hbs, mi_col + hbs,
                                 mi_row_top, mi_col_top, subsize, top_bsize,
                                 dst_buf3, dst_stride3);
      }
946
947
948
949
      for (i = 0; i < MAX_MB_PLANE; i++) {
        if (bsize == BLOCK_8X8 && i != 0)
          continue;  // Skip <4x4 chroma smoothing
        if (mi_row < cm->mi_rows && mi_col + hbs < cm->mi_cols) {
Yaowu Xu's avatar
Yaowu Xu committed
950
          av1_build_masked_inter_predictor_complex(
951
952
953
954
              xd, dst_buf[i], dst_stride[i], dst_buf1[i], dst_stride1[i],
              mi_row, mi_col, mi_row_top, mi_col_top, bsize, top_bsize,
              PARTITION_VERT, i);
          if (mi_row + hbs < cm->mi_rows) {
Yaowu Xu's avatar
Yaowu Xu committed