decodeframe.c 187 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4
5
6
7
8
9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10
11
12
13
14
 */

#include <assert.h>
#include <stdlib.h>  // qsort()

Yaowu Xu's avatar
Yaowu Xu committed
15
16
17
#include "./aom_config.h"
#include "./aom_dsp_rtcd.h"
#include "./aom_scale_rtcd.h"
Jingning Han's avatar
Jingning Han committed
18
#include "./av1_rtcd.h"
Jingning Han's avatar
Jingning Han committed
19

20
#include "aom/aom_codec.h"
Yaowu Xu's avatar
Yaowu Xu committed
21
#include "aom_dsp/aom_dsp_common.h"
Jingning Han's avatar
Jingning Han committed
22
23
#include "aom_dsp/bitreader.h"
#include "aom_dsp/bitreader_buffer.h"
Yaowu Xu's avatar
Yaowu Xu committed
24
#include "aom_mem/aom_mem.h"
25
26
#include "aom_ports/mem.h"
#include "aom_ports/mem_ops.h"
Yaowu Xu's avatar
Yaowu Xu committed
27
28
#include "aom_scale/aom_scale.h"
#include "aom_util/aom_thread.h"
Jingning Han's avatar
Jingning Han committed
29

30
#include "av1/common/alloccommon.h"
31
#if CONFIG_CDEF
32
#include "av1/common/cdef.h"
33
#include "av1/common/clpf.h"
34
#endif
35
36
37
#if CONFIG_INSPECTION
#include "av1/decoder/inspection.h"
#endif
38
39
40
#include "av1/common/common.h"
#include "av1/common/entropy.h"
#include "av1/common/entropymode.h"
41
#include "av1/common/entropymv.h"
42
43
44
45
#include "av1/common/idct.h"
#include "av1/common/pred_common.h"
#include "av1/common/quant_common.h"
#include "av1/common/reconinter.h"
Jingning Han's avatar
Jingning Han committed
46
#include "av1/common/reconintra.h"
47
#include "av1/common/seg_common.h"
Jingning Han's avatar
Jingning Han committed
48
#include "av1/common/thread_common.h"
49
#include "av1/common/tile_common.h"
50

51
52
53
#include "av1/decoder/decodeframe.h"
#include "av1/decoder/decodemv.h"
#include "av1/decoder/decoder.h"
Angie Chiang's avatar
Angie Chiang committed
54
55
56
#if CONFIG_LV_MAP
#include "av1/decoder/decodetxb.h"
#endif
Jingning Han's avatar
Jingning Han committed
57
#include "av1/decoder/detokenize.h"
58
#include "av1/decoder/dsubexp.h"
Jingning Han's avatar
Jingning Han committed
59

Yue Chen's avatar
Yue Chen committed
60
61
62
63
#if CONFIG_WARPED_MOTION
#include "av1/common/warped_motion.h"
#endif  // CONFIG_WARPED_MOTION

Yaowu Xu's avatar
Yaowu Xu committed
64
#define MAX_AV1_HEADER_SIZE 80
Michael Bebenita's avatar
Michael Bebenita committed
65
#define ACCT_STR __func__
Jingning Han's avatar
Jingning Han committed
66

67
68
#if CONFIG_PVQ
#include "av1/common/partition.h"
hui su's avatar
hui su committed
69
70
#include "av1/common/pvq.h"
#include "av1/common/scan.h"
71
#include "av1/decoder/decint.h"
hui su's avatar
hui su committed
72
73
#include "av1/decoder/pvq_decoder.h"
#include "av1/encoder/encodemb.h"
74
75
76
#include "av1/encoder/hybrid_fwd_txfm.h"
#endif

77
78
79
80
81
82
83
84
static struct aom_read_bit_buffer *init_read_bit_buffer(
    AV1Decoder *pbi, struct aom_read_bit_buffer *rb, const uint8_t *data,
    const uint8_t *data_end, uint8_t clear_data[MAX_AV1_HEADER_SIZE]);
static int read_compressed_header(AV1Decoder *pbi, const uint8_t *data,
                                  size_t partition_size);
static size_t read_uncompressed_header(AV1Decoder *pbi,
                                       struct aom_read_bit_buffer *rb);

Yaowu Xu's avatar
Yaowu Xu committed
85
static int is_compound_reference_allowed(const AV1_COMMON *cm) {
Jingning Han's avatar
Jingning Han committed
86
  int i;
87
  if (frame_is_intra_only(cm)) return 0;
88
  for (i = 1; i < INTER_REFS_PER_FRAME; ++i)
89
    if (cm->ref_frame_sign_bias[i + 1] != cm->ref_frame_sign_bias[1]) return 1;
Jingning Han's avatar
Jingning Han committed
90
91
92
93

  return 0;
}

Yaowu Xu's avatar
Yaowu Xu committed
94
static void setup_compound_reference_mode(AV1_COMMON *cm) {
95
#if CONFIG_EXT_REFS
96
  cm->comp_fwd_ref[0] = LAST_FRAME;
97
98
99
100
  cm->comp_fwd_ref[1] = LAST2_FRAME;
  cm->comp_fwd_ref[2] = LAST3_FRAME;
  cm->comp_fwd_ref[3] = GOLDEN_FRAME;

101
102
  cm->comp_bwd_ref[0] = BWDREF_FRAME;
  cm->comp_bwd_ref[1] = ALTREF_FRAME;
103
#else
Jingning Han's avatar
Jingning Han committed
104
  if (cm->ref_frame_sign_bias[LAST_FRAME] ==
105
      cm->ref_frame_sign_bias[GOLDEN_FRAME]) {
Jingning Han's avatar
Jingning Han committed
106
107
108
109
    cm->comp_fixed_ref = ALTREF_FRAME;
    cm->comp_var_ref[0] = LAST_FRAME;
    cm->comp_var_ref[1] = GOLDEN_FRAME;
  } else if (cm->ref_frame_sign_bias[LAST_FRAME] ==
110
             cm->ref_frame_sign_bias[ALTREF_FRAME]) {
Jingning Han's avatar
Jingning Han committed
111
112
113
114
115
116
117
118
    cm->comp_fixed_ref = GOLDEN_FRAME;
    cm->comp_var_ref[0] = LAST_FRAME;
    cm->comp_var_ref[1] = ALTREF_FRAME;
  } else {
    cm->comp_fixed_ref = LAST_FRAME;
    cm->comp_var_ref[0] = GOLDEN_FRAME;
    cm->comp_var_ref[1] = ALTREF_FRAME;
  }
119
#endif  // CONFIG_EXT_REFS
Jingning Han's avatar
Jingning Han committed
120
121
122
123
124
125
}

static int read_is_valid(const uint8_t *start, size_t len, const uint8_t *end) {
  return len != 0 && len <= (size_t)(end - start);
}

Yaowu Xu's avatar
Yaowu Xu committed
126
127
static int decode_unsigned_max(struct aom_read_bit_buffer *rb, int max) {
  const int data = aom_rb_read_literal(rb, get_unsigned_bits(max));
Jingning Han's avatar
Jingning Han committed
128
129
130
  return data > max ? max : data;
}

131
132
133
static TX_MODE read_tx_mode(AV1_COMMON *cm, MACROBLOCKD *xd,
                            struct aom_read_bit_buffer *rb) {
  int i, all_lossless = 1;
134
#if CONFIG_TX64X64
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
  TX_MODE tx_mode;
#endif

  if (cm->seg.enabled) {
    for (i = 0; i < MAX_SEGMENTS; ++i) {
      if (!xd->lossless[i]) {
        all_lossless = 0;
        break;
      }
    }
  } else {
    all_lossless = xd->lossless[0];
  }

  if (all_lossless) return ONLY_4X4;
#if CONFIG_TX64X64
  tx_mode = aom_rb_read_bit(rb) ? TX_MODE_SELECT : aom_rb_read_literal(rb, 2);
152
153
154
  if (tx_mode == ALLOW_32X32) tx_mode += aom_rb_read_bit(rb);
  return tx_mode;
#else
Yaowu Xu's avatar
Yaowu Xu committed
155
  return aom_rb_read_bit(rb) ? TX_MODE_SELECT : aom_rb_read_literal(rb, 2);
156
#endif  // CONFIG_TX64X64
157
}
Jingning Han's avatar
Jingning Han committed
158

159
#if !CONFIG_EC_ADAPT
Yaowu Xu's avatar
Yaowu Xu committed
160
161
162
163
164
165
166
static void read_tx_size_probs(FRAME_CONTEXT *fc, aom_reader *r) {
  int i, j, k;
  for (i = 0; i < MAX_TX_DEPTH; ++i)
    for (j = 0; j < TX_SIZE_CONTEXTS; ++j)
      for (k = 0; k < i + 1; ++k)
        av1_diff_update_prob(r, &fc->tx_size_probs[i][j][k], ACCT_STR);
}
167
#endif
Yaowu Xu's avatar
Yaowu Xu committed
168

169
#if !CONFIG_EC_ADAPT
Yaowu Xu's avatar
Yaowu Xu committed
170
static void read_switchable_interp_probs(FRAME_CONTEXT *fc, aom_reader *r) {
Jingning Han's avatar
Jingning Han committed
171
  int i, j;
172
  for (j = 0; j < SWITCHABLE_FILTER_CONTEXTS; ++j) {
Jingning Han's avatar
Jingning Han committed
173
    for (i = 0; i < SWITCHABLE_FILTERS - 1; ++i)
Michael Bebenita's avatar
Michael Bebenita committed
174
      av1_diff_update_prob(r, &fc->switchable_interp_prob[j][i], ACCT_STR);
175
  }
Jingning Han's avatar
Jingning Han committed
176
}
177
#endif
Jingning Han's avatar
Jingning Han committed
178

Yaowu Xu's avatar
Yaowu Xu committed
179
static void read_inter_mode_probs(FRAME_CONTEXT *fc, aom_reader *r) {
180
#if CONFIG_REF_MV
181
  int i;
182
  for (i = 0; i < NEWMV_MODE_CONTEXTS; ++i)
Michael Bebenita's avatar
Michael Bebenita committed
183
    av1_diff_update_prob(r, &fc->newmv_prob[i], ACCT_STR);
184
  for (i = 0; i < ZEROMV_MODE_CONTEXTS; ++i)
Michael Bebenita's avatar
Michael Bebenita committed
185
    av1_diff_update_prob(r, &fc->zeromv_prob[i], ACCT_STR);
186
  for (i = 0; i < REFMV_MODE_CONTEXTS; ++i)
Michael Bebenita's avatar
Michael Bebenita committed
187
    av1_diff_update_prob(r, &fc->refmv_prob[i], ACCT_STR);
188
  for (i = 0; i < DRL_MODE_CONTEXTS; ++i)
Michael Bebenita's avatar
Michael Bebenita committed
189
    av1_diff_update_prob(r, &fc->drl_prob[i], ACCT_STR);
Yue Chen's avatar
Yue Chen committed
190
#if CONFIG_EXT_INTER
Michael Bebenita's avatar
Michael Bebenita committed
191
  av1_diff_update_prob(r, &fc->new2mv_prob, ACCT_STR);
Yue Chen's avatar
Yue Chen committed
192
#endif  // CONFIG_EXT_INTER
193
#else
194
#if !CONFIG_EC_ADAPT
195
  int i, j;
196
  for (i = 0; i < INTER_MODE_CONTEXTS; ++i) {
Jingning Han's avatar
Jingning Han committed
197
    for (j = 0; j < INTER_MODES - 1; ++j)
Michael Bebenita's avatar
Michael Bebenita committed
198
      av1_diff_update_prob(r, &fc->inter_mode_probs[i][j], ACCT_STR);
199
  }
200
201
202
#else
  (void)fc;
  (void)r;
203
#endif
204
#endif
Jingning Han's avatar
Jingning Han committed
205
206
}

207
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
208
static void read_inter_compound_mode_probs(FRAME_CONTEXT *fc, aom_reader *r) {
209
  int i, j;
Michael Bebenita's avatar
Michael Bebenita committed
210
  if (aom_read(r, GROUP_DIFF_UPDATE_PROB, ACCT_STR)) {
211
212
    for (j = 0; j < INTER_MODE_CONTEXTS; ++j) {
      for (i = 0; i < INTER_COMPOUND_MODES - 1; ++i) {
Michael Bebenita's avatar
Michael Bebenita committed
213
        av1_diff_update_prob(r, &fc->inter_compound_mode_probs[j][i], ACCT_STR);
214
215
216
217
218
      }
    }
  }
}
#endif  // CONFIG_EXT_INTER
219
#if !CONFIG_EC_ADAPT
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
#if !CONFIG_EXT_TX
static void read_ext_tx_probs(FRAME_CONTEXT *fc, aom_reader *r) {
  int i, j, k;
  if (aom_read(r, GROUP_DIFF_UPDATE_PROB, ACCT_STR)) {
    for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
      for (j = 0; j < TX_TYPES; ++j) {
        for (k = 0; k < TX_TYPES - 1; ++k)
          av1_diff_update_prob(r, &fc->intra_ext_tx_prob[i][j][k], ACCT_STR);
      }
    }
  }
  if (aom_read(r, GROUP_DIFF_UPDATE_PROB, ACCT_STR)) {
    for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
      for (k = 0; k < TX_TYPES - 1; ++k)
        av1_diff_update_prob(r, &fc->inter_ext_tx_prob[i][k], ACCT_STR);
    }
  }
}
#endif
#endif
240

241
static REFERENCE_MODE read_frame_reference_mode(
Yaowu Xu's avatar
Yaowu Xu committed
242
    const AV1_COMMON *cm, struct aom_read_bit_buffer *rb) {
243
  if (is_compound_reference_allowed(cm)) {
Zoe Liu's avatar
Zoe Liu committed
244
245
246
#if CONFIG_REF_ADAPT
    return aom_rb_read_bit(rb) ? REFERENCE_MODE_SELECT : SINGLE_REFERENCE;
#else
Yaowu Xu's avatar
Yaowu Xu committed
247
    return aom_rb_read_bit(rb)
248
               ? REFERENCE_MODE_SELECT
Yaowu Xu's avatar
Yaowu Xu committed
249
               : (aom_rb_read_bit(rb) ? COMPOUND_REFERENCE : SINGLE_REFERENCE);
Zoe Liu's avatar
Zoe Liu committed
250
#endif  // CONFIG_REF_ADAPT
251
252
253
254
  } else {
    return SINGLE_REFERENCE;
  }
}
Jingning Han's avatar
Jingning Han committed
255

Yaowu Xu's avatar
Yaowu Xu committed
256
static void read_frame_reference_mode_probs(AV1_COMMON *cm, aom_reader *r) {
Jingning Han's avatar
Jingning Han committed
257
  FRAME_CONTEXT *const fc = cm->fc;
258
  int i, j;
Jingning Han's avatar
Jingning Han committed
259
260
261

  if (cm->reference_mode == REFERENCE_MODE_SELECT)
    for (i = 0; i < COMP_INTER_CONTEXTS; ++i)
Michael Bebenita's avatar
Michael Bebenita committed
262
      av1_diff_update_prob(r, &fc->comp_inter_prob[i], ACCT_STR);
Jingning Han's avatar
Jingning Han committed
263

264
  if (cm->reference_mode != COMPOUND_REFERENCE) {
Jingning Han's avatar
Jingning Han committed
265
    for (i = 0; i < REF_CONTEXTS; ++i) {
266
      for (j = 0; j < (SINGLE_REFS - 1); ++j) {
Michael Bebenita's avatar
Michael Bebenita committed
267
        av1_diff_update_prob(r, &fc->single_ref_prob[i][j], ACCT_STR);
268
      }
Jingning Han's avatar
Jingning Han committed
269
    }
270
  }
Jingning Han's avatar
Jingning Han committed
271

272
273
  if (cm->reference_mode != SINGLE_REFERENCE) {
    for (i = 0; i < REF_CONTEXTS; ++i) {
274
#if CONFIG_EXT_REFS
275
      for (j = 0; j < (FWD_REFS - 1); ++j)
Michael Bebenita's avatar
Michael Bebenita committed
276
        av1_diff_update_prob(r, &fc->comp_ref_prob[i][j], ACCT_STR);
277
      for (j = 0; j < (BWD_REFS - 1); ++j)
Michael Bebenita's avatar
Michael Bebenita committed
278
        av1_diff_update_prob(r, &fc->comp_bwdref_prob[i][j], ACCT_STR);
279
280
#else
      for (j = 0; j < (COMP_REFS - 1); ++j)
Michael Bebenita's avatar
Michael Bebenita committed
281
        av1_diff_update_prob(r, &fc->comp_ref_prob[i][j], ACCT_STR);
282
#endif  // CONFIG_EXT_REFS
283
284
    }
  }
Jingning Han's avatar
Jingning Han committed
285
286
}

Yaowu Xu's avatar
Yaowu Xu committed
287
static void update_mv_probs(aom_prob *p, int n, aom_reader *r) {
Jingning Han's avatar
Jingning Han committed
288
  int i;
Michael Bebenita's avatar
Michael Bebenita committed
289
  for (i = 0; i < n; ++i) av1_diff_update_prob(r, &p[i], ACCT_STR);
Jingning Han's avatar
Jingning Han committed
290
291
}

Yaowu Xu's avatar
Yaowu Xu committed
292
static void read_mv_probs(nmv_context *ctx, int allow_hp, aom_reader *r) {
293
  int i;
Jingning Han's avatar
Jingning Han committed
294

Yaowu Xu's avatar
Yaowu Xu committed
295
#if !CONFIG_EC_ADAPT
296
  int j;
Jingning Han's avatar
Jingning Han committed
297
298
299
300
301
302
303
304
305
306
307
  update_mv_probs(ctx->joints, MV_JOINTS - 1, r);

  for (i = 0; i < 2; ++i) {
    nmv_component *const comp_ctx = &ctx->comps[i];
    update_mv_probs(&comp_ctx->sign, 1, r);
    update_mv_probs(comp_ctx->classes, MV_CLASSES - 1, r);
    update_mv_probs(comp_ctx->class0, CLASS0_SIZE - 1, r);
    update_mv_probs(comp_ctx->bits, MV_OFFSET_BITS, r);
  }
  for (i = 0; i < 2; ++i) {
    nmv_component *const comp_ctx = &ctx->comps[i];
308
    for (j = 0; j < CLASS0_SIZE; ++j) {
Jingning Han's avatar
Jingning Han committed
309
      update_mv_probs(comp_ctx->class0_fp[j], MV_FP_SIZE - 1, r);
310
    }
311
    update_mv_probs(comp_ctx->fp, MV_FP_SIZE - 1, r);
Jingning Han's avatar
Jingning Han committed
312
  }
313
#endif  // !CONFIG_EC_ADAPT
Jingning Han's avatar
Jingning Han committed
314
315
316
317
318
319
320
321
322
323

  if (allow_hp) {
    for (i = 0; i < 2; ++i) {
      nmv_component *const comp_ctx = &ctx->comps[i];
      update_mv_probs(&comp_ctx->class0_hp, 1, r);
      update_mv_probs(&comp_ctx->hp, 1, r);
    }
  }
}

324
static void inverse_transform_block(MACROBLOCKD *xd, int plane,
325
                                    const TX_TYPE tx_type,
326
                                    const TX_SIZE tx_size, uint8_t *dst,
327
                                    int stride, int16_t scan_line, int eob) {
Jingning Han's avatar
Jingning Han committed
328
  struct macroblockd_plane *const pd = &xd->plane[plane];
329
330
331
332
333
334
  tran_low_t *const dqcoeff = pd->dqcoeff;
  INV_TXFM_PARAM inv_txfm_param;
  inv_txfm_param.tx_type = tx_type;
  inv_txfm_param.tx_size = tx_size;
  inv_txfm_param.eob = eob;
  inv_txfm_param.lossless = xd->lossless[xd->mi[0]->mbmi.segment_id];
Angie Chiang's avatar
Angie Chiang committed
335

Yaowu Xu's avatar
Yaowu Xu committed
336
#if CONFIG_AOM_HIGHBITDEPTH
337
338
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
    inv_txfm_param.bd = xd->bd;
hui su's avatar
hui su committed
339
    av1_highbd_inv_txfm_add(dqcoeff, dst, stride, &inv_txfm_param);
340
  } else {
Yaowu Xu's avatar
Yaowu Xu committed
341
#endif  // CONFIG_AOM_HIGHBITDEPTH
hui su's avatar
hui su committed
342
    av1_inv_txfm_add(dqcoeff, dst, stride, &inv_txfm_param);
Yaowu Xu's avatar
Yaowu Xu committed
343
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
344
  }
345
346
#endif  // CONFIG_AOM_HIGHBITDEPTH
  memset(dqcoeff, 0, (scan_line + 1) * sizeof(dqcoeff[0]));
Jingning Han's avatar
Jingning Han committed
347
348
}

349
#if CONFIG_PVQ
Thomas Daede's avatar
Thomas Daede committed
350
static int av1_pvq_decode_helper(MACROBLOCKD *xd, tran_low_t *ref_coeff,
351
                                 tran_low_t *dqcoeff, int16_t *quant, int pli,
352
                                 int bs, TX_TYPE tx_type, int xdec,
ltrudeau's avatar
ltrudeau committed
353
                                 PVQ_SKIP_TYPE ac_dc_coded) {
354
355
356
357
  unsigned int flags;  // used for daala's stream analyzer.
  int off;
  const int is_keyframe = 0;
  const int has_dc_skip = 1;
358
  int coeff_shift = 3 - get_tx_scale(bs);
Thomas Daede's avatar
Thomas Daede committed
359
  int hbd_downshift = 0;
360
  int rounding_mask;
361
362
363
364
365
366
  // DC quantizer for PVQ
  int pvq_dc_quant;
  int lossless = (quant[0] == 0);
  const int blk_size = tx_size_wide[bs];
  int eob = 0;
  int i;
Thomas Daede's avatar
Thomas Daede committed
367
  od_dec_ctx *dec = &xd->daala_dec;
368
  int use_activity_masking = dec->use_activity_masking;
369
370
  DECLARE_ALIGNED(16, tran_low_t, dqcoeff_pvq[OD_TXSIZE_MAX * OD_TXSIZE_MAX]);
  DECLARE_ALIGNED(16, tran_low_t, ref_coeff_pvq[OD_TXSIZE_MAX * OD_TXSIZE_MAX]);
371

372
373
  od_coeff ref_int32[OD_TXSIZE_MAX * OD_TXSIZE_MAX];
  od_coeff out_int32[OD_TXSIZE_MAX * OD_TXSIZE_MAX];
374

Thomas Daede's avatar
Thomas Daede committed
375
376
377
378
#if CONFIG_AOM_HIGHBITDEPTH
  hbd_downshift = xd->bd - 8;
#endif  // CONFIG_AOM_HIGHBITDEPTH

379
380
381
  od_raster_to_coding_order(ref_coeff_pvq, blk_size, tx_type, ref_coeff,
                            blk_size);

Thomas Daede's avatar
Thomas Daede committed
382
  assert(OD_COEFF_SHIFT >= 4);
383
384
385
  if (lossless)
    pvq_dc_quant = 1;
  else {
386
387
    if (use_activity_masking)
      pvq_dc_quant = OD_MAXI(
Thomas Daede's avatar
Thomas Daede committed
388
          1, (quant[0] << (OD_COEFF_SHIFT - 3) >> hbd_downshift) *
389
390
391
                     dec->state.pvq_qm_q4[pli][od_qm_get_index(bs, 0)] >>
                 4);
    else
Thomas Daede's avatar
Thomas Daede committed
392
393
      pvq_dc_quant =
          OD_MAXI(1, quant[0] << (OD_COEFF_SHIFT - 3) >> hbd_downshift);
394
395
396
397
398
  }

  off = od_qm_offset(bs, xdec);

  // copy int16 inputs to int32
399
  for (i = 0; i < blk_size * blk_size; i++) {
400
    ref_int32[i] =
Thomas Daede's avatar
Thomas Daede committed
401
402
        AOM_SIGNED_SHL(ref_coeff_pvq[i], OD_COEFF_SHIFT - coeff_shift) >>
        hbd_downshift;
403
  }
404

Thomas Daede's avatar
Thomas Daede committed
405
406
  od_pvq_decode(dec, ref_int32, out_int32,
                OD_MAXI(1, quant[1] << (OD_COEFF_SHIFT - 3) >> hbd_downshift),
407
                pli, bs, OD_PVQ_BETA[use_activity_masking][pli][bs],
408
409
                OD_ROBUST_STREAM, is_keyframe, &flags, ac_dc_coded,
                dec->state.qm + off, dec->state.qm_inv + off);
410

411
412
  if (!has_dc_skip || out_int32[0]) {
    out_int32[0] =
413
414
415
        has_dc_skip + generic_decode(dec->r, &dec->state.adapt.model_dc[pli],
                                     -1, &dec->state.adapt.ex_dc[pli][bs][0], 2,
                                     "dc:mag");
416
417
418
419
420
421
422
423
    if (out_int32[0]) out_int32[0] *= aom_read_bit(dec->r, "dc:sign") ? -1 : 1;
  }
  out_int32[0] = out_int32[0] * pvq_dc_quant + ref_int32[0];

  // copy int32 result back to int16
  assert(OD_COEFF_SHIFT > coeff_shift);
  rounding_mask = (1 << (OD_COEFF_SHIFT - coeff_shift - 1)) - 1;
  for (i = 0; i < blk_size * blk_size; i++) {
Thomas Daede's avatar
Thomas Daede committed
424
    out_int32[i] = AOM_SIGNED_SHL(out_int32[i], hbd_downshift);
425
426
    dqcoeff_pvq[i] = (out_int32[i] + (out_int32[i] < 0) + rounding_mask) >>
                     (OD_COEFF_SHIFT - coeff_shift);
427
428
429
430
431
432
433
434
435
  }

  od_coding_order_to_raster(dqcoeff, blk_size, tx_type, dqcoeff_pvq, blk_size);

  eob = blk_size * blk_size;

  return eob;
}

ltrudeau's avatar
ltrudeau committed
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
static PVQ_SKIP_TYPE read_pvq_skip(AV1_COMMON *cm, MACROBLOCKD *const xd,
                                   int plane, TX_SIZE tx_size) {
  // decode ac/dc coded flag. bit0: DC coded, bit1 : AC coded
  // NOTE : we don't use 5 symbols for luma here in aom codebase,
  // since block partition is taken care of by aom.
  // So, only AC/DC skip info is coded
  const int ac_dc_coded = aom_decode_cdf_adapt(
      xd->daala_dec.r,
      xd->daala_dec.state.adapt.skip_cdf[2 * tx_size + (plane != 0)], 4,
      xd->daala_dec.state.adapt.skip_increment, "skip");
  if (ac_dc_coded < 0 || ac_dc_coded > 3) {
    aom_internal_error(&cm->error, AOM_CODEC_INVALID_PARAM,
                       "Invalid PVQ Skip Type");
  }
  return ac_dc_coded;
}

static int av1_pvq_decode_helper2(AV1_COMMON *cm, MACROBLOCKD *const xd,
454
455
                                  MB_MODE_INFO *const mbmi, int plane, int row,
                                  int col, TX_SIZE tx_size, TX_TYPE tx_type) {
456
457
458
459
460
461
462
463
464
465
  struct macroblockd_plane *const pd = &xd->plane[plane];
  // transform block size in pixels
  int tx_blk_size = tx_size_wide[tx_size];
  int i, j;
  tran_low_t *pvq_ref_coeff = pd->pvq_ref_coeff;
  const int diff_stride = tx_blk_size;
  int16_t *pred = pd->pred;
  tran_low_t *const dqcoeff = pd->dqcoeff;
  uint8_t *dst;
  int eob;
ltrudeau's avatar
ltrudeau committed
466
  const PVQ_SKIP_TYPE ac_dc_coded = read_pvq_skip(cm, xd, plane, tx_size);
467
468
469
470
471
472
473
474
475

  eob = 0;
  dst = &pd->dst.buf[4 * row * pd->dst.stride + 4 * col];

  if (ac_dc_coded) {
    int xdec = pd->subsampling_x;
    int seg_id = mbmi->segment_id;
    int16_t *quant;
    FWD_TXFM_PARAM fwd_txfm_param;
476
477
    // ToDo(yaowu): correct this with optimal number from decoding process.
    const int max_scan_line = tx_size_2d[tx_size];
Thomas Daede's avatar
Thomas Daede committed
478
479
480
481
482
483
484
485
486
487
488
489
490
491
#if CONFIG_AOM_HIGHBITDEPTH
    if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
      for (j = 0; j < tx_blk_size; j++)
        for (i = 0; i < tx_blk_size; i++)
          pred[diff_stride * j + i] =
              CONVERT_TO_SHORTPTR(dst)[pd->dst.stride * j + i];
    } else {
#endif
      for (j = 0; j < tx_blk_size; j++)
        for (i = 0; i < tx_blk_size; i++)
          pred[diff_stride * j + i] = dst[pd->dst.stride * j + i];
#if CONFIG_AOM_HIGHBITDEPTH
    }
#endif
492
493
494
495
496

    fwd_txfm_param.tx_type = tx_type;
    fwd_txfm_param.tx_size = tx_size;
    fwd_txfm_param.lossless = xd->lossless[seg_id];

Thomas Daede's avatar
Thomas Daede committed
497
498
499
500
501
502
503
504
505
506
#if CONFIG_AOM_HIGHBITDEPTH
    if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
      fwd_txfm_param.bd = xd->bd;
      highbd_fwd_txfm(pred, pvq_ref_coeff, diff_stride, &fwd_txfm_param);
    } else {
#endif  // CONFIG_AOM_HIGHBITDEPTH
      fwd_txfm(pred, pvq_ref_coeff, diff_stride, &fwd_txfm_param);
#if CONFIG_AOM_HIGHBITDEPTH
    }
#endif  // CONFIG_AOM_HIGHBITDEPTH
507
508
509

    quant = &pd->seg_dequant[seg_id][0];  // aom's quantizer

Thomas Daede's avatar
Thomas Daede committed
510
511
    eob = av1_pvq_decode_helper(xd, pvq_ref_coeff, dqcoeff, quant, plane,
                                tx_size, tx_type, xdec, ac_dc_coded);
512

Thomas Daede's avatar
Thomas Daede committed
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
// Since av1 does not have separate inverse transform
// but also contains adding to predicted image,
// pass blank dummy image to av1_inv_txfm_add_*x*(), i.e. set dst as zeros
#if CONFIG_AOM_HIGHBITDEPTH
    if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
      for (j = 0; j < tx_blk_size; j++)
        for (i = 0; i < tx_blk_size; i++)
          CONVERT_TO_SHORTPTR(dst)[j * pd->dst.stride + i] = 0;
    } else {
#endif
      for (j = 0; j < tx_blk_size; j++)
        for (i = 0; i < tx_blk_size; i++) dst[j * pd->dst.stride + i] = 0;
#if CONFIG_AOM_HIGHBITDEPTH
    }
#endif
528

529
530
    inverse_transform_block(xd, plane, tx_type, tx_size, dst, pd->dst.stride,
                            max_scan_line, eob);
531
532
533
534
535
536
  }

  return eob;
}
#endif

537
538
539
static void predict_and_reconstruct_intra_block(
    AV1_COMMON *cm, MACROBLOCKD *const xd, aom_reader *const r,
    MB_MODE_INFO *const mbmi, int plane, int row, int col, TX_SIZE tx_size) {
Jingning Han's avatar
Jingning Han committed
540
541
  struct macroblockd_plane *const pd = &xd->plane[plane];
  PREDICTION_MODE mode = (plane == 0) ? mbmi->mode : mbmi->uv_mode;
542
  PLANE_TYPE plane_type = get_plane_type(plane);
Jingning Han's avatar
Jingning Han committed
543
  uint8_t *dst;
544
  const int block_idx = (row << 1) + col;
545
546
547
#if CONFIG_PVQ
  (void)r;
#endif
548
  dst = &pd->dst.buf[(row * pd->dst.stride + col) << tx_size_wide_log2[0]];
Jingning Han's avatar
Jingning Han committed
549

Jingning Han's avatar
Jingning Han committed
550
#if !CONFIG_CB4X4
Jingning Han's avatar
Jingning Han committed
551
  if (mbmi->sb_type < BLOCK_8X8)
552
    if (plane == 0) mode = xd->mi[0]->bmi[block_idx].as_mode;
Jingning Han's avatar
Jingning Han committed
553
#endif
554
555
556
  av1_predict_intra_block(xd, pd->width, pd->height, txsize_to_bsize[tx_size],
                          mode, dst, pd->dst.stride, dst, pd->dst.stride, col,
                          row, plane);
Jingning Han's avatar
Jingning Han committed
557
558

  if (!mbmi->skip) {
hui su's avatar
hui su committed
559
    TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, tx_size);
560
#if !CONFIG_PVQ
Angie Chiang's avatar
Angie Chiang committed
561
562
#if CONFIG_LV_MAP
    int16_t max_scan_line = 0;
563
564
565
    int eob;
    av1_read_coeffs_txb_facade(cm, xd, r, row, col, block_idx, plane,
                               pd->dqcoeff, &max_scan_line, &eob);
Angie Chiang's avatar
Angie Chiang committed
566
#else   // CONFIG_LV_MAP
Angie Chiang's avatar
Angie Chiang committed
567
    const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, 0);
568
569
570
571
    int16_t max_scan_line = 0;
    const int eob =
        av1_decode_block_tokens(xd, plane, scan_order, col, row, tx_size,
                                tx_type, &max_scan_line, r, mbmi->segment_id);
Angie Chiang's avatar
Angie Chiang committed
572
#endif  // CONFIG_LV_MAP
Angie Chiang's avatar
Angie Chiang committed
573
#if CONFIG_ADAPT_SCAN
574
575
576
    if (xd->counts)
      av1_update_scan_count_facade(cm, xd->counts, tx_size, tx_type,
                                   pd->dqcoeff, eob);
Angie Chiang's avatar
Angie Chiang committed
577
#endif
578
579
580
    if (eob)
      inverse_transform_block(xd, plane, tx_type, tx_size, dst, pd->dst.stride,
                              max_scan_line, eob);
581
#else
ltrudeau's avatar
ltrudeau committed
582
    av1_pvq_decode_helper2(cm, xd, mbmi, plane, row, col, tx_size, tx_type);
583
#endif
Jingning Han's avatar
Jingning Han committed
584
585
586
  }
}

587
#if CONFIG_VAR_TX && !CONFIG_COEF_INTERLEAVE
Angie Chiang's avatar
Angie Chiang committed
588
589
static void decode_reconstruct_tx(AV1_COMMON *cm, MACROBLOCKD *const xd,
                                  aom_reader *r, MB_MODE_INFO *const mbmi,
590
                                  int plane, BLOCK_SIZE plane_bsize,
591
592
                                  int blk_row, int blk_col, TX_SIZE tx_size,
                                  int *eob_total) {
593
  const struct macroblockd_plane *const pd = &xd->plane[plane];
594
  const BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
595
596
  const int tx_row = blk_row >> (1 - pd->subsampling_y);
  const int tx_col = blk_col >> (1 - pd->subsampling_x);
597
  const TX_SIZE plane_tx_size =
598
      plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0]
599
            : mbmi->inter_tx_size[tx_row][tx_col];
600
  // Scale to match transform block unit.
601
602
  const int max_blocks_high = max_block_high(xd, plane_bsize, plane);
  const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane);
603

604
  if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
605

606
  if (tx_size == plane_tx_size) {
607
    PLANE_TYPE plane_type = get_plane_type(plane);
608
609
    int block_idx = (blk_row << 1) + blk_col;
    TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, plane_tx_size);
Angie Chiang's avatar
Angie Chiang committed
610
611
612
#if CONFIG_LV_MAP
    (void)segment_id;
    int16_t max_scan_line = 0;
613
614
615
    int eob;
    av1_read_coeffs_txb_facade(cm, xd, r, row, col, block_idx, plane,
                               pd->dqcoeff, &max_scan_line, &eob);
Angie Chiang's avatar
Angie Chiang committed
616
#else   // CONFIG_LV_MAP
Angie Chiang's avatar
Angie Chiang committed
617
    const SCAN_ORDER *sc = get_scan(cm, plane_tx_size, tx_type, 1);
618
    int16_t max_scan_line = 0;
619
    const int eob =
Yaowu Xu's avatar
Yaowu Xu committed
620
        av1_decode_block_tokens(xd, plane, sc, blk_col, blk_row, plane_tx_size,
621
                                tx_type, &max_scan_line, r, mbmi->segment_id);
Angie Chiang's avatar
Angie Chiang committed
622
#endif  // CONFIG_LV_MAP
623
#if CONFIG_ADAPT_SCAN
624
625
626
    if (xd->counts)
      av1_update_scan_count_facade(cm, xd->counts, tx_size, tx_type,
                                   pd->dqcoeff, eob);
627
#endif
628
629
630
631
    inverse_transform_block(xd, plane, tx_type, plane_tx_size,
                            &pd->dst.buf[(blk_row * pd->dst.stride + blk_col)
                                         << tx_size_wide_log2[0]],
                            pd->dst.stride, max_scan_line, eob);
632
633
    *eob_total += eob;
  } else {
634
635
    const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
    const int bsl = tx_size_wide_unit[sub_txs];
636
637
638
639
640
    int i;

    assert(bsl > 0);

    for (i = 0; i < 4; ++i) {
641
642
      const int offsetr = blk_row + (i >> 1) * bsl;
      const int offsetc = blk_col + (i & 0x01) * bsl;
643

644
      if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;
645

646
      decode_reconstruct_tx(cm, xd, r, mbmi, plane, plane_bsize, offsetr,
647
                            offsetc, sub_txs, eob_total);
648
649
650
    }
  }
}
651
#endif  // CONFIG_VAR_TX
652

653
#if !CONFIG_VAR_TX || CONFIG_SUPERTX || CONFIG_COEF_INTERLEAVE || \
654
    (!CONFIG_VAR_TX && CONFIG_EXT_TX && CONFIG_RECT_TX)
Angie Chiang's avatar
Angie Chiang committed
655
static int reconstruct_inter_block(AV1_COMMON *cm, MACROBLOCKD *const xd,
656
657
                                   aom_reader *const r, int segment_id,
                                   int plane, int row, int col,
658
                                   TX_SIZE tx_size) {
659
  PLANE_TYPE plane_type = get_plane_type(plane);
hui su's avatar
hui su committed
660
  int block_idx = (row << 1) + col;
hui su's avatar
hui su committed
661
  TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, tx_size);
662
663
664
#if CONFIG_PVQ
  int eob;
  (void)r;
665
666
667
  (void)segment_id;
#else
  struct macroblockd_plane *const pd = &xd->plane[plane];
668
669
670
#endif

#if !CONFIG_PVQ
Angie Chiang's avatar
Angie Chiang committed
671
672
#if CONFIG_LV_MAP
  (void)segment_id;
673
  int16_t max_scan_line = 0;
674
675
676
  int eob;
  av1_read_coeffs_txb_facade(cm, xd, r, row, col, block_idx, plane, pd->dqcoeff,
                             &max_scan_line, &eob);
Angie Chiang's avatar
Angie Chiang committed
677
678
679
#else   // CONFIG_LV_MAP
  int16_t max_scan_line = 0;
  const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, 1);
680
681
682
  const int eob =
      av1_decode_block_tokens(xd, plane, scan_order, col, row, tx_size, tx_type,
                              &max_scan_line, r, segment_id);
Angie Chiang's avatar
Angie Chiang committed
683
#endif  // CONFIG_LV_MAP
684
685
  uint8_t *dst =
      &pd->dst.buf[(row * pd->dst.stride + col) << tx_size_wide_log2[0]];
Angie Chiang's avatar
Angie Chiang committed
686
#if CONFIG_ADAPT_SCAN
687
688
689
  if (xd->counts)
    av1_update_scan_count_facade(cm, xd->counts, tx_size, tx_type, pd->dqcoeff,
                                 eob);
Angie Chiang's avatar
Angie Chiang committed
690
#endif
691
  if (eob)
692
693
    inverse_transform_block(xd, plane, tx_type, tx_size, dst, pd->dst.stride,
                            max_scan_line, eob);
694
#else
ltrudeau's avatar
ltrudeau committed
695
696
  eob = av1_pvq_decode_helper2(cm, xd, &xd->mi[0]->mbmi, plane, row, col,
                               tx_size, tx_type);
697
#endif
Jingning Han's avatar
Jingning Han committed
698
699
  return eob;
}
700
#endif  // !CONFIG_VAR_TX || CONFIG_SUPER_TX
Jingning Han's avatar
Jingning Han committed
701

702
703
704
static void set_offsets(AV1_COMMON *const cm, MACROBLOCKD *const xd,
                        BLOCK_SIZE bsize, int mi_row, int mi_col, int bw,
                        int bh, int x_mis, int y_mis) {
Jingning Han's avatar
Jingning Han committed
705
706
707
708
709
710
711
712
713
  const int offset = mi_row * cm->mi_stride + mi_col;
  int x, y;
  const TileInfo *const tile = &xd->tile;

  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = &cm->mi[offset];
  // TODO(slavarnway): Generate sb_type based on bwl and bhl, instead of
  // passing bsize from decode_partition().
  xd->mi[0]->mbmi.sb_type = bsize;
714
715
716
717
#if CONFIG_RD_DEBUG
  xd->mi[0]->mbmi.mi_row = mi_row;
  xd->mi[0]->mbmi.mi_col = mi_col;
#endif
Jingning Han's avatar
Jingning Han committed
718
  for (y = 0; y < y_mis; ++y)
Jingning Han's avatar
Jingning Han committed
719
    for (x = !y; x < x_mis; ++x) xd->mi[y * cm->mi_stride + x] = xd->mi[0];
Jingning Han's avatar
Jingning Han committed
720

Jingning Han's avatar
Jingning Han committed
721
  set_plane_n4(xd, bw, bh);
Jingning Han's avatar
Jingning Han committed
722
723
  set_skip_context(xd, mi_row, mi_col);

724
725
726
727
#if CONFIG_VAR_TX
  xd->max_tx_size = max_txsize_lookup[bsize];
#endif

728
729
730
731
732
733
// Distance of Mb to the various image edges. These are specified to 8th pel
// as they are always compared to values that are in 1/8th pel units
#if CONFIG_DEPENDENT_HORZTILES
  set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, cm->mi_rows, cm->mi_cols,
                 cm->dependent_horz_tiles);
#else
Jingning Han's avatar
Jingning Han committed
734
  set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, cm->mi_rows, cm->mi_cols);
735
#endif
Jingning Han's avatar
Jingning Han committed
736

Yaowu Xu's avatar
Yaowu Xu committed
737
  av1_setup_dst_planes(xd->plane, get_frame_new_buffer(cm), mi_row, mi_col);
Jingning Han's avatar
Jingning Han committed
738
739
}

740
#if CONFIG_SUPERTX
Yaowu Xu's avatar
Yaowu Xu committed
741
static MB_MODE_INFO *set_offsets_extend(AV1_COMMON *const cm,
742
743
                                        MACROBLOCKD *const xd,
                                        const TileInfo *const tile,
744
745
746
                                        BLOCK_SIZE bsize_pred, int mi_row_pred,
                                        int mi_col_pred, int mi_row_ori,
                                        int mi_col_ori) {
747
748
749
  // Used in supertx
  // (mi_row_ori, mi_col_ori): location for mv
  // (mi_row_pred, mi_col_pred, bsize_pred): region to predict
Jingning Han's avatar
Jingning Han committed
750
751
  const int bw = mi_size_wide[bsize_pred];
  const int bh = mi_size_high[bsize_pred];
752
753
754
  const int offset = mi_row_ori * cm->mi_stride + mi_col_ori;
  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = cm->mi + offset;
755
756
757
758
#if CONFIG_DEPENDENT_HORZTILES
  set_mi_row_col(xd, tile, mi_row_pred, bh, mi_col_pred, bw, cm->mi_rows,
                 cm->mi_cols, cm->dependent_horz_tiles);
#else
759
760
  set_mi_row_col(xd, tile, mi_row_pred, bh, mi_col_pred, bw, cm->mi_rows,
                 cm->mi_cols);
761
#endif
762

763
764
  xd->up_available = (mi_row_ori > tile->mi_row_start);
  xd->left_available = (mi_col_ori > tile->mi_col_start);
765

Jingning Han's avatar
Jingning Han committed
766
  set_plane_n4(xd, bw, bh);
767
768
769
770

  return &xd->mi[0]->mbmi;
}

Angie Chiang's avatar
Angie Chiang committed
771
#if CONFIG_SUPERTX
Yaowu Xu's avatar
Yaowu Xu committed
772
773
774
static MB_MODE_INFO *set_mb_offsets(AV1_COMMON *const cm, MACROBLOCKD *const xd,
                                    BLOCK_SIZE bsize, int mi_row, int mi_col,
                                    int bw, int bh, int x_mis, int y_mis) {
775
776
777
778
779
780
781
782
  const int offset = mi_row * cm->mi_stride + mi_col;
  const TileInfo *const tile = &xd->tile;
  int x, y;

  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = cm->mi + offset;
  xd->mi[0]->mbmi.sb_type = bsize;
  for (y = 0; y < y_mis; ++y)
783
    for (x = !y; x < x_mis; ++x) xd->mi[y * cm->mi_stride + x] = xd->mi[0];
784

785
786
787
788
#if CONFIG_DEPENDENT_HORZTILES
  set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, cm->mi_rows, cm->mi_cols,
                 cm->dependent_horz_tiles);
#else
789
  set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, cm->mi_rows, cm->mi_cols);
790
#endif
791
792
  return &xd->mi[0]->mbmi;
}
Angie Chiang's avatar
Angie Chiang committed
793
#endif
794

Yaowu Xu's avatar
Yaowu Xu committed
795
static void set_offsets_topblock(AV1_COMMON *const cm, MACROBLOCKD *const xd,
796
797
                                 const TileInfo *const tile, BLOCK_SIZE bsize,
                                 int mi_row, int mi_col) {
Jingning Han's avatar
Jingning Han committed
798
799
  const int bw = mi_size_wide[bsize];
  const int bh = mi_size_high[bsize];
800
801
802
803
804
  const int offset = mi_row * cm->mi_stride + mi_col;

  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = cm->mi + offset;

Jingning Han's avatar
Jingning Han committed
805
  set_plane_n4(xd, bw, bh);
806

807
808
809
810
#if CONFIG_DEPENDENT_HORZTILES
  set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, cm->mi_rows, cm->mi_cols,
                 cm->dependent_horz_tiles);
#else
811
  set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, cm->mi_rows, cm->mi_cols);
812
#endif
813

Yaowu Xu's avatar
Yaowu Xu committed
814
  av1_setup_dst_planes(xd->plane, get_frame_new_buffer(cm), mi_row, mi_col);
815
816
}

Yaowu Xu's avatar
Yaowu Xu committed
817
static void set_param_topblock(AV1_COMMON *const cm, MACROBLOCKD *const xd,
818
                               BLOCK_SIZE bsize, int mi_row, int mi_col,
819
                               int txfm, int skip) {
Jingning Han's avatar
Jingning Han committed
820
821
  const int bw = mi_size_wide[bsize];
  const int bh = mi_size_high[bsize];
Yaowu Xu's avatar
Yaowu Xu committed
822
823
  const int x_mis = AOMMIN(bw, cm->mi_cols - mi_col);
  const int y_mis = AOMMIN(bh, cm->mi_rows - mi_row);
824
825
826
827
828
829
830
831
832
833
834
  const int offset = mi_row * cm->mi_stride + mi_col;
  int x, y;

  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = cm->mi + offset;

  for (y = 0; y < y_mis; ++y)
    for (x = 0; x < x_mis; ++x) {
      xd->mi[y * cm->mi_stride + x]->mbmi.skip = skip;
      xd->mi[y * cm->mi_stride + x]->mbmi.tx_type = txfm;
    }
835
836
#if CONFIG_VAR_TX
  xd->above_txfm_context = cm->above_txfm_context + mi_col;
837
  xd->left_txfm_context =
838
      xd->left_txfm_context_buffer + (mi_row & MAX_MIB_MASK);
839
  set_txfm_ctxs(xd->mi[0]->mbmi.tx_size, bw, bh, skip, xd);
840
#endif
841
842
}

Yaowu Xu's avatar
Yaowu Xu committed
843
static void set_ref(AV1_COMMON *const cm, MACROBLOCKD *const xd, int idx,
844
                    int mi_row, int mi_col) {
845
846
847
  MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
  RefBuffer *ref_buffer = &cm->frame_refs[mbmi->ref_frame[idx] - LAST_FRAME];
  xd->block_refs[idx] = ref_buffer;
Yaowu Xu's avatar
Yaowu Xu committed
848
849
  if (!av1_is_valid_scale(&ref_buffer->sf))
    aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM,
850
                       "Invalid scale factors");
Yaowu Xu's avatar
Yaowu Xu committed
851
852
  av1_setup_pre_planes(xd, idx, ref_buffer->buf, mi_row, mi_col,
                       &ref_buffer->sf);
853
  aom_merge_corrupted_flag(&xd->corrupted, ref_buffer->buf->corrupted);
854
855
856
}

static void dec_predict_b_extend(
Yaowu Xu's avatar
Yaowu Xu committed
857
    AV1Decoder *const pbi, MACROBLOCKD *const xd, const TileInfo *const tile,
858
859
860
    int block, int mi_row_ori, int mi_col_ori, int mi_row_pred, int mi_col_pred,
    int mi_row_top, int mi_col_top, uint8_t *dst_buf[3], int dst_stride[3],
    BLOCK_SIZE bsize_top, BLOCK_SIZE bsize_pred, int b_sub8x8, int bextend) {
861
862
863
864
865
866
867
868
869
  // Used in supertx
  // (mi_row_ori, mi_col_ori): location for mv
  // (mi_row_pred, mi_col_pred, bsize_pred): region to predict
  // (mi_row_top, mi_col_top, bsize_top): region of the top partition size
  // block: sub location of sub8x8 blocks
  // b_sub8x8: 1: ori is sub8x8; 0: ori is not sub8x8
  // bextend: 1: region to predict is an extension of ori; 0: not
  int r = (mi_row_pred - mi_row_top) * MI_SIZE;
  int c = (mi_col_pred - mi_col_top) * MI_SIZE;
Jingning Han's avatar
Jingning Han committed
870
871
  const int mi_width_top = mi_size_wide[bsize_top];
  const int mi_height_top = mi_size_high[bsize_top];
872
  MB_MODE_INFO *mbmi;
Yaowu Xu's avatar
Yaowu Xu committed
873
  AV1_COMMON *const cm = &pbi->common;
874
875
876

  if (mi_row_pred < mi_row_top || mi_col_pred < mi_col_top ||
      mi_row_pred >= mi_row_top + mi_height_top ||
877
878
      mi_col_pred >= mi_col_top + mi_width_top || mi_row_pred >= cm->mi_rows ||
      mi_col_pred >= cm->mi_cols)
879
880
    return;

881
  mbmi = set_offsets_extend(cm, xd, tile, bsize_pred, mi_row_pred, mi_col_pred,
882
883
884
885
886
                            mi_row_ori, mi_col_ori);
  set_ref(cm, xd, 0, mi_row_pred, mi_col_pred);
  if (has_second_ref(&xd->mi[0]->mbmi))
    set_ref(cm, xd, 1, mi_row_pred, mi_col_pred);

887
  if (!bextend) mbmi->tx_size = max_txsize_lookup[bsize_top];
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902

  xd->plane[0].dst.stride = dst_stride[0];
  xd->plane[1].dst.stride = dst_stride[1];
  xd->plane[2].dst.stride = dst_stride[2];
  xd->plane[0].dst.buf = dst_buf[0] +
                         (r >> xd->plane[0].subsampling_y) * dst_stride[0] +
                         (c >> xd->plane[0].subsampling_x);
  xd->plane[1].dst.buf = dst_buf[1] +
                         (r >> xd->plane[1].subsampling_y) * dst_stride[1] +
                         (c >> xd->plane[1].subsampling_x);
  xd->plane[2].dst.buf = dst_buf[2] +
                         (r >> xd->plane[2].subsampling_y) * dst_stride[2] +
                         (c >> xd->plane[2].subsampling_x);

  if (!b_sub8x8)
Yaowu Xu's avatar
Yaowu Xu committed
903
    av1_build_inter_predictors_sb_extend(xd,
904
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
905
                                         mi_row_ori, mi_col_ori,
906
#endif  // CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
907
                                         mi_row_pred, mi_col_pred, bsize_pred);
908
  else
Yaowu Xu's avatar
Yaowu Xu committed
909
    av1_build_inter_predictors_sb_sub8x8_extend(xd,
910
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
911
                                                mi_row_ori, mi_col_ori,
912
#endif  // CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
913
914
                                                mi_row_pred, mi_col_pred,
                                                bsize_pred, block);
915
916
}

Yaowu Xu's avatar
Yaowu Xu committed
917
static void dec_extend_dir(AV1Decoder *const pbi, MACROBLOCKD *const xd,
918
                           const TileInfo *const tile, int block,
919
920
921
                           BLOCK_SIZE bsize, BLOCK_SIZE top_bsize, int mi_row,
                           int mi_col, int mi_row_top, int mi_col_top,
                           uint8_t *dst_buf[3], int dst_stride[3], int dir) {
922
923
  // dir: 0-lower, 1-upper, 2-left, 3-right
  //      4-lowerleft, 5-upperleft, 6-lowerright, 7-upperright
Jingning Han's avatar
Jingning Han committed
924
925
  const int mi_width = mi_size_wide[bsize];
  const int mi_height = mi_size_high[bsize];
926
927
  int xss = xd->plane[1].subsampling_x;
  int yss = xd->plane[1].subsampling_y;
Jingning Han's avatar
Jingning Han committed
928
929
930
931
932
933
#if CONFIG_CB4X4
  const int unify_bsize = 1;
#else
  const int unify_bsize = 0;
#endif
  int b_sub8x8 = (bsize < BLOCK_8X8) && !unify_bsize ? 1 : 0;
934
  BLOCK_SIZE extend_bsize;
935
936
937
938
939
  int mi_row_pred, mi_col_pred;

  int wide_unit, high_unit;
  int i, j;
  int ext_offset = 0;
940
941

  if (dir == 0 || dir == 1) {
Jingning Han's avatar
Jingning Han committed
942
943
944
945
    extend_bsize =
        (mi_width == mi_size_wide[BLOCK_8X8] || bsize < BLOCK_8X8 || xss < yss)
            ? BLOCK_8X8
            : BLOCK_16X8;
946
947
948
949
950
951
#if CONFIG_CB4X4
    if (bsize < BLOCK_8X8) {
      extend_bsize = BLOCK_4X4;
      ext_offset = mi_size_wide[BLOCK_8X8];
    }
#endif
952

953
954
    wide_unit = mi_size_wide[extend_bsize];
    high_unit = mi_size_high[extend_bsize];
955

956
957
958
959
960
961
962
963
964
    mi_row_pred = mi_row + ((dir == 0) ? mi_height : -(mi_height + ext_offset));
    mi_col_pred = mi_col;

    for (j = 0; j < mi_height + ext_offset; j += high_unit)
      for (i = 0; i < mi_width + ext_offset; i += wide_unit)
        dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col,
                             mi_row_pred + j, mi_col_pred + i, mi_row_top,
                             mi_col_top, dst_buf, dst_stride, top_bsize,
                             extend_bsize, b_sub8x8, 1);
965
  } else if (dir == 2 || dir == 3) {
Jingning Han's avatar
Jingning Han committed
966
967
968
969
    extend_bsize =
        (mi_height == mi_size_high[BLOCK_8X8] || bsize < BLOCK_8X8 || yss < xss)
            ? BLOCK_8X8
            : BLOCK_8X16;
970
971
972
973
974
975
#if CONFIG_CB4X4
    if (bsize < BLOCK_8X8) {
      extend_bsize = BLOCK_4X4;
      ext_offset = mi_size_wide[BLOCK_8X8];
    }
#endif
976

977
978
    wide_unit = mi_size_wide[extend_bsize];
    high_unit = mi_size_high[extend_bsize];
979

980
981
982
983
984
985
986
987
988
    mi_row_pred = mi_row;
    mi_col_pred = mi_col + ((dir == 3) ? mi_width : -(mi_width + ext_offset));

    for (j = 0; j < mi_height + ext_offset; j += high_unit)
      for (i = 0; i < mi_width + ext_offset; i += wide_unit)
        dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col,
                             mi_row_pred + j, mi_col_pred + i, mi_row_top,
                             mi_col_top, dst_buf, dst_stride, top_bsize,
                             extend_bsize, b_sub8x8, 1);
989
990
  } else {
    extend_bsize = BLOCK_8X8;
991
992
993
994
995
996
997
998
999
#if CONFIG_CB4X4
    if (bsize < BLOCK_8X8) {
      extend_bsize = BLOCK_4X4;
      ext_offset = mi_size_wide[BLOCK_8X8];
    }
#endif
    wide_unit = mi_size_wide[extend_bsize];
    high_unit = mi_size_high[extend_bsize];

Jingning Han's avatar
Jingning Han committed
1000
    mi_row_pred = mi_row + ((dir == 4 || dir == 6) ? mi_height
1001
                                                   : -(mi_height + ext_offset));
Jingning Han's avatar
Jingning Han committed
1002
    mi_col_pred =
1003
1004
1005
1006
1007
1008
1009
1010
        mi_col + ((dir == 6 || dir == 7) ? mi_width : -(mi_width + ext_offset));

    for (j = 0; j < mi_height + ext_offset; j += high_unit)
      for (i = 0; i < mi_width + ext_offset; i += wide_unit)
        dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col,
                             mi_row_pred + j, mi_col_pred + i, mi_row_top,
                             mi_col_top, dst_buf, dst_stride, top_bsize,
                             extend_bsize, b_sub8x8, 1);
1011
1012
1013
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
1014
static void dec_extend_all(AV1Decoder *const pbi, MACROBLOCKD *const xd,
1015
                           const TileInfo *const tile, int block,
1016
1017
1018
                           BLOCK_SIZE bsize, BLOCK_SIZE top_bsize, int mi_row,
                           int mi_col, int mi_row_top, int mi_col_top,
                           uint8_t *dst_buf[3], int dst_stride[3]) {
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 0);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 1);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 2);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 3);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 4);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 5);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 6);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 7);
}

Yaowu Xu's avatar
Yaowu Xu committed
1037
static void dec_predict_sb_complex(AV1Decoder *const pbi, MACROBLOCKD *const xd,
1038
1039
                                   const TileInfo *const tile, int mi_row,
                                   int mi_col, int mi_row_top, int mi_col_top,
1040
1041
                                   BLOCK_SIZE bsize, BLOCK_SIZE top_bsize,
                                   uint8_t *dst_buf[3], int dst_stride[3]) {
Yaowu Xu's avatar
Yaowu Xu committed
1042
  const AV1_COMMON *const cm = &pbi->common;
Jingning Han's avatar
Jingning Han committed
1043
  const int hbs = mi_size_wide[bsize] / 2;
1044
1045
  const PARTITION_TYPE partition = get_partition(cm, mi_row, mi_col, bsize);
  const BLOCK_SIZE subsize = get_subsize(bsize, partition);
1046
#if CONFIG_EXT_PARTITION_TYPES
1047
  const BLOCK_SIZE bsize2 = get_subsize(bsize, PARTITION_SPLIT);
1048
#endif
1049
1050
  int i;
  const int mi_offset = mi_row * cm->mi_stride + mi_col;
1051
  uint8_t *dst_buf1[3], *dst_buf2[3], *dst_buf3[3];
Jingning Han's avatar
Jingning Han committed
1052
1053
1054
1055
1056
#if CONFIG_CB4X4
  const int unify_bsize = 1;
#else
  const int unify_bsize = 0;
#endif
1057

1058
1059
1060
1061
1062
1063
  DECLARE_ALIGNED(16, uint8_t, tmp_buf1[MAX_MB_PLANE * MAX_TX_SQUARE * 2]);
  DECLARE_ALIGNED(16, uint8_t, tmp_buf2[MAX_MB_PLANE * MAX_TX_SQUARE * 2]);
  DECLARE_ALIGNED(16, uint8_t, tmp_buf3[MAX_MB_PLANE * MAX_TX_SQUARE * 2]);
  int dst_stride1[3] = { MAX_TX_SIZE, MAX_TX_SIZE, MAX_TX_SIZE };
  int dst_stride2[3] = { MAX_TX_SIZE, MAX_TX_SIZE, MAX_TX_SIZE };
  int dst_stride3[3] = { MAX_TX_SIZE, MAX_TX_SIZE, MAX_TX_SIZE };
1064

Yaowu Xu's avatar
Yaowu Xu committed
1065
#if CONFIG_AOM_HIGHBITDEPTH
1066
1067
1068
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
    int len = sizeof(uint16_t);
    dst_buf1[0] = CONVERT_TO_BYTEPTR(tmp_buf1);