decodeframe.c 162 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4 5 6 7 8 9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10 11 12 13 14
 */

#include <assert.h>
#include <stdlib.h>  // qsort()

Yaowu Xu's avatar
Yaowu Xu committed
15 16 17
#include "./aom_config.h"
#include "./aom_dsp_rtcd.h"
#include "./aom_scale_rtcd.h"
Jingning Han's avatar
Jingning Han committed
18
#include "./av1_rtcd.h"
Jingning Han's avatar
Jingning Han committed
19

20
#include "aom/aom_codec.h"
Yaowu Xu's avatar
Yaowu Xu committed
21
#include "aom_dsp/aom_dsp_common.h"
Jingning Han's avatar
Jingning Han committed
22 23
#include "aom_dsp/bitreader.h"
#include "aom_dsp/bitreader_buffer.h"
Yaowu Xu's avatar
Yaowu Xu committed
24
#include "aom_mem/aom_mem.h"
25 26
#include "aom_ports/mem.h"
#include "aom_ports/mem_ops.h"
Yaowu Xu's avatar
Yaowu Xu committed
27 28
#include "aom_scale/aom_scale.h"
#include "aom_util/aom_thread.h"
Jingning Han's avatar
Jingning Han committed
29

30
#include "av1/common/alloccommon.h"
31
#if CONFIG_CLPF
Steinar Midtskogen's avatar
Steinar Midtskogen committed
32
#include "aom/aom_image.h"
33
#include "av1/common/clpf.h"
34
#endif
35
#include "av1/common/common.h"
Yaowu Xu's avatar
Yaowu Xu committed
36
#if CONFIG_DERING
37
#include "av1/common/dering.h"
Yaowu Xu's avatar
Yaowu Xu committed
38
#endif  // CONFIG_DERING
39 40 41 42 43 44
#include "av1/common/entropy.h"
#include "av1/common/entropymode.h"
#include "av1/common/idct.h"
#include "av1/common/pred_common.h"
#include "av1/common/quant_common.h"
#include "av1/common/reconinter.h"
Jingning Han's avatar
Jingning Han committed
45
#include "av1/common/reconintra.h"
46
#include "av1/common/seg_common.h"
Jingning Han's avatar
Jingning Han committed
47
#include "av1/common/thread_common.h"
48
#include "av1/common/tile_common.h"
49

50 51 52
#include "av1/decoder/decodeframe.h"
#include "av1/decoder/decodemv.h"
#include "av1/decoder/decoder.h"
Jingning Han's avatar
Jingning Han committed
53
#include "av1/decoder/detokenize.h"
54
#include "av1/decoder/dsubexp.h"
Jingning Han's avatar
Jingning Han committed
55

Yaowu Xu's avatar
Yaowu Xu committed
56
#define MAX_AV1_HEADER_SIZE 80
Michael Bebenita's avatar
Michael Bebenita committed
57
#define ACCT_STR __func__
Jingning Han's avatar
Jingning Han committed
58

59 60 61 62 63 64 65 66
static struct aom_read_bit_buffer *init_read_bit_buffer(
    AV1Decoder *pbi, struct aom_read_bit_buffer *rb, const uint8_t *data,
    const uint8_t *data_end, uint8_t clear_data[MAX_AV1_HEADER_SIZE]);
static int read_compressed_header(AV1Decoder *pbi, const uint8_t *data,
                                  size_t partition_size);
static size_t read_uncompressed_header(AV1Decoder *pbi,
                                       struct aom_read_bit_buffer *rb);

Yaowu Xu's avatar
Yaowu Xu committed
67
static int is_compound_reference_allowed(const AV1_COMMON *cm) {
Jingning Han's avatar
Jingning Han committed
68
  int i;
69
  if (frame_is_intra_only(cm)) return 0;
70
  for (i = 1; i < INTER_REFS_PER_FRAME; ++i)
71
    if (cm->ref_frame_sign_bias[i + 1] != cm->ref_frame_sign_bias[1]) return 1;
Jingning Han's avatar
Jingning Han committed
72 73 74 75

  return 0;
}

Yaowu Xu's avatar
Yaowu Xu committed
76
static void setup_compound_reference_mode(AV1_COMMON *cm) {
77
#if CONFIG_EXT_REFS
78
  cm->comp_fwd_ref[0] = LAST_FRAME;
79 80 81 82
  cm->comp_fwd_ref[1] = LAST2_FRAME;
  cm->comp_fwd_ref[2] = LAST3_FRAME;
  cm->comp_fwd_ref[3] = GOLDEN_FRAME;

83 84
  cm->comp_bwd_ref[0] = BWDREF_FRAME;
  cm->comp_bwd_ref[1] = ALTREF_FRAME;
85
#else
Jingning Han's avatar
Jingning Han committed
86
  if (cm->ref_frame_sign_bias[LAST_FRAME] ==
87
      cm->ref_frame_sign_bias[GOLDEN_FRAME]) {
Jingning Han's avatar
Jingning Han committed
88 89 90 91
    cm->comp_fixed_ref = ALTREF_FRAME;
    cm->comp_var_ref[0] = LAST_FRAME;
    cm->comp_var_ref[1] = GOLDEN_FRAME;
  } else if (cm->ref_frame_sign_bias[LAST_FRAME] ==
92
             cm->ref_frame_sign_bias[ALTREF_FRAME]) {
Jingning Han's avatar
Jingning Han committed
93 94 95 96 97 98 99 100
    cm->comp_fixed_ref = GOLDEN_FRAME;
    cm->comp_var_ref[0] = LAST_FRAME;
    cm->comp_var_ref[1] = ALTREF_FRAME;
  } else {
    cm->comp_fixed_ref = LAST_FRAME;
    cm->comp_var_ref[0] = GOLDEN_FRAME;
    cm->comp_var_ref[1] = ALTREF_FRAME;
  }
101
#endif  // CONFIG_EXT_REFS
Jingning Han's avatar
Jingning Han committed
102 103 104 105 106 107
}

static int read_is_valid(const uint8_t *start, size_t len, const uint8_t *end) {
  return len != 0 && len <= (size_t)(end - start);
}

Yaowu Xu's avatar
Yaowu Xu committed
108 109
static int decode_unsigned_max(struct aom_read_bit_buffer *rb, int max) {
  const int data = aom_rb_read_literal(rb, get_unsigned_bits(max));
Jingning Han's avatar
Jingning Han committed
110 111 112
  return data > max ? max : data;
}

Yaowu Xu's avatar
Yaowu Xu committed
113 114
static TX_MODE read_tx_mode(struct aom_read_bit_buffer *rb) {
  return aom_rb_read_bit(rb) ? TX_MODE_SELECT : aom_rb_read_literal(rb, 2);
115
}
Jingning Han's avatar
Jingning Han committed
116

117
#if !CONFIG_EC_ADAPT || !CONFIG_DAALA_EC
Yaowu Xu's avatar
Yaowu Xu committed
118
static void read_switchable_interp_probs(FRAME_CONTEXT *fc, aom_reader *r) {
Jingning Han's avatar
Jingning Han committed
119
  int i, j;
120
  for (j = 0; j < SWITCHABLE_FILTER_CONTEXTS; ++j) {
Jingning Han's avatar
Jingning Han committed
121
    for (i = 0; i < SWITCHABLE_FILTERS - 1; ++i)
Michael Bebenita's avatar
Michael Bebenita committed
122
      av1_diff_update_prob(r, &fc->switchable_interp_prob[j][i], ACCT_STR);
123 124 125 126 127
#if CONFIG_DAALA_EC
    av1_tree_to_cdf(av1_switchable_interp_tree, fc->switchable_interp_prob[j],
                    fc->switchable_interp_cdf[j]);
#endif
  }
Jingning Han's avatar
Jingning Han committed
128 129
}

Yaowu Xu's avatar
Yaowu Xu committed
130
static void read_inter_mode_probs(FRAME_CONTEXT *fc, aom_reader *r) {
131 132 133
  int i;
#if CONFIG_REF_MV
  for (i = 0; i < NEWMV_MODE_CONTEXTS; ++i)
Michael Bebenita's avatar
Michael Bebenita committed
134
    av1_diff_update_prob(r, &fc->newmv_prob[i], ACCT_STR);
135
  for (i = 0; i < ZEROMV_MODE_CONTEXTS; ++i)
Michael Bebenita's avatar
Michael Bebenita committed
136
    av1_diff_update_prob(r, &fc->zeromv_prob[i], ACCT_STR);
137
  for (i = 0; i < REFMV_MODE_CONTEXTS; ++i)
Michael Bebenita's avatar
Michael Bebenita committed
138
    av1_diff_update_prob(r, &fc->refmv_prob[i], ACCT_STR);
139
  for (i = 0; i < DRL_MODE_CONTEXTS; ++i)
Michael Bebenita's avatar
Michael Bebenita committed
140
    av1_diff_update_prob(r, &fc->drl_prob[i], ACCT_STR);
Yue Chen's avatar
Yue Chen committed
141
#if CONFIG_EXT_INTER
Michael Bebenita's avatar
Michael Bebenita committed
142
  av1_diff_update_prob(r, &fc->new2mv_prob, ACCT_STR);
Yue Chen's avatar
Yue Chen committed
143
#endif  // CONFIG_EXT_INTER
144 145
#else
  int j;
146
  for (i = 0; i < INTER_MODE_CONTEXTS; ++i) {
Jingning Han's avatar
Jingning Han committed
147
    for (j = 0; j < INTER_MODES - 1; ++j)
Michael Bebenita's avatar
Michael Bebenita committed
148
      av1_diff_update_prob(r, &fc->inter_mode_probs[i][j], ACCT_STR);
149 150 151 152 153
#if CONFIG_DAALA_EC
    av1_tree_to_cdf(av1_inter_mode_tree, fc->inter_mode_probs[i],
                    fc->inter_mode_cdf[i]);
#endif
  }
154
#endif
Jingning Han's avatar
Jingning Han committed
155 156
}

157
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
158
static void read_inter_compound_mode_probs(FRAME_CONTEXT *fc, aom_reader *r) {
159
  int i, j;
Michael Bebenita's avatar
Michael Bebenita committed
160
  if (aom_read(r, GROUP_DIFF_UPDATE_PROB, ACCT_STR)) {
161 162
    for (j = 0; j < INTER_MODE_CONTEXTS; ++j) {
      for (i = 0; i < INTER_COMPOUND_MODES - 1; ++i) {
Michael Bebenita's avatar
Michael Bebenita committed
163
        av1_diff_update_prob(r, &fc->inter_compound_mode_probs[j][i], ACCT_STR);
164 165 166 167 168
      }
    }
  }
}
#endif  // CONFIG_EXT_INTER
169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196
#if !CONFIG_EXT_TX
static void read_ext_tx_probs(FRAME_CONTEXT *fc, aom_reader *r) {
  int i, j, k;
  if (aom_read(r, GROUP_DIFF_UPDATE_PROB, ACCT_STR)) {
    for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
      for (j = 0; j < TX_TYPES; ++j) {
        for (k = 0; k < TX_TYPES - 1; ++k)
          av1_diff_update_prob(r, &fc->intra_ext_tx_prob[i][j][k], ACCT_STR);
#if CONFIG_DAALA_EC
        av1_tree_to_cdf(av1_ext_tx_tree, fc->intra_ext_tx_prob[i][j],
                        fc->intra_ext_tx_cdf[i][j]);
#endif
      }
    }
  }
  if (aom_read(r, GROUP_DIFF_UPDATE_PROB, ACCT_STR)) {
    for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
      for (k = 0; k < TX_TYPES - 1; ++k)
        av1_diff_update_prob(r, &fc->inter_ext_tx_prob[i][k], ACCT_STR);
#if CONFIG_DAALA_EC
      av1_tree_to_cdf(av1_ext_tx_tree, fc->inter_ext_tx_prob[i],
                      fc->inter_ext_tx_cdf[i]);
#endif
    }
  }
}
#endif
#endif
197

198
static REFERENCE_MODE read_frame_reference_mode(
Yaowu Xu's avatar
Yaowu Xu committed
199
    const AV1_COMMON *cm, struct aom_read_bit_buffer *rb) {
200
  if (is_compound_reference_allowed(cm)) {
Yaowu Xu's avatar
Yaowu Xu committed
201
    return aom_rb_read_bit(rb)
202
               ? REFERENCE_MODE_SELECT
Yaowu Xu's avatar
Yaowu Xu committed
203
               : (aom_rb_read_bit(rb) ? COMPOUND_REFERENCE : SINGLE_REFERENCE);
204 205 206 207
  } else {
    return SINGLE_REFERENCE;
  }
}
Jingning Han's avatar
Jingning Han committed
208

Yaowu Xu's avatar
Yaowu Xu committed
209
static void read_frame_reference_mode_probs(AV1_COMMON *cm, aom_reader *r) {
Jingning Han's avatar
Jingning Han committed
210
  FRAME_CONTEXT *const fc = cm->fc;
211
  int i, j;
Jingning Han's avatar
Jingning Han committed
212 213 214

  if (cm->reference_mode == REFERENCE_MODE_SELECT)
    for (i = 0; i < COMP_INTER_CONTEXTS; ++i)
Michael Bebenita's avatar
Michael Bebenita committed
215
      av1_diff_update_prob(r, &fc->comp_inter_prob[i], ACCT_STR);
Jingning Han's avatar
Jingning Han committed
216

217
  if (cm->reference_mode != COMPOUND_REFERENCE) {
Jingning Han's avatar
Jingning Han committed
218
    for (i = 0; i < REF_CONTEXTS; ++i) {
219
      for (j = 0; j < (SINGLE_REFS - 1); ++j) {
Michael Bebenita's avatar
Michael Bebenita committed
220
        av1_diff_update_prob(r, &fc->single_ref_prob[i][j], ACCT_STR);
221
      }
Jingning Han's avatar
Jingning Han committed
222
    }
223
  }
Jingning Han's avatar
Jingning Han committed
224

225 226
  if (cm->reference_mode != SINGLE_REFERENCE) {
    for (i = 0; i < REF_CONTEXTS; ++i) {
227
#if CONFIG_EXT_REFS
228
      for (j = 0; j < (FWD_REFS - 1); ++j)
Michael Bebenita's avatar
Michael Bebenita committed
229
        av1_diff_update_prob(r, &fc->comp_ref_prob[i][j], ACCT_STR);
230
      for (j = 0; j < (BWD_REFS - 1); ++j)
Michael Bebenita's avatar
Michael Bebenita committed
231
        av1_diff_update_prob(r, &fc->comp_bwdref_prob[i][j], ACCT_STR);
232 233
#else
      for (j = 0; j < (COMP_REFS - 1); ++j)
Michael Bebenita's avatar
Michael Bebenita committed
234
        av1_diff_update_prob(r, &fc->comp_ref_prob[i][j], ACCT_STR);
235
#endif  // CONFIG_EXT_REFS
236 237
    }
  }
Jingning Han's avatar
Jingning Han committed
238 239
}

Yaowu Xu's avatar
Yaowu Xu committed
240
static void update_mv_probs(aom_prob *p, int n, aom_reader *r) {
Jingning Han's avatar
Jingning Han committed
241
  int i;
Michael Bebenita's avatar
Michael Bebenita committed
242
  for (i = 0; i < n; ++i) av1_diff_update_prob(r, &p[i], ACCT_STR);
Jingning Han's avatar
Jingning Han committed
243 244
}

Yaowu Xu's avatar
Yaowu Xu committed
245
static void read_mv_probs(nmv_context *ctx, int allow_hp, aom_reader *r) {
246
  int i;
Jingning Han's avatar
Jingning Han committed
247

248
#if !CONFIG_EC_ADAPT || !CONFIG_EC_MULTISYMBOL
249
  int j;
Jingning Han's avatar
Jingning Han committed
250
  update_mv_probs(ctx->joints, MV_JOINTS - 1, r);
251
#if CONFIG_EC_MULTISYMBOL
252 253
  av1_tree_to_cdf(av1_mv_joint_tree, ctx->joints, ctx->joint_cdf);
#endif
Jingning Han's avatar
Jingning Han committed
254 255 256 257 258 259 260

  for (i = 0; i < 2; ++i) {
    nmv_component *const comp_ctx = &ctx->comps[i];
    update_mv_probs(&comp_ctx->sign, 1, r);
    update_mv_probs(comp_ctx->classes, MV_CLASSES - 1, r);
    update_mv_probs(comp_ctx->class0, CLASS0_SIZE - 1, r);
    update_mv_probs(comp_ctx->bits, MV_OFFSET_BITS, r);
261
#if CONFIG_EC_MULTISYMBOL
262 263
    av1_tree_to_cdf(av1_mv_class_tree, comp_ctx->classes, comp_ctx->class_cdf);
#endif
Jingning Han's avatar
Jingning Han committed
264 265 266
  }
  for (i = 0; i < 2; ++i) {
    nmv_component *const comp_ctx = &ctx->comps[i];
267
    for (j = 0; j < CLASS0_SIZE; ++j) {
Jingning Han's avatar
Jingning Han committed
268
      update_mv_probs(comp_ctx->class0_fp[j], MV_FP_SIZE - 1, r);
269
#if CONFIG_EC_MULTISYMBOL
270 271 272 273
      av1_tree_to_cdf(av1_mv_fp_tree, comp_ctx->class0_fp[j],
                      comp_ctx->class0_fp_cdf[j]);
#endif
    }
274
    update_mv_probs(comp_ctx->fp, MV_FP_SIZE - 1, r);
275
#if CONFIG_EC_MULTISYMBOL
276 277
    av1_tree_to_cdf(av1_mv_fp_tree, comp_ctx->fp, comp_ctx->fp_cdf);
#endif
Jingning Han's avatar
Jingning Han committed
278
  }
279
#endif  // !CONFIG_EC_ADAPT
Jingning Han's avatar
Jingning Han committed
280 281 282 283 284 285 286 287 288 289

  if (allow_hp) {
    for (i = 0; i < 2; ++i) {
      nmv_component *const comp_ctx = &ctx->comps[i];
      update_mv_probs(&comp_ctx->class0_hp, 1, r);
      update_mv_probs(&comp_ctx->hp, 1, r);
    }
  }
}

290
static void inverse_transform_block(MACROBLOCKD *xd, int plane,
291
                                    const TX_TYPE tx_type,
292
                                    const TX_SIZE tx_size, uint8_t *dst,
293
                                    int stride, int16_t scan_line, int eob) {
Jingning Han's avatar
Jingning Han committed
294
  struct macroblockd_plane *const pd = &xd->plane[plane];
295 296 297 298 299 300
  tran_low_t *const dqcoeff = pd->dqcoeff;
  INV_TXFM_PARAM inv_txfm_param;
  inv_txfm_param.tx_type = tx_type;
  inv_txfm_param.tx_size = tx_size;
  inv_txfm_param.eob = eob;
  inv_txfm_param.lossless = xd->lossless[xd->mi[0]->mbmi.segment_id];
Angie Chiang's avatar
Angie Chiang committed
301

Yaowu Xu's avatar
Yaowu Xu committed
302
#if CONFIG_AOM_HIGHBITDEPTH
303 304 305 306
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
    inv_txfm_param.bd = xd->bd;
    highbd_inv_txfm_add(dqcoeff, dst, stride, &inv_txfm_param);
  } else {
Yaowu Xu's avatar
Yaowu Xu committed
307
#endif  // CONFIG_AOM_HIGHBITDEPTH
308
    inv_txfm_add(dqcoeff, dst, stride, &inv_txfm_param);
Yaowu Xu's avatar
Yaowu Xu committed
309
#if CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
310
  }
311 312
#endif  // CONFIG_AOM_HIGHBITDEPTH
  memset(dqcoeff, 0, (scan_line + 1) * sizeof(dqcoeff[0]));
Jingning Han's avatar
Jingning Han committed
313 314
}

Angie Chiang's avatar
Angie Chiang committed
315 316
static void predict_and_reconstruct_intra_block(AV1_COMMON *cm,
                                                MACROBLOCKD *const xd,
Alex Converse's avatar
Alex Converse committed
317 318 319
#if CONFIG_ANS
                                                struct AnsDecoder *const r,
#else
Yaowu Xu's avatar
Yaowu Xu committed
320
                                                aom_reader *r,
Alex Converse's avatar
Alex Converse committed
321
#endif  // CONFIG_ANS
Jingning Han's avatar
Jingning Han committed
322
                                                MB_MODE_INFO *const mbmi,
323
                                                int plane, int row, int col,
Jingning Han's avatar
Jingning Han committed
324 325 326
                                                TX_SIZE tx_size) {
  struct macroblockd_plane *const pd = &xd->plane[plane];
  PREDICTION_MODE mode = (plane == 0) ? mbmi->mode : mbmi->uv_mode;
hui su's avatar
hui su committed
327
  PLANE_TYPE plane_type = (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
Jingning Han's avatar
Jingning Han committed
328
  uint8_t *dst;
hui su's avatar
hui su committed
329
  int block_idx = (row << 1) + col;
Jingning Han's avatar
Jingning Han committed
330 331 332
  dst = &pd->dst.buf[4 * row * pd->dst.stride + 4 * col];

  if (mbmi->sb_type < BLOCK_8X8)
333
    if (plane == 0) mode = xd->mi[0]->bmi[(row << 1) + col].as_mode;
Jingning Han's avatar
Jingning Han committed
334

335
  av1_predict_intra_block(xd, pd->width, pd->height, tx_size, mode, dst,
Yaowu Xu's avatar
Yaowu Xu committed
336
                          pd->dst.stride, dst, pd->dst.stride, col, row, plane);
Jingning Han's avatar
Jingning Han committed
337 338

  if (!mbmi->skip) {
hui su's avatar
hui su committed
339
    TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, tx_size);
Angie Chiang's avatar
Angie Chiang committed
340
    const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, 0);
341 342 343 344
    int16_t max_scan_line = 0;
    const int eob =
        av1_decode_block_tokens(xd, plane, scan_order, col, row, tx_size,
                                tx_type, &max_scan_line, r, mbmi->segment_id);
Angie Chiang's avatar
Angie Chiang committed
345 346 347
#if CONFIG_ADAPT_SCAN
    av1_update_scan_count_facade(cm, tx_size, tx_type, pd->dqcoeff, eob);
#endif
348 349 350
    if (eob)
      inverse_transform_block(xd, plane, tx_type, tx_size, dst, pd->dst.stride,
                              max_scan_line, eob);
Jingning Han's avatar
Jingning Han committed
351 352 353
  }
}

354
#if CONFIG_VAR_TX
Angie Chiang's avatar
Angie Chiang committed
355 356
static void decode_reconstruct_tx(AV1_COMMON *cm, MACROBLOCKD *const xd,
                                  aom_reader *r, MB_MODE_INFO *const mbmi,
357
                                  int plane, BLOCK_SIZE plane_bsize,
358 359
                                  int blk_row, int blk_col, TX_SIZE tx_size,
                                  int *eob_total) {
360
  const struct macroblockd_plane *const pd = &xd->plane[plane];
361
  const BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
362 363
  const int tx_row = blk_row >> (1 - pd->subsampling_y);
  const int tx_col = blk_col >> (1 - pd->subsampling_x);
364
  const TX_SIZE plane_tx_size =
365
      plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0]
366
            : mbmi->inter_tx_size[tx_row][tx_col];
367 368 369
  // Scale to match transform block unit.
  int max_blocks_high = block_size_high[plane_bsize];
  int max_blocks_wide = block_size_wide[plane_bsize];
370 371

  if (xd->mb_to_bottom_edge < 0)
372
    max_blocks_high += xd->mb_to_bottom_edge >> (3 + pd->subsampling_y);
373
  if (xd->mb_to_right_edge < 0)
374 375 376 377
    max_blocks_wide += xd->mb_to_right_edge >> (3 + pd->subsampling_x);

  max_blocks_high >>= tx_size_wide_log2[0];
  max_blocks_wide >>= tx_size_wide_log2[0];
378

379
  if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
380

381
  if (tx_size == plane_tx_size) {
382
    PLANE_TYPE plane_type = (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
383 384
    int block_idx = (blk_row << 1) + blk_col;
    TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, plane_tx_size);
Angie Chiang's avatar
Angie Chiang committed
385
    const SCAN_ORDER *sc = get_scan(cm, plane_tx_size, tx_type, 1);
386
    int16_t max_scan_line = 0;
387
    const int eob =
Yaowu Xu's avatar
Yaowu Xu committed
388
        av1_decode_block_tokens(xd, plane, sc, blk_col, blk_row, plane_tx_size,
389
                                tx_type, &max_scan_line, r, mbmi->segment_id);
390 391
    inverse_transform_block(
        xd, plane, tx_type, plane_tx_size,
392
        &pd->dst.buf[4 * blk_row * pd->dst.stride + 4 * blk_col],
393
        pd->dst.stride, max_scan_line, eob);
394 395
    *eob_total += eob;
  } else {
396
    int bsl = block_size_wide[bsize] >> (tx_size_wide_log2[0] + 1);
397 398 399 400 401
    int i;

    assert(bsl > 0);

    for (i = 0; i < 4; ++i) {
402 403
      const int offsetr = blk_row + (i >> 1) * bsl;
      const int offsetc = blk_col + (i & 0x01) * bsl;
404

405
      if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;
406

407 408
      decode_reconstruct_tx(cm, xd, r, mbmi, plane, plane_bsize, offsetr,
                            offsetc, tx_size - 1, eob_total);
409 410 411
    }
  }
}
412
#endif  // CONFIG_VAR_TX
413

414
#if !CONFIG_VAR_TX || CONFIG_SUPERTX || (CONFIG_EXT_TX && CONFIG_RECT_TX)
Angie Chiang's avatar
Angie Chiang committed
415
static int reconstruct_inter_block(AV1_COMMON *cm, MACROBLOCKD *const xd,
Alex Converse's avatar
Alex Converse committed
416 417 418
#if CONFIG_ANS
                                   struct AnsDecoder *const r,
#else
Yaowu Xu's avatar
Yaowu Xu committed
419
                                   aom_reader *r,
Alex Converse's avatar
Alex Converse committed
420
#endif
421 422
                                   int segment_id, int plane, int row, int col,
                                   TX_SIZE tx_size) {
Jingning Han's avatar
Jingning Han committed
423
  struct macroblockd_plane *const pd = &xd->plane[plane];
hui su's avatar
hui su committed
424 425
  PLANE_TYPE plane_type = (plane == 0) ? PLANE_TYPE_Y : PLANE_TYPE_UV;
  int block_idx = (row << 1) + col;
hui su's avatar
hui su committed
426
  TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, tx_size);
Angie Chiang's avatar
Angie Chiang committed
427
  const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, 1);
428 429 430 431
  int16_t max_scan_line = 0;
  const int eob =
      av1_decode_block_tokens(xd, plane, scan_order, col, row, tx_size, tx_type,
                              &max_scan_line, r, segment_id);
Angie Chiang's avatar
Angie Chiang committed
432 433 434
#if CONFIG_ADAPT_SCAN
  av1_update_scan_count_facade(cm, tx_size, tx_type, pd->dqcoeff, eob);
#endif
435 436 437 438
  if (eob)
    inverse_transform_block(xd, plane, tx_type, tx_size,
                            &pd->dst.buf[4 * row * pd->dst.stride + 4 * col],
                            pd->dst.stride, max_scan_line, eob);
Jingning Han's avatar
Jingning Han committed
439 440
  return eob;
}
441
#endif  // !CONFIG_VAR_TX || CONFIG_SUPER_TX
Jingning Han's avatar
Jingning Han committed
442 443 444 445 446 447 448 449 450 451

static INLINE void dec_reset_skip_context(MACROBLOCKD *xd) {
  int i;
  for (i = 0; i < MAX_MB_PLANE; i++) {
    struct macroblockd_plane *const pd = &xd->plane[i];
    memset(pd->above_context, 0, sizeof(ENTROPY_CONTEXT) * pd->n4_w);
    memset(pd->left_context, 0, sizeof(ENTROPY_CONTEXT) * pd->n4_h);
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
452
static MB_MODE_INFO *set_offsets(AV1_COMMON *const cm, MACROBLOCKD *const xd,
Jingning Han's avatar
Jingning Han committed
453
                                 BLOCK_SIZE bsize, int mi_row, int mi_col,
454 455
                                 int bw, int bh, int x_mis, int y_mis, int bwl,
                                 int bhl) {
Jingning Han's avatar
Jingning Han committed
456 457 458 459 460 461 462 463 464 465
  const int offset = mi_row * cm->mi_stride + mi_col;
  int x, y;
  const TileInfo *const tile = &xd->tile;

  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = &cm->mi[offset];
  // TODO(slavarnway): Generate sb_type based on bwl and bhl, instead of
  // passing bsize from decode_partition().
  xd->mi[0]->mbmi.sb_type = bsize;
  for (y = 0; y < y_mis; ++y)
Jingning Han's avatar
Jingning Han committed
466
    for (x = !y; x < x_mis; ++x) xd->mi[y * cm->mi_stride + x] = xd->mi[0];
Jingning Han's avatar
Jingning Han committed
467 468 469 470

  set_plane_n4(xd, bw, bh, bwl, bhl);
  set_skip_context(xd, mi_row, mi_col);

471 472 473 474
#if CONFIG_VAR_TX
  xd->max_tx_size = max_txsize_lookup[bsize];
#endif

Jingning Han's avatar
Jingning Han committed
475 476 477 478
  // Distance of Mb to the various image edges. These are specified to 8th pel
  // as they are always compared to values that are in 1/8th pel units
  set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, cm->mi_rows, cm->mi_cols);

Yaowu Xu's avatar
Yaowu Xu committed
479
  av1_setup_dst_planes(xd->plane, get_frame_new_buffer(cm), mi_row, mi_col);
Jingning Han's avatar
Jingning Han committed
480 481 482
  return &xd->mi[0]->mbmi;
}

483
#if CONFIG_SUPERTX
Yaowu Xu's avatar
Yaowu Xu committed
484
static MB_MODE_INFO *set_offsets_extend(AV1_COMMON *const cm,
485 486
                                        MACROBLOCKD *const xd,
                                        const TileInfo *const tile,
487 488 489
                                        BLOCK_SIZE bsize_pred, int mi_row_pred,
                                        int mi_col_pred, int mi_row_ori,
                                        int mi_col_ori) {
490 491 492 493 494 495 496 497 498 499
  // Used in supertx
  // (mi_row_ori, mi_col_ori): location for mv
  // (mi_row_pred, mi_col_pred, bsize_pred): region to predict
  const int bw = num_8x8_blocks_wide_lookup[bsize_pred];
  const int bh = num_8x8_blocks_high_lookup[bsize_pred];
  const int offset = mi_row_ori * cm->mi_stride + mi_col_ori;
  const int bwl = b_width_log2_lookup[bsize_pred];
  const int bhl = b_height_log2_lookup[bsize_pred];
  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = cm->mi + offset;
500 501
  set_mi_row_col(xd, tile, mi_row_pred, bh, mi_col_pred, bw, cm->mi_rows,
                 cm->mi_cols);
502

503 504
  xd->up_available = (mi_row_ori > tile->mi_row_start);
  xd->left_available = (mi_col_ori > tile->mi_col_start);
505 506 507 508 509 510

  set_plane_n4(xd, bw, bh, bwl, bhl);

  return &xd->mi[0]->mbmi;
}

Yaowu Xu's avatar
Yaowu Xu committed
511 512 513
static MB_MODE_INFO *set_mb_offsets(AV1_COMMON *const cm, MACROBLOCKD *const xd,
                                    BLOCK_SIZE bsize, int mi_row, int mi_col,
                                    int bw, int bh, int x_mis, int y_mis) {
514 515 516 517 518 519 520 521
  const int offset = mi_row * cm->mi_stride + mi_col;
  const TileInfo *const tile = &xd->tile;
  int x, y;

  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = cm->mi + offset;
  xd->mi[0]->mbmi.sb_type = bsize;
  for (y = 0; y < y_mis; ++y)
522
    for (x = !y; x < x_mis; ++x) xd->mi[y * cm->mi_stride + x] = xd->mi[0];
523 524 525 526 527

  set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, cm->mi_rows, cm->mi_cols);
  return &xd->mi[0]->mbmi;
}

Yaowu Xu's avatar
Yaowu Xu committed
528
static void set_offsets_topblock(AV1_COMMON *const cm, MACROBLOCKD *const xd,
529 530
                                 const TileInfo *const tile, BLOCK_SIZE bsize,
                                 int mi_row, int mi_col) {
531 532 533 534 535 536 537 538 539 540 541 542 543
  const int bw = num_8x8_blocks_wide_lookup[bsize];
  const int bh = num_8x8_blocks_high_lookup[bsize];
  const int offset = mi_row * cm->mi_stride + mi_col;
  const int bwl = b_width_log2_lookup[bsize];
  const int bhl = b_height_log2_lookup[bsize];

  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = cm->mi + offset;

  set_plane_n4(xd, bw, bh, bwl, bhl);

  set_mi_row_col(xd, tile, mi_row, bh, mi_col, bw, cm->mi_rows, cm->mi_cols);

Yaowu Xu's avatar
Yaowu Xu committed
544
  av1_setup_dst_planes(xd->plane, get_frame_new_buffer(cm), mi_row, mi_col);
545 546
}

Yaowu Xu's avatar
Yaowu Xu committed
547
static void set_param_topblock(AV1_COMMON *const cm, MACROBLOCKD *const xd,
548
                               BLOCK_SIZE bsize, int mi_row, int mi_col,
549
                               int txfm, int skip) {
550 551
  const int bw = num_8x8_blocks_wide_lookup[bsize];
  const int bh = num_8x8_blocks_high_lookup[bsize];
Yaowu Xu's avatar
Yaowu Xu committed
552 553
  const int x_mis = AOMMIN(bw, cm->mi_cols - mi_col);
  const int y_mis = AOMMIN(bh, cm->mi_rows - mi_row);
554 555 556 557 558 559 560 561 562 563 564
  const int offset = mi_row * cm->mi_stride + mi_col;
  int x, y;

  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = cm->mi + offset;

  for (y = 0; y < y_mis; ++y)
    for (x = 0; x < x_mis; ++x) {
      xd->mi[y * cm->mi_stride + x]->mbmi.skip = skip;
      xd->mi[y * cm->mi_stride + x]->mbmi.tx_type = txfm;
    }
565 566
#if CONFIG_VAR_TX
  xd->above_txfm_context = cm->above_txfm_context + mi_col;
567
  xd->left_txfm_context =
568
      xd->left_txfm_context_buffer + (mi_row & MAX_MIB_MASK);
569
  set_txfm_ctxs(xd->mi[0]->mbmi.tx_size, bw, bh, xd);
570
#endif
571 572
}

Yaowu Xu's avatar
Yaowu Xu committed
573
static void set_ref(AV1_COMMON *const cm, MACROBLOCKD *const xd, int idx,
574
                    int mi_row, int mi_col) {
575 576 577
  MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
  RefBuffer *ref_buffer = &cm->frame_refs[mbmi->ref_frame[idx] - LAST_FRAME];
  xd->block_refs[idx] = ref_buffer;
Yaowu Xu's avatar
Yaowu Xu committed
578 579
  if (!av1_is_valid_scale(&ref_buffer->sf))
    aom_internal_error(&cm->error, AOM_CODEC_UNSUP_BITSTREAM,
580
                       "Invalid scale factors");
Yaowu Xu's avatar
Yaowu Xu committed
581 582
  av1_setup_pre_planes(xd, idx, ref_buffer->buf, mi_row, mi_col,
                       &ref_buffer->sf);
583 584 585 586
  xd->corrupted |= ref_buffer->buf->corrupted;
}

static void dec_predict_b_extend(
Yaowu Xu's avatar
Yaowu Xu committed
587
    AV1Decoder *const pbi, MACROBLOCKD *const xd, const TileInfo *const tile,
588 589 590
    int block, int mi_row_ori, int mi_col_ori, int mi_row_pred, int mi_col_pred,
    int mi_row_top, int mi_col_top, uint8_t *dst_buf[3], int dst_stride[3],
    BLOCK_SIZE bsize_top, BLOCK_SIZE bsize_pred, int b_sub8x8, int bextend) {
591 592 593 594 595 596 597 598 599 600 601 602
  // Used in supertx
  // (mi_row_ori, mi_col_ori): location for mv
  // (mi_row_pred, mi_col_pred, bsize_pred): region to predict
  // (mi_row_top, mi_col_top, bsize_top): region of the top partition size
  // block: sub location of sub8x8 blocks
  // b_sub8x8: 1: ori is sub8x8; 0: ori is not sub8x8
  // bextend: 1: region to predict is an extension of ori; 0: not
  int r = (mi_row_pred - mi_row_top) * MI_SIZE;
  int c = (mi_col_pred - mi_col_top) * MI_SIZE;
  const int mi_width_top = num_8x8_blocks_wide_lookup[bsize_top];
  const int mi_height_top = num_8x8_blocks_high_lookup[bsize_top];
  MB_MODE_INFO *mbmi;
Yaowu Xu's avatar
Yaowu Xu committed
603
  AV1_COMMON *const cm = &pbi->common;
604 605 606

  if (mi_row_pred < mi_row_top || mi_col_pred < mi_col_top ||
      mi_row_pred >= mi_row_top + mi_height_top ||
607 608
      mi_col_pred >= mi_col_top + mi_width_top || mi_row_pred >= cm->mi_rows ||
      mi_col_pred >= cm->mi_cols)
609 610
    return;

611
  mbmi = set_offsets_extend(cm, xd, tile, bsize_pred, mi_row_pred, mi_col_pred,
612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634
                            mi_row_ori, mi_col_ori);
  set_ref(cm, xd, 0, mi_row_pred, mi_col_pred);
  if (has_second_ref(&xd->mi[0]->mbmi))
    set_ref(cm, xd, 1, mi_row_pred, mi_col_pred);

  if (!bextend) {
    mbmi->tx_size = b_width_log2_lookup[bsize_top];
  }

  xd->plane[0].dst.stride = dst_stride[0];
  xd->plane[1].dst.stride = dst_stride[1];
  xd->plane[2].dst.stride = dst_stride[2];
  xd->plane[0].dst.buf = dst_buf[0] +
                         (r >> xd->plane[0].subsampling_y) * dst_stride[0] +
                         (c >> xd->plane[0].subsampling_x);
  xd->plane[1].dst.buf = dst_buf[1] +
                         (r >> xd->plane[1].subsampling_y) * dst_stride[1] +
                         (c >> xd->plane[1].subsampling_x);
  xd->plane[2].dst.buf = dst_buf[2] +
                         (r >> xd->plane[2].subsampling_y) * dst_stride[2] +
                         (c >> xd->plane[2].subsampling_x);

  if (!b_sub8x8)
Yaowu Xu's avatar
Yaowu Xu committed
635
    av1_build_inter_predictors_sb_extend(xd,
636
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
637
                                         mi_row_ori, mi_col_ori,
638
#endif  // CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
639
                                         mi_row_pred, mi_col_pred, bsize_pred);
640
  else
Yaowu Xu's avatar
Yaowu Xu committed
641
    av1_build_inter_predictors_sb_sub8x8_extend(xd,
642
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
643
                                                mi_row_ori, mi_col_ori,
644
#endif  // CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
645 646
                                                mi_row_pred, mi_col_pred,
                                                bsize_pred, block);
647 648
}

Yaowu Xu's avatar
Yaowu Xu committed
649
static void dec_extend_dir(AV1Decoder *const pbi, MACROBLOCKD *const xd,
650
                           const TileInfo *const tile, int block,
651 652 653
                           BLOCK_SIZE bsize, BLOCK_SIZE top_bsize, int mi_row,
                           int mi_col, int mi_row_top, int mi_col_top,
                           uint8_t *dst_buf[3], int dst_stride[3], int dir) {
654 655 656 657 658 659 660 661 662 663 664
  // dir: 0-lower, 1-upper, 2-left, 3-right
  //      4-lowerleft, 5-upperleft, 6-lowerright, 7-upperright
  const int mi_width = num_8x8_blocks_wide_lookup[bsize];
  const int mi_height = num_8x8_blocks_high_lookup[bsize];
  int xss = xd->plane[1].subsampling_x;
  int yss = xd->plane[1].subsampling_y;
  int b_sub8x8 = (bsize < BLOCK_8X8) ? 1 : 0;
  BLOCK_SIZE extend_bsize;
  int unit, mi_row_pred, mi_col_pred;

  if (dir == 0 || dir == 1) {
665 666 667
    extend_bsize = (mi_width == 1 || bsize < BLOCK_8X8 || xss < yss)
                       ? BLOCK_8X8
                       : BLOCK_16X8;
668 669 670 671
    unit = num_8x8_blocks_wide_lookup[extend_bsize];
    mi_row_pred = mi_row + ((dir == 0) ? mi_height : -1);
    mi_col_pred = mi_col;

672 673 674
    dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col, mi_row_pred,
                         mi_col_pred, mi_row_top, mi_col_top, dst_buf,
                         dst_stride, top_bsize, extend_bsize, b_sub8x8, 1);
675 676 677 678

    if (mi_width > unit) {
      int i;
      assert(!b_sub8x8);
679
      for (i = 0; i < mi_width / unit - 1; i++) {
680
        mi_col_pred += unit;
681 682 683
        dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col, mi_row_pred,
                             mi_col_pred, mi_row_top, mi_col_top, dst_buf,
                             dst_stride, top_bsize, extend_bsize, b_sub8x8, 1);
684 685 686
      }
    }
  } else if (dir == 2 || dir == 3) {
687 688 689
    extend_bsize = (mi_height == 1 || bsize < BLOCK_8X8 || yss < xss)
                       ? BLOCK_8X8
                       : BLOCK_8X16;
690 691 692 693
    unit = num_8x8_blocks_high_lookup[extend_bsize];
    mi_row_pred = mi_row;
    mi_col_pred = mi_col + ((dir == 3) ? mi_width : -1);

694 695 696
    dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col, mi_row_pred,
                         mi_col_pred, mi_row_top, mi_col_top, dst_buf,
                         dst_stride, top_bsize, extend_bsize, b_sub8x8, 1);
697 698 699

    if (mi_height > unit) {
      int i;
700
      for (i = 0; i < mi_height / unit - 1; i++) {
701
        mi_row_pred += unit;
702 703 704
        dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col, mi_row_pred,
                             mi_col_pred, mi_row_top, mi_col_top, dst_buf,
                             dst_stride, top_bsize, extend_bsize, b_sub8x8, 1);
705 706 707 708 709 710
      }
    }
  } else {
    extend_bsize = BLOCK_8X8;
    mi_row_pred = mi_row + ((dir == 4 || dir == 6) ? mi_height : -1);
    mi_col_pred = mi_col + ((dir == 6 || dir == 7) ? mi_width : -1);
711 712 713
    dec_predict_b_extend(pbi, xd, tile, block, mi_row, mi_col, mi_row_pred,
                         mi_col_pred, mi_row_top, mi_col_top, dst_buf,
                         dst_stride, top_bsize, extend_bsize, b_sub8x8, 1);
714 715 716
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
717
static void dec_extend_all(AV1Decoder *const pbi, MACROBLOCKD *const xd,
718
                           const TileInfo *const tile, int block,
719 720 721
                           BLOCK_SIZE bsize, BLOCK_SIZE top_bsize, int mi_row,
                           int mi_col, int mi_row_top, int mi_col_top,
                           uint8_t *dst_buf[3], int dst_stride[3]) {
722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 0);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 1);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 2);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 3);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 4);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 5);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 6);
  dec_extend_dir(pbi, xd, tile, block, bsize, top_bsize, mi_row, mi_col,
                 mi_row_top, mi_col_top, dst_buf, dst_stride, 7);
}

Yaowu Xu's avatar
Yaowu Xu committed
740
static void dec_predict_sb_complex(AV1Decoder *const pbi, MACROBLOCKD *const xd,
741 742
                                   const TileInfo *const tile, int mi_row,
                                   int mi_col, int mi_row_top, int mi_col_top,
743 744
                                   BLOCK_SIZE bsize, BLOCK_SIZE top_bsize,
                                   uint8_t *dst_buf[3], int dst_stride[3]) {
Yaowu Xu's avatar
Yaowu Xu committed
745
  const AV1_COMMON *const cm = &pbi->common;
746 747 748
  const int hbs = num_8x8_blocks_wide_lookup[bsize] / 2;
  const PARTITION_TYPE partition = get_partition(cm, mi_row, mi_col, bsize);
  const BLOCK_SIZE subsize = get_subsize(bsize, partition);
749
#if CONFIG_EXT_PARTITION_TYPES
750
  const BLOCK_SIZE bsize2 = get_subsize(bsize, PARTITION_SPLIT);
751
#endif
752 753
  int i;
  const int mi_offset = mi_row * cm->mi_stride + mi_col;
754 755
  uint8_t *dst_buf1[3], *dst_buf2[3], *dst_buf3[3];

756 757 758 759 760 761
  DECLARE_ALIGNED(16, uint8_t, tmp_buf1[MAX_MB_PLANE * MAX_TX_SQUARE * 2]);
  DECLARE_ALIGNED(16, uint8_t, tmp_buf2[MAX_MB_PLANE * MAX_TX_SQUARE * 2]);
  DECLARE_ALIGNED(16, uint8_t, tmp_buf3[MAX_MB_PLANE * MAX_TX_SQUARE * 2]);
  int dst_stride1[3] = { MAX_TX_SIZE, MAX_TX_SIZE, MAX_TX_SIZE };
  int dst_stride2[3] = { MAX_TX_SIZE, MAX_TX_SIZE, MAX_TX_SIZE };
  int dst_stride3[3] = { MAX_TX_SIZE, MAX_TX_SIZE, MAX_TX_SIZE };
762

Yaowu Xu's avatar
Yaowu Xu committed
763
#if CONFIG_AOM_HIGHBITDEPTH
764 765 766
  if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
    int len = sizeof(uint16_t);
    dst_buf1[0] = CONVERT_TO_BYTEPTR(tmp_buf1);
767 768
    dst_buf1[1] = CONVERT_TO_BYTEPTR(tmp_buf1 + MAX_TX_SQUARE * len);
    dst_buf1[2] = CONVERT_TO_BYTEPTR(tmp_buf1 + 2 * MAX_TX_SQUARE * len);
769
    dst_buf2[0] = CONVERT_TO_BYTEPTR(tmp_buf2);
770 771
    dst_buf2[1] = CONVERT_TO_BYTEPTR(tmp_buf2 + MAX_TX_SQUARE * len);
    dst_buf2[2] = CONVERT_TO_BYTEPTR(tmp_buf2 + 2 * MAX_TX_SQUARE * len);
772
    dst_buf3[0] = CONVERT_TO_BYTEPTR(tmp_buf3);
773 774
    dst_buf3[1] = CONVERT_TO_BYTEPTR(tmp_buf3 + MAX_TX_SQUARE * len);
    dst_buf3[2] = CONVERT_TO_BYTEPTR(tmp_buf3 + 2 * MAX_TX_SQUARE * len);
775 776 777
  } else {
#endif
    dst_buf1[0] = tmp_buf1;
778 779
    dst_buf1[1] = tmp_buf1 + MAX_TX_SQUARE;
    dst_buf1[2] = tmp_buf1 + 2 * MAX_TX_SQUARE;
780
    dst_buf2[0] = tmp_buf2;
781 782
    dst_buf2[1] = tmp_buf2 + MAX_TX_SQUARE;
    dst_buf2[2] = tmp_buf2 + 2 * MAX_TX_SQUARE;
783
    dst_buf3[0] = tmp_buf3;
784 785
    dst_buf3[1] = tmp_buf3 + MAX_TX_SQUARE;
    dst_buf3[2] = tmp_buf3 + 2 * MAX_TX_SQUARE;
Yaowu Xu's avatar
Yaowu Xu committed
786
#if CONFIG_AOM_HIGHBITDEPTH
787 788 789
  }
#endif

790
  if (mi_row >= cm->mi_rows || mi_col >= cm->mi_cols) return;
791

792 793
  xd<