decodeframe.c

/*
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
 *
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
 */

#include <assert.h>
#include <stdlib.h>  // qsort()

#include "./aom_config.h"
#include "./aom_dsp_rtcd.h"
#include "./aom_scale_rtcd.h"
#include "./av1_rtcd.h"

#include "aom/aom_codec.h"
#include "aom_dsp/aom_dsp_common.h"
#include "aom_dsp/bitreader.h"
#include "aom_dsp/bitreader_buffer.h"
#include "aom_dsp/binary_codes_reader.h"
#include "aom_mem/aom_mem.h"
#include "aom_ports/mem.h"
#include "aom_ports/mem_ops.h"
#include "aom_scale/aom_scale.h"
#include "aom_util/aom_thread.h"

#if CONFIG_BITSTREAM_DEBUG
#include "aom_util/debug_util.h"
#endif  // CONFIG_BITSTREAM_DEBUG

#include "av1/common/alloccommon.h"
#if CONFIG_CDEF
#include "av1/common/cdef.h"
#include "av1/common/clpf.h"
#endif
#if CONFIG_INSPECTION
#include "av1/decoder/inspection.h"
#endif
#include "av1/common/common.h"
#include "av1/common/entropy.h"
#include "av1/common/entropymode.h"
#include "av1/common/entropymv.h"
#include "av1/common/idct.h"
#include "av1/common/pred_common.h"
#include "av1/common/quant_common.h"
#include "av1/common/reconinter.h"
#include "av1/common/reconintra.h"
#if CONFIG_FRAME_SUPERRES
#include "av1/common/resize.h"
#endif  // CONFIG_FRAME_SUPERRES
#include "av1/common/seg_common.h"
#include "av1/common/thread_common.h"
#include "av1/common/tile_common.h"

#include "av1/decoder/decodeframe.h"
#include "av1/decoder/decodemv.h"
#include "av1/decoder/decoder.h"
#if CONFIG_LV_MAP
#include "av1/decoder/decodetxb.h"
#endif
#include "av1/decoder/detokenize.h"
#include "av1/decoder/dsubexp.h"

#if CONFIG_WARPED_MOTION || CONFIG_GLOBAL_MOTION
#include "av1/common/warped_motion.h"
#endif  // CONFIG_WARPED_MOTION || CONFIG_GLOBAL_MOTION

#define MAX_AV1_HEADER_SIZE 80
#define ACCT_STR __func__

#if CONFIG_PVQ
#include "av1/common/partition.h"
#include "av1/common/pvq.h"
#include "av1/common/scan.h"
#include "av1/decoder/decint.h"
#include "av1/decoder/pvq_decoder.h"
#include "av1/encoder/encodemb.h"
#include "av1/encoder/hybrid_fwd_txfm.h"
#endif

#if CONFIG_CFL
#include "av1/common/cfl.h"
#endif

static struct aom_read_bit_buffer *init_read_bit_buffer(
    AV1Decoder *pbi, struct aom_read_bit_buffer *rb, const uint8_t *data,
    const uint8_t *data_end, uint8_t clear_data[MAX_AV1_HEADER_SIZE]);
static int read_compressed_header(AV1Decoder *pbi, const uint8_t *data,
                                  size_t partition_size);
static size_t read_uncompressed_header(AV1Decoder *pbi,
                                       struct aom_read_bit_buffer *rb);

static int is_compound_reference_allowed(const AV1_COMMON *cm) {
#if CONFIG_ONE_SIDED_COMPOUND  // Normative in decoder
  return !frame_is_intra_only(cm);
#else
  int i;
  if (frame_is_intra_only(cm)) return 0;
  for (i = 1; i < INTER_REFS_PER_FRAME; ++i)
    if (cm->ref_frame_sign_bias[i + 1] != cm->ref_frame_sign_bias[1]) return 1;

  return 0;
#endif
}

static void setup_compound_reference_mode(AV1_COMMON *cm) {
#if CONFIG_EXT_REFS
  cm->comp_fwd_ref[0] = LAST_FRAME;
  cm->comp_fwd_ref[1] = LAST2_FRAME;
  cm->comp_fwd_ref[2] = LAST3_FRAME;
  cm->comp_fwd_ref[3] = GOLDEN_FRAME;

  cm->comp_bwd_ref[0] = BWDREF_FRAME;
  cm->comp_bwd_ref[1] = ALTREF_FRAME;
#else
  if (cm->ref_frame_sign_bias[LAST_FRAME] ==
      cm->ref_frame_sign_bias[GOLDEN_FRAME]) {
    cm->comp_fixed_ref = ALTREF_FRAME;
    cm->comp_var_ref[0] = LAST_FRAME;
    cm->comp_var_ref[1] = GOLDEN_FRAME;
  } else if (cm->ref_frame_sign_bias[LAST_FRAME] ==
             cm->ref_frame_sign_bias[ALTREF_FRAME]) {
    cm->comp_fixed_ref = GOLDEN_FRAME;
    cm->comp_var_ref[0] = LAST_FRAME;
    cm->comp_var_ref[1] = ALTREF_FRAME;
  } else {
    cm->comp_fixed_ref = LAST_FRAME;
    cm->comp_var_ref[0] = GOLDEN_FRAME;
    cm->comp_var_ref[1] = ALTREF_FRAME;
  }
#endif  // CONFIG_EXT_REFS
}

static int read_is_valid(const uint8_t *start, size_t len, const uint8_t *end) {
  return len != 0 && len <= (size_t)(end - start);
}

static int decode_unsigned_max(struct aom_read_bit_buffer *rb, int max) {
  const int data = aom_rb_read_literal(rb, get_unsigned_bits(max));
  return data > max ? max : data;
}

static TX_MODE read_tx_mode(AV1_COMMON *cm, MACROBLOCKD *xd,
                            struct aom_read_bit_buffer *rb) {
  int i, all_lossless = 1;
#if CONFIG_TX64X64
  TX_MODE tx_mode;
#endif

  if (cm->seg.enabled) {
    for (i = 0; i < MAX_SEGMENTS; ++i) {
      if (!xd->lossless[i]) {
        all_lossless = 0;
        break;
      }
    }
  } else {
    all_lossless = xd->lossless[0];
  }

  if (all_lossless) return ONLY_4X4;
#if CONFIG_TX64X64
  tx_mode = aom_rb_read_bit(rb) ? TX_MODE_SELECT : aom_rb_read_literal(rb, 2);
  if (tx_mode == ALLOW_32X32) tx_mode += aom_rb_read_bit(rb);
  return tx_mode;
#else
  return aom_rb_read_bit(rb) ? TX_MODE_SELECT : aom_rb_read_literal(rb, 2);
#endif  // CONFIG_TX64X64
}

#if !CONFIG_EC_ADAPT
static void read_tx_size_probs(FRAME_CONTEXT *fc, aom_reader *r) {
  int i, j, k;
  for (i = 0; i < MAX_TX_DEPTH; ++i)
    for (j = 0; j < TX_SIZE_CONTEXTS; ++j)
      for (k = 0; k < i + 1; ++k)
        av1_diff_update_prob(r, &fc->tx_size_probs[i][j][k], ACCT_STR);
}
#endif

#if !CONFIG_EC_ADAPT
static void read_switchable_interp_probs(FRAME_CONTEXT *fc, aom_reader *r) {
  int i, j;
  for (j = 0; j < SWITCHABLE_FILTER_CONTEXTS; ++j) {
    for (i = 0; i < SWITCHABLE_FILTERS - 1; ++i)
      av1_diff_update_prob(r, &fc->switchable_interp_prob[j][i], ACCT_STR);
  }
}
#endif

static void read_inter_mode_probs(FRAME_CONTEXT *fc, aom_reader *r) {
  int i;
  for (i = 0; i < NEWMV_MODE_CONTEXTS; ++i)
    av1_diff_update_prob(r, &fc->newmv_prob[i], ACCT_STR);
  for (i = 0; i < ZEROMV_MODE_CONTEXTS; ++i)
    av1_diff_update_prob(r, &fc->zeromv_prob[i], ACCT_STR);
  for (i = 0; i < REFMV_MODE_CONTEXTS; ++i)
    av1_diff_update_prob(r, &fc->refmv_prob[i], ACCT_STR);
  for (i = 0; i < DRL_MODE_CONTEXTS; ++i)
    av1_diff_update_prob(r, &fc->drl_prob[i], ACCT_STR);
}

#if CONFIG_EXT_INTER
static void read_inter_compound_mode_probs(FRAME_CONTEXT *fc, aom_reader *r) {
  int i, j;
  if (aom_read(r, GROUP_DIFF_UPDATE_PROB, ACCT_STR)) {
    for (j = 0; j < INTER_MODE_CONTEXTS; ++j) {
      for (i = 0; i < INTER_COMPOUND_MODES - 1; ++i) {
        av1_diff_update_prob(r, &fc->inter_compound_mode_probs[j][i], ACCT_STR);
      }
    }
  }
}

#if CONFIG_COMPOUND_SINGLEREF
static void read_inter_singleref_comp_mode_probs(FRAME_CONTEXT *fc,
                                                 aom_reader *r) {
  int i, j;
  if (aom_read(r, GROUP_DIFF_UPDATE_PROB, ACCT_STR)) {
    for (j = 0; j < INTER_MODE_CONTEXTS; ++j) {
      for (i = 0; i < INTER_SINGLEREF_COMP_MODES - 1; ++i) {
        av1_diff_update_prob(r, &fc->inter_singleref_comp_mode_probs[j][i],
                             ACCT_STR);
      }
    }
  }
}
#endif  // CONFIG_COMPOUND_SINGLEREF
#endif  // CONFIG_EXT_INTER

#if !CONFIG_EC_ADAPT
#if !CONFIG_EXT_TX
static void read_ext_tx_probs(FRAME_CONTEXT *fc, aom_reader *r) {
  int i, j, k;
  if (aom_read(r, GROUP_DIFF_UPDATE_PROB, ACCT_STR)) {
    for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
      for (j = 0; j < TX_TYPES; ++j) {
        for (k = 0; k < TX_TYPES - 1; ++k)
          av1_diff_update_prob(r, &fc->intra_ext_tx_prob[i][j][k], ACCT_STR);
      }
    }
  }
  if (aom_read(r, GROUP_DIFF_UPDATE_PROB, ACCT_STR)) {
    for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
      for (k = 0; k < TX_TYPES - 1; ++k)
        av1_diff_update_prob(r, &fc->inter_ext_tx_prob[i][k], ACCT_STR);
    }
  }
}
#endif
#endif

static REFERENCE_MODE read_frame_reference_mode(
    const AV1_COMMON *cm, struct aom_read_bit_buffer *rb) {
  if (is_compound_reference_allowed(cm)) {
#if CONFIG_REF_ADAPT
    return aom_rb_read_bit(rb) ? REFERENCE_MODE_SELECT : SINGLE_REFERENCE;
#else
    return aom_rb_read_bit(rb)
               ? REFERENCE_MODE_SELECT
               : (aom_rb_read_bit(rb) ? COMPOUND_REFERENCE : SINGLE_REFERENCE);
#endif  // CONFIG_REF_ADAPT
  } else {
    return SINGLE_REFERENCE;
  }
}

static void read_frame_reference_mode_probs(AV1_COMMON *cm, aom_reader *r) {
  FRAME_CONTEXT *const fc = cm->fc;
  int i, j;

  if (cm->reference_mode == REFERENCE_MODE_SELECT)
    for (i = 0; i < COMP_INTER_CONTEXTS; ++i)
      av1_diff_update_prob(r, &fc->comp_inter_prob[i], ACCT_STR);

  if (cm->reference_mode != COMPOUND_REFERENCE) {
    for (i = 0; i < REF_CONTEXTS; ++i) {
      for (j = 0; j < (SINGLE_REFS - 1); ++j) {
        av1_diff_update_prob(r, &fc->single_ref_prob[i][j], ACCT_STR);
      }
    }
  }

  if (cm->reference_mode != SINGLE_REFERENCE) {
    for (i = 0; i < REF_CONTEXTS; ++i) {
#if CONFIG_EXT_REFS
      for (j = 0; j < (FWD_REFS - 1); ++j)
        av1_diff_update_prob(r, &fc->comp_ref_prob[i][j], ACCT_STR);
      for (j = 0; j < (BWD_REFS - 1); ++j)
        av1_diff_update_prob(r, &fc->comp_bwdref_prob[i][j], ACCT_STR);
#else
      for (j = 0; j < (COMP_REFS - 1); ++j)
        av1_diff_update_prob(r, &fc->comp_ref_prob[i][j], ACCT_STR);
#endif  // CONFIG_EXT_REFS
    }
  }
}

static void update_mv_probs(aom_prob *p, int n, aom_reader *r) {
  int i;
  for (i = 0; i < n; ++i) av1_diff_update_prob(r, &p[i], ACCT_STR);
}

static void read_mv_probs(nmv_context *ctx, int allow_hp, aom_reader *r) {
  int i;

#if !CONFIG_EC_ADAPT
  int j;
  update_mv_probs(ctx->joints, MV_JOINTS - 1, r);

  for (i = 0; i < 2; ++i) {
    nmv_component *const comp_ctx = &ctx->comps[i];
    update_mv_probs(&comp_ctx->sign, 1, r);
    update_mv_probs(comp_ctx->classes, MV_CLASSES - 1, r);
    update_mv_probs(comp_ctx->class0, CLASS0_SIZE - 1, r);
    update_mv_probs(comp_ctx->bits, MV_OFFSET_BITS, r);
  }
  for (i = 0; i < 2; ++i) {
    nmv_component *const comp_ctx = &ctx->comps[i];
    for (j = 0; j < CLASS0_SIZE; ++j) {
      update_mv_probs(comp_ctx->class0_fp[j], MV_FP_SIZE - 1, r);
    }
    update_mv_probs(comp_ctx->fp, MV_FP_SIZE - 1, r);
  }
#endif  // !CONFIG_EC_ADAPT

  if (allow_hp) {
    for (i = 0; i < 2; ++i) {
      nmv_component *const comp_ctx = &ctx->comps[i];
      update_mv_probs(&comp_ctx->class0_hp, 1, r);
      update_mv_probs(&comp_ctx->hp, 1, r);
    }
  }
}

static void inverse_transform_block(MACROBLOCKD *xd, int plane,
                                    const TX_TYPE tx_type,
                                    const TX_SIZE tx_size, uint8_t *dst,
                                    int stride, int16_t scan_line, int eob) {
  struct macroblockd_plane *const pd = &xd->plane[plane];
  tran_low_t *const dqcoeff = pd->dqcoeff;
  av1_inverse_transform_block(xd, dqcoeff, tx_type, tx_size, dst, stride, eob);
  memset(dqcoeff, 0, (scan_line + 1) * sizeof(dqcoeff[0]));
}

#if CONFIG_PVQ
static int av1_pvq_decode_helper(MACROBLOCKD *xd, tran_low_t *ref_coeff,
                                 tran_low_t *dqcoeff, int16_t *quant, int pli,
                                 int bs, TX_TYPE tx_type, int xdec,
                                 PVQ_SKIP_TYPE ac_dc_coded) {
  unsigned int flags;  // used for daala's stream analyzer.
  int off;
  const int is_keyframe = 0;
  const int has_dc_skip = 1;
  int coeff_shift = 3 - av1_get_tx_scale(bs);
  int hbd_downshift = 0;
  int rounding_mask;
  // DC quantizer for PVQ
  int pvq_dc_quant;
  int lossless = (quant[0] == 0);
  const int blk_size = tx_size_wide[bs];
  int eob = 0;
  int i;
  od_dec_ctx *dec = &xd->daala_dec;
  int use_activity_masking = dec->use_activity_masking;
  DECLARE_ALIGNED(16, tran_low_t, dqcoeff_pvq[OD_TXSIZE_MAX * OD_TXSIZE_MAX]);
  DECLARE_ALIGNED(16, tran_low_t, ref_coeff_pvq[OD_TXSIZE_MAX * OD_TXSIZE_MAX]);

  od_coeff ref_int32[OD_TXSIZE_MAX * OD_TXSIZE_MAX];
  od_coeff out_int32[OD_TXSIZE_MAX * OD_TXSIZE_MAX];

  hbd_downshift = xd->bd - 8;

  od_raster_to_coding_order(ref_coeff_pvq, blk_size, tx_type, ref_coeff,
                            blk_size);

  assert(OD_COEFF_SHIFT >= 4);
  if (lossless)
    pvq_dc_quant = 1;
  else {
    if (use_activity_masking)
      pvq_dc_quant = OD_MAXI(
          1, (quant[0] << (OD_COEFF_SHIFT - 3) >> hbd_downshift) *
                     dec->state.pvq_qm_q4[pli][od_qm_get_index(bs, 0)] >>
                 4);
    else
      pvq_dc_quant =
          OD_MAXI(1, quant[0] << (OD_COEFF_SHIFT - 3) >> hbd_downshift);
  }

  off = od_qm_offset(bs, xdec);

  // copy int16 inputs to int32
  for (i = 0; i < blk_size * blk_size; i++) {
    ref_int32[i] =
        AOM_SIGNED_SHL(ref_coeff_pvq[i], OD_COEFF_SHIFT - coeff_shift) >>
        hbd_downshift;
  }

  od_pvq_decode(dec, ref_int32, out_int32,
                OD_MAXI(1, quant[1] << (OD_COEFF_SHIFT - 3) >> hbd_downshift),
                pli, bs, OD_PVQ_BETA[use_activity_masking][pli][bs],
                is_keyframe, &flags, ac_dc_coded, dec->state.qm + off,
                dec->state.qm_inv + off);

  if (!has_dc_skip || out_int32[0]) {
    out_int32[0] =
        has_dc_skip + generic_decode(dec->r, &dec->state.adapt->model_dc[pli],
                                     &dec->state.adapt->ex_dc[pli][bs][0], 2,
                                     "dc:mag");
    if (out_int32[0]) out_int32[0] *= aom_read_bit(dec->r, "dc:sign") ? -1 : 1;
  }
  out_int32[0] = out_int32[0] * pvq_dc_quant + ref_int32[0];

  // copy int32 result back to int16
  assert(OD_COEFF_SHIFT > coeff_shift);
  rounding_mask = (1 << (OD_COEFF_SHIFT - coeff_shift - 1)) - 1;
  for (i = 0; i < blk_size * blk_size; i++) {
    out_int32[i] = AOM_SIGNED_SHL(out_int32[i], hbd_downshift);
    dqcoeff_pvq[i] = (out_int32[i] + (out_int32[i] < 0) + rounding_mask) >>
                     (OD_COEFF_SHIFT - coeff_shift);
  }

  od_coding_order_to_raster(dqcoeff, blk_size, tx_type, dqcoeff_pvq, blk_size);

  eob = blk_size * blk_size;

  return eob;
}

static PVQ_SKIP_TYPE read_pvq_skip(AV1_COMMON *cm, MACROBLOCKD *const xd,
                                   int plane, TX_SIZE tx_size) {
  // decode ac/dc coded flag. bit0: DC coded, bit1 : AC coded
  // NOTE : we don't use 5 symbols for luma here in aom codebase,
  // since block partition is taken care of by aom.
  // So, only AC/DC skip info is coded
  const int ac_dc_coded = aom_read_symbol(
      xd->daala_dec.r,
      xd->daala_dec.state.adapt->skip_cdf[2 * tx_size + (plane != 0)], 4,
      "skip");
  if (ac_dc_coded < 0 || ac_dc_coded > 3) {
    aom_internal_error(&cm->error, AOM_CODEC_INVALID_PARAM,
                       "Invalid PVQ Skip Type");
  }
  return ac_dc_coded;
}

static int av1_pvq_decode_helper2(AV1_COMMON *cm, MACROBLOCKD *const xd,
                                  MB_MODE_INFO *const mbmi, int plane, int row,
                                  int col, TX_SIZE tx_size, TX_TYPE tx_type) {
  struct macroblockd_plane *const pd = &xd->plane[plane];
  // transform block size in pixels
  int tx_blk_size = tx_size_wide[tx_size];
  int i, j;
  tran_low_t *pvq_ref_coeff = pd->pvq_ref_coeff;
  const int diff_stride = tx_blk_size;
  int16_t *pred = pd->pred;
  tran_low_t *const dqcoeff = pd->dqcoeff;
  uint8_t *dst;
  int eob;
  const PVQ_SKIP_TYPE ac_dc_coded = read_pvq_skip(cm, xd, plane, tx_size);

  eob = 0;
  dst = &pd->dst.buf[4 * row * pd->dst.stride + 4 * col];

  if (ac_dc_coded) {
    int xdec = pd->subsampling_x;
    int seg_id = mbmi->segment_id;
    int16_t *quant;
    FWD_TXFM_PARAM fwd_txfm_param;
    // ToDo(yaowu): correct this with optimal number from decoding process.
    const int max_scan_line = tx_size_2d[tx_size];
#if CONFIG_HIGHBITDEPTH
    if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
      for (j = 0; j < tx_blk_size; j++)
        for (i = 0; i < tx_blk_size; i++)
          pred[diff_stride * j + i] =
              CONVERT_TO_SHORTPTR(dst)[pd->dst.stride * j + i];
    } else {
#endif
      for (j = 0; j < tx_blk_size; j++)
        for (i = 0; i < tx_blk_size; i++)
          pred[diff_stride * j + i] = dst[pd->dst.stride * j + i];
#if CONFIG_HIGHBITDEPTH
    }
#endif

    fwd_txfm_param.tx_type = tx_type;
    fwd_txfm_param.tx_size = tx_size;
    fwd_txfm_param.lossless = xd->lossless[seg_id];

#if CONFIG_HIGHBITDEPTH
    if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
      fwd_txfm_param.bd = xd->bd;
      av1_highbd_fwd_txfm(pred, pvq_ref_coeff, diff_stride, &fwd_txfm_param);
    } else {
#endif  // CONFIG_HIGHBITDEPTH
      av1_fwd_txfm(pred, pvq_ref_coeff, diff_stride, &fwd_txfm_param);
#if CONFIG_HIGHBITDEPTH
    }
#endif  // CONFIG_HIGHBITDEPTH

    quant = &pd->seg_dequant[seg_id][0];  // aom's quantizer

    eob = av1_pvq_decode_helper(xd, pvq_ref_coeff, dqcoeff, quant, plane,
                                tx_size, tx_type, xdec, ac_dc_coded);

    inverse_transform_block(xd, plane, tx_type, tx_size, dst, pd->dst.stride,
                            max_scan_line, eob);
  }

  return eob;
}
#endif

static int get_block_idx(const MACROBLOCKD *xd, int plane, int row, int col) {
  const int bsize = xd->mi[0]->mbmi.sb_type;
  const struct macroblockd_plane *pd = &xd->plane[plane];
#if CONFIG_CB4X4
#if CONFIG_CHROMA_2X2
  const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd);
#else
  const BLOCK_SIZE plane_bsize =
      AOMMAX(BLOCK_4X4, get_plane_block_size(bsize, pd));
#endif  // CONFIG_CHROMA_2X2
#else
  const BLOCK_SIZE plane_bsize =
      get_plane_block_size(AOMMAX(BLOCK_8X8, bsize), pd);
#endif
  const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane);
  const TX_SIZE tx_size = get_tx_size(plane, xd);
  const uint8_t txh_unit = tx_size_high_unit[tx_size];
  return row * max_blocks_wide + col * txh_unit;
}

#if CONFIG_DPCM_INTRA
static void process_block_dpcm_vert(TX_SIZE tx_size, TX_TYPE_1D tx_type_1d,
                                    const tran_low_t *dqcoeff, uint8_t *dst,
                                    int dst_stride) {
  const int tx1d_width = tx_size_wide[tx_size];
  const int tx1d_height = tx_size_high[tx_size];
  dpcm_inv_txfm_add_func inverse_tx =
      av1_get_dpcm_inv_txfm_add_func(tx1d_width);
  for (int r = 0; r < tx1d_height; ++r) {
    if (r > 0) memcpy(dst, dst - dst_stride, tx1d_width * sizeof(dst[0]));
    inverse_tx(dqcoeff, 1, tx_type_1d, dst);
    dqcoeff += tx1d_width;
    dst += dst_stride;
  }
}

static void process_block_dpcm_horz(TX_SIZE tx_size, TX_TYPE_1D tx_type_1d,
                                    const tran_low_t *dqcoeff, uint8_t *dst,
                                    int dst_stride) {
  const int tx1d_width = tx_size_wide[tx_size];
  const int tx1d_height = tx_size_high[tx_size];
  dpcm_inv_txfm_add_func inverse_tx =
      av1_get_dpcm_inv_txfm_add_func(tx1d_height);
  tran_low_t tx_buff[64];
  for (int c = 0; c < tx1d_width; ++c, ++dqcoeff, ++dst) {
    for (int r = 0; r < tx1d_height; ++r) {
      if (c > 0) dst[r * dst_stride] = dst[r * dst_stride - 1];
      tx_buff[r] = dqcoeff[r * tx1d_width];
    }
    inverse_tx(tx_buff, dst_stride, tx_type_1d, dst);
  }
}

#if CONFIG_HIGHBITDEPTH
static void hbd_process_block_dpcm_vert(TX_SIZE tx_size, TX_TYPE_1D tx_type_1d,
                                        int bd, const tran_low_t *dqcoeff,
                                        uint8_t *dst8, int dst_stride) {
  uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
  const int tx1d_width = tx_size_wide[tx_size];
  const int tx1d_height = tx_size_high[tx_size];
  hbd_dpcm_inv_txfm_add_func inverse_tx =
      av1_get_hbd_dpcm_inv_txfm_add_func(tx1d_width);
  for (int r = 0; r < tx1d_height; ++r) {
    if (r > 0) memcpy(dst, dst - dst_stride, tx1d_width * sizeof(dst[0]));
    inverse_tx(dqcoeff, 1, tx_type_1d, bd, dst, 1);
    dqcoeff += tx1d_width;
    dst += dst_stride;
  }
}

static void hbd_process_block_dpcm_horz(TX_SIZE tx_size, TX_TYPE_1D tx_type_1d,
                                        int bd, const tran_low_t *dqcoeff,
                                        uint8_t *dst8, int dst_stride) {
  uint16_t *dst = CONVERT_TO_SHORTPTR(dst8);
  const int tx1d_width = tx_size_wide[tx_size];
  const int tx1d_height = tx_size_high[tx_size];
  hbd_dpcm_inv_txfm_add_func inverse_tx =
      av1_get_hbd_dpcm_inv_txfm_add_func(tx1d_height);
  tran_low_t tx_buff[64];
  switch (tx1d_height) {
    case 4: inverse_tx = av1_hbd_dpcm_inv_txfm_add_4_c; break;
    case 8: inverse_tx = av1_hbd_dpcm_inv_txfm_add_8_c; break;
    case 16: inverse_tx = av1_hbd_dpcm_inv_txfm_add_16_c; break;
    case 32: inverse_tx = av1_hbd_dpcm_inv_txfm_add_32_c; break;
    default: assert(0);
  }

  for (int c = 0; c < tx1d_width; ++c, ++dqcoeff, ++dst) {
    for (int r = 0; r < tx1d_height; ++r) {
      if (c > 0) dst[r * dst_stride] = dst[r * dst_stride - 1];
      tx_buff[r] = dqcoeff[r * tx1d_width];
    }
    inverse_tx(tx_buff, dst_stride, tx_type_1d, bd, dst, 0);
  }
}
#endif  // CONFIG_HIGHBITDEPTH

static void inverse_transform_block_dpcm(MACROBLOCKD *xd, int plane,
                                         PREDICTION_MODE mode, TX_SIZE tx_size,
                                         TX_TYPE tx_type, uint8_t *dst,
                                         int dst_stride, int16_t scan_line) {
  struct macroblockd_plane *const pd = &xd->plane[plane];
  tran_low_t *const dqcoeff = pd->dqcoeff;
  TX_TYPE_1D tx_type_1d = DCT_1D;
  switch (tx_type) {
    case IDTX: tx_type_1d = IDTX_1D; break;
    case V_DCT:
      assert(mode == H_PRED);
      tx_type_1d = DCT_1D;
      break;
    case H_DCT:
      assert(mode == V_PRED);
      tx_type_1d = DCT_1D;
      break;
    default: assert(0);
  }
  switch (mode) {
    case V_PRED:
#if CONFIG_HIGHBITDEPTH
      if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
        hbd_process_block_dpcm_vert(tx_size, tx_type_1d, xd->bd, dqcoeff, dst,
                                    dst_stride);
      } else {
#endif  // CONFIG_HIGHBITDEPTH
        process_block_dpcm_vert(tx_size, tx_type_1d, dqcoeff, dst, dst_stride);
#if CONFIG_HIGHBITDEPTH
      }
#endif  // CONFIG_HIGHBITDEPTH
      break;
    case H_PRED:
#if CONFIG_HIGHBITDEPTH
      if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
        hbd_process_block_dpcm_horz(tx_size, tx_type_1d, xd->bd, dqcoeff, dst,
                                    dst_stride);
      } else {
#endif  // CONFIG_HIGHBITDEPTH
        process_block_dpcm_horz(tx_size, tx_type_1d, dqcoeff, dst, dst_stride);
#if CONFIG_HIGHBITDEPTH
      }
#endif  // CONFIG_HIGHBITDEPTH
      break;
    default: assert(0);
  }
  memset(dqcoeff, 0, (scan_line + 1) * sizeof(dqcoeff[0]));
}
#endif  // CONFIG_DPCM_INTRA

static void predict_and_reconstruct_intra_block(
    AV1_COMMON *cm, MACROBLOCKD *const xd, aom_reader *const r,
    MB_MODE_INFO *const mbmi, int plane, int row, int col, TX_SIZE tx_size) {
  PLANE_TYPE plane_type = get_plane_type(plane);
  const int block_idx = get_block_idx(xd, plane, row, col);
#if CONFIG_PVQ
  (void)r;
#endif
  av1_predict_intra_block_facade(xd, plane, block_idx, col, row, tx_size);

  if (!mbmi->skip) {
#if !CONFIG_PVQ
    struct macroblockd_plane *const pd = &xd->plane[plane];
#if CONFIG_LV_MAP
    int16_t max_scan_line = 0;
    int eob;
    av1_read_coeffs_txb_facade(cm, xd, r, row, col, block_idx, plane,
                               pd->dqcoeff, &max_scan_line, &eob);
    // tx_type will be read out in av1_read_coeffs_txb_facade
    TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, tx_size);
#else   // CONFIG_LV_MAP
    TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, tx_size);
    const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, 0);
    int16_t max_scan_line = 0;
    const int eob =
        av1_decode_block_tokens(cm, xd, plane, scan_order, col, row, tx_size,
                                tx_type, &max_scan_line, r, mbmi->segment_id);
#endif  // CONFIG_LV_MAP
    if (eob) {
      uint8_t *dst =
          &pd->dst.buf[(row * pd->dst.stride + col) << tx_size_wide_log2[0]];
#if CONFIG_DPCM_INTRA
      const int block_raster_idx =
          av1_block_index_to_raster_order(tx_size, block_idx);
      const PREDICTION_MODE mode = (plane == 0)
                                       ? get_y_mode(xd->mi[0], block_raster_idx)
                                       : mbmi->uv_mode;
      if (av1_use_dpcm_intra(plane, mode, tx_type, mbmi)) {
        inverse_transform_block_dpcm(xd, plane, mode, tx_size, tx_type, dst,
                                     pd->dst.stride, max_scan_line);
      } else {
#endif  // CONFIG_DPCM_INTRA
        inverse_transform_block(xd, plane, tx_type, tx_size, dst,
                                pd->dst.stride, max_scan_line, eob);
#if CONFIG_DPCM_INTRA
      }
#endif  // CONFIG_DPCM_INTRA
    }
#else
    TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, tx_size);
    av1_pvq_decode_helper2(cm, xd, mbmi, plane, row, col, tx_size, tx_type);
#endif
  }
#if CONFIG_CFL
  if (plane == AOM_PLANE_Y) {
    struct macroblockd_plane *const pd = &xd->plane[plane];
    uint8_t *dst =
        &pd->dst.buf[(row * pd->dst.stride + col) << tx_size_wide_log2[0]];
    cfl_store(xd->cfl, dst, pd->dst.stride, row, col, tx_size);
  }
#endif
}

#if CONFIG_VAR_TX && !CONFIG_COEF_INTERLEAVE
static void decode_reconstruct_tx(AV1_COMMON *cm, MACROBLOCKD *const xd,
                                  aom_reader *r, MB_MODE_INFO *const mbmi,
                                  int plane, BLOCK_SIZE plane_bsize,
                                  int blk_row, int blk_col, TX_SIZE tx_size,
                                  int *eob_total) {
  const struct macroblockd_plane *const pd = &xd->plane[plane];
  const BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
  const int tx_row = blk_row >> (1 - pd->subsampling_y);
  const int tx_col = blk_col >> (1 - pd->subsampling_x);
  const TX_SIZE plane_tx_size =
      plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0]
            : mbmi->inter_tx_size[tx_row][tx_col];
  // Scale to match transform block unit.
  const int max_blocks_high = max_block_high(xd, plane_bsize, plane);
  const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane);

  if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;

  if (tx_size == plane_tx_size) {
    PLANE_TYPE plane_type = get_plane_type(plane);
    int block_idx = get_block_idx(xd, plane, blk_row, blk_col);
#if CONFIG_LV_MAP
    int16_t max_scan_line = 0;
    int eob;
    av1_read_coeffs_txb_facade(cm, xd, r, blk_row, blk_col, block_idx, plane,
                               pd->dqcoeff, &max_scan_line, &eob);
    // tx_type will be read out in av1_read_coeffs_txb_facade
    TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, plane_tx_size);
#else   // CONFIG_LV_MAP
    TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, plane_tx_size);
    const SCAN_ORDER *sc = get_scan(cm, plane_tx_size, tx_type, 1);
    int16_t max_scan_line = 0;
    const int eob = av1_decode_block_tokens(
        cm, xd, plane, sc, blk_col, blk_row, plane_tx_size, tx_type,
        &max_scan_line, r, mbmi->segment_id);
#endif  // CONFIG_LV_MAP
    inverse_transform_block(xd, plane, tx_type, plane_tx_size,
                            &pd->dst.buf[(blk_row * pd->dst.stride + blk_col)
                                         << tx_size_wide_log2[0]],
                            pd->dst.stride, max_scan_line, eob);
    *eob_total += eob;
  } else {
    const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
    const int bsl = tx_size_wide_unit[sub_txs];
    assert(sub_txs < tx_size);
    int i;

    assert(bsl > 0);

    for (i = 0; i < 4; ++i) {
      const int offsetr = blk_row + (i >> 1) * bsl;
      const int offsetc = blk_col + (i & 0x01) * bsl;

      if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;

      decode_reconstruct_tx(cm, xd, r, mbmi, plane, plane_bsize, offsetr,
                            offsetc, sub_txs, eob_total);
    }
  }
}
#endif  // CONFIG_VAR_TX

#if !CONFIG_VAR_TX || CONFIG_SUPERTX || CONFIG_COEF_INTERLEAVE || \
    (!CONFIG_VAR_TX && CONFIG_EXT_TX && CONFIG_RECT_TX)
static int reconstruct_inter_block(AV1_COMMON *cm, MACROBLOCKD *const xd,
                                   aom_reader *const r, int segment_id,
                                   int plane, int row, int col,
                                   TX_SIZE tx_size) {
  PLANE_TYPE plane_type = get_plane_type(plane);
  int block_idx = get_block_idx(xd, plane, row, col);
#if CONFIG_PVQ
  int eob;
  (void)r;
  (void)segment_id;
#else
  struct macroblockd_plane *const pd = &xd->plane[plane];
#endif

#if !CONFIG_PVQ
#if CONFIG_LV_MAP
  (void)segment_id;
  int16_t max_scan_line = 0;
  int eob;
  av1_read_coeffs_txb_facade(cm, xd, r, row, col, block_idx, plane, pd->dqcoeff,
                             &max_scan_line, &eob);
  // tx_type will be read out in av1_read_coeffs_txb_facade
  TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, tx_size);
#else   // CONFIG_LV_MAP
  int16_t max_scan_line = 0;
  TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, tx_size);
  const SCAN_ORDER *scan_order = get_scan(cm, tx_size, tx_type, 1);
  const int eob =
      av1_decode_block_tokens(cm, xd, plane, scan_order, col, row, tx_size,
                              tx_type, &max_scan_line, r, segment_id);
#endif  // CONFIG_LV_MAP
  uint8_t *dst =
      &pd->dst.buf[(row * pd->dst.stride + col) << tx_size_wide_log2[0]];
  if (eob)
    inverse_transform_block(xd, plane, tx_type, tx_size, dst, pd->dst.stride,
                            max_scan_line, eob);
#else
  TX_TYPE tx_type = get_tx_type(plane_type, xd, block_idx, tx_size);
  eob = av1_pvq_decode_helper2(cm, xd, &xd->mi[0]->mbmi, plane, row, col,
                               tx_size, tx_type);
#endif
  return eob;
}
#endif  // !CONFIG_VAR_TX || CONFIG_SUPER_TX

static void set_offsets(AV1_COMMON *const cm, MACROBLOCKD *const xd,
                        BLOCK_SIZE bsize, int mi_row, int mi_col, int bw,
                        int bh, int x_mis, int y_mis) {
  const int offset = mi_row * cm->mi_stride + mi_col;
  int x, y;
  const TileInfo *const tile = &xd->tile;

  xd->mi = cm->mi_grid_visible + offset;
  xd->mi[0] = &cm->mi[offset];
  // TODO(slavarnway): Generate sb_type based on bwl and bhl, instead of
  // passing bsize from decode_partition().
  xd->mi[0]->mbmi.sb_type = bsize;
#if CONFIG_RD_DEBUG
  xd->mi[0]->mbmi.mi_row = mi_row;
  xd->mi[0]->mbmi.mi_col = mi_col;
#endif
  for (y = 0; y < y_mis; ++y)
    for (x = !y; x < x_mis; ++x) xd->mi[y * cm->mi_stride + x] = xd->mi[0];