bitstream.c

/*
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
 *
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
 */

#include <assert.h>
#include <limits.h>
#include <stdio.h>

#include "aom/aom_encoder.h"
#include "aom_dsp/aom_dsp_common.h"
#include "aom_dsp/binary_codes_writer.h"
#include "aom_dsp/bitwriter_buffer.h"
#include "aom_mem/aom_mem.h"
#include "aom_ports/mem_ops.h"
#include "aom_ports/system_state.h"
#if CONFIG_BITSTREAM_DEBUG
#include "aom_util/debug_util.h"
#endif  // CONFIG_BITSTREAM_DEBUG

#include "av1/common/cdef.h"
#include "av1/common/entropy.h"
#include "av1/common/entropymode.h"
#include "av1/common/entropymv.h"
#include "av1/common/mvref_common.h"
#include "av1/common/odintrin.h"
#include "av1/common/pred_common.h"
#include "av1/common/reconinter.h"
#if CONFIG_EXT_INTRA
#include "av1/common/reconintra.h"
#endif  // CONFIG_EXT_INTRA
#include "av1/common/seg_common.h"
#include "av1/common/tile_common.h"

#if CONFIG_LV_MAP
#include "av1/encoder/encodetxb.h"
#endif  // CONFIG_LV_MAP
#include "av1/encoder/bitstream.h"
#include "av1/encoder/cost.h"
#include "av1/encoder/encodemv.h"
#include "av1/encoder/mcomp.h"
#if CONFIG_PALETTE_DELTA_ENCODING
#include "av1/encoder/palette.h"
#endif  // CONFIG_PALETTE_DELTA_ENCODING
#include "av1/encoder/segmentation.h"
#include "av1/encoder/subexp.h"
#include "av1/encoder/tokenize.h"

#define ENC_MISMATCH_DEBUG 0

#if CONFIG_COMPOUND_SINGLEREF
static struct av1_token
    inter_singleref_comp_mode_encodings[INTER_SINGLEREF_COMP_MODES];
#endif  // CONFIG_COMPOUND_SINGLEREF

static INLINE void write_uniform(aom_writer *w, int n, int v) {
  const int l = get_unsigned_bits(n);
  const int m = (1 << l) - n;
  if (l == 0) return;
  if (v < m) {
    aom_write_literal(w, v, l - 1);
  } else {
    aom_write_literal(w, m + ((v - m) >> 1), l - 1);
    aom_write_literal(w, (v - m) & 1, 1);
  }
}

static struct av1_token interintra_mode_encodings[INTERINTRA_MODES];
static struct av1_token compound_type_encodings[COMPOUND_TYPES];
#if CONFIG_LOOP_RESTORATION
static void loop_restoration_write_sb_coeffs(const AV1_COMMON *const cm,
                                             MACROBLOCKD *xd,
                                             const RestorationUnitInfo *rui,
                                             aom_writer *const w, int plane);
#endif  // CONFIG_LOOP_RESTORATION
#if CONFIG_OBU
static void write_uncompressed_header_obu(AV1_COMP *cpi,
                                          struct aom_write_bit_buffer *wb);
#else
static void write_uncompressed_header_frame(AV1_COMP *cpi,
                                            struct aom_write_bit_buffer *wb);
#endif

static uint32_t write_compressed_header(AV1_COMP *cpi, uint8_t *data);

#if !CONFIG_OBU || CONFIG_EXT_TILE
static int remux_tiles(const AV1_COMMON *const cm, uint8_t *dst,
                       const uint32_t data_size, const uint32_t max_tile_size,
                       const uint32_t max_tile_col_size,
                       int *const tile_size_bytes,
                       int *const tile_col_size_bytes);
#endif
void av1_encode_token_init(void) {
  av1_tokens_from_tree(interintra_mode_encodings, av1_interintra_mode_tree);
#if CONFIG_COMPOUND_SINGLEREF
  av1_tokens_from_tree(inter_singleref_comp_mode_encodings,
                       av1_inter_singleref_comp_mode_tree);
#endif  // CONFIG_COMPOUND_SINGLEREF
  av1_tokens_from_tree(compound_type_encodings, av1_compound_type_tree);
}

static void write_intra_mode_kf(const AV1_COMMON *cm, FRAME_CONTEXT *frame_ctx,
                                const MODE_INFO *mi, const MODE_INFO *above_mi,
                                const MODE_INFO *left_mi, int block,
                                PREDICTION_MODE mode, aom_writer *w) {
#if CONFIG_INTRABC
  assert(!is_intrabc_block(&mi->mbmi));
#endif  // CONFIG_INTRABC
  aom_write_symbol(w, mode,
                   get_y_mode_cdf(frame_ctx, mi, above_mi, left_mi, block),
                   INTRA_MODES);
  (void)cm;
}

static void write_inter_mode(aom_writer *w, PREDICTION_MODE mode,
                             FRAME_CONTEXT *ec_ctx, const int16_t mode_ctx) {
  const int16_t newmv_ctx = mode_ctx & NEWMV_CTX_MASK;

#if CONFIG_NEW_MULTISYMBOL
  aom_write_symbol(w, mode != NEWMV, ec_ctx->newmv_cdf[newmv_ctx], 2);
#else
  aom_write(w, mode != NEWMV, ec_ctx->newmv_prob[newmv_ctx]);
#endif

  if (mode != NEWMV) {
    if (mode_ctx & (1 << ALL_ZERO_FLAG_OFFSET)) {
      assert(mode == GLOBALMV);
      return;
    }

    const int16_t zeromv_ctx =
        (mode_ctx >> GLOBALMV_OFFSET) & GLOBALMV_CTX_MASK;
#if CONFIG_NEW_MULTISYMBOL
    aom_write_symbol(w, mode != GLOBALMV, ec_ctx->zeromv_cdf[zeromv_ctx], 2);
#else
    aom_write(w, mode != GLOBALMV, ec_ctx->zeromv_prob[zeromv_ctx]);
#endif

    if (mode != GLOBALMV) {
      int16_t refmv_ctx = (mode_ctx >> REFMV_OFFSET) & REFMV_CTX_MASK;

      if (mode_ctx & (1 << SKIP_NEARESTMV_OFFSET)) refmv_ctx = 6;
      if (mode_ctx & (1 << SKIP_NEARMV_OFFSET)) refmv_ctx = 7;
      if (mode_ctx & (1 << SKIP_NEARESTMV_SUB8X8_OFFSET)) refmv_ctx = 8;
#if CONFIG_NEW_MULTISYMBOL
      aom_write_symbol(w, mode != NEARESTMV, ec_ctx->refmv_cdf[refmv_ctx], 2);
#else
      aom_write(w, mode != NEARESTMV, ec_ctx->refmv_prob[refmv_ctx]);
#endif
    }
  }
}

static void write_drl_idx(FRAME_CONTEXT *ec_ctx, const MB_MODE_INFO *mbmi,
                          const MB_MODE_INFO_EXT *mbmi_ext, aom_writer *w) {
  uint8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame);

  assert(mbmi->ref_mv_idx < 3);

  const int new_mv =
#if CONFIG_COMPOUND_SINGLEREF
      mbmi->mode == SR_NEW_NEWMV ||
#endif
      mbmi->mode == NEWMV || mbmi->mode == NEW_NEWMV;
  if (new_mv) {
    int idx;
    for (idx = 0; idx < 2; ++idx) {
      if (mbmi_ext->ref_mv_count[ref_frame_type] > idx + 1) {
        uint8_t drl_ctx =
            av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx);

#if CONFIG_NEW_MULTISYMBOL
        aom_write_symbol(w, mbmi->ref_mv_idx != idx, ec_ctx->drl_cdf[drl_ctx],
                         2);
#else
        aom_write(w, mbmi->ref_mv_idx != idx, ec_ctx->drl_prob[drl_ctx]);
#endif
        if (mbmi->ref_mv_idx == idx) return;
      }
    }
    return;
  }

  if (have_nearmv_in_inter_mode(mbmi->mode)) {
    int idx;
    // TODO(jingning): Temporary solution to compensate the NEARESTMV offset.
    for (idx = 1; idx < 3; ++idx) {
      if (mbmi_ext->ref_mv_count[ref_frame_type] > idx + 1) {
        uint8_t drl_ctx =
            av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx);
#if CONFIG_NEW_MULTISYMBOL
        aom_write_symbol(w, mbmi->ref_mv_idx != (idx - 1),
                         ec_ctx->drl_cdf[drl_ctx], 2);
#else
        aom_write(w, mbmi->ref_mv_idx != (idx - 1), ec_ctx->drl_prob[drl_ctx]);
#endif
        if (mbmi->ref_mv_idx == (idx - 1)) return;
      }
    }
    return;
  }
}

static void write_inter_compound_mode(AV1_COMMON *cm, MACROBLOCKD *xd,
                                      aom_writer *w, PREDICTION_MODE mode,
                                      const int16_t mode_ctx) {
  assert(is_inter_compound_mode(mode));
  (void)cm;
  aom_write_symbol(w, INTER_COMPOUND_OFFSET(mode),
                   xd->tile_ctx->inter_compound_mode_cdf[mode_ctx],
                   INTER_COMPOUND_MODES);
}

#if CONFIG_COMPOUND_SINGLEREF
static void write_inter_singleref_comp_mode(MACROBLOCKD *xd, aom_writer *w,
                                            PREDICTION_MODE mode,
                                            const int16_t mode_ctx) {
  assert(is_inter_singleref_comp_mode(mode));
  aom_cdf_prob *const inter_singleref_comp_cdf =
      xd->tile_ctx->inter_singleref_comp_mode_cdf[mode_ctx];

  aom_write_symbol(w, INTER_SINGLEREF_COMP_OFFSET(mode),
                   inter_singleref_comp_cdf, INTER_SINGLEREF_COMP_MODES);
}
#endif  // CONFIG_COMPOUND_SINGLEREF

static void encode_unsigned_max(struct aom_write_bit_buffer *wb, int data,
                                int max) {
  aom_wb_write_literal(wb, data, get_unsigned_bits(max));
}

static void write_tx_size_vartx(const AV1_COMMON *cm, MACROBLOCKD *xd,
                                const MB_MODE_INFO *mbmi, TX_SIZE tx_size,
                                int depth, int blk_row, int blk_col,
                                aom_writer *w) {
#if CONFIG_NEW_MULTISYMBOL
  FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
  (void)cm;
#endif
  const int tx_row = blk_row >> 1;
  const int tx_col = blk_col >> 1;
  const int max_blocks_high = max_block_high(xd, mbmi->sb_type, 0);
  const int max_blocks_wide = max_block_wide(xd, mbmi->sb_type, 0);

  int ctx = txfm_partition_context(xd->above_txfm_context + blk_col,
                                   xd->left_txfm_context + blk_row,
                                   mbmi->sb_type, tx_size);

  if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;

  if (depth == MAX_VARTX_DEPTH) {
    txfm_partition_update(xd->above_txfm_context + blk_col,
                          xd->left_txfm_context + blk_row, tx_size, tx_size);
    return;
  }

  const int write_txfm_partition =
#if CONFIG_RECT_TX_EXT
      tx_size == mbmi->inter_tx_size[tx_row][tx_col] ||
      mbmi->tx_size == quarter_txsize_lookup[mbmi->sb_type];
#else
      tx_size == mbmi->inter_tx_size[tx_row][tx_col];
#endif
  if (write_txfm_partition) {
#if CONFIG_NEW_MULTISYMBOL
    aom_write_symbol(w, 0, ec_ctx->txfm_partition_cdf[ctx], 2);
#else
    aom_write(w, 0, cm->fc->txfm_partition_prob[ctx]);
#endif

    txfm_partition_update(xd->above_txfm_context + blk_col,
                          xd->left_txfm_context + blk_row, tx_size, tx_size);
    // TODO(yuec): set correct txfm partition update for qttx
  } else {
    const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
    const int bsl = tx_size_wide_unit[sub_txs];
    int i;

#if CONFIG_NEW_MULTISYMBOL
    aom_write_symbol(w, 1, ec_ctx->txfm_partition_cdf[ctx], 2);
#else
    aom_write(w, 1, cm->fc->txfm_partition_prob[ctx]);
#endif

    if (sub_txs == TX_4X4) {
      txfm_partition_update(xd->above_txfm_context + blk_col,
                            xd->left_txfm_context + blk_row, sub_txs, tx_size);
      return;
    }

    assert(bsl > 0);
    for (i = 0; i < 4; ++i) {
      int offsetr = blk_row + (i >> 1) * bsl;
      int offsetc = blk_col + (i & 0x01) * bsl;
      write_tx_size_vartx(cm, xd, mbmi, sub_txs, depth + 1, offsetr, offsetc,
                          w);
    }
  }
}

#if !CONFIG_NEW_MULTISYMBOL
static void update_txfm_partition_probs(AV1_COMMON *cm, aom_writer *w,
                                        FRAME_COUNTS *counts, int probwt) {
  int k;
  for (k = 0; k < TXFM_PARTITION_CONTEXTS; ++k)
    av1_cond_prob_diff_update(w, &cm->fc->txfm_partition_prob[k],
                              counts->txfm_partition[k], probwt);
}
#endif  // CONFIG_NEW_MULTISYMBOL

static void write_selected_tx_size(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                                   aom_writer *w) {
  const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
  const BLOCK_SIZE bsize = mbmi->sb_type;
  FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
  (void)cm;
  if (block_signals_txsize(bsize)) {
    const TX_SIZE tx_size = mbmi->tx_size;
    const int tx_size_ctx = get_tx_size_context(xd);
    const int32_t tx_size_cat = intra_tx_size_cat_lookup[bsize];
    const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size];
    const int depth = tx_size_to_depth(coded_tx_size, tx_size_cat);
    const int max_depths = tx_size_cat_to_max_depth(tx_size_cat);

    assert(coded_tx_size <= tx_size_cat + 1);
    assert(depth >= 0 && depth <= max_depths);

    assert(!is_inter_block(mbmi));
    assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(xd, mbmi)));

    aom_write_symbol(w, depth, ec_ctx->tx_size_cdf[tx_size_cat][tx_size_ctx],
                     max_depths + 1);
#if CONFIG_RECT_TX_EXT
    if (is_quarter_tx_allowed(xd, mbmi, 0) && tx_size != coded_tx_size)
#if CONFIG_NEW_MULTISYMBOL
      aom_write_symbol(w, tx_size == quarter_txsize_lookup[bsize],
                       cm->fc->quarter_tx_size_cdf, 2);
#else
      aom_write(w, tx_size == quarter_txsize_lookup[bsize],
                cm->fc->quarter_tx_size_prob);
#endif
#endif
  }
}

#if !CONFIG_NEW_MULTISYMBOL
static void update_inter_mode_probs(AV1_COMMON *cm, aom_writer *w,
                                    FRAME_COUNTS *counts) {
  int i;
  const int probwt = cm->num_tg;
  for (i = 0; i < NEWMV_MODE_CONTEXTS; ++i)
    av1_cond_prob_diff_update(w, &cm->fc->newmv_prob[i], counts->newmv_mode[i],
                              probwt);
  for (i = 0; i < GLOBALMV_MODE_CONTEXTS; ++i)
    av1_cond_prob_diff_update(w, &cm->fc->zeromv_prob[i],
                              counts->zeromv_mode[i], probwt);
  for (i = 0; i < REFMV_MODE_CONTEXTS; ++i)
    av1_cond_prob_diff_update(w, &cm->fc->refmv_prob[i], counts->refmv_mode[i],
                              probwt);
  for (i = 0; i < DRL_MODE_CONTEXTS; ++i)
    av1_cond_prob_diff_update(w, &cm->fc->drl_prob[i], counts->drl_mode[i],
                              probwt);
}
#endif

static int write_skip(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                      int segment_id, const MODE_INFO *mi, aom_writer *w) {
  if (segfeature_active(&cm->seg, segment_id, SEG_LVL_SKIP)) {
    return 1;
  } else {
    const int skip = mi->mbmi.skip;
    const int ctx = av1_get_skip_context(xd);
#if CONFIG_NEW_MULTISYMBOL
    FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
    aom_write_symbol(w, skip, ec_ctx->skip_cdfs[ctx], 2);
#else
    aom_write(w, skip, cm->fc->skip_probs[ctx]);
#endif
    return skip;
  }
}

static void write_is_inter(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                           int segment_id, aom_writer *w, const int is_inter) {
  if (!segfeature_active(&cm->seg, segment_id, SEG_LVL_REF_FRAME)) {
    const int ctx = av1_get_intra_inter_context(xd);
#if CONFIG_NEW_MULTISYMBOL
    FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
    aom_write_symbol(w, is_inter, ec_ctx->intra_inter_cdf[ctx], 2);
#else
    aom_write(w, is_inter, cm->fc->intra_inter_prob[ctx]);
#endif
  }
}

static void write_motion_mode(const AV1_COMMON *cm, MACROBLOCKD *xd,
                              const MODE_INFO *mi, aom_writer *w) {
  const MB_MODE_INFO *mbmi = &mi->mbmi;

  MOTION_MODE last_motion_mode_allowed =
      motion_mode_allowed(0, cm->global_motion, xd, mi);
  switch (last_motion_mode_allowed) {
    case SIMPLE_TRANSLATION: break;
    case OBMC_CAUSAL:
#if CONFIG_NEW_MULTISYMBOL
      aom_write_symbol(w, mbmi->motion_mode == OBMC_CAUSAL,
                       xd->tile_ctx->obmc_cdf[mbmi->sb_type], 2);
#else
      aom_write(w, mbmi->motion_mode == OBMC_CAUSAL,
                cm->fc->obmc_prob[mbmi->sb_type]);
#endif
      break;
    default:
      aom_write_symbol(w, mbmi->motion_mode,
                       xd->tile_ctx->motion_mode_cdf[mbmi->sb_type],
                       MOTION_MODES);
  }
}

static void write_delta_qindex(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                               int delta_qindex, aom_writer *w) {
  int sign = delta_qindex < 0;
  int abs = sign ? -delta_qindex : delta_qindex;
  int rem_bits, thr;
  int smallval = abs < DELTA_Q_SMALL ? 1 : 0;
  FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
  (void)cm;

  aom_write_symbol(w, AOMMIN(abs, DELTA_Q_SMALL), ec_ctx->delta_q_cdf,
                   DELTA_Q_PROBS + 1);

  if (!smallval) {
    rem_bits = OD_ILOG_NZ(abs - 1) - 1;
    thr = (1 << rem_bits) + 1;
    aom_write_literal(w, rem_bits - 1, 3);
    aom_write_literal(w, abs - thr, rem_bits);
  }
  if (abs > 0) {
    aom_write_bit(w, sign);
  }
}

#if CONFIG_EXT_DELTA_Q
static void write_delta_lflevel(const AV1_COMMON *cm, const MACROBLOCKD *xd,
#if CONFIG_LOOPFILTER_LEVEL
                                int lf_id,
#endif
                                int delta_lflevel, aom_writer *w) {
  int sign = delta_lflevel < 0;
  int abs = sign ? -delta_lflevel : delta_lflevel;
  int rem_bits, thr;
  int smallval = abs < DELTA_LF_SMALL ? 1 : 0;
  FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
  (void)cm;

#if CONFIG_LOOPFILTER_LEVEL
  if (cm->delta_lf_multi) {
    assert(lf_id >= 0 && lf_id < FRAME_LF_COUNT);
    aom_write_symbol(w, AOMMIN(abs, DELTA_LF_SMALL),
                     ec_ctx->delta_lf_multi_cdf[lf_id], DELTA_LF_PROBS + 1);
  } else {
    aom_write_symbol(w, AOMMIN(abs, DELTA_LF_SMALL), ec_ctx->delta_lf_cdf,
                     DELTA_LF_PROBS + 1);
  }
#else
  aom_write_symbol(w, AOMMIN(abs, DELTA_LF_SMALL), ec_ctx->delta_lf_cdf,
                   DELTA_LF_PROBS + 1);
#endif  // CONFIG_LOOPFILTER_LEVEL

  if (!smallval) {
    rem_bits = OD_ILOG_NZ(abs - 1) - 1;
    thr = (1 << rem_bits) + 1;
    aom_write_literal(w, rem_bits - 1, 3);
    aom_write_literal(w, abs - thr, rem_bits);
  }
  if (abs > 0) {
    aom_write_bit(w, sign);
  }
}
#endif  // CONFIG_EXT_DELTA_Q

#if !CONFIG_NEW_MULTISYMBOL
static void update_skip_probs(AV1_COMMON *cm, aom_writer *w,
                              FRAME_COUNTS *counts) {
  int k;
  const int probwt = cm->num_tg;
  for (k = 0; k < SKIP_CONTEXTS; ++k) {
    av1_cond_prob_diff_update(w, &cm->fc->skip_probs[k], counts->skip[k],
                              probwt);
  }
}
#endif

static void pack_map_tokens(aom_writer *w, const TOKENEXTRA **tp, int n,
                            int num) {
  const TOKENEXTRA *p = *tp;
  write_uniform(w, n, p->token);  // The first color index.
  ++p;
  --num;
  for (int i = 0; i < num; ++i) {
    aom_write_symbol(w, p->token, p->color_map_cdf, n);
    ++p;
  }
  *tp = p;
}

#if !CONFIG_LV_MAP
#if CONFIG_NEW_MULTISYMBOL
static INLINE void write_coeff_extra(const aom_cdf_prob *const *cdf, int val,
                                     int n, aom_writer *w) {
  // Code the extra bits from LSB to MSB in groups of 4
  int i = 0;
  int count = 0;
  while (count < n) {
    const int size = AOMMIN(n - count, 4);
    const int mask = (1 << size) - 1;
    aom_write_cdf(w, val & mask, cdf[i++], 1 << size);
    val >>= size;
    count += size;
  }
}
#else
static INLINE void write_coeff_extra(const aom_prob *pb, int value,
                                     int num_bits, int skip_bits, aom_writer *w,
                                     TOKEN_STATS *token_stats) {
  // Code the extra bits from MSB to LSB 1 bit at a time
  int index;
  for (index = skip_bits; index < num_bits; ++index) {
    const int shift = num_bits - index - 1;
    const int bb = (value >> shift) & 1;
    aom_write_record(w, bb, pb[index], token_stats);
  }
}
#endif  // CONFIG_NEW_MULTISYMBOL

static void pack_mb_tokens(aom_writer *w, const TOKENEXTRA **tp,
                           const TOKENEXTRA *const stop,
                           aom_bit_depth_t bit_depth, const TX_SIZE tx_size,
#if CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
                           TX_TYPE tx_type, int is_inter,
#endif  // CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
                           TOKEN_STATS *token_stats) {
  const TOKENEXTRA *p = *tp;
  int count = 0;
  const int seg_eob = tx_size_2d[tx_size];

#if CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
  if (tx_type == MRC_DCT && ((is_inter && SIGNAL_MRC_MASK_INTER) ||
                             (!is_inter && SIGNAL_MRC_MASK_INTRA))) {
    int rows = tx_size_high[tx_size];
    int cols = tx_size_wide[tx_size];
    assert(tx_size == TX_32X32);
    assert(p < stop);
    pack_map_tokens(w, &p, 2, rows * cols);
  }
#endif  // CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK

  while (p < stop && p->token != EOSB_TOKEN) {
    const int token = p->token;
    const int8_t eob_val = p->eob_val;
    if (token == BLOCK_Z_TOKEN) {
      aom_write_symbol(w, 0, *p->head_cdf, HEAD_TOKENS + 1);
      p++;
      break;
      continue;
    }

    const av1_extra_bit *const extra_bits = &av1_extra_bits[token];
    if (eob_val == LAST_EOB) {
      // Just code a flag indicating whether the value is >1 or 1.
      aom_write_bit(w, token != ONE_TOKEN);
    } else {
      int comb_symb = 2 * AOMMIN(token, TWO_TOKEN) - eob_val + p->first_val;
      aom_write_symbol(w, comb_symb, *p->head_cdf, HEAD_TOKENS + p->first_val);
    }
    if (token > ONE_TOKEN) {
      aom_write_symbol(w, token - TWO_TOKEN, *p->tail_cdf, TAIL_TOKENS);
    }

    if (extra_bits->base_val) {
      const int bit_string = p->extra;
      const int bit_string_length = extra_bits->len;  // Length of extra bits to
      const int is_cat6 = (extra_bits->base_val == CAT6_MIN_VAL);
      // be written excluding
      // the sign bit.
      int skip_bits = is_cat6
                          ? (int)sizeof(av1_cat6_prob) -
                                av1_get_cat6_extrabits_size(tx_size, bit_depth)
                          : 0;

      assert(!(bit_string >> (bit_string_length - skip_bits + 1)));
      if (bit_string_length > 0)
#if CONFIG_NEW_MULTISYMBOL
        write_coeff_extra(extra_bits->cdf, bit_string >> 1,
                          bit_string_length - skip_bits, w);
#else
        write_coeff_extra(extra_bits->prob, bit_string >> 1, bit_string_length,
                          skip_bits, w, token_stats);
#endif

      aom_write_bit_record(w, bit_string & 1, token_stats);
    }
    ++p;

    ++count;
    if (eob_val == EARLY_EOB || count == seg_eob) break;
  }

  *tp = p;
}
#endif  // !CONFIG_LV_MAP

#if CONFIG_LV_MAP
static void pack_txb_tokens(aom_writer *w, AV1_COMMON *cm, MACROBLOCK *const x,
                            const TOKENEXTRA **tp,
                            const TOKENEXTRA *const tok_end, MACROBLOCKD *xd,
                            MB_MODE_INFO *mbmi, int plane,
                            BLOCK_SIZE plane_bsize, aom_bit_depth_t bit_depth,
                            int block, int blk_row, int blk_col,
                            TX_SIZE tx_size, TOKEN_STATS *token_stats) {
  const struct macroblockd_plane *const pd = &xd->plane[plane];
  const BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
  const int tx_row = blk_row >> (1 - pd->subsampling_y);
  const int tx_col = blk_col >> (1 - pd->subsampling_x);
  TX_SIZE plane_tx_size;
  const int max_blocks_high = max_block_high(xd, plane_bsize, plane);
  const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane);

  if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;

  plane_tx_size =
      plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0]
            : mbmi->inter_tx_size[tx_row][tx_col];

  if (tx_size == plane_tx_size) {
    TOKEN_STATS tmp_token_stats;
    init_token_stats(&tmp_token_stats);

    tran_low_t *tcoeff = BLOCK_OFFSET(x->mbmi_ext->tcoeff[plane], block);
    uint16_t eob = x->mbmi_ext->eobs[plane][block];
    TXB_CTX txb_ctx = { x->mbmi_ext->txb_skip_ctx[plane][block],
                        x->mbmi_ext->dc_sign_ctx[plane][block] };
    av1_write_coeffs_txb(cm, xd, w, blk_row, blk_col, block, plane, tx_size,
                         tcoeff, eob, &txb_ctx);
#if CONFIG_RD_DEBUG
    token_stats->txb_coeff_cost_map[blk_row][blk_col] = tmp_token_stats.cost;
    token_stats->cost += tmp_token_stats.cost;
#endif
  } else {
    const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
    const int bsl = tx_size_wide_unit[sub_txs];
    int i;

    assert(bsl > 0);

    for (i = 0; i < 4; ++i) {
      const int offsetr = blk_row + (i >> 1) * bsl;
      const int offsetc = blk_col + (i & 0x01) * bsl;
      const int step = tx_size_wide_unit[sub_txs] * tx_size_high_unit[sub_txs];

      if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;

      pack_txb_tokens(w, cm, x, tp, tok_end, xd, mbmi, plane, plane_bsize,
                      bit_depth, block, offsetr, offsetc, sub_txs, token_stats);
      block += step;
    }
  }
}
#else  // CONFIG_LV_MAP
static void pack_txb_tokens(aom_writer *w, const TOKENEXTRA **tp,
                            const TOKENEXTRA *const tok_end, MACROBLOCKD *xd,
                            MB_MODE_INFO *mbmi, int plane,
                            BLOCK_SIZE plane_bsize, aom_bit_depth_t bit_depth,
                            int block, int blk_row, int blk_col,
                            TX_SIZE tx_size, TOKEN_STATS *token_stats) {
  const struct macroblockd_plane *const pd = &xd->plane[plane];
  const BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
  const int tx_row = blk_row >> (1 - pd->subsampling_y);
  const int tx_col = blk_col >> (1 - pd->subsampling_x);
  TX_SIZE plane_tx_size;
  const int max_blocks_high = max_block_high(xd, plane_bsize, plane);
  const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane);
#if CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
  TX_TYPE tx_type = av1_get_tx_type(plane ? PLANE_TYPE_UV : PLANE_TYPE_Y, xd,
                                    blk_row, blk_col, block, tx_size);
#endif  // CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK

  if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;

  plane_tx_size =
      plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0]
            : mbmi->inter_tx_size[tx_row][tx_col];

  if (tx_size == plane_tx_size) {
    TOKEN_STATS tmp_token_stats;
    init_token_stats(&tmp_token_stats);
    pack_mb_tokens(w, tp, tok_end, bit_depth, tx_size,
#if CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
                   tx_type, is_inter_block(mbmi),
#endif  // CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
                   &tmp_token_stats);
#if CONFIG_RD_DEBUG
    token_stats->txb_coeff_cost_map[blk_row][blk_col] = tmp_token_stats.cost;
    token_stats->cost += tmp_token_stats.cost;
#endif
  } else {
#if CONFIG_RECT_TX_EXT
    int is_qttx = plane_tx_size == quarter_txsize_lookup[plane_bsize];
    const TX_SIZE sub_txs = is_qttx ? plane_tx_size : sub_tx_size_map[tx_size];
#else
    const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
#endif
    const int bsl = tx_size_wide_unit[sub_txs];
    int i;

    assert(bsl > 0);

    for (i = 0; i < 4; ++i) {
#if CONFIG_RECT_TX_EXT
      int is_wide_tx = tx_size_wide_unit[sub_txs] > tx_size_high_unit[sub_txs];
      const int offsetr =
          is_qttx ? (is_wide_tx ? i * tx_size_high_unit[sub_txs] : 0)
                  : blk_row + (i >> 1) * bsl;
      const int offsetc =
          is_qttx ? (is_wide_tx ? 0 : i * tx_size_wide_unit[sub_txs])
                  : blk_col + (i & 0x01) * bsl;
#else
      const int offsetr = blk_row + (i >> 1) * bsl;
      const int offsetc = blk_col + (i & 0x01) * bsl;
#endif
      const int step = tx_size_wide_unit[sub_txs] * tx_size_high_unit[sub_txs];

      if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;

      pack_txb_tokens(w, tp, tok_end, xd, mbmi, plane, plane_bsize, bit_depth,
                      block, offsetr, offsetc, sub_txs, token_stats);
      block += step;
    }
  }
}
#endif  // CONFIG_LV_MAP

#if CONFIG_Q_SEGMENTATION
static int neg_interleave(int x, int ref, int max) {
  const int diff = x - ref;
  if (!ref) return x;
  if (ref >= (max - 1)) return -diff;
  if (2 * ref < max) {
    if (abs(diff) <= ref) {
      if (diff > 0)
        return (diff << 1) - 1;
      else
        return ((-diff) << 1);
    }
    return x;
  } else {
    if (abs(diff) < (max - ref)) {
      if (diff > 0)
        return (diff << 1) - 1;
      else
        return ((-diff) << 1);
    }
    return (max - x) - 1;
  }
}

static void write_q_segment_id(const AV1_COMMON *cm, int skip,
                               const MB_MODE_INFO *const mbmi, aom_writer *w,
                               const struct segmentation *seg,
                               struct segmentation_probs *segp,
                               BLOCK_SIZE bsize, int mi_row, int mi_col) {
  int prev_ul = 0; /* Top left segment_id */
  int prev_l = 0;  /* Current left segment_id */
  int prev_u = 0;  /* Current top segment_id */

  if (!seg->q_lvls) return;

  MODE_INFO *const mi = cm->mi + mi_row * cm->mi_stride + mi_col;
  int tinfo = mi->mbmi.boundary_info;
  int above = (!(tinfo & TILE_ABOVE_BOUNDARY)) && ((mi_row - 1) >= 0);
  int left = (!(tinfo & TILE_LEFT_BOUNDARY)) && ((mi_col - 1) >= 0);

  if (above && left)
    prev_ul =
        get_segment_id(cm, cm->q_seg_map, BLOCK_4X4, mi_row - 1, mi_col - 1);

  if (above)
    prev_u = get_segment_id(cm, cm->q_seg_map, BLOCK_4X4, mi_row - 1, mi_col);

  if (left)
    prev_l = get_segment_id(cm, cm->q_seg_map, BLOCK_4X4, mi_row, mi_col - 1);

  int cdf_num = pick_q_seg_cdf(prev_ul, prev_u, prev_l);
  int pred = pick_q_seg_pred(prev_ul, prev_u, prev_l);

  if (skip) {
    set_q_segment_id(cm, cm->q_seg_map, mbmi->sb_type, mi_row, mi_col, pred);
    return;
  }

  int coded_id = neg_interleave(mbmi->q_segment_id, pred, seg->q_lvls);

#if CONFIG_NEW_MULTISYMBOL
  aom_cdf_prob *pred_cdf = segp->q_seg_cdf[cdf_num];
  aom_write_symbol(w, coded_id, pred_cdf, 8);
#else
  aom_prob pred_cdf = segp->q_seg_cdf[cdf_num];
  aom_write(w, coded_id, pred_prob);
#endif

  set_q_segment_id(cm, cm->q_seg_map, bsize, mi_row, mi_col,
                   mbmi->q_segment_id);
}
#endif

static void write_segment_id(aom_writer *w, const struct segmentation *seg,
                             struct segmentation_probs *segp, int segment_id) {
  if (seg->enabled && seg->update_map) {
    aom_write_symbol(w, segment_id, segp->tree_cdf, MAX_SEGMENTS);
  }
}

#if CONFIG_NEW_MULTISYMBOL
#define WRITE_REF_BIT(bname, pname) \
  aom_write_symbol(w, bname, av1_get_pred_cdf_##pname(cm, xd), 2)
#define WRITE_REF_BIT2(bname, pname) \
  aom_write_symbol(w, bname, av1_get_pred_cdf_##pname(xd), 2)
#else
#define WRITE_REF_BIT(bname, pname) \
  aom_write(w, bname, av1_get_pred_prob_##pname(cm, xd))
#define WRITE_REF_BIT2(bname, pname) \
  aom_write(w, bname, av1_get_pred_prob_##pname(cm, xd))
#endif