bitstream.c 203 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4 5 6 7 8 9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10 11 12 13
 */

#include <assert.h>
#include <limits.h>
14
#include <stdio.h>
Jingning Han's avatar
Jingning Han committed
15

Yaowu Xu's avatar
Yaowu Xu committed
16 17
#include "aom/aom_encoder.h"
#include "aom_dsp/aom_dsp_common.h"
18
#include "aom_dsp/binary_codes_writer.h"
19
#include "aom_dsp/bitwriter_buffer.h"
Yaowu Xu's avatar
Yaowu Xu committed
20
#include "aom_mem/aom_mem.h"
21 22
#include "aom_ports/mem_ops.h"
#include "aom_ports/system_state.h"
23
#if CONFIG_BITSTREAM_DEBUG
24
#include "aom_util/debug_util.h"
25
#endif  // CONFIG_BITSTREAM_DEBUG
Jingning Han's avatar
Jingning Han committed
26

27
#if CONFIG_CDEF
28
#include "av1/common/cdef.h"
29
#endif  // CONFIG_CDEF
30 31 32 33
#include "av1/common/entropy.h"
#include "av1/common/entropymode.h"
#include "av1/common/entropymv.h"
#include "av1/common/mvref_common.h"
34
#include "av1/common/odintrin.h"
35 36
#include "av1/common/pred_common.h"
#include "av1/common/reconinter.h"
hui su's avatar
hui su committed
37 38 39
#if CONFIG_EXT_INTRA
#include "av1/common/reconintra.h"
#endif  // CONFIG_EXT_INTRA
40 41
#include "av1/common/seg_common.h"
#include "av1/common/tile_common.h"
Jingning Han's avatar
Jingning Han committed
42

Angie Chiang's avatar
Angie Chiang committed
43 44 45
#if CONFIG_LV_MAP
#include "av1/encoder/encodetxb.h"
#endif  // CONFIG_LV_MAP
46 47 48 49
#include "av1/encoder/bitstream.h"
#include "av1/encoder/cost.h"
#include "av1/encoder/encodemv.h"
#include "av1/encoder/mcomp.h"
Urvang Joshi's avatar
Urvang Joshi committed
50
#if CONFIG_PALETTE_DELTA_ENCODING
hui su's avatar
hui su committed
51
#include "av1/encoder/palette.h"
Urvang Joshi's avatar
Urvang Joshi committed
52
#endif  // CONFIG_PALETTE_DELTA_ENCODING
53 54 55
#include "av1/encoder/segmentation.h"
#include "av1/encoder/subexp.h"
#include "av1/encoder/tokenize.h"
56 57 58
#if CONFIG_PVQ
#include "av1/encoder/pvq_encoder.h"
#endif
Jingning Han's avatar
Jingning Han committed
59

60
#define ENC_MISMATCH_DEBUG 0
61

62
#if CONFIG_COMPOUND_SINGLEREF
63 64
static struct av1_token
    inter_singleref_comp_mode_encodings[INTER_SINGLEREF_COMP_MODES];
65
#endif  // CONFIG_COMPOUND_SINGLEREF
66

67 68
// TODO(anybody) : remove this flag when PVQ supports pallete coding tool
#if !CONFIG_PVQ || CONFIG_EXT_INTRA
Yaowu Xu's avatar
Yaowu Xu committed
69
static INLINE void write_uniform(aom_writer *w, int n, int v) {
70 71
  const int l = get_unsigned_bits(n);
  const int m = (1 << l) - n;
72
  if (l == 0) return;
hui su's avatar
hui su committed
73
  if (v < m) {
Yaowu Xu's avatar
Yaowu Xu committed
74
    aom_write_literal(w, v, l - 1);
hui su's avatar
hui su committed
75
  } else {
Yaowu Xu's avatar
Yaowu Xu committed
76 77
    aom_write_literal(w, m + ((v - m) >> 1), l - 1);
    aom_write_literal(w, (v - m) & 1, 1);
hui su's avatar
hui su committed
78 79
  }
}
80
#endif  // !CONFIG_PVQ || CONFIG_EXT_INTRA
Jingning Han's avatar
Jingning Han committed
81

82
#if CONFIG_INTERINTRA
Yaowu Xu's avatar
Yaowu Xu committed
83
static struct av1_token interintra_mode_encodings[INTERINTRA_MODES];
84
#endif
85
#if CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE
86
static struct av1_token compound_type_encodings[COMPOUND_TYPES];
87
#endif  // CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE
88
#if CONFIG_LOOP_RESTORATION
89 90 91 92
static void loop_restoration_write_sb_coeffs(const AV1_COMMON *const cm,
                                             MACROBLOCKD *xd,
                                             aom_writer *const w, int plane,
                                             int rtile_idx);
93
#endif  // CONFIG_LOOP_RESTORATION
94 95 96 97 98 99 100 101
#if CONFIG_OBU
static void write_uncompressed_header_obu(AV1_COMP *cpi,
                                          struct aom_write_bit_buffer *wb);
#else
static void write_uncompressed_header_frame(AV1_COMP *cpi,
                                            struct aom_write_bit_buffer *wb);
#endif

102
static uint32_t write_compressed_header(AV1_COMP *cpi, uint8_t *data);
103

104
#if !CONFIG_OBU || CONFIG_EXT_TILE
105 106 107 108 109
static int remux_tiles(const AV1_COMMON *const cm, uint8_t *dst,
                       const uint32_t data_size, const uint32_t max_tile_size,
                       const uint32_t max_tile_col_size,
                       int *const tile_size_bytes,
                       int *const tile_col_size_bytes);
110
#endif
Yaowu Xu's avatar
Yaowu Xu committed
111
void av1_encode_token_init(void) {
112
#if CONFIG_INTERINTRA
Yaowu Xu's avatar
Yaowu Xu committed
113
  av1_tokens_from_tree(interintra_mode_encodings, av1_interintra_mode_tree);
114
#endif  // CONFIG_INTERINTRA
115 116 117 118
#if CONFIG_COMPOUND_SINGLEREF
  av1_tokens_from_tree(inter_singleref_comp_mode_encodings,
                       av1_inter_singleref_comp_mode_tree);
#endif  // CONFIG_COMPOUND_SINGLEREF
119
#if CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE
120
  av1_tokens_from_tree(compound_type_encodings, av1_compound_type_tree);
121
#endif  // CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE
122 123
}

Jingning Han's avatar
Jingning Han committed
124 125 126 127
static void write_intra_mode_kf(const AV1_COMMON *cm, FRAME_CONTEXT *frame_ctx,
                                const MODE_INFO *mi, const MODE_INFO *above_mi,
                                const MODE_INFO *left_mi, int block,
                                PREDICTION_MODE mode, aom_writer *w) {
Alex Converse's avatar
Alex Converse committed
128 129 130
#if CONFIG_INTRABC
  assert(!is_intrabc_block(&mi->mbmi));
#endif  // CONFIG_INTRABC
Hui Su's avatar
Hui Su committed
131
  aom_write_symbol(w, mode,
Jingning Han's avatar
Jingning Han committed
132 133 134 135
                   get_y_mode_cdf(frame_ctx, mi, above_mi, left_mi, block),
                   INTRA_MODES);
  (void)cm;
}
Jingning Han's avatar
Jingning Han committed
136

137
static void write_inter_mode(aom_writer *w, PREDICTION_MODE mode,
138
                             FRAME_CONTEXT *ec_ctx, const int16_t mode_ctx) {
139
  const int16_t newmv_ctx = mode_ctx & NEWMV_CTX_MASK;
Yue Chen's avatar
Yue Chen committed
140

141 142 143 144 145
#if CONFIG_NEW_MULTISYMBOL
  aom_write_symbol(w, mode != NEWMV, ec_ctx->newmv_cdf[newmv_ctx], 2);
#else
  aom_write(w, mode != NEWMV, ec_ctx->newmv_prob[newmv_ctx]);
#endif
146

Jingning Han's avatar
Jingning Han committed
147
  if (mode != NEWMV) {
148 149 150 151 152
    if (mode_ctx & (1 << ALL_ZERO_FLAG_OFFSET)) {
      assert(mode == ZEROMV);
      return;
    }

153 154 155 156 157 158
    const int16_t zeromv_ctx = (mode_ctx >> ZEROMV_OFFSET) & ZEROMV_CTX_MASK;
#if CONFIG_NEW_MULTISYMBOL
    aom_write_symbol(w, mode != ZEROMV, ec_ctx->zeromv_cdf[zeromv_ctx], 2);
#else
    aom_write(w, mode != ZEROMV, ec_ctx->zeromv_prob[zeromv_ctx]);
#endif
159 160

    if (mode != ZEROMV) {
161 162
      int16_t refmv_ctx = (mode_ctx >> REFMV_OFFSET) & REFMV_CTX_MASK;

163 164 165
      if (mode_ctx & (1 << SKIP_NEARESTMV_OFFSET)) refmv_ctx = 6;
      if (mode_ctx & (1 << SKIP_NEARMV_OFFSET)) refmv_ctx = 7;
      if (mode_ctx & (1 << SKIP_NEARESTMV_SUB8X8_OFFSET)) refmv_ctx = 8;
166 167 168 169 170
#if CONFIG_NEW_MULTISYMBOL
      aom_write_symbol(w, mode != NEARESTMV, ec_ctx->refmv_cdf[refmv_ctx], 2);
#else
      aom_write(w, mode != NEARESTMV, ec_ctx->refmv_prob[refmv_ctx]);
#endif
171 172
    }
  }
Jingning Han's avatar
Jingning Han committed
173 174
}

175
static void write_drl_idx(FRAME_CONTEXT *ec_ctx, const MB_MODE_INFO *mbmi,
Yaowu Xu's avatar
Yaowu Xu committed
176 177
                          const MB_MODE_INFO_EXT *mbmi_ext, aom_writer *w) {
  uint8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame);
178 179 180

  assert(mbmi->ref_mv_idx < 3);

181 182 183 184
#if CONFIG_COMPOUND_SINGLEREF
  if (mbmi->mode == NEWMV || mbmi->mode == NEW_NEWMV ||
      mbmi->mode == SR_NEW_NEWMV) {
#else   // !CONFIG_COMPOUND_SINGLEREF
185
  if (mbmi->mode == NEWMV || mbmi->mode == NEW_NEWMV) {
186
#endif  // CONFIG_COMPOUND_SINGLEREF
187 188 189 190
    int idx;
    for (idx = 0; idx < 2; ++idx) {
      if (mbmi_ext->ref_mv_count[ref_frame_type] > idx + 1) {
        uint8_t drl_ctx =
Yaowu Xu's avatar
Yaowu Xu committed
191
            av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx);
192

193 194 195 196 197 198
#if CONFIG_NEW_MULTISYMBOL
        aom_write_symbol(w, mbmi->ref_mv_idx != idx, ec_ctx->drl_cdf[drl_ctx],
                         2);
#else
        aom_write(w, mbmi->ref_mv_idx != idx, ec_ctx->drl_prob[drl_ctx]);
#endif
199
        if (mbmi->ref_mv_idx == idx) return;
200
      }
201 202 203 204
    }
    return;
  }

David Barker's avatar
David Barker committed
205
  if (have_nearmv_in_inter_mode(mbmi->mode)) {
206 207 208 209 210
    int idx;
    // TODO(jingning): Temporary solution to compensate the NEARESTMV offset.
    for (idx = 1; idx < 3; ++idx) {
      if (mbmi_ext->ref_mv_count[ref_frame_type] > idx + 1) {
        uint8_t drl_ctx =
Yaowu Xu's avatar
Yaowu Xu committed
211
            av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx);
212 213 214 215 216 217
#if CONFIG_NEW_MULTISYMBOL
        aom_write_symbol(w, mbmi->ref_mv_idx != (idx - 1),
                         ec_ctx->drl_cdf[drl_ctx], 2);
#else
        aom_write(w, mbmi->ref_mv_idx != (idx - 1), ec_ctx->drl_prob[drl_ctx]);
#endif
218
        if (mbmi->ref_mv_idx == (idx - 1)) return;
219
      }
220
    }
221
    return;
222 223 224
  }
}

225 226
static void write_inter_compound_mode(AV1_COMMON *cm, MACROBLOCKD *xd,
                                      aom_writer *w, PREDICTION_MODE mode,
227
                                      const int16_t mode_ctx) {
228 229 230 231 232
  assert(is_inter_compound_mode(mode));
  (void)cm;
  aom_write_symbol(w, INTER_COMPOUND_OFFSET(mode),
                   xd->tile_ctx->inter_compound_mode_cdf[mode_ctx],
                   INTER_COMPOUND_MODES);
233
}
234 235

#if CONFIG_COMPOUND_SINGLEREF
236
static void write_inter_singleref_comp_mode(MACROBLOCKD *xd, aom_writer *w,
237 238 239
                                            PREDICTION_MODE mode,
                                            const int16_t mode_ctx) {
  assert(is_inter_singleref_comp_mode(mode));
240 241
  aom_cdf_prob *const inter_singleref_comp_cdf =
      xd->tile_ctx->inter_singleref_comp_mode_cdf[mode_ctx];
242

243 244
  aom_write_symbol(w, INTER_SINGLEREF_COMP_OFFSET(mode),
                   inter_singleref_comp_cdf, INTER_SINGLEREF_COMP_MODES);
245 246
}
#endif  // CONFIG_COMPOUND_SINGLEREF
247

Yaowu Xu's avatar
Yaowu Xu committed
248
static void encode_unsigned_max(struct aom_write_bit_buffer *wb, int data,
249
                                int max) {
Yaowu Xu's avatar
Yaowu Xu committed
250
  aom_wb_write_literal(wb, data, get_unsigned_bits(max));
Jingning Han's avatar
Jingning Han committed
251 252
}

253
#if CONFIG_VAR_TX
254
static void write_tx_size_vartx(const AV1_COMMON *cm, MACROBLOCKD *xd,
255
                                const MB_MODE_INFO *mbmi, TX_SIZE tx_size,
256 257
                                int depth, int blk_row, int blk_col,
                                aom_writer *w) {
258 259 260 261
#if CONFIG_NEW_MULTISYMBOL
  FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
  (void)cm;
#endif
262 263
  const int tx_row = blk_row >> 1;
  const int tx_col = blk_col >> 1;
264 265 266
  const int max_blocks_high = max_block_high(xd, mbmi->sb_type, 0);
  const int max_blocks_wide = max_block_wide(xd, mbmi->sb_type, 0);

267 268
  int ctx = txfm_partition_context(xd->above_txfm_context + blk_col,
                                   xd->left_txfm_context + blk_row,
269
                                   mbmi->sb_type, tx_size);
270

271
  if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
272

Jingning Han's avatar
Jingning Han committed
273
  if (depth == MAX_VARTX_DEPTH) {
274 275
    txfm_partition_update(xd->above_txfm_context + blk_col,
                          xd->left_txfm_context + blk_row, tx_size, tx_size);
276 277 278
    return;
  }

Yue Chen's avatar
Yue Chen committed
279 280 281 282
#if CONFIG_RECT_TX_EXT
  if (tx_size == mbmi->inter_tx_size[tx_row][tx_col] ||
      mbmi->tx_size == quarter_txsize_lookup[mbmi->sb_type]) {
#else
283
  if (tx_size == mbmi->inter_tx_size[tx_row][tx_col]) {
Yue Chen's avatar
Yue Chen committed
284
#endif
285 286 287
#if CONFIG_NEW_MULTISYMBOL
    aom_write_symbol(w, 0, ec_ctx->txfm_partition_cdf[ctx], 2);
#else
Yaowu Xu's avatar
Yaowu Xu committed
288
    aom_write(w, 0, cm->fc->txfm_partition_prob[ctx]);
289 290
#endif

291 292
    txfm_partition_update(xd->above_txfm_context + blk_col,
                          xd->left_txfm_context + blk_row, tx_size, tx_size);
Yue Chen's avatar
Yue Chen committed
293
    // TODO(yuec): set correct txfm partition update for qttx
294
  } else {
295 296
    const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
    const int bsl = tx_size_wide_unit[sub_txs];
297
    int i;
298

299 300 301
#if CONFIG_NEW_MULTISYMBOL
    aom_write_symbol(w, 1, ec_ctx->txfm_partition_cdf[ctx], 2);
#else
Yaowu Xu's avatar
Yaowu Xu committed
302
    aom_write(w, 1, cm->fc->txfm_partition_prob[ctx]);
303
#endif
304

David Barker's avatar
David Barker committed
305
    if (sub_txs == TX_4X4) {
306 307
      txfm_partition_update(xd->above_txfm_context + blk_col,
                            xd->left_txfm_context + blk_row, sub_txs, tx_size);
308
      return;
309
    }
310 311 312

    assert(bsl > 0);
    for (i = 0; i < 4; ++i) {
313 314 315 316
      int offsetr = blk_row + (i >> 1) * bsl;
      int offsetc = blk_col + (i & 0x01) * bsl;
      write_tx_size_vartx(cm, xd, mbmi, sub_txs, depth + 1, offsetr, offsetc,
                          w);
317 318 319
    }
  }
}
320

321
#if !CONFIG_NEW_MULTISYMBOL
Yaowu Xu's avatar
Yaowu Xu committed
322
static void update_txfm_partition_probs(AV1_COMMON *cm, aom_writer *w,
323
                                        FRAME_COUNTS *counts, int probwt) {
324 325
  int k;
  for (k = 0; k < TXFM_PARTITION_CONTEXTS; ++k)
Yaowu Xu's avatar
Yaowu Xu committed
326
    av1_cond_prob_diff_update(w, &cm->fc->txfm_partition_prob[k],
327
                              counts->txfm_partition[k], probwt);
328
}
329
#endif  // CONFIG_NEW_MULTISYMBOL
David Barker's avatar
David Barker committed
330
#endif  // CONFIG_VAR_TX
331

Yaowu Xu's avatar
Yaowu Xu committed
332 333
static void write_selected_tx_size(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                                   aom_writer *w) {
334 335
  const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
  const BLOCK_SIZE bsize = mbmi->sb_type;
336 337
  FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
  (void)cm;
338
  if (block_signals_txsize(bsize)) {
339 340
    const TX_SIZE tx_size = mbmi->tx_size;
    const int is_inter = is_inter_block(mbmi);
341 342
    const int tx_size_ctx = get_tx_size_context(xd);
    const int32_t tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
343
                                         : intra_tx_size_cat_lookup[bsize];
344
    const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size];
345
    const int depth = tx_size_to_depth(coded_tx_size);
346
#if CONFIG_EXT_TX && CONFIG_RECT_TX
347
    assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(xd, mbmi)));
348 349
#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX

350
    aom_write_symbol(w, depth, ec_ctx->tx_size_cdf[tx_size_cat][tx_size_ctx],
351
                     tx_size_cat + 2);
Yue Chen's avatar
Yue Chen committed
352
#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
353
    if (is_quarter_tx_allowed(xd, mbmi, is_inter) && tx_size != coded_tx_size)
354 355 356 357
#if CONFIG_NEW_MULTISYMBOL
      aom_write_symbol(w, tx_size == quarter_txsize_lookup[bsize],
                       cm->fc->quarter_tx_size_cdf, 2);
#else
358 359
      aom_write(w, tx_size == quarter_txsize_lookup[bsize],
                cm->fc->quarter_tx_size_prob);
360
#endif
Yue Chen's avatar
Yue Chen committed
361
#endif
Jingning Han's avatar
Jingning Han committed
362 363 364
  }
}

365
#if !CONFIG_NEW_MULTISYMBOL
Yaowu Xu's avatar
Yaowu Xu committed
366
static void update_inter_mode_probs(AV1_COMMON *cm, aom_writer *w,
367 368
                                    FRAME_COUNTS *counts) {
  int i;
369
  const int probwt = cm->num_tg;
370
  for (i = 0; i < NEWMV_MODE_CONTEXTS; ++i)
371 372
    av1_cond_prob_diff_update(w, &cm->fc->newmv_prob[i], counts->newmv_mode[i],
                              probwt);
373
  for (i = 0; i < ZEROMV_MODE_CONTEXTS; ++i)
Yaowu Xu's avatar
Yaowu Xu committed
374
    av1_cond_prob_diff_update(w, &cm->fc->zeromv_prob[i],
375
                              counts->zeromv_mode[i], probwt);
376
  for (i = 0; i < REFMV_MODE_CONTEXTS; ++i)
377 378
    av1_cond_prob_diff_update(w, &cm->fc->refmv_prob[i], counts->refmv_mode[i],
                              probwt);
379
  for (i = 0; i < DRL_MODE_CONTEXTS; ++i)
380 381
    av1_cond_prob_diff_update(w, &cm->fc->drl_prob[i], counts->drl_mode[i],
                              probwt);
382
}
383
#endif
384

Yaowu Xu's avatar
Yaowu Xu committed
385 386
static int write_skip(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                      int segment_id, const MODE_INFO *mi, aom_writer *w) {
Jingning Han's avatar
Jingning Han committed
387 388 389 390
  if (segfeature_active(&cm->seg, segment_id, SEG_LVL_SKIP)) {
    return 1;
  } else {
    const int skip = mi->mbmi.skip;
Zoe Liu's avatar
Zoe Liu committed
391
    const int ctx = av1_get_skip_context(xd);
392 393 394 395
#if CONFIG_NEW_MULTISYMBOL
    FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
    aom_write_symbol(w, skip, ec_ctx->skip_cdfs[ctx], 2);
#else
Zoe Liu's avatar
Zoe Liu committed
396
    aom_write(w, skip, cm->fc->skip_probs[ctx]);
397
#endif
Jingning Han's avatar
Jingning Han committed
398 399 400 401
    return skip;
  }
}

402 403 404 405 406 407 408 409 410 411 412 413 414
static void write_is_inter(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                           int segment_id, aom_writer *w, const int is_inter) {
  if (!segfeature_active(&cm->seg, segment_id, SEG_LVL_REF_FRAME)) {
#if CONFIG_NEW_MULTISYMBOL
    FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
    const int ctx = av1_get_intra_inter_context(xd);
    aom_write_symbol(w, is_inter, ec_ctx->intra_inter_cdf[ctx], 2);
#else
    aom_write(w, is_inter, av1_get_intra_inter_prob(cm, xd));
#endif
  }
}

Yue Chen's avatar
Yue Chen committed
415
#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
416 417
static void write_motion_mode(const AV1_COMMON *cm, MACROBLOCKD *xd,
                              const MODE_INFO *mi, aom_writer *w) {
418
  const MB_MODE_INFO *mbmi = &mi->mbmi;
419

420 421 422 423 424 425
#if !CONFIG_GLOBAL_MOTION
  // The cm parameter is only used with global_motion or with
  // motion_var and warped_motion. In other cases, explicitly ignore
  // it to avoid a compiler warning.
  (void)cm;
#endif
426
  MOTION_MODE last_motion_mode_allowed = motion_mode_allowed(
427
#if CONFIG_GLOBAL_MOTION
428
      0, cm->global_motion,
429
#endif  // CONFIG_GLOBAL_MOTION
430 431 432
#if CONFIG_WARPED_MOTION
      xd,
#endif
433
      mi);
Yue Chen's avatar
Yue Chen committed
434 435
  if (last_motion_mode_allowed == SIMPLE_TRANSLATION) return;
#if CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION
436 437 438 439 440 441 442 443 444 445
#if CONFIG_NCOBMC_ADAPT_WEIGHT
  if (last_motion_mode_allowed == NCOBMC_ADAPT_WEIGHT) {
    aom_write_symbol(w, mbmi->motion_mode,
                     xd->tile_ctx->ncobmc_cdf[mbmi->sb_type],
                     OBMC_FAMILY_MODES);
  } else if (last_motion_mode_allowed == OBMC_CAUSAL) {
    aom_write_symbol(w, mbmi->motion_mode == OBMC_CAUSAL,
                     xd->tile_ctx->obmc_cdf[mbmi->sb_type], 2);
  } else {
#else
Yue Chen's avatar
Yue Chen committed
446
  if (last_motion_mode_allowed == OBMC_CAUSAL) {
447 448 449 450
#if CONFIG_NEW_MULTISYMBOL
    aom_write_symbol(w, mbmi->motion_mode == OBMC_CAUSAL,
                     xd->tile_ctx->obmc_cdf[mbmi->sb_type], 2);
#else
Yue Chen's avatar
Yue Chen committed
451 452
    aom_write(w, mbmi->motion_mode == OBMC_CAUSAL,
              cm->fc->obmc_prob[mbmi->sb_type]);
453
#endif
Yue Chen's avatar
Yue Chen committed
454
  } else {
455
#endif  // CONFIG_NCOBMC_ADAPT_WEIGHT
Yue Chen's avatar
Yue Chen committed
456
#endif  // CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION
Thomas Davies's avatar
Thomas Davies committed
457 458 459
    aom_write_symbol(w, mbmi->motion_mode,
                     xd->tile_ctx->motion_mode_cdf[mbmi->sb_type],
                     MOTION_MODES);
Yue Chen's avatar
Yue Chen committed
460 461 462 463
#if CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION
  }
#endif  // CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION
}
464 465

#if CONFIG_NCOBMC_ADAPT_WEIGHT
466
static void write_ncobmc_mode(MACROBLOCKD *xd, const MODE_INFO *mi,
467 468 469
                              aom_writer *w) {
  const MB_MODE_INFO *mbmi = &mi->mbmi;
  ADAPT_OVERLAP_BLOCK ao_block = adapt_overlap_block_lookup[mbmi->sb_type];
470
  if (mbmi->motion_mode != NCOBMC_ADAPT_WEIGHT) return;
471

472 473
  aom_write_symbol(w, mbmi->ncobmc_mode[0],
                   xd->tile_ctx->ncobmc_mode_cdf[ao_block], MAX_NCOBMC_MODES);
474
  if (mi_size_wide[mbmi->sb_type] != mi_size_high[mbmi->sb_type]) {
475 476
    aom_write_symbol(w, mbmi->ncobmc_mode[1],
                     xd->tile_ctx->ncobmc_mode_cdf[ao_block], MAX_NCOBMC_MODES);
477 478 479
  }
}
#endif
Yue Chen's avatar
Yue Chen committed
480 481
#endif  // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION

Thomas Davies's avatar
Thomas Davies committed
482 483
static void write_delta_qindex(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                               int delta_qindex, aom_writer *w) {
484 485
  int sign = delta_qindex < 0;
  int abs = sign ? -delta_qindex : delta_qindex;
Thomas Davies's avatar
Thomas Davies committed
486
  int rem_bits, thr;
487
  int smallval = abs < DELTA_Q_SMALL ? 1 : 0;
Thomas Davies's avatar
Thomas Davies committed
488 489
  FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
  (void)cm;
490

Thomas Davies's avatar
Thomas Davies committed
491 492
  aom_write_symbol(w, AOMMIN(abs, DELTA_Q_SMALL), ec_ctx->delta_q_cdf,
                   DELTA_Q_PROBS + 1);
493 494 495 496

  if (!smallval) {
    rem_bits = OD_ILOG_NZ(abs - 1) - 1;
    thr = (1 << rem_bits) + 1;
497
    aom_write_literal(w, rem_bits - 1, 3);
498
    aom_write_literal(w, abs - thr, rem_bits);
499 500 501 502 503
  }
  if (abs > 0) {
    aom_write_bit(w, sign);
  }
}
504

Fangwen Fu's avatar
Fangwen Fu committed
505 506
#if CONFIG_EXT_DELTA_Q
static void write_delta_lflevel(const AV1_COMMON *cm, const MACROBLOCKD *xd,
507 508 509
#if CONFIG_LOOPFILTER_LEVEL
                                int lf_id,
#endif
Fangwen Fu's avatar
Fangwen Fu committed
510 511 512 513 514 515 516 517
                                int delta_lflevel, aom_writer *w) {
  int sign = delta_lflevel < 0;
  int abs = sign ? -delta_lflevel : delta_lflevel;
  int rem_bits, thr;
  int smallval = abs < DELTA_LF_SMALL ? 1 : 0;
  FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
  (void)cm;

518
#if CONFIG_LOOPFILTER_LEVEL
519 520 521 522 523 524 525 526
  if (cm->delta_lf_multi) {
    assert(lf_id >= 0 && lf_id < FRAME_LF_COUNT);
    aom_write_symbol(w, AOMMIN(abs, DELTA_LF_SMALL),
                     ec_ctx->delta_lf_multi_cdf[lf_id], DELTA_LF_PROBS + 1);
  } else {
    aom_write_symbol(w, AOMMIN(abs, DELTA_LF_SMALL), ec_ctx->delta_lf_cdf,
                     DELTA_LF_PROBS + 1);
  }
527
#else
Fangwen Fu's avatar
Fangwen Fu committed
528 529
  aom_write_symbol(w, AOMMIN(abs, DELTA_LF_SMALL), ec_ctx->delta_lf_cdf,
                   DELTA_LF_PROBS + 1);
530
#endif  // CONFIG_LOOPFILTER_LEVEL
Fangwen Fu's avatar
Fangwen Fu committed
531 532 533 534

  if (!smallval) {
    rem_bits = OD_ILOG_NZ(abs - 1) - 1;
    thr = (1 << rem_bits) + 1;
535
    aom_write_literal(w, rem_bits - 1, 3);
Fangwen Fu's avatar
Fangwen Fu committed
536 537 538 539 540 541 542
    aom_write_literal(w, abs - thr, rem_bits);
  }
  if (abs > 0) {
    aom_write_bit(w, sign);
  }
}
#endif  // CONFIG_EXT_DELTA_Q
543

544
#if !CONFIG_NEW_MULTISYMBOL
Yaowu Xu's avatar
Yaowu Xu committed
545
static void update_skip_probs(AV1_COMMON *cm, aom_writer *w,
Jingning Han's avatar
Jingning Han committed
546 547
                              FRAME_COUNTS *counts) {
  int k;
548 549 550 551 552
  const int probwt = cm->num_tg;
  for (k = 0; k < SKIP_CONTEXTS; ++k) {
    av1_cond_prob_diff_update(w, &cm->fc->skip_probs[k], counts->skip[k],
                              probwt);
  }
Jingning Han's avatar
Jingning Han committed
553
}
554
#endif
Jingning Han's avatar
Jingning Han committed
555

556 557
// TODO(anybody) : remove this flag when PVQ supports pallete coding tool
#if !CONFIG_PVQ
Sarah Parker's avatar
Sarah Parker committed
558 559
static void pack_map_tokens(aom_writer *w, const TOKENEXTRA **tp, int n,
                            int num) {
560
  const TOKENEXTRA *p = *tp;
561 562 563 564
  write_uniform(w, n, p->token);  // The first color index.
  ++p;
  --num;
  for (int i = 0; i < num; ++i) {
565
    aom_write_symbol(w, p->token, p->color_map_cdf, n);
hui su's avatar
hui su committed
566 567 568 569
    ++p;
  }
  *tp = p;
}
570
#endif  // !CONFIG_PVQ
571

572
#if !CONFIG_PVQ
573

574
#if !CONFIG_LV_MAP
575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600
#if CONFIG_NEW_MULTISYMBOL
static INLINE void write_coeff_extra(const aom_cdf_prob *const *cdf, int val,
                                     int n, aom_writer *w) {
  // Code the extra bits from LSB to MSB in groups of 4
  int i = 0;
  int count = 0;
  while (count < n) {
    const int size = AOMMIN(n - count, 4);
    const int mask = (1 << size) - 1;
    aom_write_cdf(w, val & mask, cdf[i++], 1 << size);
    val >>= size;
    count += size;
  }
}
#else
static INLINE void write_coeff_extra(const aom_prob *pb, int value,
                                     int num_bits, int skip_bits, aom_writer *w,
                                     TOKEN_STATS *token_stats) {
  // Code the extra bits from MSB to LSB 1 bit at a time
  int index;
  for (index = skip_bits; index < num_bits; ++index) {
    const int shift = num_bits - index - 1;
    const int bb = (value >> shift) & 1;
    aom_write_record(w, bb, pb[index], token_stats);
  }
}
601
#endif  // CONFIG_NEW_MULTISYMBOL
602

Yaowu Xu's avatar
Yaowu Xu committed
603
static void pack_mb_tokens(aom_writer *w, const TOKENEXTRA **tp,
604
                           const TOKENEXTRA *const stop,
605
                           aom_bit_depth_t bit_depth, const TX_SIZE tx_size,
Sarah Parker's avatar
Sarah Parker committed
606 607 608
#if CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
                           TX_TYPE tx_type, int is_inter,
#endif  // CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
609
                           TOKEN_STATS *token_stats) {
610
  const TOKENEXTRA *p = *tp;
Fangwen Fu's avatar
Fangwen Fu committed
611
#if CONFIG_VAR_TX
612
  int count = 0;
613
  const int seg_eob = tx_size_2d[tx_size];
614
#endif
Jingning Han's avatar
Jingning Han committed
615

Sarah Parker's avatar
Sarah Parker committed
616 617 618 619 620 621 622 623 624 625 626
#if CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
  if (tx_type == MRC_DCT && ((is_inter && SIGNAL_MRC_MASK_INTER) ||
                             (!is_inter && SIGNAL_MRC_MASK_INTRA))) {
    int rows = tx_size_high[tx_size];
    int cols = tx_size_wide[tx_size];
    assert(tx_size == TX_32X32);
    assert(p < stop);
    pack_map_tokens(w, &p, 2, rows * cols);
  }
#endif  // CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK

Jingning Han's avatar
Jingning Han committed
627
  while (p < stop && p->token != EOSB_TOKEN) {
628
    const int token = p->token;
629
    const int eob_val = p->eob_val;
630
    if (token == BLOCK_Z_TOKEN) {
631
      aom_write_symbol(w, 0, *p->head_cdf, HEAD_TOKENS + 1);
632
      p++;
633 634 635
#if CONFIG_VAR_TX
      break;
#endif
636 637
      continue;
    }
Yaowu Xu's avatar
Yaowu Xu committed
638 639

    const av1_extra_bit *const extra_bits = &av1_extra_bits[token];
640
    if (eob_val == LAST_EOB) {
641 642 643
      // Just code a flag indicating whether the value is >1 or 1.
      aom_write_bit(w, token != ONE_TOKEN);
    } else {
644
      int comb_symb = 2 * AOMMIN(token, TWO_TOKEN) - eob_val + p->first_val;
645
      aom_write_symbol(w, comb_symb, *p->head_cdf, HEAD_TOKENS + p->first_val);
646
    }
647
    if (token > ONE_TOKEN) {
648
      aom_write_symbol(w, token - TWO_TOKEN, *p->tail_cdf, TAIL_TOKENS);
Alex Converse's avatar
Alex Converse committed
649
    }
650 651 652 653

    if (extra_bits->base_val) {
      const int bit_string = p->extra;
      const int bit_string_length = extra_bits->len;  // Length of extra bits to
654
      const int is_cat6 = (extra_bits->base_val == CAT6_MIN_VAL);
655 656
      // be written excluding
      // the sign bit.
657
      int skip_bits = is_cat6
658 659 660
                          ? (int)sizeof(av1_cat6_prob) -
                                av1_get_cat6_extrabits_size(tx_size, bit_depth)
                          : 0;
661

662 663 664 665 666 667 668 669 670
      assert(!(bit_string >> (bit_string_length - skip_bits + 1)));
      if (bit_string_length > 0)
#if CONFIG_NEW_MULTISYMBOL
        write_coeff_extra(extra_bits->cdf, bit_string >> 1,
                          bit_string_length - skip_bits, w);
#else
        write_coeff_extra(extra_bits->prob, bit_string >> 1, bit_string_length,
                          skip_bits, w, token_stats);
#endif
671 672 673 674 675

      aom_write_bit_record(w, bit_string & 1, token_stats);
    }
    ++p;

Fangwen Fu's avatar
Fangwen Fu committed
676
#if CONFIG_VAR_TX
677
    ++count;
678
    if (eob_val == EARLY_EOB || count == seg_eob) break;
679 680 681 682 683
#endif
  }

  *tp = p;
}
Angie Chiang's avatar
Angie Chiang committed
684
#endif  // !CONFIG_LV_MAP
685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707
#else   // !CONFIG_PVQ
static PVQ_INFO *get_pvq_block(PVQ_QUEUE *pvq_q) {
  PVQ_INFO *pvq;

  assert(pvq_q->curr_pos <= pvq_q->last_pos);
  assert(pvq_q->curr_pos < pvq_q->buf_len);

  pvq = pvq_q->buf + pvq_q->curr_pos;
  ++pvq_q->curr_pos;

  return pvq;
}

static void pack_pvq_tokens(aom_writer *w, MACROBLOCK *const x,
                            MACROBLOCKD *const xd, int plane, BLOCK_SIZE bsize,
                            const TX_SIZE tx_size) {
  PVQ_INFO *pvq;
  int idx, idy;
  const struct macroblockd_plane *const pd = &xd->plane[plane];
  od_adapt_ctx *adapt;
  int max_blocks_wide;
  int max_blocks_high;
  int step = (1 << tx_size);
708 709

#if CONFIG_CHROMA_SUB8X8
710
  const BLOCK_SIZE plane_bsize =
711 712 713 714 715 716 717
      AOMMAX(BLOCK_4X4, get_plane_block_size(bsize, pd));
#elif CONFIG_CB4X4
  const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd);
#else
  const BLOCK_SIZE plane_bsize =
      get_plane_block_size(AOMMAX(BLOCK_8X8, bsize), pd);
#endif
718

719
  adapt = x->daala_enc.state.adapt;
720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737

  max_blocks_wide = max_block_wide(xd, plane_bsize, plane);
  max_blocks_high = max_block_high(xd, plane_bsize, plane);

  for (idy = 0; idy < max_blocks_high; idy += step) {
    for (idx = 0; idx < max_blocks_wide; idx += step) {
      const int is_keyframe = 0;
      const int encode_flip = 0;
      const int flip = 0;
      int i;
      const int has_dc_skip = 1;
      int *exg = &adapt->pvq.pvq_exg[plane][tx_size][0];
      int *ext = adapt->pvq.pvq_ext + tx_size * PVQ_MAX_PARTITIONS;
      generic_encoder *model = adapt->pvq.pvq_param_model;

      pvq = get_pvq_block(x->pvq_q);

      // encode block skip info
738 739
      aom_write_symbol(w, pvq->ac_dc_coded,
                       adapt->skip_cdf[2 * tx_size + (plane != 0)], 4);
740 741 742 743 744 745 746 747

      // AC coeffs coded?
      if (pvq->ac_dc_coded & AC_CODED) {
        assert(pvq->bs == tx_size);
        for (i = 0; i < pvq->nb_bands; i++) {
          if (i == 0 ||
              (!pvq->skip_rest && !(pvq->skip_dir & (1 << ((i - 1) % 3))))) {
            pvq_encode_partition(
748 749
                w, pvq->qg[i], pvq->theta[i], pvq->y + pvq->off[i],
                pvq->size[i], pvq->k[i], model, adapt, exg + i, ext + i,
750 751 752 753 754 755
                (plane != 0) * OD_TXSIZES * PVQ_MAX_PARTITIONS +
                    pvq->bs * PVQ_MAX_PARTITIONS + i,
                is_keyframe, i == 0 && (i < pvq->nb_bands - 1), pvq->skip_rest,
                encode_flip, flip);
          }
          if (i == 0 && !pvq->skip_rest && pvq->bs > 0) {
756
            aom_write_symbol(
757 758 759
                w, pvq->skip_dir,
                &adapt->pvq
                     .pvq_skip_dir_cdf[(plane != 0) + 2 * (pvq->bs - 1)][0],
760
                7);
761 762 763 764 765 766
          }
        }
      }
      // Encode residue of DC coeff, if exist.
      if (!has_dc_skip || (pvq->ac_dc_coded & DC_CODED)) {
        generic_encode(w, &adapt->model_dc[plane],
767
                       abs(pvq->dq_dc_residue) - has_dc_skip,
768 769 770 771 772 773 774 775
                       &adapt->ex_dc[plane][pvq->bs][0], 2);
      }
      if ((pvq->ac_dc_coded & DC_CODED)) {
        aom_write_bit(w, pvq->dq_dc_residue < 0);
      }
    }
  }  // for (idy = 0;
}
776
#endif  // !CONFIG_PVG
777

778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815
#if CONFIG_VAR_TX && !CONFIG_COEF_INTERLEAVE
#if CONFIG_LV_MAP
static void pack_txb_tokens(aom_writer *w,
#if CONFIG_LV_MAP
                            AV1_COMMON *cm,
#endif  // CONFIG_LV_MAP
                            const TOKENEXTRA **tp,
                            const TOKENEXTRA *const tok_end,
#if CONFIG_PVQ || CONFIG_LV_MAP
                            MACROBLOCK *const x,
#endif
                            MACROBLOCKD *xd, MB_MODE_INFO *mbmi, int plane,
                            BLOCK_SIZE plane_bsize, aom_bit_depth_t bit_depth,
                            int block, int blk_row, int blk_col,
                            TX_SIZE tx_size, TOKEN_STATS *token_stats) {
  const struct macroblockd_plane *const pd = &xd->plane[plane];
  const BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
  const int tx_row = blk_row >> (1 - pd->subsampling_y);
  const int tx_col = blk_col >> (1 - pd->subsampling_x);
  TX_SIZE plane_tx_size;
  const int max_blocks_high = max_block_high(xd, plane_bsize, plane);
  const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane);

  if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;

  plane_tx_size =
      plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0]
            : mbmi->inter_tx_size[tx_row][tx_col];

  if (tx_size == plane_tx_size) {
    TOKEN_STATS tmp_token_stats;
    init_token_stats(&tmp_token_stats);

#if !CONFIG_PVQ
    tran_low_t *tcoeff = BLOCK_OFFSET(x->mbmi_ext->tcoeff[plane], block);
    uint16_t eob = x->mbmi_ext->eobs[plane][block];
    TXB_CTX txb_ctx = { x->mbmi_ext->txb_skip_ctx[plane][block],
                        x->mbmi_ext->dc_sign_ctx[plane][block] };
Jingning Han's avatar
Jingning Han committed
816 817
    av1_write_coeffs_txb(cm, xd, w, blk_row, blk_col, block, plane, tx_size,
                         tcoeff, eob, &txb_ctx);
818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853
#else
    pack_pvq_tokens(w, x, xd, plane, bsize, tx_size);
#endif
#if CONFIG_RD_DEBUG
    token_stats->txb_coeff_cost_map[blk_row][blk_col] = tmp_token_stats.cost;
    token_stats->cost += tmp_token_stats.cost;
#endif
  } else {
    const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
    const int bsl = tx_size_wide_unit[sub_txs];
    int i;

    assert(bsl > 0);

    for (i = 0; i < 4; ++i) {
      const int offsetr = blk_row + (i >> 1) * bsl;
      const int offsetc = blk_col + (i & 0x01) * bsl;
      const int step = tx_size_wide_unit[sub_txs] * tx_size_high_unit[sub_txs];

      if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;

      pack_txb_tokens(w,
#if CONFIG_LV_MAP
                      cm,
#endif
                      tp, tok_end,
#if CONFIG_PVQ || CONFIG_LV_MAP
                      x,
#endif
                      xd, mbmi, plane, plane_bsize, bit_depth, block, offsetr,
                      offsetc, sub_txs, token_stats);
      block += step;
    }
  }
}
#else  // CONFIG_LV_MAP
Yaowu Xu's avatar
Yaowu Xu committed
854
static void pack_txb_tokens(aom_writer *w, const TOKENEXTRA **tp,
855 856 857 858 859
                            const TOKENEXTRA *const tok_end,
#if CONFIG_PVQ
                            MACROBLOCK *const x,
#endif
                            MACROBLOCKD *xd, MB_MODE_INFO *mbmi, int plane,
Yaowu Xu's avatar
Yaowu Xu committed
860
                            BLOCK_SIZE plane_bsize, aom_bit_depth_t bit_depth,
861
                            int block, int blk_row, int blk_col,
862
                            TX_SIZE tx_size, TOKEN_STATS *token_stats) {
863 864
  const struct macroblockd_plane *const pd = &xd->plane[plane];
  const BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
865 866
  const int tx_row = blk_row >> (1 - pd->subsampling_y);
  const int tx_col = blk_col >> (1 - pd->subsampling_x);
Debargha Mukherjee's avatar
Debargha Mukherjee committed
867
  TX_SIZE plane_tx_size;
868 869
  const int max_blocks_high = max_block_high(xd, plane_bsize, plane);
  const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane);
Sarah Parker's avatar
Sarah Parker committed
870 871 872 873
#if CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
  TX_TYPE tx_type = av1_get_tx_type(plane ? PLANE_TYPE_UV : PLANE_TYPE_Y, xd,
                                    blk_row, blk_col, block, tx_size);
#endif  // CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
874

875
  if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
876

877 878 879
  plane_tx_size =
      plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0]
            : mbmi->inter_tx_size[tx_row][tx_col];
Debargha Mukherjee's avatar
Debargha Mukherjee committed
880

881
  if (tx_size == plane_tx_size) {
882 883
    TOKEN_STATS tmp_token_stats;
    init_token_stats(&tmp_token_stats);
884
#if !CONFIG_PVQ
Sarah Parker's avatar
Sarah Parker committed
885 886 887 888 889
    pack_mb_tokens(w, tp, tok_end, bit_depth, tx_size,
#if CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
                   tx_type, is_inter_block(mbmi),
#endif  // CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
                   &tmp_token_stats);
890 891 892
#else
    pack_pvq_tokens(w, x, xd, plane, bsize, tx_size);
#endif
893 894 895 896
#if CONFIG_RD_DEBUG
    token_stats->txb_coeff_cost_map[blk_row][blk_col] = tmp_token_stats.cost;
    token_stats->cost += tmp_token_stats.cost;
#endif
897
  } else {
Yue Chen's avatar
Yue Chen committed
898 899 900 901
#if CONFIG_RECT_TX_EXT
    int is_qttx = plane_tx_size == quarter_txsize_lookup[plane_bsize];
    const TX_SIZE sub_txs = is_qttx ? plane_tx_size : sub_tx_size_map[tx_size];
#else
902
    const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
Yue Chen's avatar
Yue Chen committed
903
#endif
904
    const int bsl = tx_size_wide_unit[sub_txs];
905 906 907 908 909
    int i;

    assert(bsl > 0);

    for (i = 0; i < 4; ++i) {
Yue Chen's avatar
Yue Chen committed
910 911 912 913 914 915 916 917 918
#if CONFIG_RECT_TX_EXT
      int is_wide_tx = tx_size_wide_unit[sub_txs] > tx_size_high_unit[sub_txs];
      const int offsetr =
          is_qttx ? (is_wide_tx ? i * tx_size_high_unit[sub_txs] : 0)
                  : blk_row + (i >> 1) * bsl;
      const int offsetc =
          is_qttx ? (is_wide_tx ? 0 : i * tx_size_wide_unit[sub_txs])
                  : blk_col + (i & 0x01) * bsl;
#else
919 920
      const int offsetr = blk_row + (i >> 1) * bsl;
      const int offsetc = blk_col + (i & 0x01) * bsl;
Yue Chen's avatar
Yue Chen committed
921
#endif
922
      const int step = tx_size_wide_unit[sub_txs] * tx_size_high_unit[sub_txs];
923

924
      if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;
925

926 927 928 929 930 931
      pack_txb_tokens(w, tp, tok_end<