bitstream.c 189 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4 5 6 7 8 9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10 11 12 13
 */

#include <assert.h>
#include <limits.h>
14
#include <stdio.h>
Jingning Han's avatar
Jingning Han committed
15

Yaowu Xu's avatar
Yaowu Xu committed
16 17
#include "aom/aom_encoder.h"
#include "aom_dsp/aom_dsp_common.h"
18
#include "aom_dsp/binary_codes_writer.h"
19
#include "aom_dsp/bitwriter_buffer.h"
Yaowu Xu's avatar
Yaowu Xu committed
20
#include "aom_mem/aom_mem.h"
21 22
#include "aom_ports/mem_ops.h"
#include "aom_ports/system_state.h"
23
#if CONFIG_BITSTREAM_DEBUG
24
#include "aom_util/debug_util.h"
25
#endif  // CONFIG_BITSTREAM_DEBUG
Jingning Han's avatar
Jingning Han committed
26

27
#if CONFIG_CDEF
28
#include "av1/common/cdef.h"
29
#endif  // CONFIG_CDEF
30 31 32 33
#include "av1/common/entropy.h"
#include "av1/common/entropymode.h"
#include "av1/common/entropymv.h"
#include "av1/common/mvref_common.h"
34
#include "av1/common/odintrin.h"
35 36
#include "av1/common/pred_common.h"
#include "av1/common/reconinter.h"
hui su's avatar
hui su committed
37 38 39
#if CONFIG_EXT_INTRA
#include "av1/common/reconintra.h"
#endif  // CONFIG_EXT_INTRA
40 41
#include "av1/common/seg_common.h"
#include "av1/common/tile_common.h"
Jingning Han's avatar
Jingning Han committed
42

Angie Chiang's avatar
Angie Chiang committed
43 44 45
#if CONFIG_LV_MAP
#include "av1/encoder/encodetxb.h"
#endif  // CONFIG_LV_MAP
46 47 48 49
#include "av1/encoder/bitstream.h"
#include "av1/encoder/cost.h"
#include "av1/encoder/encodemv.h"
#include "av1/encoder/mcomp.h"
Urvang Joshi's avatar
Urvang Joshi committed
50
#if CONFIG_PALETTE_DELTA_ENCODING
hui su's avatar
hui su committed
51
#include "av1/encoder/palette.h"
Urvang Joshi's avatar
Urvang Joshi committed
52
#endif  // CONFIG_PALETTE_DELTA_ENCODING
53 54 55
#include "av1/encoder/segmentation.h"
#include "av1/encoder/subexp.h"
#include "av1/encoder/tokenize.h"
Jingning Han's avatar
Jingning Han committed
56

57
#define ENC_MISMATCH_DEBUG 0
58

59
#if CONFIG_COMPOUND_SINGLEREF
60 61
static struct av1_token
    inter_singleref_comp_mode_encodings[INTER_SINGLEREF_COMP_MODES];
62
#endif  // CONFIG_COMPOUND_SINGLEREF
63

Yaowu Xu's avatar
Yaowu Xu committed
64
static INLINE void write_uniform(aom_writer *w, int n, int v) {
65 66
  const int l = get_unsigned_bits(n);
  const int m = (1 << l) - n;
67
  if (l == 0) return;
hui su's avatar
hui su committed
68
  if (v < m) {
Yaowu Xu's avatar
Yaowu Xu committed
69
    aom_write_literal(w, v, l - 1);
hui su's avatar
hui su committed
70
  } else {
Yaowu Xu's avatar
Yaowu Xu committed
71 72
    aom_write_literal(w, m + ((v - m) >> 1), l - 1);
    aom_write_literal(w, (v - m) & 1, 1);
hui su's avatar
hui su committed
73 74
  }
}
Jingning Han's avatar
Jingning Han committed
75

76
#if CONFIG_INTERINTRA
Yaowu Xu's avatar
Yaowu Xu committed
77
static struct av1_token interintra_mode_encodings[INTERINTRA_MODES];
78
#endif
79
#if CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE
80
static struct av1_token compound_type_encodings[COMPOUND_TYPES];
81
#endif  // CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE
82
#if CONFIG_LOOP_RESTORATION
83 84
static void loop_restoration_write_sb_coeffs(const AV1_COMMON *const cm,
                                             MACROBLOCKD *xd,
85 86
                                             const RestorationUnitInfo *rui,
                                             aom_writer *const w, int plane);
87
#endif  // CONFIG_LOOP_RESTORATION
88 89 90 91 92 93 94 95
#if CONFIG_OBU
static void write_uncompressed_header_obu(AV1_COMP *cpi,
                                          struct aom_write_bit_buffer *wb);
#else
static void write_uncompressed_header_frame(AV1_COMP *cpi,
                                            struct aom_write_bit_buffer *wb);
#endif

96
static uint32_t write_compressed_header(AV1_COMP *cpi, uint8_t *data);
97

98
#if !CONFIG_OBU || CONFIG_EXT_TILE
99 100 101 102 103
static int remux_tiles(const AV1_COMMON *const cm, uint8_t *dst,
                       const uint32_t data_size, const uint32_t max_tile_size,
                       const uint32_t max_tile_col_size,
                       int *const tile_size_bytes,
                       int *const tile_col_size_bytes);
104
#endif
Yaowu Xu's avatar
Yaowu Xu committed
105
void av1_encode_token_init(void) {
106
#if CONFIG_INTERINTRA
Yaowu Xu's avatar
Yaowu Xu committed
107
  av1_tokens_from_tree(interintra_mode_encodings, av1_interintra_mode_tree);
108
#endif  // CONFIG_INTERINTRA
109 110 111 112
#if CONFIG_COMPOUND_SINGLEREF
  av1_tokens_from_tree(inter_singleref_comp_mode_encodings,
                       av1_inter_singleref_comp_mode_tree);
#endif  // CONFIG_COMPOUND_SINGLEREF
113
#if CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE
114
  av1_tokens_from_tree(compound_type_encodings, av1_compound_type_tree);
115
#endif  // CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE
116 117
}

Jingning Han's avatar
Jingning Han committed
118 119 120 121
static void write_intra_mode_kf(const AV1_COMMON *cm, FRAME_CONTEXT *frame_ctx,
                                const MODE_INFO *mi, const MODE_INFO *above_mi,
                                const MODE_INFO *left_mi, int block,
                                PREDICTION_MODE mode, aom_writer *w) {
Alex Converse's avatar
Alex Converse committed
122 123 124
#if CONFIG_INTRABC
  assert(!is_intrabc_block(&mi->mbmi));
#endif  // CONFIG_INTRABC
Hui Su's avatar
Hui Su committed
125
  aom_write_symbol(w, mode,
Jingning Han's avatar
Jingning Han committed
126 127 128 129
                   get_y_mode_cdf(frame_ctx, mi, above_mi, left_mi, block),
                   INTRA_MODES);
  (void)cm;
}
Jingning Han's avatar
Jingning Han committed
130

131
static void write_inter_mode(aom_writer *w, PREDICTION_MODE mode,
132
                             FRAME_CONTEXT *ec_ctx, const int16_t mode_ctx) {
133
  const int16_t newmv_ctx = mode_ctx & NEWMV_CTX_MASK;
Yue Chen's avatar
Yue Chen committed
134

135 136 137 138 139
#if CONFIG_NEW_MULTISYMBOL
  aom_write_symbol(w, mode != NEWMV, ec_ctx->newmv_cdf[newmv_ctx], 2);
#else
  aom_write(w, mode != NEWMV, ec_ctx->newmv_prob[newmv_ctx]);
#endif
140

Jingning Han's avatar
Jingning Han committed
141
  if (mode != NEWMV) {
142 143 144 145 146
    if (mode_ctx & (1 << ALL_ZERO_FLAG_OFFSET)) {
      assert(mode == ZEROMV);
      return;
    }

147 148 149 150 151 152
    const int16_t zeromv_ctx = (mode_ctx >> ZEROMV_OFFSET) & ZEROMV_CTX_MASK;
#if CONFIG_NEW_MULTISYMBOL
    aom_write_symbol(w, mode != ZEROMV, ec_ctx->zeromv_cdf[zeromv_ctx], 2);
#else
    aom_write(w, mode != ZEROMV, ec_ctx->zeromv_prob[zeromv_ctx]);
#endif
153 154

    if (mode != ZEROMV) {
155 156
      int16_t refmv_ctx = (mode_ctx >> REFMV_OFFSET) & REFMV_CTX_MASK;

157 158 159
      if (mode_ctx & (1 << SKIP_NEARESTMV_OFFSET)) refmv_ctx = 6;
      if (mode_ctx & (1 << SKIP_NEARMV_OFFSET)) refmv_ctx = 7;
      if (mode_ctx & (1 << SKIP_NEARESTMV_SUB8X8_OFFSET)) refmv_ctx = 8;
160 161 162 163 164
#if CONFIG_NEW_MULTISYMBOL
      aom_write_symbol(w, mode != NEARESTMV, ec_ctx->refmv_cdf[refmv_ctx], 2);
#else
      aom_write(w, mode != NEARESTMV, ec_ctx->refmv_prob[refmv_ctx]);
#endif
165 166
    }
  }
Jingning Han's avatar
Jingning Han committed
167 168
}

169
static void write_drl_idx(FRAME_CONTEXT *ec_ctx, const MB_MODE_INFO *mbmi,
Yaowu Xu's avatar
Yaowu Xu committed
170 171
                          const MB_MODE_INFO_EXT *mbmi_ext, aom_writer *w) {
  uint8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame);
172 173 174

  assert(mbmi->ref_mv_idx < 3);

175 176 177 178
#if CONFIG_COMPOUND_SINGLEREF
  if (mbmi->mode == NEWMV || mbmi->mode == NEW_NEWMV ||
      mbmi->mode == SR_NEW_NEWMV) {
#else   // !CONFIG_COMPOUND_SINGLEREF
179
  if (mbmi->mode == NEWMV || mbmi->mode == NEW_NEWMV) {
180
#endif  // CONFIG_COMPOUND_SINGLEREF
181 182 183 184
    int idx;
    for (idx = 0; idx < 2; ++idx) {
      if (mbmi_ext->ref_mv_count[ref_frame_type] > idx + 1) {
        uint8_t drl_ctx =
Yaowu Xu's avatar
Yaowu Xu committed
185
            av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx);
186

187 188 189 190 191 192
#if CONFIG_NEW_MULTISYMBOL
        aom_write_symbol(w, mbmi->ref_mv_idx != idx, ec_ctx->drl_cdf[drl_ctx],
                         2);
#else
        aom_write(w, mbmi->ref_mv_idx != idx, ec_ctx->drl_prob[drl_ctx]);
#endif
193
        if (mbmi->ref_mv_idx == idx) return;
194
      }
195 196 197 198
    }
    return;
  }

David Barker's avatar
David Barker committed
199
  if (have_nearmv_in_inter_mode(mbmi->mode)) {
200 201 202 203 204
    int idx;
    // TODO(jingning): Temporary solution to compensate the NEARESTMV offset.
    for (idx = 1; idx < 3; ++idx) {
      if (mbmi_ext->ref_mv_count[ref_frame_type] > idx + 1) {
        uint8_t drl_ctx =
Yaowu Xu's avatar
Yaowu Xu committed
205
            av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx);
206 207 208 209 210 211
#if CONFIG_NEW_MULTISYMBOL
        aom_write_symbol(w, mbmi->ref_mv_idx != (idx - 1),
                         ec_ctx->drl_cdf[drl_ctx], 2);
#else
        aom_write(w, mbmi->ref_mv_idx != (idx - 1), ec_ctx->drl_prob[drl_ctx]);
#endif
212
        if (mbmi->ref_mv_idx == (idx - 1)) return;
213
      }
214
    }
215
    return;
216 217 218
  }
}

219 220
static void write_inter_compound_mode(AV1_COMMON *cm, MACROBLOCKD *xd,
                                      aom_writer *w, PREDICTION_MODE mode,
221
                                      const int16_t mode_ctx) {
222 223 224 225 226
  assert(is_inter_compound_mode(mode));
  (void)cm;
  aom_write_symbol(w, INTER_COMPOUND_OFFSET(mode),
                   xd->tile_ctx->inter_compound_mode_cdf[mode_ctx],
                   INTER_COMPOUND_MODES);
227
}
228 229

#if CONFIG_COMPOUND_SINGLEREF
230
static void write_inter_singleref_comp_mode(MACROBLOCKD *xd, aom_writer *w,
231 232 233
                                            PREDICTION_MODE mode,
                                            const int16_t mode_ctx) {
  assert(is_inter_singleref_comp_mode(mode));
234 235
  aom_cdf_prob *const inter_singleref_comp_cdf =
      xd->tile_ctx->inter_singleref_comp_mode_cdf[mode_ctx];
236

237 238
  aom_write_symbol(w, INTER_SINGLEREF_COMP_OFFSET(mode),
                   inter_singleref_comp_cdf, INTER_SINGLEREF_COMP_MODES);
239 240
}
#endif  // CONFIG_COMPOUND_SINGLEREF
241

Yaowu Xu's avatar
Yaowu Xu committed
242
static void encode_unsigned_max(struct aom_write_bit_buffer *wb, int data,
243
                                int max) {
Yaowu Xu's avatar
Yaowu Xu committed
244
  aom_wb_write_literal(wb, data, get_unsigned_bits(max));
Jingning Han's avatar
Jingning Han committed
245 246
}

247
static void write_tx_size_vartx(const AV1_COMMON *cm, MACROBLOCKD *xd,
248
                                const MB_MODE_INFO *mbmi, TX_SIZE tx_size,
249 250
                                int depth, int blk_row, int blk_col,
                                aom_writer *w) {
251 252 253 254
#if CONFIG_NEW_MULTISYMBOL
  FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
  (void)cm;
#endif
255 256
  const int tx_row = blk_row >> 1;
  const int tx_col = blk_col >> 1;
257 258 259
  const int max_blocks_high = max_block_high(xd, mbmi->sb_type, 0);
  const int max_blocks_wide = max_block_wide(xd, mbmi->sb_type, 0);

260 261
  int ctx = txfm_partition_context(xd->above_txfm_context + blk_col,
                                   xd->left_txfm_context + blk_row,
262
                                   mbmi->sb_type, tx_size);
263

264
  if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
265

Jingning Han's avatar
Jingning Han committed
266
  if (depth == MAX_VARTX_DEPTH) {
267 268
    txfm_partition_update(xd->above_txfm_context + blk_col,
                          xd->left_txfm_context + blk_row, tx_size, tx_size);
269 270 271
    return;
  }

Yue Chen's avatar
Yue Chen committed
272 273 274 275
#if CONFIG_RECT_TX_EXT
  if (tx_size == mbmi->inter_tx_size[tx_row][tx_col] ||
      mbmi->tx_size == quarter_txsize_lookup[mbmi->sb_type]) {
#else
276
  if (tx_size == mbmi->inter_tx_size[tx_row][tx_col]) {
Yue Chen's avatar
Yue Chen committed
277
#endif
278 279 280
#if CONFIG_NEW_MULTISYMBOL
    aom_write_symbol(w, 0, ec_ctx->txfm_partition_cdf[ctx], 2);
#else
Yaowu Xu's avatar
Yaowu Xu committed
281
    aom_write(w, 0, cm->fc->txfm_partition_prob[ctx]);
282 283
#endif

284 285
    txfm_partition_update(xd->above_txfm_context + blk_col,
                          xd->left_txfm_context + blk_row, tx_size, tx_size);
Yue Chen's avatar
Yue Chen committed
286
    // TODO(yuec): set correct txfm partition update for qttx
287
  } else {
288 289
    const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
    const int bsl = tx_size_wide_unit[sub_txs];
290
    int i;
291

292 293 294
#if CONFIG_NEW_MULTISYMBOL
    aom_write_symbol(w, 1, ec_ctx->txfm_partition_cdf[ctx], 2);
#else
Yaowu Xu's avatar
Yaowu Xu committed
295
    aom_write(w, 1, cm->fc->txfm_partition_prob[ctx]);
296
#endif
297

David Barker's avatar
David Barker committed
298
    if (sub_txs == TX_4X4) {
299 300
      txfm_partition_update(xd->above_txfm_context + blk_col,
                            xd->left_txfm_context + blk_row, sub_txs, tx_size);
301
      return;
302
    }
303 304 305

    assert(bsl > 0);
    for (i = 0; i < 4; ++i) {
306 307 308 309
      int offsetr = blk_row + (i >> 1) * bsl;
      int offsetc = blk_col + (i & 0x01) * bsl;
      write_tx_size_vartx(cm, xd, mbmi, sub_txs, depth + 1, offsetr, offsetc,
                          w);
310 311 312
    }
  }
}
313

314
#if !CONFIG_NEW_MULTISYMBOL
Yaowu Xu's avatar
Yaowu Xu committed
315
static void update_txfm_partition_probs(AV1_COMMON *cm, aom_writer *w,
316
                                        FRAME_COUNTS *counts, int probwt) {
317 318
  int k;
  for (k = 0; k < TXFM_PARTITION_CONTEXTS; ++k)
Yaowu Xu's avatar
Yaowu Xu committed
319
    av1_cond_prob_diff_update(w, &cm->fc->txfm_partition_prob[k],
320
                              counts->txfm_partition[k], probwt);
321
}
322
#endif  // CONFIG_NEW_MULTISYMBOL
323

Yaowu Xu's avatar
Yaowu Xu committed
324 325
static void write_selected_tx_size(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                                   aom_writer *w) {
326 327
  const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
  const BLOCK_SIZE bsize = mbmi->sb_type;
328 329
  FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
  (void)cm;
330
  if (block_signals_txsize(bsize)) {
331 332
    const TX_SIZE tx_size = mbmi->tx_size;
    const int is_inter = is_inter_block(mbmi);
333 334
    const int tx_size_ctx = get_tx_size_context(xd);
    const int32_t tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
335
                                         : intra_tx_size_cat_lookup[bsize];
336
    const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size];
337
    const int depth = tx_size_to_depth(coded_tx_size);
338
#if CONFIG_EXT_TX && CONFIG_RECT_TX
339
    assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(xd, mbmi)));
340 341
#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX

342
    aom_write_symbol(w, depth, ec_ctx->tx_size_cdf[tx_size_cat][tx_size_ctx],
343
                     tx_size_cat + 2);
344
#if CONFIG_RECT_TX_EXT
345
    if (is_quarter_tx_allowed(xd, mbmi, is_inter) && tx_size != coded_tx_size)
346 347 348 349
#if CONFIG_NEW_MULTISYMBOL
      aom_write_symbol(w, tx_size == quarter_txsize_lookup[bsize],
                       cm->fc->quarter_tx_size_cdf, 2);
#else
350 351
      aom_write(w, tx_size == quarter_txsize_lookup[bsize],
                cm->fc->quarter_tx_size_prob);
352
#endif
Yue Chen's avatar
Yue Chen committed
353
#endif
Jingning Han's avatar
Jingning Han committed
354 355 356
  }
}

357
#if !CONFIG_NEW_MULTISYMBOL
Yaowu Xu's avatar
Yaowu Xu committed
358
static void update_inter_mode_probs(AV1_COMMON *cm, aom_writer *w,
359 360
                                    FRAME_COUNTS *counts) {
  int i;
361
  const int probwt = cm->num_tg;
362
  for (i = 0; i < NEWMV_MODE_CONTEXTS; ++i)
363 364
    av1_cond_prob_diff_update(w, &cm->fc->newmv_prob[i], counts->newmv_mode[i],
                              probwt);
365
  for (i = 0; i < ZEROMV_MODE_CONTEXTS; ++i)
Yaowu Xu's avatar
Yaowu Xu committed
366
    av1_cond_prob_diff_update(w, &cm->fc->zeromv_prob[i],
367
                              counts->zeromv_mode[i], probwt);
368
  for (i = 0; i < REFMV_MODE_CONTEXTS; ++i)
369 370
    av1_cond_prob_diff_update(w, &cm->fc->refmv_prob[i], counts->refmv_mode[i],
                              probwt);
371
  for (i = 0; i < DRL_MODE_CONTEXTS; ++i)
372 373
    av1_cond_prob_diff_update(w, &cm->fc->drl_prob[i], counts->drl_mode[i],
                              probwt);
374
}
375
#endif
376

Yaowu Xu's avatar
Yaowu Xu committed
377 378
static int write_skip(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                      int segment_id, const MODE_INFO *mi, aom_writer *w) {
Jingning Han's avatar
Jingning Han committed
379 380 381 382
  if (segfeature_active(&cm->seg, segment_id, SEG_LVL_SKIP)) {
    return 1;
  } else {
    const int skip = mi->mbmi.skip;
Zoe Liu's avatar
Zoe Liu committed
383
    const int ctx = av1_get_skip_context(xd);
384 385 386 387
#if CONFIG_NEW_MULTISYMBOL
    FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
    aom_write_symbol(w, skip, ec_ctx->skip_cdfs[ctx], 2);
#else
Zoe Liu's avatar
Zoe Liu committed
388
    aom_write(w, skip, cm->fc->skip_probs[ctx]);
389
#endif
Jingning Han's avatar
Jingning Han committed
390 391 392 393
    return skip;
  }
}

394 395 396
static void write_is_inter(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                           int segment_id, aom_writer *w, const int is_inter) {
  if (!segfeature_active(&cm->seg, segment_id, SEG_LVL_REF_FRAME)) {
397
    const int ctx = av1_get_intra_inter_context(xd);
398 399 400 401
#if CONFIG_NEW_MULTISYMBOL
    FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
    aom_write_symbol(w, is_inter, ec_ctx->intra_inter_cdf[ctx], 2);
#else
402
    aom_write(w, is_inter, cm->fc->intra_inter_prob[ctx]);
403 404 405 406
#endif
  }
}

Yue Chen's avatar
Yue Chen committed
407
#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
408 409
static void write_motion_mode(const AV1_COMMON *cm, MACROBLOCKD *xd,
                              const MODE_INFO *mi, aom_writer *w) {
410
  const MB_MODE_INFO *mbmi = &mi->mbmi;
411

412 413 414 415 416 417
#if !CONFIG_GLOBAL_MOTION
  // The cm parameter is only used with global_motion or with
  // motion_var and warped_motion. In other cases, explicitly ignore
  // it to avoid a compiler warning.
  (void)cm;
#endif
418
  MOTION_MODE last_motion_mode_allowed = motion_mode_allowed(
419
#if CONFIG_GLOBAL_MOTION
420
      0, cm->global_motion,
421
#endif  // CONFIG_GLOBAL_MOTION
422 423 424
#if CONFIG_WARPED_MOTION
      xd,
#endif
425
      mi);
Yue Chen's avatar
Yue Chen committed
426 427
  if (last_motion_mode_allowed == SIMPLE_TRANSLATION) return;
#if CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION
428 429 430 431 432 433 434 435 436 437
#if CONFIG_NCOBMC_ADAPT_WEIGHT
  if (last_motion_mode_allowed == NCOBMC_ADAPT_WEIGHT) {
    aom_write_symbol(w, mbmi->motion_mode,
                     xd->tile_ctx->ncobmc_cdf[mbmi->sb_type],
                     OBMC_FAMILY_MODES);
  } else if (last_motion_mode_allowed == OBMC_CAUSAL) {
    aom_write_symbol(w, mbmi->motion_mode == OBMC_CAUSAL,
                     xd->tile_ctx->obmc_cdf[mbmi->sb_type], 2);
  } else {
#else
Yue Chen's avatar
Yue Chen committed
438
  if (last_motion_mode_allowed == OBMC_CAUSAL) {
439 440 441 442
#if CONFIG_NEW_MULTISYMBOL
    aom_write_symbol(w, mbmi->motion_mode == OBMC_CAUSAL,
                     xd->tile_ctx->obmc_cdf[mbmi->sb_type], 2);
#else
Yue Chen's avatar
Yue Chen committed
443 444
    aom_write(w, mbmi->motion_mode == OBMC_CAUSAL,
              cm->fc->obmc_prob[mbmi->sb_type]);
445
#endif
Yue Chen's avatar
Yue Chen committed
446
  } else {
447
#endif  // CONFIG_NCOBMC_ADAPT_WEIGHT
Yue Chen's avatar
Yue Chen committed
448
#endif  // CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION
Thomas Davies's avatar
Thomas Davies committed
449 450 451
    aom_write_symbol(w, mbmi->motion_mode,
                     xd->tile_ctx->motion_mode_cdf[mbmi->sb_type],
                     MOTION_MODES);
Yue Chen's avatar
Yue Chen committed
452 453 454 455
#if CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION
  }
#endif  // CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION
}
456 457

#if CONFIG_NCOBMC_ADAPT_WEIGHT
458
static void write_ncobmc_mode(MACROBLOCKD *xd, const MODE_INFO *mi,
459 460 461
                              aom_writer *w) {
  const MB_MODE_INFO *mbmi = &mi->mbmi;
  ADAPT_OVERLAP_BLOCK ao_block = adapt_overlap_block_lookup[mbmi->sb_type];
462
  if (mbmi->motion_mode != NCOBMC_ADAPT_WEIGHT) return;
463

464 465
  aom_write_symbol(w, mbmi->ncobmc_mode[0],
                   xd->tile_ctx->ncobmc_mode_cdf[ao_block], MAX_NCOBMC_MODES);
466
  if (mi_size_wide[mbmi->sb_type] != mi_size_high[mbmi->sb_type]) {
467 468
    aom_write_symbol(w, mbmi->ncobmc_mode[1],
                     xd->tile_ctx->ncobmc_mode_cdf[ao_block], MAX_NCOBMC_MODES);
469 470 471
  }
}
#endif
Yue Chen's avatar
Yue Chen committed
472 473
#endif  // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION

Thomas Davies's avatar
Thomas Davies committed
474 475
static void write_delta_qindex(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                               int delta_qindex, aom_writer *w) {
476 477
  int sign = delta_qindex < 0;
  int abs = sign ? -delta_qindex : delta_qindex;
Thomas Davies's avatar
Thomas Davies committed
478
  int rem_bits, thr;
479
  int smallval = abs < DELTA_Q_SMALL ? 1 : 0;
Thomas Davies's avatar
Thomas Davies committed
480 481
  FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
  (void)cm;
482

Thomas Davies's avatar
Thomas Davies committed
483 484
  aom_write_symbol(w, AOMMIN(abs, DELTA_Q_SMALL), ec_ctx->delta_q_cdf,
                   DELTA_Q_PROBS + 1);
485 486 487 488

  if (!smallval) {
    rem_bits = OD_ILOG_NZ(abs - 1) - 1;
    thr = (1 << rem_bits) + 1;
489
    aom_write_literal(w, rem_bits - 1, 3);
490
    aom_write_literal(w, abs - thr, rem_bits);
491 492 493 494 495
  }
  if (abs > 0) {
    aom_write_bit(w, sign);
  }
}
496

Fangwen Fu's avatar
Fangwen Fu committed
497 498
#if CONFIG_EXT_DELTA_Q
static void write_delta_lflevel(const AV1_COMMON *cm, const MACROBLOCKD *xd,
499 500 501
#if CONFIG_LOOPFILTER_LEVEL
                                int lf_id,
#endif
Fangwen Fu's avatar
Fangwen Fu committed
502 503 504 505 506 507 508 509
                                int delta_lflevel, aom_writer *w) {
  int sign = delta_lflevel < 0;
  int abs = sign ? -delta_lflevel : delta_lflevel;
  int rem_bits, thr;
  int smallval = abs < DELTA_LF_SMALL ? 1 : 0;
  FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
  (void)cm;

510
#if CONFIG_LOOPFILTER_LEVEL
511 512 513 514 515 516 517 518
  if (cm->delta_lf_multi) {
    assert(lf_id >= 0 && lf_id < FRAME_LF_COUNT);
    aom_write_symbol(w, AOMMIN(abs, DELTA_LF_SMALL),
                     ec_ctx->delta_lf_multi_cdf[lf_id], DELTA_LF_PROBS + 1);
  } else {
    aom_write_symbol(w, AOMMIN(abs, DELTA_LF_SMALL), ec_ctx->delta_lf_cdf,
                     DELTA_LF_PROBS + 1);
  }
519
#else
Fangwen Fu's avatar
Fangwen Fu committed
520 521
  aom_write_symbol(w, AOMMIN(abs, DELTA_LF_SMALL), ec_ctx->delta_lf_cdf,
                   DELTA_LF_PROBS + 1);
522
#endif  // CONFIG_LOOPFILTER_LEVEL
Fangwen Fu's avatar
Fangwen Fu committed
523 524 525 526

  if (!smallval) {
    rem_bits = OD_ILOG_NZ(abs - 1) - 1;
    thr = (1 << rem_bits) + 1;
527
    aom_write_literal(w, rem_bits - 1, 3);
Fangwen Fu's avatar
Fangwen Fu committed
528 529 530 531 532 533 534
    aom_write_literal(w, abs - thr, rem_bits);
  }
  if (abs > 0) {
    aom_write_bit(w, sign);
  }
}
#endif  // CONFIG_EXT_DELTA_Q
535

536
#if !CONFIG_NEW_MULTISYMBOL
Yaowu Xu's avatar
Yaowu Xu committed
537
static void update_skip_probs(AV1_COMMON *cm, aom_writer *w,
Jingning Han's avatar
Jingning Han committed
538 539
                              FRAME_COUNTS *counts) {
  int k;
540 541 542 543 544
  const int probwt = cm->num_tg;
  for (k = 0; k < SKIP_CONTEXTS; ++k) {
    av1_cond_prob_diff_update(w, &cm->fc->skip_probs[k], counts->skip[k],
                              probwt);
  }
Jingning Han's avatar
Jingning Han committed
545
}
546
#endif
Jingning Han's avatar
Jingning Han committed
547

Sarah Parker's avatar
Sarah Parker committed
548 549
static void pack_map_tokens(aom_writer *w, const TOKENEXTRA **tp, int n,
                            int num) {
550
  const TOKENEXTRA *p = *tp;
551 552 553 554
  write_uniform(w, n, p->token);  // The first color index.
  ++p;
  --num;
  for (int i = 0; i < num; ++i) {
555
    aom_write_symbol(w, p->token, p->color_map_cdf, n);
hui su's avatar
hui su committed
556 557 558 559
    ++p;
  }
  *tp = p;
}
560

561
#if !CONFIG_LV_MAP
562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587
#if CONFIG_NEW_MULTISYMBOL
static INLINE void write_coeff_extra(const aom_cdf_prob *const *cdf, int val,
                                     int n, aom_writer *w) {
  // Code the extra bits from LSB to MSB in groups of 4
  int i = 0;
  int count = 0;
  while (count < n) {
    const int size = AOMMIN(n - count, 4);
    const int mask = (1 << size) - 1;
    aom_write_cdf(w, val & mask, cdf[i++], 1 << size);
    val >>= size;
    count += size;
  }
}
#else
static INLINE void write_coeff_extra(const aom_prob *pb, int value,
                                     int num_bits, int skip_bits, aom_writer *w,
                                     TOKEN_STATS *token_stats) {
  // Code the extra bits from MSB to LSB 1 bit at a time
  int index;
  for (index = skip_bits; index < num_bits; ++index) {
    const int shift = num_bits - index - 1;
    const int bb = (value >> shift) & 1;
    aom_write_record(w, bb, pb[index], token_stats);
  }
}
588
#endif  // CONFIG_NEW_MULTISYMBOL
589

Yaowu Xu's avatar
Yaowu Xu committed
590
static void pack_mb_tokens(aom_writer *w, const TOKENEXTRA **tp,
591
                           const TOKENEXTRA *const stop,
592
                           aom_bit_depth_t bit_depth, const TX_SIZE tx_size,
Sarah Parker's avatar
Sarah Parker committed
593 594 595
#if CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
                           TX_TYPE tx_type, int is_inter,
#endif  // CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
596
                           TOKEN_STATS *token_stats) {
597
  const TOKENEXTRA *p = *tp;
598
  int count = 0;
599
  const int seg_eob = tx_size_2d[tx_size];
Jingning Han's avatar
Jingning Han committed
600

Sarah Parker's avatar
Sarah Parker committed
601 602 603 604 605 606 607 608 609 610 611
#if CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
  if (tx_type == MRC_DCT && ((is_inter && SIGNAL_MRC_MASK_INTER) ||
                             (!is_inter && SIGNAL_MRC_MASK_INTRA))) {
    int rows = tx_size_high[tx_size];
    int cols = tx_size_wide[tx_size];
    assert(tx_size == TX_32X32);
    assert(p < stop);
    pack_map_tokens(w, &p, 2, rows * cols);
  }
#endif  // CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK

Jingning Han's avatar
Jingning Han committed
612
  while (p < stop && p->token != EOSB_TOKEN) {
613
    const int token = p->token;
Yaowu Xu's avatar
Yaowu Xu committed
614
    const int8_t eob_val = p->eob_val;
615
    if (token == BLOCK_Z_TOKEN) {
616
      aom_write_symbol(w, 0, *p->head_cdf, HEAD_TOKENS + 1);
617
      p++;
618
      break;
619 620
      continue;
    }
Yaowu Xu's avatar
Yaowu Xu committed
621 622

    const av1_extra_bit *const extra_bits = &av1_extra_bits[token];
623
    if (eob_val == LAST_EOB) {
624 625 626
      // Just code a flag indicating whether the value is >1 or 1.
      aom_write_bit(w, token != ONE_TOKEN);
    } else {
627
      int comb_symb = 2 * AOMMIN(token, TWO_TOKEN) - eob_val + p->first_val;
628
      aom_write_symbol(w, comb_symb, *p->head_cdf, HEAD_TOKENS + p->first_val);
629
    }
630
    if (token > ONE_TOKEN) {
631
      aom_write_symbol(w, token - TWO_TOKEN, *p->tail_cdf, TAIL_TOKENS);
Alex Converse's avatar
Alex Converse committed
632
    }
633 634 635 636

    if (extra_bits->base_val) {
      const int bit_string = p->extra;
      const int bit_string_length = extra_bits->len;  // Length of extra bits to
637
      const int is_cat6 = (extra_bits->base_val == CAT6_MIN_VAL);
638 639
      // be written excluding
      // the sign bit.
640
      int skip_bits = is_cat6
641 642 643
                          ? (int)sizeof(av1_cat6_prob) -
                                av1_get_cat6_extrabits_size(tx_size, bit_depth)
                          : 0;
644

645 646 647 648 649 650 651 652 653
      assert(!(bit_string >> (bit_string_length - skip_bits + 1)));
      if (bit_string_length > 0)
#if CONFIG_NEW_MULTISYMBOL
        write_coeff_extra(extra_bits->cdf, bit_string >> 1,
                          bit_string_length - skip_bits, w);
#else
        write_coeff_extra(extra_bits->prob, bit_string >> 1, bit_string_length,
                          skip_bits, w, token_stats);
#endif
654 655 656 657 658 659

      aom_write_bit_record(w, bit_string & 1, token_stats);
    }
    ++p;

    ++count;
660
    if (eob_val == EARLY_EOB || count == seg_eob) break;
661 662 663 664
  }

  *tp = p;
}
Angie Chiang's avatar
Angie Chiang committed
665
#endif  // !CONFIG_LV_MAP
666

667
#if !CONFIG_COEF_INTERLEAVE
668 669 670 671 672 673 674
#if CONFIG_LV_MAP
static void pack_txb_tokens(aom_writer *w,
#if CONFIG_LV_MAP
                            AV1_COMMON *cm,
#endif  // CONFIG_LV_MAP
                            const TOKENEXTRA **tp,
                            const TOKENEXTRA *const tok_end,
Yushin Cho's avatar
Yushin Cho committed
675
#if CONFIG_LV_MAP
676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703
                            MACROBLOCK *const x,
#endif
                            MACROBLOCKD *xd, MB_MODE_INFO *mbmi, int plane,
                            BLOCK_SIZE plane_bsize, aom_bit_depth_t bit_depth,
                            int block, int blk_row, int blk_col,
                            TX_SIZE tx_size, TOKEN_STATS *token_stats) {
  const struct macroblockd_plane *const pd = &xd->plane[plane];
  const BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
  const int tx_row = blk_row >> (1 - pd->subsampling_y);
  const int tx_col = blk_col >> (1 - pd->subsampling_x);
  TX_SIZE plane_tx_size;
  const int max_blocks_high = max_block_high(xd, plane_bsize, plane);
  const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane);

  if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;

  plane_tx_size =
      plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0]
            : mbmi->inter_tx_size[tx_row][tx_col];

  if (tx_size == plane_tx_size) {
    TOKEN_STATS tmp_token_stats;
    init_token_stats(&tmp_token_stats);

    tran_low_t *tcoeff = BLOCK_OFFSET(x->mbmi_ext->tcoeff[plane], block);
    uint16_t eob = x->mbmi_ext->eobs[plane][block];
    TXB_CTX txb_ctx = { x->mbmi_ext->txb_skip_ctx[plane][block],
                        x->mbmi_ext->dc_sign_ctx[plane][block] };
Jingning Han's avatar
Jingning Han committed
704 705
    av1_write_coeffs_txb(cm, xd, w, blk_row, blk_col, block, plane, tx_size,
                         tcoeff, eob, &txb_ctx);
706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728
#if CONFIG_RD_DEBUG
    token_stats->txb_coeff_cost_map[blk_row][blk_col] = tmp_token_stats.cost;
    token_stats->cost += tmp_token_stats.cost;
#endif
  } else {
    const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
    const int bsl = tx_size_wide_unit[sub_txs];
    int i;

    assert(bsl > 0);

    for (i = 0; i < 4; ++i) {
      const int offsetr = blk_row + (i >> 1) * bsl;
      const int offsetc = blk_col + (i & 0x01) * bsl;
      const int step = tx_size_wide_unit[sub_txs] * tx_size_high_unit[sub_txs];

      if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;

      pack_txb_tokens(w,
#if CONFIG_LV_MAP
                      cm,
#endif
                      tp, tok_end,
Yushin Cho's avatar
Yushin Cho committed
729
#if CONFIG_LV_MAP
730 731 732 733 734 735 736 737 738
                      x,
#endif
                      xd, mbmi, plane, plane_bsize, bit_depth, block, offsetr,
                      offsetc, sub_txs, token_stats);
      block += step;
    }
  }
}
#else  // CONFIG_LV_MAP
Yaowu Xu's avatar
Yaowu Xu committed
739
static void pack_txb_tokens(aom_writer *w, const TOKENEXTRA **tp,
Yushin Cho's avatar
Yushin Cho committed
740 741
                            const TOKENEXTRA *const tok_end, MACROBLOCKD *xd,
                            MB_MODE_INFO *mbmi, int plane,
Yaowu Xu's avatar
Yaowu Xu committed
742
                            BLOCK_SIZE plane_bsize, aom_bit_depth_t bit_depth,
743
                            int block, int blk_row, int blk_col,
744
                            TX_SIZE tx_size, TOKEN_STATS *token_stats) {
745 746
  const struct macroblockd_plane *const pd = &xd->plane[plane];
  const BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
747 748
  const int tx_row = blk_row >> (1 - pd->subsampling_y);
  const int tx_col = blk_col >> (1 - pd->subsampling_x);
Debargha Mukherjee's avatar
Debargha Mukherjee committed
749
  TX_SIZE plane_tx_size;
750 751
  const int max_blocks_high = max_block_high(xd, plane_bsize, plane);
  const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane);
Sarah Parker's avatar
Sarah Parker committed
752 753 754 755
#if CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
  TX_TYPE tx_type = av1_get_tx_type(plane ? PLANE_TYPE_UV : PLANE_TYPE_Y, xd,
                                    blk_row, blk_col, block, tx_size);
#endif  // CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
756

757
  if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
758

759 760 761
  plane_tx_size =
      plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0]
            : mbmi->inter_tx_size[tx_row][tx_col];
Debargha Mukherjee's avatar
Debargha Mukherjee committed
762

763
  if (tx_size == plane_tx_size) {
764 765
    TOKEN_STATS tmp_token_stats;
    init_token_stats(&tmp_token_stats);
Sarah Parker's avatar
Sarah Parker committed
766 767 768 769 770
    pack_mb_tokens(w, tp, tok_end, bit_depth, tx_size,
#if CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
                   tx_type, is_inter_block(mbmi),
#endif  // CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
                   &tmp_token_stats);
771 772 773 774
#if CONFIG_RD_DEBUG
    token_stats->txb_coeff_cost_map[blk_row][blk_col] = tmp_token_stats.cost;
    token_stats->cost += tmp_token_stats.cost;
#endif
775
  } else {
Yue Chen's avatar
Yue Chen committed
776 777 778 779
#if CONFIG_RECT_TX_EXT
    int is_qttx = plane_tx_size == quarter_txsize_lookup[plane_bsize];
    const TX_SIZE sub_txs = is_qttx ? plane_tx_size : sub_tx_size_map[tx_size];
#else
780
    const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
Yue Chen's avatar
Yue Chen committed
781
#endif
782
    const int bsl = tx_size_wide_unit[sub_txs];
783 784 785 786 787
    int i;

    assert(bsl > 0);

    for (i = 0; i < 4; ++i) {
Yue Chen's avatar
Yue Chen committed
788 789 790 791 792 793 794 795 796
#if CONFIG_RECT_TX_EXT
      int is_wide_tx = tx_size_wide_unit[sub_txs] > tx_size_high_unit[sub_txs];
      const int offsetr =
          is_qttx ? (is_wide_tx ? i * tx_size_high_unit[sub_txs] : 0)
                  : blk_row + (i >> 1) * bsl;
      const int offsetc =
          is_qttx ? (is_wide_tx ? 0 : i * tx_size_wide_unit[sub_txs])
                  : blk_col + (i & 0x01) * bsl;
#else
797 798
      const int offsetr = blk_row + (i >> 1) * bsl;
      const int offsetc = blk_col + (i & 0x01) * bsl;
Yue Chen's avatar
Yue Chen committed
799
#endif
800
      const int step = tx_size_wide_unit[sub_txs] * tx_size_high_unit[sub_txs];
801

802
      if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;
803

Yushin Cho's avatar
Yushin Cho committed
804 805
      pack_txb_tokens(w, tp, tok_end, xd, mbmi, plane, plane_bsize, bit_depth,
                      block, offsetr, offsetc, sub_txs, token_stats);
806
      block += step;
807 808 809
    }
  }
}
810
#endif  // CONFIG_LV_MAP
811
#endif
812

Yaowu Xu's avatar
Yaowu Xu committed
813
static void write_segment_id(aom_writer *w, const struct segmentation *seg,
814
                             struct segmentation_probs *segp, int segment_id) {
815 816 817
  if (seg->enabled && seg->update_map) {
    aom_write_symbol(w, segment_id, segp->tree_cdf, MAX_SEGMENTS);
  }
Jingning Han's avatar
Jingning Han committed
818 819
}

820 821
#if CONFIG_NEW_MULTISYMBOL
#define WRITE_REF_BIT(bname, pname) \
822
  aom_write_symbol(w, bname, av1_get_pred_cdf_##pname(cm, xd), 2)
823 824
#define WRITE_REF_BIT2(bname, pname) \
  aom_write_symbol(w, bname, av1_get_pred_cdf_##pname(xd), 2)
825 826 827
#else
#define WRITE_REF_BIT(bname, pname) \
  aom_write(w, bname, av1_get_pred_prob_##pname(cm, xd))
828 829
#define WRITE_REF_BIT2(bname, pname) \
  aom_write(w, bname, av1_get_pred_prob_##pname(cm, xd))
830 831
#endif

Jingning Han's avatar
Jingning Han committed
832
// This function encodes the reference frame
Yaowu Xu's avatar
Yaowu Xu committed
833 834
static void write_ref_frames(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                             aom_writer *w) {
Jingning Han's avatar
Jingning Han committed
835 836 837 838 839 840 841 842 843
  const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
  const int is_compound = has_second_ref(mbmi);
  const int segment_id = mbmi->segment_id;

  // If segment level coding of this signal is disabled...
  // or the segment allows multiple reference frame options
  if (segfeature_active(&cm->seg, segment_id, SEG_LVL_REF_FRAME)) {
    assert(!is_compound);
    assert(mbmi->ref_frame[0] ==
844
           get_segdata(&cm->seg, segment_id, SEG_LVL_REF_FRAME));
845 846 847 848 849 850 851 852 853 854
  }
#if CONFIG_SEGMENT_ZEROMV
  else if (segfeature_active(&cm->seg, segment_id, SEG_LVL_SKIP) ||
           segfeature_active(&cm->seg, segment_id, SEG_LVL_ZEROMV))
#else
  else if (segfeature_active(&cm->seg, segment_id, SEG_LVL_SKIP))
#endif
  {
    assert(!is_compound);
    assert(mbmi->ref_frame[0] == LAST_FRAME);
Jingning Han's avatar
Jingning Han committed
855 856 857 858
  } else {
    // does the feature use compound prediction or not
    // (if not specified at the frame/segment level)
    if (cm->reference_mode == REFERENCE_MODE_SELECT) {
859
      if (is_comp_ref_allowed(mbmi->sb_type))
860 861 862
#if CONFIG_NEW_MULTISYMBOL
        aom_write_symbol(w, is_compound, av1_get_reference_mode_cdf(cm, xd), 2);
#else
863 864
        aom_write(w, is_compound, av1_get_reference_mode_prob(cm, xd));
#endif  // CONFIG_NEW_MULTISYMBOL
Jingning Han's avatar
Jingning Han committed
865
    } else {
866
      assert((!is_compound) == (cm->reference_mode == SINGLE_REFERENCE));
Jingning Han's avatar
Jingning Han committed
867 868 869
    }

    if (is_compound) {
Zoe Liu's avatar
Zoe Liu committed
870 871 872 873 874
#if CONFIG_EXT_COMP_REFS
      const COMP_REFERENCE_TYPE comp_ref_type = has_uni_comp_refs(mbmi)
                                                    ? UNIDIR_COMP_REFERENCE
                                                    : BIDIR_COMP_REFERENCE;
#if USE_UNI_COMP_REFS
875 876 877 878
#if CONFIG_VAR_REFS
      if ((L_OR_L2(cm) || L3_OR_G(cm)) && BWD_OR_ALT(cm))
        if (L_AND_L2(cm) || L_AND_L3(cm) || L_AND_G(cm) || BWD_AND_ALT(cm))
#endif  // CONFIG_VAR_REFS
879 880 881 882 883 884
#if CONFIG_NEW_MULTISYMBOL
          aom_write_symbol(w, comp_ref_type,
                           av1_get_comp_reference_type_cdf(xd), 2);
#else
      aom_write(w, comp_ref_type, av1_get_comp_reference_type_prob(cm, xd));
#endif
885 886 887 888 889 890
#if CONFIG_VAR_REFS
        else
          assert(comp_ref_type == BIDIR_COMP_REFERENCE);
      else
        assert(comp_ref_type == UNIDIR_COMP_REFERENCE);
#endif  // CONFIG_VAR_REFS
Zoe Liu's avatar
Zoe Liu committed
891 892 893 894 895 896 897
#else   // !USE_UNI_COMP_REFS
      // NOTE: uni-directional comp refs disabled
      assert(comp_ref_type == BIDIR_COMP_REFERENCE);
#endif  // USE_UNI_COMP_REFS

      if (comp_ref_type == UNIDIR_COMP_REFERENCE) {
        const int bit = mbmi->ref_frame[0] == BWDREF_FRAME;