bitstream.c 137 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4 5 6 7 8 9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10 11 12 13
 */

#include <assert.h>
#include <limits.h>
14
#include <stdio.h>
Jingning Han's avatar
Jingning Han committed
15

Yaowu Xu's avatar
Yaowu Xu committed
16
#include "aom/aom_encoder.h"
17
#include "aom_dsp/bitwriter_buffer.h"
Yaowu Xu's avatar
Yaowu Xu committed
18 19
#include "aom_dsp/aom_dsp_common.h"
#include "aom_mem/aom_mem.h"
20 21
#include "aom_ports/mem_ops.h"
#include "aom_ports/system_state.h"
22
#if CONFIG_BITSTREAM_DEBUG
23
#include "aom_util/debug_util.h"
24
#endif  // CONFIG_BITSTREAM_DEBUG
Jingning Han's avatar
Jingning Han committed
25

26
#if CONFIG_CLPF
27
#include "av1/common/clpf.h"
28
#endif
Yaowu Xu's avatar
Yaowu Xu committed
29
#if CONFIG_DERING
30
#include "av1/common/dering.h"
Yaowu Xu's avatar
Yaowu Xu committed
31
#endif  // CONFIG_DERING
32 33 34 35 36 37 38 39
#include "av1/common/entropy.h"
#include "av1/common/entropymode.h"
#include "av1/common/entropymv.h"
#include "av1/common/mvref_common.h"
#include "av1/common/pred_common.h"
#include "av1/common/reconinter.h"
#include "av1/common/seg_common.h"
#include "av1/common/tile_common.h"
Jingning Han's avatar
Jingning Han committed
40

Alex Converse's avatar
Alex Converse committed
41
#if CONFIG_ANS
Alex Converse's avatar
Alex Converse committed
42
#include "aom_dsp/buf_ans.h"
Alex Converse's avatar
Alex Converse committed
43
#endif  // CONFIG_ANS
44 45 46 47 48 49 50
#include "av1/encoder/bitstream.h"
#include "av1/encoder/cost.h"
#include "av1/encoder/encodemv.h"
#include "av1/encoder/mcomp.h"
#include "av1/encoder/segmentation.h"
#include "av1/encoder/subexp.h"
#include "av1/encoder/tokenize.h"
Jingning Han's avatar
Jingning Han committed
51

Yaowu Xu's avatar
Yaowu Xu committed
52
static const struct av1_token intra_mode_encodings[INTRA_MODES] = {
53 54 55
  { 0, 1 },  { 6, 3 },   { 28, 5 },  { 30, 5 }, { 58, 6 },
  { 59, 6 }, { 126, 7 }, { 127, 7 }, { 62, 6 }, { 2, 2 }
};
Jingning Han's avatar
Jingning Han committed
56
#if CONFIG_EXT_INTERP
Yaowu Xu's avatar
Yaowu Xu committed
57
static const struct av1_token switchable_interp_encodings[SWITCHABLE_FILTERS] =
58
    { { 0, 1 }, { 4, 3 }, { 6, 3 }, { 5, 3 }, { 7, 3 } };
59
#else
Yaowu Xu's avatar
Yaowu Xu committed
60
static const struct av1_token switchable_interp_encodings[SWITCHABLE_FILTERS] =
61
    { { 0, 1 }, { 2, 2 }, { 3, 2 } };
Jingning Han's avatar
Jingning Han committed
62
#endif  // CONFIG_EXT_INTERP
63
#if CONFIG_EXT_PARTITION_TYPES
Yaowu Xu's avatar
Yaowu Xu committed
64
static const struct av1_token ext_partition_encodings[EXT_PARTITION_TYPES] = {
65 66 67
  { 0, 1 },  { 4, 3 },  { 12, 4 }, { 7, 3 },
  { 10, 4 }, { 11, 4 }, { 26, 5 }, { 27, 5 }
};
68
#endif
Yaowu Xu's avatar
Yaowu Xu committed
69
static const struct av1_token partition_encodings[PARTITION_TYPES] = {
70 71
  { 0, 1 }, { 2, 2 }, { 6, 3 }, { 7, 3 }
};
72
#if !CONFIG_REF_MV
Yaowu Xu's avatar
Yaowu Xu committed
73
static const struct av1_token inter_mode_encodings[INTER_MODES] =
Yue Chen's avatar
Yue Chen committed
74
#if CONFIG_EXT_INTER
75
    { { 2, 2 }, { 6, 3 }, { 0, 1 }, { 14, 4 }, { 15, 4 } };
Yue Chen's avatar
Yue Chen committed
76
#else
77
    { { 2, 2 }, { 6, 3 }, { 0, 1 }, { 7, 3 } };
Yue Chen's avatar
Yue Chen committed
78
#endif  // CONFIG_EXT_INTER
79
#endif
80
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
81
static const struct av1_token
82 83 84 85
    inter_compound_mode_encodings[INTER_COMPOUND_MODES] = {
      { 2, 2 },  { 50, 6 }, { 51, 6 }, { 24, 5 }, { 52, 6 },
      { 53, 6 }, { 54, 6 }, { 55, 6 }, { 0, 1 },  { 7, 3 }
    };
86
#endif  // CONFIG_EXT_INTER
87
#if CONFIG_PALETTE
Yaowu Xu's avatar
Yaowu Xu committed
88
static const struct av1_token palette_size_encodings[] = {
89
  { 0, 1 }, { 2, 2 }, { 6, 3 }, { 14, 4 }, { 30, 5 }, { 62, 6 }, { 63, 6 },
hui su's avatar
hui su committed
90
};
Yaowu Xu's avatar
Yaowu Xu committed
91
static const struct av1_token
92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118
    palette_color_encodings[PALETTE_MAX_SIZE - 1][PALETTE_MAX_SIZE] = {
      { { 0, 1 }, { 1, 1 } },                                  // 2 colors
      { { 0, 1 }, { 2, 2 }, { 3, 2 } },                        // 3 colors
      { { 0, 1 }, { 2, 2 }, { 6, 3 }, { 7, 3 } },              // 4 colors
      { { 0, 1 }, { 2, 2 }, { 6, 3 }, { 14, 4 }, { 15, 4 } },  // 5 colors
      { { 0, 1 },
        { 2, 2 },
        { 6, 3 },
        { 14, 4 },
        { 30, 5 },
        { 31, 5 } },  // 6 colors
      { { 0, 1 },
        { 2, 2 },
        { 6, 3 },
        { 14, 4 },
        { 30, 5 },
        { 62, 6 },
        { 63, 6 } },  // 7 colors
      { { 0, 1 },
        { 2, 2 },
        { 6, 3 },
        { 14, 4 },
        { 30, 5 },
        { 62, 6 },
        { 126, 7 },
        { 127, 7 } },  // 8 colors
    };
119
#endif  // CONFIG_PALETTE
120

Yaowu Xu's avatar
Yaowu Xu committed
121
static const struct av1_token tx_size_encodings[TX_SIZES - 1][TX_SIZES] = {
122 123 124
  { { 0, 1 }, { 1, 1 } },                      // Max tx_size is 8X8
  { { 0, 1 }, { 2, 2 }, { 3, 2 } },            // Max tx_size is 16X16
  { { 0, 1 }, { 2, 2 }, { 6, 3 }, { 7, 3 } },  // Max tx_size is 32X32
125 126
};

127
#if CONFIG_EXT_INTRA || CONFIG_PALETTE
Yaowu Xu's avatar
Yaowu Xu committed
128
static INLINE void write_uniform(aom_writer *w, int n, int v) {
hui su's avatar
hui su committed
129 130
  int l = get_unsigned_bits(n);
  int m = (1 << l) - n;
131
  if (l == 0) return;
hui su's avatar
hui su committed
132
  if (v < m) {
Yaowu Xu's avatar
Yaowu Xu committed
133
    aom_write_literal(w, v, l - 1);
hui su's avatar
hui su committed
134
  } else {
Yaowu Xu's avatar
Yaowu Xu committed
135 136
    aom_write_literal(w, m + ((v - m) >> 1), l - 1);
    aom_write_literal(w, (v - m) & 1, 1);
hui su's avatar
hui su committed
137 138
  }
}
139
#endif  // CONFIG_EXT_INTRA || CONFIG_PALETTE
Jingning Han's avatar
Jingning Han committed
140

141
#if CONFIG_EXT_TX
Yaowu Xu's avatar
Yaowu Xu committed
142 143
static struct av1_token ext_tx_inter_encodings[EXT_TX_SETS_INTER][TX_TYPES];
static struct av1_token ext_tx_intra_encodings[EXT_TX_SETS_INTRA][TX_TYPES];
144
#else
Yaowu Xu's avatar
Yaowu Xu committed
145
static struct av1_token ext_tx_encodings[TX_TYPES];
146
#endif  // CONFIG_EXT_TX
147
#if CONFIG_GLOBAL_MOTION
Yaowu Xu's avatar
Yaowu Xu committed
148
static struct av1_token global_motion_types_encodings[GLOBAL_MOTION_TYPES];
149
#endif  // CONFIG_GLOBAL_MOTION
150
#if CONFIG_EXT_INTRA
Yaowu Xu's avatar
Yaowu Xu committed
151
static struct av1_token intra_filter_encodings[INTRA_FILTERS];
152
#endif  // CONFIG_EXT_INTRA
153
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
154
static struct av1_token interintra_mode_encodings[INTERINTRA_MODES];
155
#endif  // CONFIG_EXT_INTER
Yue Chen's avatar
Yue Chen committed
156 157 158
#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
static struct av1_token motion_mode_encodings[MOTION_MODES];
#endif  // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
159 160 161
#if CONFIG_LOOP_RESTORATION
static struct av1_token switchable_restore_encodings[RESTORE_SWITCHABLE_TYPES];
#endif  // CONFIG_LOOP_RESTORATION
162

Yaowu Xu's avatar
Yaowu Xu committed
163
void av1_encode_token_init(void) {
164
#if CONFIG_EXT_TX
165 166
  int s;
  for (s = 1; s < EXT_TX_SETS_INTER; ++s) {
Yaowu Xu's avatar
Yaowu Xu committed
167
    av1_tokens_from_tree(ext_tx_inter_encodings[s], av1_ext_tx_inter_tree[s]);
168 169
  }
  for (s = 1; s < EXT_TX_SETS_INTRA; ++s) {
Yaowu Xu's avatar
Yaowu Xu committed
170
    av1_tokens_from_tree(ext_tx_intra_encodings[s], av1_ext_tx_intra_tree[s]);
171
  }
172
#else
Yaowu Xu's avatar
Yaowu Xu committed
173
  av1_tokens_from_tree(ext_tx_encodings, av1_ext_tx_tree);
174
#endif  // CONFIG_EXT_TX
175
#if CONFIG_EXT_INTRA
Yaowu Xu's avatar
Yaowu Xu committed
176
  av1_tokens_from_tree(intra_filter_encodings, av1_intra_filter_tree);
177
#endif  // CONFIG_EXT_INTRA
178
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
179
  av1_tokens_from_tree(interintra_mode_encodings, av1_interintra_mode_tree);
180
#endif  // CONFIG_EXT_INTER
Yue Chen's avatar
Yue Chen committed
181 182 183
#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
  av1_tokens_from_tree(motion_mode_encodings, av1_motion_mode_tree);
#endif  // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
184
#if CONFIG_GLOBAL_MOTION
Yaowu Xu's avatar
Yaowu Xu committed
185 186
  av1_tokens_from_tree(global_motion_types_encodings,
                       av1_global_motion_types_tree);
187
#endif  // CONFIG_GLOBAL_MOTION
188 189 190 191
#if CONFIG_LOOP_RESTORATION
  av1_tokens_from_tree(switchable_restore_encodings,
                       av1_switchable_restore_tree);
#endif  // CONFIG_LOOP_RESTORATION
192 193 194 195 196 197 198

#if CONFIG_DAALA_EC
  /* This hack is necessary when CONFIG_EXT_INTERP is enabled because the five
      SWITCHABLE_FILTERS are not consecutive, e.g., 0, 1, 2, 3, 4, when doing
      an in-order traversal of the av1_switchable_interp_tree structure. */
  av1_indices_from_tree(av1_switchable_interp_ind, av1_switchable_interp_inv,
                        SWITCHABLE_FILTERS, av1_switchable_interp_tree);
199 200 201 202 203
  /* This hack is necessary because the four TX_TYPES are not consecutive,
      e.g., 0, 1, 2, 3, when doing an in-order traversal of the av1_ext_tx_tree
      structure. */
  av1_indices_from_tree(av1_ext_tx_ind, av1_ext_tx_inv, TX_TYPES,
                        av1_ext_tx_tree);
204
#endif
205 206
}

Yaowu Xu's avatar
Yaowu Xu committed
207 208 209
static void write_intra_mode(aom_writer *w, PREDICTION_MODE mode,
                             const aom_prob *probs) {
  av1_write_token(w, av1_intra_mode_tree, probs, &intra_mode_encodings[mode]);
Jingning Han's avatar
Jingning Han committed
210 211
}

212
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
213 214 215 216
static void write_interintra_mode(aom_writer *w, INTERINTRA_MODE mode,
                                  const aom_prob *probs) {
  av1_write_token(w, av1_interintra_mode_tree, probs,
                  &interintra_mode_encodings[mode]);
217 218 219
}
#endif  // CONFIG_EXT_INTER

Yaowu Xu's avatar
Yaowu Xu committed
220
static void write_inter_mode(AV1_COMMON *cm, aom_writer *w,
221
                             PREDICTION_MODE mode,
Yue Chen's avatar
Yue Chen committed
222 223 224
#if CONFIG_REF_MV && CONFIG_EXT_INTER
                             int is_compound,
#endif  // CONFIG_REF_MV && CONFIG_EXT_INTER
225
                             const int16_t mode_ctx) {
226
#if CONFIG_REF_MV
227
  const int16_t newmv_ctx = mode_ctx & NEWMV_CTX_MASK;
Yaowu Xu's avatar
Yaowu Xu committed
228
  const aom_prob newmv_prob = cm->fc->newmv_prob[newmv_ctx];
Yue Chen's avatar
Yue Chen committed
229
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
230
  aom_write(w, mode != NEWMV && mode != NEWFROMNEARMV, newmv_prob);
Yue Chen's avatar
Yue Chen committed
231 232

  if (!is_compound && (mode == NEWMV || mode == NEWFROMNEARMV))
Yaowu Xu's avatar
Yaowu Xu committed
233
    aom_write(w, mode == NEWFROMNEARMV, cm->fc->new2mv_prob);
Yue Chen's avatar
Yue Chen committed
234 235 236

  if (mode != NEWMV && mode != NEWFROMNEARMV) {
#else
Yaowu Xu's avatar
Yaowu Xu committed
237
  aom_write(w, mode != NEWMV, newmv_prob);
238 239

  if (mode != NEWMV) {
Yue Chen's avatar
Yue Chen committed
240
#endif  // CONFIG_EXT_INTER
241
    const int16_t zeromv_ctx = (mode_ctx >> ZEROMV_OFFSET) & ZEROMV_CTX_MASK;
Yaowu Xu's avatar
Yaowu Xu committed
242
    const aom_prob zeromv_prob = cm->fc->zeromv_prob[zeromv_ctx];
243 244 245 246 247 248

    if (mode_ctx & (1 << ALL_ZERO_FLAG_OFFSET)) {
      assert(mode == ZEROMV);
      return;
    }

Yaowu Xu's avatar
Yaowu Xu committed
249
    aom_write(w, mode != ZEROMV, zeromv_prob);
250 251

    if (mode != ZEROMV) {
252
      int16_t refmv_ctx = (mode_ctx >> REFMV_OFFSET) & REFMV_CTX_MASK;
Yaowu Xu's avatar
Yaowu Xu committed
253
      aom_prob refmv_prob;
254

255 256 257
      if (mode_ctx & (1 << SKIP_NEARESTMV_OFFSET)) refmv_ctx = 6;
      if (mode_ctx & (1 << SKIP_NEARMV_OFFSET)) refmv_ctx = 7;
      if (mode_ctx & (1 << SKIP_NEARESTMV_SUB8X8_OFFSET)) refmv_ctx = 8;
258 259

      refmv_prob = cm->fc->refmv_prob[refmv_ctx];
Yaowu Xu's avatar
Yaowu Xu committed
260
      aom_write(w, mode != NEARESTMV, refmv_prob);
261 262 263
    }
  }
#else
Yaowu Xu's avatar
Yaowu Xu committed
264
  const aom_prob *const inter_probs = cm->fc->inter_mode_probs[mode_ctx];
Jingning Han's avatar
Jingning Han committed
265
  assert(is_inter_mode(mode));
Yaowu Xu's avatar
Yaowu Xu committed
266 267
  av1_write_token(w, av1_inter_mode_tree, inter_probs,
                  &inter_mode_encodings[INTER_OFFSET(mode)]);
268
#endif
Jingning Han's avatar
Jingning Han committed
269 270
}

271
#if CONFIG_REF_MV
Yaowu Xu's avatar
Yaowu Xu committed
272 273 274
static void write_drl_idx(const AV1_COMMON *cm, const MB_MODE_INFO *mbmi,
                          const MB_MODE_INFO_EXT *mbmi_ext, aom_writer *w) {
  uint8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame);
275 276 277

  assert(mbmi->ref_mv_idx < 3);

278 279 280 281 282
  if (mbmi->mode == NEWMV) {
    int idx;
    for (idx = 0; idx < 2; ++idx) {
      if (mbmi_ext->ref_mv_count[ref_frame_type] > idx + 1) {
        uint8_t drl_ctx =
Yaowu Xu's avatar
Yaowu Xu committed
283 284
            av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx);
        aom_prob drl_prob = cm->fc->drl_prob[drl_ctx];
285

Yaowu Xu's avatar
Yaowu Xu committed
286
        aom_write(w, mbmi->ref_mv_idx != idx, drl_prob);
287
        if (mbmi->ref_mv_idx == idx) return;
288
      }
289 290 291 292
    }
    return;
  }

293 294 295 296 297 298
  if (mbmi->mode == NEARMV) {
    int idx;
    // TODO(jingning): Temporary solution to compensate the NEARESTMV offset.
    for (idx = 1; idx < 3; ++idx) {
      if (mbmi_ext->ref_mv_count[ref_frame_type] > idx + 1) {
        uint8_t drl_ctx =
Yaowu Xu's avatar
Yaowu Xu committed
299 300
            av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx);
        aom_prob drl_prob = cm->fc->drl_prob[drl_ctx];
301

Yaowu Xu's avatar
Yaowu Xu committed
302
        aom_write(w, mbmi->ref_mv_idx != (idx - 1), drl_prob);
303
        if (mbmi->ref_mv_idx == (idx - 1)) return;
304
      }
305
    }
306
    return;
307 308 309 310
  }
}
#endif

311
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
312
static void write_inter_compound_mode(AV1_COMMON *cm, aom_writer *w,
313 314
                                      PREDICTION_MODE mode,
                                      const int16_t mode_ctx) {
Yaowu Xu's avatar
Yaowu Xu committed
315
  const aom_prob *const inter_compound_probs =
316
      cm->fc->inter_compound_mode_probs[mode_ctx];
317 318

  assert(is_inter_compound_mode(mode));
Yaowu Xu's avatar
Yaowu Xu committed
319 320
  av1_write_token(w, av1_inter_compound_mode_tree, inter_compound_probs,
                  &inter_compound_mode_encodings[INTER_COMPOUND_OFFSET(mode)]);
321 322 323
}
#endif  // CONFIG_EXT_INTER

Yaowu Xu's avatar
Yaowu Xu committed
324
static void encode_unsigned_max(struct aom_write_bit_buffer *wb, int data,
325
                                int max) {
Yaowu Xu's avatar
Yaowu Xu committed
326
  aom_wb_write_literal(wb, data, get_unsigned_bits(max));
Jingning Han's avatar
Jingning Han committed
327 328
}

Yaowu Xu's avatar
Yaowu Xu committed
329 330
static void prob_diff_update(const aom_tree_index *tree,
                             aom_prob probs[/*n - 1*/],
331
                             const unsigned int counts[/*n - 1*/], int n,
Yaowu Xu's avatar
Yaowu Xu committed
332
                             aom_writer *w) {
Jingning Han's avatar
Jingning Han committed
333 334 335 336 337 338
  int i;
  unsigned int branch_ct[32][2];

  // Assuming max number of probabilities <= 32
  assert(n <= 32);

Yaowu Xu's avatar
Yaowu Xu committed
339
  av1_tree_probs_from_distribution(tree, branch_ct, counts);
Jingning Han's avatar
Jingning Han committed
340
  for (i = 0; i < n - 1; ++i)
Yaowu Xu's avatar
Yaowu Xu committed
341
    av1_cond_prob_diff_update(w, &probs[i], branch_ct[i]);
Jingning Han's avatar
Jingning Han committed
342 343
}

Yaowu Xu's avatar
Yaowu Xu committed
344 345
static int prob_diff_update_savings(const aom_tree_index *tree,
                                    aom_prob probs[/*n - 1*/],
346 347 348 349 350 351 352 353
                                    const unsigned int counts[/*n - 1*/],
                                    int n) {
  int i;
  unsigned int branch_ct[32][2];
  int savings = 0;

  // Assuming max number of probabilities <= 32
  assert(n <= 32);
Yaowu Xu's avatar
Yaowu Xu committed
354
  av1_tree_probs_from_distribution(tree, branch_ct, counts);
355
  for (i = 0; i < n - 1; ++i) {
Yaowu Xu's avatar
Yaowu Xu committed
356
    savings += av1_cond_prob_diff_update_savings(&probs[i], branch_ct[i]);
357 358 359 360
  }
  return savings;
}

361
#if CONFIG_VAR_TX
Yaowu Xu's avatar
Yaowu Xu committed
362
static void write_tx_size_vartx(const AV1_COMMON *cm, const MACROBLOCKD *xd,
363
                                const MB_MODE_INFO *mbmi, TX_SIZE tx_size,
Yaowu Xu's avatar
Yaowu Xu committed
364
                                int blk_row, int blk_col, aom_writer *w) {
365 366
  const int tx_row = blk_row >> 1;
  const int tx_col = blk_col >> 1;
367 368
  int max_blocks_high = num_4x4_blocks_high_lookup[mbmi->sb_type];
  int max_blocks_wide = num_4x4_blocks_wide_lookup[mbmi->sb_type];
369
  int ctx = txfm_partition_context(xd->above_txfm_context + tx_col,
370
                                   xd->left_txfm_context + tx_row, tx_size);
371

372 373
  if (xd->mb_to_bottom_edge < 0) max_blocks_high += xd->mb_to_bottom_edge >> 5;
  if (xd->mb_to_right_edge < 0) max_blocks_wide += xd->mb_to_right_edge >> 5;
374

375
  if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
376

377
  if (tx_size == mbmi->inter_tx_size[tx_row][tx_col]) {
Yaowu Xu's avatar
Yaowu Xu committed
378
    aom_write(w, 0, cm->fc->txfm_partition_prob[ctx]);
379 380
    txfm_partition_update(xd->above_txfm_context + tx_col,
                          xd->left_txfm_context + tx_row, tx_size);
381 382
  } else {
    const BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
383
    int bsl = b_width_log2_lookup[bsize];
384
    int i;
Yaowu Xu's avatar
Yaowu Xu committed
385
    aom_write(w, 1, cm->fc->txfm_partition_prob[ctx]);
386

387
    if (tx_size == TX_8X8) {
388 389
      txfm_partition_update(xd->above_txfm_context + tx_col,
                            xd->left_txfm_context + tx_row, TX_4X4);
390
      return;
391
    }
392 393 394 395

    assert(bsl > 0);
    --bsl;
    for (i = 0; i < 4; ++i) {
396 397
      int offsetr = blk_row + ((i >> 1) << bsl);
      int offsetc = blk_col + ((i & 0x01) << bsl);
Debargha Mukherjee's avatar
Debargha Mukherjee committed
398
      write_tx_size_vartx(cm, xd, mbmi, tx_size - 1, offsetr, offsetc, w);
399 400 401
    }
  }
}
402

Yaowu Xu's avatar
Yaowu Xu committed
403
static void update_txfm_partition_probs(AV1_COMMON *cm, aom_writer *w,
404 405 406
                                        FRAME_COUNTS *counts) {
  int k;
  for (k = 0; k < TXFM_PARTITION_CONTEXTS; ++k)
Yaowu Xu's avatar
Yaowu Xu committed
407 408
    av1_cond_prob_diff_update(w, &cm->fc->txfm_partition_prob[k],
                              counts->txfm_partition[k]);
409
}
410 411
#endif

Yaowu Xu's avatar
Yaowu Xu committed
412 413
static void write_selected_tx_size(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                                   aom_writer *w) {
414 415
  const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
  const BLOCK_SIZE bsize = mbmi->sb_type;
416 417
  // For sub8x8 blocks the tx_size symbol does not need to be sent
  if (bsize >= BLOCK_8X8) {
418 419 420 421 422 423 424 425
    const TX_SIZE tx_size = mbmi->tx_size;
    const int is_inter = is_inter_block(mbmi);
    const int tx_size_ctx = get_tx_size_context(xd);
    const int tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
                                     : intra_tx_size_cat_lookup[bsize];
    const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size];

#if CONFIG_EXT_TX && CONFIG_RECT_TX
426
    assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(xd, mbmi)));
427 428 429 430
    assert(
        IMPLIES(is_rect_tx(tx_size), tx_size == max_txsize_rect_lookup[bsize]));
#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX

Yaowu Xu's avatar
Yaowu Xu committed
431 432 433
    av1_write_token(w, av1_tx_size_tree[tx_size_cat],
                    cm->fc->tx_size_probs[tx_size_cat][tx_size_ctx],
                    &tx_size_encodings[tx_size_cat][coded_tx_size]);
Jingning Han's avatar
Jingning Han committed
434 435 436
  }
}

437
#if CONFIG_REF_MV
Yaowu Xu's avatar
Yaowu Xu committed
438
static void update_inter_mode_probs(AV1_COMMON *cm, aom_writer *w,
439 440 441
                                    FRAME_COUNTS *counts) {
  int i;
  for (i = 0; i < NEWMV_MODE_CONTEXTS; ++i)
Yaowu Xu's avatar
Yaowu Xu committed
442
    av1_cond_prob_diff_update(w, &cm->fc->newmv_prob[i], counts->newmv_mode[i]);
443
  for (i = 0; i < ZEROMV_MODE_CONTEXTS; ++i)
Yaowu Xu's avatar
Yaowu Xu committed
444 445
    av1_cond_prob_diff_update(w, &cm->fc->zeromv_prob[i],
                              counts->zeromv_mode[i]);
446
  for (i = 0; i < REFMV_MODE_CONTEXTS; ++i)
Yaowu Xu's avatar
Yaowu Xu committed
447
    av1_cond_prob_diff_update(w, &cm->fc->refmv_prob[i], counts->refmv_mode[i]);
448
  for (i = 0; i < DRL_MODE_CONTEXTS; ++i)
Yaowu Xu's avatar
Yaowu Xu committed
449
    av1_cond_prob_diff_update(w, &cm->fc->drl_prob[i], counts->drl_mode[i]);
Yue Chen's avatar
Yue Chen committed
450
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
451
  av1_cond_prob_diff_update(w, &cm->fc->new2mv_prob, counts->new2mv_mode);
Yue Chen's avatar
Yue Chen committed
452
#endif  // CONFIG_EXT_INTER
453 454 455
}
#endif

456
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
457 458 459
static void update_inter_compound_mode_probs(AV1_COMMON *cm, aom_writer *w) {
  const int savings_thresh = av1_cost_one(GROUP_DIFF_UPDATE_PROB) -
                             av1_cost_zero(GROUP_DIFF_UPDATE_PROB);
460 461 462 463
  int i;
  int savings = 0;
  int do_update = 0;
  for (i = 0; i < INTER_MODE_CONTEXTS; ++i) {
464
    savings += prob_diff_update_savings(
Yaowu Xu's avatar
Yaowu Xu committed
465
        av1_inter_compound_mode_tree, cm->fc->inter_compound_mode_probs[i],
466
        cm->counts.inter_compound_mode[i], INTER_COMPOUND_MODES);
467 468
  }
  do_update = savings > savings_thresh;
Yaowu Xu's avatar
Yaowu Xu committed
469
  aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
470 471
  if (do_update) {
    for (i = 0; i < INTER_MODE_CONTEXTS; ++i) {
472
      prob_diff_update(
Yaowu Xu's avatar
Yaowu Xu committed
473
          av1_inter_compound_mode_tree, cm->fc->inter_compound_mode_probs[i],
474
          cm->counts.inter_compound_mode[i], INTER_COMPOUND_MODES, w);
475 476 477 478 479
    }
  }
}
#endif  // CONFIG_EXT_INTER

Yaowu Xu's avatar
Yaowu Xu committed
480 481
static int write_skip(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                      int segment_id, const MODE_INFO *mi, aom_writer *w) {
Jingning Han's avatar
Jingning Han committed
482 483 484 485
  if (segfeature_active(&cm->seg, segment_id, SEG_LVL_SKIP)) {
    return 1;
  } else {
    const int skip = mi->mbmi.skip;
Yaowu Xu's avatar
Yaowu Xu committed
486
    aom_write(w, skip, av1_get_skip_prob(cm, xd));
Jingning Han's avatar
Jingning Han committed
487 488 489 490
    return skip;
  }
}

491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506
#if CONFIG_DELTA_Q
static void write_delta_qindex(int delta_qindex, aom_writer *w) {
  int sign = delta_qindex < 0;
  int abs = sign ? -delta_qindex : delta_qindex;
  if (abs < 3) {
    aom_write_literal(w, 1, abs + 1);
  } else {
    aom_write_literal(w, 0, 3);
    aom_write_literal(w, abs, 6);
  }
  if (abs > 0) {
    aom_write_bit(w, sign);
  }
}
#endif

Yaowu Xu's avatar
Yaowu Xu committed
507
static void update_skip_probs(AV1_COMMON *cm, aom_writer *w,
Jingning Han's avatar
Jingning Han committed
508 509 510 511
                              FRAME_COUNTS *counts) {
  int k;

  for (k = 0; k < SKIP_CONTEXTS; ++k)
Yaowu Xu's avatar
Yaowu Xu committed
512
    av1_cond_prob_diff_update(w, &cm->fc->skip_probs[k], counts->skip[k]);
Jingning Han's avatar
Jingning Han committed
513 514
}

Yaowu Xu's avatar
Yaowu Xu committed
515
static void update_switchable_interp_probs(AV1_COMMON *cm, aom_writer *w,
Jingning Han's avatar
Jingning Han committed
516 517
                                           FRAME_COUNTS *counts) {
  int j;
518
  for (j = 0; j < SWITCHABLE_FILTER_CONTEXTS; ++j) {
Yaowu Xu's avatar
Yaowu Xu committed
519
    prob_diff_update(av1_switchable_interp_tree,
Jingning Han's avatar
Jingning Han committed
520 521
                     cm->fc->switchable_interp_prob[j],
                     counts->switchable_interp[j], SWITCHABLE_FILTERS, w);
522 523 524 525 526 527
#if CONFIG_DAALA_EC
    av1_tree_to_cdf(av1_switchable_interp_tree,
                    cm->fc->switchable_interp_prob[j],
                    cm->fc->switchable_interp_cdf[j]);
#endif
  }
Jingning Han's avatar
Jingning Han committed
528 529
}

530
#if CONFIG_EXT_TX
Yaowu Xu's avatar
Yaowu Xu committed
531 532 533
static void update_ext_tx_probs(AV1_COMMON *cm, aom_writer *w) {
  const int savings_thresh = av1_cost_one(GROUP_DIFF_UPDATE_PROB) -
                             av1_cost_zero(GROUP_DIFF_UPDATE_PROB);
534
  int i, j;
535 536 537 538 539
  int s;
  for (s = 1; s < EXT_TX_SETS_INTER; ++s) {
    int savings = 0;
    int do_update = 0;
    for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
540
      if (!use_inter_ext_tx_for_txsize[s][i]) continue;
541
      savings += prob_diff_update_savings(
Yaowu Xu's avatar
Yaowu Xu committed
542
          av1_ext_tx_inter_tree[s], cm->fc->inter_ext_tx_prob[s][i],
543 544 545
          cm->counts.inter_ext_tx[s][i], num_ext_tx_set_inter[s]);
    }
    do_update = savings > savings_thresh;
Yaowu Xu's avatar
Yaowu Xu committed
546
    aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
547 548
    if (do_update) {
      for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
549
        if (!use_inter_ext_tx_for_txsize[s][i]) continue;
550
        prob_diff_update(
Yaowu Xu's avatar
Yaowu Xu committed
551
            av1_ext_tx_inter_tree[s], cm->fc->inter_ext_tx_prob[s][i],
552
            cm->counts.inter_ext_tx[s][i], num_ext_tx_set_inter[s], w);
553
      }
554 555
    }
  }
556

557 558 559 560
  for (s = 1; s < EXT_TX_SETS_INTRA; ++s) {
    int savings = 0;
    int do_update = 0;
    for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
561
      if (!use_intra_ext_tx_for_txsize[s][i]) continue;
562
      for (j = 0; j < INTRA_MODES; ++j)
563
        savings += prob_diff_update_savings(
Yaowu Xu's avatar
Yaowu Xu committed
564
            av1_ext_tx_intra_tree[s], cm->fc->intra_ext_tx_prob[s][i][j],
565 566 567
            cm->counts.intra_ext_tx[s][i][j], num_ext_tx_set_intra[s]);
    }
    do_update = savings > savings_thresh;
Yaowu Xu's avatar
Yaowu Xu committed
568
    aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
569 570
    if (do_update) {
      for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
571
        if (!use_intra_ext_tx_for_txsize[s][i]) continue;
572
        for (j = 0; j < INTRA_MODES; ++j)
573
          prob_diff_update(
Yaowu Xu's avatar
Yaowu Xu committed
574
              av1_ext_tx_intra_tree[s], cm->fc->intra_ext_tx_prob[s][i][j],
575
              cm->counts.intra_ext_tx[s][i][j], num_ext_tx_set_intra[s], w);
576 577
      }
    }
578
  }
579
}
Debargha Mukherjee's avatar
Debargha Mukherjee committed
580

581
#else
Debargha Mukherjee's avatar
Debargha Mukherjee committed
582

Yaowu Xu's avatar
Yaowu Xu committed
583 584 585
static void update_ext_tx_probs(AV1_COMMON *cm, aom_writer *w) {
  const int savings_thresh = av1_cost_one(GROUP_DIFF_UPDATE_PROB) -
                             av1_cost_zero(GROUP_DIFF_UPDATE_PROB);
586 587 588 589 590 591 592
  int i, j;

  int savings = 0;
  int do_update = 0;
  for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
    for (j = 0; j < TX_TYPES; ++j)
      savings += prob_diff_update_savings(
Yaowu Xu's avatar
Yaowu Xu committed
593
          av1_ext_tx_tree, cm->fc->intra_ext_tx_prob[i][j],
594 595 596
          cm->counts.intra_ext_tx[i][j], TX_TYPES);
  }
  do_update = savings > savings_thresh;
Yaowu Xu's avatar
Yaowu Xu committed
597
  aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
598 599
  if (do_update) {
    for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
600
      for (j = 0; j < TX_TYPES; ++j) {
Yaowu Xu's avatar
Yaowu Xu committed
601
        prob_diff_update(av1_ext_tx_tree, cm->fc->intra_ext_tx_prob[i][j],
602
                         cm->counts.intra_ext_tx[i][j], TX_TYPES, w);
603 604 605 606 607
#if CONFIG_DAALA_EC
        av1_tree_to_cdf(av1_ext_tx_tree, cm->fc->intra_ext_tx_prob[i][j],
                        cm->fc->intra_ext_tx_cdf[i][j]);
#endif
      }
608 609 610 611
    }
  }
  savings = 0;
  for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
612
    savings +=
Yaowu Xu's avatar
Yaowu Xu committed
613
        prob_diff_update_savings(av1_ext_tx_tree, cm->fc->inter_ext_tx_prob[i],
614
                                 cm->counts.inter_ext_tx[i], TX_TYPES);
615 616
  }
  do_update = savings > savings_thresh;
Yaowu Xu's avatar
Yaowu Xu committed
617
  aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
618 619
  if (do_update) {
    for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
Yaowu Xu's avatar
Yaowu Xu committed
620
      prob_diff_update(av1_ext_tx_tree, cm->fc->inter_ext_tx_prob[i],
621
                       cm->counts.inter_ext_tx[i], TX_TYPES, w);
622 623 624 625
#if CONFIG_DAALA_EC
      av1_tree_to_cdf(av1_ext_tx_tree, cm->fc->inter_ext_tx_prob[i],
                      cm->fc->inter_ext_tx_cdf[i]);
#endif
626 627 628
    }
  }
}
629 630
#endif  // CONFIG_EXT_TX

631
#if CONFIG_PALETTE
Yaowu Xu's avatar
Yaowu Xu committed
632
static void pack_palette_tokens(aom_writer *w, const TOKENEXTRA **tp, int n,
633
                                int num) {
hui su's avatar
hui su committed
634
  int i;
635
  const TOKENEXTRA *p = *tp;
hui su's avatar
hui su committed
636

637
  for (i = 0; i < num; ++i) {
Yaowu Xu's avatar
Yaowu Xu committed
638 639
    av1_write_token(w, av1_palette_color_tree[n - 2], p->context_tree,
                    &palette_color_encodings[n - 2][p->token]);
hui su's avatar
hui su committed
640 641 642 643 644
    ++p;
  }

  *tp = p;
}
645
#endif  // CONFIG_PALETTE
hui su's avatar
hui su committed
646

647
#if CONFIG_SUPERTX
Yaowu Xu's avatar
Yaowu Xu committed
648 649 650
static void update_supertx_probs(AV1_COMMON *cm, aom_writer *w) {
  const int savings_thresh = av1_cost_one(GROUP_DIFF_UPDATE_PROB) -
                             av1_cost_zero(GROUP_DIFF_UPDATE_PROB);
651 652 653 654 655
  int i, j;
  int savings = 0;
  int do_update = 0;
  for (i = 0; i < PARTITION_SUPERTX_CONTEXTS; ++i) {
    for (j = 1; j < TX_SIZES; ++j) {
Yaowu Xu's avatar
Yaowu Xu committed
656 657
      savings += av1_cond_prob_diff_update_savings(&cm->fc->supertx_prob[i][j],
                                                   cm->counts.supertx[i][j]);
658 659 660
    }
  }
  do_update = savings > savings_thresh;
Yaowu Xu's avatar
Yaowu Xu committed
661
  aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
662 663 664
  if (do_update) {
    for (i = 0; i < PARTITION_SUPERTX_CONTEXTS; ++i) {
      for (j = 1; j < TX_SIZES; ++j) {
Yaowu Xu's avatar
Yaowu Xu committed
665 666
        av1_cond_prob_diff_update(w, &cm->fc->supertx_prob[i][j],
                                  cm->counts.supertx[i][j]);
667 668 669 670 671 672
      }
    }
  }
}
#endif  // CONFIG_SUPERTX

Yaowu Xu's avatar
Yaowu Xu committed
673
static void pack_mb_tokens(aom_writer *w, const TOKENEXTRA **tp,
674
                           const TOKENEXTRA *const stop,
Yaowu Xu's avatar
Yaowu Xu committed
675
                           aom_bit_depth_t bit_depth, const TX_SIZE tx) {
676
  const TOKENEXTRA *p = *tp;
677 678
#if CONFIG_VAR_TX
  int count = 0;
Debargha Mukherjee's avatar
Debargha Mukherjee committed
679
  const int seg_eob = get_tx2d_size(tx);
680
#endif
Jingning Han's avatar
Jingning Han committed
681 682

  while (p < stop && p->token != EOSB_TOKEN) {
683 684
    const int token = p->token;
    aom_tree_index index = 0;
Alex Converse's avatar
Alex Converse committed
685
#if !CONFIG_ANS
686 687 688
    const struct av1_token *const coef_encoding = &av1_coef_encodings[token];
    int coef_value = coef_encoding->value;
    int coef_length = coef_encoding->len;
Alex Converse's avatar
Alex Converse committed
689
#endif  // !CONFIG_ANS
Yaowu Xu's avatar
Yaowu Xu committed
690
#if CONFIG_AOM_HIGHBITDEPTH
691 692 693 694 695
    const av1_extra_bit *const extra_bits_av1 =
        (bit_depth == AOM_BITS_12)
            ? &av1_extra_bits_high12[token]
            : (bit_depth == AOM_BITS_10) ? &av1_extra_bits_high10[token]
                                         : &av1_extra_bits[token];
Jingning Han's avatar
Jingning Han committed
696
#else
697
    const av1_extra_bit *const extra_bits_av1 = &av1_extra_bits[token];
698
    (void)bit_depth;
Yaowu Xu's avatar
Yaowu Xu committed
699
#endif  // CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
700

Alex Converse's avatar
Alex Converse committed
701 702
#if CONFIG_ANS
    /* skip one or two nodes */
703
    if (!p->skip_eob_node) aom_write(w, token != EOB_TOKEN, p->context_tree[0]);
Alex Converse's avatar
Alex Converse committed
704

705 706
    if (token != EOB_TOKEN) {
      aom_write(w, token != ZERO_TOKEN, p->context_tree[1]);
707

708 709
      if (token != ZERO_TOKEN) {
        aom_write_symbol(w, token - ONE_TOKEN, *p->token_cdf,
710
                         CATEGORY6_TOKEN - ONE_TOKEN + 1);
711
      }
Alex Converse's avatar
Alex Converse committed
712 713
    }
#else
Jingning Han's avatar
Jingning Han committed
714
    /* skip one or two nodes */
715
    if (p->skip_eob_node)
716
      coef_length -= p->skip_eob_node;
717
    else
718
      aom_write(w, token != EOB_TOKEN, p->context_tree[0]);
719

720 721
    if (token != EOB_TOKEN) {
      aom_write(w, token != ZERO_TOKEN, p->context_tree[1]);
Jingning Han's avatar
Jingning Han committed
722

723 724
      if (token != ZERO_TOKEN) {
        aom_write(w, token != ONE_TOKEN, p->context_tree[2]);
725

726 727
        if (token != ONE_TOKEN) {
          const int unconstrained_len = UNCONSTRAINED_NODES - p->skip_eob_node;
728
          aom_write_tree(w, av1_coef_con_tree,
729 730
                         av1_pareto8_full[p->context_tree[PIVOT_NODE] - 1],
                         coef_value, coef_length - unconstrained_len, 0);
731 732
        }
      }
Jingning Han's avatar
Jingning Han committed
733
    }
Alex Converse's avatar
Alex Converse committed
734
#endif  // CONFIG_ANS
Jingning Han's avatar
Jingning Han committed
735

736 737 738 739
    if (extra_bits_av1->base_val) {
      const int extra_bits = p->extra;
      const int extra_bits_av1_length = extra_bits_av1->len;
      int skip_bits = (extra_bits_av1->base_val == CAT6_MIN_VAL)
Debargha Mukherjee's avatar
Debargha Mukherjee committed
740 741
                          ? TX_SIZES - 1 - txsize_sqr_up_map[tx]
                          : 0;
Jingning Han's avatar
Jingning Han committed
742

743 744 745 746 747
      if (extra_bits_av1_length) {
        const unsigned char *pb = extra_bits_av1->prob;
        const int value = extra_bits >> 1;
        int num_bits = extra_bits_av1_length;  // number of bits in value
        assert(num_bits > 0);
Jingning Han's avatar
Jingning Han committed
748

749
        index = 0;
Jingning Han's avatar
Jingning Han committed
750
        do {
751
          const int bb = (value >> --num_bits) & 1;
752
          if (skip_bits) {
753
            --skip_bits;
754 755
            assert(!bb);
          } else {
756
            aom_write(w, bb, pb[index >> 1]);
757
          }
758 759
          index = extra_bits_av1->tree[index + bb];
        } while (num_bits);
Jingning Han's avatar
Jingning Han committed
760 761
      }

762
      aom_write_bit(w, extra_bits & 1);
Jingning Han's avatar
Jingning Han committed
763 764
    }
    ++p;
765 766 767

#if CONFIG_VAR_TX
    ++count;
768
    if (token == EOB_TOKEN || count == seg_eob) break;
769
#endif
Jingning Han's avatar
Jingning Han committed
770 771
  }

772
  *tp = p;
Jingning Han's avatar
Jingning Han committed
773 774
}

775
#if CONFIG_VAR_TX
Yaowu Xu's avatar
Yaowu Xu committed
776
static void pack_txb_tokens(aom_writer *w, const TOKENEXTRA **tp,
777 778
                            const TOKENEXTRA *const tok_end, MACROBLOCKD *xd,
                            MB_MODE_INFO *mbmi, int plane,
Yaowu Xu's avatar
Yaowu Xu committed
779
                            BLOCK_SIZE plane_bsize, aom_bit_depth_t bit_depth,
780 781
                            int block, int blk_row, int blk_col,
                            TX_SIZE tx_size) {
782 783
  const struct macroblockd_plane *const pd = &xd->plane[plane];
  const BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
784 785
  const int tx_row = blk_row >> (1 - pd->subsampling_y);
  const int tx_col = blk_col >> (1 - pd->subsampling_x);
Debargha Mukherjee's avatar
Debargha Mukherjee committed
786
  TX_SIZE plane_tx_size;
787 788 789 790 791 792 793 794
  int max_blocks_high = num_4x4_blocks_high_lookup[plane_bsize];