bitstream.c 157 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4 5 6 7 8 9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10 11 12 13
 */

#include <assert.h>
#include <limits.h>
14
#include <stdio.h>
Jingning Han's avatar
Jingning Han committed
15

Yaowu Xu's avatar
Yaowu Xu committed
16
#include "aom/aom_encoder.h"
17
#include "aom_dsp/bitwriter_buffer.h"
Yaowu Xu's avatar
Yaowu Xu committed
18 19
#include "aom_dsp/aom_dsp_common.h"
#include "aom_mem/aom_mem.h"
20 21
#include "aom_ports/mem_ops.h"
#include "aom_ports/system_state.h"
22
#if CONFIG_BITSTREAM_DEBUG
23
#include "aom_util/debug_util.h"
24
#endif  // CONFIG_BITSTREAM_DEBUG
Jingning Han's avatar
Jingning Han committed
25

26
#if CONFIG_CLPF
27
#include "av1/common/clpf.h"
28
#endif
Yaowu Xu's avatar
Yaowu Xu committed
29
#if CONFIG_DERING
30
#include "av1/common/dering.h"
Yaowu Xu's avatar
Yaowu Xu committed
31
#endif  // CONFIG_DERING
32 33 34 35
#include "av1/common/entropy.h"
#include "av1/common/entropymode.h"
#include "av1/common/entropymv.h"
#include "av1/common/mvref_common.h"
36
#include "av1/common/odintrin.h"
37 38 39 40
#include "av1/common/pred_common.h"
#include "av1/common/reconinter.h"
#include "av1/common/seg_common.h"
#include "av1/common/tile_common.h"
Jingning Han's avatar
Jingning Han committed
41

Alex Converse's avatar
Alex Converse committed
42
#if CONFIG_ANS
Alex Converse's avatar
Alex Converse committed
43
#include "aom_dsp/buf_ans.h"
Alex Converse's avatar
Alex Converse committed
44
#endif  // CONFIG_ANS
45 46 47 48 49 50 51
#include "av1/encoder/bitstream.h"
#include "av1/encoder/cost.h"
#include "av1/encoder/encodemv.h"
#include "av1/encoder/mcomp.h"
#include "av1/encoder/segmentation.h"
#include "av1/encoder/subexp.h"
#include "av1/encoder/tokenize.h"
52 53 54
#if CONFIG_PVQ
#include "av1/encoder/pvq_encoder.h"
#endif
Jingning Han's avatar
Jingning Han committed
55

56 57
static struct av1_token intra_mode_encodings[INTRA_MODES];
static struct av1_token switchable_interp_encodings[SWITCHABLE_FILTERS];
58
#if CONFIG_EXT_PARTITION_TYPES
Yaowu Xu's avatar
Yaowu Xu committed
59
static const struct av1_token ext_partition_encodings[EXT_PARTITION_TYPES] = {
60 61 62
  { 0, 1 },  { 4, 3 },  { 12, 4 }, { 7, 3 },
  { 10, 4 }, { 11, 4 }, { 26, 5 }, { 27, 5 }
};
63
#endif
64
static struct av1_token partition_encodings[PARTITION_TYPES];
65
#if !CONFIG_REF_MV
66
static struct av1_token inter_mode_encodings[INTER_MODES];
67
#endif
68
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
69
static const struct av1_token
70 71 72 73
    inter_compound_mode_encodings[INTER_COMPOUND_MODES] = {
      { 2, 2 },  { 50, 6 }, { 51, 6 }, { 24, 5 }, { 52, 6 },
      { 53, 6 }, { 54, 6 }, { 55, 6 }, { 0, 1 },  { 7, 3 }
    };
74
#endif  // CONFIG_EXT_INTER
75
#if CONFIG_PALETTE
76 77 78
static struct av1_token palette_size_encodings[PALETTE_MAX_SIZE - 1];
static struct av1_token palette_color_encodings[PALETTE_MAX_SIZE - 1]
                                               [PALETTE_MAX_SIZE];
79
#endif  // CONFIG_PALETTE
Jingning Han's avatar
Jingning Han committed
80
static const struct av1_token tx_size_encodings[MAX_TX_DEPTH][TX_SIZES] = {
81 82 83
  { { 0, 1 }, { 1, 1 } },                      // Max tx_size is 8X8
  { { 0, 1 }, { 2, 2 }, { 3, 2 } },            // Max tx_size is 16X16
  { { 0, 1 }, { 2, 2 }, { 6, 3 }, { 7, 3 } },  // Max tx_size is 32X32
84 85
};

86
#if CONFIG_EXT_INTRA || CONFIG_FILTER_INTRA || CONFIG_PALETTE
Yaowu Xu's avatar
Yaowu Xu committed
87
static INLINE void write_uniform(aom_writer *w, int n, int v) {
hui su's avatar
hui su committed
88 89
  int l = get_unsigned_bits(n);
  int m = (1 << l) - n;
90
  if (l == 0) return;
hui su's avatar
hui su committed
91
  if (v < m) {
Yaowu Xu's avatar
Yaowu Xu committed
92
    aom_write_literal(w, v, l - 1);
hui su's avatar
hui su committed
93
  } else {
Yaowu Xu's avatar
Yaowu Xu committed
94 95
    aom_write_literal(w, m + ((v - m) >> 1), l - 1);
    aom_write_literal(w, (v - m) & 1, 1);
hui su's avatar
hui su committed
96 97
  }
}
98
#endif  // CONFIG_EXT_INTRA || CONFIG_FILTER_INTRA || CONFIG_PALETTE
Jingning Han's avatar
Jingning Han committed
99

100
#if CONFIG_EXT_TX
Yaowu Xu's avatar
Yaowu Xu committed
101 102
static struct av1_token ext_tx_inter_encodings[EXT_TX_SETS_INTER][TX_TYPES];
static struct av1_token ext_tx_intra_encodings[EXT_TX_SETS_INTRA][TX_TYPES];
103
#else
Yaowu Xu's avatar
Yaowu Xu committed
104
static struct av1_token ext_tx_encodings[TX_TYPES];
105
#endif  // CONFIG_EXT_TX
106
#if CONFIG_GLOBAL_MOTION
Yaowu Xu's avatar
Yaowu Xu committed
107
static struct av1_token global_motion_types_encodings[GLOBAL_MOTION_TYPES];
108
#endif  // CONFIG_GLOBAL_MOTION
109
#if CONFIG_EXT_INTRA
Yaowu Xu's avatar
Yaowu Xu committed
110
static struct av1_token intra_filter_encodings[INTRA_FILTERS];
111
#endif  // CONFIG_EXT_INTRA
112
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
113
static struct av1_token interintra_mode_encodings[INTERINTRA_MODES];
114
#endif  // CONFIG_EXT_INTER
Yue Chen's avatar
Yue Chen committed
115 116 117
#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
static struct av1_token motion_mode_encodings[MOTION_MODES];
#endif  // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
118 119 120
#if CONFIG_LOOP_RESTORATION
static struct av1_token switchable_restore_encodings[RESTORE_SWITCHABLE_TYPES];
#endif  // CONFIG_LOOP_RESTORATION
121 122 123
static void write_uncompressed_header(AV1_COMP *cpi,
                                      struct aom_write_bit_buffer *wb);
static uint32_t write_compressed_header(AV1_COMP *cpi, uint8_t *data);
124

Yaowu Xu's avatar
Yaowu Xu committed
125
void av1_encode_token_init(void) {
126
#if CONFIG_EXT_TX || CONFIG_PALETTE
127
  int s;
128 129
#endif  // CONFIG_EXT_TX || CONFIG_PALETTE
#if CONFIG_EXT_TX
130
  for (s = 1; s < EXT_TX_SETS_INTER; ++s) {
Yaowu Xu's avatar
Yaowu Xu committed
131
    av1_tokens_from_tree(ext_tx_inter_encodings[s], av1_ext_tx_inter_tree[s]);
132 133
  }
  for (s = 1; s < EXT_TX_SETS_INTRA; ++s) {
Yaowu Xu's avatar
Yaowu Xu committed
134
    av1_tokens_from_tree(ext_tx_intra_encodings[s], av1_ext_tx_intra_tree[s]);
135
  }
136
#else
Yaowu Xu's avatar
Yaowu Xu committed
137
  av1_tokens_from_tree(ext_tx_encodings, av1_ext_tx_tree);
138
#endif  // CONFIG_EXT_TX
139 140 141 142 143 144 145
  av1_tokens_from_tree(intra_mode_encodings, av1_intra_mode_tree);
  av1_tokens_from_tree(switchable_interp_encodings, av1_switchable_interp_tree);
  av1_tokens_from_tree(partition_encodings, av1_partition_tree);
#if !CONFIG_REF_MV
  av1_tokens_from_tree(inter_mode_encodings, av1_inter_mode_tree);
#endif

146 147 148 149 150 151 152
#if CONFIG_PALETTE
  av1_tokens_from_tree(palette_size_encodings, av1_palette_size_tree);
  for (s = 0; s < PALETTE_MAX_SIZE - 1; ++s) {
    av1_tokens_from_tree(palette_color_encodings[s], av1_palette_color_tree[s]);
  }
#endif  // CONFIG_PALETTE

153
#if CONFIG_EXT_INTRA
Yaowu Xu's avatar
Yaowu Xu committed
154
  av1_tokens_from_tree(intra_filter_encodings, av1_intra_filter_tree);
155
#endif  // CONFIG_EXT_INTRA
156
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
157
  av1_tokens_from_tree(interintra_mode_encodings, av1_interintra_mode_tree);
158
#endif  // CONFIG_EXT_INTER
Yue Chen's avatar
Yue Chen committed
159 160 161
#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
  av1_tokens_from_tree(motion_mode_encodings, av1_motion_mode_tree);
#endif  // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
162
#if CONFIG_GLOBAL_MOTION
Yaowu Xu's avatar
Yaowu Xu committed
163 164
  av1_tokens_from_tree(global_motion_types_encodings,
                       av1_global_motion_types_tree);
165
#endif  // CONFIG_GLOBAL_MOTION
166 167 168 169
#if CONFIG_LOOP_RESTORATION
  av1_tokens_from_tree(switchable_restore_encodings,
                       av1_switchable_restore_tree);
#endif  // CONFIG_LOOP_RESTORATION
170 171 172 173 174 175 176

#if CONFIG_DAALA_EC
  /* This hack is necessary when CONFIG_EXT_INTERP is enabled because the five
      SWITCHABLE_FILTERS are not consecutive, e.g., 0, 1, 2, 3, 4, when doing
      an in-order traversal of the av1_switchable_interp_tree structure. */
  av1_indices_from_tree(av1_switchable_interp_ind, av1_switchable_interp_inv,
                        SWITCHABLE_FILTERS, av1_switchable_interp_tree);
177 178 179
  /* This hack is necessary because the four TX_TYPES are not consecutive,
      e.g., 0, 1, 2, 3, when doing an in-order traversal of the av1_ext_tx_tree
      structure. */
David Barker's avatar
David Barker committed
180
#if !CONFIG_EXT_TX
181 182
  av1_indices_from_tree(av1_ext_tx_ind, av1_ext_tx_inv, TX_TYPES,
                        av1_ext_tx_tree);
David Barker's avatar
David Barker committed
183
#endif
184 185
  av1_indices_from_tree(av1_intra_mode_ind, av1_intra_mode_inv, INTRA_MODES,
                        av1_intra_mode_tree);
186 187
  av1_indices_from_tree(av1_inter_mode_ind, av1_inter_mode_inv, INTER_MODES,
                        av1_inter_mode_tree);
188
#endif
189 190
}

191
#if !CONFIG_DAALA_EC
Yaowu Xu's avatar
Yaowu Xu committed
192 193 194
static void write_intra_mode(aom_writer *w, PREDICTION_MODE mode,
                             const aom_prob *probs) {
  av1_write_token(w, av1_intra_mode_tree, probs, &intra_mode_encodings[mode]);
Jingning Han's avatar
Jingning Han committed
195
}
196
#endif
Jingning Han's avatar
Jingning Han committed
197

198
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
199 200 201 202
static void write_interintra_mode(aom_writer *w, INTERINTRA_MODE mode,
                                  const aom_prob *probs) {
  av1_write_token(w, av1_interintra_mode_tree, probs,
                  &interintra_mode_encodings[mode]);
203 204 205
}
#endif  // CONFIG_EXT_INTER

Yaowu Xu's avatar
Yaowu Xu committed
206
static void write_inter_mode(AV1_COMMON *cm, aom_writer *w,
207
                             PREDICTION_MODE mode,
Yue Chen's avatar
Yue Chen committed
208 209 210
#if CONFIG_REF_MV && CONFIG_EXT_INTER
                             int is_compound,
#endif  // CONFIG_REF_MV && CONFIG_EXT_INTER
211
                             const int16_t mode_ctx) {
212
#if CONFIG_REF_MV
213
  const int16_t newmv_ctx = mode_ctx & NEWMV_CTX_MASK;
Yaowu Xu's avatar
Yaowu Xu committed
214
  const aom_prob newmv_prob = cm->fc->newmv_prob[newmv_ctx];
Yue Chen's avatar
Yue Chen committed
215
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
216
  aom_write(w, mode != NEWMV && mode != NEWFROMNEARMV, newmv_prob);
Yue Chen's avatar
Yue Chen committed
217 218

  if (!is_compound && (mode == NEWMV || mode == NEWFROMNEARMV))
Yaowu Xu's avatar
Yaowu Xu committed
219
    aom_write(w, mode == NEWFROMNEARMV, cm->fc->new2mv_prob);
Yue Chen's avatar
Yue Chen committed
220 221 222

  if (mode != NEWMV && mode != NEWFROMNEARMV) {
#else
Yaowu Xu's avatar
Yaowu Xu committed
223
  aom_write(w, mode != NEWMV, newmv_prob);
224 225

  if (mode != NEWMV) {
Yue Chen's avatar
Yue Chen committed
226
#endif  // CONFIG_EXT_INTER
227
    const int16_t zeromv_ctx = (mode_ctx >> ZEROMV_OFFSET) & ZEROMV_CTX_MASK;
Yaowu Xu's avatar
Yaowu Xu committed
228
    const aom_prob zeromv_prob = cm->fc->zeromv_prob[zeromv_ctx];
229 230 231 232 233 234

    if (mode_ctx & (1 << ALL_ZERO_FLAG_OFFSET)) {
      assert(mode == ZEROMV);
      return;
    }

Yaowu Xu's avatar
Yaowu Xu committed
235
    aom_write(w, mode != ZEROMV, zeromv_prob);
236 237

    if (mode != ZEROMV) {
238
      int16_t refmv_ctx = (mode_ctx >> REFMV_OFFSET) & REFMV_CTX_MASK;
Yaowu Xu's avatar
Yaowu Xu committed
239
      aom_prob refmv_prob;
240

241 242 243
      if (mode_ctx & (1 << SKIP_NEARESTMV_OFFSET)) refmv_ctx = 6;
      if (mode_ctx & (1 << SKIP_NEARMV_OFFSET)) refmv_ctx = 7;
      if (mode_ctx & (1 << SKIP_NEARESTMV_SUB8X8_OFFSET)) refmv_ctx = 8;
244 245

      refmv_prob = cm->fc->refmv_prob[refmv_ctx];
Yaowu Xu's avatar
Yaowu Xu committed
246
      aom_write(w, mode != NEARESTMV, refmv_prob);
247 248 249
    }
  }
#else
Jingning Han's avatar
Jingning Han committed
250
  assert(is_inter_mode(mode));
251 252 253 254 255 256 257 258 259 260
#if CONFIG_DAALA_EC
  aom_write_symbol(w, av1_inter_mode_ind[INTER_OFFSET(mode)],
                   cm->fc->inter_mode_cdf[mode_ctx], INTER_MODES);
#else
  {
    const aom_prob *const inter_probs = cm->fc->inter_mode_probs[mode_ctx];
    av1_write_token(w, av1_inter_mode_tree, inter_probs,
                    &inter_mode_encodings[INTER_OFFSET(mode)]);
  }
#endif
261
#endif
Jingning Han's avatar
Jingning Han committed
262 263
}

264
#if CONFIG_REF_MV
Yaowu Xu's avatar
Yaowu Xu committed
265 266 267
static void write_drl_idx(const AV1_COMMON *cm, const MB_MODE_INFO *mbmi,
                          const MB_MODE_INFO_EXT *mbmi_ext, aom_writer *w) {
  uint8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame);
268 269 270

  assert(mbmi->ref_mv_idx < 3);

271 272 273 274 275
  if (mbmi->mode == NEWMV) {
    int idx;
    for (idx = 0; idx < 2; ++idx) {
      if (mbmi_ext->ref_mv_count[ref_frame_type] > idx + 1) {
        uint8_t drl_ctx =
Yaowu Xu's avatar
Yaowu Xu committed
276 277
            av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx);
        aom_prob drl_prob = cm->fc->drl_prob[drl_ctx];
278

Yaowu Xu's avatar
Yaowu Xu committed
279
        aom_write(w, mbmi->ref_mv_idx != idx, drl_prob);
280
        if (mbmi->ref_mv_idx == idx) return;
281
      }
282 283 284 285
    }
    return;
  }

286 287 288 289 290 291
  if (mbmi->mode == NEARMV) {
    int idx;
    // TODO(jingning): Temporary solution to compensate the NEARESTMV offset.
    for (idx = 1; idx < 3; ++idx) {
      if (mbmi_ext->ref_mv_count[ref_frame_type] > idx + 1) {
        uint8_t drl_ctx =
Yaowu Xu's avatar
Yaowu Xu committed
292 293
            av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx);
        aom_prob drl_prob = cm->fc->drl_prob[drl_ctx];
294

Yaowu Xu's avatar
Yaowu Xu committed
295
        aom_write(w, mbmi->ref_mv_idx != (idx - 1), drl_prob);
296
        if (mbmi->ref_mv_idx == (idx - 1)) return;
297
      }
298
    }
299
    return;
300 301 302 303
  }
}
#endif

304
#if CONFIG_EXT_INTER
Yaowu Xu's avatar
Yaowu Xu committed
305
static void write_inter_compound_mode(AV1_COMMON *cm, aom_writer *w,
306 307
                                      PREDICTION_MODE mode,
                                      const int16_t mode_ctx) {
Yaowu Xu's avatar
Yaowu Xu committed
308
  const aom_prob *const inter_compound_probs =
309
      cm->fc->inter_compound_mode_probs[mode_ctx];
310 311

  assert(is_inter_compound_mode(mode));
Yaowu Xu's avatar
Yaowu Xu committed
312 313
  av1_write_token(w, av1_inter_compound_mode_tree, inter_compound_probs,
                  &inter_compound_mode_encodings[INTER_COMPOUND_OFFSET(mode)]);
314 315 316
}
#endif  // CONFIG_EXT_INTER

Yaowu Xu's avatar
Yaowu Xu committed
317
static void encode_unsigned_max(struct aom_write_bit_buffer *wb, int data,
318
                                int max) {
Yaowu Xu's avatar
Yaowu Xu committed
319
  aom_wb_write_literal(wb, data, get_unsigned_bits(max));
Jingning Han's avatar
Jingning Han committed
320 321
}

Yaowu Xu's avatar
Yaowu Xu committed
322 323
static void prob_diff_update(const aom_tree_index *tree,
                             aom_prob probs[/*n - 1*/],
324
                             const unsigned int counts[/*n - 1*/], int n,
325
                             int probwt, aom_writer *w) {
Jingning Han's avatar
Jingning Han committed
326 327 328 329 330 331
  int i;
  unsigned int branch_ct[32][2];

  // Assuming max number of probabilities <= 32
  assert(n <= 32);

Yaowu Xu's avatar
Yaowu Xu committed
332
  av1_tree_probs_from_distribution(tree, branch_ct, counts);
Jingning Han's avatar
Jingning Han committed
333
  for (i = 0; i < n - 1; ++i)
334
    av1_cond_prob_diff_update(w, &probs[i], branch_ct[i], probwt);
Jingning Han's avatar
Jingning Han committed
335 336
}

337
#if !CONFIG_EC_ADAPT
Yaowu Xu's avatar
Yaowu Xu committed
338 339
static int prob_diff_update_savings(const aom_tree_index *tree,
                                    aom_prob probs[/*n - 1*/],
340 341
                                    const unsigned int counts[/*n - 1*/], int n,
                                    int probwt) {
342 343 344 345 346 347
  int i;
  unsigned int branch_ct[32][2];
  int savings = 0;

  // Assuming max number of probabilities <= 32
  assert(n <= 32);
Yaowu Xu's avatar
Yaowu Xu committed
348
  av1_tree_probs_from_distribution(tree, branch_ct, counts);
349
  for (i = 0; i < n - 1; ++i) {
350 351
    savings +=
        av1_cond_prob_diff_update_savings(&probs[i], branch_ct[i], probwt);
352 353 354
  }
  return savings;
}
355
#endif
356

357
#if CONFIG_VAR_TX
Yaowu Xu's avatar
Yaowu Xu committed
358
static void write_tx_size_vartx(const AV1_COMMON *cm, const MACROBLOCKD *xd,
359
                                const MB_MODE_INFO *mbmi, TX_SIZE tx_size,
360 361
                                int depth, int blk_row, int blk_col,
                                aom_writer *w) {
362 363
  const int tx_row = blk_row >> 1;
  const int tx_col = blk_col >> 1;
364 365 366
  const int max_blocks_high = max_block_high(xd, mbmi->sb_type, 0);
  const int max_blocks_wide = max_block_wide(xd, mbmi->sb_type, 0);

367
  int ctx = txfm_partition_context(xd->above_txfm_context + tx_col,
368 369
                                   xd->left_txfm_context + tx_row,
                                   mbmi->sb_type, tx_size);
370

371
  if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
372

Jingning Han's avatar
Jingning Han committed
373
  if (depth == MAX_VARTX_DEPTH) {
374 375 376 377 378
    txfm_partition_update(xd->above_txfm_context + tx_col,
                          xd->left_txfm_context + tx_row, tx_size);
    return;
  }

379
  if (tx_size == mbmi->inter_tx_size[tx_row][tx_col]) {
Yaowu Xu's avatar
Yaowu Xu committed
380
    aom_write(w, 0, cm->fc->txfm_partition_prob[ctx]);
381 382
    txfm_partition_update(xd->above_txfm_context + tx_col,
                          xd->left_txfm_context + tx_row, tx_size);
383
  } else {
384 385
    const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
    const int bsl = tx_size_wide_unit[sub_txs];
386
    int i;
387

Yaowu Xu's avatar
Yaowu Xu committed
388
    aom_write(w, 1, cm->fc->txfm_partition_prob[ctx]);
389

390
    if (tx_size == TX_8X8) {
391 392
      txfm_partition_update(xd->above_txfm_context + tx_col,
                            xd->left_txfm_context + tx_row, TX_4X4);
393
      return;
394
    }
395 396 397

    assert(bsl > 0);
    for (i = 0; i < 4; ++i) {
398 399 400 401
      int offsetr = blk_row + (i >> 1) * bsl;
      int offsetc = blk_col + (i & 0x01) * bsl;
      write_tx_size_vartx(cm, xd, mbmi, sub_txs, depth + 1, offsetr, offsetc,
                          w);
402 403 404
    }
  }
}
405

Yaowu Xu's avatar
Yaowu Xu committed
406
static void update_txfm_partition_probs(AV1_COMMON *cm, aom_writer *w,
407
                                        FRAME_COUNTS *counts, int probwt) {
408 409
  int k;
  for (k = 0; k < TXFM_PARTITION_CONTEXTS; ++k)
Yaowu Xu's avatar
Yaowu Xu committed
410
    av1_cond_prob_diff_update(w, &cm->fc->txfm_partition_prob[k],
411
                              counts->txfm_partition[k], probwt);
412
}
413 414
#endif

Yaowu Xu's avatar
Yaowu Xu committed
415 416
static void write_selected_tx_size(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                                   aom_writer *w) {
417 418
  const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
  const BLOCK_SIZE bsize = mbmi->sb_type;
419 420
  // For sub8x8 blocks the tx_size symbol does not need to be sent
  if (bsize >= BLOCK_8X8) {
421 422 423 424 425 426
    const TX_SIZE tx_size = mbmi->tx_size;
    const int is_inter = is_inter_block(mbmi);
    const int tx_size_ctx = get_tx_size_context(xd);
    const int tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
                                     : intra_tx_size_cat_lookup[bsize];
    const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size];
427
    const int depth = tx_size_to_depth(coded_tx_size);
428 429

#if CONFIG_EXT_TX && CONFIG_RECT_TX
430
    assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(xd, mbmi)));
431 432 433 434
    assert(
        IMPLIES(is_rect_tx(tx_size), tx_size == max_txsize_rect_lookup[bsize]));
#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX

Yaowu Xu's avatar
Yaowu Xu committed
435 436
    av1_write_token(w, av1_tx_size_tree[tx_size_cat],
                    cm->fc->tx_size_probs[tx_size_cat][tx_size_ctx],
437
                    &tx_size_encodings[tx_size_cat][depth]);
Jingning Han's avatar
Jingning Han committed
438 439 440
  }
}

441
#if CONFIG_REF_MV
Yaowu Xu's avatar
Yaowu Xu committed
442
static void update_inter_mode_probs(AV1_COMMON *cm, aom_writer *w,
443 444
                                    FRAME_COUNTS *counts) {
  int i;
445 446 447 448 449
#if CONFIG_TILE_GROUPS
  const int probwt = cm->num_tg;
#else
  const int probwt = 1;
#endif
450
  for (i = 0; i < NEWMV_MODE_CONTEXTS; ++i)
451 452
    av1_cond_prob_diff_update(w, &cm->fc->newmv_prob[i], counts->newmv_mode[i],
                              probwt);
453
  for (i = 0; i < ZEROMV_MODE_CONTEXTS; ++i)
Yaowu Xu's avatar
Yaowu Xu committed
454
    av1_cond_prob_diff_update(w, &cm->fc->zeromv_prob[i],
455
                              counts->zeromv_mode[i], probwt);
456
  for (i = 0; i < REFMV_MODE_CONTEXTS; ++i)
457 458
    av1_cond_prob_diff_update(w, &cm->fc->refmv_prob[i], counts->refmv_mode[i],
                              probwt);
459
  for (i = 0; i < DRL_MODE_CONTEXTS; ++i)
460 461
    av1_cond_prob_diff_update(w, &cm->fc->drl_prob[i], counts->drl_mode[i],
                              probwt);
Yue Chen's avatar
Yue Chen committed
462
#if CONFIG_EXT_INTER
463 464
  av1_cond_prob_diff_update(w, &cm->fc->new2mv_prob, counts->new2mv_mode,
                            probwt);
Yue Chen's avatar
Yue Chen committed
465
#endif  // CONFIG_EXT_INTER
466 467 468
}
#endif

469
#if CONFIG_EXT_INTER
470 471
static void update_inter_compound_mode_probs(AV1_COMMON *cm, int probwt,
                                             aom_writer *w) {
Yaowu Xu's avatar
Yaowu Xu committed
472 473
  const int savings_thresh = av1_cost_one(GROUP_DIFF_UPDATE_PROB) -
                             av1_cost_zero(GROUP_DIFF_UPDATE_PROB);
474 475 476 477
  int i;
  int savings = 0;
  int do_update = 0;
  for (i = 0; i < INTER_MODE_CONTEXTS; ++i) {
478
    savings += prob_diff_update_savings(
Yaowu Xu's avatar
Yaowu Xu committed
479
        av1_inter_compound_mode_tree, cm->fc->inter_compound_mode_probs[i],
480
        cm->counts.inter_compound_mode[i], INTER_COMPOUND_MODES, probwt);
481 482
  }
  do_update = savings > savings_thresh;
Yaowu Xu's avatar
Yaowu Xu committed
483
  aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
484 485
  if (do_update) {
    for (i = 0; i < INTER_MODE_CONTEXTS; ++i) {
486
      prob_diff_update(
Yaowu Xu's avatar
Yaowu Xu committed
487
          av1_inter_compound_mode_tree, cm->fc->inter_compound_mode_probs[i],
488
          cm->counts.inter_compound_mode[i], INTER_COMPOUND_MODES, probwt, w);
489 490 491 492 493
    }
  }
}
#endif  // CONFIG_EXT_INTER

Yaowu Xu's avatar
Yaowu Xu committed
494 495
static int write_skip(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                      int segment_id, const MODE_INFO *mi, aom_writer *w) {
Jingning Han's avatar
Jingning Han committed
496 497 498 499
  if (segfeature_active(&cm->seg, segment_id, SEG_LVL_SKIP)) {
    return 1;
  } else {
    const int skip = mi->mbmi.skip;
Yaowu Xu's avatar
Yaowu Xu committed
500
    aom_write(w, skip, av1_get_skip_prob(cm, xd));
Jingning Han's avatar
Jingning Han committed
501 502 503 504
    return skip;
  }
}

505
#if CONFIG_DELTA_Q
506 507
static void write_delta_qindex(const AV1_COMMON *cm, int delta_qindex,
                               aom_writer *w) {
508 509
  int sign = delta_qindex < 0;
  int abs = sign ? -delta_qindex : delta_qindex;
510 511 512 513 514 515 516 517 518 519 520 521 522 523
  int rem_bits, thr, i = 0;
  int smallval = abs < DELTA_Q_SMALL ? 1 : 0;

  while (i < DELTA_Q_SMALL && i <= abs) {
    int bit = (i < abs);
    aom_write(w, bit, cm->fc->delta_q_prob[i]);
    i++;
  }

  if (!smallval) {
    rem_bits = OD_ILOG_NZ(abs - 1) - 1;
    thr = (1 << rem_bits) + 1;
    aom_write_literal(w, rem_bits, 3);
    aom_write_literal(w, abs - thr, rem_bits);
524 525 526 527 528
  }
  if (abs > 0) {
    aom_write_bit(w, sign);
  }
}
529 530 531 532

static void update_delta_q_probs(AV1_COMMON *cm, aom_writer *w,
                                 FRAME_COUNTS *counts) {
  int k;
533 534 535 536 537
#if CONFIG_TILE_GROUPS
  const int probwt = cm->num_tg;
#else
  const int probwt = 1;
#endif
538
  for (k = 0; k < DELTA_Q_CONTEXTS; ++k) {
539 540
    av1_cond_prob_diff_update(w, &cm->fc->delta_q_prob[k], counts->delta_q[k],
                              probwt);
541 542
  }
}
543 544
#endif

Yaowu Xu's avatar
Yaowu Xu committed
545
static void update_skip_probs(AV1_COMMON *cm, aom_writer *w,
Jingning Han's avatar
Jingning Han committed
546 547
                              FRAME_COUNTS *counts) {
  int k;
548 549 550 551 552 553 554 555 556
#if CONFIG_TILE_GROUPS
  const int probwt = cm->num_tg;
#else
  const int probwt = 1;
#endif
  for (k = 0; k < SKIP_CONTEXTS; ++k) {
    av1_cond_prob_diff_update(w, &cm->fc->skip_probs[k], counts->skip[k],
                              probwt);
  }
Jingning Han's avatar
Jingning Han committed
557 558
}

559
#if !CONFIG_EC_ADAPT
Yaowu Xu's avatar
Yaowu Xu committed
560
static void update_switchable_interp_probs(AV1_COMMON *cm, aom_writer *w,
Jingning Han's avatar
Jingning Han committed
561 562
                                           FRAME_COUNTS *counts) {
  int j;
563
  for (j = 0; j < SWITCHABLE_FILTER_CONTEXTS; ++j) {
564 565 566 567 568 569 570 571
#if CONFIG_TILE_GROUPS
    const int probwt = cm->num_tg;
#else
    const int probwt = 1;
#endif
    prob_diff_update(
        av1_switchable_interp_tree, cm->fc->switchable_interp_prob[j],
        counts->switchable_interp[j], SWITCHABLE_FILTERS, probwt, w);
572
  }
Jingning Han's avatar
Jingning Han committed
573
}
574
#endif
Jingning Han's avatar
Jingning Han committed
575

576
#if CONFIG_EXT_TX
Yaowu Xu's avatar
Yaowu Xu committed
577 578 579
static void update_ext_tx_probs(AV1_COMMON *cm, aom_writer *w) {
  const int savings_thresh = av1_cost_one(GROUP_DIFF_UPDATE_PROB) -
                             av1_cost_zero(GROUP_DIFF_UPDATE_PROB);
580
  int i, j;
581
  int s;
582 583 584 585 586
#if CONFIG_TILE_GROUPS
  const int probwt = cm->num_tg;
#else
  const int probwt = 1;
#endif
587 588 589 590
  for (s = 1; s < EXT_TX_SETS_INTER; ++s) {
    int savings = 0;
    int do_update = 0;
    for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
591
      if (!use_inter_ext_tx_for_txsize[s][i]) continue;
592
      savings += prob_diff_update_savings(
Yaowu Xu's avatar
Yaowu Xu committed
593
          av1_ext_tx_inter_tree[s], cm->fc->inter_ext_tx_prob[s][i],
594
          cm->counts.inter_ext_tx[s][i], num_ext_tx_set_inter[s], probwt);
595 596
    }
    do_update = savings > savings_thresh;
Yaowu Xu's avatar
Yaowu Xu committed
597
    aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
598 599
    if (do_update) {
      for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
600
        if (!use_inter_ext_tx_for_txsize[s][i]) continue;
601
        prob_diff_update(
Yaowu Xu's avatar
Yaowu Xu committed
602
            av1_ext_tx_inter_tree[s], cm->fc->inter_ext_tx_prob[s][i],
603
            cm->counts.inter_ext_tx[s][i], num_ext_tx_set_inter[s], probwt, w);
604
      }
605 606
    }
  }
607

608 609 610 611
  for (s = 1; s < EXT_TX_SETS_INTRA; ++s) {
    int savings = 0;
    int do_update = 0;
    for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
612
      if (!use_intra_ext_tx_for_txsize[s][i]) continue;
613
      for (j = 0; j < INTRA_MODES; ++j)
614
        savings += prob_diff_update_savings(
Yaowu Xu's avatar
Yaowu Xu committed
615
            av1_ext_tx_intra_tree[s], cm->fc->intra_ext_tx_prob[s][i][j],
616
            cm->counts.intra_ext_tx[s][i][j], num_ext_tx_set_intra[s], probwt);
617 618
    }
    do_update = savings > savings_thresh;
Yaowu Xu's avatar
Yaowu Xu committed
619
    aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
620 621
    if (do_update) {
      for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
622
        if (!use_intra_ext_tx_for_txsize[s][i]) continue;
623
        for (j = 0; j < INTRA_MODES; ++j)
624 625 626 627
          prob_diff_update(av1_ext_tx_intra_tree[s],
                           cm->fc->intra_ext_tx_prob[s][i][j],
                           cm->counts.intra_ext_tx[s][i][j],
                           num_ext_tx_set_intra[s], probwt, w);
628 629
      }
    }
630
  }
631
}
Debargha Mukherjee's avatar
Debargha Mukherjee committed
632

633
#else
634
#if !CONFIG_EC_ADAPT
Yaowu Xu's avatar
Yaowu Xu committed
635 636 637
static void update_ext_tx_probs(AV1_COMMON *cm, aom_writer *w) {
  const int savings_thresh = av1_cost_one(GROUP_DIFF_UPDATE_PROB) -
                             av1_cost_zero(GROUP_DIFF_UPDATE_PROB);
638 639 640 641
  int i, j;

  int savings = 0;
  int do_update = 0;
642 643 644 645 646
#if CONFIG_TILE_GROUPS
  const int probwt = cm->num_tg;
#else
  const int probwt = 1;
#endif
647 648 649
  for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
    for (j = 0; j < TX_TYPES; ++j)
      savings += prob_diff_update_savings(
Yaowu Xu's avatar
Yaowu Xu committed
650
          av1_ext_tx_tree, cm->fc->intra_ext_tx_prob[i][j],
651
          cm->counts.intra_ext_tx[i][j], TX_TYPES, probwt);
652 653
  }
  do_update = savings > savings_thresh;
Yaowu Xu's avatar
Yaowu Xu committed
654
  aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
655 656
  if (do_update) {
    for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
657
      for (j = 0; j < TX_TYPES; ++j) {
Yaowu Xu's avatar
Yaowu Xu committed
658
        prob_diff_update(av1_ext_tx_tree, cm->fc->intra_ext_tx_prob[i][j],
659
                         cm->counts.intra_ext_tx[i][j], TX_TYPES, probwt, w);
660
      }
661 662
    }
  }
663

664 665
  savings = 0;
  for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
666
    savings +=
Yaowu Xu's avatar
Yaowu Xu committed
667
        prob_diff_update_savings(av1_ext_tx_tree, cm->fc->inter_ext_tx_prob[i],
668
                                 cm->counts.inter_ext_tx[i], TX_TYPES, probwt);
669 670
  }
  do_update = savings > savings_thresh;
Yaowu Xu's avatar
Yaowu Xu committed
671
  aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
672 673
  if (do_update) {
    for (i = TX_4X4; i < EXT_TX_SIZES; ++i) {
Yaowu Xu's avatar
Yaowu Xu committed
674
      prob_diff_update(av1_ext_tx_tree, cm->fc->inter_ext_tx_prob[i],
675
                       cm->counts.inter_ext_tx[i], TX_TYPES, probwt, w);
676 677 678
    }
  }
}
679
#endif  // CONFIG_EXT_TX
680
#endif
681
#if CONFIG_PALETTE
Yaowu Xu's avatar
Yaowu Xu committed
682
static void pack_palette_tokens(aom_writer *w, const TOKENEXTRA **tp, int n,
683
                                int num) {
hui su's avatar
hui su committed
684
  int i;
685
  const TOKENEXTRA *p = *tp;
hui su's avatar
hui su committed
686

687
  for (i = 0; i < num; ++i) {
Yaowu Xu's avatar
Yaowu Xu committed
688 689
    av1_write_token(w, av1_palette_color_tree[n - 2], p->context_tree,
                    &palette_color_encodings[n - 2][p->token]);
hui su's avatar
hui su committed
690 691 692 693 694
    ++p;
  }

  *tp = p;
}
695
#endif  // CONFIG_PALETTE
696
#if !CONFIG_PVQ
697
#if CONFIG_SUPERTX
698
static void update_supertx_probs(AV1_COMMON *cm, int probwt, aom_writer *w) {
Yaowu Xu's avatar
Yaowu Xu committed
699 700
  const int savings_thresh = av1_cost_one(GROUP_DIFF_UPDATE_PROB) -
                             av1_cost_zero(GROUP_DIFF_UPDATE_PROB);
701 702 703 704 705
  int i, j;
  int savings = 0;
  int do_update = 0;
  for (i = 0; i < PARTITION_SUPERTX_CONTEXTS; ++i) {
    for (j = 1; j < TX_SIZES; ++j) {
706 707
      savings += av1_cond_prob_diff_update_savings(
          &cm->fc->supertx_prob[i][j], cm->counts.supertx[i][j], probwt);
708 709 710
    }
  }
  do_update = savings > savings_thresh;
Yaowu Xu's avatar
Yaowu Xu committed
711
  aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
712 713 714
  if (do_update) {
    for (i = 0; i < PARTITION_SUPERTX_CONTEXTS; ++i) {
      for (j = 1; j < TX_SIZES; ++j) {
Yaowu Xu's avatar
Yaowu Xu committed
715
        av1_cond_prob_diff_update(w, &cm->fc->supertx_prob[i][j],
716
                                  cm->counts.supertx[i][j], probwt);
717 718 719 720 721 722
      }
    }
  }
}
#endif  // CONFIG_SUPERTX

Yaowu Xu's avatar
Yaowu Xu committed
723
static void pack_mb_tokens(aom_writer *w, const TOKENEXTRA **tp,
724
                           const TOKENEXTRA *const stop,
725 726
                           aom_bit_depth_t bit_depth, const TX_SIZE tx_size,
                           TOKEN_STATS *token_stats) {
727
  const TOKENEXTRA *p = *tp;
728 729
#if CONFIG_VAR_TX
  int count = 0;
730
  const int seg_eob = tx_size_2d[tx_size];
731
#endif
Alex Converse's avatar
Alex Converse committed
732 733 734 735 736 737 738 739 740
#if CONFIG_AOM_HIGHBITDEPTH
  const av1_extra_bit *const extra_bits_table =
      (bit_depth == AOM_BITS_12)
          ? av1_extra_bits_high12
          : (bit_depth == AOM_BITS_10) ? av1_extra_bits_high10 : av1_extra_bits;
#else
  const av1_extra_bit *const extra_bits_table = av1_extra_bits;
  (void)bit_depth;
#endif  // CONFIG_AOM_HIGHBITDEPTH
Jingning Han's avatar
Jingning Han committed
741 742

  while (p < stop && p->token != EOSB_TOKEN) {
743 744
    const int token = p->token;
    aom_tree_index index = 0;
745
#if !CONFIG_EC_MULTISYMBOL
746 747 748
    const struct av1_token *const coef_encoding = &av1_coef_encodings[token];
    int coef_value = coef_encoding->value;
    int coef_length = coef_encoding->len;
749
#endif  // !CONFIG_EC_MULTISYMBOL
Alex Converse's avatar
Alex Converse committed
750
    const av1_extra_bit *const extra_bits = &extra_bits_table[token];
Jingning Han's avatar
Jingning Han committed
751

752
#if CONFIG_EC_MULTISYMBOL
Alex Converse's avatar
Alex Converse committed
753
    /* skip one or two nodes */
754 755
    if (!p->skip_eob_node)
      aom_write_record(w, token != EOB_TOKEN, p->context_tree[0], token_stats);
Alex Converse's avatar
Alex Converse committed
756

757
    if (token != EOB_TOKEN) {
758
      aom_write_record(w, token != ZERO_TOKEN, p->context_tree[1], token_stats);
759

760 761
      if (token != ZERO_TOKEN) {
        aom_write_symbol(w, token - ONE_TOKEN, *p->token_cdf,
762
                         CATEGORY6_TOKEN - ONE_TOKEN + 1);
763
      }
Alex Converse's avatar
Alex Converse committed
764 765
    }
#else
Jingning Han's avatar
Jingning Han committed
766
    /* skip one or two nodes */
767
    if (p->skip_eob_node)
768
      coef_length -= p->skip_eob_node;
769
    else
770
      aom_write_record(w, token != EOB_TOKEN, p->context_tree[0], token_stats);
771

772
    if (token != EOB_TOKEN) {
773
      aom_write_record(w, token != ZERO_TOKEN, p->context_tree[1], token_stats);
Jingning Han's avatar
Jingning Han committed
774

775
      if (token != ZERO_TOKEN) {
776 777
        aom_write_record(w, token != ONE_TOKEN, p->context_tree[2],
                         token_stats);
778

779 780
        if (token != ONE_TOKEN) {
          const int unconstrained_len = UNCONSTRAINED_NODES - p->skip_eob_node;
781 782 783 784
          aom_write_tree_record(
              w, av1_coef_con_tree,
              av1_pareto8_full[p->context_tree[PIVOT_NODE] - 1], coef_value,
              coef_length - unconstrained_len, 0, token_stats);
785 786
        }
      }
Jingning Han's avatar
Jingning Han committed
787
    }
788
#endif  // CONFIG_EC_MULTISYMBOL
Jingning Han's avatar
Jingning Han committed
789

Alex Converse's avatar
Alex Converse committed
790 791 792 793 794 795
    if (extra_bits->base_val) {
      const int bit_string = p->extra;
      const int bit_string_length = extra_bits->len;  // Length of extra bits to
                                                      // be written excluding
                                                      // the sign bit.
      int skip_bits = (extra_bits->base_val == CAT6_MIN_VAL)
796
                          ? TX_SIZES - 1 - txsize_sqr_up_map[tx_size]
Debargha Mukherjee's avatar
Debargha Mukherjee committed
797
                          : 0;