bitstream.c 177 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4
5
6
7
8
9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10
11
12
13
 */

#include <assert.h>
#include <limits.h>
14
#include <stdio.h>
Jingning Han's avatar
Jingning Han committed
15

Yaowu Xu's avatar
Yaowu Xu committed
16
#include "aom/aom_encoder.h"
17
#include "aom_dsp/bitwriter_buffer.h"
Yaowu Xu's avatar
Yaowu Xu committed
18
#include "aom_dsp/aom_dsp_common.h"
19
#include "aom_dsp/binary_codes_writer.h"
Yaowu Xu's avatar
Yaowu Xu committed
20
#include "aom_mem/aom_mem.h"
21
22
#include "aom_ports/mem_ops.h"
#include "aom_ports/system_state.h"
23
#if CONFIG_BITSTREAM_DEBUG
24
#include "aom_util/debug_util.h"
25
#endif  // CONFIG_BITSTREAM_DEBUG
Jingning Han's avatar
Jingning Han committed
26

27
#if CONFIG_CDEF
28
#include "av1/common/cdef.h"
29
#endif  // CONFIG_CDEF
30
31
32
33
#include "av1/common/entropy.h"
#include "av1/common/entropymode.h"
#include "av1/common/entropymv.h"
#include "av1/common/mvref_common.h"
34
#include "av1/common/odintrin.h"
35
36
#include "av1/common/pred_common.h"
#include "av1/common/reconinter.h"
hui su's avatar
hui su committed
37
38
39
#if CONFIG_EXT_INTRA
#include "av1/common/reconintra.h"
#endif  // CONFIG_EXT_INTRA
40
41
#include "av1/common/seg_common.h"
#include "av1/common/tile_common.h"
Jingning Han's avatar
Jingning Han committed
42

Angie Chiang's avatar
Angie Chiang committed
43
44
45
#if CONFIG_LV_MAP
#include "av1/encoder/encodetxb.h"
#endif  // CONFIG_LV_MAP
46
47
48
49
#include "av1/encoder/bitstream.h"
#include "av1/encoder/cost.h"
#include "av1/encoder/encodemv.h"
#include "av1/encoder/mcomp.h"
Urvang Joshi's avatar
Urvang Joshi committed
50
#if CONFIG_PALETTE_DELTA_ENCODING
hui su's avatar
hui su committed
51
#include "av1/encoder/palette.h"
Urvang Joshi's avatar
Urvang Joshi committed
52
#endif  // CONFIG_PALETTE_DELTA_ENCODING
53
54
55
#include "av1/encoder/segmentation.h"
#include "av1/encoder/subexp.h"
#include "av1/encoder/tokenize.h"
56
57
58
#if CONFIG_PVQ
#include "av1/encoder/pvq_encoder.h"
#endif
Jingning Han's avatar
Jingning Han committed
59

60
#define ENC_MISMATCH_DEBUG 0
61

62
#if CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF
63
64
static struct av1_token
    inter_singleref_comp_mode_encodings[INTER_SINGLEREF_COMP_MODES];
65
#endif  // CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF
66

Yaowu Xu's avatar
Yaowu Xu committed
67
static INLINE void write_uniform(aom_writer *w, int n, int v) {
68
69
  const int l = get_unsigned_bits(n);
  const int m = (1 << l) - n;
70
  if (l == 0) return;
hui su's avatar
hui su committed
71
  if (v < m) {
Yaowu Xu's avatar
Yaowu Xu committed
72
    aom_write_literal(w, v, l - 1);
hui su's avatar
hui su committed
73
  } else {
Yaowu Xu's avatar
Yaowu Xu committed
74
75
    aom_write_literal(w, m + ((v - m) >> 1), l - 1);
    aom_write_literal(w, (v - m) & 1, 1);
hui su's avatar
hui su committed
76
77
  }
}
Jingning Han's avatar
Jingning Han committed
78

79
#if CONFIG_EXT_TX
Yaowu Xu's avatar
Yaowu Xu committed
80
81
static struct av1_token ext_tx_inter_encodings[EXT_TX_SETS_INTER][TX_TYPES];
static struct av1_token ext_tx_intra_encodings[EXT_TX_SETS_INTRA][TX_TYPES];
82
#else
Yaowu Xu's avatar
Yaowu Xu committed
83
static struct av1_token ext_tx_encodings[TX_TYPES];
84
#endif  // CONFIG_EXT_TX
85
#if CONFIG_EXT_INTRA
hui su's avatar
hui su committed
86
#if CONFIG_INTRA_INTERP
Yaowu Xu's avatar
Yaowu Xu committed
87
static struct av1_token intra_filter_encodings[INTRA_FILTERS];
hui su's avatar
hui su committed
88
#endif  // CONFIG_INTRA_INTERP
89
#endif  // CONFIG_EXT_INTRA
90
#if CONFIG_EXT_INTER
91
#if CONFIG_INTERINTRA
Yaowu Xu's avatar
Yaowu Xu committed
92
static struct av1_token interintra_mode_encodings[INTERINTRA_MODES];
93
#endif
94
#if CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE
95
static struct av1_token compound_type_encodings[COMPOUND_TYPES];
96
#endif  // CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE
97
#endif  // CONFIG_EXT_INTER
Yue Chen's avatar
Yue Chen committed
98
#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
99
100
101
#if CONFIG_NCOBMC_ADAPT_WEIGHT
static struct av1_token ncobmc_mode_encodings[MAX_NCOBMC_MODES];
#endif
Yue Chen's avatar
Yue Chen committed
102
#endif  // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
103
104
#if CONFIG_LOOP_RESTORATION
static struct av1_token switchable_restore_encodings[RESTORE_SWITCHABLE_TYPES];
105
106
107
108
static void loop_restoration_write_sb_coeffs(const AV1_COMMON *const cm,
                                             MACROBLOCKD *xd,
                                             aom_writer *const w, int plane,
                                             int rtile_idx);
109
#endif  // CONFIG_LOOP_RESTORATION
110
111
112
static void write_uncompressed_header(AV1_COMP *cpi,
                                      struct aom_write_bit_buffer *wb);
static uint32_t write_compressed_header(AV1_COMP *cpi, uint8_t *data);
113
114
115
116
117
static int remux_tiles(const AV1_COMMON *const cm, uint8_t *dst,
                       const uint32_t data_size, const uint32_t max_tile_size,
                       const uint32_t max_tile_col_size,
                       int *const tile_size_bytes,
                       int *const tile_col_size_bytes);
118

Yaowu Xu's avatar
Yaowu Xu committed
119
void av1_encode_token_init(void) {
120
#if CONFIG_EXT_TX
121
  int s;
122
#endif  // CONFIG_EXT_TX
123
#if CONFIG_EXT_TX
124
  for (s = 1; s < EXT_TX_SETS_INTER; ++s) {
Yaowu Xu's avatar
Yaowu Xu committed
125
    av1_tokens_from_tree(ext_tx_inter_encodings[s], av1_ext_tx_inter_tree[s]);
126
127
  }
  for (s = 1; s < EXT_TX_SETS_INTRA; ++s) {
Yaowu Xu's avatar
Yaowu Xu committed
128
    av1_tokens_from_tree(ext_tx_intra_encodings[s], av1_ext_tx_intra_tree[s]);
129
  }
130
#else
Yaowu Xu's avatar
Yaowu Xu committed
131
  av1_tokens_from_tree(ext_tx_encodings, av1_ext_tx_tree);
132
#endif  // CONFIG_EXT_TX
133

134
#if CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP
Yaowu Xu's avatar
Yaowu Xu committed
135
  av1_tokens_from_tree(intra_filter_encodings, av1_intra_filter_tree);
136
#endif  // CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP
137
#if CONFIG_EXT_INTER
138
#if CONFIG_INTERINTRA
Yaowu Xu's avatar
Yaowu Xu committed
139
  av1_tokens_from_tree(interintra_mode_encodings, av1_interintra_mode_tree);
140
#endif  // CONFIG_INTERINTRA
141
142
143
144
#if CONFIG_COMPOUND_SINGLEREF
  av1_tokens_from_tree(inter_singleref_comp_mode_encodings,
                       av1_inter_singleref_comp_mode_tree);
#endif  // CONFIG_COMPOUND_SINGLEREF
145
#if CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE
146
  av1_tokens_from_tree(compound_type_encodings, av1_compound_type_tree);
147
#endif  // CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE
148
#endif  // CONFIG_EXT_INTER
Yue Chen's avatar
Yue Chen committed
149
#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
150
151
152
#if CONFIG_NCOBMC_ADAPT_WEIGHT
  av1_tokens_from_tree(ncobmc_mode_encodings, av1_ncobmc_mode_tree);
#endif
Yue Chen's avatar
Yue Chen committed
153
#endif  // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
154
155
156
157
#if CONFIG_LOOP_RESTORATION
  av1_tokens_from_tree(switchable_restore_encodings,
                       av1_switchable_restore_tree);
#endif  // CONFIG_LOOP_RESTORATION
158

159
  /* This hack is necessary when CONFIG_DUAL_FILTER is enabled because the five
160
161
162
      SWITCHABLE_FILTERS are not consecutive, e.g., 0, 1, 2, 3, 4, when doing
      an in-order traversal of the av1_switchable_interp_tree structure. */
  av1_indices_from_tree(av1_switchable_interp_ind, av1_switchable_interp_inv,
Jingning Han's avatar
Jingning Han committed
163
                        av1_switchable_interp_tree);
164
165
166
/* This hack is necessary because the four TX_TYPES are not consecutive,
    e.g., 0, 1, 2, 3, when doing an in-order traversal of the av1_ext_tx_tree
    structure. */
167
168
169
#if CONFIG_EXT_TX
  for (s = 1; s < EXT_TX_SETS_INTRA; ++s)
    av1_indices_from_tree(av1_ext_tx_intra_ind[s], av1_ext_tx_intra_inv[s],
Jingning Han's avatar
Jingning Han committed
170
                          av1_ext_tx_intra_tree[s]);
171
172
  for (s = 1; s < EXT_TX_SETS_INTER; ++s)
    av1_indices_from_tree(av1_ext_tx_inter_ind[s], av1_ext_tx_inter_inv[s],
Jingning Han's avatar
Jingning Han committed
173
                          av1_ext_tx_inter_tree[s]);
174
#else
Jingning Han's avatar
Jingning Han committed
175
  av1_indices_from_tree(av1_ext_tx_ind, av1_ext_tx_inv, av1_ext_tx_tree);
David Barker's avatar
David Barker committed
176
#endif
177
178
}

Jingning Han's avatar
Jingning Han committed
179
180
181
182
static void write_intra_mode_kf(const AV1_COMMON *cm, FRAME_CONTEXT *frame_ctx,
                                const MODE_INFO *mi, const MODE_INFO *above_mi,
                                const MODE_INFO *left_mi, int block,
                                PREDICTION_MODE mode, aom_writer *w) {
Alex Converse's avatar
Alex Converse committed
183
184
185
#if CONFIG_INTRABC
  assert(!is_intrabc_block(&mi->mbmi));
#endif  // CONFIG_INTRABC
Jingning Han's avatar
Jingning Han committed
186
187
188
189
190
  aom_write_symbol(w, av1_intra_mode_ind[mode],
                   get_y_mode_cdf(frame_ctx, mi, above_mi, left_mi, block),
                   INTRA_MODES);
  (void)cm;
}
Jingning Han's avatar
Jingning Han committed
191

192
static void write_inter_mode(aom_writer *w, PREDICTION_MODE mode,
193
                             FRAME_CONTEXT *ec_ctx, const int16_t mode_ctx) {
194
  const int16_t newmv_ctx = mode_ctx & NEWMV_CTX_MASK;
Yue Chen's avatar
Yue Chen committed
195

196
197
198
199
200
#if CONFIG_NEW_MULTISYMBOL
  aom_write_symbol(w, mode != NEWMV, ec_ctx->newmv_cdf[newmv_ctx], 2);
#else
  aom_write(w, mode != NEWMV, ec_ctx->newmv_prob[newmv_ctx]);
#endif
201

Jingning Han's avatar
Jingning Han committed
202
  if (mode != NEWMV) {
203
204
205
206
207
    if (mode_ctx & (1 << ALL_ZERO_FLAG_OFFSET)) {
      assert(mode == ZEROMV);
      return;
    }

208
209
210
211
212
213
    const int16_t zeromv_ctx = (mode_ctx >> ZEROMV_OFFSET) & ZEROMV_CTX_MASK;
#if CONFIG_NEW_MULTISYMBOL
    aom_write_symbol(w, mode != ZEROMV, ec_ctx->zeromv_cdf[zeromv_ctx], 2);
#else
    aom_write(w, mode != ZEROMV, ec_ctx->zeromv_prob[zeromv_ctx]);
#endif
214
215

    if (mode != ZEROMV) {
216
217
      int16_t refmv_ctx = (mode_ctx >> REFMV_OFFSET) & REFMV_CTX_MASK;

218
219
220
      if (mode_ctx & (1 << SKIP_NEARESTMV_OFFSET)) refmv_ctx = 6;
      if (mode_ctx & (1 << SKIP_NEARMV_OFFSET)) refmv_ctx = 7;
      if (mode_ctx & (1 << SKIP_NEARESTMV_SUB8X8_OFFSET)) refmv_ctx = 8;
221
222
223
224
225
#if CONFIG_NEW_MULTISYMBOL
      aom_write_symbol(w, mode != NEARESTMV, ec_ctx->refmv_cdf[refmv_ctx], 2);
#else
      aom_write(w, mode != NEARESTMV, ec_ctx->refmv_prob[refmv_ctx]);
#endif
226
227
    }
  }
Jingning Han's avatar
Jingning Han committed
228
229
}

230
static void write_drl_idx(FRAME_CONTEXT *ec_ctx, const MB_MODE_INFO *mbmi,
Yaowu Xu's avatar
Yaowu Xu committed
231
232
                          const MB_MODE_INFO_EXT *mbmi_ext, aom_writer *w) {
  uint8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame);
233
234
235

  assert(mbmi->ref_mv_idx < 3);

236
#if CONFIG_EXT_INTER
237
238
239
240
#if CONFIG_COMPOUND_SINGLEREF
  if (mbmi->mode == NEWMV || mbmi->mode == NEW_NEWMV ||
      mbmi->mode == SR_NEW_NEWMV) {
#else   // !CONFIG_COMPOUND_SINGLEREF
241
  if (mbmi->mode == NEWMV || mbmi->mode == NEW_NEWMV) {
242
243
#endif  // CONFIG_COMPOUND_SINGLEREF
#else   // !CONFIG_EXT_INTER
244
  if (mbmi->mode == NEWMV) {
245
#endif  // CONFIG_EXT_INTER
246
247
248
249
    int idx;
    for (idx = 0; idx < 2; ++idx) {
      if (mbmi_ext->ref_mv_count[ref_frame_type] > idx + 1) {
        uint8_t drl_ctx =
Yaowu Xu's avatar
Yaowu Xu committed
250
            av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx);
251

252
253
254
255
256
257
#if CONFIG_NEW_MULTISYMBOL
        aom_write_symbol(w, mbmi->ref_mv_idx != idx, ec_ctx->drl_cdf[drl_ctx],
                         2);
#else
        aom_write(w, mbmi->ref_mv_idx != idx, ec_ctx->drl_prob[drl_ctx]);
#endif
258
        if (mbmi->ref_mv_idx == idx) return;
259
      }
260
261
262
263
    }
    return;
  }

David Barker's avatar
David Barker committed
264
  if (have_nearmv_in_inter_mode(mbmi->mode)) {
265
266
267
268
269
    int idx;
    // TODO(jingning): Temporary solution to compensate the NEARESTMV offset.
    for (idx = 1; idx < 3; ++idx) {
      if (mbmi_ext->ref_mv_count[ref_frame_type] > idx + 1) {
        uint8_t drl_ctx =
Yaowu Xu's avatar
Yaowu Xu committed
270
            av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx);
271
272
273
274
275
276
#if CONFIG_NEW_MULTISYMBOL
        aom_write_symbol(w, mbmi->ref_mv_idx != (idx - 1),
                         ec_ctx->drl_cdf[drl_ctx], 2);
#else
        aom_write(w, mbmi->ref_mv_idx != (idx - 1), ec_ctx->drl_prob[drl_ctx]);
#endif
277
        if (mbmi->ref_mv_idx == (idx - 1)) return;
278
      }
279
    }
280
    return;
281
282
283
  }
}

284
#if CONFIG_EXT_INTER
285
286
static void write_inter_compound_mode(AV1_COMMON *cm, MACROBLOCKD *xd,
                                      aom_writer *w, PREDICTION_MODE mode,
287
                                      const int16_t mode_ctx) {
288
289
290
291
292
  assert(is_inter_compound_mode(mode));
  (void)cm;
  aom_write_symbol(w, INTER_COMPOUND_OFFSET(mode),
                   xd->tile_ctx->inter_compound_mode_cdf[mode_ctx],
                   INTER_COMPOUND_MODES);
293
}
294
295

#if CONFIG_COMPOUND_SINGLEREF
296
static void write_inter_singleref_comp_mode(MACROBLOCKD *xd, aom_writer *w,
297
298
299
                                            PREDICTION_MODE mode,
                                            const int16_t mode_ctx) {
  assert(is_inter_singleref_comp_mode(mode));
300
301
  aom_cdf_prob *const inter_singleref_comp_cdf =
      xd->tile_ctx->inter_singleref_comp_mode_cdf[mode_ctx];
302

303
304
  aom_write_symbol(w, INTER_SINGLEREF_COMP_OFFSET(mode),
                   inter_singleref_comp_cdf, INTER_SINGLEREF_COMP_MODES);
305
306
}
#endif  // CONFIG_COMPOUND_SINGLEREF
307
308
#endif  // CONFIG_EXT_INTER

Yaowu Xu's avatar
Yaowu Xu committed
309
static void encode_unsigned_max(struct aom_write_bit_buffer *wb, int data,
310
                                int max) {
Yaowu Xu's avatar
Yaowu Xu committed
311
  aom_wb_write_literal(wb, data, get_unsigned_bits(max));
Jingning Han's avatar
Jingning Han committed
312
313
}

314
#if CONFIG_NCOBMC_ADAPT_WEIGHT
Yaowu Xu's avatar
Yaowu Xu committed
315
316
static void prob_diff_update(const aom_tree_index *tree,
                             aom_prob probs[/*n - 1*/],
317
                             const unsigned int counts[/* n */], int n,
318
                             int probwt, aom_writer *w) {
Jingning Han's avatar
Jingning Han committed
319
320
321
322
323
324
  int i;
  unsigned int branch_ct[32][2];

  // Assuming max number of probabilities <= 32
  assert(n <= 32);

Yaowu Xu's avatar
Yaowu Xu committed
325
  av1_tree_probs_from_distribution(tree, branch_ct, counts);
Jingning Han's avatar
Jingning Han committed
326
  for (i = 0; i < n - 1; ++i)
327
    av1_cond_prob_diff_update(w, &probs[i], branch_ct[i], probwt);
Jingning Han's avatar
Jingning Han committed
328
}
329
330
#endif

331
#if CONFIG_VAR_TX
332
static void write_tx_size_vartx(const AV1_COMMON *cm, MACROBLOCKD *xd,
333
                                const MB_MODE_INFO *mbmi, TX_SIZE tx_size,
334
335
                                int depth, int blk_row, int blk_col,
                                aom_writer *w) {
336
337
338
339
#if CONFIG_NEW_MULTISYMBOL
  FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
  (void)cm;
#endif
340
341
  const int tx_row = blk_row >> 1;
  const int tx_col = blk_col >> 1;
342
343
344
  const int max_blocks_high = max_block_high(xd, mbmi->sb_type, 0);
  const int max_blocks_wide = max_block_wide(xd, mbmi->sb_type, 0);

345
346
  int ctx = txfm_partition_context(xd->above_txfm_context + blk_col,
                                   xd->left_txfm_context + blk_row,
347
                                   mbmi->sb_type, tx_size);
348

349
  if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
350

Jingning Han's avatar
Jingning Han committed
351
  if (depth == MAX_VARTX_DEPTH) {
352
353
    txfm_partition_update(xd->above_txfm_context + blk_col,
                          xd->left_txfm_context + blk_row, tx_size, tx_size);
354
355
356
    return;
  }

Yue Chen's avatar
Yue Chen committed
357
358
359
360
#if CONFIG_RECT_TX_EXT
  if (tx_size == mbmi->inter_tx_size[tx_row][tx_col] ||
      mbmi->tx_size == quarter_txsize_lookup[mbmi->sb_type]) {
#else
361
  if (tx_size == mbmi->inter_tx_size[tx_row][tx_col]) {
Yue Chen's avatar
Yue Chen committed
362
#endif
363
364
365
#if CONFIG_NEW_MULTISYMBOL
    aom_write_symbol(w, 0, ec_ctx->txfm_partition_cdf[ctx], 2);
#else
Yaowu Xu's avatar
Yaowu Xu committed
366
    aom_write(w, 0, cm->fc->txfm_partition_prob[ctx]);
367
368
#endif

369
370
    txfm_partition_update(xd->above_txfm_context + blk_col,
                          xd->left_txfm_context + blk_row, tx_size, tx_size);
Yue Chen's avatar
Yue Chen committed
371
    // TODO(yuec): set correct txfm partition update for qttx
372
  } else {
373
374
    const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
    const int bsl = tx_size_wide_unit[sub_txs];
375
    int i;
376

377
378
379
#if CONFIG_NEW_MULTISYMBOL
    aom_write_symbol(w, 1, ec_ctx->txfm_partition_cdf[ctx], 2);
#else
Yaowu Xu's avatar
Yaowu Xu committed
380
    aom_write(w, 1, cm->fc->txfm_partition_prob[ctx]);
381
#endif
382

David Barker's avatar
David Barker committed
383
    if (sub_txs == TX_4X4) {
384
385
      txfm_partition_update(xd->above_txfm_context + blk_col,
                            xd->left_txfm_context + blk_row, sub_txs, tx_size);
386
      return;
387
    }
388
389
390

    assert(bsl > 0);
    for (i = 0; i < 4; ++i) {
391
392
393
394
      int offsetr = blk_row + (i >> 1) * bsl;
      int offsetc = blk_col + (i & 0x01) * bsl;
      write_tx_size_vartx(cm, xd, mbmi, sub_txs, depth + 1, offsetr, offsetc,
                          w);
395
396
397
    }
  }
}
398

399
#if !CONFIG_NEW_MULTISYMBOL
Yaowu Xu's avatar
Yaowu Xu committed
400
static void update_txfm_partition_probs(AV1_COMMON *cm, aom_writer *w,
401
                                        FRAME_COUNTS *counts, int probwt) {
402
403
  int k;
  for (k = 0; k < TXFM_PARTITION_CONTEXTS; ++k)
Yaowu Xu's avatar
Yaowu Xu committed
404
    av1_cond_prob_diff_update(w, &cm->fc->txfm_partition_prob[k],
405
                              counts->txfm_partition[k], probwt);
406
}
407
#endif  // CONFIG_NEW_MULTISYMBOL
David Barker's avatar
David Barker committed
408
#endif  // CONFIG_VAR_TX
409

Yaowu Xu's avatar
Yaowu Xu committed
410
411
static void write_selected_tx_size(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                                   aom_writer *w) {
412
413
  const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
  const BLOCK_SIZE bsize = mbmi->sb_type;
414
415
  FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
  (void)cm;
416
// For sub8x8 blocks the tx_size symbol does not need to be sent
417
#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX
418
419
  if (bsize > BLOCK_4X4) {
#else
420
  if (bsize >= BLOCK_8X8) {
421
#endif
422
423
424
425
426
427
    const TX_SIZE tx_size = mbmi->tx_size;
    const int is_inter = is_inter_block(mbmi);
    const int tx_size_ctx = get_tx_size_context(xd);
    const int tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
                                     : intra_tx_size_cat_lookup[bsize];
    const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size];
428
    const int depth = tx_size_to_depth(coded_tx_size);
429
#if CONFIG_EXT_TX && CONFIG_RECT_TX
430
    assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(xd, mbmi)));
431
432
#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX

433
    aom_write_symbol(w, depth, ec_ctx->tx_size_cdf[tx_size_cat][tx_size_ctx],
434
                     tx_size_cat + 2);
Yue Chen's avatar
Yue Chen committed
435
#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
436
437
438
    if (is_quarter_tx_allowed(xd, mbmi, is_inter) && tx_size != coded_tx_size)
      aom_write(w, tx_size == quarter_txsize_lookup[bsize],
                cm->fc->quarter_tx_size_prob);
Yue Chen's avatar
Yue Chen committed
439
#endif
Jingning Han's avatar
Jingning Han committed
440
441
442
  }
}

443
#if !CONFIG_NEW_MULTISYMBOL
Yaowu Xu's avatar
Yaowu Xu committed
444
static void update_inter_mode_probs(AV1_COMMON *cm, aom_writer *w,
445
446
                                    FRAME_COUNTS *counts) {
  int i;
447
  const int probwt = cm->num_tg;
448
  for (i = 0; i < NEWMV_MODE_CONTEXTS; ++i)
449
450
    av1_cond_prob_diff_update(w, &cm->fc->newmv_prob[i], counts->newmv_mode[i],
                              probwt);
451
  for (i = 0; i < ZEROMV_MODE_CONTEXTS; ++i)
Yaowu Xu's avatar
Yaowu Xu committed
452
    av1_cond_prob_diff_update(w, &cm->fc->zeromv_prob[i],
453
                              counts->zeromv_mode[i], probwt);
454
  for (i = 0; i < REFMV_MODE_CONTEXTS; ++i)
455
456
    av1_cond_prob_diff_update(w, &cm->fc->refmv_prob[i], counts->refmv_mode[i],
                              probwt);
457
  for (i = 0; i < DRL_MODE_CONTEXTS; ++i)
458
459
    av1_cond_prob_diff_update(w, &cm->fc->drl_prob[i], counts->drl_mode[i],
                              probwt);
460
}
461
#endif
462

Yaowu Xu's avatar
Yaowu Xu committed
463
464
static int write_skip(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                      int segment_id, const MODE_INFO *mi, aom_writer *w) {
Jingning Han's avatar
Jingning Han committed
465
466
467
468
  if (segfeature_active(&cm->seg, segment_id, SEG_LVL_SKIP)) {
    return 1;
  } else {
    const int skip = mi->mbmi.skip;
469
470
471
472
473
#if CONFIG_NEW_MULTISYMBOL
    FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
    const int ctx = av1_get_skip_context(xd);
    aom_write_symbol(w, skip, ec_ctx->skip_cdfs[ctx], 2);
#else
Yaowu Xu's avatar
Yaowu Xu committed
474
    aom_write(w, skip, av1_get_skip_prob(cm, xd));
475
#endif
Jingning Han's avatar
Jingning Han committed
476
477
478
479
    return skip;
  }
}

480
481
482
483
484
485
486
487
488
489
490
491
492
static void write_is_inter(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                           int segment_id, aom_writer *w, const int is_inter) {
  if (!segfeature_active(&cm->seg, segment_id, SEG_LVL_REF_FRAME)) {
#if CONFIG_NEW_MULTISYMBOL
    FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
    const int ctx = av1_get_intra_inter_context(xd);
    aom_write_symbol(w, is_inter, ec_ctx->intra_inter_cdf[ctx], 2);
#else
    aom_write(w, is_inter, av1_get_intra_inter_prob(cm, xd));
#endif
  }
}

Yue Chen's avatar
Yue Chen committed
493
#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
494
495
static void write_motion_mode(const AV1_COMMON *cm, MACROBLOCKD *xd,
                              const MODE_INFO *mi, aom_writer *w) {
496
  const MB_MODE_INFO *mbmi = &mi->mbmi;
497

498
499
500
501
502
503
#if !CONFIG_GLOBAL_MOTION
  // The cm parameter is only used with global_motion or with
  // motion_var and warped_motion. In other cases, explicitly ignore
  // it to avoid a compiler warning.
  (void)cm;
#endif
504
  MOTION_MODE last_motion_mode_allowed = motion_mode_allowed(
505
#if CONFIG_GLOBAL_MOTION
506
      0, cm->global_motion,
507
#endif  // CONFIG_GLOBAL_MOTION
508
509
510
#if CONFIG_WARPED_MOTION
      xd,
#endif
511
      mi);
Yue Chen's avatar
Yue Chen committed
512
513
514
  if (last_motion_mode_allowed == SIMPLE_TRANSLATION) return;
#if CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION
  if (last_motion_mode_allowed == OBMC_CAUSAL) {
515
516
517
518
#if CONFIG_NEW_MULTISYMBOL
    aom_write_symbol(w, mbmi->motion_mode == OBMC_CAUSAL,
                     xd->tile_ctx->obmc_cdf[mbmi->sb_type], 2);
#else
Yue Chen's avatar
Yue Chen committed
519
520
    aom_write(w, mbmi->motion_mode == OBMC_CAUSAL,
              cm->fc->obmc_prob[mbmi->sb_type]);
521
#endif
Yue Chen's avatar
Yue Chen committed
522
523
  } else {
#endif  // CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION
Thomas Davies's avatar
Thomas Davies committed
524
525
526
    aom_write_symbol(w, mbmi->motion_mode,
                     xd->tile_ctx->motion_mode_cdf[mbmi->sb_type],
                     MOTION_MODES);
Yue Chen's avatar
Yue Chen committed
527
528
529
530
#if CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION
  }
#endif  // CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION
}
531
532

#if CONFIG_NCOBMC_ADAPT_WEIGHT
533
static void write_ncobmc_mode(MACROBLOCKD *xd, const MODE_INFO *mi,
534
535
536
                              aom_writer *w) {
  const MB_MODE_INFO *mbmi = &mi->mbmi;
  ADAPT_OVERLAP_BLOCK ao_block = adapt_overlap_block_lookup[mbmi->sb_type];
537
  if (mbmi->motion_mode != NCOBMC_ADAPT_WEIGHT) return;
538

539
540
  aom_write_symbol(w, mbmi->ncobmc_mode[0],
                   xd->tile_ctx->ncobmc_mode_cdf[ao_block], MAX_NCOBMC_MODES);
541
  if (mi_size_wide[mbmi->sb_type] != mi_size_high[mbmi->sb_type]) {
542
543
    aom_write_symbol(w, mbmi->ncobmc_mode[1],
                     xd->tile_ctx->ncobmc_mode_cdf[ao_block], MAX_NCOBMC_MODES);
544
545
546
  }
}
#endif
Yue Chen's avatar
Yue Chen committed
547
548
#endif  // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION

549
#if CONFIG_DELTA_Q
Thomas Davies's avatar
Thomas Davies committed
550
551
static void write_delta_qindex(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                               int delta_qindex, aom_writer *w) {
552
553
  int sign = delta_qindex < 0;
  int abs = sign ? -delta_qindex : delta_qindex;
Thomas Davies's avatar
Thomas Davies committed
554
  int rem_bits, thr;
555
  int smallval = abs < DELTA_Q_SMALL ? 1 : 0;
Thomas Davies's avatar
Thomas Davies committed
556
557
  FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
  (void)cm;
558

Thomas Davies's avatar
Thomas Davies committed
559
560
  aom_write_symbol(w, AOMMIN(abs, DELTA_Q_SMALL), ec_ctx->delta_q_cdf,
                   DELTA_Q_PROBS + 1);
561
562
563
564
565
566

  if (!smallval) {
    rem_bits = OD_ILOG_NZ(abs - 1) - 1;
    thr = (1 << rem_bits) + 1;
    aom_write_literal(w, rem_bits, 3);
    aom_write_literal(w, abs - thr, rem_bits);
567
568
569
570
571
  }
  if (abs > 0) {
    aom_write_bit(w, sign);
  }
}
572

Fangwen Fu's avatar
Fangwen Fu committed
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
#if CONFIG_EXT_DELTA_Q
static void write_delta_lflevel(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                                int delta_lflevel, aom_writer *w) {
  int sign = delta_lflevel < 0;
  int abs = sign ? -delta_lflevel : delta_lflevel;
  int rem_bits, thr;
  int smallval = abs < DELTA_LF_SMALL ? 1 : 0;
  FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
  (void)cm;

  aom_write_symbol(w, AOMMIN(abs, DELTA_LF_SMALL), ec_ctx->delta_lf_cdf,
                   DELTA_LF_PROBS + 1);

  if (!smallval) {
    rem_bits = OD_ILOG_NZ(abs - 1) - 1;
    thr = (1 << rem_bits) + 1;
    aom_write_literal(w, rem_bits, 3);
    aom_write_literal(w, abs - thr, rem_bits);
  }
  if (abs > 0) {
    aom_write_bit(w, sign);
  }
}
#endif  // CONFIG_EXT_DELTA_Q
Thomas Davies's avatar
Thomas Davies committed
597
#endif  // CONFIG_DELTA_Q
598

599
#if !CONFIG_NEW_MULTISYMBOL
Yaowu Xu's avatar
Yaowu Xu committed
600
static void update_skip_probs(AV1_COMMON *cm, aom_writer *w,
Jingning Han's avatar
Jingning Han committed
601
602
                              FRAME_COUNTS *counts) {
  int k;
603
604
605
606
607
  const int probwt = cm->num_tg;
  for (k = 0; k < SKIP_CONTEXTS; ++k) {
    av1_cond_prob_diff_update(w, &cm->fc->skip_probs[k], counts->skip[k],
                              probwt);
  }
Jingning Han's avatar
Jingning Han committed
608
}
609
#endif
Jingning Han's avatar
Jingning Han committed
610

Yaowu Xu's avatar
Yaowu Xu committed
611
static void pack_palette_tokens(aom_writer *w, const TOKENEXTRA **tp, int n,
612
                                int num) {
613
  const TOKENEXTRA *p = *tp;
614
615
616
617
  write_uniform(w, n, p->token);  // The first color index.
  ++p;
  --num;
  for (int i = 0; i < num; ++i) {
618
    aom_write_symbol(w, p->token, p->color_map_cdf, n);
hui su's avatar
hui su committed
619
620
621
622
    ++p;
  }
  *tp = p;
}
623

624
#if !CONFIG_PVQ
625
#if CONFIG_SUPERTX
626
static void update_supertx_probs(AV1_COMMON *cm, int probwt, aom_writer *w) {
Yaowu Xu's avatar
Yaowu Xu committed
627
628
  const int savings_thresh = av1_cost_one(GROUP_DIFF_UPDATE_PROB) -
                             av1_cost_zero(GROUP_DIFF_UPDATE_PROB);
629
630
631
632
  int i, j;
  int savings = 0;
  int do_update = 0;
  for (i = 0; i < PARTITION_SUPERTX_CONTEXTS; ++i) {
Jingning Han's avatar
Jingning Han committed
633
    for (j = TX_8X8; j < TX_SIZES; ++j) {
634
635
      savings += av1_cond_prob_diff_update_savings(
          &cm->fc->supertx_prob[i][j], cm->counts.supertx[i][j], probwt);
636
637
638
    }
  }
  do_update = savings > savings_thresh;
Yaowu Xu's avatar
Yaowu Xu committed
639
  aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
640
641
  if (do_update) {
    for (i = 0; i < PARTITION_SUPERTX_CONTEXTS; ++i) {
Jingning Han's avatar
Jingning Han committed
642
      for (j = TX_8X8; j < TX_SIZES; ++j) {
Yaowu Xu's avatar
Yaowu Xu committed
643
        av1_cond_prob_diff_update(w, &cm->fc->supertx_prob[i][j],
644
                                  cm->counts.supertx[i][j], probwt);
645
646
647
648
649
650
      }
    }
  }
}
#endif  // CONFIG_SUPERTX

651
#if !CONFIG_LV_MAP
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
#if CONFIG_NEW_MULTISYMBOL
static INLINE void write_coeff_extra(const aom_cdf_prob *const *cdf, int val,
                                     int n, aom_writer *w) {
  // Code the extra bits from LSB to MSB in groups of 4
  int i = 0;
  int count = 0;
  while (count < n) {
    const int size = AOMMIN(n - count, 4);
    const int mask = (1 << size) - 1;
    aom_write_cdf(w, val & mask, cdf[i++], 1 << size);
    val >>= size;
    count += size;
  }
}
#else
static INLINE void write_coeff_extra(const aom_prob *pb, int value,
                                     int num_bits, int skip_bits, aom_writer *w,
                                     TOKEN_STATS *token_stats) {
  // Code the extra bits from MSB to LSB 1 bit at a time
  int index;
  for (index = skip_bits; index < num_bits; ++index) {
    const int shift = num_bits - index - 1;
    const int bb = (value >> shift) & 1;
    aom_write_record(w, bb, pb[index], token_stats);
  }
}
678
#endif  // CONFIG_NEW_MULTISYMBOL
679

Yaowu Xu's avatar
Yaowu Xu committed
680
static void pack_mb_tokens(aom_writer *w, const TOKENEXTRA **tp,
681
                           const TOKENEXTRA *const stop,
682
683
                           aom_bit_depth_t bit_depth, const TX_SIZE tx_size,
                           TOKEN_STATS *token_stats) {
684
  const TOKENEXTRA *p = *tp;
Fangwen Fu's avatar
Fangwen Fu committed
685
#if CONFIG_VAR_TX
686
  int count = 0;
687
  const int seg_eob = tx_size_2d[tx_size];
688
#endif
Jingning Han's avatar
Jingning Han committed
689
690

  while (p < stop && p->token != EOSB_TOKEN) {
691
    const int token = p->token;
692
    const int eob_val = p->eob_val;
693
    if (token == BLOCK_Z_TOKEN) {
694
      aom_write_symbol(w, 0, *p->head_cdf, HEAD_TOKENS + 1);
695
      p++;
696
697
698
#if CONFIG_VAR_TX
      break;
#endif
699
700
      continue;
    }
Yaowu Xu's avatar
Yaowu Xu committed
701
702

    const av1_extra_bit *const extra_bits = &av1_extra_bits[token];
703
    if (eob_val == LAST_EOB) {
704
705
706
      // Just code a flag indicating whether the value is >1 or 1.
      aom_write_bit(w, token != ONE_TOKEN);
    } else {
707
      int comb_symb = 2 * AOMMIN(token, TWO_TOKEN) - eob_val + p->first_val;
708
      aom_write_symbol(w, comb_symb, *p->head_cdf, HEAD_TOKENS + p->first_val);
709
    }
710
    if (token > ONE_TOKEN) {
711
      aom_write_symbol(w, token - TWO_TOKEN, *p->tail_cdf, TAIL_TOKENS);
Alex Converse's avatar
Alex Converse committed
712
    }
713
714
715
716

    if (extra_bits->base_val) {
      const int bit_string = p->extra;
      const int bit_string_length = extra_bits->len;  // Length of extra bits to
717
      const int is_cat6 = (extra_bits->base_val == CAT6_MIN_VAL);
718
719
      // be written excluding
      // the sign bit.
720
      int skip_bits = is_cat6
721
722
723
                          ? (int)sizeof(av1_cat6_prob) -
                                av1_get_cat6_extrabits_size(tx_size, bit_depth)
                          : 0;
724

725
726
727
728
729
730
731
732
733
      assert(!(bit_string >> (bit_string_length - skip_bits + 1)));
      if (bit_string_length > 0)
#if CONFIG_NEW_MULTISYMBOL
        write_coeff_extra(extra_bits->cdf, bit_string >> 1,
                          bit_string_length - skip_bits, w);
#else
        write_coeff_extra(extra_bits->prob, bit_string >> 1, bit_string_length,
                          skip_bits, w, token_stats);
#endif
734
735
736
737
738

      aom_write_bit_record(w, bit_string & 1, token_stats);
    }
    ++p;

Fangwen Fu's avatar
Fangwen Fu committed
739
#if CONFIG_VAR_TX
740
    ++count;
741
    if (eob_val == EARLY_EOB || count == seg_eob) break;
742
743
744
745
746
#endif
  }

  *tp = p;
}
Angie Chiang's avatar
Angie Chiang committed
747
#endif  // !CONFIG_LV_MAP
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
#else   // !CONFIG_PVQ
static PVQ_INFO *get_pvq_block(PVQ_QUEUE *pvq_q) {
  PVQ_INFO *pvq;

  assert(pvq_q->curr_pos <= pvq_q->last_pos);
  assert(pvq_q->curr_pos < pvq_q->buf_len);

  pvq = pvq_q->buf + pvq_q->curr_pos;
  ++pvq_q->curr_pos;

  return pvq;
}

static void pack_pvq_tokens(aom_writer *w, MACROBLOCK *const x,
                            MACROBLOCKD *const xd, int plane, BLOCK_SIZE bsize,
                            const TX_SIZE tx_size) {
  PVQ_INFO *pvq;
  int idx, idy;
  const struct macroblockd_plane *const pd = &xd->plane[plane];
  od_adapt_ctx *adapt;
  int max_blocks_wide;
  int max_blocks_high;
  int step = (1 << tx_size);
771
772

#if CONFIG_CHROMA_SUB8X8
773
  const BLOCK_SIZE plane_bsize =
774
775
776
777
778
779
780
      AOMMAX(BLOCK_4X4, get_plane_block_size(bsize, pd));
#elif CONFIG_CB4X4
  const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd);
#else
  const BLOCK_SIZE plane_bsize =
      get_plane_block_size(AOMMAX(BLOCK_8X8, bsize), pd);
#endif
781

782
  adapt = x->daala_enc.state.adapt;
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800

  max_blocks_wide = max_block_wide(xd, plane_bsize, plane);
  max_blocks_high = max_block_high(xd, plane_bsize, plane);

  for (idy = 0; idy < max_blocks_high; idy += step) {
    for (idx = 0; idx < max_blocks_wide; idx += step) {
      const int is_keyframe = 0;
      const int encode_flip = 0;
      const int flip = 0;
      int i;
      const int has_dc_skip = 1;
      int *exg = &adapt->pvq.pvq_exg[plane][tx_size][0];
      int *ext = adapt->pvq.pvq_ext + tx_size * PVQ_MAX_PARTITIONS;
      generic_encoder *model = adapt->pvq.pvq_param_model;

      pvq = get_pvq_block(x->pvq_q);

      // encode block skip info
801
802
      aom_write_symbol(w, pvq->ac_dc_coded,
                       adapt->skip_cdf[2 * tx_size + (plane != 0)], 4);
803
804
805
806
807
808
809
810

      // AC coeffs coded?
      if (pvq->ac_dc_coded & AC_CODED) {
        assert(pvq->bs == tx_size);
        for (i = 0; i < pvq->nb_bands; i++) {
          if (i == 0 ||
              (!pvq->skip_rest && !(pvq->skip_dir & (1 << ((i - 1) % 3))))) {
            pvq_encode_partition(
811
812
                w, pvq->qg[i], pvq->theta[i], pvq->y + pvq->off[i],
                pvq->size[i], pvq->k[i], model, adapt, exg + i, ext + i,
813
814
815
816
817
818
                (plane != 0) * OD_TXSIZES * PVQ_MAX_PARTITIONS +
                    pvq->bs * PVQ_MAX_PARTITIONS + i,
                is_keyframe, i == 0 && (i < pvq->nb_bands - 1), pvq->skip_rest,
                encode_flip, flip);
          }
          if (i == 0 && !pvq->skip_rest && pvq->bs > 0) {
819
            aom_write_symbol(
820
821
822
                w, pvq->skip_dir,
                &adapt->pvq
                     .pvq_skip_dir_cdf[(plane != 0) + 2 * (pvq->bs - 1)][0],
823
                7);
824
825
826
827
828
829
          }
        }
      }
      // Encode residue of DC coeff, if exist.
      if (!has_dc_skip || (pvq->ac_dc_coded & DC_CODED)) {
        generic_encode(w, &adapt->model_dc[plane],
830
                       abs(pvq->dq_dc_residue) - has_dc_skip,
831
832
833
834
835
836
837
838
                       &adapt->ex_dc[plane][pvq->bs][0], 2);
      }
      if ((pvq->ac_dc_coded & DC_CODED)) {
        aom_write_bit(w, pvq->dq_dc_residue < 0);
      }
    }
  }  // for (idy = 0;
}
839
#endif  // !CONFIG_PVG
840

841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
#if CONFIG_VAR_TX && !CONFIG_COEF_INTERLEAVE
#if CONFIG_LV_MAP
static void pack_txb_tokens(aom_writer *w,
#if CONFIG_LV_MAP
                            AV1_COMMON *cm,
#endif  // CONFIG_LV_MAP
                            const TOKENEXTRA **tp,
                            const TOKENEXTRA *const tok_end,
#if CONFIG_PVQ || CONFIG_LV_MAP
                            MACROBLOCK *const x,
#endif
                            MACROBLOCKD *xd, MB_MODE_INFO *mbmi, int plane,
                            BLOCK_SIZE plane_bsize, aom_bit_depth_t bit_depth,
                            int block, int blk_row, int blk_col,
                            TX_SIZE tx_size, TOKEN_STATS *token_stats) {
  const struct macroblockd_plane *const pd = &xd->plane[plane];
  const BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
  const int tx_row = blk_row >> (1 - pd->subsampling_y);
  const int tx_col = blk_col >> (1 - pd->subsampling_x);
  TX_SIZE plane_tx_size;
  const int max_blocks_high = max_block_high(xd, plane_bsize, plane);
  const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane);

  if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;

  plane_tx_size =
      plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0]
            : mbmi->inter_tx_size[tx_row][tx_col];

  if (tx_size == plane_tx_size) {
    TOKEN_STATS tmp_token_stats;
    init_token_stats(&tmp_token_stats);

#if !CONFIG_PVQ
    tran_low_t *tcoeff = BLOCK_OFFSET(x->mbmi_ext->tcoeff[plane], block);
    uint16_t eob = x->mbmi_ext->eobs[plane][block];
    TXB_CTX txb_ctx = { x->mbmi_ext->txb_skip_ctx[plane][block],
                        x->mbmi_ext->dc_sign_ctx[plane][block] };
Jingning Han's avatar
Jingning Han committed
879
880
    av1_write_coeffs_txb(cm, xd, w, blk_row, blk_col, block, plane, tx_size,
                         tcoeff, eob, &txb_ctx);
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
#else
    pack_pvq_tokens(w, x, xd, plane, bsize, tx_size);
#endif
#if CONFIG_RD_DEBUG
    token_stats->txb_coeff_cost_map[blk_row][blk_col] = tmp_token_stats.cost;
    token_stats->cost += tmp_token_stats.cost;
#endif
  } else {
    const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
    const int bsl = tx_size_wide_unit[sub_txs];
    int i;

    assert(bsl > 0);

    for (i = 0; i < 4; ++i) {
      const int offsetr = blk_row + (i >> 1) * bsl;
      const int offsetc = blk_col + (i & 0x01) * bsl;
      const int step = tx_size_wide_unit[sub_txs] * tx_size_high_unit[sub_txs];

      if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;

      pack_txb_tokens(w,
#if CONFIG_LV_MAP
                      cm,
#endif
                      tp, tok_end,
#if CONFIG_PVQ || CONFIG_LV_MAP
                      x,
#endif
                      xd, mbmi, plane, plane_bsize, bit_depth, block, offsetr,
                      offsetc, sub_txs, token_stats);
      block += step;
    }
  }
}
#else  // CONFIG_LV_MAP
Yaowu Xu's avatar
Yaowu Xu committed
917
static void pack_txb_tokens(aom_writer *w, const TOKENEXTRA **tp,
918
919
920
921
922
                            const TOKENEXTRA *const tok_end,
#if CONFIG_PVQ
                            MACROBLOCK *const x,
#endif
                            MACROBLOCKD *xd, MB_MODE_INFO *mbmi, int plane,
Yaowu Xu's avatar
Yaowu Xu committed
923
                            BLOCK_SIZE plane_bsize, aom_bit_depth_t bit_depth,
924
                            int block, int blk_row, int blk_col,
925
                            TX_SIZE tx_size, TOKEN_STATS *token_stats) {
926
927
  const struct macroblockd_plane *const pd = &xd->plane[plane];
  const BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
928
929
  const int tx_row = blk_row >> (1 - pd->subsampling_y);
  const int tx_col = blk_col >> (1 - pd->subsampling_x);
Debargha Mukherjee's avatar
Debargha Mukherjee committed
930
  TX_SIZE plane_tx_size;
931
932
  const int max_blocks_high = max_block_high(xd, plane_bsize, plane);
  const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane);
933

934
  if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
935

936
937
938
  plane_tx_size =
      plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0]
            : mbmi->inter_tx_size[tx_row][tx_col];
Debargha Mukherjee's avatar
Debargha Mukherjee committed
939

940
  if (tx_size == plane_tx_size) {
941
942
    TOKEN_STATS tmp_token_stats;
    init_token_stats(&tmp_token_stats);
943
#if !CONFIG_PVQ
944
    pack_mb_tokens(w, tp, tok_end, bit_depth, tx_size, &tmp_token_stats);
945
946
947
#else
    pack_pvq_tokens(w, x, xd, plane, bsize, tx_size);
#endif
948
949
950
951
#if CONFIG_RD_DEBUG
    token_stats->txb_coeff_cost_map[blk_row][blk_col] = tmp_token_stats.cost;
    token_stats->cost += tmp_token_stats.cost;
#endif
952
  } else {
Yue Chen's avatar
Yue Chen committed
953
954
955
956
#if CONFIG_RECT_TX_EXT
    int is_qttx = plane_tx_size == quarter_txsize_lookup[plane_bsize];
    const TX_SIZE sub_txs = is_qttx ? plane_tx_size : sub_tx_size_map[tx_size];
#else
957
    const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
Yue Chen's avatar
Yue Chen committed
958
#endif
959
    const int bsl = tx_size_wide_unit[sub_txs];
960
961
962
963
964
    int i;

    assert(bsl > 0);

    for (i = 0; i < 4; ++i) {