bitstream.c 179 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4
5
6
7
8
9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10
11
12
13
 */

#include <assert.h>
#include <limits.h>
14
#include <stdio.h>
Jingning Han's avatar
Jingning Han committed
15

Yaowu Xu's avatar
Yaowu Xu committed
16
#include "aom/aom_encoder.h"
17
#include "aom_dsp/bitwriter_buffer.h"
Yaowu Xu's avatar
Yaowu Xu committed
18
#include "aom_dsp/aom_dsp_common.h"
19
#include "aom_dsp/binary_codes_writer.h"
Yaowu Xu's avatar
Yaowu Xu committed
20
#include "aom_mem/aom_mem.h"
21
22
#include "aom_ports/mem_ops.h"
#include "aom_ports/system_state.h"
23
#if CONFIG_BITSTREAM_DEBUG
24
#include "aom_util/debug_util.h"
25
#endif  // CONFIG_BITSTREAM_DEBUG
Jingning Han's avatar
Jingning Han committed
26

27
#if CONFIG_CDEF
28
#include "av1/common/cdef.h"
29
#endif  // CONFIG_CDEF
30
31
32
33
#include "av1/common/entropy.h"
#include "av1/common/entropymode.h"
#include "av1/common/entropymv.h"
#include "av1/common/mvref_common.h"
34
#include "av1/common/odintrin.h"
35
36
#include "av1/common/pred_common.h"
#include "av1/common/reconinter.h"
hui su's avatar
hui su committed
37
38
39
#if CONFIG_EXT_INTRA
#include "av1/common/reconintra.h"
#endif  // CONFIG_EXT_INTRA
40
41
#include "av1/common/seg_common.h"
#include "av1/common/tile_common.h"
Jingning Han's avatar
Jingning Han committed
42

Angie Chiang's avatar
Angie Chiang committed
43
44
45
#if CONFIG_LV_MAP
#include "av1/encoder/encodetxb.h"
#endif  // CONFIG_LV_MAP
46
47
48
49
#include "av1/encoder/bitstream.h"
#include "av1/encoder/cost.h"
#include "av1/encoder/encodemv.h"
#include "av1/encoder/mcomp.h"
Urvang Joshi's avatar
Urvang Joshi committed
50
#if CONFIG_PALETTE_DELTA_ENCODING
hui su's avatar
hui su committed
51
#include "av1/encoder/palette.h"
Urvang Joshi's avatar
Urvang Joshi committed
52
#endif  // CONFIG_PALETTE_DELTA_ENCODING
53
54
55
#include "av1/encoder/segmentation.h"
#include "av1/encoder/subexp.h"
#include "av1/encoder/tokenize.h"
56
57
58
#if CONFIG_PVQ
#include "av1/encoder/pvq_encoder.h"
#endif
Jingning Han's avatar
Jingning Han committed
59

60
#define ENC_MISMATCH_DEBUG 0
61

62
#if CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF
63
64
static struct av1_token
    inter_singleref_comp_mode_encodings[INTER_SINGLEREF_COMP_MODES];
65
#endif  // CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF
66

Yaowu Xu's avatar
Yaowu Xu committed
67
static INLINE void write_uniform(aom_writer *w, int n, int v) {
68
69
  const int l = get_unsigned_bits(n);
  const int m = (1 << l) - n;
70
  if (l == 0) return;
hui su's avatar
hui su committed
71
  if (v < m) {
Yaowu Xu's avatar
Yaowu Xu committed
72
    aom_write_literal(w, v, l - 1);
hui su's avatar
hui su committed
73
  } else {
Yaowu Xu's avatar
Yaowu Xu committed
74
75
    aom_write_literal(w, m + ((v - m) >> 1), l - 1);
    aom_write_literal(w, (v - m) & 1, 1);
hui su's avatar
hui su committed
76
77
  }
}
Jingning Han's avatar
Jingning Han committed
78

79
#if CONFIG_EXT_TX
Yaowu Xu's avatar
Yaowu Xu committed
80
81
static struct av1_token ext_tx_inter_encodings[EXT_TX_SETS_INTER][TX_TYPES];
static struct av1_token ext_tx_intra_encodings[EXT_TX_SETS_INTRA][TX_TYPES];
82
#else
Yaowu Xu's avatar
Yaowu Xu committed
83
static struct av1_token ext_tx_encodings[TX_TYPES];
84
#endif  // CONFIG_EXT_TX
85
#if CONFIG_EXT_INTRA
hui su's avatar
hui su committed
86
#if CONFIG_INTRA_INTERP
Yaowu Xu's avatar
Yaowu Xu committed
87
static struct av1_token intra_filter_encodings[INTRA_FILTERS];
hui su's avatar
hui su committed
88
#endif  // CONFIG_INTRA_INTERP
89
#endif  // CONFIG_EXT_INTRA
90
#if CONFIG_EXT_INTER
91
#if CONFIG_INTERINTRA
Yaowu Xu's avatar
Yaowu Xu committed
92
static struct av1_token interintra_mode_encodings[INTERINTRA_MODES];
93
#endif
94
#if CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE
95
static struct av1_token compound_type_encodings[COMPOUND_TYPES];
96
#endif  // CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE
97
#endif  // CONFIG_EXT_INTER
Yue Chen's avatar
Yue Chen committed
98
#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
99
100
101
#if CONFIG_NCOBMC_ADAPT_WEIGHT
static struct av1_token ncobmc_mode_encodings[MAX_NCOBMC_MODES];
#endif
Yue Chen's avatar
Yue Chen committed
102
#endif  // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
103
104
105
#if CONFIG_LOOP_RESTORATION
static struct av1_token switchable_restore_encodings[RESTORE_SWITCHABLE_TYPES];
#endif  // CONFIG_LOOP_RESTORATION
106
107
108
static void write_uncompressed_header(AV1_COMP *cpi,
                                      struct aom_write_bit_buffer *wb);
static uint32_t write_compressed_header(AV1_COMP *cpi, uint8_t *data);
109
110
111
112
113
static int remux_tiles(const AV1_COMMON *const cm, uint8_t *dst,
                       const uint32_t data_size, const uint32_t max_tile_size,
                       const uint32_t max_tile_col_size,
                       int *const tile_size_bytes,
                       int *const tile_col_size_bytes);
114

Yaowu Xu's avatar
Yaowu Xu committed
115
void av1_encode_token_init(void) {
116
#if CONFIG_EXT_TX
117
  int s;
118
#endif  // CONFIG_EXT_TX
119
#if CONFIG_EXT_TX
120
  for (s = 1; s < EXT_TX_SETS_INTER; ++s) {
Yaowu Xu's avatar
Yaowu Xu committed
121
    av1_tokens_from_tree(ext_tx_inter_encodings[s], av1_ext_tx_inter_tree[s]);
122
123
  }
  for (s = 1; s < EXT_TX_SETS_INTRA; ++s) {
Yaowu Xu's avatar
Yaowu Xu committed
124
    av1_tokens_from_tree(ext_tx_intra_encodings[s], av1_ext_tx_intra_tree[s]);
125
  }
126
#else
Yaowu Xu's avatar
Yaowu Xu committed
127
  av1_tokens_from_tree(ext_tx_encodings, av1_ext_tx_tree);
128
#endif  // CONFIG_EXT_TX
129

130
#if CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP
Yaowu Xu's avatar
Yaowu Xu committed
131
  av1_tokens_from_tree(intra_filter_encodings, av1_intra_filter_tree);
132
#endif  // CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP
133
#if CONFIG_EXT_INTER
134
#if CONFIG_INTERINTRA
Yaowu Xu's avatar
Yaowu Xu committed
135
  av1_tokens_from_tree(interintra_mode_encodings, av1_interintra_mode_tree);
136
#endif  // CONFIG_INTERINTRA
137
138
139
140
#if CONFIG_COMPOUND_SINGLEREF
  av1_tokens_from_tree(inter_singleref_comp_mode_encodings,
                       av1_inter_singleref_comp_mode_tree);
#endif  // CONFIG_COMPOUND_SINGLEREF
141
#if CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE
142
  av1_tokens_from_tree(compound_type_encodings, av1_compound_type_tree);
143
#endif  // CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE
144
#endif  // CONFIG_EXT_INTER
Yue Chen's avatar
Yue Chen committed
145
#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
146
147
148
#if CONFIG_NCOBMC_ADAPT_WEIGHT
  av1_tokens_from_tree(ncobmc_mode_encodings, av1_ncobmc_mode_tree);
#endif
Yue Chen's avatar
Yue Chen committed
149
#endif  // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
150
151
152
153
#if CONFIG_LOOP_RESTORATION
  av1_tokens_from_tree(switchable_restore_encodings,
                       av1_switchable_restore_tree);
#endif  // CONFIG_LOOP_RESTORATION
154

155
  /* This hack is necessary when CONFIG_DUAL_FILTER is enabled because the five
156
157
158
      SWITCHABLE_FILTERS are not consecutive, e.g., 0, 1, 2, 3, 4, when doing
      an in-order traversal of the av1_switchable_interp_tree structure. */
  av1_indices_from_tree(av1_switchable_interp_ind, av1_switchable_interp_inv,
Jingning Han's avatar
Jingning Han committed
159
                        av1_switchable_interp_tree);
160
161
162
/* This hack is necessary because the four TX_TYPES are not consecutive,
    e.g., 0, 1, 2, 3, when doing an in-order traversal of the av1_ext_tx_tree
    structure. */
163
164
165
#if CONFIG_EXT_TX
  for (s = 1; s < EXT_TX_SETS_INTRA; ++s)
    av1_indices_from_tree(av1_ext_tx_intra_ind[s], av1_ext_tx_intra_inv[s],
Jingning Han's avatar
Jingning Han committed
166
                          av1_ext_tx_intra_tree[s]);
167
168
  for (s = 1; s < EXT_TX_SETS_INTER; ++s)
    av1_indices_from_tree(av1_ext_tx_inter_ind[s], av1_ext_tx_inter_inv[s],
Jingning Han's avatar
Jingning Han committed
169
                          av1_ext_tx_inter_tree[s]);
170
#else
Jingning Han's avatar
Jingning Han committed
171
  av1_indices_from_tree(av1_ext_tx_ind, av1_ext_tx_inv, av1_ext_tx_tree);
David Barker's avatar
David Barker committed
172
#endif
173
174
}

Jingning Han's avatar
Jingning Han committed
175
176
177
178
static void write_intra_mode_kf(const AV1_COMMON *cm, FRAME_CONTEXT *frame_ctx,
                                const MODE_INFO *mi, const MODE_INFO *above_mi,
                                const MODE_INFO *left_mi, int block,
                                PREDICTION_MODE mode, aom_writer *w) {
Alex Converse's avatar
Alex Converse committed
179
180
181
#if CONFIG_INTRABC
  assert(!is_intrabc_block(&mi->mbmi));
#endif  // CONFIG_INTRABC
Jingning Han's avatar
Jingning Han committed
182
183
184
185
186
  aom_write_symbol(w, av1_intra_mode_ind[mode],
                   get_y_mode_cdf(frame_ctx, mi, above_mi, left_mi, block),
                   INTRA_MODES);
  (void)cm;
}
Jingning Han's avatar
Jingning Han committed
187

188
static void write_inter_mode(aom_writer *w, PREDICTION_MODE mode,
189
                             FRAME_CONTEXT *ec_ctx, const int16_t mode_ctx) {
190
  const int16_t newmv_ctx = mode_ctx & NEWMV_CTX_MASK;
Yue Chen's avatar
Yue Chen committed
191

192
193
194
195
196
#if CONFIG_NEW_MULTISYMBOL
  aom_write_symbol(w, mode != NEWMV, ec_ctx->newmv_cdf[newmv_ctx], 2);
#else
  aom_write(w, mode != NEWMV, ec_ctx->newmv_prob[newmv_ctx]);
#endif
197

Jingning Han's avatar
Jingning Han committed
198
  if (mode != NEWMV) {
199
200
201
202
203
    if (mode_ctx & (1 << ALL_ZERO_FLAG_OFFSET)) {
      assert(mode == ZEROMV);
      return;
    }

204
205
206
207
208
209
    const int16_t zeromv_ctx = (mode_ctx >> ZEROMV_OFFSET) & ZEROMV_CTX_MASK;
#if CONFIG_NEW_MULTISYMBOL
    aom_write_symbol(w, mode != ZEROMV, ec_ctx->zeromv_cdf[zeromv_ctx], 2);
#else
    aom_write(w, mode != ZEROMV, ec_ctx->zeromv_prob[zeromv_ctx]);
#endif
210
211

    if (mode != ZEROMV) {
212
213
      int16_t refmv_ctx = (mode_ctx >> REFMV_OFFSET) & REFMV_CTX_MASK;

214
215
216
      if (mode_ctx & (1 << SKIP_NEARESTMV_OFFSET)) refmv_ctx = 6;
      if (mode_ctx & (1 << SKIP_NEARMV_OFFSET)) refmv_ctx = 7;
      if (mode_ctx & (1 << SKIP_NEARESTMV_SUB8X8_OFFSET)) refmv_ctx = 8;
217
218
219
220
221
#if CONFIG_NEW_MULTISYMBOL
      aom_write_symbol(w, mode != NEARESTMV, ec_ctx->refmv_cdf[refmv_ctx], 2);
#else
      aom_write(w, mode != NEARESTMV, ec_ctx->refmv_prob[refmv_ctx]);
#endif
222
223
    }
  }
Jingning Han's avatar
Jingning Han committed
224
225
}

226
static void write_drl_idx(FRAME_CONTEXT *ec_ctx, const MB_MODE_INFO *mbmi,
Yaowu Xu's avatar
Yaowu Xu committed
227
228
                          const MB_MODE_INFO_EXT *mbmi_ext, aom_writer *w) {
  uint8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame);
229
230
231

  assert(mbmi->ref_mv_idx < 3);

232
#if CONFIG_EXT_INTER
233
234
235
236
#if CONFIG_COMPOUND_SINGLEREF
  if (mbmi->mode == NEWMV || mbmi->mode == NEW_NEWMV ||
      mbmi->mode == SR_NEW_NEWMV) {
#else   // !CONFIG_COMPOUND_SINGLEREF
237
  if (mbmi->mode == NEWMV || mbmi->mode == NEW_NEWMV) {
238
239
#endif  // CONFIG_COMPOUND_SINGLEREF
#else   // !CONFIG_EXT_INTER
240
  if (mbmi->mode == NEWMV) {
241
#endif  // CONFIG_EXT_INTER
242
243
244
245
    int idx;
    for (idx = 0; idx < 2; ++idx) {
      if (mbmi_ext->ref_mv_count[ref_frame_type] > idx + 1) {
        uint8_t drl_ctx =
Yaowu Xu's avatar
Yaowu Xu committed
246
            av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx);
247

248
249
250
251
252
253
#if CONFIG_NEW_MULTISYMBOL
        aom_write_symbol(w, mbmi->ref_mv_idx != idx, ec_ctx->drl_cdf[drl_ctx],
                         2);
#else
        aom_write(w, mbmi->ref_mv_idx != idx, ec_ctx->drl_prob[drl_ctx]);
#endif
254
        if (mbmi->ref_mv_idx == idx) return;
255
      }
256
257
258
259
    }
    return;
  }

David Barker's avatar
David Barker committed
260
  if (have_nearmv_in_inter_mode(mbmi->mode)) {
261
262
263
264
265
    int idx;
    // TODO(jingning): Temporary solution to compensate the NEARESTMV offset.
    for (idx = 1; idx < 3; ++idx) {
      if (mbmi_ext->ref_mv_count[ref_frame_type] > idx + 1) {
        uint8_t drl_ctx =
Yaowu Xu's avatar
Yaowu Xu committed
266
            av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx);
267
268
269
270
271
272
#if CONFIG_NEW_MULTISYMBOL
        aom_write_symbol(w, mbmi->ref_mv_idx != (idx - 1),
                         ec_ctx->drl_cdf[drl_ctx], 2);
#else
        aom_write(w, mbmi->ref_mv_idx != (idx - 1), ec_ctx->drl_prob[drl_ctx]);
#endif
273
        if (mbmi->ref_mv_idx == (idx - 1)) return;
274
      }
275
    }
276
    return;
277
278
279
  }
}

280
#if CONFIG_EXT_INTER
281
282
static void write_inter_compound_mode(AV1_COMMON *cm, MACROBLOCKD *xd,
                                      aom_writer *w, PREDICTION_MODE mode,
283
                                      const int16_t mode_ctx) {
284
285
286
287
288
  assert(is_inter_compound_mode(mode));
  (void)cm;
  aom_write_symbol(w, INTER_COMPOUND_OFFSET(mode),
                   xd->tile_ctx->inter_compound_mode_cdf[mode_ctx],
                   INTER_COMPOUND_MODES);
289
}
290
291

#if CONFIG_COMPOUND_SINGLEREF
292
static void write_inter_singleref_comp_mode(MACROBLOCKD *xd, aom_writer *w,
293
294
295
                                            PREDICTION_MODE mode,
                                            const int16_t mode_ctx) {
  assert(is_inter_singleref_comp_mode(mode));
296
297
  aom_cdf_prob *const inter_singleref_comp_cdf =
      xd->tile_ctx->inter_singleref_comp_mode_cdf[mode_ctx];
298

299
300
  aom_write_symbol(w, INTER_SINGLEREF_COMP_OFFSET(mode),
                   inter_singleref_comp_cdf, INTER_SINGLEREF_COMP_MODES);
301
302
}
#endif  // CONFIG_COMPOUND_SINGLEREF
303
304
#endif  // CONFIG_EXT_INTER

Yaowu Xu's avatar
Yaowu Xu committed
305
static void encode_unsigned_max(struct aom_write_bit_buffer *wb, int data,
306
                                int max) {
Yaowu Xu's avatar
Yaowu Xu committed
307
  aom_wb_write_literal(wb, data, get_unsigned_bits(max));
Jingning Han's avatar
Jingning Han committed
308
309
}

310
#if CONFIG_NCOBMC_ADAPT_WEIGHT
Yaowu Xu's avatar
Yaowu Xu committed
311
312
static void prob_diff_update(const aom_tree_index *tree,
                             aom_prob probs[/*n - 1*/],
313
                             const unsigned int counts[/* n */], int n,
314
                             int probwt, aom_writer *w) {
Jingning Han's avatar
Jingning Han committed
315
316
317
318
319
320
  int i;
  unsigned int branch_ct[32][2];

  // Assuming max number of probabilities <= 32
  assert(n <= 32);

Yaowu Xu's avatar
Yaowu Xu committed
321
  av1_tree_probs_from_distribution(tree, branch_ct, counts);
Jingning Han's avatar
Jingning Han committed
322
  for (i = 0; i < n - 1; ++i)
323
    av1_cond_prob_diff_update(w, &probs[i], branch_ct[i], probwt);
Jingning Han's avatar
Jingning Han committed
324
}
325
326
#endif

327
#if CONFIG_VAR_TX
328
static void write_tx_size_vartx(const AV1_COMMON *cm, MACROBLOCKD *xd,
329
                                const MB_MODE_INFO *mbmi, TX_SIZE tx_size,
330
331
                                int depth, int blk_row, int blk_col,
                                aom_writer *w) {
332
333
334
335
#if CONFIG_NEW_MULTISYMBOL
  FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
  (void)cm;
#endif
336
337
  const int tx_row = blk_row >> 1;
  const int tx_col = blk_col >> 1;
338
339
340
  const int max_blocks_high = max_block_high(xd, mbmi->sb_type, 0);
  const int max_blocks_wide = max_block_wide(xd, mbmi->sb_type, 0);

341
342
  int ctx = txfm_partition_context(xd->above_txfm_context + blk_col,
                                   xd->left_txfm_context + blk_row,
343
                                   mbmi->sb_type, tx_size);
344

345
  if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
346

Jingning Han's avatar
Jingning Han committed
347
  if (depth == MAX_VARTX_DEPTH) {
348
349
    txfm_partition_update(xd->above_txfm_context + blk_col,
                          xd->left_txfm_context + blk_row, tx_size, tx_size);
350
351
352
    return;
  }

Yue Chen's avatar
Yue Chen committed
353
354
355
356
#if CONFIG_RECT_TX_EXT
  if (tx_size == mbmi->inter_tx_size[tx_row][tx_col] ||
      mbmi->tx_size == quarter_txsize_lookup[mbmi->sb_type]) {
#else
357
  if (tx_size == mbmi->inter_tx_size[tx_row][tx_col]) {
Yue Chen's avatar
Yue Chen committed
358
#endif
359
360
361
#if CONFIG_NEW_MULTISYMBOL
    aom_write_symbol(w, 0, ec_ctx->txfm_partition_cdf[ctx], 2);
#else
Yaowu Xu's avatar
Yaowu Xu committed
362
    aom_write(w, 0, cm->fc->txfm_partition_prob[ctx]);
363
364
#endif

365
366
    txfm_partition_update(xd->above_txfm_context + blk_col,
                          xd->left_txfm_context + blk_row, tx_size, tx_size);
Yue Chen's avatar
Yue Chen committed
367
    // TODO(yuec): set correct txfm partition update for qttx
368
  } else {
369
370
    const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
    const int bsl = tx_size_wide_unit[sub_txs];
371
    int i;
372

373
374
375
#if CONFIG_NEW_MULTISYMBOL
    aom_write_symbol(w, 1, ec_ctx->txfm_partition_cdf[ctx], 2);
#else
Yaowu Xu's avatar
Yaowu Xu committed
376
    aom_write(w, 1, cm->fc->txfm_partition_prob[ctx]);
377
#endif
378

379
    if (tx_size == TX_8X8) {
380
381
      txfm_partition_update(xd->above_txfm_context + blk_col,
                            xd->left_txfm_context + blk_row, sub_txs, tx_size);
382
      return;
383
    }
384
385
386

    assert(bsl > 0);
    for (i = 0; i < 4; ++i) {
387
388
389
390
      int offsetr = blk_row + (i >> 1) * bsl;
      int offsetc = blk_col + (i & 0x01) * bsl;
      write_tx_size_vartx(cm, xd, mbmi, sub_txs, depth + 1, offsetr, offsetc,
                          w);
391
392
393
    }
  }
}
394

395
#if !CONFIG_NEW_MULTISYMBOL
Yaowu Xu's avatar
Yaowu Xu committed
396
static void update_txfm_partition_probs(AV1_COMMON *cm, aom_writer *w,
397
                                        FRAME_COUNTS *counts, int probwt) {
398
399
  int k;
  for (k = 0; k < TXFM_PARTITION_CONTEXTS; ++k)
Yaowu Xu's avatar
Yaowu Xu committed
400
    av1_cond_prob_diff_update(w, &cm->fc->txfm_partition_prob[k],
401
                              counts->txfm_partition[k], probwt);
402
}
403
#endif  // CONFIG_NEW_MULTISYMBOL
404
405
#endif

Yaowu Xu's avatar
Yaowu Xu committed
406
407
static void write_selected_tx_size(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                                   aom_writer *w) {
408
409
  const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
  const BLOCK_SIZE bsize = mbmi->sb_type;
410
411
  FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
  (void)cm;
412
// For sub8x8 blocks the tx_size symbol does not need to be sent
413
#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX
414
415
  if (bsize > BLOCK_4X4) {
#else
416
  if (bsize >= BLOCK_8X8) {
417
#endif
418
419
420
421
422
423
    const TX_SIZE tx_size = mbmi->tx_size;
    const int is_inter = is_inter_block(mbmi);
    const int tx_size_ctx = get_tx_size_context(xd);
    const int tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
                                     : intra_tx_size_cat_lookup[bsize];
    const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size];
424
    const int depth = tx_size_to_depth(coded_tx_size);
425
#if CONFIG_EXT_TX && CONFIG_RECT_TX
426
    assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(xd, mbmi)));
427
428
#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX

429
    aom_write_symbol(w, depth, ec_ctx->tx_size_cdf[tx_size_cat][tx_size_ctx],
430
                     tx_size_cat + 2);
Yue Chen's avatar
Yue Chen committed
431
#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
432
433
434
    if (is_quarter_tx_allowed(xd, mbmi, is_inter) && tx_size != coded_tx_size)
      aom_write(w, tx_size == quarter_txsize_lookup[bsize],
                cm->fc->quarter_tx_size_prob);
Yue Chen's avatar
Yue Chen committed
435
#endif
Jingning Han's avatar
Jingning Han committed
436
437
438
  }
}

439
#if !CONFIG_NEW_MULTISYMBOL
Yaowu Xu's avatar
Yaowu Xu committed
440
static void update_inter_mode_probs(AV1_COMMON *cm, aom_writer *w,
441
442
                                    FRAME_COUNTS *counts) {
  int i;
443
  const int probwt = cm->num_tg;
444
  for (i = 0; i < NEWMV_MODE_CONTEXTS; ++i)
445
446
    av1_cond_prob_diff_update(w, &cm->fc->newmv_prob[i], counts->newmv_mode[i],
                              probwt);
447
  for (i = 0; i < ZEROMV_MODE_CONTEXTS; ++i)
Yaowu Xu's avatar
Yaowu Xu committed
448
    av1_cond_prob_diff_update(w, &cm->fc->zeromv_prob[i],
449
                              counts->zeromv_mode[i], probwt);
450
  for (i = 0; i < REFMV_MODE_CONTEXTS; ++i)
451
452
    av1_cond_prob_diff_update(w, &cm->fc->refmv_prob[i], counts->refmv_mode[i],
                              probwt);
453
  for (i = 0; i < DRL_MODE_CONTEXTS; ++i)
454
455
    av1_cond_prob_diff_update(w, &cm->fc->drl_prob[i], counts->drl_mode[i],
                              probwt);
456
}
457
#endif
458

Yaowu Xu's avatar
Yaowu Xu committed
459
460
static int write_skip(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                      int segment_id, const MODE_INFO *mi, aom_writer *w) {
Jingning Han's avatar
Jingning Han committed
461
462
463
464
  if (segfeature_active(&cm->seg, segment_id, SEG_LVL_SKIP)) {
    return 1;
  } else {
    const int skip = mi->mbmi.skip;
465
466
467
468
469
#if CONFIG_NEW_MULTISYMBOL
    FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
    const int ctx = av1_get_skip_context(xd);
    aom_write_symbol(w, skip, ec_ctx->skip_cdfs[ctx], 2);
#else
Yaowu Xu's avatar
Yaowu Xu committed
470
    aom_write(w, skip, av1_get_skip_prob(cm, xd));
471
#endif
Jingning Han's avatar
Jingning Han committed
472
473
474
475
    return skip;
  }
}

476
477
478
479
480
481
482
483
484
485
486
487
488
static void write_is_inter(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                           int segment_id, aom_writer *w, const int is_inter) {
  if (!segfeature_active(&cm->seg, segment_id, SEG_LVL_REF_FRAME)) {
#if CONFIG_NEW_MULTISYMBOL
    FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
    const int ctx = av1_get_intra_inter_context(xd);
    aom_write_symbol(w, is_inter, ec_ctx->intra_inter_cdf[ctx], 2);
#else
    aom_write(w, is_inter, av1_get_intra_inter_prob(cm, xd));
#endif
  }
}

Yue Chen's avatar
Yue Chen committed
489
#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
490
491
static void write_motion_mode(const AV1_COMMON *cm, MACROBLOCKD *xd,
                              const MODE_INFO *mi, aom_writer *w) {
492
  const MB_MODE_INFO *mbmi = &mi->mbmi;
493

494
495
496
497
498
499
500
#if !CONFIG_GLOBAL_MOTION
  // The cm parameter is only used with global_motion or with
  // motion_var and warped_motion. In other cases, explicitly ignore
  // it to avoid a compiler warning.
  (void)cm;
#endif

501
502
503
#if CONFIG_NCOBMC_ADAPT_WEIGHT
  MOTION_MODE last_motion_mode_allowed =
      motion_mode_allowed_wrapper(0,
504
#if CONFIG_GLOBAL_MOTION
505
                                  0, cm->global_motion,
506
#endif  // CONFIG_GLOBAL_MOTION
507
508
509
#if CONFIG_WARPED_MOTION
                                  xd,
#endif
510
511
                                  mi);
#else
512
  MOTION_MODE last_motion_mode_allowed = motion_mode_allowed(
513
#if CONFIG_GLOBAL_MOTION
514
      0, cm->global_motion,
515
#endif  // CONFIG_GLOBAL_MOTION
516
517
518
#if CONFIG_WARPED_MOTION
      xd,
#endif
519
      mi);
520
#endif  // CONFIG_NCOBMC_ADAPT_WEIGHT
Yue Chen's avatar
Yue Chen committed
521
522
523
  if (last_motion_mode_allowed == SIMPLE_TRANSLATION) return;
#if CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION
  if (last_motion_mode_allowed == OBMC_CAUSAL) {
524
525
526
527
#if CONFIG_NEW_MULTISYMBOL
    aom_write_symbol(w, mbmi->motion_mode == OBMC_CAUSAL,
                     xd->tile_ctx->obmc_cdf[mbmi->sb_type], 2);
#else
Yue Chen's avatar
Yue Chen committed
528
529
    aom_write(w, mbmi->motion_mode == OBMC_CAUSAL,
              cm->fc->obmc_prob[mbmi->sb_type]);
530
#endif
Yue Chen's avatar
Yue Chen committed
531
532
  } else {
#endif  // CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION
Thomas Davies's avatar
Thomas Davies committed
533
534
535
    aom_write_symbol(w, mbmi->motion_mode,
                     xd->tile_ctx->motion_mode_cdf[mbmi->sb_type],
                     MOTION_MODES);
Yue Chen's avatar
Yue Chen committed
536
537
538
539
#if CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION
  }
#endif  // CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION
}
540
541

#if CONFIG_NCOBMC_ADAPT_WEIGHT
542
static void write_ncobmc_mode(MACROBLOCKD *xd, const MODE_INFO *mi,
543
544
545
                              aom_writer *w) {
  const MB_MODE_INFO *mbmi = &mi->mbmi;
  ADAPT_OVERLAP_BLOCK ao_block = adapt_overlap_block_lookup[mbmi->sb_type];
546
  if (mbmi->motion_mode != NCOBMC_ADAPT_WEIGHT) return;
547

548
549
  aom_write_symbol(w, mbmi->ncobmc_mode[0],
                   xd->tile_ctx->ncobmc_mode_cdf[ao_block], MAX_NCOBMC_MODES);
550
  if (mi_size_wide[mbmi->sb_type] != mi_size_high[mbmi->sb_type]) {
551
552
    aom_write_symbol(w, mbmi->ncobmc_mode[1],
                     xd->tile_ctx->ncobmc_mode_cdf[ao_block], MAX_NCOBMC_MODES);
553
554
555
  }
}
#endif
Yue Chen's avatar
Yue Chen committed
556
557
#endif  // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION

558
#if CONFIG_DELTA_Q
Thomas Davies's avatar
Thomas Davies committed
559
560
static void write_delta_qindex(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                               int delta_qindex, aom_writer *w) {
561
562
  int sign = delta_qindex < 0;
  int abs = sign ? -delta_qindex : delta_qindex;
Thomas Davies's avatar
Thomas Davies committed
563
  int rem_bits, thr;
564
  int smallval = abs < DELTA_Q_SMALL ? 1 : 0;
Thomas Davies's avatar
Thomas Davies committed
565
566
  FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
  (void)cm;
567

Thomas Davies's avatar
Thomas Davies committed
568
569
  aom_write_symbol(w, AOMMIN(abs, DELTA_Q_SMALL), ec_ctx->delta_q_cdf,
                   DELTA_Q_PROBS + 1);
570
571
572
573
574
575

  if (!smallval) {
    rem_bits = OD_ILOG_NZ(abs - 1) - 1;
    thr = (1 << rem_bits) + 1;
    aom_write_literal(w, rem_bits, 3);
    aom_write_literal(w, abs - thr, rem_bits);
576
577
578
579
580
  }
  if (abs > 0) {
    aom_write_bit(w, sign);
  }
}
581

Fangwen Fu's avatar
Fangwen Fu committed
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
#if CONFIG_EXT_DELTA_Q
static void write_delta_lflevel(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                                int delta_lflevel, aom_writer *w) {
  int sign = delta_lflevel < 0;
  int abs = sign ? -delta_lflevel : delta_lflevel;
  int rem_bits, thr;
  int smallval = abs < DELTA_LF_SMALL ? 1 : 0;
  FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
  (void)cm;

  aom_write_symbol(w, AOMMIN(abs, DELTA_LF_SMALL), ec_ctx->delta_lf_cdf,
                   DELTA_LF_PROBS + 1);

  if (!smallval) {
    rem_bits = OD_ILOG_NZ(abs - 1) - 1;
    thr = (1 << rem_bits) + 1;
    aom_write_literal(w, rem_bits, 3);
    aom_write_literal(w, abs - thr, rem_bits);
  }
  if (abs > 0) {
    aom_write_bit(w, sign);
  }
}
#endif  // CONFIG_EXT_DELTA_Q
Thomas Davies's avatar
Thomas Davies committed
606
#endif  // CONFIG_DELTA_Q
607

608
#if !CONFIG_NEW_MULTISYMBOL
Yaowu Xu's avatar
Yaowu Xu committed
609
static void update_skip_probs(AV1_COMMON *cm, aom_writer *w,
Jingning Han's avatar
Jingning Han committed
610
611
                              FRAME_COUNTS *counts) {
  int k;
612
613
614
615
616
  const int probwt = cm->num_tg;
  for (k = 0; k < SKIP_CONTEXTS; ++k) {
    av1_cond_prob_diff_update(w, &cm->fc->skip_probs[k], counts->skip[k],
                              probwt);
  }
Jingning Han's avatar
Jingning Han committed
617
}
618
#endif
Jingning Han's avatar
Jingning Han committed
619

Yaowu Xu's avatar
Yaowu Xu committed
620
static void pack_palette_tokens(aom_writer *w, const TOKENEXTRA **tp, int n,
621
                                int num) {
622
  const TOKENEXTRA *p = *tp;
623
624
625
626
  write_uniform(w, n, p->token);  // The first color index.
  ++p;
  --num;
  for (int i = 0; i < num; ++i) {
627
    aom_write_symbol(w, p->token, p->palette_cdf, n);
hui su's avatar
hui su committed
628
629
630
631
    ++p;
  }
  *tp = p;
}
632

633
#if !CONFIG_PVQ
634
#if CONFIG_SUPERTX
635
static void update_supertx_probs(AV1_COMMON *cm, int probwt, aom_writer *w) {
Yaowu Xu's avatar
Yaowu Xu committed
636
637
  const int savings_thresh = av1_cost_one(GROUP_DIFF_UPDATE_PROB) -
                             av1_cost_zero(GROUP_DIFF_UPDATE_PROB);
638
639
640
641
  int i, j;
  int savings = 0;
  int do_update = 0;
  for (i = 0; i < PARTITION_SUPERTX_CONTEXTS; ++i) {
Jingning Han's avatar
Jingning Han committed
642
    for (j = TX_8X8; j < TX_SIZES; ++j) {
643
644
      savings += av1_cond_prob_diff_update_savings(
          &cm->fc->supertx_prob[i][j], cm->counts.supertx[i][j], probwt);
645
646
647
    }
  }
  do_update = savings > savings_thresh;
Yaowu Xu's avatar
Yaowu Xu committed
648
  aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
649
650
  if (do_update) {
    for (i = 0; i < PARTITION_SUPERTX_CONTEXTS; ++i) {
Jingning Han's avatar
Jingning Han committed
651
      for (j = TX_8X8; j < TX_SIZES; ++j) {
Yaowu Xu's avatar
Yaowu Xu committed
652
        av1_cond_prob_diff_update(w, &cm->fc->supertx_prob[i][j],
653
                                  cm->counts.supertx[i][j], probwt);
654
655
656
657
658
659
      }
    }
  }
}
#endif  // CONFIG_SUPERTX

660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
#if CONFIG_NEW_MULTISYMBOL
static INLINE void write_coeff_extra(const aom_cdf_prob *const *cdf, int val,
                                     int n, aom_writer *w) {
  // Code the extra bits from LSB to MSB in groups of 4
  int i = 0;
  int count = 0;
  while (count < n) {
    const int size = AOMMIN(n - count, 4);
    const int mask = (1 << size) - 1;
    aom_write_cdf(w, val & mask, cdf[i++], 1 << size);
    val >>= size;
    count += size;
  }
}
#else
static INLINE void write_coeff_extra(const aom_prob *pb, int value,
                                     int num_bits, int skip_bits, aom_writer *w,
                                     TOKEN_STATS *token_stats) {
  // Code the extra bits from MSB to LSB 1 bit at a time
  int index;
  for (index = skip_bits; index < num_bits; ++index) {
    const int shift = num_bits - index - 1;
    const int bb = (value >> shift) & 1;
    aom_write_record(w, bb, pb[index], token_stats);
  }
}
#endif

688
#if !CONFIG_LV_MAP
Yaowu Xu's avatar
Yaowu Xu committed
689
static void pack_mb_tokens(aom_writer *w, const TOKENEXTRA **tp,
690
                           const TOKENEXTRA *const stop,
691
692
                           aom_bit_depth_t bit_depth, const TX_SIZE tx_size,
                           TOKEN_STATS *token_stats) {
693
  const TOKENEXTRA *p = *tp;
Fangwen Fu's avatar
Fangwen Fu committed
694
#if CONFIG_VAR_TX
695
  int count = 0;
696
  const int seg_eob = tx_size_2d[tx_size];
697
#endif
Jingning Han's avatar
Jingning Han committed
698
699

  while (p < stop && p->token != EOSB_TOKEN) {
700
    const int token = p->token;
701
    const int eob_val = p->eob_val;
702
    if (token == BLOCK_Z_TOKEN) {
703
      aom_write_symbol(w, 0, *p->head_cdf, HEAD_TOKENS + 1);
704
      p++;
705
706
707
#if CONFIG_VAR_TX
      break;
#endif
708
709
      continue;
    }
Yaowu Xu's avatar
Yaowu Xu committed
710
711

    const av1_extra_bit *const extra_bits = &av1_extra_bits[token];
712
    if (eob_val == LAST_EOB) {
713
714
715
      // Just code a flag indicating whether the value is >1 or 1.
      aom_write_bit(w, token != ONE_TOKEN);
    } else {
716
      int comb_symb = 2 * AOMMIN(token, TWO_TOKEN) - eob_val + p->first_val;
717
      aom_write_symbol(w, comb_symb, *p->head_cdf, HEAD_TOKENS + p->first_val);
718
    }
719
    if (token > ONE_TOKEN) {
720
      aom_write_symbol(w, token - TWO_TOKEN, *p->tail_cdf, TAIL_TOKENS);
Alex Converse's avatar
Alex Converse committed
721
    }
722
723
724
725

    if (extra_bits->base_val) {
      const int bit_string = p->extra;
      const int bit_string_length = extra_bits->len;  // Length of extra bits to
726
      const int is_cat6 = (extra_bits->base_val == CAT6_MIN_VAL);
727
728
      // be written excluding
      // the sign bit.
729
      int skip_bits = is_cat6
730
731
732
                          ? (int)sizeof(av1_cat6_prob) -
                                av1_get_cat6_extrabits_size(tx_size, bit_depth)
                          : 0;
733

734
735
736
737
738
739
740
741
742
      assert(!(bit_string >> (bit_string_length - skip_bits + 1)));
      if (bit_string_length > 0)
#if CONFIG_NEW_MULTISYMBOL
        write_coeff_extra(extra_bits->cdf, bit_string >> 1,
                          bit_string_length - skip_bits, w);
#else
        write_coeff_extra(extra_bits->prob, bit_string >> 1, bit_string_length,
                          skip_bits, w, token_stats);
#endif
743
744
745
746
747

      aom_write_bit_record(w, bit_string & 1, token_stats);
    }
    ++p;

Fangwen Fu's avatar
Fangwen Fu committed
748
#if CONFIG_VAR_TX
749
    ++count;
750
    if (eob_val == EARLY_EOB || count == seg_eob) break;
751
752
753
754
755
#endif
  }

  *tp = p;
}
Angie Chiang's avatar
Angie Chiang committed
756
#endif  // !CONFIG_LV_MAP
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
#else   // !CONFIG_PVQ
static PVQ_INFO *get_pvq_block(PVQ_QUEUE *pvq_q) {
  PVQ_INFO *pvq;

  assert(pvq_q->curr_pos <= pvq_q->last_pos);
  assert(pvq_q->curr_pos < pvq_q->buf_len);

  pvq = pvq_q->buf + pvq_q->curr_pos;
  ++pvq_q->curr_pos;

  return pvq;
}

static void pack_pvq_tokens(aom_writer *w, MACROBLOCK *const x,
                            MACROBLOCKD *const xd, int plane, BLOCK_SIZE bsize,
                            const TX_SIZE tx_size) {
  PVQ_INFO *pvq;
  int idx, idy;
  const struct macroblockd_plane *const pd = &xd->plane[plane];
  od_adapt_ctx *adapt;
  int max_blocks_wide;
  int max_blocks_high;
  int step = (1 << tx_size);
780
781

#if CONFIG_CHROMA_SUB8X8
782
  const BLOCK_SIZE plane_bsize =
783
784
785
786
787
788
789
      AOMMAX(BLOCK_4X4, get_plane_block_size(bsize, pd));
#elif CONFIG_CB4X4
  const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd);
#else
  const BLOCK_SIZE plane_bsize =
      get_plane_block_size(AOMMAX(BLOCK_8X8, bsize), pd);
#endif
790

791
  adapt = x->daala_enc.state.adapt;
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809

  max_blocks_wide = max_block_wide(xd, plane_bsize, plane);
  max_blocks_high = max_block_high(xd, plane_bsize, plane);

  for (idy = 0; idy < max_blocks_high; idy += step) {
    for (idx = 0; idx < max_blocks_wide; idx += step) {
      const int is_keyframe = 0;
      const int encode_flip = 0;
      const int flip = 0;
      int i;
      const int has_dc_skip = 1;
      int *exg = &adapt->pvq.pvq_exg[plane][tx_size][0];
      int *ext = adapt->pvq.pvq_ext + tx_size * PVQ_MAX_PARTITIONS;
      generic_encoder *model = adapt->pvq.pvq_param_model;

      pvq = get_pvq_block(x->pvq_q);

      // encode block skip info
810
811
      aom_write_symbol(w, pvq->ac_dc_coded,
                       adapt->skip_cdf[2 * tx_size + (plane != 0)], 4);
812
813
814
815
816
817
818
819

      // AC coeffs coded?
      if (pvq->ac_dc_coded & AC_CODED) {
        assert(pvq->bs == tx_size);
        for (i = 0; i < pvq->nb_bands; i++) {
          if (i == 0 ||
              (!pvq->skip_rest && !(pvq->skip_dir & (1 << ((i - 1) % 3))))) {
            pvq_encode_partition(
820
821
                w, pvq->qg[i], pvq->theta[i], pvq->y + pvq->off[i],
                pvq->size[i], pvq->k[i], model, adapt, exg + i, ext + i,
822
823
824
825
826
827
                (plane != 0) * OD_TXSIZES * PVQ_MAX_PARTITIONS +
                    pvq->bs * PVQ_MAX_PARTITIONS + i,
                is_keyframe, i == 0 && (i < pvq->nb_bands - 1), pvq->skip_rest,
                encode_flip, flip);
          }
          if (i == 0 && !pvq->skip_rest && pvq->bs > 0) {
828
            aom_write_symbol(
829
830
831
                w, pvq->skip_dir,
                &adapt->pvq
                     .pvq_skip_dir_cdf[(plane != 0) + 2 * (pvq->bs - 1)][0],
832
                7);
833
834
835
836
837
838
          }
        }
      }
      // Encode residue of DC coeff, if exist.
      if (!has_dc_skip || (pvq->ac_dc_coded & DC_CODED)) {
        generic_encode(w, &adapt->model_dc[plane],
839
                       abs(pvq->dq_dc_residue) - has_dc_skip,
840
841
842
843
844
845
846
847
                       &adapt->ex_dc[plane][pvq->bs][0], 2);
      }
      if ((pvq->ac_dc_coded & DC_CODED)) {
        aom_write_bit(w, pvq->dq_dc_residue < 0);
      }
    }
  }  // for (idy = 0;
}
848
#endif  // !CONFIG_PVG
849

850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
#if CONFIG_VAR_TX && !CONFIG_COEF_INTERLEAVE
#if CONFIG_LV_MAP
static void pack_txb_tokens(aom_writer *w,
#if CONFIG_LV_MAP
                            AV1_COMMON *cm,
#endif  // CONFIG_LV_MAP
                            const TOKENEXTRA **tp,
                            const TOKENEXTRA *const tok_end,
#if CONFIG_PVQ || CONFIG_LV_MAP
                            MACROBLOCK *const x,
#endif
                            MACROBLOCKD *xd, MB_MODE_INFO *mbmi, int plane,
                            BLOCK_SIZE plane_bsize, aom_bit_depth_t bit_depth,
                            int block, int blk_row, int blk_col,
                            TX_SIZE tx_size, TOKEN_STATS *token_stats) {
  const struct macroblockd_plane *const pd = &xd->plane[plane];
  const BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
  const int tx_row = blk_row >> (1 - pd->subsampling_y);
  const int tx_col = blk_col >> (1 - pd->subsampling_x);
  TX_SIZE plane_tx_size;
  const int max_blocks_high = max_block_high(xd, plane_bsize, plane);
  const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane);

  if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;

  plane_tx_size =
      plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0]
            : mbmi->inter_tx_size[tx_row][tx_col];

  if (tx_size == plane_tx_size) {
    TOKEN_STATS tmp_token_stats;
    init_token_stats(&tmp_token_stats);

#if !CONFIG_PVQ
    tran_low_t *tcoeff = BLOCK_OFFSET(x->mbmi_ext->tcoeff[plane], block);
    uint16_t eob = x->mbmi_ext->eobs[plane][block];
    TXB_CTX txb_ctx = { x->mbmi_ext->txb_skip_ctx[plane][block],
                        x->mbmi_ext->dc_sign_ctx[plane][block] };
Jingning Han's avatar
Jingning Han committed
888
889
    av1_write_coeffs_txb(cm, xd, w, blk_row, blk_col, block, plane, tx_size,
                         tcoeff, eob, &txb_ctx);
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
#else
    pack_pvq_tokens(w, x, xd, plane, bsize, tx_size);
#endif
#if CONFIG_RD_DEBUG
    token_stats->txb_coeff_cost_map[blk_row][blk_col] = tmp_token_stats.cost;
    token_stats->cost += tmp_token_stats.cost;
#endif
  } else {
    const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
    const int bsl = tx_size_wide_unit[sub_txs];
    int i;

    assert(bsl > 0);

    for (i = 0; i < 4; ++i) {
      const int offsetr = blk_row + (i >> 1) * bsl;
      const int offsetc = blk_col + (i & 0x01) * bsl;
      const int step = tx_size_wide_unit[sub_txs] * tx_size_high_unit[sub_txs];

      if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;

      pack_txb_tokens(w,
#if CONFIG_LV_MAP
                      cm,
#endif
                      tp, tok_end,
#if CONFIG_PVQ || CONFIG_LV_MAP
                      x,
#endif
                      xd, mbmi, plane, plane_bsize, bit_depth, block, offsetr,
                      offsetc, sub_txs, token_stats);
      block += step;
    }
  }
}
#else  // CONFIG_LV_MAP
Yaowu Xu's avatar
Yaowu Xu committed
926
static void pack_txb_tokens(aom_writer *w, const TOKENEXTRA **tp,
927
928
929
930
931
                            const TOKENEXTRA *const tok_end,
#if CONFIG_PVQ
                            MACROBLOCK *const x,
#endif
                            MACROBLOCKD *xd, MB_MODE_INFO *mbmi, int plane,
Yaowu Xu's avatar
Yaowu Xu committed
932
                            BLOCK_SIZE plane_bsize, aom_bit_depth_t bit_depth,
933
                            int block, int blk_row, int blk_col,
934
                            TX_SIZE tx_size, TOKEN_STATS *token_stats) {
935
936
  const struct macroblockd_plane *const pd = &xd->plane[plane];
  const BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
937
938
  const int tx_row = blk_row >> (1 - pd->subsampling_y);
  const int tx_col = blk_col >> (1 - pd->subsampling_x);
Debargha Mukherjee's avatar
Debargha Mukherjee committed
939
  TX_SIZE plane_tx_size;
940
941
  const int max_blocks_high = max_block_high(xd, plane_bsize, plane);
  const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane);
942

943
  if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
944

945
946
947
  plane_tx_size =
      plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0]
            : mbmi->inter_tx_size[tx_row][tx_col];