bitstream.c 179 KB
Newer Older
<
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4
5
6
7
8
9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10
11
12
13
 */

#include <assert.h>
#include <limits.h>
14
#include <stdio.h>
Jingning Han's avatar
Jingning Han committed
15

Yaowu Xu's avatar
Yaowu Xu committed
16
#include "aom/aom_encoder.h"
17
#include "aom_dsp/bitwriter_buffer.h"
Yaowu Xu's avatar
Yaowu Xu committed
18
#include "aom_dsp/aom_dsp_common.h"
19
#include "aom_dsp/binary_codes_writer.h"
Yaowu Xu's avatar
Yaowu Xu committed
20
#include "aom_mem/aom_mem.h"
21
22
#include "aom_ports/mem_ops.h"
#include "aom_ports/system_state.h"
23
#if CONFIG_BITSTREAM_DEBUG
24
#include "aom_util/debug_util.h"
25
#endif  // CONFIG_BITSTREAM_DEBUG
Jingning Han's avatar
Jingning Han committed
26

27
#if CONFIG_CDEF
28
#include "av1/common/cdef.h"
29
#endif  // CONFIG_CDEF
30
31
32
33
#include "av1/common/entropy.h"
#include "av1/common/entropymode.h"
#include "av1/common/entropymv.h"
#include "av1/common/mvref_common.h"
34
#include "av1/common/odintrin.h"
35
36
#include "av1/common/pred_common.h"
#include "av1/common/reconinter.h"
hui su's avatar
hui su committed
37
38
39
#if CONFIG_EXT_INTRA
#include "av1/common/reconintra.h"
#endif  // CONFIG_EXT_INTRA
40
41
#include "av1/common/seg_common.h"
#include "av1/common/tile_common.h"
Jingning Han's avatar
Jingning Han committed
42

Angie Chiang's avatar
Angie Chiang committed
43
44
45
#if CONFIG_LV_MAP
#include "av1/encoder/encodetxb.h"
#endif  // CONFIG_LV_MAP
46
47
48
49
#include "av1/encoder/bitstream.h"
#include "av1/encoder/cost.h"
#include "av1/encoder/encodemv.h"
#include "av1/encoder/mcomp.h"
Urvang Joshi's avatar
Urvang Joshi committed
50
#if CONFIG_PALETTE_DELTA_ENCODING
hui su's avatar
hui su committed
51
#include "av1/encoder/palette.h"
Urvang Joshi's avatar
Urvang Joshi committed
52
#endif  // CONFIG_PALETTE_DELTA_ENCODING
53
54
55
#include "av1/encoder/segmentation.h"
#include "av1/encoder/subexp.h"
#include "av1/encoder/tokenize.h"
56
57
58
#if CONFIG_PVQ
#include "av1/encoder/pvq_encoder.h"
#endif
Jingning Han's avatar
Jingning Han committed
59

60
#define ENC_MISMATCH_DEBUG 0
61

62
#if CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF
63
64
static struct av1_token
    inter_singleref_comp_mode_encodings[INTER_SINGLEREF_COMP_MODES];
65
#endif  // CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF
66

Yaowu Xu's avatar
Yaowu Xu committed
67
static INLINE void write_uniform(aom_writer *w, int n, int v) {
68
69
  const int l = get_unsigned_bits(n);
  const int m = (1 << l) - n;
70
  if (l == 0) return;
hui su's avatar
hui su committed
71
  if (v < m) {
Yaowu Xu's avatar
Yaowu Xu committed
72
    aom_write_literal(w, v, l - 1);
hui su's avatar
hui su committed
73
  } else {
Yaowu Xu's avatar
Yaowu Xu committed
74
75
    aom_write_literal(w, m + ((v - m) >> 1), l - 1);
    aom_write_literal(w, (v - m) & 1, 1);
hui su's avatar
hui su committed
76
77
  }
}
Jingning Han's avatar
Jingning Han committed
78

79
#if CONFIG_EXT_TX
Yaowu Xu's avatar
Yaowu Xu committed
80
81
static struct av1_token ext_tx_inter_encodings[EXT_TX_SETS_INTER][TX_TYPES];
static struct av1_token ext_tx_intra_encodings[EXT_TX_SETS_INTRA][TX_TYPES];
82
#else
Yaowu Xu's avatar
Yaowu Xu committed
83
static struct av1_token ext_tx_encodings[TX_TYPES];
84
#endif  // CONFIG_EXT_TX
85
#if CONFIG_EXT_INTRA
hui su's avatar
hui su committed
86
#if CONFIG_INTRA_INTERP
Yaowu Xu's avatar
Yaowu Xu committed
87
static struct av1_token intra_filter_encodings[INTRA_FILTERS];
hui su's avatar
hui su committed
88
#endif  // CONFIG_INTRA_INTERP
89
#endif  // CONFIG_EXT_INTRA
90
#if CONFIG_EXT_INTER
91
#if CONFIG_INTERINTRA
Yaowu Xu's avatar
Yaowu Xu committed
92
static struct av1_token interintra_mode_encodings[INTERINTRA_MODES];
93
#endif
94
#if CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE
95
static struct av1_token compound_type_encodings[COMPOUND_TYPES];
96
#endif  // CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE
97
#endif  // CONFIG_EXT_INTER
98
99
#if CONFIG_LOOP_RESTORATION
static struct av1_token switchable_restore_encodings[RESTORE_SWITCHABLE_TYPES];
100
101
102
103
static void loop_restoration_write_sb_coeffs(const AV1_COMMON *const cm,
                                             MACROBLOCKD *xd,
                                             aom_writer *const w, int plane,
                                             int rtile_idx);
104
#endif  // CONFIG_LOOP_RESTORATION
105
106
107
static void write_uncompressed_header(AV1_COMP *cpi,
                                      struct aom_write_bit_buffer *wb);
static uint32_t write_compressed_header(AV1_COMP *cpi, uint8_t *data);
108
109
110
111
112
static int remux_tiles(const AV1_COMMON *const cm, uint8_t *dst,
                       const uint32_t data_size, const uint32_t max_tile_size,
                       const uint32_t max_tile_col_size,
                       int *const tile_size_bytes,
                       int *const tile_col_size_bytes);
113

Yaowu Xu's avatar
Yaowu Xu committed
114
void av1_encode_token_init(void) {
115
#if CONFIG_EXT_TX
116
  int s;
117
#endif  // CONFIG_EXT_TX
118
#if CONFIG_EXT_TX
119
  for (s = 1; s < EXT_TX_SETS_INTER; ++s) {
Yaowu Xu's avatar
Yaowu Xu committed
120
    av1_tokens_from_tree(ext_tx_inter_encodings[s], av1_ext_tx_inter_tree[s]);
121
122
  }
  for (s = 1; s < EXT_TX_SETS_INTRA; ++s) {
Yaowu Xu's avatar
Yaowu Xu committed
123
    av1_tokens_from_tree(ext_tx_intra_encodings[s], av1_ext_tx_intra_tree[s]);
124
  }
125
#else
Yaowu Xu's avatar
Yaowu Xu committed
126
  av1_tokens_from_tree(ext_tx_encodings, av1_ext_tx_tree);
127
#endif  // CONFIG_EXT_TX
128

129
#if CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP
Yaowu Xu's avatar
Yaowu Xu committed
130
  av1_tokens_from_tree(intra_filter_encodings, av1_intra_filter_tree);
131
#endif  // CONFIG_EXT_INTRA && CONFIG_INTRA_INTERP
132
#if CONFIG_EXT_INTER
133
#if CONFIG_INTERINTRA
Yaowu Xu's avatar
Yaowu Xu committed
134
  av1_tokens_from_tree(interintra_mode_encodings, av1_interintra_mode_tree);
135
#endif  // CONFIG_INTERINTRA
136
137
138
139
#if CONFIG_COMPOUND_SINGLEREF
  av1_tokens_from_tree(inter_singleref_comp_mode_encodings,
                       av1_inter_singleref_comp_mode_tree);
#endif  // CONFIG_COMPOUND_SINGLEREF
140
#if CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE
141
  av1_tokens_from_tree(compound_type_encodings, av1_compound_type_tree);
142
#endif  // CONFIG_COMPOUND_SEGMENT || CONFIG_WEDGE
143
#endif  // CONFIG_EXT_INTER
144
145
146
147
#if CONFIG_LOOP_RESTORATION
  av1_tokens_from_tree(switchable_restore_encodings,
                       av1_switchable_restore_tree);
#endif  // CONFIG_LOOP_RESTORATION
148

149
  /* This hack is necessary when CONFIG_DUAL_FILTER is enabled because the five
150
151
152
      SWITCHABLE_FILTERS are not consecutive, e.g., 0, 1, 2, 3, 4, when doing
      an in-order traversal of the av1_switchable_interp_tree structure. */
  av1_indices_from_tree(av1_switchable_interp_ind, av1_switchable_interp_inv,
Jingning Han's avatar
Jingning Han committed
153
                        av1_switchable_interp_tree);
154
155
156
/* This hack is necessary because the four TX_TYPES are not consecutive,
    e.g., 0, 1, 2, 3, when doing an in-order traversal of the av1_ext_tx_tree
    structure. */
157
158
159
#if CONFIG_EXT_TX
  for (s = 1; s < EXT_TX_SETS_INTRA; ++s)
    av1_indices_from_tree(av1_ext_tx_intra_ind[s], av1_ext_tx_intra_inv[s],
Jingning Han's avatar
Jingning Han committed
160
                          av1_ext_tx_intra_tree[s]);
161
162
  for (s = 1; s < EXT_TX_SETS_INTER; ++s)
    av1_indices_from_tree(av1_ext_tx_inter_ind[s], av1_ext_tx_inter_inv[s],
Jingning Han's avatar
Jingning Han committed
163
                          av1_ext_tx_inter_tree[s]);
164
#else
Jingning Han's avatar
Jingning Han committed
165
  av1_indices_from_tree(av1_ext_tx_ind, av1_ext_tx_inv, av1_ext_tx_tree);
David Barker's avatar
David Barker committed
166
#endif
167
168
}

Jingning Han's avatar
Jingning Han committed
169
170
171
172
static void write_intra_mode_kf(const AV1_COMMON *cm, FRAME_CONTEXT *frame_ctx,
                                const MODE_INFO *mi, const MODE_INFO *above_mi,
                                const MODE_INFO *left_mi, int block,
                                PREDICTION_MODE mode, aom_writer *w) {
Alex Converse's avatar
Alex Converse committed
173
174
175
#if CONFIG_INTRABC
  assert(!is_intrabc_block(&mi->mbmi));
#endif  // CONFIG_INTRABC
Jingning Han's avatar
Jingning Han committed
176
177
178
179
180
  aom_write_symbol(w, av1_intra_mode_ind[mode],
                   get_y_mode_cdf(frame_ctx, mi, above_mi, left_mi, block),
                   INTRA_MODES);
  (void)cm;
}
Jingning Han's avatar
Jingning Han committed
181

182
static void write_inter_mode(aom_writer *w, PREDICTION_MODE mode,
183
                             FRAME_CONTEXT *ec_ctx, const int16_t mode_ctx) {
184
  const int16_t newmv_ctx = mode_ctx & NEWMV_CTX_MASK;
Yue Chen's avatar
Yue Chen committed
185

186
187
188
189
190
#if CONFIG_NEW_MULTISYMBOL
  aom_write_symbol(w, mode != NEWMV, ec_ctx->newmv_cdf[newmv_ctx], 2);
#else
  aom_write(w, mode != NEWMV, ec_ctx->newmv_prob[newmv_ctx]);
#endif
191

Jingning Han's avatar
Jingning Han committed
192
  if (mode != NEWMV) {
193
194
195
196
197
    if (mode_ctx & (1 << ALL_ZERO_FLAG_OFFSET)) {
      assert(mode == ZEROMV);
      return;
    }

198
199
200
201
202
203
    const int16_t zeromv_ctx = (mode_ctx >> ZEROMV_OFFSET) & ZEROMV_CTX_MASK;
#if CONFIG_NEW_MULTISYMBOL
    aom_write_symbol(w, mode != ZEROMV, ec_ctx->zeromv_cdf[zeromv_ctx], 2);
#else
    aom_write(w, mode != ZEROMV, ec_ctx->zeromv_prob[zeromv_ctx]);
#endif
204
205

    if (mode != ZEROMV) {
206
207
      int16_t refmv_ctx = (mode_ctx >> REFMV_OFFSET) & REFMV_CTX_MASK;

208
209
210
      if (mode_ctx & (1 << SKIP_NEARESTMV_OFFSET)) refmv_ctx = 6;
      if (mode_ctx & (1 << SKIP_NEARMV_OFFSET)) refmv_ctx = 7;
      if (mode_ctx & (1 << SKIP_NEARESTMV_SUB8X8_OFFSET)) refmv_ctx = 8;
211
212
213
214
215
#if CONFIG_NEW_MULTISYMBOL
      aom_write_symbol(w, mode != NEARESTMV, ec_ctx->refmv_cdf[refmv_ctx], 2);
#else
      aom_write(w, mode != NEARESTMV, ec_ctx->refmv_prob[refmv_ctx]);
#endif
216
217
    }
  }
Jingning Han's avatar
Jingning Han committed
218
219
}

220
static void write_drl_idx(FRAME_CONTEXT *ec_ctx, const MB_MODE_INFO *mbmi,
Yaowu Xu's avatar
Yaowu Xu committed
221
222
                          const MB_MODE_INFO_EXT *mbmi_ext, aom_writer *w) {
  uint8_t ref_frame_type = av1_ref_frame_type(mbmi->ref_frame);
223
224
225

  assert(mbmi->ref_mv_idx < 3);

226
#if CONFIG_EXT_INTER
227
228
229
230
#if CONFIG_COMPOUND_SINGLEREF
  if (mbmi->mode == NEWMV || mbmi->mode == NEW_NEWMV ||
      mbmi->mode == SR_NEW_NEWMV) {
#else   // !CONFIG_COMPOUND_SINGLEREF
231
  if (mbmi->mode == NEWMV || mbmi->mode == NEW_NEWMV) {
232
233
#endif  // CONFIG_COMPOUND_SINGLEREF
#else   // !CONFIG_EXT_INTER
234
  if (mbmi->mode == NEWMV) {
235
#endif  // CONFIG_EXT_INTER
236
237
238
239
    int idx;
    for (idx = 0; idx < 2; ++idx) {
      if (mbmi_ext->ref_mv_count[ref_frame_type] > idx + 1) {
        uint8_t drl_ctx =
Yaowu Xu's avatar
Yaowu Xu committed
240
            av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx);
241

242
243
244
245
246
247
#if CONFIG_NEW_MULTISYMBOL
        aom_write_symbol(w, mbmi->ref_mv_idx != idx, ec_ctx->drl_cdf[drl_ctx],
                         2);
#else
        aom_write(w, mbmi->ref_mv_idx != idx, ec_ctx->drl_prob[drl_ctx]);
#endif
248
        if (mbmi->ref_mv_idx == idx) return;
249
      }
250
251
252
253
    }
    return;
  }

David Barker's avatar
David Barker committed
254
  if (have_nearmv_in_inter_mode(mbmi->mode)) {
255
256
257
258
259
    int idx;
    // TODO(jingning): Temporary solution to compensate the NEARESTMV offset.
    for (idx = 1; idx < 3; ++idx) {
      if (mbmi_ext->ref_mv_count[ref_frame_type] > idx + 1) {
        uint8_t drl_ctx =
Yaowu Xu's avatar
Yaowu Xu committed
260
            av1_drl_ctx(mbmi_ext->ref_mv_stack[ref_frame_type], idx);
261
262
263
264
265
266
#if CONFIG_NEW_MULTISYMBOL
        aom_write_symbol(w, mbmi->ref_mv_idx != (idx - 1),
                         ec_ctx->drl_cdf[drl_ctx], 2);
#else
        aom_write(w, mbmi->ref_mv_idx != (idx - 1), ec_ctx->drl_prob[drl_ctx]);
#endif
267
        if (mbmi->ref_mv_idx == (idx - 1)) return;
268
      }
269
    }
270
    return;
271
272
273
  }
}

274
#if CONFIG_EXT_INTER
275
276
static void write_inter_compound_mode(AV1_COMMON *cm, MACROBLOCKD *xd,
                                      aom_writer *w, PREDICTION_MODE mode,
277
                                      const int16_t mode_ctx) {
278
279
280
281
282
  assert(is_inter_compound_mode(mode));
  (void)cm;
  aom_write_symbol(w, INTER_COMPOUND_OFFSET(mode),
                   xd->tile_ctx->inter_compound_mode_cdf[mode_ctx],
                   INTER_COMPOUND_MODES);
283
}
284
285

#if CONFIG_COMPOUND_SINGLEREF
286
static void write_inter_singleref_comp_mode(MACROBLOCKD *xd, aom_writer *w,
287
288
289
                                            PREDICTION_MODE mode,
                                            const int16_t mode_ctx) {
  assert(is_inter_singleref_comp_mode(mode));
290
291
  aom_cdf_prob *const inter_singleref_comp_cdf =
      xd->tile_ctx->inter_singleref_comp_mode_cdf[mode_ctx];
292

293
294
  aom_write_symbol(w, INTER_SINGLEREF_COMP_OFFSET(mode),
                   inter_singleref_comp_cdf, INTER_SINGLEREF_COMP_MODES);
295
296
}
#endif  // CONFIG_COMPOUND_SINGLEREF
297
298
#endif  // CONFIG_EXT_INTER

Yaowu Xu's avatar
Yaowu Xu committed
299
static void encode_unsigned_max(struct aom_write_bit_buffer *wb, int data,
300
                                int max) {
Yaowu Xu's avatar
Yaowu Xu committed
301
  aom_wb_write_literal(wb, data, get_unsigned_bits(max));
Jingning Han's avatar
Jingning Han committed
302
303
}

304
#if CONFIG_VAR_TX
305
static void write_tx_size_vartx(const AV1_COMMON *cm, MACROBLOCKD *xd,
306
                                const MB_MODE_INFO *mbmi, TX_SIZE tx_size,
307
308
                                int depth, int blk_row, int blk_col,
                                aom_writer *w) {
309
310
311
312
#if CONFIG_NEW_MULTISYMBOL
  FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
  (void)cm;
#endif
313
314
  const int tx_row = blk_row >> 1;
  const int tx_col = blk_col >> 1;
315
316
317
  const int max_blocks_high = max_block_high(xd, mbmi->sb_type, 0);
  const int max_blocks_wide = max_block_wide(xd, mbmi->sb_type, 0);

318
319
  int ctx = txfm_partition_context(xd->above_txfm_context + blk_col,
                                   xd->left_txfm_context + blk_row,
320
                                   mbmi->sb_type, tx_size);
321

322
  if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
323

Jingning Han's avatar
Jingning Han committed
324
  if (depth == MAX_VARTX_DEPTH) {
325
326
    txfm_partition_update(xd->above_txfm_context + blk_col,
                          xd->left_txfm_context + blk_row, tx_size, tx_size);
327
328
329
    return;
  }

Yue Chen's avatar
Yue Chen committed
330
331
332
333
#if CONFIG_RECT_TX_EXT
  if (tx_size == mbmi->inter_tx_size[tx_row][tx_col] ||
      mbmi->tx_size == quarter_txsize_lookup[mbmi->sb_type]) {
#else
334
  if (tx_size == mbmi->inter_tx_size[tx_row][tx_col]) {
Yue Chen's avatar
Yue Chen committed
335
#endif
336
337
338
#if CONFIG_NEW_MULTISYMBOL
    aom_write_symbol(w, 0, ec_ctx->txfm_partition_cdf[ctx], 2);
#else
Yaowu Xu's avatar
Yaowu Xu committed
339
    aom_write(w, 0, cm->fc->txfm_partition_prob[ctx]);
340
341
#endif

342
343
    txfm_partition_update(xd->above_txfm_context + blk_col,
                          xd->left_txfm_context + blk_row, tx_size, tx_size);
Yue Chen's avatar
Yue Chen committed
344
    // TODO(yuec): set correct txfm partition update for qttx
345
  } else {
346
347
    const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
    const int bsl = tx_size_wide_unit[sub_txs];
348
    int i;
349

350
351
352
#if CONFIG_NEW_MULTISYMBOL
    aom_write_symbol(w, 1, ec_ctx->txfm_partition_cdf[ctx], 2);
#else
Yaowu Xu's avatar
Yaowu Xu committed
353
    aom_write(w, 1, cm->fc->txfm_partition_prob[ctx]);
354
#endif
355

David Barker's avatar
David Barker committed
356
    if (sub_txs == TX_4X4) {
357
358
      txfm_partition_update(xd->above_txfm_context + blk_col,
                            xd->left_txfm_context + blk_row, sub_txs, tx_size);
359
      return;
360
    }
361
362
363

    assert(bsl > 0);
    for (i = 0; i < 4; ++i) {
364
365
366
367
      int offsetr = blk_row + (i >> 1) * bsl;
      int offsetc = blk_col + (i & 0x01) * bsl;
      write_tx_size_vartx(cm, xd, mbmi, sub_txs, depth + 1, offsetr, offsetc,
                          w);
368
369
370
    }
  }
}
371

372
#if !CONFIG_NEW_MULTISYMBOL
Yaowu Xu's avatar
Yaowu Xu committed
373
static void update_txfm_partition_probs(AV1_COMMON *cm, aom_writer *w,
374
                                        FRAME_COUNTS *counts, int probwt) {
375
376
  int k;
  for (k = 0; k < TXFM_PARTITION_CONTEXTS; ++k)
Yaowu Xu's avatar
Yaowu Xu committed
377
    av1_cond_prob_diff_update(w, &cm->fc->txfm_partition_prob[k],
378
                              counts->txfm_partition[k], probwt);
379
}
380
#endif  // CONFIG_NEW_MULTISYMBOL
David Barker's avatar
David Barker committed
381
#endif  // CONFIG_VAR_TX
382

Yaowu Xu's avatar
Yaowu Xu committed
383
384
static void write_selected_tx_size(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                                   aom_writer *w) {
385
386
  const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
  const BLOCK_SIZE bsize = mbmi->sb_type;
387
388
  FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
  (void)cm;
389
// For sub8x8 blocks the tx_size symbol does not need to be sent
390
#if CONFIG_CB4X4 && (CONFIG_VAR_TX || CONFIG_EXT_TX) && CONFIG_RECT_TX
391
392
  if (bsize > BLOCK_4X4) {
#else
393
  if (bsize >= BLOCK_8X8) {
394
#endif
395
396
397
398
399
400
    const TX_SIZE tx_size = mbmi->tx_size;
    const int is_inter = is_inter_block(mbmi);
    const int tx_size_ctx = get_tx_size_context(xd);
    const int tx_size_cat = is_inter ? inter_tx_size_cat_lookup[bsize]
                                     : intra_tx_size_cat_lookup[bsize];
    const TX_SIZE coded_tx_size = txsize_sqr_up_map[tx_size];
401
    const int depth = tx_size_to_depth(coded_tx_size);
402
#if CONFIG_EXT_TX && CONFIG_RECT_TX
403
    assert(IMPLIES(is_rect_tx(tx_size), is_rect_tx_allowed(xd, mbmi)));
404
405
#endif  // CONFIG_EXT_TX && CONFIG_RECT_TX

406
    aom_write_symbol(w, depth, ec_ctx->tx_size_cdf[tx_size_cat][tx_size_ctx],
407
                     tx_size_cat + 2);
Yue Chen's avatar
Yue Chen committed
408
#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
409
410
411
    if (is_quarter_tx_allowed(xd, mbmi, is_inter) && tx_size != coded_tx_size)
      aom_write(w, tx_size == quarter_txsize_lookup[bsize],
                cm->fc->quarter_tx_size_prob);
Yue Chen's avatar
Yue Chen committed
412
#endif
Jingning Han's avatar
Jingning Han committed
413
414
415
  }
}

416
#if !CONFIG_NEW_MULTISYMBOL
Yaowu Xu's avatar
Yaowu Xu committed
417
static void update_inter_mode_probs(AV1_COMMON *cm, aom_writer *w,
418
419
                                    FRAME_COUNTS *counts) {
  int i;
420
  const int probwt = cm->num_tg;
421
  for (i = 0; i < NEWMV_MODE_CONTEXTS; ++i)
422
423
    av1_cond_prob_diff_update(w, &cm->fc->newmv_prob[i], counts->newmv_mode[i],
                              probwt);
424
  for (i = 0; i < ZEROMV_MODE_CONTEXTS; ++i)
Yaowu Xu's avatar
Yaowu Xu committed
425
    av1_cond_prob_diff_update(w, &cm->fc->zeromv_prob[i],
426
                              counts->zeromv_mode[i], probwt);
427
  for (i = 0; i < REFMV_MODE_CONTEXTS; ++i)
428
429
    av1_cond_prob_diff_update(w, &cm->fc->refmv_prob[i], counts->refmv_mode[i],
                              probwt);
430
  for (i = 0; i < DRL_MODE_CONTEXTS; ++i)
431
432
    av1_cond_prob_diff_update(w, &cm->fc->drl_prob[i], counts->drl_mode[i],
                              probwt);
433
}
434
#endif
435

Yaowu Xu's avatar
Yaowu Xu committed
436
437
static int write_skip(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                      int segment_id, const MODE_INFO *mi, aom_writer *w) {
Jingning Han's avatar
Jingning Han committed
438
439
440
441
  if (segfeature_active(&cm->seg, segment_id, SEG_LVL_SKIP)) {
    return 1;
  } else {
    const int skip = mi->mbmi.skip;
442
443
444
445
446
#if CONFIG_NEW_MULTISYMBOL
    FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
    const int ctx = av1_get_skip_context(xd);
    aom_write_symbol(w, skip, ec_ctx->skip_cdfs[ctx], 2);
#else
Yaowu Xu's avatar
Yaowu Xu committed
447
    aom_write(w, skip, av1_get_skip_prob(cm, xd));
448
#endif
Jingning Han's avatar
Jingning Han committed
449
450
451
452
    return skip;
  }
}

453
454
455
456
457
458
459
460
461
462
463
464
465
static void write_is_inter(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                           int segment_id, aom_writer *w, const int is_inter) {
  if (!segfeature_active(&cm->seg, segment_id, SEG_LVL_REF_FRAME)) {
#if CONFIG_NEW_MULTISYMBOL
    FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
    const int ctx = av1_get_intra_inter_context(xd);
    aom_write_symbol(w, is_inter, ec_ctx->intra_inter_cdf[ctx], 2);
#else
    aom_write(w, is_inter, av1_get_intra_inter_prob(cm, xd));
#endif
  }
}

Yue Chen's avatar
Yue Chen committed
466
#if CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION
467
468
static void write_motion_mode(const AV1_COMMON *cm, MACROBLOCKD *xd,
                              const MODE_INFO *mi, aom_writer *w) {
469
  const MB_MODE_INFO *mbmi = &mi->mbmi;
470

471
472
473
474
475
476
#if !CONFIG_GLOBAL_MOTION
  // The cm parameter is only used with global_motion or with
  // motion_var and warped_motion. In other cases, explicitly ignore
  // it to avoid a compiler warning.
  (void)cm;
#endif
477
  MOTION_MODE last_motion_mode_allowed = motion_mode_allowed(
478
#if CONFIG_GLOBAL_MOTION
479
      0, cm->global_motion,
480
#endif  // CONFIG_GLOBAL_MOTION
481
482
483
#if CONFIG_WARPED_MOTION
      xd,
#endif
484
      mi);
Yue Chen's avatar
Yue Chen committed
485
486
  if (last_motion_mode_allowed == SIMPLE_TRANSLATION) return;
#if CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION
487
488
489
490
491
492
493
494
495
496
#if CONFIG_NCOBMC_ADAPT_WEIGHT
  if (last_motion_mode_allowed == NCOBMC_ADAPT_WEIGHT) {
    aom_write_symbol(w, mbmi->motion_mode,
                     xd->tile_ctx->ncobmc_cdf[mbmi->sb_type],
                     OBMC_FAMILY_MODES);
  } else if (last_motion_mode_allowed == OBMC_CAUSAL) {
    aom_write_symbol(w, mbmi->motion_mode == OBMC_CAUSAL,
                     xd->tile_ctx->obmc_cdf[mbmi->sb_type], 2);
  } else {
#else
Yue Chen's avatar
Yue Chen committed
497
  if (last_motion_mode_allowed == OBMC_CAUSAL) {
498
499
500
501
#if CONFIG_NEW_MULTISYMBOL
    aom_write_symbol(w, mbmi->motion_mode == OBMC_CAUSAL,
                     xd->tile_ctx->obmc_cdf[mbmi->sb_type], 2);
#else
Yue Chen's avatar
Yue Chen committed
502
503
    aom_write(w, mbmi->motion_mode == OBMC_CAUSAL,
              cm->fc->obmc_prob[mbmi->sb_type]);
504
#endif
Yue Chen's avatar
Yue Chen committed
505
  } else {
506
#endif  // CONFIG_NCOBMC_ADAPT_WEIGHT
Yue Chen's avatar
Yue Chen committed
507
#endif  // CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION
Thomas Davies's avatar
Thomas Davies committed
508
509
510
    aom_write_symbol(w, mbmi->motion_mode,
                     xd->tile_ctx->motion_mode_cdf[mbmi->sb_type],
                     MOTION_MODES);
Yue Chen's avatar
Yue Chen committed
511
512
513
514
#if CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION
  }
#endif  // CONFIG_MOTION_VAR && CONFIG_WARPED_MOTION
}
515
516

#if CONFIG_NCOBMC_ADAPT_WEIGHT
517
static void write_ncobmc_mode(MACROBLOCKD *xd, const MODE_INFO *mi,
518
519
520
                              aom_writer *w) {
  const MB_MODE_INFO *mbmi = &mi->mbmi;
  ADAPT_OVERLAP_BLOCK ao_block = adapt_overlap_block_lookup[mbmi->sb_type];
521
  if (mbmi->motion_mode != NCOBMC_ADAPT_WEIGHT) return;
522

523
524
  aom_write_symbol(w, mbmi->ncobmc_mode[0],
                   xd->tile_ctx->ncobmc_mode_cdf[ao_block], MAX_NCOBMC_MODES);
525
  if (mi_size_wide[mbmi->sb_type] != mi_size_high[mbmi->sb_type]) {
526
527
    aom_write_symbol(w, mbmi->ncobmc_mode[1],
                     xd->tile_ctx->ncobmc_mode_cdf[ao_block], MAX_NCOBMC_MODES);
528
529
530
  }
}
#endif
Yue Chen's avatar
Yue Chen committed
531
532
#endif  // CONFIG_MOTION_VAR || CONFIG_WARPED_MOTION

533
#if CONFIG_DELTA_Q
Thomas Davies's avatar
Thomas Davies committed
534
535
static void write_delta_qindex(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                               int delta_qindex, aom_writer *w) {
536
537
  int sign = delta_qindex < 0;
  int abs = sign ? -delta_qindex : delta_qindex;
Thomas Davies's avatar
Thomas Davies committed
538
  int rem_bits, thr;
539
  int smallval = abs < DELTA_Q_SMALL ? 1 : 0;
Thomas Davies's avatar
Thomas Davies committed
540
541
  FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
  (void)cm;
542

Thomas Davies's avatar
Thomas Davies committed
543
544
  aom_write_symbol(w, AOMMIN(abs, DELTA_Q_SMALL), ec_ctx->delta_q_cdf,
                   DELTA_Q_PROBS + 1);
545
546
547
548
549
550

  if (!smallval) {
    rem_bits = OD_ILOG_NZ(abs - 1) - 1;
    thr = (1 << rem_bits) + 1;
    aom_write_literal(w, rem_bits, 3);
    aom_write_literal(w, abs - thr, rem_bits);
551
552
553
554
555
  }
  if (abs > 0) {
    aom_write_bit(w, sign);
  }
}
556

Fangwen Fu's avatar
Fangwen Fu committed
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
#if CONFIG_EXT_DELTA_Q
static void write_delta_lflevel(const AV1_COMMON *cm, const MACROBLOCKD *xd,
                                int delta_lflevel, aom_writer *w) {
  int sign = delta_lflevel < 0;
  int abs = sign ? -delta_lflevel : delta_lflevel;
  int rem_bits, thr;
  int smallval = abs < DELTA_LF_SMALL ? 1 : 0;
  FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
  (void)cm;

  aom_write_symbol(w, AOMMIN(abs, DELTA_LF_SMALL), ec_ctx->delta_lf_cdf,
                   DELTA_LF_PROBS + 1);

  if (!smallval) {
    rem_bits = OD_ILOG_NZ(abs - 1) - 1;
    thr = (1 << rem_bits) + 1;
    aom_write_literal(w, rem_bits, 3);
    aom_write_literal(w, abs - thr, rem_bits);
  }
  if (abs > 0) {
    aom_write_bit(w, sign);
  }
}
#endif  // CONFIG_EXT_DELTA_Q
Thomas Davies's avatar
Thomas Davies committed
581
#endif  // CONFIG_DELTA_Q
582

583
#if !CONFIG_NEW_MULTISYMBOL
Yaowu Xu's avatar
Yaowu Xu committed
584
static void update_skip_probs(AV1_COMMON *cm, aom_writer *w,
Jingning Han's avatar
Jingning Han committed
585
586
                              FRAME_COUNTS *counts) {
  int k;
587
588
589
590
591
  const int probwt = cm->num_tg;
  for (k = 0; k < SKIP_CONTEXTS; ++k) {
    av1_cond_prob_diff_update(w, &cm->fc->skip_probs[k], counts->skip[k],
                              probwt);
  }
Jingning Han's avatar
Jingning Han committed
592
}
593
#endif
Jingning Han's avatar
Jingning Han committed
594

Sarah Parker's avatar
Sarah Parker committed
595
596
static void pack_map_tokens(aom_writer *w, const TOKENEXTRA **tp, int n,
                            int num) {
597
  const TOKENEXTRA *p = *tp;
598
599
600
601
  write_uniform(w, n, p->token);  // The first color index.
  ++p;
  --num;
  for (int i = 0; i < num; ++i) {
602
    aom_write_symbol(w, p->token, p->color_map_cdf, n);
hui su's avatar
hui su committed
603
604
605
606
    ++p;
  }
  *tp = p;
}
607

608
#if !CONFIG_PVQ
609
#if CONFIG_SUPERTX
610
static void update_supertx_probs(AV1_COMMON *cm, int probwt, aom_writer *w) {
Yaowu Xu's avatar
Yaowu Xu committed
611
612
  const int savings_thresh = av1_cost_one(GROUP_DIFF_UPDATE_PROB) -
                             av1_cost_zero(GROUP_DIFF_UPDATE_PROB);
613
614
615
616
  int i, j;
  int savings = 0;
  int do_update = 0;
  for (i = 0; i < PARTITION_SUPERTX_CONTEXTS; ++i) {
Jingning Han's avatar
Jingning Han committed
617
    for (j = TX_8X8; j < TX_SIZES; ++j) {
618
619
      savings += av1_cond_prob_diff_update_savings(
          &cm->fc->supertx_prob[i][j], cm->counts.supertx[i][j], probwt);
620
621
622
    }
  }
  do_update = savings > savings_thresh;
Yaowu Xu's avatar
Yaowu Xu committed
623
  aom_write(w, do_update, GROUP_DIFF_UPDATE_PROB);
624
625
  if (do_update) {
    for (i = 0; i < PARTITION_SUPERTX_CONTEXTS; ++i) {
Jingning Han's avatar
Jingning Han committed
626
      for (j = TX_8X8; j < TX_SIZES; ++j) {
Yaowu Xu's avatar
Yaowu Xu committed
627
        av1_cond_prob_diff_update(w, &cm->fc->supertx_prob[i][j],
628
                                  cm->counts.supertx[i][j], probwt);
629
630
631
632
633
634
      }
    }
  }
}
#endif  // CONFIG_SUPERTX

635
#if !CONFIG_LV_MAP
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
#if CONFIG_NEW_MULTISYMBOL
static INLINE void write_coeff_extra(const aom_cdf_prob *const *cdf, int val,
                                     int n, aom_writer *w) {
  // Code the extra bits from LSB to MSB in groups of 4
  int i = 0;
  int count = 0;
  while (count < n) {
    const int size = AOMMIN(n - count, 4);
    const int mask = (1 << size) - 1;
    aom_write_cdf(w, val & mask, cdf[i++], 1 << size);
    val >>= size;
    count += size;
  }
}
#else
static INLINE void write_coeff_extra(const aom_prob *pb, int value,
                                     int num_bits, int skip_bits, aom_writer *w,
                                     TOKEN_STATS *token_stats) {
  // Code the extra bits from MSB to LSB 1 bit at a time
  int index;
  for (index = skip_bits; index < num_bits; ++index) {
    const int shift = num_bits - index - 1;
    const int bb = (value >> shift) & 1;
    aom_write_record(w, bb, pb[index], token_stats);
  }
}
662
#endif  // CONFIG_NEW_MULTISYMBOL
663

Yaowu Xu's avatar
Yaowu Xu committed
664
static void pack_mb_tokens(aom_writer *w, const TOKENEXTRA **tp,
665
                           const TOKENEXTRA *const stop,
666
                           aom_bit_depth_t bit_depth, const TX_SIZE tx_size,
Sarah Parker's avatar
Sarah Parker committed
667
668
669
#if CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
                           TX_TYPE tx_type, int is_inter,
#endif  // CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
670
                           TOKEN_STATS *token_stats) {
671
  const TOKENEXTRA *p = *tp;
Fangwen Fu's avatar
Fangwen Fu committed
672
#if CONFIG_VAR_TX
673
  int count = 0;
674
  const int seg_eob = tx_size_2d[tx_size];
675
#endif
Jingning Han's avatar
Jingning Han committed
676

Sarah Parker's avatar
Sarah Parker committed
677
678
679
680
681
682
683
684
685
686
687
#if CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
  if (tx_type == MRC_DCT && ((is_inter && SIGNAL_MRC_MASK_INTER) ||
                             (!is_inter && SIGNAL_MRC_MASK_INTRA))) {
    int rows = tx_size_high[tx_size];
    int cols = tx_size_wide[tx_size];
    assert(tx_size == TX_32X32);
    assert(p < stop);
    pack_map_tokens(w, &p, 2, rows * cols);
  }
#endif  // CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK

Jingning Han's avatar
Jingning Han committed
688
  while (p < stop && p->token != EOSB_TOKEN) {
689
    const int token = p->token;
690
    const int eob_val = p->eob_val;
691
    if (token == BLOCK_Z_TOKEN) {
692
      aom_write_symbol(w, 0, *p->head_cdf, HEAD_TOKENS + 1);
693
      p++;
694
695
696
#if CONFIG_VAR_TX
      break;
#endif
697
698
      continue;
    }
Yaowu Xu's avatar
Yaowu Xu committed
699
700

    const av1_extra_bit *const extra_bits = &av1_extra_bits[token];
701
    if (eob_val == LAST_EOB) {
702
703
704
      // Just code a flag indicating whether the value is >1 or 1.
      aom_write_bit(w, token != ONE_TOKEN);
    } else {
705
      int comb_symb = 2 * AOMMIN(token, TWO_TOKEN) - eob_val + p->first_val;
706
      aom_write_symbol(w, comb_symb, *p->head_cdf, HEAD_TOKENS + p->first_val);
707
    }
708
    if (token > ONE_TOKEN) {
709
      aom_write_symbol(w, token - TWO_TOKEN, *p->tail_cdf, TAIL_TOKENS);
Alex Converse's avatar
Alex Converse committed
710
    }
711
712
713
714

    if (extra_bits->base_val) {
      const int bit_string = p->extra;
      const int bit_string_length = extra_bits->len;  // Length of extra bits to
715
      const int is_cat6 = (extra_bits->base_val == CAT6_MIN_VAL);
716
717
      // be written excluding
      // the sign bit.
718
      int skip_bits = is_cat6
719
720
721
                          ? (int)sizeof(av1_cat6_prob) -
                                av1_get_cat6_extrabits_size(tx_size, bit_depth)
                          : 0;
722

723
724
725
726
727
728
729
730
731
      assert(!(bit_string >> (bit_string_length - skip_bits + 1)));
      if (bit_string_length > 0)
#if CONFIG_NEW_MULTISYMBOL
        write_coeff_extra(extra_bits->cdf, bit_string >> 1,
                          bit_string_length - skip_bits, w);
#else
        write_coeff_extra(extra_bits->prob, bit_string >> 1, bit_string_length,
                          skip_bits, w, token_stats);
#endif
732
733
734
735
736

      aom_write_bit_record(w, bit_string & 1, token_stats);
    }
    ++p;

Fangwen Fu's avatar
Fangwen Fu committed
737
#if CONFIG_VAR_TX
738
    ++count;
739
    if (eob_val == EARLY_EOB || count == seg_eob) break;
740
741
742
743
744
#endif
  }

  *tp = p;
}
Angie Chiang's avatar
Angie Chiang committed
745
#endif  // !CONFIG_LV_MAP
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
#else   // !CONFIG_PVQ
static PVQ_INFO *get_pvq_block(PVQ_QUEUE *pvq_q) {
  PVQ_INFO *pvq;

  assert(pvq_q->curr_pos <= pvq_q->last_pos);
  assert(pvq_q->curr_pos < pvq_q->buf_len);

  pvq = pvq_q->buf + pvq_q->curr_pos;
  ++pvq_q->curr_pos;

  return pvq;
}

static void pack_pvq_tokens(aom_writer *w, MACROBLOCK *const x,
                            MACROBLOCKD *const xd, int plane, BLOCK_SIZE bsize,
                            const TX_SIZE tx_size) {
  PVQ_INFO *pvq;
  int idx, idy;
  const struct macroblockd_plane *const pd = &xd->plane[plane];
  od_adapt_ctx *adapt;
  int max_blocks_wide;
  int max_blocks_high;
  int step = (1 << tx_size);
769
770

#if CONFIG_CHROMA_SUB8X8
771
  const BLOCK_SIZE plane_bsize =
772
773
774
775
776
777
778
      AOMMAX(BLOCK_4X4, get_plane_block_size(bsize, pd));
#elif CONFIG_CB4X4
  const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd);
#else
  const BLOCK_SIZE plane_bsize =
      get_plane_block_size(AOMMAX(BLOCK_8X8, bsize), pd);
#endif
779

780
  adapt = x->daala_enc.state.adapt;
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798

  max_blocks_wide = max_block_wide(xd, plane_bsize, plane);
  max_blocks_high = max_block_high(xd, plane_bsize, plane);

  for (idy = 0; idy < max_blocks_high; idy += step) {
    for (idx = 0; idx < max_blocks_wide; idx += step) {
      const int is_keyframe = 0;
      const int encode_flip = 0;
      const int flip = 0;
      int i;
      const int has_dc_skip = 1;
      int *exg = &adapt->pvq.pvq_exg[plane][tx_size][0];
      int *ext = adapt->pvq.pvq_ext + tx_size * PVQ_MAX_PARTITIONS;
      generic_encoder *model = adapt->pvq.pvq_param_model;

      pvq = get_pvq_block(x->pvq_q);

      // encode block skip info
799
800
      aom_write_symbol(w, pvq->ac_dc_coded,
                       adapt->skip_cdf[2 * tx_size + (plane != 0)], 4);
801
802
803
804
805
806
807
808

      // AC coeffs coded?
      if (pvq->ac_dc_coded & AC_CODED) {
        assert(pvq->bs == tx_size);
        for (i = 0; i < pvq->nb_bands; i++) {
          if (i == 0 ||
              (!pvq->skip_rest && !(pvq->skip_dir & (1 << ((i - 1) % 3))))) {
            pvq_encode_partition(
809
810
                w, pvq->qg[i], pvq->theta[i], pvq->y + pvq->off[i],
                pvq->size[i], pvq->k[i], model, adapt, exg + i, ext + i,
811
812
813
814
815
816
                (plane != 0) * OD_TXSIZES * PVQ_MAX_PARTITIONS +
                    pvq->bs * PVQ_MAX_PARTITIONS + i,
                is_keyframe, i == 0 && (i < pvq->nb_bands - 1), pvq->skip_rest,
                encode_flip, flip);
          }
          if (i == 0 && !pvq->skip_rest && pvq->bs > 0) {
817
            aom_write_symbol(
818
819
820
                w, pvq->skip_dir,
                &adapt->pvq
                     .pvq_skip_dir_cdf[(plane != 0) + 2 * (pvq->bs - 1)][0],
821
                7);
822
823
824
825
826
827
          }
        }
      }
      // Encode residue of DC coeff, if exist.
      if (!has_dc_skip || (pvq->ac_dc_coded & DC_CODED)) {
        generic_encode(w, &adapt->model_dc[plane],
828
                       abs(pvq->dq_dc_residue) - has_dc_skip,
829
830
831
832
833
834
835
836
                       &adapt->ex_dc[plane][pvq->bs][0], 2);
      }
      if ((pvq->ac_dc_coded & DC_CODED)) {
        aom_write_bit(w, pvq->dq_dc_residue < 0);
      }
    }
  }  // for (idy = 0;
}
837
#endif  // !CONFIG_PVG
838

839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
#if CONFIG_VAR_TX && !CONFIG_COEF_INTERLEAVE
#if CONFIG_LV_MAP
static void pack_txb_tokens(aom_writer *w,
#if CONFIG_LV_MAP
                            AV1_COMMON *cm,
#endif  // CONFIG_LV_MAP
                            const TOKENEXTRA **tp,
                            const TOKENEXTRA *const tok_end,
#if CONFIG_PVQ || CONFIG_LV_MAP
                            MACROBLOCK *const x,
#endif
                            MACROBLOCKD *xd, MB_MODE_INFO *mbmi, int plane,
                            BLOCK_SIZE plane_bsize, aom_bit_depth_t bit_depth,
                            int block, int blk_row, int blk_col,
                            TX_SIZE tx_size, TOKEN_STATS *token_stats) {
  const struct macroblockd_plane *const pd = &xd->plane[plane];
  const BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
  const int tx_row = blk_row >> (1 - pd->subsampling_y);
  const int tx_col = blk_col >> (1 - pd->subsampling_x);
  TX_SIZE plane_tx_size;
  const int max_blocks_high = max_block_high(xd, plane_bsize, plane);
  const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane);

  if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;

  plane_tx_size =
      plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0]
            : mbmi->inter_tx_size[tx_row][tx_col];

  if (tx_size == plane_tx_size) {
    TOKEN_STATS tmp_token_stats;
    init_token_stats(&tmp_token_stats);

#if !CONFIG_PVQ
    tran_low_t *tcoeff = BLOCK_OFFSET(x->mbmi_ext->tcoeff[plane], block);
    uint16_t eob = x->mbmi_ext->eobs[plane][block];
    TXB_CTX txb_ctx = { x->mbmi_ext->txb_skip_ctx[plane][block],
                        x->mbmi_ext->dc_sign_ctx[plane][block] };
Jingning Han's avatar
Jingning Han committed
877
878
    av1_write_coeffs_txb(cm, xd, w, blk_row, blk_col, block, plane, tx_size,
                         tcoeff, eob, &txb_ctx);
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
#else
    pack_pvq_tokens(w, x, xd, plane, bsize, tx_size);
#endif
#if CONFIG_RD_DEBUG
    token_stats->txb_coeff_cost_map[blk_row][blk_col] = tmp_token_stats.cost;
    token_stats->cost += tmp_token_stats.cost;
#endif
  } else {
    const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
    const int bsl = tx_size_wide_unit[sub_txs];
    int i;

    assert(bsl > 0);

    for (i = 0; i < 4; ++i) {
      const int offsetr = blk_row + (i >> 1) * bsl;
      const int offsetc = blk_col + (i & 0x01) * bsl;
      const int step = tx_size_wide_unit[sub_txs] * tx_size_high_unit[sub_txs];

      if (offsetr >= max_blocks_high || offsetc >= max_blocks_wide) continue;

      pack_txb_tokens(w,
#if CONFIG_LV_MAP
                      cm,
#endif
                      tp, tok_end,
#if CONFIG_PVQ || CONFIG_LV_MAP
                      x,
#endif
                      xd, mbmi, plane, plane_bsize, bit_depth, block, offsetr,
                      offsetc, sub_txs, token_stats);
      block += step;
    }
  }
}
#else  // CONFIG_LV_MAP
Yaowu Xu's avatar
Yaowu Xu committed
915
static void pack_txb_tokens(aom_writer *w, const TOKENEXTRA **tp,
916
917
918
919
920
                            const TOKENEXTRA *const tok_end,
#if CONFIG_PVQ
                            MACROBLOCK *const x,
#endif
                            MACROBLOCKD *xd, MB_MODE_INFO *mbmi, int plane,
Yaowu Xu's avatar
Yaowu Xu committed
921
                            BLOCK_SIZE plane_bsize, aom_bit_depth_t bit_depth,
922
                            int block, int blk_row, int blk_col,
923
                            TX_SIZE tx_size, TOKEN_STATS *token_stats) {
924
925
  const struct macroblockd_plane *const pd = &xd->plane[plane];
  const BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
926
927
  const int tx_row = blk_row >> (1 - pd->subsampling_y);
  const int tx_col = blk_col >> (1 - pd->subsampling_x);
Debargha Mukherjee's avatar
Debargha Mukherjee committed
928
  TX_SIZE plane_tx_size;
929
930
  const int max_blocks_high = max_block_high(xd, plane_bsize, plane);
  const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane);
Sarah Parker's avatar
Sarah Parker committed
931
932
933
934
#if CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
  TX_TYPE tx_type = av1_get_tx_type(plane ? PLANE_TYPE_UV : PLANE_TYPE_Y, xd,
                                    blk_row, blk_col, block, tx_size);
#endif  // CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
935

936
  if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
937

938
939
940
  plane_tx_size =
      plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0]
            : mbmi->inter_tx_size[tx_row][tx_col];
Debargha Mukherjee's avatar
Debargha Mukherjee committed
941

942
  if (tx_size == plane_tx_size) {
943
944
    TOKEN_STATS tmp_token_stats;
    init_token_stats(&tmp_token_stats);
945
#if !CONFIG_PVQ
Sarah Parker's avatar
Sarah Parker committed
946
947
948
949
950
    pack_mb_tokens(w, tp, tok_end, bit_depth, tx_size,
#if CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
                   tx_type, is_inter_block(mbmi),
#endif  // CONFIG_MRC_TX && SIGNAL_ANY_MRC_MASK
                   &tmp_token_stats);
951
952
953
#else
    pack_pvq_tokens(w, x, xd, plane, bsize, tx_size);
#endif
954
955
956
957
#if CONFIG_RD_DEBUG
    token_stats->txb_coeff_cost_map[blk_row][blk_col] = tmp_token_stats.cost;
    token_stats->cost += tmp_token_stats.cost;
#endif
958
  } else {
Yue Chen's avatar
Yue Chen committed
959
960
961
962
#if CONFIG_RECT_TX_EXT
    int is_qttx = plane_tx_size == quarter_txsize_lookup[plane_bsize];
    const TX_SIZE sub_txs = is_qttx ? plane_tx_size : sub_tx_size_map[tx_size];
#else
963
    const TX_SIZE sub_txs = sub_tx_size_map[tx_size];
Yue Chen's avatar
Yue Chen committed
964
#endif
965
    const int bsl = tx_size_wide_unit[sub_txs];
966
967
968
969
970
    int i;

    assert(bsl > 0);

    for (i = 0; i < 4; ++i) {
Yue Chen's avatar
Yue Chen committed
971
972
973
974
975
976
977
978
979
#if CONFIG_RECT_TX_EXT
      int is_wide_tx = tx_size_wide_unit[sub_txs] > tx_size_high_unit[sub_txs];
      const int offsetr =
          is_qttx ? (is_wide_tx ? i * tx_size_high_unit[sub_txs] : 0)
                  : blk_row + (i >> 1) * bsl;
      const int offsetc =
          is_qttx ? (is_wide_tx ? 0 : i * tx_size_wide_unit[sub_txs])
                  : blk_col + (i & 0x01) * bsl;
#else
980
981
      const int offsetr = blk_row + (i >> 1) * bsl;
      const int offsetc = blk_col + (i & 0x01) * bsl;
Yue Chen's avatar
Yue Chen committed
982