tokenize.c 43 KB
Newer Older
John Koleszar's avatar
John Koleszar committed
1
/*
2
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
John Koleszar's avatar
John Koleszar committed
3
 *
4
 *  Use of this source code is governed by a BSD-style license
5
6
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
7
 *  in the file PATENTS.  All contributing project authors may
8
 *  be found in the AUTHORS file in the root of the source tree.
John Koleszar's avatar
John Koleszar committed
9
10
11
12
13
14
15
16
17
18
19
 */


#include <math.h>
#include <stdio.h>
#include <string.h>
#include <assert.h>
#include "onyx_int.h"
#include "tokenize.h"
#include "vpx_mem/vpx_mem.h"

20
#include "vp8/common/pred_common.h"
21
22
#include "vp8/common/seg_common.h"

John Koleszar's avatar
John Koleszar committed
23
24
25
26
/* Global event counters used for accumulating statistics across several
   compressions, then generating context.c = initial stats. */

#ifdef ENTROPY_STATS
27
INT64 context_counters[BLOCK_TYPES] [COEF_BANDS] [PREV_COEF_CONTEXTS] [MAX_ENTROPY_TOKENS];
28
INT64 context_counters_8x8[BLOCK_TYPES_8X8] [COEF_BANDS] [PREV_COEF_CONTEXTS] [MAX_ENTROPY_TOKENS];
Daniel Kang's avatar
Daniel Kang committed
29
30
31
32
33
34
35
36
37
38
39
#if CONFIG_TX16X16
INT64 context_counters_16x16[BLOCK_TYPES_16X16] [COEF_BANDS] [PREV_COEF_CONTEXTS] [MAX_ENTROPY_TOKENS];
#endif
extern unsigned int tree_update_hist[BLOCK_TYPES][COEF_BANDS]
                    [PREV_COEF_CONTEXTS][ENTROPY_NODES][2];
extern unsigned int tree_update_hist_8x8[BLOCK_TYPES_8X8][COEF_BANDS]
                    [PREV_COEF_CONTEXTS][ENTROPY_NODES] [2];
#if CONFIG_TX16X16
extern unsigned int tree_update_hist_16x16[BLOCK_TYPES_16X16][COEF_BANDS]
                    [PREV_COEF_CONTEXTS][ENTROPY_NODES] [2];
#endif
40
#endif
41
42
43
44
void vp8_stuff_mb(VP8_COMP *cpi,
                  MACROBLOCKD *x, TOKENEXTRA **t, int dry_run);
void vp8_stuff_mb_8x8(VP8_COMP *cpi,
                      MACROBLOCKD *x, TOKENEXTRA **t, int dry_run);
Daniel Kang's avatar
Daniel Kang committed
45
#if CONFIG_TX16X16
46
47
void vp8_stuff_mb_16x16(VP8_COMP *cpi, MACROBLOCKD *x,
                        TOKENEXTRA **t, int dry_run);
Daniel Kang's avatar
Daniel Kang committed
48
#endif
49
void vp8_fix_contexts(MACROBLOCKD *x);
John Koleszar's avatar
John Koleszar committed
50

John Koleszar's avatar
John Koleszar committed
51
static TOKENVALUE dct_value_tokens[DCT_MAX_VALUE * 2];
52
const TOKENVALUE *vp8_dct_value_tokens_ptr;
John Koleszar's avatar
John Koleszar committed
53
static int dct_value_cost[DCT_MAX_VALUE * 2];
54
const int *vp8_dct_value_cost_ptr;
55
56
57
58
59

#ifdef ENC_DEBUG
extern int mb_row_debug;
extern int mb_col_debug;
extern int enc_debug;
John Koleszar's avatar
John Koleszar committed
60
#endif
61

John Koleszar's avatar
John Koleszar committed
62
static void fill_value_tokens() {
John Koleszar's avatar
John Koleszar committed
63

John Koleszar's avatar
John Koleszar committed
64
65
  TOKENVALUE *const t = dct_value_tokens + DCT_MAX_VALUE;
  vp8_extra_bit_struct *const e = vp8_extra_bits;
John Koleszar's avatar
John Koleszar committed
66

John Koleszar's avatar
John Koleszar committed
67
68
  int i = -DCT_MAX_VALUE;
  int sign = 1;
John Koleszar's avatar
John Koleszar committed
69

John Koleszar's avatar
John Koleszar committed
70
71
72
  do {
    if (!i)
      sign = 0;
John Koleszar's avatar
John Koleszar committed
73

John Koleszar's avatar
John Koleszar committed
74
75
76
    {
      const int a = sign ? -i : i;
      int eb = sign;
John Koleszar's avatar
John Koleszar committed
77

John Koleszar's avatar
John Koleszar committed
78
79
      if (a > 4) {
        int j = 4;
John Koleszar's avatar
John Koleszar committed
80

John Koleszar's avatar
John Koleszar committed
81
        while (++j < 11  &&  e[j].base_val <= a) {}
John Koleszar's avatar
John Koleszar committed
82

John Koleszar's avatar
John Koleszar committed
83
84
85
86
        t[i].Token = --j;
        eb |= (a - e[j].base_val) << 1;
      } else
        t[i].Token = a;
John Koleszar's avatar
John Koleszar committed
87

John Koleszar's avatar
John Koleszar committed
88
89
      t[i].Extra = eb;
    }
John Koleszar's avatar
John Koleszar committed
90

John Koleszar's avatar
John Koleszar committed
91
92
93
94
    // initialize the cost for extra bits for all possible coefficient value.
    {
      int cost = 0;
      vp8_extra_bit_struct *p = vp8_extra_bits + t[i].Token;
John Koleszar's avatar
John Koleszar committed
95

John Koleszar's avatar
John Koleszar committed
96
97
98
      if (p->base_val) {
        const int extra = t[i].Extra;
        const int Length = p->Len;
John Koleszar's avatar
John Koleszar committed
99

John Koleszar's avatar
John Koleszar committed
100
101
        if (Length)
          cost += vp8_treed_cost(p->tree, p->prob, extra >> 1, Length);
John Koleszar's avatar
John Koleszar committed
102

John Koleszar's avatar
John Koleszar committed
103
104
105
        cost += vp8_cost_bit(vp8_prob_half, extra & 1); /* sign */
        dct_value_cost[i + DCT_MAX_VALUE] = cost;
      }
John Koleszar's avatar
John Koleszar committed
106
107
108

    }

John Koleszar's avatar
John Koleszar committed
109
110
111
112
  } while (++i < DCT_MAX_VALUE);

  vp8_dct_value_tokens_ptr = dct_value_tokens + DCT_MAX_VALUE;
  vp8_dct_value_cost_ptr   = dct_value_cost + DCT_MAX_VALUE;
John Koleszar's avatar
John Koleszar committed
113
114
}

Daniel Kang's avatar
Daniel Kang committed
115
#if CONFIG_TX16X16
116
117
118
119
120
121
122
123
124
static void tokenize1st_order_b_16x16(MACROBLOCKD *xd,
                                      const BLOCKD *const b,
                                      TOKENEXTRA **tp,
                                      const int type,
                                      const FRAME_TYPE frametype,
                                      ENTROPY_CONTEXT *a,
                                      ENTROPY_CONTEXT *l,
                                      VP8_COMP *cpi,
                                      int dry_run) {
Daniel Kang's avatar
Daniel Kang committed
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
  int pt; /* near block/prev token context index */
  int c = 0;                  /* start at DC unless type 0 */
  const int eob = b->eob;     /* one beyond last nonzero coeff */
  TOKENEXTRA *t = *tp;        /* store tokens starting here */
  int x;
  const short *qcoeff_ptr = b->qcoeff;

  int seg_eob = 256;
  int segment_id = xd->mode_info_context->mbmi.segment_id;

  if (segfeature_active(xd, segment_id, SEG_LVL_EOB))
    seg_eob = get_segdata(xd, segment_id, SEG_LVL_EOB);

  VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l);

  do {
    const int band = vp8_coef_bands_16x16[c];
    int v;

    x = DCT_EOB_TOKEN;
    if (c < eob) {
      int rc = vp8_default_zig_zag1d_16x16[c];
      v = qcoeff_ptr[rc];

      assert(-DCT_MAX_VALUE <= v  &&  v < (DCT_MAX_VALUE));

      t->Extra = vp8_dct_value_tokens_ptr[v].Extra;
      x        = vp8_dct_value_tokens_ptr[v].Token;
    }

    t->Token = x;
    t->context_tree = cpi->common.fc.coef_probs_16x16[type][band][pt];

    t->skip_eob_node = pt == 0 && ((band > 0 && type > 0) || (band > 1 && type == 0));
159
160
    if (!dry_run)
      ++cpi->coef_counts_16x16[type][band][pt][x];
Daniel Kang's avatar
Daniel Kang committed
161
162
163
164
165
166
167
168
  } while (pt = vp8_prev_token_class[x], ++t, c < eob  &&  ++c < seg_eob);

  *tp = t;
  pt = (c != !type); /* 0 <-> all coeff data is zero */
  *a = *l = pt;
}
#endif

169
170
static void tokenize2nd_order_b_8x8
(
John Koleszar's avatar
John Koleszar committed
171
172
173
174
175
176
177
  MACROBLOCKD *xd,
  const BLOCKD *const b,
  TOKENEXTRA **tp,
  const int type,     /* which plane: 0=Y no DC, 1=Y2, 2=UV, 3=Y with DC */
  const FRAME_TYPE frametype,
  ENTROPY_CONTEXT *a,
  ENTROPY_CONTEXT *l,
178
179
  VP8_COMP *cpi,
  int dry_run) {
John Koleszar's avatar
John Koleszar committed
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
  int pt; /* near block/prev token context index */
  int c = 0;          /* start at DC */
  const int eob = b->eob;     /* one beyond last nonzero coeff */
  TOKENEXTRA *t = *tp;        /* store tokens starting here */
  int x;
  const short *qcoeff_ptr = b->qcoeff;

  int seg_eob = 4;
  int segment_id = xd->mode_info_context->mbmi.segment_id;

  if (segfeature_active(xd, segment_id, SEG_LVL_EOB)) {
    seg_eob = get_segdata(xd, segment_id, SEG_LVL_EOB);
  }

  VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l);

  assert(eob <= 4);

  do {
    const int band = vp8_coef_bands[c];
    int v = 0;

    if (c < eob) {
      int rc = vp8_default_zig_zag1d[c];
      v = qcoeff_ptr[rc];

      assert(-DCT_MAX_VALUE <= v  &&  v < (DCT_MAX_VALUE));

      t->Extra = vp8_dct_value_tokens_ptr[v].Extra;
      x        = vp8_dct_value_tokens_ptr[v].Token;
    } else
      x = DCT_EOB_TOKEN;

    t->Token = x;
    // printf("Token : %d\n", x);
    t->context_tree = cpi->common.fc.coef_probs_8x8 [type] [band] [pt];

    t->skip_eob_node = pt == 0 && ((band > 0 && type > 0) || (band > 1 && type == 0));
218
219

#ifdef ENC_DEBUG
John Koleszar's avatar
John Koleszar committed
220
221
222
223
    if (t->skip_eob_node && vp8_coef_encodings[x].Len == 1)
      printf("Trouble 2 x=%d Len=%d skip=%d eob=%d c=%d band=%d type=%d: [%d %d %d]\n",
             x, vp8_coef_encodings[x].Len, t->skip_eob_node, eob, c, band, type,
             cpi->count, mb_row_debug, mb_col_debug);
224
#endif
225
226
    if (!dry_run)
      ++cpi->coef_counts_8x8       [type] [band] [pt] [x];
John Koleszar's avatar
John Koleszar committed
227
  } while (pt = vp8_prev_token_class[x], ++t, c < eob  &&  ++c < seg_eob);
228

229

John Koleszar's avatar
John Koleszar committed
230
231
232
  *tp = t;
  pt = (c != !type); /* 0 <-> all coeff data is zero */
  *a = *l = pt;
233
234
235

}

236
237
238
239
static void tokenize2nd_order_b(MACROBLOCKD *xd,
                                TOKENEXTRA **tp,
                                VP8_COMP *cpi,
                                int dry_run) {
John Koleszar's avatar
John Koleszar committed
240
241
242
243
244
245
246
247
248
249
250
251
  int pt;             /* near block/prev token context index */
  int c;              /* start at DC */
  TOKENEXTRA *t = *tp;/* store tokens starting here */
  const BLOCKD *b;
  const short *qcoeff_ptr;
  ENTROPY_CONTEXT *a;
  ENTROPY_CONTEXT *l;
  int band, rc, v, token;

  int seg_eob = 16;
  int segment_id = xd->mode_info_context->mbmi.segment_id;

Daniel Kang's avatar
Daniel Kang committed
252
  if (segfeature_active(xd, segment_id, SEG_LVL_EOB))
John Koleszar's avatar
John Koleszar committed
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
    seg_eob = get_segdata(xd, segment_id, SEG_LVL_EOB);

  b = xd->block + 24;
  qcoeff_ptr = b->qcoeff;
  a = (ENTROPY_CONTEXT *)xd->above_context + 8;
  l = (ENTROPY_CONTEXT *)xd->left_context + 8;

  VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l);

  for (c = 0; c < b->eob; c++) {
    rc = vp8_default_zig_zag1d[c];
    band = vp8_coef_bands[c];
    v = qcoeff_ptr[rc];

    t->Extra = vp8_dct_value_tokens_ptr[v].Extra;
    token    = vp8_dct_value_tokens_ptr[v].Token;

    t->Token = token;
    t->context_tree = cpi->common.fc.coef_probs [1] [band] [pt];

    t->skip_eob_node = ((pt == 0) && (band > 0));

275
276
    if (!dry_run)
      ++cpi->coef_counts       [1] [band] [pt] [token];
John Koleszar's avatar
John Koleszar committed
277
278
279
280
281
282
283
284
285

    pt = vp8_prev_token_class[token];
    t++;
  }

  if (c < seg_eob) {
    band = vp8_coef_bands[c];
    t->Token = DCT_EOB_TOKEN;
    t->context_tree = cpi->common.fc.coef_probs [1] [band] [pt];
286

John Koleszar's avatar
John Koleszar committed
287
    t->skip_eob_node = ((pt == 0) && (band > 0));
John Koleszar's avatar
John Koleszar committed
288

289
290
    if (!dry_run)
      ++cpi->coef_counts       [1] [band] [pt] [DCT_EOB_TOKEN];
291

John Koleszar's avatar
John Koleszar committed
292
293
    t++;
  }
John Koleszar's avatar
John Koleszar committed
294
295


John Koleszar's avatar
John Koleszar committed
296
297
298
  *tp = t;
  pt = (c != 0); /* 0 <-> all coeff data is zero */
  *a = *l = pt;
John Koleszar's avatar
John Koleszar committed
299
300

}
Yaowu Xu's avatar
Yaowu Xu committed
301

302
303
static void tokenize1st_order_b_8x8
(
John Koleszar's avatar
John Koleszar committed
304
305
306
307
308
309
310
  MACROBLOCKD *xd,
  const BLOCKD *const b,
  TOKENEXTRA **tp,
  const int type,     /* which plane: 0=Y no DC, 1=Y2, 2=UV, 3=Y with DC */
  const FRAME_TYPE frametype,
  ENTROPY_CONTEXT *a,
  ENTROPY_CONTEXT *l,
311
312
  VP8_COMP *cpi,
  int dry_run) {
John Koleszar's avatar
John Koleszar committed
313
314
315
316
317
318
319
320
  int pt; /* near block/prev token context index */
  int c = type ? 0 : 1;       /* start at DC unless type 0 */
  TOKENEXTRA *t = *tp;        /* store tokens starting here */
  const short *qcoeff_ptr = b->qcoeff;

  int seg_eob = 64;
  int segment_id = xd->mode_info_context->mbmi.segment_id;

321
  if (segfeature_active(xd, segment_id, SEG_LVL_EOB))
John Koleszar's avatar
John Koleszar committed
322
323
324
325
    seg_eob = get_segdata(xd, segment_id, SEG_LVL_EOB);

  VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l);

326
  for (; c < b->eob; ++c) {
John Koleszar's avatar
John Koleszar committed
327
    const int band = vp8_coef_bands_8x8[c];
328
329
    int rc = vp8_default_zig_zag1d_8x8[c];
    int v = qcoeff_ptr[rc], x;
John Koleszar's avatar
John Koleszar committed
330

331
    assert(-DCT_MAX_VALUE <= v  &&  v < (DCT_MAX_VALUE));
John Koleszar's avatar
John Koleszar committed
332

333
334
    t->Extra = vp8_dct_value_tokens_ptr[v].Extra;
    x        = vp8_dct_value_tokens_ptr[v].Token;
335

John Koleszar's avatar
John Koleszar committed
336
    t->Token = x;
337
    t->context_tree = cpi->common.fc.coef_probs_8x8[type][band][pt];
338

John Koleszar's avatar
John Koleszar committed
339
    t->skip_eob_node = pt == 0 && ((band > 0 && type > 0) || (band > 1 && type == 0));
340
341
342

    if (!dry_run)
      ++cpi->coef_counts_8x8[type][band][pt][x];
343

344
345
346
    pt = vp8_prev_token_class[x];
    ++t;
  }
347

348
349
350
351
352
353
354
  if (c < seg_eob) {
    const int band = vp8_coef_bands_8x8[c];
    t->Token = DCT_EOB_TOKEN;

    t->context_tree = cpi->common.fc.coef_probs_8x8 [type] [band] [pt];
    t->skip_eob_node = pt == 0 && ((band > 0 && type > 0) || (band > 1 && type == 0));

355
356
    if (!dry_run)
      ++cpi->coef_counts_8x8[type][band][pt][DCT_EOB_TOKEN];
357
358
    ++t;
  }
359

John Koleszar's avatar
John Koleszar committed
360
361
362
  *tp = t;
  pt = (c != !type); /* 0 <-> all coeff data is zero */
  *a = *l = pt;
363
364
}

Jingning Han's avatar
Jingning Han committed
365
366
367
368
#if CONFIG_HYBRIDTRANSFORM
static void tokenize1st_order_ht(   MACROBLOCKD *xd,
                                    TOKENEXTRA **tp,
                                    int type,
369
370
                                    VP8_COMP *cpi,
                                    int dry_run) {
Jingning Han's avatar
Jingning Han committed
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
  unsigned int block;
  const BLOCKD *b;
  int pt;             /* near block/prev token context index */
  int c;
  int token;
  TOKENEXTRA *t = *tp;/* store tokens starting here */
  const short *qcoeff_ptr;
  ENTROPY_CONTEXT * a;
  ENTROPY_CONTEXT * l;
  int band, rc, v;
  int tmp1, tmp2;

  int const *pt_scan ;

  int seg_eob = 16;
  int segment_id = xd->mode_info_context->mbmi.segment_id;

  if ( segfeature_active( xd, segment_id, SEG_LVL_EOB ) ) {
    seg_eob = get_segdata( xd, segment_id, SEG_LVL_EOB );
  }

  b = xd->block;

  /* Luma */
  for (block = 0; block < 16; block++, b++) {
    B_PREDICTION_MODE b_mode;

    if( xd->mode_info_context->mbmi.mode == B_PRED ) {
      b_mode = b->bmi.as_mode.first;
    }

    // assign scanning order for luma components coded in intra4x4 mode
403
404
    if( (xd->mode_info_context->mbmi.mode == B_PRED) &&
        (type == PLANE_TYPE_Y_WITH_DC) ) {
Jingning Han's avatar
Jingning Han committed
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
      switch(b_mode) {
        case B_VE_PRED :
        case B_VR_PRED :
          pt_scan = vp8_row_scan;
          break;

        case B_HE_PRED :
        case B_HD_PRED :
        case B_HU_PRED :
          pt_scan = vp8_col_scan;
          break;

        default :
          pt_scan = vp8_default_zig_zag1d;
          break;
      }
    } else {
      pt_scan = vp8_default_zig_zag1d;
    }

    tmp1 = vp8_block2above[block];
    tmp2 = vp8_block2left[block];
    qcoeff_ptr = b->qcoeff;
    a = (ENTROPY_CONTEXT *)xd->above_context + tmp1;
    l = (ENTROPY_CONTEXT *)xd->left_context + tmp2;
    VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l);

    c = type ? 0 : 1;

    for (; c < b->eob; c++) {
      rc = pt_scan[c];
      band = vp8_coef_bands[c];
      v = qcoeff_ptr[rc];

      t->Extra = vp8_dct_value_tokens_ptr[v].Extra;
      token    = vp8_dct_value_tokens_ptr[v].Token;

      t->Token = token;
      t->context_tree = cpi->common.fc.coef_probs [type] [band] [pt];

      t->skip_eob_node = pt == 0 &&
          ((band > 0 && type > 0) || (band > 1 && type == 0));

448
449
      if (!dry_run)
        ++cpi->coef_counts       [type] [band] [pt] [token];
Jingning Han's avatar
Jingning Han committed
450
451
452
453
454
455
456
457
458
459
460
461

      pt = vp8_prev_token_class[token];
      t++;
    }

    if (c < seg_eob) {
      band = vp8_coef_bands[c];
      t->Token = DCT_EOB_TOKEN;
      t->context_tree = cpi->common.fc.coef_probs [type] [band] [pt];

      t->skip_eob_node = pt == 0 &&
          ((band > 0 && type > 0) || (band > 1 && type == 0));
462
463
      if (!dry_run)
        ++cpi->coef_counts       [type] [band] [pt] [DCT_EOB_TOKEN];
Jingning Han's avatar
Jingning Han committed
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498

      t++;
    }

    *tp = t;
    pt = (c != !type); /* 0 <-> all coeff data is zero */
    *a = *l = pt;
  }

  // reset scanning order for chroma components
  pt_scan = vp8_default_zig_zag1d ;

  /* Chroma */
  for (block = 16; block < 24; block++, b++) {
    tmp1 = vp8_block2above[block];
    tmp2 = vp8_block2left[block];
    qcoeff_ptr = b->qcoeff;
    a = (ENTROPY_CONTEXT *)xd->above_context + tmp1;
    l = (ENTROPY_CONTEXT *)xd->left_context + tmp2;

    VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l);

    for (c = 0; c < b->eob; c++) {
      rc = pt_scan[c];
      band = vp8_coef_bands[c];
      v = qcoeff_ptr[rc];

      t->Extra = vp8_dct_value_tokens_ptr[v].Extra;
      token    = vp8_dct_value_tokens_ptr[v].Token;

      t->Token = token;
      t->context_tree = cpi->common.fc.coef_probs [2] [band] [pt];

      t->skip_eob_node = ((pt == 0) && (band > 0));

499
500
      if (!dry_run)
        ++cpi->coef_counts       [2] [band] [pt] [token];
Jingning Han's avatar
Jingning Han committed
501
502
503
504
505
506
507
508
509
510
511

      pt = vp8_prev_token_class[token];
      t++;
  }

    if (c < seg_eob) {
      band = vp8_coef_bands[c];
      t->Token = DCT_EOB_TOKEN;
      t->context_tree = cpi->common.fc.coef_probs [2] [band] [pt];

      t->skip_eob_node = ((pt == 0) && (band > 0));
512
513
      if (!dry_run)
        ++cpi->coef_counts       [2] [band] [pt] [DCT_EOB_TOKEN];
Jingning Han's avatar
Jingning Han committed
514
515
516
517
518
519
520
521
522
523
      t++;
    }

    *tp = t;
    pt = (c != 0); /* 0 <-> all coeff data is zero */
    *a = *l = pt;
  }
}
#endif

524

525
#if CONFIG_HYBRIDTRANSFORM8X8
526
527
528
529
530
static void tokenize1st_order_chroma
(
  MACROBLOCKD *xd,
  TOKENEXTRA **tp,
  int type,           /* which plane: 0=Y no DC, 1=Y2, 2=UV, 3=Y with DC */
531
532
  VP8_COMP *cpi,
  int dry_run) {
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
  unsigned int block;
  const BLOCKD *b;
  int pt;             /* near block/prev token context index */
  int c;
  int token;
  TOKENEXTRA *t = *tp;/* store tokens starting here */
  const short *qcoeff_ptr;
  ENTROPY_CONTEXT *a;
  ENTROPY_CONTEXT *l;
  int band, rc, v;
  int tmp1, tmp2;

  int seg_eob = 16;
  int segment_id = xd->mode_info_context->mbmi.segment_id;

  if (segfeature_active(xd, segment_id, SEG_LVL_EOB)) {
    seg_eob = get_segdata(xd, segment_id, SEG_LVL_EOB);
  }

  b = xd->block;
  b += 16;

  /* Chroma */
  for (block = 16; block < 24; block++, b++) {
    tmp1 = vp8_block2above[block];
    tmp2 = vp8_block2left[block];
    qcoeff_ptr = b->qcoeff;
    a = (ENTROPY_CONTEXT *)xd->above_context + tmp1;
    l = (ENTROPY_CONTEXT *)xd->left_context + tmp2;

    VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l);

    for (c = 0; c < b->eob; c++) {
      rc = vp8_default_zig_zag1d[c];
      band = vp8_coef_bands[c];
      v = qcoeff_ptr[rc];

      t->Extra = vp8_dct_value_tokens_ptr[v].Extra;
      token    = vp8_dct_value_tokens_ptr[v].Token;

      t->Token = token;
      t->context_tree = cpi->common.fc.coef_probs [2] [band] [pt];

      t->skip_eob_node = ((pt == 0) && (band > 0));

578
579
      if (!dry_run)
        ++cpi->coef_counts       [2] [band] [pt] [token];
580
581
582
583
584
585
586
587
588
589
590
591

      pt = vp8_prev_token_class[token];
      t++;
    }

    if (c < seg_eob) {
      band = vp8_coef_bands[c];
      t->Token = DCT_EOB_TOKEN;
      t->context_tree = cpi->common.fc.coef_probs [2] [band] [pt];

      t->skip_eob_node = ((pt == 0) && (band > 0));

592
593
      if (!dry_run)
        ++cpi->coef_counts       [2] [band] [pt] [DCT_EOB_TOKEN];
594
595
596
597
598
599
600
601
602
603

      t++;
    }
    *tp = t;
    pt = (c != 0); /* 0 <-> all coeff data is zero */
    *a = *l = pt;
  }
}
#endif

John Koleszar's avatar
John Koleszar committed
604
605
static void tokenize1st_order_b
(
John Koleszar's avatar
John Koleszar committed
606
607
608
  MACROBLOCKD *xd,
  TOKENEXTRA **tp,
  int type,           /* which plane: 0=Y no DC, 1=Y2, 2=UV, 3=Y with DC */
609
610
  VP8_COMP *cpi,
  int dry_run) {
John Koleszar's avatar
John Koleszar committed
611
612
613
  unsigned int block;
  const BLOCKD *b;
  int pt;             /* near block/prev token context index */
Daniel Kang's avatar
Daniel Kang committed
614
  int band, rc, v, c, token;
John Koleszar's avatar
John Koleszar committed
615
616
  TOKENEXTRA *t = *tp;/* store tokens starting here */
  const short *qcoeff_ptr;
Daniel Kang's avatar
Daniel Kang committed
617
  ENTROPY_CONTEXT *a, *l;
John Koleszar's avatar
John Koleszar committed
618
619
620
621
622
623
624
625
626
627
628
629

  int seg_eob = 16;
  int segment_id = xd->mode_info_context->mbmi.segment_id;

  if (segfeature_active(xd, segment_id, SEG_LVL_EOB)) {
    seg_eob = get_segdata(xd, segment_id, SEG_LVL_EOB);
  }

  b = xd->block;
  /* Luma */
  for (block = 0; block < 16; block++, b++) {
    qcoeff_ptr = b->qcoeff;
Daniel Kang's avatar
Daniel Kang committed
630
631
    a = (ENTROPY_CONTEXT *)xd->above_context + vp8_block2above[block];
    l = (ENTROPY_CONTEXT *)xd->left_context + vp8_block2left[block];
John Koleszar's avatar
John Koleszar committed
632
    VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l);
John Koleszar's avatar
John Koleszar committed
633

John Koleszar's avatar
John Koleszar committed
634
    c = type ? 0 : 1;
Tero Rintaluoma's avatar
Tero Rintaluoma committed
635

John Koleszar's avatar
John Koleszar committed
636
637
638
639
    for (; c < b->eob; c++) {
      rc = vp8_default_zig_zag1d[c];
      band = vp8_coef_bands[c];
      v = qcoeff_ptr[rc];
Tero Rintaluoma's avatar
Tero Rintaluoma committed
640

John Koleszar's avatar
John Koleszar committed
641
642
      t->Extra = vp8_dct_value_tokens_ptr[v].Extra;
      token    = vp8_dct_value_tokens_ptr[v].Token;
Tero Rintaluoma's avatar
Tero Rintaluoma committed
643

John Koleszar's avatar
John Koleszar committed
644
645
      t->Token = token;
      t->context_tree = cpi->common.fc.coef_probs [type] [band] [pt];
Tero Rintaluoma's avatar
Tero Rintaluoma committed
646

John Koleszar's avatar
John Koleszar committed
647
648
      t->skip_eob_node = pt == 0 &&
                         ((band > 0 && type > 0) || (band > 1 && type == 0));
649
650
      if (!dry_run)
        ++cpi->coef_counts       [type] [band] [pt] [token];
651

John Koleszar's avatar
John Koleszar committed
652
653
654
      pt = vp8_prev_token_class[token];
      t++;
    }
John Koleszar's avatar
John Koleszar committed
655

John Koleszar's avatar
John Koleszar committed
656
657
658
659
    if (c < seg_eob) {
      band = vp8_coef_bands[c];
      t->Token = DCT_EOB_TOKEN;
      t->context_tree = cpi->common.fc.coef_probs [type] [band] [pt];
660

John Koleszar's avatar
John Koleszar committed
661
662
      t->skip_eob_node = pt == 0 &&
                         ((band > 0 && type > 0) || (band > 1 && type == 0));
663
664
      if (!dry_run)
        ++cpi->coef_counts       [type] [band] [pt] [DCT_EOB_TOKEN];
John Koleszar's avatar
John Koleszar committed
665

John Koleszar's avatar
John Koleszar committed
666
      t++;
John Koleszar's avatar
John Koleszar committed
667
    }
John Koleszar's avatar
John Koleszar committed
668
669
670
    *tp = t;
    pt = (c != !type); /* 0 <-> all coeff data is zero */
    *a = *l = pt;
John Koleszar's avatar
John Koleszar committed
671

John Koleszar's avatar
John Koleszar committed
672
673
674
675
  }
  /* Chroma */
  for (block = 16; block < 24; block++, b++) {
    qcoeff_ptr = b->qcoeff;
Daniel Kang's avatar
Daniel Kang committed
676
677
    a = (ENTROPY_CONTEXT *)xd->above_context + vp8_block2above[block];
    l = (ENTROPY_CONTEXT *)xd->left_context + vp8_block2left[block];
Tero Rintaluoma's avatar
Tero Rintaluoma committed
678

John Koleszar's avatar
John Koleszar committed
679
    VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l);
Tero Rintaluoma's avatar
Tero Rintaluoma committed
680

John Koleszar's avatar
John Koleszar committed
681
682
683
684
    for (c = 0; c < b->eob; c++) {
      rc = vp8_default_zig_zag1d[c];
      band = vp8_coef_bands[c];
      v = qcoeff_ptr[rc];
Tero Rintaluoma's avatar
Tero Rintaluoma committed
685

John Koleszar's avatar
John Koleszar committed
686
687
      t->Extra = vp8_dct_value_tokens_ptr[v].Extra;
      token    = vp8_dct_value_tokens_ptr[v].Token;
Tero Rintaluoma's avatar
Tero Rintaluoma committed
688

John Koleszar's avatar
John Koleszar committed
689
690
      t->Token = token;
      t->context_tree = cpi->common.fc.coef_probs [2] [band] [pt];
Tero Rintaluoma's avatar
Tero Rintaluoma committed
691

John Koleszar's avatar
John Koleszar committed
692
      t->skip_eob_node = ((pt == 0) && (band > 0));
Tero Rintaluoma's avatar
Tero Rintaluoma committed
693

694
695
      if (!dry_run)
        ++cpi->coef_counts       [2] [band] [pt] [token];
John Koleszar's avatar
John Koleszar committed
696
697
698
699

      pt = vp8_prev_token_class[token];
      t++;
    }
700

John Koleszar's avatar
John Koleszar committed
701
702
703
704
    if (c < seg_eob) {
      band = vp8_coef_bands[c];
      t->Token = DCT_EOB_TOKEN;
      t->context_tree = cpi->common.fc.coef_probs [2] [band] [pt];
Tero Rintaluoma's avatar
Tero Rintaluoma committed
705

John Koleszar's avatar
John Koleszar committed
706
      t->skip_eob_node = ((pt == 0) && (band > 0));
Tero Rintaluoma's avatar
Tero Rintaluoma committed
707

708
709
      if (!dry_run)
        ++cpi->coef_counts       [2] [band] [pt] [DCT_EOB_TOKEN];
Tero Rintaluoma's avatar
Tero Rintaluoma committed
710

John Koleszar's avatar
John Koleszar committed
711
      t++;
Tero Rintaluoma's avatar
Tero Rintaluoma committed
712
    }
John Koleszar's avatar
John Koleszar committed
713
714
715
716
    *tp = t;
    pt = (c != 0); /* 0 <-> all coeff data is zero */
    *a = *l = pt;
  }
John Koleszar's avatar
John Koleszar committed
717
718
}

719

John Koleszar's avatar
John Koleszar committed
720
721
722
int mby_is_skippable(MACROBLOCKD *x, int has_y2_block) {
  int skip = 1;
  int i = 0;
723

John Koleszar's avatar
John Koleszar committed
724
725
726
727
728
729
730
731
732
  if (has_y2_block) {
    for (i = 0; i < 16; i++)
      skip &= (x->block[i].eob < 2);
    skip &= (!x->block[24].eob);
  } else {
    for (i = 0; i < 16; i++)
      skip &= (!x->block[i].eob);
  }
  return skip;
733
}
734

John Koleszar's avatar
John Koleszar committed
735
736
737
int mbuv_is_skippable(MACROBLOCKD *x) {
  int skip = 1;
  int i;
738

John Koleszar's avatar
John Koleszar committed
739
740
741
  for (i = 16; i < 24; i++)
    skip &= (!x->block[i].eob);
  return skip;
742
743
}

John Koleszar's avatar
John Koleszar committed
744
745
746
int mb_is_skippable(MACROBLOCKD *x, int has_y2_block) {
  return (mby_is_skippable(x, has_y2_block) &
          mbuv_is_skippable(x));
747
}
Yaowu Xu's avatar
Yaowu Xu committed
748

John Koleszar's avatar
John Koleszar committed
749
750
751
int mby_is_skippable_8x8(MACROBLOCKD *x) {
  int skip = 1;
  int i = 0;
752

John Koleszar's avatar
John Koleszar committed
753
754
755
756
  for (i = 0; i < 16; i += 4)
    skip &= (x->block[i].eob < 2);
  skip &= (!x->block[24].eob);
  return skip;
757
}
758

John Koleszar's avatar
John Koleszar committed
759
760
int mbuv_is_skippable_8x8(MACROBLOCKD *x) {
  return (!x->block[16].eob) & (!x->block[20].eob);
761
}
762

John Koleszar's avatar
John Koleszar committed
763
764
int mb_is_skippable_8x8(MACROBLOCKD *x) {
  return (mby_is_skippable_8x8(x) & mbuv_is_skippable_8x8(x));
765
}
Yaowu Xu's avatar
Yaowu Xu committed
766

Daniel Kang's avatar
Daniel Kang committed
767
768
769
770
771
772
773
774
775
776
777
778
779
780
#if CONFIG_TX16X16
int mby_is_skippable_16x16(MACROBLOCKD *x) {
  int skip = 1;
  //skip &= (x->block[0].eob < 2); // I think this should be commented? No second order == DC must be coded
  //skip &= (x->block[0].eob < 1);
  //skip &= (!x->block[24].eob);
  skip &= !x->block[0].eob;
  return skip;
}

int mb_is_skippable_16x16(MACROBLOCKD *x) {
  return (mby_is_skippable_16x16(x) & mbuv_is_skippable_8x8(x));
}
#endif
781

782
783
784
785
void vp8_tokenize_mb(VP8_COMP *cpi,
                     MACROBLOCKD *x,
                     TOKENEXTRA **t,
                     int dry_run) {
John Koleszar's avatar
John Koleszar committed
786
787
788
789
790
  int plane_type;
  int has_y2_block;
  int b;
  int tx_type = x->mode_info_context->mbmi.txfm_size;
  int mb_skip_context = get_pred_context(&cpi->common, x, PRED_MBSKIP);
791
  TOKENEXTRA *t_backup = *t;
Paul Wilkins's avatar
Paul Wilkins committed
792

John Koleszar's avatar
John Koleszar committed
793
794
795
796
797
  // If the MB is going to be skipped because of a segment level flag
  // exclude this from the skip count stats used to calculate the
  // transmitted skip probability;
  int skip_inc;
  int segment_id = x->mode_info_context->mbmi.segment_id;
798

Jingning Han's avatar
Jingning Han committed
799
800
#if CONFIG_HYBRIDTRANSFORM
    int QIndex = cpi->mb.q_index;
801
802
    int active_ht = (QIndex < ACTIVE_HT) &&
                    (x->mode_info_context->mbmi.mode == B_PRED);
Jingning Han's avatar
Jingning Han committed
803
804
#endif

John Koleszar's avatar
John Koleszar committed
805
806
807
808
809
  if (!segfeature_active(x, segment_id, SEG_LVL_EOB) ||
      (get_segdata(x, segment_id, SEG_LVL_EOB) != 0)) {
    skip_inc = 1;
  } else
    skip_inc = 0;
810

John Koleszar's avatar
John Koleszar committed
811
812
813
  has_y2_block = (x->mode_info_context->mbmi.mode != B_PRED
                  && x->mode_info_context->mbmi.mode != I8X8_PRED
                  && x->mode_info_context->mbmi.mode != SPLITMV);
Daniel Kang's avatar
Daniel Kang committed
814
815
816
#if CONFIG_TX16X16
  if (tx_type == TX_16X16) has_y2_block = 0; // Because of inter frames
#endif
John Koleszar's avatar
John Koleszar committed
817

Daniel Kang's avatar
Daniel Kang committed
818
819
820
821
822
823
824
825
826
827
828
829
830
  switch (tx_type) {
#if CONFIG_TX16X16
    case TX_16X16:
      x->mode_info_context->mbmi.mb_skip_coeff = mb_is_skippable_16x16(x);
      break;
#endif
    case TX_8X8:
      x->mode_info_context->mbmi.mb_skip_coeff = mb_is_skippable_8x8(x);
      break;
    default:
      x->mode_info_context->mbmi.mb_skip_coeff = mb_is_skippable(x, has_y2_block);
      break;
  }
831

John Koleszar's avatar
John Koleszar committed
832
  if (x->mode_info_context->mbmi.mb_skip_coeff) {
833
834
    if (!dry_run)
      cpi->skip_true_count[mb_skip_context] += skip_inc;
John Koleszar's avatar
John Koleszar committed
835
    if (!cpi->common.mb_no_coeff_skip) {
Daniel Kang's avatar
Daniel Kang committed
836
837
#if CONFIG_TX16X16
      if (tx_type == TX_16X16)
838
        vp8_stuff_mb_16x16(cpi, x, t, dry_run);
Daniel Kang's avatar
Daniel Kang committed
839
840
      else
#endif
John Koleszar's avatar
John Koleszar committed
841
      if (tx_type == TX_8X8)
842
        vp8_stuff_mb_8x8(cpi, x, t, dry_run);
John Koleszar's avatar
John Koleszar committed
843
      else
844
        vp8_stuff_mb(cpi, x, t, dry_run);
John Koleszar's avatar
John Koleszar committed
845
846
    } else {
      vp8_fix_contexts(x);
John Koleszar's avatar
John Koleszar committed
847
    }
848
849
    if (dry_run)
      *t = t_backup;
John Koleszar's avatar
John Koleszar committed
850
851
852
    return;
  }

853
854
  if (!dry_run)
    cpi->skip_false_count[mb_skip_context] += skip_inc;
855

John Koleszar's avatar
John Koleszar committed
856
857
858
859
860
861
862
863
  plane_type = 3;
  if (has_y2_block) {
    if (tx_type == TX_8X8) {
      ENTROPY_CONTEXT *A = (ENTROPY_CONTEXT *)x->above_context;
      ENTROPY_CONTEXT *L = (ENTROPY_CONTEXT *)x->left_context;
      tokenize2nd_order_b_8x8(x,
                              x->block + 24, t, 1, x->frame_type,
                              A + vp8_block2above_8x8[24],
864
865
                              L + vp8_block2left_8x8[24],
                              cpi, dry_run);
John Koleszar's avatar
John Koleszar committed
866
    } else
867
      tokenize2nd_order_b(x, t, cpi, dry_run);
John Koleszar's avatar
John Koleszar committed
868

John Koleszar's avatar
John Koleszar committed
869
870
871
    plane_type = 0;
  }

Daniel Kang's avatar
Daniel Kang committed
872
873
874
875
#if CONFIG_TX16X16
  if (tx_type == TX_16X16) {
    ENTROPY_CONTEXT * A = (ENTROPY_CONTEXT *)x->above_context;
    ENTROPY_CONTEXT * L = (ENTROPY_CONTEXT *)x->left_context;
876
877
    tokenize1st_order_b_16x16(x, x->block, t, 3,
                              x->frame_type, A, L, cpi, dry_run);
Daniel Kang's avatar
Daniel Kang committed
878
879
880
881
882
883
    for (b = 1; b < 16; b++) {
      *(A + vp8_block2above[b]) = *(A);
      *(L + vp8_block2left[b] ) = *(L);
    }
    for (b = 16; b < 24; b += 4) {
      tokenize1st_order_b_8x8(x, x->block + b, t, 2, x->frame_type,
884
          A + vp8_block2above_8x8[b], L + vp8_block2left_8x8[b], cpi, dry_run);
Daniel Kang's avatar
Daniel Kang committed
885
886
887
888
889
890
891
892
      *(A + vp8_block2above_8x8[b]+1) = *(A + vp8_block2above_8x8[b]);
      *(L + vp8_block2left_8x8[b]+1 ) = *(L + vp8_block2left_8x8[b]);
    }
    vpx_memset(&A[8], 0, sizeof(A[8]));
    vpx_memset(&L[8], 0, sizeof(L[8]));
  }
  else
#endif
John Koleszar's avatar
John Koleszar committed
893
894
895
896
897
898
899
900
  if (tx_type == TX_8X8) {
    ENTROPY_CONTEXT *A = (ENTROPY_CONTEXT *)x->above_context;
    ENTROPY_CONTEXT *L = (ENTROPY_CONTEXT *)x->left_context;
    for (b = 0; b < 16; b += 4) {
      tokenize1st_order_b_8x8(x,
                              x->block + b, t, plane_type, x->frame_type,
                              A + vp8_block2above_8x8[b],
                              L + vp8_block2left_8x8[b],
901
                              cpi, dry_run);
John Koleszar's avatar
John Koleszar committed
902
903
904
905
906
907
908
909
      *(A + vp8_block2above_8x8[b] + 1) = *(A + vp8_block2above_8x8[b]);
      *(L + vp8_block2left_8x8[b] + 1)  = *(L + vp8_block2left_8x8[b]);
    }
    for (b = 16; b < 24; b += 4) {
      tokenize1st_order_b_8x8(x,
                              x->block + b, t, 2, x->frame_type,
                              A + vp8_block2above_8x8[b],
                              L + vp8_block2left_8x8[b],
910
                              cpi, dry_run);
John Koleszar's avatar
John Koleszar committed
911
912
      *(A + vp8_block2above_8x8[b] + 1) = *(A + vp8_block2above_8x8[b]);
      *(L + vp8_block2left_8x8[b] + 1) = *(L + vp8_block2left_8x8[b]);
913
    }
Jingning Han's avatar
Jingning Han committed
914
915
916
  } else {
#if CONFIG_HYBRIDTRANSFORM
    if(active_ht) {
917
      tokenize1st_order_ht(x, t, plane_type, cpi, dry_run);
Jingning Han's avatar
Jingning Han committed
918
    } else {
919

920
#if CONFIG_HYBRIDTRANSFORM8X8
921
922
923
924
925
926
927
928
929
      if (x->mode_info_context->mbmi.mode == I8X8_PRED) {
        ENTROPY_CONTEXT *A = (ENTROPY_CONTEXT *)x->above_context;
        ENTROPY_CONTEXT *L = (ENTROPY_CONTEXT *)x->left_context;
        for (b = 0; b < 16; b += 4) {
          tokenize1st_order_b_8x8(x,
                                  x->block + b, t, PLANE_TYPE_Y_WITH_DC,
                                  x->frame_type,
                                  A + vp8_block2above_8x8[b],
                                  L + vp8_block2left_8x8[b],
930
                                  cpi, dry_run);
931
932
933
          *(A + vp8_block2above_8x8[b] + 1) = *(A + vp8_block2above_8x8[b]);
          *(L + vp8_block2left_8x8[b] + 1)  = *(L + vp8_block2left_8x8[b]);
        }
934
        tokenize1st_order_chroma(x, t, PLANE_TYPE_UV, cpi, dry_run);
935
      } else {
936
        tokenize1st_order_b(x, t, plane_type, cpi, dry_run);
937
938
      }
#else
939
      tokenize1st_order_b(x, t, plane_type, cpi, dry_run);
940
941
#endif

Jingning Han's avatar
Jingning Han committed
942
943
    }
#else
944
    tokenize1st_order_b(x, t, plane_type, cpi, dry_run);
Jingning Han's avatar
Jingning Han committed
945
946
#endif
  }
947
948
  if (dry_run)
    *t = t_backup;
John Koleszar's avatar
John Koleszar committed
949
}
950

John Koleszar's avatar
John Koleszar committed
951
952

#ifdef ENTROPY_STATS
John Koleszar's avatar
John Koleszar committed
953
954
955
956
957
void init_context_counters(void) {
  FILE *f = fopen("context.bin", "rb");
  if (!f) {
    vpx_memset(context_counters, 0, sizeof(context_counters));
    vpx_memset(context_counters_8x8, 0, sizeof(context_counters_8x8));
Daniel Kang's avatar
Daniel Kang committed
958
959
960
#if CONFIG_TX16X16
    vpx_memset(context_counters_16x16, 0, sizeof(context_counters_16x16));
#endif
John Koleszar's avatar
John Koleszar committed
961
962
963
  } else {
    fread(context_counters, sizeof(context_counters), 1, f);
    fread(context_counters_8x8, sizeof(context_counters_8x8), 1, f);
Daniel Kang's avatar
Daniel Kang committed
964
965
966
#if CONFIG_TX16X16
    fread(context_counters_16x16, sizeof(context_counters_16x16), 1, f);
#endif
John Koleszar's avatar
John Koleszar committed
967
968
969
970
971
972
973
    fclose(f);
  }

  f = fopen("treeupdate.bin", "rb");
  if (!f) {
    vpx_memset(tree_update_hist, 0, sizeof(tree_update_hist));
    vpx_memset(tree_update_hist_8x8, 0, sizeof(tree_update_hist_8x8));
Daniel Kang's avatar
Daniel Kang committed
974
975
976
#if CONFIG_TX16X16
    vpx_memset(tree_update_hist_16x16, 0, sizeof(tree_update_hist_16x16));
#endif
John Koleszar's avatar
John Koleszar committed
977
978
979
  } else {
    fread(tree_update_hist, sizeof(tree_update_hist), 1, f);
    fread(tree_update_hist_8x8, sizeof(tree_update_hist_8x8), 1, f);
Daniel Kang's avatar
Daniel Kang committed
980
981
982
#if CONFIG_TX16X16
    fread(tree_update_hist_16x16, sizeof(tree_update_hist_16x16), 1, f);
#endif
John Koleszar's avatar
John Koleszar committed
983
984
    fclose(f);
  }
John Koleszar's avatar
John Koleszar committed
985
986
}

John Koleszar's avatar
John Koleszar committed
987
988
989
void print_context_counters() {
  int type, band, pt, t;
  FILE *f = fopen("context.c", "w");
John Koleszar's avatar
John Koleszar committed
990

John Koleszar's avatar
John Koleszar committed
991
992
993
994
995
996
997
  fprintf(f, "#include \"entropy.h\"\n");
  fprintf(f, "\n/* *** GENERATED FILE: DO NOT EDIT *** */\n\n");
  fprintf(f, "static const unsigned int\n"
          "vp8_default_coef_counts[BLOCK_TYPES]\n"
          "                      [COEF_BANDS]\n"
          "                      [PREV_COEF_CONTEXTS]\n"
          "                      [MAX_ENTROPY_TOKENS]={\n");
John Koleszar's avatar
John Koleszar committed
998
999

# define Comma( X) (X? ",":"")
John Koleszar's avatar
John Koleszar committed
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
  type = 0;
  do {
    fprintf(f, "%s\n  { /* block Type %d */", Comma(type), type);
    band = 0;
    do {
      fprintf(f, "%s\n    { /* Coeff Band %d */", Comma(band), band);
      pt = 0;
      do {
        fprintf(f, "%s\n      {", Comma(pt));

        t = 0;
        do {
          const INT64 x = context_counters [type] [band] [pt] [t];
          const int y = (int) x;
          assert(x == (INT64) y);  /* no overflow handling yet */
          fprintf(f, "%s %d", Comma(t), y);
        } while (++t < MAX_ENTROPY_TOKENS);
        fprintf(f, "}");
      } while (++pt < PREV_COEF_CONTEXTS);
      fprintf(f, "\n    }");
    } while (++band < COEF_BANDS);
    fprintf(f, "\n  }");
  } while (++type < BLOCK_TYPES);
  fprintf(f, "\n};\n");
1024

John Koleszar's avatar
John Koleszar committed
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
  fprintf(f, "static const unsigned int\nvp8_default_coef_counts_8x8"
          "[BLOCK_TYPES_8X8] [COEF_BANDS]"
          "[PREV_COEF_CONTEXTS] [MAX_ENTROPY_TOKENS] = {");
  type = 0;
  do {
    fprintf(f, "%s\n  { /* block Type %d */", Comma(type), type);
    band = 0;
    do {
      fprintf(f, "%s\n    { /* Coeff Band %d */", Comma(band), band);
      pt = 0;
      do {
        fprintf(f, "%s\n      {", Comma(pt));
        t = 0;
        do {
          const INT64 x = context_counters_8x8 [type] [band] [pt] [t];
          const int y = (int) x;
1041

John Koleszar's avatar
John Koleszar committed
1042
1043
          assert(x == (INT64) y);  /* no overflow handling yet */
          fprintf(f, "%s %d", Comma(t), y);
John Koleszar's avatar
John Koleszar committed
1044

John Koleszar's avatar
John Koleszar committed
1045
        } while (++t < MAX_ENTROPY_TOKENS);
1046

John Koleszar's avatar
John Koleszar committed
1047
1048
        fprintf(f, "}");
      } while (++pt < PREV_COEF_CONTEXTS);
1049

John Koleszar's avatar
John Koleszar committed
1050
      fprintf(f, "\n    }");
1051

John Koleszar's avatar
John Koleszar committed
1052
    } while (++band < COEF_BANDS);
1053

John Koleszar's avatar
John Koleszar committed
1054
1055
    fprintf(f, "\n  }");
  } while (++type < BLOCK_TYPES_8X8);
Daniel Kang's avatar
Daniel Kang committed
1056
  fprintf(f, "\n};\n");
1057

Daniel Kang's avatar
Daniel Kang committed
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
#if CONFIG_TX16X16
  fprintf(f, "static const unsigned int\nvp8_default_coef_counts_16x16"
          "[BLOCK_TYPES_16X16] [COEF_BANDS]"
          "[PREV_COEF_CONTEXTS] [MAX_ENTROPY_TOKENS] = {");
  type = 0;
  do {
    fprintf(f, "%s\n  { /* block Type %d */", Comma(type), type);
    band = 0;
    do {
      fprintf(f, "%s\n    { /* Coeff Band %d */", Comma(band), band);
      pt = 0;
      do {
        fprintf(f, "%s\n      {", Comma(pt));
        t = 0;
        do {
          const INT64 x = context_counters_16x16 [type] [band] [pt] [t];
          const int y = (int) x;

          assert(x == (INT64) y);  /* no overflow handling yet */
          fprintf(f, "%s %d", Comma(t), y);

        } while (++t < MAX_ENTROPY_TOKENS);

        fprintf(f, "}");
      } while (++pt < PREV_COEF_CONTEXTS);

      fprintf(f, "\n    }");

    } while (++band < COEF_BANDS);

    fprintf(f, "\n  }");
  } while (++type < BLOCK_TYPES_16X16);
John Koleszar's avatar
John Koleszar committed
1090
  fprintf(f, "\n};\n");
Daniel Kang's avatar
Daniel Kang committed
1091
#endif
1092

John Koleszar's avatar
John Koleszar committed
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
  fprintf(f, "static const vp8_prob\n"
          "vp8_default_coef_probs[BLOCK_TYPES] [COEF_BANDS] \n"
          "[PREV_COEF_CONTEXTS] [ENTROPY_NODES] = {");
  type = 0;
  do {
    fprintf(f, "%s\n  { /* block Type %d */", Comma(type), type);
    band = 0;
    do {
      fprintf(f, "%s\n    { /* Coeff Band %d */", Comma(band), band);
      pt = 0;
      do {
        unsigned int branch_ct [ENTROPY_NODES] [2];
        unsigned int coef_counts[MAX_ENTROPY_TOKENS];
        vp8_prob coef_probs[ENTROPY_NODES];
        for (t = 0; t < MAX_ENTROPY_TOKENS; ++t)
          coef_counts[t] = context_counters [type] [band] [pt] [t];
        vp8_tree_probs_from_distribution(
          MAX_ENTROPY_TOKENS, vp8_coef_encodings, vp8_coef_tree,
          coef_probs, branch_ct, coef_counts, 256, 1);
        fprintf(f, "%s\n      {", Comma(pt));
1113

John Koleszar's avatar
John Koleszar committed
1114
1115
1116
        t = 0;
        do {
          fprintf(f, "%s %d", Comma(t), coef_probs[t]);
1117

John Koleszar's avatar
John Koleszar committed
1118
        } while (++t < ENTROPY_NODES);
1119

John Koleszar's avatar
John Koleszar committed
1120
1121
1122
1123
1124
1125
1126
        fprintf(f, "}");
      } while (++pt < PREV_COEF_CONTEXTS);
      fprintf(f, "\n    }");
    } while (++band < COEF_BANDS);
    fprintf(f, "\n  }");
  } while (++type < BLOCK_TYPES);
  fprintf(f, "\n};\n");
1127

John Koleszar's avatar
John Koleszar committed
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
  fprintf(f, "static const vp8_prob\n"
          "vp8_default_coef_probs_8x8[BLOCK_TYPES_8X8] [COEF_BANDS]\n"
          "[PREV_COEF_CONTEXTS] [ENTROPY_NODES] = {");
  type = 0;
  do {
    fprintf(f, "%s\n  { /* block Type %d */", Comma(type), type);
    band = 0;
    do {
      fprintf(f, "%s\n    { /* Coeff Band %d */", Comma(band), band);
      pt = 0;
      do {
        unsigned int branch_ct [ENTROPY_NODES] [2];
        unsigned int coef_counts[MAX_ENTROPY_TOKENS];
        vp8_prob coef_probs[ENTROPY_NODES];
        for (t = 0; t < MAX_ENTROPY_TOKENS; ++t)
          coef_counts[t] = context_counters_8x8[type] [band] [pt] [t];
        vp8_tree_probs_from_distribution(
          MAX_ENTROPY_TOKENS, vp8_coef_encodings, vp8_coef_tree,
          coef_probs, branch_ct, coef_counts, 256, 1);
        fprintf(f, "%s\n      {", Comma(pt));
1148

Daniel Kang's avatar
Daniel Kang committed
1149
        t = 0;
John Koleszar's avatar
John Koleszar committed
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
        do {
          fprintf(f, "%s %d", Comma(t), coef_probs[t]);
        } while (++t < ENTROPY_NODES);
        fprintf(f, "}");
      } while (++pt < PREV_COEF_CONTEXTS);
      fprintf(f, "\n    }");
    } while (++band < COEF_BANDS);
    fprintf(f, "\n  }");
  } while (++type < BLOCK_TYPES_8X8);
  fprintf(f, "\n};\n");

Daniel Kang's avatar
Daniel Kang committed
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
#if CONFIG_TX16X16
  fprintf(f, "static const vp8_prob\n"
          "vp8_default_coef_probs_16x16[BLOCK_TYPES_16X16] [COEF_BANDS]\n"
          "[PREV_COEF_CONTEXTS] [ENTROPY_NODES] = {");
  type = 0;
  do {
    fprintf(f, "%s\n  { /* block Type %d */", Comma(type), type);
    band = 0;
    do {
      fprintf(f, "%s\n    { /* Coeff Band %d */", Comma(band), band);
      pt = 0;
      do {
        unsigned int branch_ct [ENTROPY_NODES] [2];
        unsigned int coef_counts[MAX_ENTROPY_TOKENS];
        vp8_prob coef_probs[ENTROPY_NODES];
        for (t = 0; t < MAX_ENTROPY_TOKENS; ++t)
          coef_counts[t] = context_counters_16x16[type] [band] [pt] [t];
        vp8_tree_probs_from_distribution(
          MAX_ENTROPY_TOKENS, vp8_coef_encodings, vp8_coef_tree,
          coef_probs, branch_ct, coef_counts, 256, 1);
        fprintf(f, "%s\n      {", Comma(pt));

        t = 0;
        do {
          fprintf(f, "%s %d", Comma(t), coef_probs[t]);
        } while (++t < ENTROPY_NODES);
        fprintf(f, "}");
      } while (++pt < PREV_COEF_CONTEXTS);
      fprintf(f, "\n    }");
    } while (++band < COEF_BANDS);
    fprintf(f, "\n  }");
  } while (++type < BLOCK_TYPES_16X16);
  fprintf(f, "\n};\n");
#endif

John Koleszar's avatar
John Koleszar committed
1196
1197
1198
1199
1200
  fclose(f);

  f = fopen("context.bin", "wb");
  fwrite(context_counters, sizeof(context_counters), 1, f);
  fwrite(context_counters_8x8, sizeof(context_counters_8x8), 1, f);
Daniel Kang's avatar
Daniel Kang committed
1201
1202
1203
#if CONFIG_TX16X16
  fwrite(context_counters_16x16, sizeof(context_counters_16x16), 1, f);
#endif
John Koleszar's avatar
John Koleszar committed
1204
  fclose(f);
John Koleszar's avatar
John Koleszar committed
1205
1206
1207
1208
}
#endif


John Koleszar's avatar
John Koleszar committed
1209
1210
void vp8_tokenize_initialize() {
  fill_value_tokens();
John Koleszar's avatar
John Koleszar committed
1211
1212
}

Yaowu Xu's avatar
Yaowu Xu committed
1213

1214
static __inline void stuff2nd_order_b_8x8(
John Koleszar's avatar
John Koleszar committed
1215
1216
1217
1218
1219
1220
  const BLOCKD *const b,
  TOKENEXTRA **tp,
  const int type,     /* which plane: 0=Y no DC, 1=Y2, 2=UV, 3=Y with DC */
  const FRAME_TYPE frametype,
  ENTROPY_CONTEXT *a,
  ENTROPY_CONTEXT *l,
1221
1222
  VP8_COMP *cpi,
  int dry_run) {
John Koleszar's avatar
John Koleszar committed
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
  int pt; /* near block/prev token context index */
  TOKENEXTRA *t = *tp;        /* store tokens starting here */
  VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l);
  (void) frametype;
  (void) type;
  (void) b;

  t->Token = DCT_EOB_TOKEN;
  t->context_tree = cpi->common.fc.coef_probs_8x8 [1] [0] [pt];
  // t->section = 11;
  t->skip_eob_node = 0;
  ++t;

  *tp = t;
1237
1238
  if (!dry_run)
    ++cpi->coef_counts_8x8       [1] [0] [pt] [DCT_EOB_TOKEN];
John Koleszar's avatar
John Koleszar committed
1239
1240
  pt = 0;
  *a = *l = pt;
1241
1242
1243
1244
1245

}

static __inline void stuff1st_order_b_8x8
(
John Koleszar's avatar
John Koleszar committed
1246
1247
1248
1249
1250
1251
  const BLOCKD *const b,
  TOKENEXTRA **tp,
  const int type,     /* which plane: 0=Y no DC, 1=Y2, 2=UV, 3=Y with DC */
  const FRAME_TYPE frametype,
  ENTROPY_CONTEXT *a,
  ENTROPY_CONTEXT *l,
1252
1253
  VP8_COMP *cpi,
  int dry_run) {
John Koleszar's avatar
John Koleszar committed
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
  int pt; /* near block/prev token context index */
  TOKENEXTRA *t = *tp;        /* store tokens starting here */
  VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l);
  (void) frametype;
  (void) type;
  (void) b;

  t->Token = DCT_EOB_TOKEN;
  t->context_tree = cpi->common.fc.coef_probs_8x8 [0] [1] [pt];
  // t->section = 8;
  t->skip_eob_node = 0;
  ++t;
  *tp = t;
1267
1268
  if (!dry_run)
    ++cpi->coef_counts_8x8[0] [1] [pt] [DCT_EOB_TOKEN];
John Koleszar's avatar
John Koleszar committed
1269
1270
  pt = 0; /* 0 <-> all coeff data is zero */
  *a = *l = pt;
1271
1272
1273
1274
1275
1276
1277


}

static __inline
void stuff1st_order_buv_8x8
(
John Koleszar's avatar
John Koleszar committed
1278
1279
1280
1281
1282
1283
  const BLOCKD *const b,
  TOKENEXTRA **tp,
  const int type,     /* which plane: 0=Y no DC, 1=Y2, 2=UV, 3=Y with DC */
  const FRAME_TYPE frametype,
  ENTROPY_CONTEXT *a,
  ENTROPY_CONTEXT *l,
1284
1285
  VP8_COMP *cpi,
  int dry_run) {
John Koleszar's avatar
John Koleszar committed
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
  int pt; /* near block/prev token context index */
  TOKENEXTRA *t = *tp;        /* store tokens starting here */
  VP8_COMBINEENTROPYCONTEXTS(pt, *a, *l);
  (void) frametype;
  (void) type;
  (void) b;

  t->Token = DCT_EOB_TOKEN;
  t->context_tree = cpi->common.fc.coef_probs_8x8 [2] [0] [pt];
  // t->section = 13;
  t->skip_eob_node = 0;
  ++t;
  *tp = t;
1299
1300
  if (!dry_run)
    ++cpi->coef_counts_8x8[2] [0] [pt] [DCT_EOB_TOKEN];
John Koleszar's avatar
John Koleszar committed
1301
1302
  pt = 0; /* 0 <-> all coeff data is zero */
  *a = *l = pt;
1303
1304
1305

}