rdopt.c 303 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
/*
 *  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include <assert.h>
#include <math.h>

#include "./vp10_rtcd.h"
#include "./vpx_dsp_rtcd.h"

17
#include "vpx_dsp/vpx_dsp_common.h"
Jingning Han's avatar
Jingning Han committed
18
19
#include "vpx_mem/vpx_mem.h"
#include "vpx_ports/mem.h"
20
#include "vpx_ports/system_state.h"
Jingning Han's avatar
Jingning Han committed
21

22
23
24
25
26
27
28
29
30
31
32
#include "vp10/common/common.h"
#include "vp10/common/entropy.h"
#include "vp10/common/entropymode.h"
#include "vp10/common/idct.h"
#include "vp10/common/mvref_common.h"
#include "vp10/common/pred_common.h"
#include "vp10/common/quant_common.h"
#include "vp10/common/reconinter.h"
#include "vp10/common/reconintra.h"
#include "vp10/common/scan.h"
#include "vp10/common/seg_common.h"
Jingning Han's avatar
Jingning Han committed
33

34
35
36
37
#include "vp10/encoder/cost.h"
#include "vp10/encoder/encodemb.h"
#include "vp10/encoder/encodemv.h"
#include "vp10/encoder/encoder.h"
Angie Chiang's avatar
Angie Chiang committed
38
#include "vp10/encoder/hybrid_fwd_txfm.h"
39
#include "vp10/encoder/mcomp.h"
hui su's avatar
hui su committed
40
#include "vp10/encoder/palette.h"
41
42
43
44
45
#include "vp10/encoder/quantize.h"
#include "vp10/encoder/ratectrl.h"
#include "vp10/encoder/rd.h"
#include "vp10/encoder/rdopt.h"
#include "vp10/encoder/aq_variance.h"
Jingning Han's avatar
Jingning Han committed
46

47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
#if CONFIG_EXT_REFS

#define LAST_FRAME_MODE_MASK    ((1 << GOLDEN_FRAME) | (1 << ALTREF_FRAME) | \
                                 (1 << LAST2_FRAME) | (1 << INTRA_FRAME) | \
                                 (1 << LAST3_FRAME) | (1 << LAST4_FRAME))
#define LAST2_FRAME_MODE_MASK   ((1 << GOLDEN_FRAME) | (1 << ALTREF_FRAME) | \
                                 (1 << LAST_FRAME) | (1 << INTRA_FRAME) | \
                                 (1 << LAST3_FRAME) | (1 << LAST4_FRAME))
#define LAST3_FRAME_MODE_MASK   ((1 << GOLDEN_FRAME) | (1 << ALTREF_FRAME) | \
                                 (1 << LAST_FRAME) | (1 << INTRA_FRAME) | \
                                 (1 << LAST2_FRAME) | (1 << LAST4_FRAME))
#define LAST4_FRAME_MODE_MASK   ((1 << GOLDEN_FRAME) | (1 << ALTREF_FRAME) | \
                                 (1 << LAST_FRAME) | (1 << INTRA_FRAME) | \
                                 (1 << LAST2_FRAME) | (1 << LAST3_FRAME))
#define GOLDEN_FRAME_MODE_MASK  ((1 << LAST_FRAME) | (1 << ALTREF_FRAME) | \
                                 (1 << LAST2_FRAME) | (1 << INTRA_FRAME) | \
                                 (1 << LAST3_FRAME) | (1 << LAST4_FRAME))
#define ALT_REF_MODE_MASK       ((1 << LAST_FRAME) | (1 << GOLDEN_FRAME) | \
                                 (1 << LAST2_FRAME) | (1 << INTRA_FRAME) | \
                                 (1 << LAST3_FRAME) | (1 << LAST4_FRAME))

#else

Jingning Han's avatar
Jingning Han committed
70
71
72
73
74
75
76
#define LAST_FRAME_MODE_MASK    ((1 << GOLDEN_FRAME) | (1 << ALTREF_FRAME) | \
                                 (1 << INTRA_FRAME))
#define GOLDEN_FRAME_MODE_MASK  ((1 << LAST_FRAME) | (1 << ALTREF_FRAME) | \
                                 (1 << INTRA_FRAME))
#define ALT_REF_MODE_MASK       ((1 << LAST_FRAME) | (1 << GOLDEN_FRAME) | \
                                 (1 << INTRA_FRAME))

77
78
#endif  // CONFIG_EXT_REFS

Jingning Han's avatar
Jingning Han committed
79
80
81
82
83
#define SECOND_REF_FRAME_MASK   ((1 << ALTREF_FRAME) | 0x01)

#define MIN_EARLY_TERM_INDEX    3
#define NEW_MV_DISCOUNT_FACTOR  8

84
#if CONFIG_EXT_TX
85
const double ext_tx_th = 0.98;
86
#else
87
const double ext_tx_th = 0.99;
88
89
#endif

90

Jingning Han's avatar
Jingning Han committed
91
92
93
94
95
96
97
98
99
100
typedef struct {
  PREDICTION_MODE mode;
  MV_REFERENCE_FRAME ref_frame[2];
} MODE_DEFINITION;

typedef struct {
  MV_REFERENCE_FRAME ref_frame[2];
} REF_DEFINITION;

struct rdcost_block_args {
Jingning Han's avatar
Jingning Han committed
101
102
103
#if CONFIG_VAR_TX
  const VP10_COMP *cpi;
#endif
Jingning Han's avatar
Jingning Han committed
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
  MACROBLOCK *x;
  ENTROPY_CONTEXT t_above[16];
  ENTROPY_CONTEXT t_left[16];
  int this_rate;
  int64_t this_dist;
  int64_t this_sse;
  int64_t this_rd;
  int64_t best_rd;
  int exit_early;
  int use_fast_coef_costing;
  const scan_order *so;
  uint8_t skippable;
};

#define LAST_NEW_MV_INDEX 6
static const MODE_DEFINITION vp10_mode_order[MAX_MODES] = {
  {NEARESTMV, {LAST_FRAME,   NONE}},
121
122
123
124
125
#if CONFIG_EXT_REFS
  {NEARESTMV, {LAST2_FRAME,  NONE}},
  {NEARESTMV, {LAST3_FRAME,  NONE}},
  {NEARESTMV, {LAST4_FRAME,  NONE}},
#endif  // CONFIG_EXT_REFS
Jingning Han's avatar
Jingning Han committed
126
127
128
129
130
131
  {NEARESTMV, {ALTREF_FRAME, NONE}},
  {NEARESTMV, {GOLDEN_FRAME, NONE}},

  {DC_PRED,   {INTRA_FRAME,  NONE}},

  {NEWMV,     {LAST_FRAME,   NONE}},
132
133
134
135
136
#if CONFIG_EXT_REFS
  {NEWMV,     {LAST2_FRAME,  NONE}},
  {NEWMV,     {LAST3_FRAME,  NONE}},
  {NEWMV,     {LAST4_FRAME,  NONE}},
#endif  // CONFIG_EXT_REFS
Jingning Han's avatar
Jingning Han committed
137
138
139
140
  {NEWMV,     {ALTREF_FRAME, NONE}},
  {NEWMV,     {GOLDEN_FRAME, NONE}},

  {NEARMV,    {LAST_FRAME,   NONE}},
141
142
143
144
145
#if CONFIG_EXT_REFS
  {NEARMV,    {LAST2_FRAME,  NONE}},
  {NEARMV,    {LAST3_FRAME,  NONE}},
  {NEARMV,    {LAST4_FRAME,  NONE}},
#endif  // CONFIG_EXT_REFS
Jingning Han's avatar
Jingning Han committed
146
147
148
  {NEARMV,    {ALTREF_FRAME, NONE}},
  {NEARMV,    {GOLDEN_FRAME, NONE}},

Yue Chen's avatar
Yue Chen committed
149
150
#if CONFIG_EXT_INTER
  {NEWFROMNEARMV,    {LAST_FRAME,   NONE}},
151
#if CONFIG_EXT_REFS
Yue Chen's avatar
Yue Chen committed
152
153
154
  {NEWFROMNEARMV,    {LAST2_FRAME,  NONE}},
  {NEWFROMNEARMV,    {LAST3_FRAME,  NONE}},
  {NEWFROMNEARMV,    {LAST4_FRAME,  NONE}},
155
#endif  // CONFIG_EXT_REFS
Yue Chen's avatar
Yue Chen committed
156
157
158
159
  {NEWFROMNEARMV,    {ALTREF_FRAME, NONE}},
  {NEWFROMNEARMV,    {GOLDEN_FRAME, NONE}},
#endif  // CONFIG_EXT_INTER

Jingning Han's avatar
Jingning Han committed
160
  {ZEROMV,    {LAST_FRAME,   NONE}},
161
162
163
164
165
#if CONFIG_EXT_REFS
  {ZEROMV,    {LAST2_FRAME,  NONE}},
  {ZEROMV,    {LAST3_FRAME,  NONE}},
  {ZEROMV,    {LAST4_FRAME,  NONE}},
#endif  // CONFIG_EXT_REFS
Jingning Han's avatar
Jingning Han committed
166
167
168
  {ZEROMV,    {GOLDEN_FRAME, NONE}},
  {ZEROMV,    {ALTREF_FRAME, NONE}},

169
170
171
172
173
174
175
176
177
#if CONFIG_EXT_INTER
  {NEAREST_NEARESTMV, {LAST_FRAME,   ALTREF_FRAME}},
#if CONFIG_EXT_REFS
  {NEAREST_NEARESTMV, {LAST2_FRAME,  ALTREF_FRAME}},
  {NEAREST_NEARESTMV, {LAST3_FRAME,  ALTREF_FRAME}},
  {NEAREST_NEARESTMV, {LAST4_FRAME,  ALTREF_FRAME}},
#endif  // CONFIG_EXT_REFS
  {NEAREST_NEARESTMV, {GOLDEN_FRAME, ALTREF_FRAME}},
#else  // CONFIG_EXT_INTER
Jingning Han's avatar
Jingning Han committed
178
  {NEARESTMV, {LAST_FRAME,   ALTREF_FRAME}},
179
180
181
182
183
#if CONFIG_EXT_REFS
  {NEARESTMV, {LAST2_FRAME,  ALTREF_FRAME}},
  {NEARESTMV, {LAST3_FRAME,  ALTREF_FRAME}},
  {NEARESTMV, {LAST4_FRAME,  ALTREF_FRAME}},
#endif  // CONFIG_EXT_REFS
Jingning Han's avatar
Jingning Han committed
184
  {NEARESTMV, {GOLDEN_FRAME, ALTREF_FRAME}},
185
#endif  // CONFIG_EXT_INTER
Jingning Han's avatar
Jingning Han committed
186
187
188

  {TM_PRED,   {INTRA_FRAME,  NONE}},

189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
#if CONFIG_EXT_INTER
  {NEAR_NEARESTMV, {LAST_FRAME,   ALTREF_FRAME}},
  {NEAR_NEARESTMV, {GOLDEN_FRAME, ALTREF_FRAME}},
  {NEAREST_NEARMV, {LAST_FRAME,   ALTREF_FRAME}},
  {NEAREST_NEARMV, {GOLDEN_FRAME, ALTREF_FRAME}},
  {NEW_NEARESTMV, {LAST_FRAME,   ALTREF_FRAME}},
  {NEW_NEARESTMV, {GOLDEN_FRAME, ALTREF_FRAME}},
  {NEAREST_NEWMV, {LAST_FRAME,   ALTREF_FRAME}},
  {NEAREST_NEWMV, {GOLDEN_FRAME, ALTREF_FRAME}},
  {NEW_NEARMV, {LAST_FRAME,   ALTREF_FRAME}},
  {NEW_NEARMV, {GOLDEN_FRAME, ALTREF_FRAME}},
  {NEAR_NEWMV, {LAST_FRAME,   ALTREF_FRAME}},
  {NEAR_NEWMV, {GOLDEN_FRAME, ALTREF_FRAME}},
  {NEW_NEWMV,      {LAST_FRAME,   ALTREF_FRAME}},
  {NEW_NEWMV,      {GOLDEN_FRAME, ALTREF_FRAME}},
  {ZERO_ZEROMV,    {LAST_FRAME,   ALTREF_FRAME}},
  {ZERO_ZEROMV,    {GOLDEN_FRAME, ALTREF_FRAME}},
#if CONFIG_EXT_REFS
  {NEAR_NEARESTMV, {LAST2_FRAME,  ALTREF_FRAME}},
  {NEAREST_NEARMV, {LAST2_FRAME,  ALTREF_FRAME}},
  {NEW_NEARESTMV,  {LAST2_FRAME,  ALTREF_FRAME}},
  {NEAREST_NEWMV,  {LAST2_FRAME,  ALTREF_FRAME}},
  {NEW_NEARMV,     {LAST2_FRAME,  ALTREF_FRAME}},
  {NEAR_NEWMV,     {LAST2_FRAME,  ALTREF_FRAME}},
  {NEW_NEWMV,      {LAST2_FRAME,  ALTREF_FRAME}},
  {ZERO_ZEROMV,    {LAST2_FRAME,  ALTREF_FRAME}},

  {NEAR_NEARESTMV, {LAST3_FRAME,  ALTREF_FRAME}},
  {NEAREST_NEARMV, {LAST3_FRAME,  ALTREF_FRAME}},
  {NEW_NEARESTMV,  {LAST3_FRAME,  ALTREF_FRAME}},
  {NEAREST_NEWMV,  {LAST3_FRAME,  ALTREF_FRAME}},
  {NEW_NEARMV,     {LAST3_FRAME,  ALTREF_FRAME}},
  {NEAR_NEWMV,     {LAST3_FRAME,  ALTREF_FRAME}},
  {NEW_NEWMV,      {LAST3_FRAME,  ALTREF_FRAME}},
  {ZERO_ZEROMV,    {LAST3_FRAME,  ALTREF_FRAME}},

  {NEAR_NEARESTMV, {LAST4_FRAME,  ALTREF_FRAME}},
  {NEAREST_NEARMV, {LAST4_FRAME,  ALTREF_FRAME}},
  {NEW_NEARESTMV,  {LAST4_FRAME,  ALTREF_FRAME}},
  {NEAREST_NEWMV,  {LAST4_FRAME,  ALTREF_FRAME}},
  {NEW_NEARMV,     {LAST4_FRAME,  ALTREF_FRAME}},
  {NEAR_NEWMV,     {LAST4_FRAME,  ALTREF_FRAME}},
  {NEW_NEWMV,      {LAST4_FRAME,  ALTREF_FRAME}},
  {ZERO_ZEROMV,    {LAST4_FRAME,  ALTREF_FRAME}},
#endif  // CONFIG_EXT_REFS
#else
Jingning Han's avatar
Jingning Han committed
235
236
  {NEARMV,    {LAST_FRAME,   ALTREF_FRAME}},
  {NEWMV,     {LAST_FRAME,   ALTREF_FRAME}},
237
238
239
240
241
242
243
244
#if CONFIG_EXT_REFS
  {NEARMV,    {LAST2_FRAME,  ALTREF_FRAME}},
  {NEWMV,     {LAST2_FRAME,  ALTREF_FRAME}},
  {NEARMV,    {LAST3_FRAME,  ALTREF_FRAME}},
  {NEWMV,     {LAST3_FRAME,  ALTREF_FRAME}},
  {NEARMV,    {LAST4_FRAME,  ALTREF_FRAME}},
  {NEWMV,     {LAST4_FRAME,  ALTREF_FRAME}},
#endif  // CONFIG_EXT_REFS
Jingning Han's avatar
Jingning Han committed
245
246
247
248
  {NEARMV,    {GOLDEN_FRAME, ALTREF_FRAME}},
  {NEWMV,     {GOLDEN_FRAME, ALTREF_FRAME}},

  {ZEROMV,    {LAST_FRAME,   ALTREF_FRAME}},
249
250
251
252
253
#if CONFIG_EXT_REFS
  {ZEROMV,    {LAST3_FRAME,  ALTREF_FRAME}},
  {ZEROMV,    {LAST2_FRAME,  ALTREF_FRAME}},
  {ZEROMV,    {LAST4_FRAME,  ALTREF_FRAME}},
#endif  // CONFIG_EXT_REFS
Jingning Han's avatar
Jingning Han committed
254
  {ZEROMV,    {GOLDEN_FRAME, ALTREF_FRAME}},
255
#endif  // CONFIG_EXT_INTER
Jingning Han's avatar
Jingning Han committed
256
257
258
259
260
261
262
263
264
265
266
267
268

  {H_PRED,    {INTRA_FRAME,  NONE}},
  {V_PRED,    {INTRA_FRAME,  NONE}},
  {D135_PRED, {INTRA_FRAME,  NONE}},
  {D207_PRED, {INTRA_FRAME,  NONE}},
  {D153_PRED, {INTRA_FRAME,  NONE}},
  {D63_PRED,  {INTRA_FRAME,  NONE}},
  {D117_PRED, {INTRA_FRAME,  NONE}},
  {D45_PRED,  {INTRA_FRAME,  NONE}},
};

static const REF_DEFINITION vp10_ref_order[MAX_REFS] = {
  {{LAST_FRAME,   NONE}},
269
270
271
272
273
#if CONFIG_EXT_REFS
  {{LAST2_FRAME,  NONE}},
  {{LAST3_FRAME,  NONE}},
  {{LAST4_FRAME,  NONE}},
#endif  // CONFIG_EXT_REFS
Jingning Han's avatar
Jingning Han committed
274
275
276
  {{GOLDEN_FRAME, NONE}},
  {{ALTREF_FRAME, NONE}},
  {{LAST_FRAME,   ALTREF_FRAME}},
277
278
279
280
281
#if CONFIG_EXT_REFS
  {{LAST2_FRAME,  ALTREF_FRAME}},
  {{LAST3_FRAME,  ALTREF_FRAME}},
  {{LAST4_FRAME,  ALTREF_FRAME}},
#endif  // CONFIG_EXT_REFS
Jingning Han's avatar
Jingning Han committed
282
283
284
285
  {{GOLDEN_FRAME, ALTREF_FRAME}},
  {{INTRA_FRAME,  NONE}},
};

hui su's avatar
hui su committed
286
287
288
289
290
291
292
293
294
295
static INLINE int write_uniform_cost(int n, int v) {
  int l = get_unsigned_bits(n), m = (1 << l) - n;
  if (l == 0)
    return 0;
  if (v < m)
    return (l - 1) * vp10_cost_bit(128, 0);
  else
    return l * vp10_cost_bit(128, 0);
}

Jingning Han's avatar
Jingning Han committed
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
static void swap_block_ptr(MACROBLOCK *x, PICK_MODE_CONTEXT *ctx,
                           int m, int n, int min_plane, int max_plane) {
  int i;

  for (i = min_plane; i < max_plane; ++i) {
    struct macroblock_plane *const p = &x->plane[i];
    struct macroblockd_plane *const pd = &x->e_mbd.plane[i];

    p->coeff    = ctx->coeff_pbuf[i][m];
    p->qcoeff   = ctx->qcoeff_pbuf[i][m];
    pd->dqcoeff = ctx->dqcoeff_pbuf[i][m];
    p->eobs     = ctx->eobs_pbuf[i][m];

    ctx->coeff_pbuf[i][m]   = ctx->coeff_pbuf[i][n];
    ctx->qcoeff_pbuf[i][m]  = ctx->qcoeff_pbuf[i][n];
    ctx->dqcoeff_pbuf[i][m] = ctx->dqcoeff_pbuf[i][n];
    ctx->eobs_pbuf[i][m]    = ctx->eobs_pbuf[i][n];

    ctx->coeff_pbuf[i][n]   = p->coeff;
    ctx->qcoeff_pbuf[i][n]  = p->qcoeff;
    ctx->dqcoeff_pbuf[i][n] = pd->dqcoeff;
    ctx->eobs_pbuf[i][n]    = p->eobs;
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
321
static void model_rd_for_sb(VP10_COMP *cpi, BLOCK_SIZE bsize,
Jingning Han's avatar
Jingning Han committed
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
                            MACROBLOCK *x, MACROBLOCKD *xd,
                            int *out_rate_sum, int64_t *out_dist_sum,
                            int *skip_txfm_sb, int64_t *skip_sse_sb) {
  // Note our transform coeffs are 8 times an orthogonal transform.
  // Hence quantizer step is also 8 times. To get effective quantizer
  // we need to divide by 8 before sending to modeling function.
  int i;
  int64_t rate_sum = 0;
  int64_t dist_sum = 0;
  const int ref = xd->mi[0]->mbmi.ref_frame[0];
  unsigned int sse;
  unsigned int var = 0;
  unsigned int sum_sse = 0;
  int64_t total_sse = 0;
  int skip_flag = 1;
  const int shift = 6;
  int rate;
  int64_t dist;
  const int dequant_shift =
#if CONFIG_VP9_HIGHBITDEPTH
      (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) ?
          xd->bd - 5 :
#endif  // CONFIG_VP9_HIGHBITDEPTH
          3;

  x->pred_sse[ref] = 0;

  for (i = 0; i < MAX_MB_PLANE; ++i) {
    struct macroblock_plane *const p = &x->plane[i];
    struct macroblockd_plane *const pd = &xd->plane[i];
    const BLOCK_SIZE bs = get_plane_block_size(bsize, pd);
    const TX_SIZE max_tx_size = max_txsize_lookup[bs];
    const BLOCK_SIZE unit_size = txsize_to_bsize[max_tx_size];
    const int64_t dc_thr = p->quant_thred[0] >> shift;
    const int64_t ac_thr = p->quant_thred[1] >> shift;
    // The low thresholds are used to measure if the prediction errors are
    // low enough so that we can skip the mode search.
359
360
    const int64_t low_dc_thr = VPXMIN(50, dc_thr >> 2);
    const int64_t low_ac_thr = VPXMIN(80, ac_thr >> 2);
Jingning Han's avatar
Jingning Han committed
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
    int bw = 1 << (b_width_log2_lookup[bs] - b_width_log2_lookup[unit_size]);
    int bh = 1 << (b_height_log2_lookup[bs] - b_width_log2_lookup[unit_size]);
    int idx, idy;
    int lw = b_width_log2_lookup[unit_size] + 2;
    int lh = b_height_log2_lookup[unit_size] + 2;

    sum_sse = 0;

    for (idy = 0; idy < bh; ++idy) {
      for (idx = 0; idx < bw; ++idx) {
        uint8_t *src = p->src.buf + (idy * p->src.stride << lh) + (idx << lw);
        uint8_t *dst = pd->dst.buf + (idy * pd->dst.stride << lh) + (idx << lh);
        int block_idx = (idy << 1) + idx;
        int low_err_skip = 0;

        var = cpi->fn_ptr[unit_size].vf(src, p->src.stride,
                                        dst, pd->dst.stride, &sse);
        x->bsse[(i << 2) + block_idx] = sse;
        sum_sse += sse;

        x->skip_txfm[(i << 2) + block_idx] = SKIP_TXFM_NONE;
        if (!x->select_tx_size) {
          // Check if all ac coefficients can be quantized to zero.
          if (var < ac_thr || var == 0) {
            x->skip_txfm[(i << 2) + block_idx] = SKIP_TXFM_AC_ONLY;

            // Check if dc coefficient can be quantized to zero.
            if (sse - var < dc_thr || sse == var) {
              x->skip_txfm[(i << 2) + block_idx] = SKIP_TXFM_AC_DC;

              if (!sse || (var < low_ac_thr && sse - var < low_dc_thr))
                low_err_skip = 1;
            }
          }
        }

        if (skip_flag && !low_err_skip)
          skip_flag = 0;

        if (i == 0)
          x->pred_sse[ref] += sse;
      }
    }

    total_sse += sum_sse;

    // Fast approximate the modelling function.
    if (cpi->sf.simple_model_rd_from_var) {
      int64_t rate;
      const int64_t square_error = sum_sse;
      int quantizer = (pd->dequant[1] >> dequant_shift);

      if (quantizer < 120)
414
        rate = (square_error * (280 - quantizer)) >> (16 - VP9_PROB_COST_SHIFT);
Jingning Han's avatar
Jingning Han committed
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
      else
        rate = 0;
      dist = (square_error * quantizer) >> 8;
      rate_sum += rate;
      dist_sum += dist;
    } else {
      vp10_model_rd_from_var_lapndz(sum_sse, num_pels_log2_lookup[bs],
                                   pd->dequant[1] >> dequant_shift,
                                   &rate, &dist);
      rate_sum += rate;
      dist_sum += dist;
    }
  }

  *skip_txfm_sb = skip_flag;
  *skip_sse_sb = total_sse << 4;
  *out_rate_sum = (int)rate_sum;
  *out_dist_sum = dist_sum << 4;
}

int64_t vp10_block_error_c(const tran_low_t *coeff, const tran_low_t *dqcoeff,
                          intptr_t block_size, int64_t *ssz) {
  int i;
  int64_t error = 0, sqcoeff = 0;

  for (i = 0; i < block_size; i++) {
    const int diff = coeff[i] - dqcoeff[i];
    error +=  diff * diff;
    sqcoeff += coeff[i] * coeff[i];
  }

  *ssz = sqcoeff;
  return error;
}

int64_t vp10_block_error_fp_c(const int16_t *coeff, const int16_t *dqcoeff,
                             int block_size) {
  int i;
  int64_t error = 0;

  for (i = 0; i < block_size; i++) {
    const int diff = coeff[i] - dqcoeff[i];
    error +=  diff * diff;
  }

  return error;
}

#if CONFIG_VP9_HIGHBITDEPTH
int64_t vp10_highbd_block_error_c(const tran_low_t *coeff,
                                 const tran_low_t *dqcoeff,
                                 intptr_t block_size,
                                 int64_t *ssz, int bd) {
  int i;
  int64_t error = 0, sqcoeff = 0;
  int shift = 2 * (bd - 8);
  int rounding = shift > 0 ? 1 << (shift - 1) : 0;

  for (i = 0; i < block_size; i++) {
    const int64_t diff = coeff[i] - dqcoeff[i];
    error +=  diff * diff;
    sqcoeff += (int64_t)coeff[i] * (int64_t)coeff[i];
  }
  assert(error >= 0 && sqcoeff >= 0);
  error = (error + rounding) >> shift;
  sqcoeff = (sqcoeff + rounding) >> shift;

  *ssz = sqcoeff;
  return error;
}
#endif  // CONFIG_VP9_HIGHBITDEPTH

/* The trailing '0' is a terminator which is used inside cost_coeffs() to
 * decide whether to include cost of a trailing EOB node or not (i.e. we
 * can skip this if the last coefficient in this transform block, e.g. the
 * 16th coefficient in a 4x4 block or the 64th coefficient in a 8x8 block,
 * were non-zero). */
static const int16_t band_counts[TX_SIZES][8] = {
  { 1, 2, 3, 4,  3,   16 - 13, 0 },
  { 1, 2, 3, 4, 11,   64 - 21, 0 },
  { 1, 2, 3, 4, 11,  256 - 21, 0 },
  { 1, 2, 3, 4, 11, 1024 - 21, 0 },
};
static int cost_coeffs(MACROBLOCK *x,
                       int plane, int block,
500
501
502
#if CONFIG_VAR_TX
                       int coeff_ctx,
#else
Jingning Han's avatar
Jingning Han committed
503
                       ENTROPY_CONTEXT *A, ENTROPY_CONTEXT *L,
504
#endif
Jingning Han's avatar
Jingning Han committed
505
506
507
508
509
510
511
512
513
514
515
516
517
518
                       TX_SIZE tx_size,
                       const int16_t *scan, const int16_t *nb,
                       int use_fast_coef_costing) {
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *mbmi = &xd->mi[0]->mbmi;
  const struct macroblock_plane *p = &x->plane[plane];
  const struct macroblockd_plane *pd = &xd->plane[plane];
  const PLANE_TYPE type = pd->plane_type;
  const int16_t *band_count = &band_counts[tx_size][1];
  const int eob = p->eobs[block];
  const tran_low_t *const qcoeff = BLOCK_OFFSET(p->qcoeff, block);
  unsigned int (*token_costs)[2][COEFF_CONTEXTS][ENTROPY_TOKENS] =
                   x->token_costs[tx_size][type][is_inter_block(mbmi)];
  uint8_t token_cache[32 * 32];
519
520
521
#if CONFIG_VAR_TX
  int pt = coeff_ctx;
#else
Jingning Han's avatar
Jingning Han committed
522
  int pt = combine_entropy_contexts(*A, *L);
523
#endif
Jingning Han's avatar
Jingning Han committed
524
525
  int c, cost;
#if CONFIG_VP9_HIGHBITDEPTH
526
  const int *cat6_high_cost = vp10_get_high_cost_table(xd->bd);
Jingning Han's avatar
Jingning Han committed
527
#else
528
  const int *cat6_high_cost = vp10_get_high_cost_table(8);
Jingning Han's avatar
Jingning Han committed
529
530
#endif

531
#if !CONFIG_VAR_TX && !CONFIG_SUPERTX
Jingning Han's avatar
Jingning Han committed
532
533
534
  // Check for consistency of tx_size with mode info
  assert(type == PLANE_TYPE_Y ? mbmi->tx_size == tx_size
                              : get_uv_tx_size(mbmi, pd) == tx_size);
535
#endif  // !CONFIG_VAR_TX && !CONFIG_SUPERTX
Jingning Han's avatar
Jingning Han committed
536
537
538
539
540
541

  if (eob == 0) {
    // single eob token
    cost = token_costs[0][0][pt][EOB_TOKEN];
    c = 0;
  } else {
Julia Robson's avatar
Julia Robson committed
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
    if (use_fast_coef_costing) {
      int band_left = *band_count++;

      // dc token
      int v = qcoeff[0];
      int16_t prev_t;
      cost = vp10_get_token_cost(v, &prev_t, cat6_high_cost);
      cost += (*token_costs)[0][pt][prev_t];

      token_cache[0] = vp10_pt_energy_class[prev_t];
      ++token_costs;

      // ac tokens
      for (c = 1; c < eob; c++) {
        const int rc = scan[c];
        int16_t t;

        v = qcoeff[rc];
        cost += vp10_get_token_cost(v, &t, cat6_high_cost);
        cost += (*token_costs)[!prev_t][!prev_t][t];
        prev_t = t;
        if (!--band_left) {
          band_left = *band_count++;
          ++token_costs;
        }
Jingning Han's avatar
Jingning Han committed
567
568
      }

Julia Robson's avatar
Julia Robson committed
569
570
      // eob token
      if (band_left)
Jingning Han's avatar
Jingning Han committed
571
        cost += (*token_costs)[0][!prev_t][EOB_TOKEN];
Julia Robson's avatar
Julia Robson committed
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605

    } else {  // !use_fast_coef_costing
      int band_left = *band_count++;

      // dc token
      int v = qcoeff[0];
      int16_t tok;
      unsigned int (*tok_cost_ptr)[COEFF_CONTEXTS][ENTROPY_TOKENS];
      cost = vp10_get_token_cost(v, &tok, cat6_high_cost);
      cost += (*token_costs)[0][pt][tok];

      token_cache[0] = vp10_pt_energy_class[tok];
      ++token_costs;

      tok_cost_ptr = &((*token_costs)[!tok]);

      // ac tokens
      for (c = 1; c < eob; c++) {
        const int rc = scan[c];

        v = qcoeff[rc];
        cost += vp10_get_token_cost(v, &tok, cat6_high_cost);
        pt = get_coef_context(nb, token_cache, c);
        cost += (*tok_cost_ptr)[pt][tok];
        token_cache[rc] = vp10_pt_energy_class[tok];
        if (!--band_left) {
          band_left = *band_count++;
          ++token_costs;
        }
        tok_cost_ptr = &((*token_costs)[!tok]);
      }

      // eob token
      if (band_left) {
Jingning Han's avatar
Jingning Han committed
606
607
608
609
610
611
        pt = get_coef_context(nb, token_cache, c);
        cost += (*token_costs)[0][pt][EOB_TOKEN];
      }
    }
  }

612
#if !CONFIG_VAR_TX
Jingning Han's avatar
Jingning Han committed
613
614
  // is eob first coefficient;
  *A = *L = (c > 0);
615
#endif
Jingning Han's avatar
Jingning Han committed
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640

  return cost;
}

static void dist_block(MACROBLOCK *x, int plane, int block, TX_SIZE tx_size,
                       int64_t *out_dist, int64_t *out_sse) {
  const int ss_txfrm_size = tx_size << 1;
  MACROBLOCKD* const xd = &x->e_mbd;
  const struct macroblock_plane *const p = &x->plane[plane];
  const struct macroblockd_plane *const pd = &xd->plane[plane];
  int64_t this_sse;
  int shift = tx_size == TX_32X32 ? 0 : 2;
  tran_low_t *const coeff = BLOCK_OFFSET(p->coeff, block);
  tran_low_t *const dqcoeff = BLOCK_OFFSET(pd->dqcoeff, block);
#if CONFIG_VP9_HIGHBITDEPTH
  const int bd = (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) ? xd->bd : 8;
  *out_dist = vp10_highbd_block_error(coeff, dqcoeff, 16 << ss_txfrm_size,
                                     &this_sse, bd) >> shift;
#else
  *out_dist = vp10_block_error(coeff, dqcoeff, 16 << ss_txfrm_size,
                              &this_sse) >> shift;
#endif  // CONFIG_VP9_HIGHBITDEPTH
  *out_sse = this_sse >> shift;
}

641
static int rate_block(int plane, int block, int blk_row, int blk_col,
Jingning Han's avatar
Jingning Han committed
642
                      TX_SIZE tx_size, struct rdcost_block_args* args) {
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
#if CONFIG_VAR_TX
  int coeff_ctx = combine_entropy_contexts(*(args->t_above + blk_col),
                                           *(args->t_left + blk_row));
  int coeff_cost = cost_coeffs(args->x, plane, block, coeff_ctx,
                               tx_size, args->so->scan, args->so->neighbors,
                               args->use_fast_coef_costing);
  const struct macroblock_plane *p = &args->x->plane[plane];
  *(args->t_above + blk_col) = !(p->eobs[block] == 0);
  *(args->t_left  + blk_row) = !(p->eobs[block] == 0);
  return coeff_cost;
#else
  return cost_coeffs(args->x, plane, block,
                     args->t_above + blk_col,
                     args->t_left + blk_row,
                     tx_size, args->so->scan, args->so->neighbors,
Jingning Han's avatar
Jingning Han committed
658
                     args->use_fast_coef_costing);
659
#endif
Jingning Han's avatar
Jingning Han committed
660
661
}

662
663
static void block_rd_txfm(int plane, int block, int blk_row, int blk_col,
                          BLOCK_SIZE plane_bsize,
Jingning Han's avatar
Jingning Han committed
664
665
666
667
668
669
670
671
672
673
674
675
676
677
                          TX_SIZE tx_size, void *arg) {
  struct rdcost_block_args *args = arg;
  MACROBLOCK *const x = args->x;
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
  int64_t rd1, rd2, rd;
  int rate;
  int64_t dist;
  int64_t sse;

  if (args->exit_early)
    return;

  if (!is_inter_block(mbmi)) {
Jingning Han's avatar
Jingning Han committed
678
    struct encode_b_args arg = {x, NULL, &mbmi->skip};
679
#if CONFIG_VAR_TX
Jingning Han's avatar
Jingning Han committed
680
681
    vp10_encode_block_intra(plane, block, blk_row, blk_col,
                            plane_bsize, tx_size, &arg);
682
683

    {
684
685
686
687
#if CONFIG_VP9_HIGHBITDEPTH
      const VP10_COMP *cpi = args->cpi;
      const uint32_t hbd_shift = (cpi->common.bit_depth - 8) * 2;
#endif
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
      const int bs = 4 << tx_size;
      const BLOCK_SIZE tx_bsize = txsize_to_bsize[tx_size];
      const vpx_variance_fn_t variance = args->cpi->fn_ptr[tx_bsize].vf;

      const struct macroblock_plane *const p = &x->plane[plane];
      const struct macroblockd_plane *const pd = &xd->plane[plane];

      const int src_stride = p->src.stride;
      const int dst_stride = pd->dst.stride;
      const int diff_stride = 4 * num_4x4_blocks_wide_lookup[plane_bsize];

      const uint8_t *src = &p->src.buf[4 * (blk_row * src_stride + blk_col)];
      const uint8_t *dst = &pd->dst.buf[4 * (blk_row * dst_stride + blk_col)];
      const int16_t *diff = &p->src_diff[4 * (blk_row * diff_stride + blk_col)];

      unsigned int tmp;
704
705
706
707
#if CONFIG_VP9_HIGHBITDEPTH
      sse = (int64_t)ROUND_POWER_OF_TWO(
          vpx_sum_squares_2d_i16(diff, diff_stride, bs), hbd_shift) * 16;
#else
708
      sse = (int64_t)vpx_sum_squares_2d_i16(diff, diff_stride, bs) * 16;
709
#endif
710
711
712
      variance(src, src_stride, dst, dst_stride, &tmp);
      dist = (int64_t)tmp * 16;
    }
Jingning Han's avatar
Jingning Han committed
713
#else
714
715
    vp10_encode_block_intra(plane, block, blk_row, blk_col,
                            plane_bsize, tx_size, &arg);
Jingning Han's avatar
Jingning Han committed
716
    dist_block(x, plane, block, tx_size, &dist, &sse);
Jingning Han's avatar
Jingning Han committed
717
#endif
Jingning Han's avatar
Jingning Han committed
718
719
720
721
  } else if (max_txsize_lookup[plane_bsize] == tx_size) {
    if (x->skip_txfm[(plane << 2) + (block >> (tx_size << 1))] ==
        SKIP_TXFM_NONE) {
      // full forward transform and quantization
722
      vp10_xform_quant(x, plane, block, blk_row, blk_col,
Angie Chiang's avatar
Angie Chiang committed
723
                       plane_bsize, tx_size, VP10_XFORM_QUANT_B);
Jingning Han's avatar
Jingning Han committed
724
725
726
727
728
729
      dist_block(x, plane, block, tx_size, &dist, &sse);
    } else if (x->skip_txfm[(plane << 2) + (block >> (tx_size << 1))] ==
               SKIP_TXFM_AC_ONLY) {
      // compute DC coefficient
      tran_low_t *const coeff   = BLOCK_OFFSET(x->plane[plane].coeff, block);
      tran_low_t *const dqcoeff = BLOCK_OFFSET(xd->plane[plane].dqcoeff, block);
Angie Chiang's avatar
Angie Chiang committed
730
731
      vp10_xform_quant(x, plane, block, blk_row, blk_col,
                          plane_bsize, tx_size, VP10_XFORM_QUANT_DC);
Jingning Han's avatar
Jingning Han committed
732
733
734
735
736
737
738
739
740
741
742
743
      sse  = x->bsse[(plane << 2) + (block >> (tx_size << 1))] << 4;
      dist = sse;
      if (x->plane[plane].eobs[block]) {
        const int64_t orig_sse = (int64_t)coeff[0] * coeff[0];
        const int64_t resd_sse = coeff[0] - dqcoeff[0];
        int64_t dc_correct = orig_sse - resd_sse * resd_sse;
#if CONFIG_VP9_HIGHBITDEPTH
        dc_correct >>= ((xd->bd - 8) * 2);
#endif
        if (tx_size != TX_32X32)
          dc_correct >>= 2;

744
        dist = VPXMAX(0, sse - dc_correct);
Jingning Han's avatar
Jingning Han committed
745
746
747
748
749
750
751
752
753
754
      }
    } else {
      // SKIP_TXFM_AC_DC
      // skip forward transform
      x->plane[plane].eobs[block] = 0;
      sse  = x->bsse[(plane << 2) + (block >> (tx_size << 1))] << 4;
      dist = sse;
    }
  } else {
    // full forward transform and quantization
Angie Chiang's avatar
Angie Chiang committed
755
756
    vp10_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize, tx_size,
                     VP10_XFORM_QUANT_B);
Jingning Han's avatar
Jingning Han committed
757
758
759
760
761
762
763
764
765
    dist_block(x, plane, block, tx_size, &dist, &sse);
  }

  rd = RDCOST(x->rdmult, x->rddiv, 0, dist);
  if (args->this_rd + rd > args->best_rd) {
    args->exit_early = 1;
    return;
  }

766
  rate = rate_block(plane, block, blk_row, blk_col, tx_size, args);
Jingning Han's avatar
Jingning Han committed
767
768
769
770
  rd1 = RDCOST(x->rdmult, x->rddiv, rate, dist);
  rd2 = RDCOST(x->rdmult, x->rddiv, 0, sse);

  // TODO(jingning): temporarily enabled only for luma component
771
  rd = VPXMIN(rd1, rd2);
Jingning Han's avatar
Jingning Han committed
772
773
  if (plane == 0)
    x->zcoeff_blk[tx_size][block] = !x->plane[plane].eobs[block] ||
774
        (rd1 > rd2 && !xd->lossless[mbmi->segment_id]);
Jingning Han's avatar
Jingning Han committed
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789

  args->this_rate += rate;
  args->this_dist += dist;
  args->this_sse += sse;
  args->this_rd += rd;

  if (args->this_rd > args->best_rd) {
    args->exit_early = 1;
    return;
  }

  args->skippable &= !x->plane[plane].eobs[block];
}

static void txfm_rd_in_plane(MACROBLOCK *x,
Jingning Han's avatar
Jingning Han committed
790
791
792
#if CONFIG_VAR_TX
                             const VP10_COMP *cpi,
#endif
Jingning Han's avatar
Jingning Han committed
793
794
795
796
797
798
799
                             int *rate, int64_t *distortion,
                             int *skippable, int64_t *sse,
                             int64_t ref_best_rd, int plane,
                             BLOCK_SIZE bsize, TX_SIZE tx_size,
                             int use_fast_coef_casting) {
  MACROBLOCKD *const xd = &x->e_mbd;
  const struct macroblockd_plane *const pd = &xd->plane[plane];
hui su's avatar
hui su committed
800
  TX_TYPE tx_type;
Jingning Han's avatar
Jingning Han committed
801
802
803
  struct rdcost_block_args args;
  vp10_zero(args);
  args.x = x;
Jingning Han's avatar
Jingning Han committed
804
805
806
#if CONFIG_VAR_TX
  args.cpi = cpi;
#endif
Jingning Han's avatar
Jingning Han committed
807
808
809
810
811
812
813
814
815
  args.best_rd = ref_best_rd;
  args.use_fast_coef_costing = use_fast_coef_casting;
  args.skippable = 1;

  if (plane == 0)
    xd->mi[0]->mbmi.tx_size = tx_size;

  vp10_get_entropy_contexts(bsize, tx_size, pd, args.t_above, args.t_left);

hui su's avatar
hui su committed
816
  tx_type = get_tx_type(pd->plane_type, xd, 0, tx_size);
817
  args.so = get_scan(tx_size, tx_type, is_inter_block(&xd->mi[0]->mbmi));
Jingning Han's avatar
Jingning Han committed
818
819

  vp10_foreach_transformed_block_in_plane(xd, bsize, plane,
820
                                          block_rd_txfm, &args);
Jingning Han's avatar
Jingning Han committed
821
822
823
824
825
826
827
828
829
830
831
832
833
  if (args.exit_early) {
    *rate       = INT_MAX;
    *distortion = INT64_MAX;
    *sse        = INT64_MAX;
    *skippable  = 0;
  } else {
    *distortion = args.this_dist;
    *rate       = args.this_rate;
    *sse        = args.this_sse;
    *skippable  = args.skippable;
  }
}

834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
#if CONFIG_SUPERTX
void vp10_txfm_rd_in_plane_supertx(MACROBLOCK *x,
                                   int *rate, int64_t *distortion,
                                   int *skippable, int64_t *sse,
                                   int64_t ref_best_rd, int plane,
                                   BLOCK_SIZE bsize, TX_SIZE tx_size,
                                   int use_fast_coef_casting) {
  MACROBLOCKD *const xd = &x->e_mbd;
  const struct macroblockd_plane *const pd = &xd->plane[plane];
  struct rdcost_block_args args;
  TX_TYPE tx_type;

  vp10_zero(args);
  args.x = x;
  args.best_rd = ref_best_rd;
  args.use_fast_coef_costing = use_fast_coef_casting;

  if (plane == 0)
    xd->mi[0]->mbmi.tx_size = tx_size;

  vp10_get_entropy_contexts(bsize, tx_size, pd, args.t_above, args.t_left);

  tx_type = get_tx_type(pd->plane_type, xd, 0, tx_size);
  args.so = get_scan(tx_size, tx_type, is_inter_block(&xd->mi[0]->mbmi));

  block_rd_txfm(plane, 0, 0, 0, get_plane_block_size(bsize, pd),
                tx_size, &args);

  if (args.exit_early) {
    *rate       = INT_MAX;
    *distortion = INT64_MAX;
    *sse        = INT64_MAX;
    *skippable  = 0;
  } else {
    *distortion = args.this_dist;
    *rate       = args.this_rate;
    *sse        = args.this_sse;
    *skippable  = !x->plane[plane].eobs[0];
  }
}
#endif  // CONFIG_SUPERTX

Yaowu Xu's avatar
Yaowu Xu committed
876
static void choose_largest_tx_size(VP10_COMP *cpi, MACROBLOCK *x,
Jingning Han's avatar
Jingning Han committed
877
878
879
880
881
                                   int *rate, int64_t *distortion,
                                   int *skip, int64_t *sse,
                                   int64_t ref_best_rd,
                                   BLOCK_SIZE bs) {
  const TX_SIZE max_tx_size = max_txsize_lookup[bs];
Yaowu Xu's avatar
Yaowu Xu committed
882
  VP10_COMMON *const cm = &cpi->common;
Jingning Han's avatar
Jingning Han committed
883
884
885
  const TX_SIZE largest_tx_size = tx_mode_to_biggest_tx_size[cm->tx_mode];
  MACROBLOCKD *const xd = &x->e_mbd;
  MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
886
  TX_TYPE tx_type, best_tx_type = DCT_DCT;
887
888
889
890
891
  int r, s;
  int64_t d, psse, this_rd, best_rd = INT64_MAX;
  vpx_prob skip_prob = vp10_get_skip_prob(cm, xd);
  int  s0 = vp10_cost_bit(skip_prob, 0);
  int  s1 = vp10_cost_bit(skip_prob, 1);
892
#if CONFIG_EXT_TX
893
  int ext_tx_set;
894
#endif  // CONFIG_EXT_TX
895
  const int is_inter = is_inter_block(mbmi);
Jingning Han's avatar
Jingning Han committed
896

897
  mbmi->tx_size = VPXMIN(max_tx_size, largest_tx_size);
898

899
#if CONFIG_EXT_TX
900
901
  ext_tx_set = get_ext_tx_set(mbmi->tx_size, bs, is_inter);

902
  if (get_ext_tx_types(mbmi->tx_size, bs, is_inter) > 1 &&
Yaowu Xu's avatar
Yaowu Xu committed
903
      !xd->lossless[mbmi->segment_id]) {
904
905
906
907
908
    for (tx_type = 0; tx_type < TX_TYPES; ++tx_type) {
      if (is_inter) {
        if (!ext_tx_used_inter[ext_tx_set][tx_type])
          continue;
      } else {
909
        if (!ALLOW_INTRA_EXT_TX && bs >= BLOCK_8X8) {
910
          if (tx_type != intra_mode_to_tx_type_context[mbmi->mode])
911
912
            continue;
        }
913
914
915
916
917
918
919
        if (!ext_tx_used_intra[ext_tx_set][tx_type])
          continue;
      }

      mbmi->tx_type = tx_type;
      if (ext_tx_set == 1 &&
          mbmi->tx_type >= DST_ADST && mbmi->tx_type < IDTX &&
hui su's avatar
hui su committed
920
921
          best_tx_type == DCT_DCT) {
        tx_type = IDTX - 1;
922
923
        continue;
      }
924

Jingning Han's avatar
Jingning Han committed
925
926
927
928
929
      txfm_rd_in_plane(x,
#if CONFIG_VAR_TX
                       cpi,
#endif
                       &r, &d, &s,
930
931
932
933
934
                       &psse, ref_best_rd, 0, bs, mbmi->tx_size,
                       cpi->sf.use_fast_coef_costing);

      if (r == INT_MAX)
        continue;
935
936
937
938
939
940
      if (get_ext_tx_types(mbmi->tx_size, bs, is_inter) > 1) {
        if (is_inter) {
          if (ext_tx_set > 0)
            r += cpi->inter_tx_type_costs[ext_tx_set]
                                         [mbmi->tx_size][mbmi->tx_type];
        } else {
941
          if (ext_tx_set > 0 && ALLOW_INTRA_EXT_TX)
942
943
944
            r += cpi->intra_tx_type_costs[ext_tx_set][mbmi->tx_size]
                                         [mbmi->mode][mbmi->tx_type];
        }
945
      }
946
947
948
949
950

      if (s)
        this_rd = RDCOST(x->rdmult, x->rddiv, s1, psse);
      else
        this_rd = RDCOST(x->rdmult, x->rddiv, r + s0, d);
Yaowu Xu's avatar
Yaowu Xu committed
951
      if (is_inter_block(mbmi) && !xd->lossless[mbmi->segment_id] && !s)
952
953
        this_rd = VPXMIN(this_rd, RDCOST(x->rdmult, x->rddiv, s1, psse));

hui su's avatar
hui su committed
954
      if (this_rd < ((best_tx_type == DCT_DCT) ? ext_tx_th : 1) * best_rd) {
955
        best_rd = this_rd;
hui su's avatar
hui su committed
956
        best_tx_type = mbmi->tx_type;
957
958
      }
    }
959
  }
960

961
#else  // CONFIG_EXT_TX
962
963
964
965
  if (mbmi->tx_size < TX_32X32 &&
      !xd->lossless[mbmi->segment_id]) {
    for (tx_type = 0; tx_type < TX_TYPES; ++tx_type) {
      mbmi->tx_type = tx_type;
966
967
968
969
970
      txfm_rd_in_plane(x,
#if CONFIG_VAR_TX
                       cpi,
#endif
                       &r, &d, &s,
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
                       &psse, ref_best_rd, 0, bs, mbmi->tx_size,
                       cpi->sf.use_fast_coef_costing);
      if (r == INT_MAX)
        continue;
      if (is_inter)
        r += cpi->inter_tx_type_costs[mbmi->tx_size][mbmi->tx_type];
      else
        r += cpi->intra_tx_type_costs[mbmi->tx_size]
                                     [intra_mode_to_tx_type_context[mbmi->mode]]
                                     [mbmi->tx_type];
      if (s)
        this_rd = RDCOST(x->rdmult, x->rddiv, s1, psse);
      else
        this_rd = RDCOST(x->rdmult, x->rddiv, r + s0, d);
      if (is_inter && !xd->lossless[mbmi->segment_id] && !s)
        this_rd = VPXMIN(this_rd, RDCOST(x->rdmult, x->rddiv, s1, psse));
Jingning Han's avatar
Jingning Han committed
987

988
989
990
991
992
993
      if (this_rd < ((best_tx_type == DCT_DCT) ? ext_tx_th : 1) * best_rd) {
        best_rd = this_rd;
        best_tx_type = mbmi->tx_type;
      }
    }
  }
994
#endif  // CONFIG_EXT_TX
995
  mbmi->tx_type = best_tx_type;
Jingning Han's avatar
Jingning Han committed
996

Jingning Han's avatar
Jingning Han committed
997
998
999
1000
  txfm_rd_in_plane(x,
#if CONFIG_VAR_TX
                   cpi,
#endif