rdopt.c 465 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4
5
6
7
8
9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10
11
12
13
14
 */

#include <assert.h>
#include <math.h>

Yaowu Xu's avatar
Yaowu Xu committed
15
#include "./aom_dsp_rtcd.h"
Jingning Han's avatar
Jingning Han committed
16
#include "./av1_rtcd.h"
Jingning Han's avatar
Jingning Han committed
17

Yaowu Xu's avatar
Yaowu Xu committed
18
#include "aom_dsp/aom_dsp_common.h"
19
#include "aom_dsp/blend.h"
Yaowu Xu's avatar
Yaowu Xu committed
20
#include "aom_mem/aom_mem.h"
21
22
#include "aom_ports/mem.h"
#include "aom_ports/system_state.h"
Jingning Han's avatar
Jingning Han committed
23

24
25
26
27
28
29
#include "av1/common/common.h"
#include "av1/common/common_data.h"
#include "av1/common/entropy.h"
#include "av1/common/entropymode.h"
#include "av1/common/idct.h"
#include "av1/common/mvref_common.h"
30
#include "av1/common/obmc.h"
31
32
33
34
35
36
#include "av1/common/pred_common.h"
#include "av1/common/quant_common.h"
#include "av1/common/reconinter.h"
#include "av1/common/reconintra.h"
#include "av1/common/scan.h"
#include "av1/common/seg_common.h"
37
38
39
#if CONFIG_LV_MAP
#include "av1/common/txb_common.h"
#endif
Yue Chen's avatar
Yue Chen committed
40
41
42
#if CONFIG_WARPED_MOTION
#include "av1/common/warped_motion.h"
#endif  // CONFIG_WARPED_MOTION
Jingning Han's avatar
Jingning Han committed
43

Jingning Han's avatar
Jingning Han committed
44
#include "av1/encoder/aq_variance.h"
45
#include "av1/encoder/av1_quantize.h"
46
47
48
49
#include "av1/encoder/cost.h"
#include "av1/encoder/encodemb.h"
#include "av1/encoder/encodemv.h"
#include "av1/encoder/encoder.h"
50
51
52
#if CONFIG_LV_MAP
#include "av1/encoder/encodetxb.h"
#endif
53
54
55
56
57
58
#include "av1/encoder/hybrid_fwd_txfm.h"
#include "av1/encoder/mcomp.h"
#include "av1/encoder/palette.h"
#include "av1/encoder/ratectrl.h"
#include "av1/encoder/rd.h"
#include "av1/encoder/rdopt.h"
59
#include "av1/encoder/tokenize.h"
60
61
#if CONFIG_PVQ
#include "av1/encoder/pvq_encoder.h"
Yushin Cho's avatar
Yushin Cho committed
62
#include "av1/common/pvq.h"
63
#endif  // CONFIG_PVQ
64
#if CONFIG_DUAL_FILTER
Angie Chiang's avatar
Angie Chiang committed
65
#define DUAL_FILTER_SET_SIZE (SWITCHABLE_FILTERS * SWITCHABLE_FILTERS)
66
#if USE_EXTRA_FILTER
Angie Chiang's avatar
Angie Chiang committed
67
static const int filter_sets[DUAL_FILTER_SET_SIZE][2] = {
68
69
70
  { 0, 0 }, { 0, 1 }, { 0, 2 }, { 0, 3 }, { 1, 0 }, { 1, 1 },
  { 1, 2 }, { 1, 3 }, { 2, 0 }, { 2, 1 }, { 2, 2 }, { 2, 3 },
  { 3, 0 }, { 3, 1 }, { 3, 2 }, { 3, 3 },
71
};
72
73
74
75
76
77
#else   // USE_EXTRA_FILTER
static const int filter_sets[DUAL_FILTER_SET_SIZE][2] = {
  { 0, 0 }, { 0, 1 }, { 0, 2 }, { 1, 0 }, { 1, 1 },
  { 1, 2 }, { 2, 0 }, { 2, 1 }, { 2, 2 },
};
#endif  // USE_EXTRA_FILTER
Angie Chiang's avatar
Angie Chiang committed
78
#endif  // CONFIG_DUAL_FILTER
79

80
81
#if CONFIG_EXT_REFS

Zoe Liu's avatar
Zoe Liu committed
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
#if CONFIG_ALTREF2

#define LAST_FRAME_MODE_MASK                                          \
  ((1 << INTRA_FRAME) | (1 << LAST2_FRAME) | (1 << LAST3_FRAME) |     \
   (1 << GOLDEN_FRAME) | (1 << BWDREF_FRAME) | (1 << ALTREF2_FRAME) | \
   (1 << ALTREF_FRAME))
#define LAST2_FRAME_MODE_MASK                                         \
  ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | (1 << LAST3_FRAME) |      \
   (1 << GOLDEN_FRAME) | (1 << BWDREF_FRAME) | (1 << ALTREF2_FRAME) | \
   (1 << ALTREF_FRAME))
#define LAST3_FRAME_MODE_MASK                                         \
  ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | (1 << LAST2_FRAME) |      \
   (1 << GOLDEN_FRAME) | (1 << BWDREF_FRAME) | (1 << ALTREF2_FRAME) | \
   (1 << ALTREF_FRAME))
#define GOLDEN_FRAME_MODE_MASK                                       \
  ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | (1 << LAST2_FRAME) |     \
   (1 << LAST3_FRAME) | (1 << BWDREF_FRAME) | (1 << ALTREF2_FRAME) | \
   (1 << ALTREF_FRAME))
#define BWDREF_FRAME_MODE_MASK                                       \
  ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | (1 << LAST2_FRAME) |     \
   (1 << LAST3_FRAME) | (1 << GOLDEN_FRAME) | (1 << ALTREF2_FRAME) | \
   (1 << ALTREF_FRAME))
#define ALTREF2_FRAME_MODE_MASK                                     \
  ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | (1 << LAST2_FRAME) |    \
   (1 << LAST3_FRAME) | (1 << GOLDEN_FRAME) | (1 << BWDREF_FRAME) | \
   (1 << ALTREF_FRAME))
#define ALTREF_FRAME_MODE_MASK                                      \
  ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | (1 << LAST2_FRAME) |    \
   (1 << LAST3_FRAME) | (1 << GOLDEN_FRAME) | (1 << BWDREF_FRAME) | \
   (1 << ALTREF2_FRAME))

#else  // CONFIG_ALTREF2

115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
#define LAST_FRAME_MODE_MASK                                      \
  ((1 << INTRA_FRAME) | (1 << LAST2_FRAME) | (1 << LAST3_FRAME) | \
   (1 << GOLDEN_FRAME) | (1 << BWDREF_FRAME) | (1 << ALTREF_FRAME))
#define LAST2_FRAME_MODE_MASK                                    \
  ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | (1 << LAST3_FRAME) | \
   (1 << GOLDEN_FRAME) | (1 << BWDREF_FRAME) | (1 << ALTREF_FRAME))
#define LAST3_FRAME_MODE_MASK                                    \
  ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | (1 << LAST2_FRAME) | \
   (1 << GOLDEN_FRAME) | (1 << BWDREF_FRAME) | (1 << ALTREF_FRAME))
#define GOLDEN_FRAME_MODE_MASK                                   \
  ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | (1 << LAST2_FRAME) | \
   (1 << LAST3_FRAME) | (1 << BWDREF_FRAME) | (1 << ALTREF_FRAME))
#define BWDREF_FRAME_MODE_MASK                                   \
  ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | (1 << LAST2_FRAME) | \
   (1 << LAST3_FRAME) | (1 << GOLDEN_FRAME) | (1 << ALTREF_FRAME))
#define ALTREF_FRAME_MODE_MASK                                   \
  ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | (1 << LAST2_FRAME) | \
   (1 << LAST3_FRAME) | (1 << GOLDEN_FRAME) | (1 << BWDREF_FRAME))

Zoe Liu's avatar
Zoe Liu committed
134
135
136
#endif  // CONFIG_ALTREF2

#else  // !CONFIG_EXT_REFS
137
138
139
140
141
142
143

#define LAST_FRAME_MODE_MASK \
  ((1 << GOLDEN_FRAME) | (1 << ALTREF_FRAME) | (1 << INTRA_FRAME))
#define GOLDEN_FRAME_MODE_MASK \
  ((1 << LAST_FRAME) | (1 << ALTREF_FRAME) | (1 << INTRA_FRAME))
#define ALTREF_FRAME_MODE_MASK \
  ((1 << LAST_FRAME) | (1 << GOLDEN_FRAME) | (1 << INTRA_FRAME))
Jingning Han's avatar
Jingning Han committed
144

145
146
#endif  // CONFIG_EXT_REFS

147
#if CONFIG_EXT_REFS
Zoe Liu's avatar
Zoe Liu committed
148
149
150
151
152
#if CONFIG_EXT_COMP_REFS
#define SECOND_REF_FRAME_MASK                                        \
  ((1 << ALTREF_FRAME) | (1 << BWDREF_FRAME) | (1 << GOLDEN_FRAME) | \
   (1 << LAST2_FRAME) | 0x01)  // NOLINT
#else                          // !CONFIG_EXT_COMP_REFS
Zoe Liu's avatar
Zoe Liu committed
153
154
155
156
#if CONFIG_ALTREF2
#define SECOND_REF_FRAME_MASK \
  ((1 << ALTREF_FRAME) | (1 << ALTREF2_FRAME) | (1 << BWDREF_FRAME) | 0x01)
#else  // !CONFIG_ALTREF2
157
#define SECOND_REF_FRAME_MASK ((1 << ALTREF_FRAME) | (1 << BWDREF_FRAME) | 0x01)
Zoe Liu's avatar
Zoe Liu committed
158
#endif  // CONFIG_ALTREF2
Zoe Liu's avatar
Zoe Liu committed
159
160
#endif  // CONFIG_EXT_COMP_REFS
#else   // !CONFIG_EXT_REFS
161
#define SECOND_REF_FRAME_MASK ((1 << ALTREF_FRAME) | 0x01)
162
#endif  // CONFIG_EXT_REFS
Jingning Han's avatar
Jingning Han committed
163

164
165
#define MIN_EARLY_TERM_INDEX 3
#define NEW_MV_DISCOUNT_FACTOR 8
Jingning Han's avatar
Jingning Han committed
166

167
168
169
170
171
#if CONFIG_EXT_INTRA
#define ANGLE_SKIP_THRESH 10
#define FILTER_FAST_SEARCH 1
#endif  // CONFIG_EXT_INTRA

172
173
174
// Setting this to 1 will disable trellis optimization within the
// transform search. Trellis optimization will still be applied
// in the final encode.
175
#ifndef DISABLE_TRELLISQ_SEARCH
176
#define DISABLE_TRELLISQ_SEARCH 0
177
#endif
178

179
180
181
182
183
184
static const double ADST_FLIP_SVM[8] = {
  /* vertical */
  -6.6623, -2.8062, -3.2531, 3.1671,
  /* horizontal */
  -7.7051, -3.2234, -3.6193, 3.4533
};
185

Jingning Han's avatar
Jingning Han committed
186
187
188
189
190
typedef struct {
  PREDICTION_MODE mode;
  MV_REFERENCE_FRAME ref_frame[2];
} MODE_DEFINITION;

191
typedef struct { MV_REFERENCE_FRAME ref_frame[2]; } REF_DEFINITION;
Jingning Han's avatar
Jingning Han committed
192
193

struct rdcost_block_args {
Yaowu Xu's avatar
Yaowu Xu committed
194
  const AV1_COMP *cpi;
Jingning Han's avatar
Jingning Han committed
195
  MACROBLOCK *x;
196
197
  ENTROPY_CONTEXT t_above[2 * MAX_MIB_SIZE];
  ENTROPY_CONTEXT t_left[2 * MAX_MIB_SIZE];
198
  RD_STATS rd_stats;
Jingning Han's avatar
Jingning Han committed
199
200
201
202
203
204
205
  int64_t this_rd;
  int64_t best_rd;
  int exit_early;
  int use_fast_coef_costing;
};

#define LAST_NEW_MV_INDEX 6
Yaowu Xu's avatar
Yaowu Xu committed
206
static const MODE_DEFINITION av1_mode_order[MAX_MODES] = {
Emil Keyder's avatar
Emil Keyder committed
207
  { NEARESTMV, { LAST_FRAME, NONE_FRAME } },
208
#if CONFIG_EXT_REFS
Emil Keyder's avatar
Emil Keyder committed
209
210
211
  { NEARESTMV, { LAST2_FRAME, NONE_FRAME } },
  { NEARESTMV, { LAST3_FRAME, NONE_FRAME } },
  { NEARESTMV, { BWDREF_FRAME, NONE_FRAME } },
Zoe Liu's avatar
Zoe Liu committed
212
213
214
#if CONFIG_ALTREF2
  { NEARESTMV, { ALTREF2_FRAME, NONE_FRAME } },
#endif  // CONFIG_ALTREF2
215
#endif  // CONFIG_EXT_REFS
Emil Keyder's avatar
Emil Keyder committed
216
217
  { NEARESTMV, { ALTREF_FRAME, NONE_FRAME } },
  { NEARESTMV, { GOLDEN_FRAME, NONE_FRAME } },
Jingning Han's avatar
Jingning Han committed
218

Emil Keyder's avatar
Emil Keyder committed
219
  { DC_PRED, { INTRA_FRAME, NONE_FRAME } },
Jingning Han's avatar
Jingning Han committed
220

Emil Keyder's avatar
Emil Keyder committed
221
  { NEWMV, { LAST_FRAME, NONE_FRAME } },
222
#if CONFIG_EXT_REFS
Emil Keyder's avatar
Emil Keyder committed
223
224
225
  { NEWMV, { LAST2_FRAME, NONE_FRAME } },
  { NEWMV, { LAST3_FRAME, NONE_FRAME } },
  { NEWMV, { BWDREF_FRAME, NONE_FRAME } },
Zoe Liu's avatar
Zoe Liu committed
226
227
228
#if CONFIG_ALTREF2
  { NEWMV, { ALTREF2_FRAME, NONE_FRAME } },
#endif  // CONFIG_ALTREF2
229
#endif  // CONFIG_EXT_REFS
Emil Keyder's avatar
Emil Keyder committed
230
231
  { NEWMV, { ALTREF_FRAME, NONE_FRAME } },
  { NEWMV, { GOLDEN_FRAME, NONE_FRAME } },
Jingning Han's avatar
Jingning Han committed
232

Emil Keyder's avatar
Emil Keyder committed
233
  { NEARMV, { LAST_FRAME, NONE_FRAME } },
234
#if CONFIG_EXT_REFS
Emil Keyder's avatar
Emil Keyder committed
235
236
237
  { NEARMV, { LAST2_FRAME, NONE_FRAME } },
  { NEARMV, { LAST3_FRAME, NONE_FRAME } },
  { NEARMV, { BWDREF_FRAME, NONE_FRAME } },
Zoe Liu's avatar
Zoe Liu committed
238
239
240
#if CONFIG_ALTREF2
  { NEARMV, { ALTREF2_FRAME, NONE_FRAME } },
#endif  // CONFIG_ALTREF2
241
#endif  // CONFIG_EXT_REFS
Emil Keyder's avatar
Emil Keyder committed
242
243
  { NEARMV, { ALTREF_FRAME, NONE_FRAME } },
  { NEARMV, { GOLDEN_FRAME, NONE_FRAME } },
Jingning Han's avatar
Jingning Han committed
244

Emil Keyder's avatar
Emil Keyder committed
245
  { ZEROMV, { LAST_FRAME, NONE_FRAME } },
246
#if CONFIG_EXT_REFS
Emil Keyder's avatar
Emil Keyder committed
247
248
249
  { ZEROMV, { LAST2_FRAME, NONE_FRAME } },
  { ZEROMV, { LAST3_FRAME, NONE_FRAME } },
  { ZEROMV, { BWDREF_FRAME, NONE_FRAME } },
Zoe Liu's avatar
Zoe Liu committed
250
251
252
#if CONFIG_ALTREF2
  { ZEROMV, { ALTREF2_FRAME, NONE_FRAME } },
#endif  // CONFIG_ALTREF2
253
#endif  // CONFIG_EXT_REFS
Emil Keyder's avatar
Emil Keyder committed
254
255
  { ZEROMV, { GOLDEN_FRAME, NONE_FRAME } },
  { ZEROMV, { ALTREF_FRAME, NONE_FRAME } },
Jingning Han's avatar
Jingning Han committed
256

257
// TODO(zoeliu): May need to reconsider the order on the modes to check
258

259
#if CONFIG_EXT_INTER
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309

#if CONFIG_COMPOUND_SINGLEREF
  // Single ref comp mode
  { SR_NEAREST_NEARMV, { LAST_FRAME, NONE_FRAME } },
#if CONFIG_EXT_REFS
  { SR_NEAREST_NEARMV, { LAST2_FRAME, NONE_FRAME } },
  { SR_NEAREST_NEARMV, { LAST3_FRAME, NONE_FRAME } },
  { SR_NEAREST_NEARMV, { BWDREF_FRAME, NONE_FRAME } },
#endif  // CONFIG_EXT_REFS
  { SR_NEAREST_NEARMV, { GOLDEN_FRAME, NONE_FRAME } },
  { SR_NEAREST_NEARMV, { ALTREF_FRAME, NONE_FRAME } },

  /*
  { SR_NEAREST_NEWMV, { LAST_FRAME, NONE_FRAME } },
#if CONFIG_EXT_REFS
  { SR_NEAREST_NEWMV, { LAST2_FRAME, NONE_FRAME } },
  { SR_NEAREST_NEWMV, { LAST3_FRAME, NONE_FRAME } },
  { SR_NEAREST_NEWMV, { BWDREF_FRAME, NONE_FRAME } },
#endif  // CONFIG_EXT_REFS
  { SR_NEAREST_NEWMV, { GOLDEN_FRAME, NONE_FRAME } },
  { SR_NEAREST_NEWMV, { ALTREF_FRAME, NONE_FRAME } },*/

  { SR_NEAR_NEWMV, { LAST_FRAME, NONE_FRAME } },
#if CONFIG_EXT_REFS
  { SR_NEAR_NEWMV, { LAST2_FRAME, NONE_FRAME } },
  { SR_NEAR_NEWMV, { LAST3_FRAME, NONE_FRAME } },
  { SR_NEAR_NEWMV, { BWDREF_FRAME, NONE_FRAME } },
#endif  // CONFIG_EXT_REFS
  { SR_NEAR_NEWMV, { GOLDEN_FRAME, NONE_FRAME } },
  { SR_NEAR_NEWMV, { ALTREF_FRAME, NONE_FRAME } },

  { SR_ZERO_NEWMV, { LAST_FRAME, NONE_FRAME } },
#if CONFIG_EXT_REFS
  { SR_ZERO_NEWMV, { LAST2_FRAME, NONE_FRAME } },
  { SR_ZERO_NEWMV, { LAST3_FRAME, NONE_FRAME } },
  { SR_ZERO_NEWMV, { BWDREF_FRAME, NONE_FRAME } },
#endif  // CONFIG_EXT_REFS
  { SR_ZERO_NEWMV, { GOLDEN_FRAME, NONE_FRAME } },
  { SR_ZERO_NEWMV, { ALTREF_FRAME, NONE_FRAME } },

  { SR_NEW_NEWMV, { LAST_FRAME, NONE_FRAME } },
#if CONFIG_EXT_REFS
  { SR_NEW_NEWMV, { LAST2_FRAME, NONE_FRAME } },
  { SR_NEW_NEWMV, { LAST3_FRAME, NONE_FRAME } },
  { SR_NEW_NEWMV, { BWDREF_FRAME, NONE_FRAME } },
#endif  // CONFIG_EXT_REFS
  { SR_NEW_NEWMV, { GOLDEN_FRAME, NONE_FRAME } },
  { SR_NEW_NEWMV, { ALTREF_FRAME, NONE_FRAME } },
#endif  // CONFIG_COMPOUND_SINGLEREF

310
  { NEAREST_NEARESTMV, { LAST_FRAME, ALTREF_FRAME } },
311
#if CONFIG_EXT_REFS
312
313
  { NEAREST_NEARESTMV, { LAST2_FRAME, ALTREF_FRAME } },
  { NEAREST_NEARESTMV, { LAST3_FRAME, ALTREF_FRAME } },
314
#endif  // CONFIG_EXT_REFS
315
  { NEAREST_NEARESTMV, { GOLDEN_FRAME, ALTREF_FRAME } },
316
#if CONFIG_EXT_REFS
317
318
319
320
  { NEAREST_NEARESTMV, { LAST_FRAME, BWDREF_FRAME } },
  { NEAREST_NEARESTMV, { LAST2_FRAME, BWDREF_FRAME } },
  { NEAREST_NEARESTMV, { LAST3_FRAME, BWDREF_FRAME } },
  { NEAREST_NEARESTMV, { GOLDEN_FRAME, BWDREF_FRAME } },
Zoe Liu's avatar
Zoe Liu committed
321
322
323
324
325
326
#if CONFIG_ALTREF2
  { NEAREST_NEARESTMV, { LAST_FRAME, ALTREF2_FRAME } },
  { NEAREST_NEARESTMV, { LAST2_FRAME, ALTREF2_FRAME } },
  { NEAREST_NEARESTMV, { LAST3_FRAME, ALTREF2_FRAME } },
  { NEAREST_NEARESTMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
#endif  // CONFIG_ALTREF2
Zoe Liu's avatar
Zoe Liu committed
327
328
329

#if CONFIG_EXT_COMP_REFS
  { NEAREST_NEARESTMV, { LAST_FRAME, LAST2_FRAME } },
330
  { NEAREST_NEARESTMV, { LAST_FRAME, LAST3_FRAME } },
Zoe Liu's avatar
Zoe Liu committed
331
332
333
  { NEAREST_NEARESTMV, { LAST_FRAME, GOLDEN_FRAME } },
  { NEAREST_NEARESTMV, { BWDREF_FRAME, ALTREF_FRAME } },
#endif  // CONFIG_EXT_COMP_REFS
334
#endif  // CONFIG_EXT_REFS
335

336
#else  // CONFIG_EXT_INTER
337

338
  { NEARESTMV, { LAST_FRAME, ALTREF_FRAME } },
339
#if CONFIG_EXT_REFS
340
341
  { NEARESTMV, { LAST2_FRAME, ALTREF_FRAME } },
  { NEARESTMV, { LAST3_FRAME, ALTREF_FRAME } },
342
#endif  // CONFIG_EXT_REFS
343
  { NEARESTMV, { GOLDEN_FRAME, ALTREF_FRAME } },
344
#if CONFIG_EXT_REFS
345
346
347
348
  { NEARESTMV, { LAST_FRAME, BWDREF_FRAME } },
  { NEARESTMV, { LAST2_FRAME, BWDREF_FRAME } },
  { NEARESTMV, { LAST3_FRAME, BWDREF_FRAME } },
  { NEARESTMV, { GOLDEN_FRAME, BWDREF_FRAME } },
Zoe Liu's avatar
Zoe Liu committed
349
350
351
352
353
354
#if CONFIG_ALTREF2
  { NEARESTMV, { LAST_FRAME, ALTREF2_FRAME } },
  { NEARESTMV, { LAST2_FRAME, ALTREF2_FRAME } },
  { NEARESTMV, { LAST3_FRAME, ALTREF2_FRAME } },
  { NEARESTMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
#endif  // CONFIG_ALTREF2
Zoe Liu's avatar
Zoe Liu committed
355
356
357

#if CONFIG_EXT_COMP_REFS
  { NEARESTMV, { LAST_FRAME, LAST2_FRAME } },
358
  { NEARESTMV, { LAST_FRAME, LAST3_FRAME } },
Zoe Liu's avatar
Zoe Liu committed
359
360
361
  { NEARESTMV, { LAST_FRAME, GOLDEN_FRAME } },
  { NEARESTMV, { BWDREF_FRAME, ALTREF_FRAME } },
#endif  // CONFIG_EXT_COMP_REFS
362
#endif  // CONFIG_EXT_REFS
363
#endif  // CONFIG_EXT_INTER
Jingning Han's avatar
Jingning Han committed
364

Emil Keyder's avatar
Emil Keyder committed
365
  { TM_PRED, { INTRA_FRAME, NONE_FRAME } },
Jingning Han's avatar
Jingning Han committed
366

Emil Keyder's avatar
Emil Keyder committed
367
  { SMOOTH_PRED, { INTRA_FRAME, NONE_FRAME } },
Urvang Joshi's avatar
Urvang Joshi committed
368
369
370
371
#if CONFIG_SMOOTH_HV
  { SMOOTH_V_PRED, { INTRA_FRAME, NONE_FRAME } },
  { SMOOTH_H_PRED, { INTRA_FRAME, NONE_FRAME } },
#endif  // CONFIG_SMOOTH_HV
372

373
#if CONFIG_EXT_INTER
374
375
376
377
378
379
380
  { NEAR_NEARMV, { LAST_FRAME, ALTREF_FRAME } },
  { NEW_NEARESTMV, { LAST_FRAME, ALTREF_FRAME } },
  { NEAREST_NEWMV, { LAST_FRAME, ALTREF_FRAME } },
  { NEW_NEARMV, { LAST_FRAME, ALTREF_FRAME } },
  { NEAR_NEWMV, { LAST_FRAME, ALTREF_FRAME } },
  { NEW_NEWMV, { LAST_FRAME, ALTREF_FRAME } },
  { ZERO_ZEROMV, { LAST_FRAME, ALTREF_FRAME } },
381

382
#if CONFIG_EXT_REFS
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
  { NEAR_NEARMV, { LAST2_FRAME, ALTREF_FRAME } },
  { NEW_NEARESTMV, { LAST2_FRAME, ALTREF_FRAME } },
  { NEAREST_NEWMV, { LAST2_FRAME, ALTREF_FRAME } },
  { NEW_NEARMV, { LAST2_FRAME, ALTREF_FRAME } },
  { NEAR_NEWMV, { LAST2_FRAME, ALTREF_FRAME } },
  { NEW_NEWMV, { LAST2_FRAME, ALTREF_FRAME } },
  { ZERO_ZEROMV, { LAST2_FRAME, ALTREF_FRAME } },

  { NEAR_NEARMV, { LAST3_FRAME, ALTREF_FRAME } },
  { NEW_NEARESTMV, { LAST3_FRAME, ALTREF_FRAME } },
  { NEAREST_NEWMV, { LAST3_FRAME, ALTREF_FRAME } },
  { NEW_NEARMV, { LAST3_FRAME, ALTREF_FRAME } },
  { NEAR_NEWMV, { LAST3_FRAME, ALTREF_FRAME } },
  { NEW_NEWMV, { LAST3_FRAME, ALTREF_FRAME } },
  { ZERO_ZEROMV, { LAST3_FRAME, ALTREF_FRAME } },
398
#endif  // CONFIG_EXT_REFS
399

400
401
402
403
404
405
406
  { NEAR_NEARMV, { GOLDEN_FRAME, ALTREF_FRAME } },
  { NEW_NEARESTMV, { GOLDEN_FRAME, ALTREF_FRAME } },
  { NEAREST_NEWMV, { GOLDEN_FRAME, ALTREF_FRAME } },
  { NEW_NEARMV, { GOLDEN_FRAME, ALTREF_FRAME } },
  { NEAR_NEWMV, { GOLDEN_FRAME, ALTREF_FRAME } },
  { NEW_NEWMV, { GOLDEN_FRAME, ALTREF_FRAME } },
  { ZERO_ZEROMV, { GOLDEN_FRAME, ALTREF_FRAME } },
407
408

#if CONFIG_EXT_REFS
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
  { NEAR_NEARMV, { LAST_FRAME, BWDREF_FRAME } },
  { NEW_NEARESTMV, { LAST_FRAME, BWDREF_FRAME } },
  { NEAREST_NEWMV, { LAST_FRAME, BWDREF_FRAME } },
  { NEW_NEARMV, { LAST_FRAME, BWDREF_FRAME } },
  { NEAR_NEWMV, { LAST_FRAME, BWDREF_FRAME } },
  { NEW_NEWMV, { LAST_FRAME, BWDREF_FRAME } },
  { ZERO_ZEROMV, { LAST_FRAME, BWDREF_FRAME } },

  { NEAR_NEARMV, { LAST2_FRAME, BWDREF_FRAME } },
  { NEW_NEARESTMV, { LAST2_FRAME, BWDREF_FRAME } },
  { NEAREST_NEWMV, { LAST2_FRAME, BWDREF_FRAME } },
  { NEW_NEARMV, { LAST2_FRAME, BWDREF_FRAME } },
  { NEAR_NEWMV, { LAST2_FRAME, BWDREF_FRAME } },
  { NEW_NEWMV, { LAST2_FRAME, BWDREF_FRAME } },
  { ZERO_ZEROMV, { LAST2_FRAME, BWDREF_FRAME } },

  { NEAR_NEARMV, { LAST3_FRAME, BWDREF_FRAME } },
  { NEW_NEARESTMV, { LAST3_FRAME, BWDREF_FRAME } },
  { NEAREST_NEWMV, { LAST3_FRAME, BWDREF_FRAME } },
  { NEW_NEARMV, { LAST3_FRAME, BWDREF_FRAME } },
  { NEAR_NEWMV, { LAST3_FRAME, BWDREF_FRAME } },
  { NEW_NEWMV, { LAST3_FRAME, BWDREF_FRAME } },
  { ZERO_ZEROMV, { LAST3_FRAME, BWDREF_FRAME } },

  { NEAR_NEARMV, { GOLDEN_FRAME, BWDREF_FRAME } },
  { NEW_NEARESTMV, { GOLDEN_FRAME, BWDREF_FRAME } },
  { NEAREST_NEWMV, { GOLDEN_FRAME, BWDREF_FRAME } },
  { NEW_NEARMV, { GOLDEN_FRAME, BWDREF_FRAME } },
  { NEAR_NEWMV, { GOLDEN_FRAME, BWDREF_FRAME } },
  { NEW_NEWMV, { GOLDEN_FRAME, BWDREF_FRAME } },
  { ZERO_ZEROMV, { GOLDEN_FRAME, BWDREF_FRAME } },
Zoe Liu's avatar
Zoe Liu committed
440

Zoe Liu's avatar
Zoe Liu committed
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
#if CONFIG_ALTREF2
  { NEAR_NEARMV, { LAST_FRAME, ALTREF2_FRAME } },
  { NEW_NEARESTMV, { LAST_FRAME, ALTREF2_FRAME } },
  { NEAREST_NEWMV, { LAST_FRAME, ALTREF2_FRAME } },
  { NEW_NEARMV, { LAST_FRAME, ALTREF2_FRAME } },
  { NEAR_NEWMV, { LAST_FRAME, ALTREF2_FRAME } },
  { NEW_NEWMV, { LAST_FRAME, ALTREF2_FRAME } },
  { ZERO_ZEROMV, { LAST_FRAME, ALTREF2_FRAME } },

  { NEAR_NEARMV, { LAST2_FRAME, ALTREF2_FRAME } },
  { NEW_NEARESTMV, { LAST2_FRAME, ALTREF2_FRAME } },
  { NEAREST_NEWMV, { LAST2_FRAME, ALTREF2_FRAME } },
  { NEW_NEARMV, { LAST2_FRAME, ALTREF2_FRAME } },
  { NEAR_NEWMV, { LAST2_FRAME, ALTREF2_FRAME } },
  { NEW_NEWMV, { LAST2_FRAME, ALTREF2_FRAME } },
  { ZERO_ZEROMV, { LAST2_FRAME, ALTREF2_FRAME } },

  { NEAR_NEARMV, { LAST3_FRAME, ALTREF2_FRAME } },
  { NEW_NEARESTMV, { LAST3_FRAME, ALTREF2_FRAME } },
  { NEAREST_NEWMV, { LAST3_FRAME, ALTREF2_FRAME } },
  { NEW_NEARMV, { LAST3_FRAME, ALTREF2_FRAME } },
  { NEAR_NEWMV, { LAST3_FRAME, ALTREF2_FRAME } },
  { NEW_NEWMV, { LAST3_FRAME, ALTREF2_FRAME } },
  { ZERO_ZEROMV, { LAST3_FRAME, ALTREF2_FRAME } },

  { NEAR_NEARMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
  { NEW_NEARESTMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
  { NEAREST_NEWMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
  { NEW_NEARMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
  { NEAR_NEWMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
  { NEW_NEWMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
  { ZERO_ZEROMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
#endif  // CONFIG_ALTREF2

Zoe Liu's avatar
Zoe Liu committed
475
476
477
478
479
480
481
482
483
#if CONFIG_EXT_COMP_REFS
  { NEAR_NEARMV, { LAST_FRAME, LAST2_FRAME } },
  { NEW_NEARESTMV, { LAST_FRAME, LAST2_FRAME } },
  { NEAREST_NEWMV, { LAST_FRAME, LAST2_FRAME } },
  { NEW_NEARMV, { LAST_FRAME, LAST2_FRAME } },
  { NEAR_NEWMV, { LAST_FRAME, LAST2_FRAME } },
  { NEW_NEWMV, { LAST_FRAME, LAST2_FRAME } },
  { ZERO_ZEROMV, { LAST_FRAME, LAST2_FRAME } },

484
485
486
487
488
489
490
491
  { NEAR_NEARMV, { LAST_FRAME, LAST3_FRAME } },
  { NEW_NEARESTMV, { LAST_FRAME, LAST3_FRAME } },
  { NEAREST_NEWMV, { LAST_FRAME, LAST3_FRAME } },
  { NEW_NEARMV, { LAST_FRAME, LAST3_FRAME } },
  { NEAR_NEWMV, { LAST_FRAME, LAST3_FRAME } },
  { NEW_NEWMV, { LAST_FRAME, LAST3_FRAME } },
  { ZERO_ZEROMV, { LAST_FRAME, LAST3_FRAME } },

Zoe Liu's avatar
Zoe Liu committed
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
  { NEAR_NEARMV, { LAST_FRAME, GOLDEN_FRAME } },
  { NEW_NEARESTMV, { LAST_FRAME, GOLDEN_FRAME } },
  { NEAREST_NEWMV, { LAST_FRAME, GOLDEN_FRAME } },
  { NEW_NEARMV, { LAST_FRAME, GOLDEN_FRAME } },
  { NEAR_NEWMV, { LAST_FRAME, GOLDEN_FRAME } },
  { NEW_NEWMV, { LAST_FRAME, GOLDEN_FRAME } },
  { ZERO_ZEROMV, { LAST_FRAME, GOLDEN_FRAME } },

  { NEAR_NEARMV, { BWDREF_FRAME, ALTREF_FRAME } },
  { NEW_NEARESTMV, { BWDREF_FRAME, ALTREF_FRAME } },
  { NEAREST_NEWMV, { BWDREF_FRAME, ALTREF_FRAME } },
  { NEW_NEARMV, { BWDREF_FRAME, ALTREF_FRAME } },
  { NEAR_NEWMV, { BWDREF_FRAME, ALTREF_FRAME } },
  { NEW_NEWMV, { BWDREF_FRAME, ALTREF_FRAME } },
  { ZERO_ZEROMV, { BWDREF_FRAME, ALTREF_FRAME } },
#endif  // CONFIG_EXT_COMP_REFS
508
#endif  // CONFIG_EXT_REFS
509

Zoe Liu's avatar
Zoe Liu committed
510
#else  // !CONFIG_EXT_INTER
511

512
513
  { NEARMV, { LAST_FRAME, ALTREF_FRAME } },
  { NEWMV, { LAST_FRAME, ALTREF_FRAME } },
514
#if CONFIG_EXT_REFS
515
516
517
518
  { NEARMV, { LAST2_FRAME, ALTREF_FRAME } },
  { NEWMV, { LAST2_FRAME, ALTREF_FRAME } },
  { NEARMV, { LAST3_FRAME, ALTREF_FRAME } },
  { NEWMV, { LAST3_FRAME, ALTREF_FRAME } },
519
#endif  // CONFIG_EXT_REFS
520
521
  { NEARMV, { GOLDEN_FRAME, ALTREF_FRAME } },
  { NEWMV, { GOLDEN_FRAME, ALTREF_FRAME } },
522
523

#if CONFIG_EXT_REFS
524
525
526
527
528
529
530
531
  { NEARMV, { LAST_FRAME, BWDREF_FRAME } },
  { NEWMV, { LAST_FRAME, BWDREF_FRAME } },
  { NEARMV, { LAST2_FRAME, BWDREF_FRAME } },
  { NEWMV, { LAST2_FRAME, BWDREF_FRAME } },
  { NEARMV, { LAST3_FRAME, BWDREF_FRAME } },
  { NEWMV, { LAST3_FRAME, BWDREF_FRAME } },
  { NEARMV, { GOLDEN_FRAME, BWDREF_FRAME } },
  { NEWMV, { GOLDEN_FRAME, BWDREF_FRAME } },
Zoe Liu's avatar
Zoe Liu committed
532

Zoe Liu's avatar
Zoe Liu committed
533
534
535
536
537
538
539
540
541
542
543
#if CONFIG_ALTREF2
  { NEARMV, { LAST_FRAME, ALTREF2_FRAME } },
  { NEWMV, { LAST_FRAME, ALTREF2_FRAME } },
  { NEARMV, { LAST2_FRAME, ALTREF2_FRAME } },
  { NEWMV, { LAST2_FRAME, ALTREF2_FRAME } },
  { NEARMV, { LAST3_FRAME, ALTREF2_FRAME } },
  { NEWMV, { LAST3_FRAME, ALTREF2_FRAME } },
  { NEARMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
  { NEWMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
#endif  // CONFIG_ALTREF2

Zoe Liu's avatar
Zoe Liu committed
544
545
546
#if CONFIG_EXT_COMP_REFS
  { NEARMV, { LAST_FRAME, LAST2_FRAME } },
  { NEWMV, { LAST_FRAME, LAST2_FRAME } },
547
548
  { NEARMV, { LAST_FRAME, LAST3_FRAME } },
  { NEWMV, { LAST_FRAME, LAST3_FRAME } },
Zoe Liu's avatar
Zoe Liu committed
549
550
551
552
553
  { NEARMV, { LAST_FRAME, GOLDEN_FRAME } },
  { NEWMV, { LAST_FRAME, GOLDEN_FRAME } },
  { NEARMV, { BWDREF_FRAME, ALTREF_FRAME } },
  { NEWMV, { BWDREF_FRAME, ALTREF_FRAME } },
#endif  // CONFIG_EXT_COMP_REFS
554
#endif  // CONFIG_EXT_REFS
Jingning Han's avatar
Jingning Han committed
555

556
  { ZEROMV, { LAST_FRAME, ALTREF_FRAME } },
557
#if CONFIG_EXT_REFS
558
559
  { ZEROMV, { LAST2_FRAME, ALTREF_FRAME } },
  { ZEROMV, { LAST3_FRAME, ALTREF_FRAME } },
560
#endif  // CONFIG_EXT_REFS
561
  { ZEROMV, { GOLDEN_FRAME, ALTREF_FRAME } },
562
563

#if CONFIG_EXT_REFS
564
565
566
567
  { ZEROMV, { LAST_FRAME, BWDREF_FRAME } },
  { ZEROMV, { LAST2_FRAME, BWDREF_FRAME } },
  { ZEROMV, { LAST3_FRAME, BWDREF_FRAME } },
  { ZEROMV, { GOLDEN_FRAME, BWDREF_FRAME } },
Zoe Liu's avatar
Zoe Liu committed
568

Zoe Liu's avatar
Zoe Liu committed
569
570
571
572
573
574
575
#if CONFIG_ALTREF2
  { ZEROMV, { LAST_FRAME, ALTREF2_FRAME } },
  { ZEROMV, { LAST2_FRAME, ALTREF2_FRAME } },
  { ZEROMV, { LAST3_FRAME, ALTREF2_FRAME } },
  { ZEROMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
#endif  // CONFIG_ALTREF2

Zoe Liu's avatar
Zoe Liu committed
576
577
#if CONFIG_EXT_COMP_REFS
  { ZEROMV, { LAST_FRAME, LAST2_FRAME } },
578
  { ZEROMV, { LAST_FRAME, LAST3_FRAME } },
Zoe Liu's avatar
Zoe Liu committed
579
580
581
  { ZEROMV, { LAST_FRAME, GOLDEN_FRAME } },
  { ZEROMV, { BWDREF_FRAME, ALTREF_FRAME } },
#endif  // CONFIG_EXT_COMP_REFS
582
#endif  // CONFIG_EXT_REFS
583

584
#endif  // CONFIG_EXT_INTER
Jingning Han's avatar
Jingning Han committed
585

Emil Keyder's avatar
Emil Keyder committed
586
587
588
589
590
591
592
593
  { H_PRED, { INTRA_FRAME, NONE_FRAME } },
  { V_PRED, { INTRA_FRAME, NONE_FRAME } },
  { D135_PRED, { INTRA_FRAME, NONE_FRAME } },
  { D207_PRED, { INTRA_FRAME, NONE_FRAME } },
  { D153_PRED, { INTRA_FRAME, NONE_FRAME } },
  { D63_PRED, { INTRA_FRAME, NONE_FRAME } },
  { D117_PRED, { INTRA_FRAME, NONE_FRAME } },
  { D45_PRED, { INTRA_FRAME, NONE_FRAME } },
594
595

#if CONFIG_EXT_INTER
596
597
598
599
  { ZEROMV, { LAST_FRAME, INTRA_FRAME } },
  { NEARESTMV, { LAST_FRAME, INTRA_FRAME } },
  { NEARMV, { LAST_FRAME, INTRA_FRAME } },
  { NEWMV, { LAST_FRAME, INTRA_FRAME } },
600
601

#if CONFIG_EXT_REFS
602
603
604
605
606
607
608
609
610
  { ZEROMV, { LAST2_FRAME, INTRA_FRAME } },
  { NEARESTMV, { LAST2_FRAME, INTRA_FRAME } },
  { NEARMV, { LAST2_FRAME, INTRA_FRAME } },
  { NEWMV, { LAST2_FRAME, INTRA_FRAME } },

  { ZEROMV, { LAST3_FRAME, INTRA_FRAME } },
  { NEARESTMV, { LAST3_FRAME, INTRA_FRAME } },
  { NEARMV, { LAST3_FRAME, INTRA_FRAME } },
  { NEWMV, { LAST3_FRAME, INTRA_FRAME } },
611
612
#endif  // CONFIG_EXT_REFS

613
614
615
616
  { ZEROMV, { GOLDEN_FRAME, INTRA_FRAME } },
  { NEARESTMV, { GOLDEN_FRAME, INTRA_FRAME } },
  { NEARMV, { GOLDEN_FRAME, INTRA_FRAME } },
  { NEWMV, { GOLDEN_FRAME, INTRA_FRAME } },
617

618
#if CONFIG_EXT_REFS
619
620
621
622
  { ZEROMV, { BWDREF_FRAME, INTRA_FRAME } },
  { NEARESTMV, { BWDREF_FRAME, INTRA_FRAME } },
  { NEARMV, { BWDREF_FRAME, INTRA_FRAME } },
  { NEWMV, { BWDREF_FRAME, INTRA_FRAME } },
Zoe Liu's avatar
Zoe Liu committed
623
624
625
626
627
628
629

#if CONFIG_ALTREF2
  { ZEROMV, { ALTREF2_FRAME, INTRA_FRAME } },
  { NEARESTMV, { ALTREF2_FRAME, INTRA_FRAME } },
  { NEARMV, { ALTREF2_FRAME, INTRA_FRAME } },
  { NEWMV, { ALTREF2_FRAME, INTRA_FRAME } },
#endif  // CONFIG_ALTREF2
630
631
#endif  // CONFIG_EXT_REFS

632
633
634
635
  { ZEROMV, { ALTREF_FRAME, INTRA_FRAME } },
  { NEARESTMV, { ALTREF_FRAME, INTRA_FRAME } },
  { NEARMV, { ALTREF_FRAME, INTRA_FRAME } },
  { NEWMV, { ALTREF_FRAME, INTRA_FRAME } },
636
#endif  // CONFIG_EXT_INTER
Jingning Han's avatar
Jingning Han committed
637
638
};

hui su's avatar
hui su committed
639
static const PREDICTION_MODE intra_rd_search_mode_order[INTRA_MODES] = {
Urvang Joshi's avatar
Urvang Joshi committed
640
641
  DC_PRED,       H_PRED,        V_PRED,    SMOOTH_PRED, TM_PRED,
#if CONFIG_SMOOTH_HV
hui su's avatar
hui su committed
642
  SMOOTH_V_PRED, SMOOTH_H_PRED,
Urvang Joshi's avatar
Urvang Joshi committed
643
644
#endif  // CONFIG_SMOOTH_HV
  D135_PRED,     D207_PRED,     D153_PRED, D63_PRED,    D117_PRED, D45_PRED,
hui su's avatar
hui su committed
645
646
};

Luc Trudeau's avatar
Luc Trudeau committed
647
648
#if CONFIG_CFL
static const UV_PREDICTION_MODE uv_rd_search_mode_order[UV_INTRA_MODES] = {
Urvang Joshi's avatar
Urvang Joshi committed
649
650
651
  UV_DC_PRED,       UV_CFL_PRED,      UV_H_PRED,
  UV_V_PRED,        UV_SMOOTH_PRED,   UV_TM_PRED,
#if CONFIG_SMOOTH_HV
Luc Trudeau's avatar
Luc Trudeau committed
652
  UV_SMOOTH_V_PRED, UV_SMOOTH_H_PRED,
Urvang Joshi's avatar
Urvang Joshi committed
653
654
655
#endif  // CONFIG_SMOOTH_HV
  UV_D135_PRED,     UV_D207_PRED,     UV_D153_PRED,
  UV_D63_PRED,      UV_D117_PRED,     UV_D45_PRED,
Luc Trudeau's avatar
Luc Trudeau committed
656
657
658
659
660
};
#else
#define uv_rd_search_mode_order intra_rd_search_mode_order
#endif  // CONFIG_CFL

hui su's avatar
hui su committed
661
static INLINE int write_uniform_cost(int n, int v) {
662
663
  const int l = get_unsigned_bits(n);
  const int m = (1 << l) - n;
664
  if (l == 0) return 0;
hui su's avatar
hui su committed
665
  if (v < m)
Yaowu Xu's avatar
Yaowu Xu committed
666
    return (l - 1) * av1_cost_bit(128, 0);
hui su's avatar
hui su committed
667
  else
Yaowu Xu's avatar
Yaowu Xu committed
668
    return l * av1_cost_bit(128, 0);
hui su's avatar
hui su committed
669
670
}

671
672
673
// constants for prune 1 and prune 2 decision boundaries
#define FAST_EXT_TX_CORR_MID 0.0
#define FAST_EXT_TX_EDST_MID 0.1
674
675
676
#define FAST_EXT_TX_CORR_MARGIN 0.5
#define FAST_EXT_TX_EDST_MARGIN 0.3

677
#if CONFIG_DIST_8X8
Yushin Cho's avatar
Yushin Cho committed
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
static uint64_t cdef_dist_8x8_16bit(uint16_t *dst, int dstride, uint16_t *src,
                                    int sstride, int coeff_shift) {
  uint64_t svar = 0;
  uint64_t dvar = 0;
  uint64_t sum_s = 0;
  uint64_t sum_d = 0;
  uint64_t sum_s2 = 0;
  uint64_t sum_d2 = 0;
  uint64_t sum_sd = 0;
  uint64_t dist = 0;

  int i, j;
  for (i = 0; i < 8; i++) {
    for (j = 0; j < 8; j++) {
      sum_s += src[i * sstride + j];
      sum_d += dst[i * dstride + j];
      sum_s2 += src[i * sstride + j] * src[i * sstride + j];
      sum_d2 += dst[i * dstride + j] * dst[i * dstride + j];
      sum_sd += src[i * sstride + j] * dst[i * dstride + j];
    }
  }
  /* Compute the variance -- the calculation cannot go negative. */
  svar = sum_s2 - ((sum_s * sum_s + 32) >> 6);
  dvar = sum_d2 - ((sum_d * sum_d + 32) >> 6);

  // Tuning of jm's original dering distortion metric used in CDEF tool,
  // suggested by jm
  const uint64_t a = 4;
  const uint64_t b = 2;
  const uint64_t c1 = (400 * a << 2 * coeff_shift);
  const uint64_t c2 = (b * 20000 * a * a << 4 * coeff_shift);

  dist =
      (uint64_t)floor(.5 +
                      (sum_d2 + sum_s2 - 2 * sum_sd) * .5 * (svar + dvar + c1) /
                          (sqrt(svar * (double)dvar + c2)));

  // Calibrate dist to have similar rate for the same QP with MSE only
  // distortion (as in master branch)
  dist = (uint64_t)((float)dist * 0.75);

  return dist;
}

Yushin Cho's avatar
Yushin Cho committed
722
static int od_compute_var_4x4(uint16_t *x, int stride) {
Yushin Cho's avatar
Yushin Cho committed
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
  int sum;
  int s2;
  int i;
  sum = 0;
  s2 = 0;
  for (i = 0; i < 4; i++) {
    int j;
    for (j = 0; j < 4; j++) {
      int t;

      t = x[i * stride + j];
      sum += t;
      s2 += t * t;
    }
  }
Yushin Cho's avatar
Yushin Cho committed
738

Yushin Cho's avatar
Yushin Cho committed
739
740
741
  return (s2 - (sum * sum >> 4)) >> 4;
}

742
743
744
745
746
747
748
/* OD_DIST_LP_MID controls the frequency weighting filter used for computing
   the distortion. For a value X, the filter is [1 X 1]/(X + 2) and
   is applied both horizontally and vertically. For X=5, the filter is
   a good approximation for the OD_QM8_Q4_HVS quantization matrix. */
#define OD_DIST_LP_MID (5)
#define OD_DIST_LP_NORM (OD_DIST_LP_MID + 2)

Yushin Cho's avatar
Yushin Cho committed
749
750
static double od_compute_dist_8x8(int use_activity_masking, uint16_t *x,
                                  uint16_t *y, od_coeff *e_lp, int stride) {
Yushin Cho's avatar
Yushin Cho committed
751
752
753
754
755
756
757
758
759
760
761
  double sum;
  int min_var;
  double mean_var;
  double var_stat;
  double activity;
  double calibration;
  int i;
  int j;
  double vardist;

  vardist = 0;
Yushin Cho's avatar
Yushin Cho committed
762

Yushin Cho's avatar
Yushin Cho committed
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
#if 1
  min_var = INT_MAX;
  mean_var = 0;
  for (i = 0; i < 3; i++) {
    for (j = 0; j < 3; j++) {
      int varx;
      int vary;
      varx = od_compute_var_4x4(x + 2 * i * stride + 2 * j, stride);
      vary = od_compute_var_4x4(y + 2 * i * stride + 2 * j, stride);
      min_var = OD_MINI(min_var, varx);
      mean_var += 1. / (1 + varx);
      /* The cast to (double) is to avoid an overflow before the sqrt.*/
      vardist += varx - 2 * sqrt(varx * (double)vary) + vary;
    }
  }
  /* We use a different variance statistic depending on whether activity
James Zern's avatar
James Zern committed
779
     masking is used, since the harmonic mean appeared slightly worse with
Yushin Cho's avatar
Yushin Cho committed
780
781
782
783
784
785
786
787
788
789
790
791
792
793
     masking off. The calibration constant just ensures that we preserve the
     rate compared to activity=1. */
  if (use_activity_masking) {
    calibration = 1.95;
    var_stat = 9. / mean_var;
  } else {
    calibration = 1.62;
    var_stat = min_var;
  }
  /* 1.62 is a calibration constant, 0.25 is a noise floor and 1/6 is the
     activity masking constant. */
  activity = calibration * pow(.25 + var_stat, -1. / 6);
#else
  activity = 1;
794
#endif  // 1
Yushin Cho's avatar
Yushin Cho committed
795
796
797
  sum = 0;
  for (i = 0; i < 8; i++) {
    for (j = 0; j < 8; j++)
798
      sum += e_lp[i * stride + j] * (double)e_lp[i * stride + j];
Yushin Cho's avatar
Yushin Cho committed
799
  }
800
801
802
  /* Normalize the filter to unit DC response. */
  sum *= 1. / (OD_DIST_LP_NORM * OD_DIST_LP_NORM * OD_DIST_LP_NORM *
               OD_DIST_LP_NORM);
Yushin Cho's avatar
Yushin Cho committed
803
804
805
806
  return activity * activity * (sum + vardist);
}

// Note : Inputs x and y are in a pixel domain
Yushin Cho's avatar
Yushin Cho committed
807
808
static double od_compute_dist_common(int activity_masking, uint16_t *x,
                                     uint16_t *y, int bsize_w, int bsize_h,
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
                                     int qindex, od_coeff *tmp,
                                     od_coeff *e_lp) {
  int i, j;
  double sum = 0;
  const int mid = OD_DIST_LP_MID;

  for (j = 0; j < bsize_w; j++) {
    e_lp[j] = mid * tmp[j] + 2 * tmp[bsize_w + j];
    e_lp[(bsize_h - 1) * bsize_w + j] = mid * tmp[(bsize_h - 1) * bsize_w + j] +
                                        2 * tmp[(bsize_h - 2) * bsize_w + j];
  }
  for (i = 1; i < bsize_h - 1; i++) {
    for (j = 0; j < bsize_w; j++) {
      e_lp[i * bsize_w + j] = mid * tmp[i * bsize_w + j] +
                              tmp[(i - 1) * bsize_w + j] +
                              tmp[(i + 1) * bsize_w + j];
    }
  }
  for (i = 0; i < bsize_h; i += 8) {
    for (j = 0; j < bsize_w; j += 8) {
Yushin Cho's avatar
Yushin Cho committed
829
      sum += od_compute_dist_8x8(activity_masking, &x[i * bsize_w + j],
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
                                 &y[i * bsize_w + j], &e_lp[i * bsize_w + j],
                                 bsize_w);
    }
  }
  /* Scale according to linear regression against SSE, for 8x8 blocks. */
  if (activity_masking) {
    sum *= 2.2 + (1.7 - 2.2) * (qindex - 99) / (210 - 99) +
           (qindex < 99 ? 2.5 * (qindex - 99) / 99 * (qindex - 99) / 99 : 0);
  } else {
    sum *= qindex >= 128
               ? 1.4 + (0.9 - 1.4) * (qindex - 128) / (209 - 128)
               : qindex <= 43 ? 1.5 + (2.0 - 1.5) * (qindex - 43) / (16 - 43)
                              : 1.5 + (1.4 - 1.5) * (qindex - 43) / (128 - 43);
  }

  return sum;
}

Yushin Cho's avatar
Yushin Cho committed
848
849
static double od_compute_dist(uint16_t *x, uint16_t *y, int bsize_w,
                              int bsize_h, int qindex) {
Yushin Cho's avatar
Yushin Cho committed
850
851
852
853
854
855
  int i;
  double sum;
  sum = 0;

  assert(bsize_w >= 8 && bsize_h >= 8);

Yushin Cho's avatar
Yushin Cho committed
856
857
858
859
860
861
#if CONFIG_PVQ
  int activity_masking = 1;
#else
  int activity_masking = 0;
#endif
  {
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
    int j;
    DECLARE_ALIGNED(16, od_coeff, e[MAX_TX_SQUARE]);
    DECLARE_ALIGNED(16, od_coeff, tmp[MAX_TX_SQUARE]);
    DECLARE_ALIGNED(16, od_coeff, e_lp[MAX_TX_SQUARE]);
    int mid = OD_DIST_LP_MID;
    for (i = 0; i < bsize_h; i++) {
      for (j = 0; j < bsize_w; j++) {
        e[i * bsize_w + j] = x[i * bsize_w + j] - y[i * bsize_w + j];
      }
    }
    for (i = 0; i < bsize_h; i++) {
      tmp[i * bsize_w] = mid * e[i * bsize_w] + 2 * e[i * bsize_w + 1];
      tmp[i * bsize_w + bsize_w - 1] =
          mid * e[i * bsize_w + bsize_w - 1] + 2 * e[i * bsize_w + bsize_w - 2];
      for (j = 1; j < bsize_w - 1; j++) {
        tmp[i * bsize_w + j] = mid * e[i * bsize_w + j] +
                               e[i * bsize_w + j - 1] + e[i * bsize_w + j + 1];
      }
    }
Yushin Cho's avatar
Yushin Cho committed
881
    sum = od_compute_dist_common(activity_masking, x, y, bsize_w, bsize_h,
882
883
884
885
886
                                 qindex, tmp, e_lp);
  }
  return sum;
}

Yushin Cho's avatar
Yushin Cho committed
887
888
static double od_compute_dist_diff(uint16_t *x, int16_t *e, int bsize_w,
                                   int bsize_h, int qindex) {
889
890
891
892
893
894
  int i;
  double sum;
  sum = 0;

  assert(bsize_w >= 8 && bsize_h >= 8);

Yushin Cho's avatar
Yushin Cho committed
895
896
897
898
899
900
#if CONFIG_PVQ
  int activity_masking = 1;
#else
  int activity_masking = 0;
#endif
  {
901
    int j;
Yushin Cho's avatar
Yushin Cho committed
902
    DECLARE_ALIGNED(16, uint16_t, y[MAX_TX_SQUARE]);
903
904
905
906
    DECLARE_ALIGNED(16, od_coeff, tmp[MAX_TX_SQUARE]);
    DECLARE_ALIGNED(16, od_coeff, e_lp[MAX_TX_SQUARE]);
    int mid = OD_DIST_LP_MID;
    for (i = 0; i < bsize_h; i++) {
907
      for (j = 0; j < bsize_w; j++) {
908
        y[i * bsize_w + j] = x[i * bsize_w + j] - e[i * bsize_w + j];
909
910
      }
    }
911
912
913
914
915
916
917
    for (i = 0; i < bsize_h; i++) {
      tmp[i * bsize_w] = mid * e[i * bsize_w] + 2 * e[i * bsize_w + 1];
      tmp[i * bsize_w + bsize_w - 1] =
          mid * e[i * bsize_w + bsize_w - 1] + 2 * e[i * bsize_w + bsize_w - 2];
      for (j = 1; j < bsize_w - 1; j++) {
        tmp[i * bsize_w + j] = mid * e[i * bsize_w + j] +
                               e[i * bsize_w + j - 1] + e[i * bsize_w + j + 1];
Yushin Cho's avatar
Yushin Cho committed
918
919
      }
    }
Yushin Cho's avatar
Yushin Cho committed
920
    sum = od_compute_dist_common(activity_masking, x, y, bsize_w, bsize_h,
921
                                 qindex, tmp, e_lp);
Yushin Cho's avatar
Yushin Cho committed
922
923
924
925
  }
  return sum;
}

926
int64_t av1_dist_8x8(const AV1_COMP *const cpi, const MACROBLOCK *x,
Yushin Cho's avatar
Yushin Cho committed
927
928
929
930
                     const uint8_t *src, int src_stride, const uint8_t *dst,
                     int dst_stride, const BLOCK_SIZE tx_bsize, int bsw,
                     int bsh, int visible_w, int visible_h, int qindex) {
  int64_t d = 0;
Yushin Cho's avatar
Yushin Cho committed
931
  int i, j;
932
  const MACROBLOCKD *xd = &x->e_mbd;
Yushin Cho's avatar
Yushin Cho committed
933
934
935
936

  DECLARE_ALIGNED(16, uint16_t, orig[MAX_TX_SQUARE]);
  DECLARE_ALIGNED(16, uint16_t, rec[MAX_TX_SQUARE]);

937
938
939
940
  if (x->tune_metric == AOM_TUNE_CDEF_DIST ||
      x->tune_metric == AOM_TUNE_DAALA_DIST) {
    assert((bsw & 0x07) == 0);
    assert((bsh & 0x07) == 0);
Yushin Cho's avatar
Yushin Cho committed
941

942
#if CONFIG_HIGHBITDEPTH
943
    if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
944
      for (j = 0; j < bsh; j++)
945
        for (i = 0; i < bsw; i++)
946
          orig[j * bsw + i] = CONVERT_TO_SHORTPTR(src)[j * src_stride + i];
947

948
      if ((bsw == visible_w) && (bsh == visible_h)) {
949
950
        for (j = 0; j < bsh; j++)
          for (i = 0; i < bsw; i++)
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
            rec[j * bsw + i] = CONVERT_TO_SHORTPTR(dst)[j * dst_stride + i];
      } else {
        for (j = 0; j < visible_h; j++)
          for (i = 0; i < visible_w; i++)
            rec[j * bsw + i] = CONVERT_TO_SHORTPTR(dst)[j * dst_stride + i];

        if (visible_w < bsw) {
          for (j = 0; j < bsh; j++)
            for (i = visible_w; i < bsw; i++)
              rec[j * bsw + i] = CONVERT_TO_SHORTPTR(src)[j * src_stride + i];
        }

        if (visible_h < bsh) {
          for (j = visible_h; j < bsh; j++)
            for (i = 0; i < bsw; i++)
              rec[j * bsw + i] = CONVERT_TO_SHORTPTR(src)[j * src_stride + i];
        }
968
      }
969
    } else {
970
971
#endif
      for (j = 0; j < bsh; j++)
972
        for (i = 0; i < bsw; i++) orig[j * bsw + i] = src[j * src_stride + i];
973

974
      if ((bsw == visible_w) && (bsh == visible_h)) {
975
        for (j = 0; j < bsh; j++)
976
977
978
979
980
981
982
983
984
985
986
          for (i = 0; i < bsw; i++) rec[j * bsw + i] = dst[j * dst_stride + i];
      } else {
        for (j = 0; j < visible_h; j++)
          for (i = 0; i < visible_w; i++)
            rec[j * bsw + i] = dst[j * dst_stride + i];

        if (visible_w < bsw) {
          for (j = 0; j < bsh; j++)
            for (i = visible_w; i < bsw; i++)
              rec[j * bsw + i] = src[j * src_stride + i];
        }
987

988
989
990
991
992
        if (visible_h < bsh) {
          for (j = visible_h; j < bsh; j++)
            for (i = 0; i < bsw; i++)
              rec[j * bsw + i] = src[j * src_stride + i];
        }
993
994
      }
#if CONFIG_HIGHBITDEPTH
995
    }
996
#endif  // CONFIG_HIGHBITDEPTH
997
  }
Yushin Cho's avatar
Yushin Cho committed
998

999
1000
1001
  if (x->tune_metric == AOM_TUNE_DAALA_DIST) {
    d = (int64_t)od_compute_dist(orig, rec, bsw, bsh, qindex);
  } else if (x->tune_metric == AOM_TUNE_CDEF_DIST) {
Yushin Cho's avatar
Yushin Cho committed
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
    int coeff_shift = AOMMAX(xd->bd - 8, 0);

    for (i = 0; i < bsh; i += 8) {
      for (j = 0; j < bsw; j += 8) {
        d += cdef_dist_8x8_16bit(&rec[i * bsw + j], bsw, &orig[i * bsw + j],
                                 bsw, coeff_shift);
      }
    }
#if CONFIG_HIGHBITDEPTH
    if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH)
      d = ((uint64_t)d) >> 2 * coeff_shift;
#endif
1014
1015
1016
1017
1018
1019
  } else {
    // Otherwise, MSE by default
    unsigned sse;
    // TODO(Any): Use even faster function which does not calculate variance
    cpi->fn_ptr[tx_bsize].vf(src, src_stride, dst, dst_stride, &sse);
    d = sse;
Yushin Cho's avatar
Yushin Cho committed
1020
  }
1021

Yushin Cho's avatar
Yushin Cho committed
1022
1023
  return d;
}
1024

1025
static int64_t av1_dist_8x8_diff(const MACROBLOCK *x, const uint8_t *src,
Yushin Cho's avatar
Yushin Cho committed
1026
1027
1028
1029
                                 int src_stride, const int16_t *diff,
                                 int diff_stride, int bsw, int bsh,
                                 int visible_w, int visible_h, int qindex) {
  int64_t d = 0;
1030
  int i, j;
1031
  const MACROBLOCKD *xd = &x->e_mbd;
Yushin Cho's avatar
Yushin Cho committed
1032
1033
1034
1035

  DECLARE_ALIGNED(16, uint16_t, orig[MAX_TX_SQUARE]);
  DECLARE_ALIGNED(16, int16_t, diff16[MAX_TX_SQUARE]);

1036
1037
1038
1039
  if (x->tune_metric == AOM_TUNE_CDEF_DIST ||
      x->tune_metric == AOM_TUNE_DAALA_DIST) {
    assert((bsw & 0x07) == 0);
    assert((bsh & 0x07) == 0);
Yushin Cho's avatar
Yushin Cho committed
1040

1041
#if CONFIG_HIGHBITDEPTH
1042
1043
1044
1045
1046
    if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
      for (j = 0; j < bsh; j++)
        for (i = 0; i < bsw; i++)
          orig[j * bsw + i] = CONVERT_TO_SHORTPTR(src)[j * src_stride + i];
    } else {
1047
#endif
1048
1049
      for (j = 0; j < bsh; j++)
        for (i = 0; i < bsw; i++) orig[j * bsw + i] = src[j * src_stride + i];
1050
#if CONFIG_HIGHBITDEPTH
1051
    }
1052
#endif  // CONFIG_HIGHBITDEPTH
1053

1054
    if ((bsw == visible_w) && (bsh == visible_h)) {
1055
      for (j = 0; j < bsh; j++)
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
        for (i = 0; i < bsw; i++)
          diff16[j * bsw + i] = diff[j * diff_stride + i];
    } else {
      for (j = 0; j < visible_h; j++)
        for (i = 0; i < visible_w; i++)
          diff16[j * bsw + i] = diff[j * diff_stride + i];

      if (visible_w < bsw) {
        for (j = 0; j < bsh; j++)
          for (i = visible_w; i < bsw; i++) diff16[j * bsw + i] = 0;
      }
1067

1068
1069
1070
1071
      if (visible_h < bsh) {
        for (j = visible_h; j < bsh; j++)
          for (i = 0; i < bsw; i++) diff16[j * bsw + i] = 0;
      }
1072
1073
    }
  }
1074

1075
1076
1077
  if (x->tune_metric == AOM_TUNE_DAALA_DIST) {
    d = (int64_t)od_compute_dist_diff(orig, diff16, bsw, bsh, qindex);
  } else if (x->tune_metric == AOM_TUNE_CDEF_DIST) {
Yushin Cho's avatar
Yushin Cho committed
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
    int coeff_shift = AOMMAX(xd->bd - 8, 0);
    DECLARE_ALIGNED(16, uint16_t, dst16[MAX_TX_SQUARE]);

    for (i = 0; i < bsh; i++) {
      for (j = 0; j < bsw; j++) {
        dst16[i * bsw + j] = orig[i * bsw + j] - diff16[i * bsw + j];
      }
    }

    for (i = 0; i < bsh; i += 8) {
      for (j = 0; j < bsw; j += 8) {
        d += cdef_dist_8x8_16bit(&dst16[i * bsw + j], bsw, &orig[i * bsw + j],
                                 bsw, coeff_shift);
      }
    }
    // Don't scale 'd' for HBD since it will be done by caller side for diff
    // input
1095
1096
1097
  } else {
    // Otherwise, MSE by default
    d = aom_sum_squares_2d_i16(diff, diff_stride, bsw, bsh);
Yushin Cho's avatar
Yushin Cho committed
1098
  }
1099
1100
1101

  return d;
}
Yushin Cho's avatar
Yushin Cho committed
1102
#endif  // CONFIG_DIST_8X8
Yushin Cho's avatar
Yushin Cho committed
1103

Yaowu Xu's avatar
Yaowu Xu committed
1104
static void get_energy_distribution_fine(const AV1_COMP *cpi, BLOCK_SIZE bsize,
1105
1106
                                         const uint8_t *src, int src_stride,
                                         const uint8_t *dst, int dst_stride,
1107
                                         double *hordist, double *verdist) {
1108
1109
  const int bw = block_size_wide[bsize];
  const int bh = block_size_high[bsize];
1110
  unsigned int esq[16] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
1111

1112
  const int f_index = bsize - BLOCK_16X16;
1113
  if (f_index < 0) {
1114
1115
    const int w_shift = bw == 8 ? 1 : 2;
    const int h_shift = bh == 8 ? 1 : 2;
1116
#if CONFIG_HIGHBITDEPTH
1117
    if (cpi->common.use_highbitdepth) {
1118
1119
1120
1121
1122
      const uint16_t *src16 = CONVERT_TO_SHORTPTR(src);
      const uint16_t *dst16 = CONVERT_TO_SHORTPTR(dst);
      for (int i = 0; i < bh; ++i)
        for (int j = 0; j < bw; ++j) {
          const int index = (j >> w_shift) + ((i >> h_shift) << 2);
1123
1124
1125
          esq[index] +=
              (src16[j + i * src_stride] - dst16[j + i * dst_stride]) *
              (src16[j + i * src_stride] - dst16[j + i * dst_stride]);
1126
1127
        }
    } else {
1128
#endif  // CONFIG_HIGHBITDEPTH
1129

1130
1131
1132
      for (int i = 0; i < bh; ++i)
        for (int j = 0; j < bw; ++j) {
          const int index = (j >> w_shift) + ((i >> h_shift) << 2);
1133
1134
1135
          esq[index] += (src[j + i * src_stride] - dst[j + i * dst_stride]) *
                        (src[j + i * src_stride] - dst[j + i * dst_stride]);
        }
1136
#if CONFIG_HIGHBITDEPTH
1137
    }
1138
#endif  // CONFIG_HIGHBITDEPTH
1139
  } else {
1140
1141
1142
1143
1144
1145
1146
    cpi->fn_ptr[f_index].vf(src, src_stride, dst, dst_stride, &esq[0]);
    cpi->fn_ptr[f_index].vf(src + bw / 4, src_stride, dst + bw / 4, dst_stride,
                            &esq[1]);
    cpi->fn_ptr[f_index].vf(src + bw / 2, src_stride, dst + bw / 2, dst_stride,
                            &esq[2]);
    cpi->fn_ptr[f_index].vf(src + 3 * bw / 4, src_stride, dst + 3 * bw / 4,
                            dst_stride, &esq[3]);
1147
1148
1149
    src += bh / 4 * src_stride;
    dst += bh / 4 * dst_stride;

1150
1151
1152
1153
1154
1155
1156
    cpi->fn_ptr[f_index].vf(src, src_stride, dst, dst_stride, &esq[4]);
    cpi->fn_ptr[f_index].vf(src + bw / 4, src_stride, dst + bw / 4, dst_stride,
                            &esq[5]);
    cpi->fn_ptr[f_index].vf(src + bw / 2, src_stride, dst + bw / 2, dst_stride,
                            &esq[6]);
    cpi->fn_ptr[f_index].vf(src + 3 * bw / 4, src_stride, dst + 3 * bw / 4,
                            dst_stride, &esq[7]);
1157
1158
1159
    src += bh / 4 * src_stride;
    dst += bh / 4 * dst_stride;

1160
1161
1162
1163
1164
1165
1166
    cpi->fn_ptr[f_index].vf(src, src_stride, dst, dst_stride, &esq[8]);
    cpi->fn_ptr[f_index].vf(src + bw / 4, src_stride, dst + bw / 4, dst_stride,
                            &esq[9]);
    cpi->fn_ptr[f_index].vf(src + bw / 2, src_stride, dst + bw / 2, dst_stride,
                            &esq[10]);
    cpi->fn_ptr[f_index].vf(src + 3 * bw / 4, src_stride, dst + 3 * bw / 4,
                            dst_stride, &esq[11]);
1167
1168
1169
    src += bh / 4 * src_stride;
    dst += bh / 4 * dst_stride;

1170
1171
1172
1173
1174
1175
1176
    cpi->fn_ptr[f_index].vf(src, src_stride, dst, dst_stride, &esq[12]);
    cpi->fn_ptr[f_index].vf(src + bw / 4, src_stride, dst + bw / 4, dst_stride,
                            &esq[13]);
    cpi->fn_ptr[f_index].vf(src + bw / 2, src_stride, dst + bw / 2, dst_stride,
                            &esq[14]);
    cpi->fn_ptr[f_index].vf(src + 3 * bw / 4, src_stride, dst + 3 * bw / 4,
                            dst_stride, &esq[15]);
1177
1178
  }

1179
1180
1181
  double total = (double)esq[0] + esq[1] + esq[2] + esq[3] + esq[4] + esq[5] +
                 esq[6] + esq[7] + esq[8] + esq[9] + esq[10] + esq[11] +
                 esq[12] + esq[13] + esq[14] + esq[15];
1182
1183
  if (total > 0) {
    const double e_recip = 1.0 / total;
1184
1185
1186
1187
1188
1189
    hordist[0] = ((double)esq[0] + esq[4] + esq[8] + esq[12]) * e_recip;
    hordist[1] = ((double)esq[1] + esq[5] + esq[9] + esq[13]) * e_recip;
    hordist[2] = ((double)esq[2] + esq[6] + esq[10] + esq[14]) * e_recip;
    verdist[0] = ((double)esq[0] + esq[1] + esq[2] + esq[3]) * e_recip;
    verdist[1] = ((double)esq[4] + esq[5] + esq[6] + esq[7]) * e_recip;
    verdist[2] = ((double)esq[8] + esq[9] + esq[10] + esq[11]) * e_recip;
1190
1191
1192
1193
1194
1195
1196
  } else {
    hordist[0] = verdist[0] = 0.25;
    hordist[1] = verdist[1] = 0.25;
    hordist[2] = verdist[2] = 0.25;
  }
}

Urvang Joshi's avatar
Urvang Joshi committed
1197
1198
1199
static int adst_vs_flipadst(const AV1_COMP *cpi, BLOCK_SIZE bsize,
                            const uint8_t *src, int src_stride,
                            const uint8_t *dst, int dst_stride) {
1200
1201
  int prune_bitmask = 0;
  double svm_proj_h = 0, svm_proj_v = 0;
Alex Converse's avatar
Alex Converse committed
1202
  double hdist[3] = { 0, 0, 0 }, vdist[3] = { 0, 0, 0 };
1203
1204
  get_energy_distribution_fine(cpi, bsize, src, src_stride, dst, dst_stride,
                               hdist, vdist);
1205

1206
  svm_proj_v = vdist[0] * ADST_FLIP_SVM[0] + vdist[1] * ADST_FLIP_SVM[1] +
1207
               vdist[2] * ADST_FLIP_SVM[2] + ADST_FLIP_SVM[3];
1208
  svm_proj_h = hdist[0] * ADST_FLIP_SVM[4] + hdist[1] * ADST_FLIP_SVM[5] +
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
               hdist[2] * ADST_FLIP_SVM[6] + ADST_FLIP_SVM[7];
  if (svm_proj_v > FAST_EXT_TX_EDST_MID + FAST_EXT_TX_EDST_MARGIN)
    prune_bitmask |= 1 << FLIPADST_1D;
  else if (svm_proj_v < FAST_EXT_TX_EDST_MID - FAST_EXT_TX_EDST_MARGIN)
    prune_bitmask |= 1 << ADST_1D;

  if (svm_proj_h > FAST_EXT_TX_EDST_MID + FAST_EXT_TX_EDST_MARGIN)
    prune_bitmask |= 1 << (FLIPADST_1D + 8);
  else if (svm_proj_h < FAST_EXT_TX_EDST_MID - FAST_EXT_TX_EDST_MARGIN)
    prune_bitmask |= 1 << (ADST_1D + 8);

  return prune_bitmask;
}

#if CONFIG_EXT_TX
Alex Converse's avatar
Alex Converse committed
1224
1225
static void get_horver_correlation(const int16_t *diff, int stride, int w,
                                   int h, double *hcorr, double *vcorr) {
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
  // Returns hor/ver correlation coefficient
  const int num = (h - 1) * (w - 1);
  double num_r;
  int i, j;
  int64_t xy_sum = 0, xz_sum = 0;
  int64_t x_sum = 0, y_sum = 0, z_sum = 0;
  int64_t x2_sum = 0, y2_sum = 0, z2_sum = 0;
  double x_var_n, y_var_n, z_var_n, xy_var_n, xz_var_n;
  *hcorr = *vcorr = 1;

  assert(num > 0);
  num_r = 1.0 / num;
  for (i = 1; i < h; ++i) {
    for (j = 1; j < w; ++j) {
      const int16_t x = diff[i * stride + j];
      const int16_t y = diff[i * stride + j - 1];
      const int16_t z = diff[(i - 1) * stride + j];
      xy_sum += x * y;
      xz_sum += x * z;
      x_sum += x;
      y_sum += y;
      z_sum += z;
      x2_sum += x * x;
      y2_sum += y * y;
      z2_sum += z * z;
    }
  }
1253
1254
1255
  x_var_n = x2_sum - (x_sum * x_sum) * num_r;
  y_var_n = y2_sum - (y_sum * y_sum) * num_r;
  z_var_n = z2_sum - (z_sum * z_sum) * num_r;
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
  xy_var_n = xy_sum - (x_sum * y_sum) * num_r;
  xz_var_n = xz_sum - (x_sum * z_sum) * num_r;
  if (x_var_n > 0 && y_var_n > 0) {
    *hcorr = xy_var_n / sqrt(x_var_n * y_var_n);
    *hcorr = *hcorr < 0 ? 0 : *hcorr;
  }
  if (x_var_n > 0 && z_var_n > 0) {
    *vcorr = xz_var_n / sqrt(x_var_n * z_var_n);
    *vcorr = *vcorr < 0 ? 0 : *vcorr;
  }
}

Alex Converse's avatar
Alex Converse committed
1268
1269
int dct_vs_idtx(const int16_t *diff, int stride, int w, int h) {
  double hcorr, vcorr;
1270
  int prune_bitmask = 0;
Alex Converse's avatar
Alex Converse committed
1271
  get_horver_correlation(diff, stride, w, h, &hcorr, &vcorr);
1272

Alex Converse's avatar
Alex Converse committed
1273
  if (vcorr > FAST_EXT_TX_CORR_MID + FAST_EXT_TX_CORR_MARGIN)
1274
    prune_bitmask |= 1 << IDTX_1D;
Alex Converse's avatar
Alex Converse committed
1275
  else if (vcorr < FAST_EXT_TX_CORR_MID - FAST_EXT_TX_CORR_MARGIN)
1276
1277
    prune_bitmask |= 1 << DCT_1D;

Alex Converse's avatar
Alex Converse committed
1278
  if (hcorr > FAST_EXT_TX_CORR_MID + FAST_EXT_TX_CORR_MARGIN)
1279
    prune_bitmask |= 1 << (IDTX_1D + 8);
Alex Converse's avatar
Alex Converse committed
1280
  else if (hcorr < FAST_EXT_TX_CORR_MID - FAST_EXT_TX_CORR_MARGIN)
1281
1282
1283
1284
1285
    prune_bitmask |= 1 << (DCT_1D + 8);
  return prune_bitmask;
}

// Performance drop: 0.5%, Speed improvement: 24%
Yaowu Xu's avatar
Yaowu Xu committed
1286
static int prune_two_for_sby(const AV1_COMP *cpi, BLOCK_SIZE bsize,
Alex Converse's avatar
Alex Converse committed
1287
1288
                             MACROBLOCK *x, const MACROBLOCKD *xd,
                             int adst_flipadst, int dct_idtx) {
1289
  int prune = 0;
1290

Alex Converse's avatar
Alex Converse committed
1291
1292
1293
  if (adst_flipadst) {
    const struct macroblock_plane *const p = &x->plane[0];
    const struct macroblockd_plane *const pd = &xd->plane[0];
1294
    prune |= adst_vs_flipadst(cpi, bsize, p->src.buf, p->src.stride,
Alex Converse's avatar
Alex Converse committed
1295
1296
1297
1298
1299
1300
1301
1302
1303
                              pd->dst.buf, pd->dst.stride);
  }
  if (dct_idtx) {
    av1_subtract_plane(x, bsize, 0);
    const struct macroblock_plane *const p = &x->plane[0];
    const int bw = 4 << (b_width_log2_lookup[bsize]);
    const int bh = 4 << (b_height_log2_lookup[bsize]);
    prune |= dct_vs_idtx(p->src_diff, bw, bw, bh);
  }
1304
1305
1306

  return prune;
}
1307
1308
#endif  // CONFIG_EXT_TX

1309
// Performance drop: 0.3%, Speed improvement: 5%
Yaowu Xu's avatar
Yaowu Xu committed
1310
static int prune_one_for_sby(const AV1_COMP *cpi, BLOCK_SIZE bsize,
Alex Converse's avatar
Alex Converse committed
1311
1312
1313
                             const MACROBLOCK *x, const MACROBLOCKD *xd) {
  const struct macroblock_plane *const p = &x->plane[0];
  const struct macroblockd_plane *const pd = &xd->plane[0];
1314
  return adst_vs_flipadst(cpi, bsize, p->src.buf, p->src.stride, pd->dst.buf,
Alex Converse's avatar
Alex Converse committed
1315
                          pd->dst.stride);
1316
1317
}

Yaowu Xu's avatar
Yaowu Xu committed
1318
static int prune_tx_types(const AV1_COMP *cpi, BLOCK_SIZE bsize, MACROBLOCK *x,
Urvang Joshi's avatar
Urvang Joshi committed
1319
                          const MACROBLOCKD *const xd, int tx_set) {
1320
#if CONFIG_EXT_TX
Yaowu Xu's avatar
Yaowu Xu committed
1321
  const int *tx_set_1D = tx_set >= 0 ? ext_tx_used_inter_1D[tx_set] : NULL;
1322
#else
1323
  const int tx_set_1D[TX_TYPES_1D] = { 0 };
1324
#endif  // CONFIG_EXT_TX
1325

1326
  switch (cpi->sf.tx_type_search.prune_mode) {
1327
1328
    case NO_PRUNE: return 0; break;
    case PRUNE_ONE:
1329
      if ((tx_set >= 0) && !(tx_set_1D[FLIPADST_1D] & tx_set_1D[ADST_1D]))
1330
        return 0;
1331
1332
      return prune_one_for_sby(cpi, bsize, x, xd);
      break;
1333
1334
#if CONFIG_EXT_TX
    case PRUNE_TWO:
1335
      if ((tx_set >= 0) && !(tx_set_1D[FLIPADST_1D] & tx_set_1D[ADST_1D])) {
1336
        if (!(tx_set_1D[DCT_1D] & tx_set_1D[IDTX_1D])) return 0;
1337
1338
        return prune_two_for_sby(cpi, bsize, x, xd, 0, 1);
      }
1339
      if ((tx_set >= 0) && !(tx_set_1D[DCT_1D] & tx_set_1D[IDTX_1D]))
1340
1341
        return prune_two_for_sby(cpi, bsize, x, xd, 1, 0);
      return prune_two_for_sby(cpi, bsize, x, xd, 1, 1);
1342
      break;
1343
#endif  // CONFIG_EXT_TX
1344
1345
1346
1347
1348
  }
  assert(0);
  return 0;
}

1349
static int do_tx_type_search(TX_TYPE tx_type, int prune) {
1350
1351
1352
// TODO(sarahparker) implement for non ext tx
#if CONFIG_EXT_TX
  return !(((prune >> vtx_tab[tx_type]) & 1) |
1353
           ((prune >> (htx_tab[tx_type] + 8)) & 1));
1354
1355
#else
  // temporary to avoid compiler warnings
1356
1357
1358
1359
  (void)vtx_tab;
  (void)htx_tab;
  (void)tx_type;
  (void)prune;
1360
  return 1;
1361
#endif  // CONFIG_EXT_TX
1362
1363
}

Yaowu Xu's avatar
Yaowu Xu committed
1364
static void model_rd_from_sse(const AV1_COMP *const cpi,
1365
1366
                              const MACROBLOCKD *const xd, BLOCK_SIZE bsize,
                              int plane, int64_t sse, int *rate,
Geza Lore's avatar
Geza Lore committed
1367
1368
1369
                              int64_t *dist) {
  const struct macroblockd_plane *const pd = &xd->plane[plane];
  const int dequant_shift =
1370
#if CONFIG_HIGHBITDEPTH
1371
      (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) ? xd->bd - 5 :
1372
#endif  // CONFIG_HIGHBITDEPTH
1373
                                                    3;
Geza Lore's avatar
Geza Lore committed
1374
1375
1376
1377
1378
1379
1380

  // Fast approximate the modelling function.
  if (cpi->sf.simple_model_rd_from_var) {
    const int64_t square_error = sse;
    int quantizer = (pd->dequant[1] >> dequant_shift);

    if (quantizer < 120)
Yaowu Xu's avatar
Yaowu Xu committed
1381
      *rate = (int)((square_error * (280 - quantizer)) >>
Yaowu Xu's avatar
Yaowu Xu committed
1382
                    (16 - AV1_PROB_COST_SHIFT));
Geza Lore's avatar
Geza Lore committed
1383
1384
1385
1386
    else
      *rate = 0;
    *dist = (square_error * quantizer) >> 8;
  } else {
Yaowu Xu's avatar
Yaowu Xu committed
1387
1388
    av1_model_rd_from_var_lapndz(sse, num_pels_log2_lookup[bsize],
                                 pd->dequant[1] >> dequant_shift, rate, dist);
Geza Lore's avatar
Geza Lore committed
1389
1390
1391
1392
1393
  }

  *dist <<= 4;
}

Yaowu Xu's avatar
Yaowu Xu committed
1394
static void model_rd_for_sb(const AV1_COMP *const cpi, BLOCK_SIZE bsize,
1395
1396
1397
1398
                            MACROBLOCK *x, MACROBLOCKD *xd, int plane_from,
                            int plane_to, int *out_rate_sum,
                            int64_t *out_dist_sum, int *skip_txfm_sb,
                            int64_t *skip_sse_sb) {
Jingning Han's avatar
Jingning Han committed
1399
1400
1401
  // Note our transform coeffs are 8 times an orthogonal transform.
  // Hence quantizer step is also 8 times. To get effective quantizer
  // we need to divide by 8 before sending to modeling function.
Geza Lore's avatar
Geza Lore committed
1402
1403
1404
  int plane;
  const int ref = xd->mi[0]->mbmi.ref_frame[0];

Jingning Han's avatar
Jingning Han committed
1405
1406
1407
1408
1409
1410
  int64_t rate_sum = 0;
  int64_t dist_sum = 0;
  int64_t total_sse = 0;

  x->pred_sse[ref] = 0;

Geza Lore's avatar
Geza Lore committed
1411
1412
1413
  for (plane = plane_from; plane <= plane_to; ++plane) {
    struct macroblock_plane *const p = &x->plane[plane];
    struct macroblockd_plane *const pd = &xd->plane[plane];
1414
#if CONFIG_CHROMA_SUB8X8