rdopt.c 466 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4
5
6
7
8
9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10
11
12
13
14
 */

#include <assert.h>
#include <math.h>

Yaowu Xu's avatar
Yaowu Xu committed
15
#include "./aom_dsp_rtcd.h"
Jingning Han's avatar
Jingning Han committed
16
#include "./av1_rtcd.h"
Jingning Han's avatar
Jingning Han committed
17

Yaowu Xu's avatar
Yaowu Xu committed
18
#include "aom_dsp/aom_dsp_common.h"
19
#include "aom_dsp/blend.h"
Yaowu Xu's avatar
Yaowu Xu committed
20
#include "aom_mem/aom_mem.h"
21
22
#include "aom_ports/mem.h"
#include "aom_ports/system_state.h"
Jingning Han's avatar
Jingning Han committed
23

24
25
26
27
28
29
#include "av1/common/common.h"
#include "av1/common/common_data.h"
#include "av1/common/entropy.h"
#include "av1/common/entropymode.h"
#include "av1/common/idct.h"
#include "av1/common/mvref_common.h"
30
#include "av1/common/obmc.h"
31
32
33
34
35
36
#include "av1/common/pred_common.h"
#include "av1/common/quant_common.h"
#include "av1/common/reconinter.h"
#include "av1/common/reconintra.h"
#include "av1/common/scan.h"
#include "av1/common/seg_common.h"
37
38
39
#if CONFIG_LV_MAP
#include "av1/common/txb_common.h"
#endif
Yue Chen's avatar
Yue Chen committed
40
41
42
#if CONFIG_WARPED_MOTION
#include "av1/common/warped_motion.h"
#endif  // CONFIG_WARPED_MOTION
Jingning Han's avatar
Jingning Han committed
43

Jingning Han's avatar
Jingning Han committed
44
#include "av1/encoder/aq_variance.h"
45
#include "av1/encoder/av1_quantize.h"
46
47
48
49
#include "av1/encoder/cost.h"
#include "av1/encoder/encodemb.h"
#include "av1/encoder/encodemv.h"
#include "av1/encoder/encoder.h"
50
51
52
#if CONFIG_LV_MAP
#include "av1/encoder/encodetxb.h"
#endif
53
54
55
56
57
58
#include "av1/encoder/hybrid_fwd_txfm.h"
#include "av1/encoder/mcomp.h"
#include "av1/encoder/palette.h"
#include "av1/encoder/ratectrl.h"
#include "av1/encoder/rd.h"
#include "av1/encoder/rdopt.h"
59
#include "av1/encoder/tokenize.h"
60
61
#if CONFIG_PVQ
#include "av1/encoder/pvq_encoder.h"
Yushin Cho's avatar
Yushin Cho committed
62
#include "av1/common/pvq.h"
63
#endif  // CONFIG_PVQ
64
#if CONFIG_DUAL_FILTER
Angie Chiang's avatar
Angie Chiang committed
65
#define DUAL_FILTER_SET_SIZE (SWITCHABLE_FILTERS * SWITCHABLE_FILTERS)
66
#if USE_EXTRA_FILTER
Angie Chiang's avatar
Angie Chiang committed
67
static const int filter_sets[DUAL_FILTER_SET_SIZE][2] = {
68
69
70
  { 0, 0 }, { 0, 1 }, { 0, 2 }, { 0, 3 }, { 1, 0 }, { 1, 1 },
  { 1, 2 }, { 1, 3 }, { 2, 0 }, { 2, 1 }, { 2, 2 }, { 2, 3 },
  { 3, 0 }, { 3, 1 }, { 3, 2 }, { 3, 3 },
71
};
72
73
74
75
76
77
#else   // USE_EXTRA_FILTER
static const int filter_sets[DUAL_FILTER_SET_SIZE][2] = {
  { 0, 0 }, { 0, 1 }, { 0, 2 }, { 1, 0 }, { 1, 1 },
  { 1, 2 }, { 2, 0 }, { 2, 1 }, { 2, 2 },
};
#endif  // USE_EXTRA_FILTER
Angie Chiang's avatar
Angie Chiang committed
78
#endif  // CONFIG_DUAL_FILTER
79

80
81
#if CONFIG_EXT_REFS

Zoe Liu's avatar
Zoe Liu committed
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
#if CONFIG_ALTREF2

#define LAST_FRAME_MODE_MASK                                          \
  ((1 << INTRA_FRAME) | (1 << LAST2_FRAME) | (1 << LAST3_FRAME) |     \
   (1 << GOLDEN_FRAME) | (1 << BWDREF_FRAME) | (1 << ALTREF2_FRAME) | \
   (1 << ALTREF_FRAME))
#define LAST2_FRAME_MODE_MASK                                         \
  ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | (1 << LAST3_FRAME) |      \
   (1 << GOLDEN_FRAME) | (1 << BWDREF_FRAME) | (1 << ALTREF2_FRAME) | \
   (1 << ALTREF_FRAME))
#define LAST3_FRAME_MODE_MASK                                         \
  ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | (1 << LAST2_FRAME) |      \
   (1 << GOLDEN_FRAME) | (1 << BWDREF_FRAME) | (1 << ALTREF2_FRAME) | \
   (1 << ALTREF_FRAME))
#define GOLDEN_FRAME_MODE_MASK                                       \
  ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | (1 << LAST2_FRAME) |     \
   (1 << LAST3_FRAME) | (1 << BWDREF_FRAME) | (1 << ALTREF2_FRAME) | \
   (1 << ALTREF_FRAME))
#define BWDREF_FRAME_MODE_MASK                                       \
  ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | (1 << LAST2_FRAME) |     \
   (1 << LAST3_FRAME) | (1 << GOLDEN_FRAME) | (1 << ALTREF2_FRAME) | \
   (1 << ALTREF_FRAME))
#define ALTREF2_FRAME_MODE_MASK                                     \
  ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | (1 << LAST2_FRAME) |    \
   (1 << LAST3_FRAME) | (1 << GOLDEN_FRAME) | (1 << BWDREF_FRAME) | \
   (1 << ALTREF_FRAME))
#define ALTREF_FRAME_MODE_MASK                                      \
  ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | (1 << LAST2_FRAME) |    \
   (1 << LAST3_FRAME) | (1 << GOLDEN_FRAME) | (1 << BWDREF_FRAME) | \
   (1 << ALTREF2_FRAME))

#else  // CONFIG_ALTREF2

115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
#define LAST_FRAME_MODE_MASK                                      \
  ((1 << INTRA_FRAME) | (1 << LAST2_FRAME) | (1 << LAST3_FRAME) | \
   (1 << GOLDEN_FRAME) | (1 << BWDREF_FRAME) | (1 << ALTREF_FRAME))
#define LAST2_FRAME_MODE_MASK                                    \
  ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | (1 << LAST3_FRAME) | \
   (1 << GOLDEN_FRAME) | (1 << BWDREF_FRAME) | (1 << ALTREF_FRAME))
#define LAST3_FRAME_MODE_MASK                                    \
  ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | (1 << LAST2_FRAME) | \
   (1 << GOLDEN_FRAME) | (1 << BWDREF_FRAME) | (1 << ALTREF_FRAME))
#define GOLDEN_FRAME_MODE_MASK                                   \
  ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | (1 << LAST2_FRAME) | \
   (1 << LAST3_FRAME) | (1 << BWDREF_FRAME) | (1 << ALTREF_FRAME))
#define BWDREF_FRAME_MODE_MASK                                   \
  ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | (1 << LAST2_FRAME) | \
   (1 << LAST3_FRAME) | (1 << GOLDEN_FRAME) | (1 << ALTREF_FRAME))
#define ALTREF_FRAME_MODE_MASK                                   \
  ((1 << INTRA_FRAME) | (1 << LAST_FRAME) | (1 << LAST2_FRAME) | \
   (1 << LAST3_FRAME) | (1 << GOLDEN_FRAME) | (1 << BWDREF_FRAME))

Zoe Liu's avatar
Zoe Liu committed
134
135
136
#endif  // CONFIG_ALTREF2

#else  // !CONFIG_EXT_REFS
137
138
139
140
141
142
143

#define LAST_FRAME_MODE_MASK \
  ((1 << GOLDEN_FRAME) | (1 << ALTREF_FRAME) | (1 << INTRA_FRAME))
#define GOLDEN_FRAME_MODE_MASK \
  ((1 << LAST_FRAME) | (1 << ALTREF_FRAME) | (1 << INTRA_FRAME))
#define ALTREF_FRAME_MODE_MASK \
  ((1 << LAST_FRAME) | (1 << GOLDEN_FRAME) | (1 << INTRA_FRAME))
Jingning Han's avatar
Jingning Han committed
144

145
146
#endif  // CONFIG_EXT_REFS

147
#if CONFIG_EXT_REFS
Zoe Liu's avatar
Zoe Liu committed
148
#if CONFIG_EXT_COMP_REFS
149
150
151
152
153
#if CONFIG_ALTREF2
#define SECOND_REF_FRAME_MASK                                         \
  ((1 << ALTREF_FRAME) | (1 << ALTREF2_FRAME) | (1 << BWDREF_FRAME) | \
   (1 << GOLDEN_FRAME) | (1 << LAST2_FRAME) | 0x01)
#else  // !CONFIG_ALTREF2
Zoe Liu's avatar
Zoe Liu committed
154
155
156
#define SECOND_REF_FRAME_MASK                                        \
  ((1 << ALTREF_FRAME) | (1 << BWDREF_FRAME) | (1 << GOLDEN_FRAME) | \
   (1 << LAST2_FRAME) | 0x01)  // NOLINT
157
#endif                         // CONFIG_ALTREF2
Zoe Liu's avatar
Zoe Liu committed
158
#else                          // !CONFIG_EXT_COMP_REFS
Zoe Liu's avatar
Zoe Liu committed
159
160
161
162
#if CONFIG_ALTREF2
#define SECOND_REF_FRAME_MASK \
  ((1 << ALTREF_FRAME) | (1 << ALTREF2_FRAME) | (1 << BWDREF_FRAME) | 0x01)
#else  // !CONFIG_ALTREF2
163
#define SECOND_REF_FRAME_MASK ((1 << ALTREF_FRAME) | (1 << BWDREF_FRAME) | 0x01)
Zoe Liu's avatar
Zoe Liu committed
164
#endif  // CONFIG_ALTREF2
Zoe Liu's avatar
Zoe Liu committed
165
166
#endif  // CONFIG_EXT_COMP_REFS
#else   // !CONFIG_EXT_REFS
167
#define SECOND_REF_FRAME_MASK ((1 << ALTREF_FRAME) | 0x01)
168
#endif  // CONFIG_EXT_REFS
Jingning Han's avatar
Jingning Han committed
169

170
171
#define MIN_EARLY_TERM_INDEX 3
#define NEW_MV_DISCOUNT_FACTOR 8
Jingning Han's avatar
Jingning Han committed
172

173
174
175
176
177
#if CONFIG_EXT_INTRA
#define ANGLE_SKIP_THRESH 10
#define FILTER_FAST_SEARCH 1
#endif  // CONFIG_EXT_INTRA

178
179
180
// Setting this to 1 will disable trellis optimization within the
// transform search. Trellis optimization will still be applied
// in the final encode.
181
#ifndef DISABLE_TRELLISQ_SEARCH
182
#define DISABLE_TRELLISQ_SEARCH 0
183
#endif
184

185
186
187
188
189
190
static const double ADST_FLIP_SVM[8] = {
  /* vertical */
  -6.6623, -2.8062, -3.2531, 3.1671,
  /* horizontal */
  -7.7051, -3.2234, -3.6193, 3.4533
};
191

Jingning Han's avatar
Jingning Han committed
192
193
194
195
196
typedef struct {
  PREDICTION_MODE mode;
  MV_REFERENCE_FRAME ref_frame[2];
} MODE_DEFINITION;

197
typedef struct { MV_REFERENCE_FRAME ref_frame[2]; } REF_DEFINITION;
Jingning Han's avatar
Jingning Han committed
198
199

struct rdcost_block_args {
Yaowu Xu's avatar
Yaowu Xu committed
200
  const AV1_COMP *cpi;
Jingning Han's avatar
Jingning Han committed
201
  MACROBLOCK *x;
202
203
  ENTROPY_CONTEXT t_above[2 * MAX_MIB_SIZE];
  ENTROPY_CONTEXT t_left[2 * MAX_MIB_SIZE];
204
  RD_STATS rd_stats;
Jingning Han's avatar
Jingning Han committed
205
206
207
208
209
210
211
  int64_t this_rd;
  int64_t best_rd;
  int exit_early;
  int use_fast_coef_costing;
};

#define LAST_NEW_MV_INDEX 6
Yaowu Xu's avatar
Yaowu Xu committed
212
static const MODE_DEFINITION av1_mode_order[MAX_MODES] = {
Emil Keyder's avatar
Emil Keyder committed
213
  { NEARESTMV, { LAST_FRAME, NONE_FRAME } },
214
#if CONFIG_EXT_REFS
Emil Keyder's avatar
Emil Keyder committed
215
216
217
  { NEARESTMV, { LAST2_FRAME, NONE_FRAME } },
  { NEARESTMV, { LAST3_FRAME, NONE_FRAME } },
  { NEARESTMV, { BWDREF_FRAME, NONE_FRAME } },
Zoe Liu's avatar
Zoe Liu committed
218
219
220
#if CONFIG_ALTREF2
  { NEARESTMV, { ALTREF2_FRAME, NONE_FRAME } },
#endif  // CONFIG_ALTREF2
221
#endif  // CONFIG_EXT_REFS
Emil Keyder's avatar
Emil Keyder committed
222
223
  { NEARESTMV, { ALTREF_FRAME, NONE_FRAME } },
  { NEARESTMV, { GOLDEN_FRAME, NONE_FRAME } },
Jingning Han's avatar
Jingning Han committed
224

Emil Keyder's avatar
Emil Keyder committed
225
  { DC_PRED, { INTRA_FRAME, NONE_FRAME } },
Jingning Han's avatar
Jingning Han committed
226

Emil Keyder's avatar
Emil Keyder committed
227
  { NEWMV, { LAST_FRAME, NONE_FRAME } },
228
#if CONFIG_EXT_REFS
Emil Keyder's avatar
Emil Keyder committed
229
230
231
  { NEWMV, { LAST2_FRAME, NONE_FRAME } },
  { NEWMV, { LAST3_FRAME, NONE_FRAME } },
  { NEWMV, { BWDREF_FRAME, NONE_FRAME } },
Zoe Liu's avatar
Zoe Liu committed
232
233
234
#if CONFIG_ALTREF2
  { NEWMV, { ALTREF2_FRAME, NONE_FRAME } },
#endif  // CONFIG_ALTREF2
235
#endif  // CONFIG_EXT_REFS
Emil Keyder's avatar
Emil Keyder committed
236
237
  { NEWMV, { ALTREF_FRAME, NONE_FRAME } },
  { NEWMV, { GOLDEN_FRAME, NONE_FRAME } },
Jingning Han's avatar
Jingning Han committed
238

Emil Keyder's avatar
Emil Keyder committed
239
  { NEARMV, { LAST_FRAME, NONE_FRAME } },
240
#if CONFIG_EXT_REFS
Emil Keyder's avatar
Emil Keyder committed
241
242
243
  { NEARMV, { LAST2_FRAME, NONE_FRAME } },
  { NEARMV, { LAST3_FRAME, NONE_FRAME } },
  { NEARMV, { BWDREF_FRAME, NONE_FRAME } },
Zoe Liu's avatar
Zoe Liu committed
244
245
246
#if CONFIG_ALTREF2
  { NEARMV, { ALTREF2_FRAME, NONE_FRAME } },
#endif  // CONFIG_ALTREF2
247
#endif  // CONFIG_EXT_REFS
Emil Keyder's avatar
Emil Keyder committed
248
249
  { NEARMV, { ALTREF_FRAME, NONE_FRAME } },
  { NEARMV, { GOLDEN_FRAME, NONE_FRAME } },
Jingning Han's avatar
Jingning Han committed
250

Emil Keyder's avatar
Emil Keyder committed
251
  { ZEROMV, { LAST_FRAME, NONE_FRAME } },
252
#if CONFIG_EXT_REFS
Emil Keyder's avatar
Emil Keyder committed
253
254
255
  { ZEROMV, { LAST2_FRAME, NONE_FRAME } },
  { ZEROMV, { LAST3_FRAME, NONE_FRAME } },
  { ZEROMV, { BWDREF_FRAME, NONE_FRAME } },
Zoe Liu's avatar
Zoe Liu committed
256
257
258
#if CONFIG_ALTREF2
  { ZEROMV, { ALTREF2_FRAME, NONE_FRAME } },
#endif  // CONFIG_ALTREF2
259
#endif  // CONFIG_EXT_REFS
Emil Keyder's avatar
Emil Keyder committed
260
261
  { ZEROMV, { GOLDEN_FRAME, NONE_FRAME } },
  { ZEROMV, { ALTREF_FRAME, NONE_FRAME } },
Jingning Han's avatar
Jingning Han committed
262

263
// TODO(zoeliu): May need to reconsider the order on the modes to check
264

265
#if CONFIG_EXT_INTER
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315

#if CONFIG_COMPOUND_SINGLEREF
  // Single ref comp mode
  { SR_NEAREST_NEARMV, { LAST_FRAME, NONE_FRAME } },
#if CONFIG_EXT_REFS
  { SR_NEAREST_NEARMV, { LAST2_FRAME, NONE_FRAME } },
  { SR_NEAREST_NEARMV, { LAST3_FRAME, NONE_FRAME } },
  { SR_NEAREST_NEARMV, { BWDREF_FRAME, NONE_FRAME } },
#endif  // CONFIG_EXT_REFS
  { SR_NEAREST_NEARMV, { GOLDEN_FRAME, NONE_FRAME } },
  { SR_NEAREST_NEARMV, { ALTREF_FRAME, NONE_FRAME } },

  /*
  { SR_NEAREST_NEWMV, { LAST_FRAME, NONE_FRAME } },
#if CONFIG_EXT_REFS
  { SR_NEAREST_NEWMV, { LAST2_FRAME, NONE_FRAME } },
  { SR_NEAREST_NEWMV, { LAST3_FRAME, NONE_FRAME } },
  { SR_NEAREST_NEWMV, { BWDREF_FRAME, NONE_FRAME } },
#endif  // CONFIG_EXT_REFS
  { SR_NEAREST_NEWMV, { GOLDEN_FRAME, NONE_FRAME } },
  { SR_NEAREST_NEWMV, { ALTREF_FRAME, NONE_FRAME } },*/

  { SR_NEAR_NEWMV, { LAST_FRAME, NONE_FRAME } },
#if CONFIG_EXT_REFS
  { SR_NEAR_NEWMV, { LAST2_FRAME, NONE_FRAME } },
  { SR_NEAR_NEWMV, { LAST3_FRAME, NONE_FRAME } },
  { SR_NEAR_NEWMV, { BWDREF_FRAME, NONE_FRAME } },
#endif  // CONFIG_EXT_REFS
  { SR_NEAR_NEWMV, { GOLDEN_FRAME, NONE_FRAME } },
  { SR_NEAR_NEWMV, { ALTREF_FRAME, NONE_FRAME } },

  { SR_ZERO_NEWMV, { LAST_FRAME, NONE_FRAME } },
#if CONFIG_EXT_REFS
  { SR_ZERO_NEWMV, { LAST2_FRAME, NONE_FRAME } },
  { SR_ZERO_NEWMV, { LAST3_FRAME, NONE_FRAME } },
  { SR_ZERO_NEWMV, { BWDREF_FRAME, NONE_FRAME } },
#endif  // CONFIG_EXT_REFS
  { SR_ZERO_NEWMV, { GOLDEN_FRAME, NONE_FRAME } },
  { SR_ZERO_NEWMV, { ALTREF_FRAME, NONE_FRAME } },

  { SR_NEW_NEWMV, { LAST_FRAME, NONE_FRAME } },
#if CONFIG_EXT_REFS
  { SR_NEW_NEWMV, { LAST2_FRAME, NONE_FRAME } },
  { SR_NEW_NEWMV, { LAST3_FRAME, NONE_FRAME } },
  { SR_NEW_NEWMV, { BWDREF_FRAME, NONE_FRAME } },
#endif  // CONFIG_EXT_REFS
  { SR_NEW_NEWMV, { GOLDEN_FRAME, NONE_FRAME } },
  { SR_NEW_NEWMV, { ALTREF_FRAME, NONE_FRAME } },
#endif  // CONFIG_COMPOUND_SINGLEREF

316
  { NEAREST_NEARESTMV, { LAST_FRAME, ALTREF_FRAME } },
317
#if CONFIG_EXT_REFS
318
319
  { NEAREST_NEARESTMV, { LAST2_FRAME, ALTREF_FRAME } },
  { NEAREST_NEARESTMV, { LAST3_FRAME, ALTREF_FRAME } },
320
#endif  // CONFIG_EXT_REFS
321
  { NEAREST_NEARESTMV, { GOLDEN_FRAME, ALTREF_FRAME } },
322
#if CONFIG_EXT_REFS
323
324
325
326
  { NEAREST_NEARESTMV, { LAST_FRAME, BWDREF_FRAME } },
  { NEAREST_NEARESTMV, { LAST2_FRAME, BWDREF_FRAME } },
  { NEAREST_NEARESTMV, { LAST3_FRAME, BWDREF_FRAME } },
  { NEAREST_NEARESTMV, { GOLDEN_FRAME, BWDREF_FRAME } },
Zoe Liu's avatar
Zoe Liu committed
327
328
329
330
331
332
#if CONFIG_ALTREF2
  { NEAREST_NEARESTMV, { LAST_FRAME, ALTREF2_FRAME } },
  { NEAREST_NEARESTMV, { LAST2_FRAME, ALTREF2_FRAME } },
  { NEAREST_NEARESTMV, { LAST3_FRAME, ALTREF2_FRAME } },
  { NEAREST_NEARESTMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
#endif  // CONFIG_ALTREF2
Zoe Liu's avatar
Zoe Liu committed
333
334
335

#if CONFIG_EXT_COMP_REFS
  { NEAREST_NEARESTMV, { LAST_FRAME, LAST2_FRAME } },
336
  { NEAREST_NEARESTMV, { LAST_FRAME, LAST3_FRAME } },
Zoe Liu's avatar
Zoe Liu committed
337
338
339
  { NEAREST_NEARESTMV, { LAST_FRAME, GOLDEN_FRAME } },
  { NEAREST_NEARESTMV, { BWDREF_FRAME, ALTREF_FRAME } },
#endif  // CONFIG_EXT_COMP_REFS
340
#endif  // CONFIG_EXT_REFS
341

342
#else  // CONFIG_EXT_INTER
343

344
  { NEARESTMV, { LAST_FRAME, ALTREF_FRAME } },
345
#if CONFIG_EXT_REFS
346
347
  { NEARESTMV, { LAST2_FRAME, ALTREF_FRAME } },
  { NEARESTMV, { LAST3_FRAME, ALTREF_FRAME } },
348
#endif  // CONFIG_EXT_REFS
349
  { NEARESTMV, { GOLDEN_FRAME, ALTREF_FRAME } },
350
#if CONFIG_EXT_REFS
351
352
353
354
  { NEARESTMV, { LAST_FRAME, BWDREF_FRAME } },
  { NEARESTMV, { LAST2_FRAME, BWDREF_FRAME } },
  { NEARESTMV, { LAST3_FRAME, BWDREF_FRAME } },
  { NEARESTMV, { GOLDEN_FRAME, BWDREF_FRAME } },
Zoe Liu's avatar
Zoe Liu committed
355
356
357
358
359
360
#if CONFIG_ALTREF2
  { NEARESTMV, { LAST_FRAME, ALTREF2_FRAME } },
  { NEARESTMV, { LAST2_FRAME, ALTREF2_FRAME } },
  { NEARESTMV, { LAST3_FRAME, ALTREF2_FRAME } },
  { NEARESTMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
#endif  // CONFIG_ALTREF2
Zoe Liu's avatar
Zoe Liu committed
361
362
363

#if CONFIG_EXT_COMP_REFS
  { NEARESTMV, { LAST_FRAME, LAST2_FRAME } },
364
  { NEARESTMV, { LAST_FRAME, LAST3_FRAME } },
Zoe Liu's avatar
Zoe Liu committed
365
366
367
  { NEARESTMV, { LAST_FRAME, GOLDEN_FRAME } },
  { NEARESTMV, { BWDREF_FRAME, ALTREF_FRAME } },
#endif  // CONFIG_EXT_COMP_REFS
368
#endif  // CONFIG_EXT_REFS
369
#endif  // CONFIG_EXT_INTER
Jingning Han's avatar
Jingning Han committed
370

Emil Keyder's avatar
Emil Keyder committed
371
  { TM_PRED, { INTRA_FRAME, NONE_FRAME } },
Jingning Han's avatar
Jingning Han committed
372

Emil Keyder's avatar
Emil Keyder committed
373
  { SMOOTH_PRED, { INTRA_FRAME, NONE_FRAME } },
Urvang Joshi's avatar
Urvang Joshi committed
374
375
376
377
#if CONFIG_SMOOTH_HV
  { SMOOTH_V_PRED, { INTRA_FRAME, NONE_FRAME } },
  { SMOOTH_H_PRED, { INTRA_FRAME, NONE_FRAME } },
#endif  // CONFIG_SMOOTH_HV
378

379
#if CONFIG_EXT_INTER
380
381
382
383
384
385
386
  { NEAR_NEARMV, { LAST_FRAME, ALTREF_FRAME } },
  { NEW_NEARESTMV, { LAST_FRAME, ALTREF_FRAME } },
  { NEAREST_NEWMV, { LAST_FRAME, ALTREF_FRAME } },
  { NEW_NEARMV, { LAST_FRAME, ALTREF_FRAME } },
  { NEAR_NEWMV, { LAST_FRAME, ALTREF_FRAME } },
  { NEW_NEWMV, { LAST_FRAME, ALTREF_FRAME } },
  { ZERO_ZEROMV, { LAST_FRAME, ALTREF_FRAME } },
387

388
#if CONFIG_EXT_REFS
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
  { NEAR_NEARMV, { LAST2_FRAME, ALTREF_FRAME } },
  { NEW_NEARESTMV, { LAST2_FRAME, ALTREF_FRAME } },
  { NEAREST_NEWMV, { LAST2_FRAME, ALTREF_FRAME } },
  { NEW_NEARMV, { LAST2_FRAME, ALTREF_FRAME } },
  { NEAR_NEWMV, { LAST2_FRAME, ALTREF_FRAME } },
  { NEW_NEWMV, { LAST2_FRAME, ALTREF_FRAME } },
  { ZERO_ZEROMV, { LAST2_FRAME, ALTREF_FRAME } },

  { NEAR_NEARMV, { LAST3_FRAME, ALTREF_FRAME } },
  { NEW_NEARESTMV, { LAST3_FRAME, ALTREF_FRAME } },
  { NEAREST_NEWMV, { LAST3_FRAME, ALTREF_FRAME } },
  { NEW_NEARMV, { LAST3_FRAME, ALTREF_FRAME } },
  { NEAR_NEWMV, { LAST3_FRAME, ALTREF_FRAME } },
  { NEW_NEWMV, { LAST3_FRAME, ALTREF_FRAME } },
  { ZERO_ZEROMV, { LAST3_FRAME, ALTREF_FRAME } },
404
#endif  // CONFIG_EXT_REFS
405

406
407
408
409
410
411
412
  { NEAR_NEARMV, { GOLDEN_FRAME, ALTREF_FRAME } },
  { NEW_NEARESTMV, { GOLDEN_FRAME, ALTREF_FRAME } },
  { NEAREST_NEWMV, { GOLDEN_FRAME, ALTREF_FRAME } },
  { NEW_NEARMV, { GOLDEN_FRAME, ALTREF_FRAME } },
  { NEAR_NEWMV, { GOLDEN_FRAME, ALTREF_FRAME } },
  { NEW_NEWMV, { GOLDEN_FRAME, ALTREF_FRAME } },
  { ZERO_ZEROMV, { GOLDEN_FRAME, ALTREF_FRAME } },
413
414

#if CONFIG_EXT_REFS
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
  { NEAR_NEARMV, { LAST_FRAME, BWDREF_FRAME } },
  { NEW_NEARESTMV, { LAST_FRAME, BWDREF_FRAME } },
  { NEAREST_NEWMV, { LAST_FRAME, BWDREF_FRAME } },
  { NEW_NEARMV, { LAST_FRAME, BWDREF_FRAME } },
  { NEAR_NEWMV, { LAST_FRAME, BWDREF_FRAME } },
  { NEW_NEWMV, { LAST_FRAME, BWDREF_FRAME } },
  { ZERO_ZEROMV, { LAST_FRAME, BWDREF_FRAME } },

  { NEAR_NEARMV, { LAST2_FRAME, BWDREF_FRAME } },
  { NEW_NEARESTMV, { LAST2_FRAME, BWDREF_FRAME } },
  { NEAREST_NEWMV, { LAST2_FRAME, BWDREF_FRAME } },
  { NEW_NEARMV, { LAST2_FRAME, BWDREF_FRAME } },
  { NEAR_NEWMV, { LAST2_FRAME, BWDREF_FRAME } },
  { NEW_NEWMV, { LAST2_FRAME, BWDREF_FRAME } },
  { ZERO_ZEROMV, { LAST2_FRAME, BWDREF_FRAME } },

  { NEAR_NEARMV, { LAST3_FRAME, BWDREF_FRAME } },
  { NEW_NEARESTMV, { LAST3_FRAME, BWDREF_FRAME } },
  { NEAREST_NEWMV, { LAST3_FRAME, BWDREF_FRAME } },
  { NEW_NEARMV, { LAST3_FRAME, BWDREF_FRAME } },
  { NEAR_NEWMV, { LAST3_FRAME, BWDREF_FRAME } },
  { NEW_NEWMV, { LAST3_FRAME, BWDREF_FRAME } },
  { ZERO_ZEROMV, { LAST3_FRAME, BWDREF_FRAME } },

  { NEAR_NEARMV, { GOLDEN_FRAME, BWDREF_FRAME } },
  { NEW_NEARESTMV, { GOLDEN_FRAME, BWDREF_FRAME } },
  { NEAREST_NEWMV, { GOLDEN_FRAME, BWDREF_FRAME } },
  { NEW_NEARMV, { GOLDEN_FRAME, BWDREF_FRAME } },
  { NEAR_NEWMV, { GOLDEN_FRAME, BWDREF_FRAME } },
  { NEW_NEWMV, { GOLDEN_FRAME, BWDREF_FRAME } },
  { ZERO_ZEROMV, { GOLDEN_FRAME, BWDREF_FRAME } },
Zoe Liu's avatar
Zoe Liu committed
446

Zoe Liu's avatar
Zoe Liu committed
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
#if CONFIG_ALTREF2
  { NEAR_NEARMV, { LAST_FRAME, ALTREF2_FRAME } },
  { NEW_NEARESTMV, { LAST_FRAME, ALTREF2_FRAME } },
  { NEAREST_NEWMV, { LAST_FRAME, ALTREF2_FRAME } },
  { NEW_NEARMV, { LAST_FRAME, ALTREF2_FRAME } },
  { NEAR_NEWMV, { LAST_FRAME, ALTREF2_FRAME } },
  { NEW_NEWMV, { LAST_FRAME, ALTREF2_FRAME } },
  { ZERO_ZEROMV, { LAST_FRAME, ALTREF2_FRAME } },

  { NEAR_NEARMV, { LAST2_FRAME, ALTREF2_FRAME } },
  { NEW_NEARESTMV, { LAST2_FRAME, ALTREF2_FRAME } },
  { NEAREST_NEWMV, { LAST2_FRAME, ALTREF2_FRAME } },
  { NEW_NEARMV, { LAST2_FRAME, ALTREF2_FRAME } },
  { NEAR_NEWMV, { LAST2_FRAME, ALTREF2_FRAME } },
  { NEW_NEWMV, { LAST2_FRAME, ALTREF2_FRAME } },
  { ZERO_ZEROMV, { LAST2_FRAME, ALTREF2_FRAME } },

  { NEAR_NEARMV, { LAST3_FRAME, ALTREF2_FRAME } },
  { NEW_NEARESTMV, { LAST3_FRAME, ALTREF2_FRAME } },
  { NEAREST_NEWMV, { LAST3_FRAME, ALTREF2_FRAME } },
  { NEW_NEARMV, { LAST3_FRAME, ALTREF2_FRAME } },
  { NEAR_NEWMV, { LAST3_FRAME, ALTREF2_FRAME } },
  { NEW_NEWMV, { LAST3_FRAME, ALTREF2_FRAME } },
  { ZERO_ZEROMV, { LAST3_FRAME, ALTREF2_FRAME } },

  { NEAR_NEARMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
  { NEW_NEARESTMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
  { NEAREST_NEWMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
  { NEW_NEARMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
  { NEAR_NEWMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
  { NEW_NEWMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
  { ZERO_ZEROMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
#endif  // CONFIG_ALTREF2

Zoe Liu's avatar
Zoe Liu committed
481
482
483
484
485
486
487
488
489
#if CONFIG_EXT_COMP_REFS
  { NEAR_NEARMV, { LAST_FRAME, LAST2_FRAME } },
  { NEW_NEARESTMV, { LAST_FRAME, LAST2_FRAME } },
  { NEAREST_NEWMV, { LAST_FRAME, LAST2_FRAME } },
  { NEW_NEARMV, { LAST_FRAME, LAST2_FRAME } },
  { NEAR_NEWMV, { LAST_FRAME, LAST2_FRAME } },
  { NEW_NEWMV, { LAST_FRAME, LAST2_FRAME } },
  { ZERO_ZEROMV, { LAST_FRAME, LAST2_FRAME } },

490
491
492
493
494
495
496
497
  { NEAR_NEARMV, { LAST_FRAME, LAST3_FRAME } },
  { NEW_NEARESTMV, { LAST_FRAME, LAST3_FRAME } },
  { NEAREST_NEWMV, { LAST_FRAME, LAST3_FRAME } },
  { NEW_NEARMV, { LAST_FRAME, LAST3_FRAME } },
  { NEAR_NEWMV, { LAST_FRAME, LAST3_FRAME } },
  { NEW_NEWMV, { LAST_FRAME, LAST3_FRAME } },
  { ZERO_ZEROMV, { LAST_FRAME, LAST3_FRAME } },

Zoe Liu's avatar
Zoe Liu committed
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
  { NEAR_NEARMV, { LAST_FRAME, GOLDEN_FRAME } },
  { NEW_NEARESTMV, { LAST_FRAME, GOLDEN_FRAME } },
  { NEAREST_NEWMV, { LAST_FRAME, GOLDEN_FRAME } },
  { NEW_NEARMV, { LAST_FRAME, GOLDEN_FRAME } },
  { NEAR_NEWMV, { LAST_FRAME, GOLDEN_FRAME } },
  { NEW_NEWMV, { LAST_FRAME, GOLDEN_FRAME } },
  { ZERO_ZEROMV, { LAST_FRAME, GOLDEN_FRAME } },

  { NEAR_NEARMV, { BWDREF_FRAME, ALTREF_FRAME } },
  { NEW_NEARESTMV, { BWDREF_FRAME, ALTREF_FRAME } },
  { NEAREST_NEWMV, { BWDREF_FRAME, ALTREF_FRAME } },
  { NEW_NEARMV, { BWDREF_FRAME, ALTREF_FRAME } },
  { NEAR_NEWMV, { BWDREF_FRAME, ALTREF_FRAME } },
  { NEW_NEWMV, { BWDREF_FRAME, ALTREF_FRAME } },
  { ZERO_ZEROMV, { BWDREF_FRAME, ALTREF_FRAME } },
#endif  // CONFIG_EXT_COMP_REFS
514
#endif  // CONFIG_EXT_REFS
515

Zoe Liu's avatar
Zoe Liu committed
516
#else  // !CONFIG_EXT_INTER
517

518
519
  { NEARMV, { LAST_FRAME, ALTREF_FRAME } },
  { NEWMV, { LAST_FRAME, ALTREF_FRAME } },
520
#if CONFIG_EXT_REFS
521
522
523
524
  { NEARMV, { LAST2_FRAME, ALTREF_FRAME } },
  { NEWMV, { LAST2_FRAME, ALTREF_FRAME } },
  { NEARMV, { LAST3_FRAME, ALTREF_FRAME } },
  { NEWMV, { LAST3_FRAME, ALTREF_FRAME } },
525
#endif  // CONFIG_EXT_REFS
526
527
  { NEARMV, { GOLDEN_FRAME, ALTREF_FRAME } },
  { NEWMV, { GOLDEN_FRAME, ALTREF_FRAME } },
528
529

#if CONFIG_EXT_REFS
530
531
532
533
534
535
536
537
  { NEARMV, { LAST_FRAME, BWDREF_FRAME } },
  { NEWMV, { LAST_FRAME, BWDREF_FRAME } },
  { NEARMV, { LAST2_FRAME, BWDREF_FRAME } },
  { NEWMV, { LAST2_FRAME, BWDREF_FRAME } },
  { NEARMV, { LAST3_FRAME, BWDREF_FRAME } },
  { NEWMV, { LAST3_FRAME, BWDREF_FRAME } },
  { NEARMV, { GOLDEN_FRAME, BWDREF_FRAME } },
  { NEWMV, { GOLDEN_FRAME, BWDREF_FRAME } },
Zoe Liu's avatar
Zoe Liu committed
538

Zoe Liu's avatar
Zoe Liu committed
539
540
541
542
543
544
545
546
547
548
549
#if CONFIG_ALTREF2
  { NEARMV, { LAST_FRAME, ALTREF2_FRAME } },
  { NEWMV, { LAST_FRAME, ALTREF2_FRAME } },
  { NEARMV, { LAST2_FRAME, ALTREF2_FRAME } },
  { NEWMV, { LAST2_FRAME, ALTREF2_FRAME } },
  { NEARMV, { LAST3_FRAME, ALTREF2_FRAME } },
  { NEWMV, { LAST3_FRAME, ALTREF2_FRAME } },
  { NEARMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
  { NEWMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
#endif  // CONFIG_ALTREF2

Zoe Liu's avatar
Zoe Liu committed
550
551
552
#if CONFIG_EXT_COMP_REFS
  { NEARMV, { LAST_FRAME, LAST2_FRAME } },
  { NEWMV, { LAST_FRAME, LAST2_FRAME } },
553
554
  { NEARMV, { LAST_FRAME, LAST3_FRAME } },
  { NEWMV, { LAST_FRAME, LAST3_FRAME } },
Zoe Liu's avatar
Zoe Liu committed
555
556
557
558
559
  { NEARMV, { LAST_FRAME, GOLDEN_FRAME } },
  { NEWMV, { LAST_FRAME, GOLDEN_FRAME } },
  { NEARMV, { BWDREF_FRAME, ALTREF_FRAME } },
  { NEWMV, { BWDREF_FRAME, ALTREF_FRAME } },
#endif  // CONFIG_EXT_COMP_REFS
560
#endif  // CONFIG_EXT_REFS
Jingning Han's avatar
Jingning Han committed
561

562
  { ZEROMV, { LAST_FRAME, ALTREF_FRAME } },
563
#if CONFIG_EXT_REFS
564
565
  { ZEROMV, { LAST2_FRAME, ALTREF_FRAME } },
  { ZEROMV, { LAST3_FRAME, ALTREF_FRAME } },
566
#endif  // CONFIG_EXT_REFS
567
  { ZEROMV, { GOLDEN_FRAME, ALTREF_FRAME } },
568
569

#if CONFIG_EXT_REFS
570
571
572
573
  { ZEROMV, { LAST_FRAME, BWDREF_FRAME } },
  { ZEROMV, { LAST2_FRAME, BWDREF_FRAME } },
  { ZEROMV, { LAST3_FRAME, BWDREF_FRAME } },
  { ZEROMV, { GOLDEN_FRAME, BWDREF_FRAME } },
Zoe Liu's avatar
Zoe Liu committed
574

Zoe Liu's avatar
Zoe Liu committed
575
576
577
578
579
580
581
#if CONFIG_ALTREF2
  { ZEROMV, { LAST_FRAME, ALTREF2_FRAME } },
  { ZEROMV, { LAST2_FRAME, ALTREF2_FRAME } },
  { ZEROMV, { LAST3_FRAME, ALTREF2_FRAME } },
  { ZEROMV, { GOLDEN_FRAME, ALTREF2_FRAME } },
#endif  // CONFIG_ALTREF2

Zoe Liu's avatar
Zoe Liu committed
582
583
#if CONFIG_EXT_COMP_REFS
  { ZEROMV, { LAST_FRAME, LAST2_FRAME } },
584
  { ZEROMV, { LAST_FRAME, LAST3_FRAME } },
Zoe Liu's avatar
Zoe Liu committed
585
586
587
  { ZEROMV, { LAST_FRAME, GOLDEN_FRAME } },
  { ZEROMV, { BWDREF_FRAME, ALTREF_FRAME } },
#endif  // CONFIG_EXT_COMP_REFS
588
#endif  // CONFIG_EXT_REFS
589

590
#endif  // CONFIG_EXT_INTER
Jingning Han's avatar
Jingning Han committed
591

Emil Keyder's avatar
Emil Keyder committed
592
593
594
595
596
597
598
599
  { H_PRED, { INTRA_FRAME, NONE_FRAME } },
  { V_PRED, { INTRA_FRAME, NONE_FRAME } },
  { D135_PRED, { INTRA_FRAME, NONE_FRAME } },
  { D207_PRED, { INTRA_FRAME, NONE_FRAME } },
  { D153_PRED, { INTRA_FRAME, NONE_FRAME } },
  { D63_PRED, { INTRA_FRAME, NONE_FRAME } },
  { D117_PRED, { INTRA_FRAME, NONE_FRAME } },
  { D45_PRED, { INTRA_FRAME, NONE_FRAME } },
600
601

#if CONFIG_EXT_INTER
602
603
604
605
  { ZEROMV, { LAST_FRAME, INTRA_FRAME } },
  { NEARESTMV, { LAST_FRAME, INTRA_FRAME } },
  { NEARMV, { LAST_FRAME, INTRA_FRAME } },
  { NEWMV, { LAST_FRAME, INTRA_FRAME } },
606
607

#if CONFIG_EXT_REFS
608
609
610
611
612
613
614
615
616
  { ZEROMV, { LAST2_FRAME, INTRA_FRAME } },
  { NEARESTMV, { LAST2_FRAME, INTRA_FRAME } },
  { NEARMV, { LAST2_FRAME, INTRA_FRAME } },
  { NEWMV, { LAST2_FRAME, INTRA_FRAME } },

  { ZEROMV, { LAST3_FRAME, INTRA_FRAME } },
  { NEARESTMV, { LAST3_FRAME, INTRA_FRAME } },
  { NEARMV, { LAST3_FRAME, INTRA_FRAME } },
  { NEWMV, { LAST3_FRAME, INTRA_FRAME } },
617
618
#endif  // CONFIG_EXT_REFS

619
620
621
622
  { ZEROMV, { GOLDEN_FRAME, INTRA_FRAME } },
  { NEARESTMV, { GOLDEN_FRAME, INTRA_FRAME } },
  { NEARMV, { GOLDEN_FRAME, INTRA_FRAME } },
  { NEWMV, { GOLDEN_FRAME, INTRA_FRAME } },
623

624
#if CONFIG_EXT_REFS
625
626
627
628
  { ZEROMV, { BWDREF_FRAME, INTRA_FRAME } },
  { NEARESTMV, { BWDREF_FRAME, INTRA_FRAME } },
  { NEARMV, { BWDREF_FRAME, INTRA_FRAME } },
  { NEWMV, { BWDREF_FRAME, INTRA_FRAME } },
Zoe Liu's avatar
Zoe Liu committed
629
630
631
632
633
634
635

#if CONFIG_ALTREF2
  { ZEROMV, { ALTREF2_FRAME, INTRA_FRAME } },
  { NEARESTMV, { ALTREF2_FRAME, INTRA_FRAME } },
  { NEARMV, { ALTREF2_FRAME, INTRA_FRAME } },
  { NEWMV, { ALTREF2_FRAME, INTRA_FRAME } },
#endif  // CONFIG_ALTREF2
636
637
#endif  // CONFIG_EXT_REFS

638
639
640
641
  { ZEROMV, { ALTREF_FRAME, INTRA_FRAME } },
  { NEARESTMV, { ALTREF_FRAME, INTRA_FRAME } },
  { NEARMV, { ALTREF_FRAME, INTRA_FRAME } },
  { NEWMV, { ALTREF_FRAME, INTRA_FRAME } },
642
#endif  // CONFIG_EXT_INTER
Jingning Han's avatar
Jingning Han committed
643
644
};

hui su's avatar
hui su committed
645
static const PREDICTION_MODE intra_rd_search_mode_order[INTRA_MODES] = {
Urvang Joshi's avatar
Urvang Joshi committed
646
647
  DC_PRED,       H_PRED,        V_PRED,    SMOOTH_PRED, TM_PRED,
#if CONFIG_SMOOTH_HV
hui su's avatar
hui su committed
648
  SMOOTH_V_PRED, SMOOTH_H_PRED,
Urvang Joshi's avatar
Urvang Joshi committed
649
650
#endif  // CONFIG_SMOOTH_HV
  D135_PRED,     D207_PRED,     D153_PRED, D63_PRED,    D117_PRED, D45_PRED,
hui su's avatar
hui su committed
651
652
};

Luc Trudeau's avatar
Luc Trudeau committed
653
654
#if CONFIG_CFL
static const UV_PREDICTION_MODE uv_rd_search_mode_order[UV_INTRA_MODES] = {
Urvang Joshi's avatar
Urvang Joshi committed
655
656
657
  UV_DC_PRED,       UV_CFL_PRED,      UV_H_PRED,
  UV_V_PRED,        UV_SMOOTH_PRED,   UV_TM_PRED,
#if CONFIG_SMOOTH_HV
Luc Trudeau's avatar
Luc Trudeau committed
658
  UV_SMOOTH_V_PRED, UV_SMOOTH_H_PRED,
Urvang Joshi's avatar
Urvang Joshi committed
659
660
661
#endif  // CONFIG_SMOOTH_HV
  UV_D135_PRED,     UV_D207_PRED,     UV_D153_PRED,
  UV_D63_PRED,      UV_D117_PRED,     UV_D45_PRED,
Luc Trudeau's avatar
Luc Trudeau committed
662
663
664
665
666
};
#else
#define uv_rd_search_mode_order intra_rd_search_mode_order
#endif  // CONFIG_CFL

hui su's avatar
hui su committed
667
static INLINE int write_uniform_cost(int n, int v) {
668
669
  const int l = get_unsigned_bits(n);
  const int m = (1 << l) - n;
670
  if (l == 0) return 0;
hui su's avatar
hui su committed
671
  if (v < m)
Yaowu Xu's avatar
Yaowu Xu committed
672
    return (l - 1) * av1_cost_bit(128, 0);
hui su's avatar
hui su committed
673
  else
Yaowu Xu's avatar
Yaowu Xu committed
674
    return l * av1_cost_bit(128, 0);
hui su's avatar
hui su committed
675
676
}

677
678
679
// constants for prune 1 and prune 2 decision boundaries
#define FAST_EXT_TX_CORR_MID 0.0
#define FAST_EXT_TX_EDST_MID 0.1
680
681
682
#define FAST_EXT_TX_CORR_MARGIN 0.5
#define FAST_EXT_TX_EDST_MARGIN 0.3

683
#if CONFIG_DIST_8X8
Yushin Cho's avatar
Yushin Cho committed
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
static uint64_t cdef_dist_8x8_16bit(uint16_t *dst, int dstride, uint16_t *src,
                                    int sstride, int coeff_shift) {
  uint64_t svar = 0;
  uint64_t dvar = 0;
  uint64_t sum_s = 0;
  uint64_t sum_d = 0;
  uint64_t sum_s2 = 0;
  uint64_t sum_d2 = 0;
  uint64_t sum_sd = 0;
  uint64_t dist = 0;

  int i, j;
  for (i = 0; i < 8; i++) {
    for (j = 0; j < 8; j++) {
      sum_s += src[i * sstride + j];
      sum_d += dst[i * dstride + j];
      sum_s2 += src[i * sstride + j] * src[i * sstride + j];
      sum_d2 += dst[i * dstride + j] * dst[i * dstride + j];
      sum_sd += src[i * sstride + j] * dst[i * dstride + j];
    }
  }
  /* Compute the variance -- the calculation cannot go negative. */
  svar = sum_s2 - ((sum_s * sum_s + 32) >> 6);
  dvar = sum_d2 - ((sum_d * sum_d + 32) >> 6);

  // Tuning of jm's original dering distortion metric used in CDEF tool,
  // suggested by jm
  const uint64_t a = 4;
  const uint64_t b = 2;
  const uint64_t c1 = (400 * a << 2 * coeff_shift);
  const uint64_t c2 = (b * 20000 * a * a << 4 * coeff_shift);

  dist =
      (uint64_t)floor(.5 +
                      (sum_d2 + sum_s2 - 2 * sum_sd) * .5 * (svar + dvar + c1) /
                          (sqrt(svar * (double)dvar + c2)));

  // Calibrate dist to have similar rate for the same QP with MSE only
  // distortion (as in master branch)
  dist = (uint64_t)((float)dist * 0.75);

  return dist;
}

Yushin Cho's avatar
Yushin Cho committed
728
static int od_compute_var_4x4(uint16_t *x, int stride) {
Yushin Cho's avatar
Yushin Cho committed
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
  int sum;
  int s2;
  int i;
  sum = 0;
  s2 = 0;
  for (i = 0; i < 4; i++) {
    int j;
    for (j = 0; j < 4; j++) {
      int t;

      t = x[i * stride + j];
      sum += t;
      s2 += t * t;
    }
  }
Yushin Cho's avatar
Yushin Cho committed
744

Yushin Cho's avatar
Yushin Cho committed
745
746
747
  return (s2 - (sum * sum >> 4)) >> 4;
}

748
749
750
751
752
753
754
/* OD_DIST_LP_MID controls the frequency weighting filter used for computing
   the distortion. For a value X, the filter is [1 X 1]/(X + 2) and
   is applied both horizontally and vertically. For X=5, the filter is
   a good approximation for the OD_QM8_Q4_HVS quantization matrix. */
#define OD_DIST_LP_MID (5)
#define OD_DIST_LP_NORM (OD_DIST_LP_MID + 2)

Yushin Cho's avatar
Yushin Cho committed
755
756
static double od_compute_dist_8x8(int use_activity_masking, uint16_t *x,
                                  uint16_t *y, od_coeff *e_lp, int stride) {
Yushin Cho's avatar
Yushin Cho committed
757
758
759
760
761
762
763
764
765
766
767
  double sum;
  int min_var;
  double mean_var;
  double var_stat;
  double activity;
  double calibration;
  int i;
  int j;
  double vardist;

  vardist = 0;
Yushin Cho's avatar
Yushin Cho committed
768

Yushin Cho's avatar
Yushin Cho committed
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
#if 1
  min_var = INT_MAX;
  mean_var = 0;
  for (i = 0; i < 3; i++) {
    for (j = 0; j < 3; j++) {
      int varx;
      int vary;
      varx = od_compute_var_4x4(x + 2 * i * stride + 2 * j, stride);
      vary = od_compute_var_4x4(y + 2 * i * stride + 2 * j, stride);
      min_var = OD_MINI(min_var, varx);
      mean_var += 1. / (1 + varx);
      /* The cast to (double) is to avoid an overflow before the sqrt.*/
      vardist += varx - 2 * sqrt(varx * (double)vary) + vary;
    }
  }
  /* We use a different variance statistic depending on whether activity
James Zern's avatar
James Zern committed
785
     masking is used, since the harmonic mean appeared slightly worse with
Yushin Cho's avatar
Yushin Cho committed
786
787
788
789
790
791
792
793
794
795
796
797
798
799
     masking off. The calibration constant just ensures that we preserve the
     rate compared to activity=1. */
  if (use_activity_masking) {
    calibration = 1.95;
    var_stat = 9. / mean_var;
  } else {
    calibration = 1.62;
    var_stat = min_var;
  }
  /* 1.62 is a calibration constant, 0.25 is a noise floor and 1/6 is the
     activity masking constant. */
  activity = calibration * pow(.25 + var_stat, -1. / 6);
#else
  activity = 1;
800
#endif  // 1
Yushin Cho's avatar
Yushin Cho committed
801
802
803
  sum = 0;
  for (i = 0; i < 8; i++) {
    for (j = 0; j < 8; j++)
804
      sum += e_lp[i * stride + j] * (double)e_lp[i * stride + j];
Yushin Cho's avatar
Yushin Cho committed
805
  }
806
807
808
  /* Normalize the filter to unit DC response. */
  sum *= 1. / (OD_DIST_LP_NORM * OD_DIST_LP_NORM * OD_DIST_LP_NORM *
               OD_DIST_LP_NORM);
Yushin Cho's avatar
Yushin Cho committed
809
810
811
812
  return activity * activity * (sum + vardist);
}

// Note : Inputs x and y are in a pixel domain
Yushin Cho's avatar
Yushin Cho committed
813
814
static double od_compute_dist_common(int activity_masking, uint16_t *x,
                                     uint16_t *y, int bsize_w, int bsize_h,
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
                                     int qindex, od_coeff *tmp,
                                     od_coeff *e_lp) {
  int i, j;
  double sum = 0;
  const int mid = OD_DIST_LP_MID;

  for (j = 0; j < bsize_w; j++) {
    e_lp[j] = mid * tmp[j] + 2 * tmp[bsize_w + j];
    e_lp[(bsize_h - 1) * bsize_w + j] = mid * tmp[(bsize_h - 1) * bsize_w + j] +
                                        2 * tmp[(bsize_h - 2) * bsize_w + j];
  }
  for (i = 1; i < bsize_h - 1; i++) {
    for (j = 0; j < bsize_w; j++) {
      e_lp[i * bsize_w + j] = mid * tmp[i * bsize_w + j] +
                              tmp[(i - 1) * bsize_w + j] +
                              tmp[(i + 1) * bsize_w + j];
    }
  }
  for (i = 0; i < bsize_h; i += 8) {
    for (j = 0; j < bsize_w; j += 8) {
Yushin Cho's avatar
Yushin Cho committed
835
      sum += od_compute_dist_8x8(activity_masking, &x[i * bsize_w + j],
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
                                 &y[i * bsize_w + j], &e_lp[i * bsize_w + j],
                                 bsize_w);
    }
  }
  /* Scale according to linear regression against SSE, for 8x8 blocks. */
  if (activity_masking) {
    sum *= 2.2 + (1.7 - 2.2) * (qindex - 99) / (210 - 99) +
           (qindex < 99 ? 2.5 * (qindex - 99) / 99 * (qindex - 99) / 99 : 0);
  } else {
    sum *= qindex >= 128
               ? 1.4 + (0.9 - 1.4) * (qindex - 128) / (209 - 128)
               : qindex <= 43 ? 1.5 + (2.0 - 1.5) * (qindex - 43) / (16 - 43)
                              : 1.5 + (1.4 - 1.5) * (qindex - 43) / (128 - 43);
  }

  return sum;
}

Yushin Cho's avatar
Yushin Cho committed
854
855
static double od_compute_dist(uint16_t *x, uint16_t *y, int bsize_w,
                              int bsize_h, int qindex) {
Yushin Cho's avatar
Yushin Cho committed
856
857
858
859
860
861
  int i;
  double sum;
  sum = 0;

  assert(bsize_w >= 8 && bsize_h >= 8);

Yushin Cho's avatar
Yushin Cho committed
862
863
864
865
866
867
#if CONFIG_PVQ
  int activity_masking = 1;
#else
  int activity_masking = 0;
#endif
  {
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
    int j;
    DECLARE_ALIGNED(16, od_coeff, e[MAX_TX_SQUARE]);
    DECLARE_ALIGNED(16, od_coeff, tmp[MAX_TX_SQUARE]);
    DECLARE_ALIGNED(16, od_coeff, e_lp[MAX_TX_SQUARE]);
    int mid = OD_DIST_LP_MID;
    for (i = 0; i < bsize_h; i++) {
      for (j = 0; j < bsize_w; j++) {
        e[i * bsize_w + j] = x[i * bsize_w + j] - y[i * bsize_w + j];
      }
    }
    for (i = 0; i < bsize_h; i++) {
      tmp[i * bsize_w] = mid * e[i * bsize_w] + 2 * e[i * bsize_w + 1];
      tmp[i * bsize_w + bsize_w - 1] =
          mid * e[i * bsize_w + bsize_w - 1] + 2 * e[i * bsize_w + bsize_w - 2];
      for (j = 1; j < bsize_w - 1; j++) {
        tmp[i * bsize_w + j] = mid * e[i * bsize_w + j] +
                               e[i * bsize_w + j - 1] + e[i * bsize_w + j + 1];
      }
    }
Yushin Cho's avatar
Yushin Cho committed
887
    sum = od_compute_dist_common(activity_masking, x, y, bsize_w, bsize_h,
888
889
890
891
892
                                 qindex, tmp, e_lp);
  }
  return sum;
}

Yushin Cho's avatar
Yushin Cho committed
893
894
static double od_compute_dist_diff(uint16_t *x, int16_t *e, int bsize_w,
                                   int bsize_h, int qindex) {
895
896
897
898
899
900
  int i;
  double sum;
  sum = 0;

  assert(bsize_w >= 8 && bsize_h >= 8);

Yushin Cho's avatar
Yushin Cho committed
901
902
903
904
905
906
#if CONFIG_PVQ
  int activity_masking = 1;
#else
  int activity_masking = 0;
#endif
  {
907
    int j;
Yushin Cho's avatar
Yushin Cho committed
908
    DECLARE_ALIGNED(16, uint16_t, y[MAX_TX_SQUARE]);
909
910
911
912
    DECLARE_ALIGNED(16, od_coeff, tmp[MAX_TX_SQUARE]);
    DECLARE_ALIGNED(16, od_coeff, e_lp[MAX_TX_SQUARE]);
    int mid = OD_DIST_LP_MID;
    for (i = 0; i < bsize_h; i++) {
913
      for (j = 0; j < bsize_w; j++) {
914
        y[i * bsize_w + j] = x[i * bsize_w + j] - e[i * bsize_w + j];
915
916
      }
    }
917
918
919
920
921
922
923
    for (i = 0; i < bsize_h; i++) {
      tmp[i * bsize_w] = mid * e[i * bsize_w] + 2 * e[i * bsize_w + 1];
      tmp[i * bsize_w + bsize_w - 1] =
          mid * e[i * bsize_w + bsize_w - 1] + 2 * e[i * bsize_w + bsize_w - 2];
      for (j = 1; j < bsize_w - 1; j++) {
        tmp[i * bsize_w + j] = mid * e[i * bsize_w + j] +
                               e[i * bsize_w + j - 1] + e[i * bsize_w + j + 1];
Yushin Cho's avatar
Yushin Cho committed
924
925
      }
    }
Yushin Cho's avatar
Yushin Cho committed
926
    sum = od_compute_dist_common(activity_masking, x, y, bsize_w, bsize_h,
927
                                 qindex, tmp, e_lp);
Yushin Cho's avatar
Yushin Cho committed
928
929
930
931
  }
  return sum;
}

932
int64_t av1_dist_8x8(const AV1_COMP *const cpi, const MACROBLOCK *x,
Yushin Cho's avatar
Yushin Cho committed
933
934
935
936
                     const uint8_t *src, int src_stride, const uint8_t *dst,
                     int dst_stride, const BLOCK_SIZE tx_bsize, int bsw,
                     int bsh, int visible_w, int visible_h, int qindex) {
  int64_t d = 0;
Yushin Cho's avatar
Yushin Cho committed
937
  int i, j;
938
  const MACROBLOCKD *xd = &x->e_mbd;
Yushin Cho's avatar
Yushin Cho committed
939
940
941
942

  DECLARE_ALIGNED(16, uint16_t, orig[MAX_TX_SQUARE]);
  DECLARE_ALIGNED(16, uint16_t, rec[MAX_TX_SQUARE]);

943
944
945
946
  if (x->tune_metric == AOM_TUNE_CDEF_DIST ||
      x->tune_metric == AOM_TUNE_DAALA_DIST) {
    assert((bsw & 0x07) == 0);
    assert((bsh & 0x07) == 0);
Yushin Cho's avatar
Yushin Cho committed
947

948
#if CONFIG_HIGHBITDEPTH
949
    if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
950
      for (j = 0; j < bsh; j++)
951
        for (i = 0; i < bsw; i++)
952
          orig[j * bsw + i] = CONVERT_TO_SHORTPTR(src)[j * src_stride + i];
953

954
      if ((bsw == visible_w) && (bsh == visible_h)) {
955
956
        for (j = 0; j < bsh; j++)
          for (i = 0; i < bsw; i++)
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
            rec[j * bsw + i] = CONVERT_TO_SHORTPTR(dst)[j * dst_stride + i];
      } else {
        for (j = 0; j < visible_h; j++)
          for (i = 0; i < visible_w; i++)
            rec[j * bsw + i] = CONVERT_TO_SHORTPTR(dst)[j * dst_stride + i];

        if (visible_w < bsw) {
          for (j = 0; j < bsh; j++)
            for (i = visible_w; i < bsw; i++)
              rec[j * bsw + i] = CONVERT_TO_SHORTPTR(src)[j * src_stride + i];
        }

        if (visible_h < bsh) {
          for (j = visible_h; j < bsh; j++)
            for (i = 0; i < bsw; i++)
              rec[j * bsw + i] = CONVERT_TO_SHORTPTR(src)[j * src_stride + i];
        }
974
      }
975
    } else {
976
977
#endif
      for (j = 0; j < bsh; j++)
978
        for (i = 0; i < bsw; i++) orig[j * bsw + i] = src[j * src_stride + i];
979

980
      if ((bsw == visible_w) && (bsh == visible_h)) {
981
        for (j = 0; j < bsh; j++)
982
983
984
985
986
987
988
989
990
991
992
          for (i = 0; i < bsw; i++) rec[j * bsw + i] = dst[j * dst_stride + i];
      } else {
        for (j = 0; j < visible_h; j++)
          for (i = 0; i < visible_w; i++)
            rec[j * bsw + i] = dst[j * dst_stride + i];

        if (visible_w < bsw) {
          for (j = 0; j < bsh; j++)
            for (i = visible_w; i < bsw; i++)
              rec[j * bsw + i] = src[j * src_stride + i];
        }
993

994
995
996
997
998
        if (visible_h < bsh) {
          for (j = visible_h; j < bsh; j++)
            for (i = 0; i < bsw; i++)
              rec[j * bsw + i] = src[j * src_stride + i];
        }
999
1000
      }
#if CONFIG_HIGHBITDEPTH
1001
    }
1002
#endif  // CONFIG_HIGHBITDEPTH
1003
  }
Yushin Cho's avatar
Yushin Cho committed
1004

1005
1006
1007
  if (x->tune_metric == AOM_TUNE_DAALA_DIST) {
    d = (int64_t)od_compute_dist(orig, rec, bsw, bsh, qindex);
  } else if (x->tune_metric == AOM_TUNE_CDEF_DIST) {
Yushin Cho's avatar
Yushin Cho committed
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
    int coeff_shift = AOMMAX(xd->bd - 8, 0);

    for (i = 0; i < bsh; i += 8) {
      for (j = 0; j < bsw; j += 8) {
        d += cdef_dist_8x8_16bit(&rec[i * bsw + j], bsw, &orig[i * bsw + j],
                                 bsw, coeff_shift);
      }
    }
#if CONFIG_HIGHBITDEPTH
    if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH)
      d = ((uint64_t)d) >> 2 * coeff_shift;
#endif
1020
1021
1022
1023
1024
1025
  } else {
    // Otherwise, MSE by default
    unsigned sse;
    // TODO(Any): Use even faster function which does not calculate variance
    cpi->fn_ptr[tx_bsize].vf(src, src_stride, dst, dst_stride, &sse);
    d = sse;
Yushin Cho's avatar
Yushin Cho committed
1026
  }
1027

Yushin Cho's avatar
Yushin Cho committed
1028
1029
  return d;
}
1030

1031
static int64_t av1_dist_8x8_diff(const MACROBLOCK *x, const uint8_t *src,
Yushin Cho's avatar
Yushin Cho committed
1032
1033
1034
1035
                                 int src_stride, const int16_t *diff,
                                 int diff_stride, int bsw, int bsh,
                                 int visible_w, int visible_h, int qindex) {
  int64_t d = 0;
1036
  int i, j;
1037
  const MACROBLOCKD *xd = &x->e_mbd;
Yushin Cho's avatar
Yushin Cho committed
1038
1039
1040
1041

  DECLARE_ALIGNED(16, uint16_t, orig[MAX_TX_SQUARE]);
  DECLARE_ALIGNED(16, int16_t, diff16[MAX_TX_SQUARE]);

1042
1043
1044
1045
  if (x->tune_metric == AOM_TUNE_CDEF_DIST ||
      x->tune_metric == AOM_TUNE_DAALA_DIST) {
    assert((bsw & 0x07) == 0);
    assert((bsh & 0x07) == 0);
Yushin Cho's avatar
Yushin Cho committed
1046

1047
#if CONFIG_HIGHBITDEPTH
1048
1049
1050
1051
1052
    if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
      for (j = 0; j < bsh; j++)
        for (i = 0; i < bsw; i++)
          orig[j * bsw + i] = CONVERT_TO_SHORTPTR(src)[j * src_stride + i];
    } else {
1053
#endif
1054
1055
      for (j = 0; j < bsh; j++)
        for (i = 0; i < bsw; i++) orig[j * bsw + i] = src[j * src_stride + i];
1056
#if CONFIG_HIGHBITDEPTH
1057
    }
1058
#endif  // CONFIG_HIGHBITDEPTH
1059

1060
    if ((bsw == visible_w) && (bsh == visible_h)) {
1061
      for (j = 0; j < bsh; j++)
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
        for (i = 0; i < bsw; i++)
          diff16[j * bsw + i] = diff[j * diff_stride + i];
    } else {
      for (j = 0; j < visible_h; j++)
        for (i = 0; i < visible_w; i++)
          diff16[j * bsw + i] = diff[j * diff_stride + i];

      if (visible_w < bsw) {
        for (j = 0; j < bsh; j++)
          for (i = visible_w; i < bsw; i++) diff16[j * bsw + i] = 0;
      }
1073

1074
1075
1076
1077
      if (visible_h < bsh) {
        for (j = visible_h; j < bsh; j++)
          for (i = 0; i < bsw; i++) diff16[j * bsw + i] = 0;
      }
1078
1079
    }
  }
1080

1081
1082
1083
  if (x->tune_metric == AOM_TUNE_DAALA_DIST) {
    d = (int64_t)od_compute_dist_diff(orig, diff16, bsw, bsh, qindex);
  } else if (x->tune_metric == AOM_TUNE_CDEF_DIST) {
Yushin Cho's avatar
Yushin Cho committed
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
    int coeff_shift = AOMMAX(xd->bd - 8, 0);
    DECLARE_ALIGNED(16, uint16_t, dst16[MAX_TX_SQUARE]);

    for (i = 0; i < bsh; i++) {
      for (j = 0; j < bsw; j++) {
        dst16[i * bsw + j] = orig[i * bsw + j] - diff16[i * bsw + j];
      }
    }

    for (i = 0; i < bsh; i += 8) {
      for (j = 0; j < bsw; j += 8) {
        d += cdef_dist_8x8_16bit(&dst16[i * bsw + j], bsw, &orig[i * bsw + j],
                                 bsw, coeff_shift);
      }
    }
    // Don't scale 'd' for HBD since it will be done by caller side for diff
    // input
1101
1102
1103
  } else {
    // Otherwise, MSE by default
    d = aom_sum_squares_2d_i16(diff, diff_stride, bsw, bsh);
Yushin Cho's avatar
Yushin Cho committed
1104
  }
1105
1106
1107

  return d;
}
Yushin Cho's avatar
Yushin Cho committed
1108
#endif  // CONFIG_DIST_8X8
Yushin Cho's avatar
Yushin Cho committed
1109

Yaowu Xu's avatar
Yaowu Xu committed
1110
static void get_energy_distribution_fine(const AV1_COMP *cpi, BLOCK_SIZE bsize,
1111
1112
                                         const uint8_t *src, int src_stride,
                                         const uint8_t *dst, int dst_stride,
1113
                                         double *hordist, double *verdist) {
1114
1115
  const int bw = block_size_wide[bsize];
  const int bh = block_size_high[bsize];
1116
  unsigned int esq[16] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
1117

1118
  const int f_index = bsize - BLOCK_16X16;
1119
  if (f_index < 0) {
1120
1121
    const int w_shift = bw == 8 ? 1 : 2;
    const int h_shift = bh == 8 ? 1 : 2;
1122
#if CONFIG_HIGHBITDEPTH
1123
    if (cpi->common.use_highbitdepth) {
1124
1125
1126
1127
1128
      const uint16_t *src16 = CONVERT_TO_SHORTPTR(src);
      const uint16_t *dst16 = CONVERT_TO_SHORTPTR(dst);
      for (int i = 0; i < bh; ++i)
        for (int j = 0; j < bw; ++j) {
          const int index = (j >> w_shift) + ((i >> h_shift) << 2);
1129
1130
1131
          esq[index] +=
              (src16[j + i * src_stride] - dst16[j + i * dst_stride]) *
              (src16[j + i * src_stride] - dst16[j + i * dst_stride]);
1132
1133
        }
    } else {
1134
#endif  // CONFIG_HIGHBITDEPTH
1135

1136
1137
1138
      for (int i = 0; i < bh; ++i)
        for (int j = 0; j < bw; ++j) {
          const int index = (j >> w_shift) + ((i >> h_shift) << 2);
1139
1140
1141
          esq[index] += (src[j + i * src_stride] - dst[j + i * dst_stride]) *
                        (src[j + i * src_stride] - dst[j + i * dst_stride]);
        }
1142
#if CONFIG_HIGHBITDEPTH
1143
    }
1144
#endif  // CONFIG_HIGHBITDEPTH
1145
  } else {
1146
1147
1148
1149
1150
1151
1152
    cpi->fn_ptr[f_index].vf(src, src_stride, dst, dst_stride, &esq[0]);
    cpi->fn_ptr[f_index].vf(src + bw / 4, src_stride, dst + bw / 4, dst_stride,
                            &esq[1]);
    cpi->fn_ptr[f_index].vf(src + bw / 2, src_stride, dst + bw / 2, dst_stride,
                            &esq[2]);
    cpi->fn_ptr[f_index].vf(src + 3 * bw / 4, src_stride, dst + 3 * bw / 4,
                            dst_stride, &esq[3]);
1153
1154
1155
    src += bh / 4 * src_stride;
    dst += bh / 4 * dst_stride;

1156
1157
1158
1159
1160
1161
1162
    cpi->fn_ptr[f_index].vf(src, src_stride, dst, dst_stride, &esq[4]);
    cpi->fn_ptr[f_index].vf(src + bw / 4, src_stride, dst + bw / 4, dst_stride,
                            &esq[5]);
    cpi->fn_ptr[f_index].vf(src + bw / 2, src_stride, dst + bw / 2, dst_stride,
                            &esq[6]);
    cpi->fn_ptr[f_index].vf(src + 3 * bw / 4, src_stride, dst + 3 * bw / 4,
                            dst_stride, &esq[7]);
1163
1164
1165
    src += bh / 4 * src_stride;
    dst += bh / 4 * dst_stride;

1166
1167
1168
1169
1170
1171
1172
    cpi->fn_ptr[f_index].vf(src, src_stride, dst, dst_stride, &esq[8]);
    cpi->fn_ptr[f_index].vf(src + bw / 4, src_stride, dst + bw / 4, dst_stride,
                            &esq[9]);
    cpi->fn_ptr[f_index].vf(src + bw / 2, src_stride, dst + bw / 2, dst_stride,
                            &esq[10]);
    cpi->fn_ptr[f_index].vf(src + 3 * bw / 4, src_stride, dst + 3 * bw / 4,
                            dst_stride, &esq[11]);
1173
1174
1175
    src += bh / 4 * src_stride;
    dst += bh / 4 * dst_stride;

1176
1177
1178
1179
1180
1181
1182
    cpi->fn_ptr[f_index].vf(src, src_stride, dst, dst_stride, &esq[12]);
    cpi->fn_ptr[f_index].vf(src + bw / 4, src_stride, dst + bw / 4, dst_stride,
                            &esq[13]);
    cpi->fn_ptr[f_index].vf(src + bw / 2, src_stride, dst + bw / 2, dst_stride,
                            &esq[14]);
    cpi->fn_ptr[f_index].vf(src + 3 * bw / 4, src_stride, dst + 3 * bw / 4,
                            dst_stride, &esq[15]);
1183
1184
  }

1185
1186
1187
  double total = (double)esq[0] + esq[1] + esq[2] + esq[3] + esq[4] + esq[5] +
                 esq[6] + esq[7] + esq[8] + esq[9] + esq[10] + esq[11] +
                 esq[12] + esq[13] + esq[14] + esq[15];
1188
1189
  if (total > 0) {
    const double e_recip = 1.0 / total;
1190
1191
1192
1193
1194
1195
    hordist[0] = ((double)esq[0] + esq[4] + esq[8] + esq[12]) * e_recip;
    hordist[1] = ((double)esq[1] + esq[5] + esq[9] + esq[13]) * e_recip;
    hordist[2] = ((double)esq[2] + esq[6] + esq[10] + esq[14]) * e_recip;
    verdist[0] = ((double)esq[0] + esq[1] + esq[2] + esq[3]) * e_recip;
    verdist[1] = ((double)esq[4] + esq[5] + esq[6] + esq[7]) * e_recip;
    verdist[2] = ((double)esq[8] + esq[9] + esq[10] + esq[11]) * e_recip;
1196
1197
1198
1199
1200
1201
1202
  } else {
    hordist[0] = verdist[0] = 0.25;
    hordist[1] = verdist[1] = 0.25;
    hordist[2] = verdist[2] = 0.25;
  }
}

Urvang Joshi's avatar
Urvang Joshi committed
1203
1204
1205
static int adst_vs_flipadst(const AV1_COMP *cpi, BLOCK_SIZE bsize,
                            const uint8_t *src, int src_stride,
                            const uint8_t *dst, int dst_stride) {
1206
1207
  int prune_bitmask = 0;
  double svm_proj_h = 0, svm_proj_v = 0;
Alex Converse's avatar
Alex Converse committed
1208
  double hdist[3] = { 0, 0, 0 }, vdist[3] = { 0, 0, 0 };
1209
1210
  get_energy_distribution_fine(cpi, bsize, src, src_stride, dst, dst_stride,
                               hdist, vdist);
1211

1212
  svm_proj_v = vdist[0] * ADST_FLIP_SVM[0] + vdist[1] * ADST_FLIP_SVM[1] +
1213
               vdist[2] * ADST_FLIP_SVM[2] + ADST_FLIP_SVM[3];
1214
  svm_proj_h = hdist[0] * ADST_FLIP_SVM[4] + hdist[1] * ADST_FLIP_SVM[5] +
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
               hdist[2] * ADST_FLIP_SVM[6] + ADST_FLIP_SVM[7];
  if (svm_proj_v > FAST_EXT_TX_EDST_MID + FAST_EXT_TX_EDST_MARGIN)
    prune_bitmask |= 1 << FLIPADST_1D;
  else if (svm_proj_v < FAST_EXT_TX_EDST_MID - FAST_EXT_TX_EDST_MARGIN)
    prune_bitmask |= 1 << ADST_1D;

  if (svm_proj_h > FAST_EXT_TX_EDST_MID + FAST_EXT_TX_EDST_MARGIN)
    prune_bitmask |= 1 << (FLIPADST_1D + 8);
  else if (svm_proj_h < FAST_EXT_TX_EDST_MID - FAST_EXT_TX_EDST_MARGIN)
    prune_bitmask |= 1 << (ADST_1D + 8);

  return prune_bitmask;
}

#if CONFIG_EXT_TX
Alex Converse's avatar
Alex Converse committed
1230
1231
static void get_horver_correlation(const int16_t *diff, int stride, int w,
                                   int h, double *hcorr, double *vcorr) {
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
  // Returns hor/ver correlation coefficient
  const int num = (h - 1) * (w - 1);
  double num_r;
  int i, j;
  int64_t xy_sum = 0, xz_sum = 0;
  int64_t x_sum = 0, y_sum = 0, z_sum = 0;
  int64_t x2_sum = 0, y2_sum = 0, z2_sum = 0;
  double x_var_n, y_var_n, z_var_n, xy_var_n, xz_var_n;
  *hcorr = *vcorr = 1;

  assert(num > 0);
  num_r = 1.0 / num;
  for (i = 1; i < h; ++i) {
    for (j = 1; j < w; ++j) {
      const int16_t x = diff[i * stride + j];
      const int16_t y = diff[i * stride + j - 1];
      const int16_t z = diff[(i - 1) * stride + j];
      xy_sum += x * y;
      xz_sum += x * z;
      x_sum += x;
      y_sum += y;
      z_sum += z;
      x2_sum += x * x;
      y2_sum += y * y;
      z2_sum += z * z;
    }
  }
1259
1260
1261
  x_var_n = x2_sum - (x_sum * x_sum) * num_r;
  y_var_n = y2_sum - (y_sum * y_sum) * num_r;
  z_var_n = z2_sum - (z_sum * z_sum) * num_r;
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
  xy_var_n = xy_sum - (x_sum * y_sum) * num_r;
  xz_var_n = xz_sum - (x_sum * z_sum) * num_r;
  if (x_var_n > 0 && y_var_n > 0) {
    *hcorr = xy_var_n / sqrt(x_var_n * y_var_n);
    *hcorr = *hcorr < 0 ? 0 : *hcorr;
  }
  if (x_var_n > 0 && z_var_n > 0) {
    *vcorr = xz_var_n / sqrt(x_var_n * z_var_n);
    *vcorr = *vcorr < 0 ? 0 : *vcorr;
  }
}

Alex Converse's avatar
Alex Converse committed
1274
1275
int dct_vs_idtx(const int16_t *diff, int stride, int w, int h) {
  double hcorr, vcorr;
1276
  int prune_bitmask = 0;
Alex Converse's avatar
Alex Converse committed
1277
  get_horver_correlation(diff, stride, w, h, &hcorr, &vcorr);
1278

Alex Converse's avatar
Alex Converse committed
1279
  if (vcorr > FAST_EXT_TX_CORR_MID + FAST_EXT_TX_CORR_MARGIN)
1280
    prune_bitmask |= 1 << IDTX_1D;
Alex Converse's avatar
Alex Converse committed
1281
  else if (vcorr < FAST_EXT_TX_CORR_MID - FAST_EXT_TX_CORR_MARGIN)
1282
1283
    prune_bitmask |= 1 << DCT_1D;

Alex Converse's avatar
Alex Converse committed
1284
  if (hcorr > FAST_EXT_TX_CORR_MID + FAST_EXT_TX_CORR_MARGIN)
1285
    prune_bitmask |= 1 << (IDTX_1D + 8);
Alex Converse's avatar
Alex Converse committed
1286
  else if (hcorr < FAST_EXT_TX_CORR_MID - FAST_EXT_TX_CORR_MARGIN)
1287
1288
1289
1290
1291
    prune_bitmask |= 1 << (DCT_1D + 8);
  return prune_bitmask;
}

// Performance drop: 0.5%, Speed improvement: 24%
Yaowu Xu's avatar
Yaowu Xu committed
1292
static int prune_two_for_sby(const AV1_COMP *cpi, BLOCK_SIZE bsize,
Alex Converse's avatar
Alex Converse committed
1293
1294
                             MACROBLOCK *x, const MACROBLOCKD *xd,
                             int adst_flipadst, int dct_idtx) {
1295
  int prune = 0;
1296

Alex Converse's avatar
Alex Converse committed
1297
1298
1299
  if (adst_flipadst) {
    const struct macroblock_plane *const p = &x->plane[0];
    const struct macroblockd_plane *const pd = &xd->plane[0];
1300
    prune |= adst_vs_flipadst(cpi, bsize, p->src.buf, p->src.stride,
Alex Converse's avatar
Alex Converse committed
1301
1302
1303
1304
1305
1306
1307
1308
1309
                              pd->dst.buf, pd->dst.stride);
  }
  if (dct_idtx) {
    av1_subtract_plane(x, bsize, 0);
    const struct macroblock_plane *const p = &x->plane[0];
    const int bw = 4 << (b_width_log2_lookup[bsize]);
    const int bh = 4 << (b_height_log2_lookup[bsize]);
    prune |= dct_vs_idtx(p->src_diff, bw, bw, bh);
  }
1310
1311
1312

  return prune;
}
1313
1314
#endif  // CONFIG_EXT_TX

1315
// Performance drop: 0.3%, Speed improvement: 5%
Yaowu Xu's avatar
Yaowu Xu committed
1316
static int prune_one_for_sby(const AV1_COMP *cpi, BLOCK_SIZE bsize,
Alex Converse's avatar
Alex Converse committed
1317
1318
1319
                             const MACROBLOCK *x, const MACROBLOCKD *xd) {
  const struct macroblock_plane *const p = &x->plane[0];
  const struct macroblockd_plane *const pd = &xd->plane[0];
1320
  return adst_vs_flipadst(cpi, bsize, p->src.buf, p->src.stride, pd->dst.buf,
Alex Converse's avatar
Alex Converse committed
1321
                          pd->dst.stride);
1322
1323
}

Yaowu Xu's avatar
Yaowu Xu committed
1324
static int prune_tx_types(const AV1_COMP *cpi, BLOCK_SIZE bsize, MACROBLOCK *x,
Urvang Joshi's avatar
Urvang Joshi committed
1325
                          const MACROBLOCKD *const xd, int tx_set) {
1326
#if CONFIG_EXT_TX
Yaowu Xu's avatar
Yaowu Xu committed
1327
  const int *tx_set_1D = tx_set >= 0 ? ext_tx_used_inter_1D[tx_set] : NULL;
1328
#else
1329
  const int tx_set_1D[TX_TYPES_1D] = { 0 };
1330
#endif  // CONFIG_EXT_TX
1331

1332
  switch (cpi->sf.tx_type_search.prune_mode) {
1333
1334
    case NO_PRUNE: return 0; break;
    case PRUNE_ONE:
1335
      if ((tx_set >= 0) && !(tx_set_1D[FLIPADST_1D] & tx_set_1D[ADST_1D]))
1336
        return 0;
1337
1338
      return prune_one_for_sby(cpi, bsize, x, xd);
      break;
1339
1340
#if CONFIG_EXT_TX
    case PRUNE_TWO:
1341
      if ((tx_set >= 0) && !(tx_set_1D[FLIPADST_1D] & tx_set_1D[ADST_1D])) {
1342
        if (!(tx_set_1D[DCT_1D] & tx_set_1D[IDTX_1D])) return 0;
1343
1344
        return prune_two_for_sby(cpi, bsize, x, xd, 0, 1);
      }
1345
      if ((tx_set >= 0) && !(tx_set_1D[DCT_1D] & tx_set_1D[IDTX_1D]))
1346
1347
        return prune_two_for_sby(cpi, bsize, x, xd, 1, 0);
      return prune_two_for_sby(cpi, bsize, x, xd, 1, 1);
1348
      break;
1349
#endif  // CONFIG_EXT_TX
1350
1351
1352
1353
1354
  }
  assert(0);
  return 0;
}

1355
static int do_tx_type_search(TX_TYPE tx_type, int prune) {
1356
1357
1358
// TODO(sarahparker) implement for non ext tx
#if CONFIG_EXT_TX
  return !(((prune >> vtx_tab[tx_type]) & 1) |
1359
           ((prune >> (htx_tab[tx_type] + 8)) & 1));
1360
1361
#else
  // temporary to avoid compiler warnings
1362
1363
1364
1365
  (void)vtx_tab;
  (void)htx_tab;
  (void)tx_type;
  (void)prune;
1366
  return 1;
1367
#endif  // CONFIG_EXT_TX
1368
1369
}

Yaowu Xu's avatar
Yaowu Xu committed
1370
static void model_rd_from_sse(const AV1_COMP *const cpi,
1371
1372
                              const MACROBLOCKD *const xd, BLOCK_SIZE bsize,
                              int plane, int64_t sse, int *rate,
Geza Lore's avatar
Geza Lore committed
1373
1374
1375
                              int64_t *dist) {
  const struct macroblockd_plane *const pd = &xd->plane[plane];
  const int dequant_shift =
1376
#if CONFIG_HIGHBITDEPTH
1377
      (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) ? xd->bd - 5 :