rd.h 17.2 KB
Newer Older
Jingning Han's avatar
Jingning Han committed
1
/*
Yaowu Xu's avatar
Yaowu Xu committed
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Jingning Han's avatar
Jingning Han committed
3
 *
Yaowu Xu's avatar
Yaowu Xu committed
4 5 6 7 8 9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Jingning Han's avatar
Jingning Han committed
10 11
 */

Yaowu Xu's avatar
Yaowu Xu committed
12 13
#ifndef AV1_ENCODER_RD_H_
#define AV1_ENCODER_RD_H_
Jingning Han's avatar
Jingning Han committed
14 15 16

#include <limits.h>

17
#if CONFIG_ANS
Alex Converse's avatar
Alex Converse committed
18
#include "aom_dsp/ans.h"
19
#endif  // CONFIG_ANS
20
#include "av1/common/blockd.h"
Jingning Han's avatar
Jingning Han committed
21

22 23 24
#include "av1/encoder/block.h"
#include "av1/encoder/context_tree.h"
#include "av1/encoder/cost.h"
Jingning Han's avatar
Jingning Han committed
25 26 27 28 29

#ifdef __cplusplus
extern "C" {
#endif

30 31
#define RDDIV_BITS 7
#define RD_EPB_SHIFT 6
Jingning Han's avatar
Jingning Han committed
32

33 34 35
#define RDCOST(RM, R, D)                                          \
  (ROUND_POWER_OF_TWO(((int64_t)R) * (RM), AV1_PROB_COST_SHIFT) + \
   (D << RDDIV_BITS))
Debargha Mukherjee's avatar
Debargha Mukherjee committed
36

37
#define RDCOST_DBL(RM, R, D)                                       \
Yaowu Xu's avatar
Yaowu Xu committed
38
  (((((double)(R)) * (RM)) / (double)(1 << AV1_PROB_COST_SHIFT)) + \
39
   ((double)(D) * (1 << RDDIV_BITS)))
Debargha Mukherjee's avatar
Debargha Mukherjee committed
40

41
#define QIDX_SKIP_THRESH 115
Jingning Han's avatar
Jingning Han committed
42

43 44
#define MV_COST_WEIGHT 108
#define MV_COST_WEIGHT_SUB 120
Jingning Han's avatar
Jingning Han committed
45 46 47 48

#define INVALID_MV 0x80008000

#define RD_THRESH_MAX_FACT 64
49
#define RD_THRESH_INC 1
Jingning Han's avatar
Jingning Han committed
50 51

// This enumerator type needs to be kept aligned with the mode order in
Yaowu Xu's avatar
Yaowu Xu committed
52
// const MODE_DEFINITION av1_mode_order[MAX_MODES] used in the rd code.
Jingning Han's avatar
Jingning Han committed
53 54
typedef enum {
  THR_NEARESTMV,
55 56 57
#if CONFIG_EXT_REFS
  THR_NEARESTL2,
  THR_NEARESTL3,
58
  THR_NEARESTB,
Zoe Liu's avatar
Zoe Liu committed
59 60 61
#if CONFIG_ALTREF2
  THR_NEARESTA2,
#endif  // CONFIG_ALTREF2
62
#endif  // CONFIG_EXT_REFS
Jingning Han's avatar
Jingning Han committed
63 64 65 66 67 68
  THR_NEARESTA,
  THR_NEARESTG,

  THR_DC,

  THR_NEWMV,
69 70 71
#if CONFIG_EXT_REFS
  THR_NEWL2,
  THR_NEWL3,
72
  THR_NEWB,
Zoe Liu's avatar
Zoe Liu committed
73 74 75
#if CONFIG_ALTREF2
  THR_NEWA2,
#endif  // CONFIG_ALTREF2
76
#endif  // CONFIG_EXT_REFS
Jingning Han's avatar
Jingning Han committed
77 78 79 80
  THR_NEWA,
  THR_NEWG,

  THR_NEARMV,
81 82 83
#if CONFIG_EXT_REFS
  THR_NEARL2,
  THR_NEARL3,
84
  THR_NEARB,
Zoe Liu's avatar
Zoe Liu committed
85 86 87
#if CONFIG_ALTREF2
  THR_NEARA2,
#endif  // CONFIG_ALTREF2
88
#endif  // CONFIG_EXT_REFS
Jingning Han's avatar
Jingning Han committed
89 90 91 92
  THR_NEARA,
  THR_NEARG,

  THR_ZEROMV,
93 94 95
#if CONFIG_EXT_REFS
  THR_ZEROL2,
  THR_ZEROL3,
96
  THR_ZEROB,
Zoe Liu's avatar
Zoe Liu committed
97 98 99
#if CONFIG_ALTREF2
  THR_ZEROA2,
#endif  // CONFIG_ALTREF2
100
#endif  // CONFIG_EXT_REFS
Jingning Han's avatar
Jingning Han committed
101
  THR_ZEROA,
Zoe Liu's avatar
Zoe Liu committed
102
  THR_ZEROG,
Jingning Han's avatar
Jingning Han committed
103

104
#if CONFIG_EXT_INTER
105

106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153
#if CONFIG_COMPOUND_SINGLEREF
  THR_SR_NEAREST_NEARMV,
#if CONFIG_EXT_REFS
  THR_SR_NEAREST_NEARL2,
  THR_SR_NEAREST_NEARL3,
  THR_SR_NEAREST_NEARB,
#endif  // CONFIG_EXT_REFS
  THR_SR_NEAREST_NEARG,
  THR_SR_NEAREST_NEARA,

  /*
  THR_SR_NEAREST_NEWMV,
#if CONFIG_EXT_REFS
  THR_SR_NEAREST_NEWL2,
  THR_SR_NEAREST_NEWL3,
  THR_SR_NEAREST_NEWB,
#endif  // CONFIG_EXT_REFS
  THR_SR_NEAREST_NEWG,
  THR_SR_NEAREST_NEWA,*/

  THR_SR_NEAR_NEWMV,
#if CONFIG_EXT_REFS
  THR_SR_NEAR_NEWL2,
  THR_SR_NEAR_NEWL3,
  THR_SR_NEAR_NEWB,
#endif  // CONFIG_EXT_REFS
  THR_SR_NEAR_NEWG,
  THR_SR_NEAR_NEWA,

  THR_SR_ZERO_NEWMV,
#if CONFIG_EXT_REFS
  THR_SR_ZERO_NEWL2,
  THR_SR_ZERO_NEWL3,
  THR_SR_ZERO_NEWB,
#endif  // CONFIG_EXT_REFS
  THR_SR_ZERO_NEWG,
  THR_SR_ZERO_NEWA,

  THR_SR_NEW_NEWMV,
#if CONFIG_EXT_REFS
  THR_SR_NEW_NEWL2,
  THR_SR_NEW_NEWL3,
  THR_SR_NEW_NEWB,
#endif  // CONFIG_EXT_REFS
  THR_SR_NEW_NEWG,
  THR_SR_NEW_NEWA,
#endif  // CONFIG_COMPOUND_SINGLEREF

154 155 156 157
  THR_COMP_NEAREST_NEARESTLA,
#if CONFIG_EXT_REFS
  THR_COMP_NEAREST_NEARESTL2A,
  THR_COMP_NEAREST_NEARESTL3A,
158 159 160
#endif  // CONFIG_EXT_REFS
  THR_COMP_NEAREST_NEARESTGA,
#if CONFIG_EXT_REFS
161
  THR_COMP_NEAREST_NEARESTLB,
162 163
  THR_COMP_NEAREST_NEARESTL2B,
  THR_COMP_NEAREST_NEARESTL3B,
164
  THR_COMP_NEAREST_NEARESTGB,
Zoe Liu's avatar
Zoe Liu committed
165 166 167 168 169 170
#if CONFIG_ALTREF2
  THR_COMP_NEAREST_NEARESTLA2,
  THR_COMP_NEAREST_NEARESTL2A2,
  THR_COMP_NEAREST_NEARESTL3A2,
  THR_COMP_NEAREST_NEARESTGA2,
#endif  // CONFIG_ALTREF2
Zoe Liu's avatar
Zoe Liu committed
171 172
#if CONFIG_EXT_COMP_REFS
  THR_COMP_NEAREST_NEARESTLL2,
173
  THR_COMP_NEAREST_NEARESTLL3,
Zoe Liu's avatar
Zoe Liu committed
174 175 176
  THR_COMP_NEAREST_NEARESTLG,
  THR_COMP_NEAREST_NEARESTBA,
#endif  // CONFIG_EXT_COMP_REFS
177
#endif  // CONFIG_EXT_REFS
178

179
#else  // CONFIG_EXT_INTER
180

Jingning Han's avatar
Jingning Han committed
181
  THR_COMP_NEARESTLA,
182 183 184
#if CONFIG_EXT_REFS
  THR_COMP_NEARESTL2A,
  THR_COMP_NEARESTL3A,
185 186 187
#endif  // CONFIG_EXT_REFS
  THR_COMP_NEARESTGA,
#if CONFIG_EXT_REFS
188
  THR_COMP_NEARESTLB,
189 190
  THR_COMP_NEARESTL2B,
  THR_COMP_NEARESTL3B,
191
  THR_COMP_NEARESTGB,
Zoe Liu's avatar
Zoe Liu committed
192 193 194 195 196 197
#if CONFIG_ALTREF2
  THR_COMP_NEARESTLA2,
  THR_COMP_NEARESTL2A2,
  THR_COMP_NEARESTL3A2,
  THR_COMP_NEARESTGA2,
#endif  // CONFIG_ALTREF2
Zoe Liu's avatar
Zoe Liu committed
198 199
#if CONFIG_EXT_COMP_REFS
  THR_COMP_NEARESTLL2,
200
  THR_COMP_NEARESTLL3,
Zoe Liu's avatar
Zoe Liu committed
201 202 203
  THR_COMP_NEARESTLG,
  THR_COMP_NEARESTBA,
#endif  // CONFIG_EXT_COMP_REFS
204
#endif  // CONFIG_EXT_REFS
205

206
#endif  // CONFIG_EXT_INTER
Jingning Han's avatar
Jingning Han committed
207 208 209

  THR_TM,

210 211
#if CONFIG_ALT_INTRA
  THR_SMOOTH,
Urvang Joshi's avatar
Urvang Joshi committed
212 213 214 215
#if CONFIG_SMOOTH_HV
  THR_SMOOTH_V,
  THR_SMOOTH_H,
#endif  // CONFIG_SMOOTH_HV
216 217
#endif  // CONFIG_ALT_INTRA

218
#if CONFIG_EXT_INTER
219

220
  THR_COMP_NEAR_NEARLA,
221 222 223 224 225 226 227 228
  THR_COMP_NEW_NEARESTLA,
  THR_COMP_NEAREST_NEWLA,
  THR_COMP_NEW_NEARLA,
  THR_COMP_NEAR_NEWLA,
  THR_COMP_NEW_NEWLA,
  THR_COMP_ZERO_ZEROLA,

#if CONFIG_EXT_REFS
229
  THR_COMP_NEAR_NEARL2A,
230 231 232 233 234 235 236
  THR_COMP_NEW_NEARESTL2A,
  THR_COMP_NEAREST_NEWL2A,
  THR_COMP_NEW_NEARL2A,
  THR_COMP_NEAR_NEWL2A,
  THR_COMP_NEW_NEWL2A,
  THR_COMP_ZERO_ZEROL2A,

237
  THR_COMP_NEAR_NEARL3A,
238 239 240 241 242 243
  THR_COMP_NEW_NEARESTL3A,
  THR_COMP_NEAREST_NEWL3A,
  THR_COMP_NEW_NEARL3A,
  THR_COMP_NEAR_NEWL3A,
  THR_COMP_NEW_NEWL3A,
  THR_COMP_ZERO_ZEROL3A,
244
#endif  // CONFIG_EXT_REFS
245

246 247 248 249 250 251 252
  THR_COMP_NEAR_NEARGA,
  THR_COMP_NEW_NEARESTGA,
  THR_COMP_NEAREST_NEWGA,
  THR_COMP_NEW_NEARGA,
  THR_COMP_NEAR_NEWGA,
  THR_COMP_NEW_NEWGA,
  THR_COMP_ZERO_ZEROGA,
253

254 255
#if CONFIG_EXT_REFS
  THR_COMP_NEAR_NEARLB,
256 257 258 259 260 261
  THR_COMP_NEW_NEARESTLB,
  THR_COMP_NEAREST_NEWLB,
  THR_COMP_NEW_NEARLB,
  THR_COMP_NEAR_NEWLB,
  THR_COMP_NEW_NEWLB,
  THR_COMP_ZERO_ZEROLB,
262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284

  THR_COMP_NEAR_NEARL2B,
  THR_COMP_NEW_NEARESTL2B,
  THR_COMP_NEAREST_NEWL2B,
  THR_COMP_NEW_NEARL2B,
  THR_COMP_NEAR_NEWL2B,
  THR_COMP_NEW_NEWL2B,
  THR_COMP_ZERO_ZEROL2B,

  THR_COMP_NEAR_NEARL3B,
  THR_COMP_NEW_NEARESTL3B,
  THR_COMP_NEAREST_NEWL3B,
  THR_COMP_NEW_NEARL3B,
  THR_COMP_NEAR_NEWL3B,
  THR_COMP_NEW_NEWL3B,
  THR_COMP_ZERO_ZEROL3B,

  THR_COMP_NEAR_NEARGB,
  THR_COMP_NEW_NEARESTGB,
  THR_COMP_NEAREST_NEWGB,
  THR_COMP_NEW_NEARGB,
  THR_COMP_NEAR_NEWGB,
  THR_COMP_NEW_NEWGB,
285
  THR_COMP_ZERO_ZEROGB,
Zoe Liu's avatar
Zoe Liu committed
286

Zoe Liu's avatar
Zoe Liu committed
287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320
#if CONFIG_ALTREF2
  THR_COMP_NEAR_NEARLA2,
  THR_COMP_NEW_NEARESTLA2,
  THR_COMP_NEAREST_NEWLA2,
  THR_COMP_NEW_NEARLA2,
  THR_COMP_NEAR_NEWLA2,
  THR_COMP_NEW_NEWLA2,
  THR_COMP_ZERO_ZEROLA2,

  THR_COMP_NEAR_NEARL2A2,
  THR_COMP_NEW_NEARESTL2A2,
  THR_COMP_NEAREST_NEWL2A2,
  THR_COMP_NEW_NEARL2A2,
  THR_COMP_NEAR_NEWL2A2,
  THR_COMP_NEW_NEWL2A2,
  THR_COMP_ZERO_ZEROL2A2,

  THR_COMP_NEAR_NEARL3A2,
  THR_COMP_NEW_NEARESTL3A2,
  THR_COMP_NEAREST_NEWL3A2,
  THR_COMP_NEW_NEARL3A2,
  THR_COMP_NEAR_NEWL3A2,
  THR_COMP_NEW_NEWL3A2,
  THR_COMP_ZERO_ZEROL3A2,

  THR_COMP_NEAR_NEARGA2,
  THR_COMP_NEW_NEARESTGA2,
  THR_COMP_NEAREST_NEWGA2,
  THR_COMP_NEW_NEARGA2,
  THR_COMP_NEAR_NEWGA2,
  THR_COMP_NEW_NEWGA2,
  THR_COMP_ZERO_ZEROGA2,
#endif  // CONFIG_ALTREF2

Zoe Liu's avatar
Zoe Liu committed
321 322 323 324 325 326 327 328 329
#if CONFIG_EXT_COMP_REFS
  THR_COMP_NEAR_NEARLL2,
  THR_COMP_NEW_NEARESTLL2,
  THR_COMP_NEAREST_NEWLL2,
  THR_COMP_NEW_NEARLL2,
  THR_COMP_NEAR_NEWLL2,
  THR_COMP_NEW_NEWLL2,
  THR_COMP_ZERO_ZEROLL2,

330 331 332 333 334 335 336 337
  THR_COMP_NEAR_NEARLL3,
  THR_COMP_NEW_NEARESTLL3,
  THR_COMP_NEAREST_NEWLL3,
  THR_COMP_NEW_NEARLL3,
  THR_COMP_NEAR_NEWLL3,
  THR_COMP_NEW_NEWLL3,
  THR_COMP_ZERO_ZEROLL3,

Zoe Liu's avatar
Zoe Liu committed
338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353
  THR_COMP_NEAR_NEARLG,
  THR_COMP_NEW_NEARESTLG,
  THR_COMP_NEAREST_NEWLG,
  THR_COMP_NEW_NEARLG,
  THR_COMP_NEAR_NEWLG,
  THR_COMP_NEW_NEWLG,
  THR_COMP_ZERO_ZEROLG,

  THR_COMP_NEAR_NEARBA,
  THR_COMP_NEW_NEARESTBA,
  THR_COMP_NEAREST_NEWBA,
  THR_COMP_NEW_NEARBA,
  THR_COMP_NEAR_NEWBA,
  THR_COMP_NEW_NEWBA,
  THR_COMP_ZERO_ZEROBA,
#endif  // CONFIG_EXT_COMP_REFS
354
#endif  // CONFIG_EXT_REFS
355 356 357

#else  // CONFIG_EXT_INTER

Jingning Han's avatar
Jingning Han committed
358 359
  THR_COMP_NEARLA,
  THR_COMP_NEWLA,
360 361 362 363 364
#if CONFIG_EXT_REFS
  THR_COMP_NEARL2A,
  THR_COMP_NEWL2A,
  THR_COMP_NEARL3A,
  THR_COMP_NEWL3A,
365 366 367 368 369
#endif  // CONFIG_EXT_REFS
  THR_COMP_NEARGA,
  THR_COMP_NEWGA,

#if CONFIG_EXT_REFS
370 371
  THR_COMP_NEARLB,
  THR_COMP_NEWLB,
372 373 374 375
  THR_COMP_NEARL2B,
  THR_COMP_NEWL2B,
  THR_COMP_NEARL3B,
  THR_COMP_NEWL3B,
376 377
  THR_COMP_NEARGB,
  THR_COMP_NEWGB,
Zoe Liu's avatar
Zoe Liu committed
378

Zoe Liu's avatar
Zoe Liu committed
379 380 381 382 383 384 385 386 387 388 389
#if CONFIG_ALTREF2
  THR_COMP_NEARLA2,
  THR_COMP_NEWLA2,
  THR_COMP_NEARL2A2,
  THR_COMP_NEWL2A2,
  THR_COMP_NEARL3A2,
  THR_COMP_NEWL3A2,
  THR_COMP_NEARGA2,
  THR_COMP_NEWGA2,
#endif  // CONFIG_ALTREF2

Zoe Liu's avatar
Zoe Liu committed
390 391 392
#if CONFIG_EXT_COMP_REFS
  THR_COMP_NEARLL2,
  THR_COMP_NEWLL2,
393 394
  THR_COMP_NEARLL3,
  THR_COMP_NEWLL3,
Zoe Liu's avatar
Zoe Liu committed
395 396 397 398 399
  THR_COMP_NEARLG,
  THR_COMP_NEWLG,
  THR_COMP_NEARBA,
  THR_COMP_NEWBA,
#endif  // CONFIG_EXT_COMP_REFS
400
#endif  // CONFIG_EXT_REFS
Jingning Han's avatar
Jingning Han committed
401 402

  THR_COMP_ZEROLA,
403 404 405
#if CONFIG_EXT_REFS
  THR_COMP_ZEROL2A,
  THR_COMP_ZEROL3A,
406 407 408 409
#endif  // CONFIG_EXT_REFS
  THR_COMP_ZEROGA,

#if CONFIG_EXT_REFS
410
  THR_COMP_ZEROLB,
411 412
  THR_COMP_ZEROL2B,
  THR_COMP_ZEROL3B,
413
  THR_COMP_ZEROGB,
Zoe Liu's avatar
Zoe Liu committed
414

Zoe Liu's avatar
Zoe Liu committed
415 416 417 418 419 420 421
#if CONFIG_ALTREF2
  THR_COMP_ZEROLA2,
  THR_COMP_ZEROL2A2,
  THR_COMP_ZEROL3A2,
  THR_COMP_ZEROGA2,
#endif  // CONFIG_ALTEF2

Zoe Liu's avatar
Zoe Liu committed
422 423
#if CONFIG_EXT_COMP_REFS
  THR_COMP_ZEROLL2,
424
  THR_COMP_ZEROLL3,
Zoe Liu's avatar
Zoe Liu committed
425 426 427
  THR_COMP_ZEROLG,
  THR_COMP_ZEROBA,
#endif  // CONFIG_EXT_COMP_REFS
428
#endif  // CONFIG_EXT_REFS
429

430
#endif  // CONFIG_EXT_INTER
Jingning Han's avatar
Jingning Han committed
431 432 433 434 435 436 437 438 439

  THR_H_PRED,
  THR_V_PRED,
  THR_D135_PRED,
  THR_D207_PRED,
  THR_D153_PRED,
  THR_D63_PRED,
  THR_D117_PRED,
  THR_D45_PRED,
440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463

#if CONFIG_EXT_INTER
  THR_COMP_INTERINTRA_ZEROL,
  THR_COMP_INTERINTRA_NEARESTL,
  THR_COMP_INTERINTRA_NEARL,
  THR_COMP_INTERINTRA_NEWL,

#if CONFIG_EXT_REFS
  THR_COMP_INTERINTRA_ZEROL2,
  THR_COMP_INTERINTRA_NEARESTL2,
  THR_COMP_INTERINTRA_NEARL2,
  THR_COMP_INTERINTRA_NEWL2,

  THR_COMP_INTERINTRA_ZEROL3,
  THR_COMP_INTERINTRA_NEARESTL3,
  THR_COMP_INTERINTRA_NEARL3,
  THR_COMP_INTERINTRA_NEWL3,
#endif  // CONFIG_EXT_REFS

  THR_COMP_INTERINTRA_ZEROG,
  THR_COMP_INTERINTRA_NEARESTG,
  THR_COMP_INTERINTRA_NEARG,
  THR_COMP_INTERINTRA_NEWG,

464 465 466 467 468
#if CONFIG_EXT_REFS
  THR_COMP_INTERINTRA_ZEROB,
  THR_COMP_INTERINTRA_NEARESTB,
  THR_COMP_INTERINTRA_NEARB,
  THR_COMP_INTERINTRA_NEWB,
Zoe Liu's avatar
Zoe Liu committed
469 470 471 472 473 474 475

#if CONFIG_ALTREF2
  THR_COMP_INTERINTRA_ZEROA2,
  THR_COMP_INTERINTRA_NEARESTA2,
  THR_COMP_INTERINTRA_NEARA2,
  THR_COMP_INTERINTRA_NEWA2,
#endif  // CONFIG_ALTREF2
476 477
#endif  // CONFIG_EXT_REFS

478 479 480 481 482
  THR_COMP_INTERINTRA_ZEROA,
  THR_COMP_INTERINTRA_NEARESTA,
  THR_COMP_INTERINTRA_NEARA,
  THR_COMP_INTERINTRA_NEWA,
#endif  // CONFIG_EXT_INTER
483
  MAX_MODES
Jingning Han's avatar
Jingning Han committed
484 485 486 487
} THR_MODES;

typedef enum {
  THR_LAST,
488 489 490
#if CONFIG_EXT_REFS
  THR_LAST2,
  THR_LAST3,
491
  THR_BWDR,
Zoe Liu's avatar
Zoe Liu committed
492 493 494
#if CONFIG_ALTREF2
  THR_ALTR2,
#endif  // CONFIG_ALTREF2
495
#endif  // CONFIG_EXT_REFS
Jingning Han's avatar
Jingning Han committed
496 497
  THR_GOLD,
  THR_ALTR,
498

Jingning Han's avatar
Jingning Han committed
499
  THR_COMP_LA,
500 501 502
#if CONFIG_EXT_REFS
  THR_COMP_L2A,
  THR_COMP_L3A,
503 504 505 506
#endif  // CONFIG_EXT_REFS
  THR_COMP_GA,

#if CONFIG_EXT_REFS
507
  THR_COMP_LB,
508 509
  THR_COMP_L2B,
  THR_COMP_L3B,
510
  THR_COMP_GB,
Zoe Liu's avatar
Zoe Liu committed
511 512 513 514 515 516 517

#if CONFIG_ALTREF2
  THR_COMP_LA2,
  THR_COMP_L2A2,
  THR_COMP_L3A2,
  THR_COMP_GA2,
#endif  // CONFIG_ALTREF2
518
#endif  // CONFIG_EXT_REFS
519

Jingning Han's avatar
Jingning Han committed
520
  THR_INTRA,
Zoe Liu's avatar
Zoe Liu committed
521 522

  MAX_REFS
Jingning Han's avatar
Jingning Han committed
523 524 525 526 527 528 529 530 531 532
} THR_MODES_SUB8X8;

typedef struct RD_OPT {
  // Thresh_mult is used to set a threshold for the rd score. A higher value
  // means that we will accept the best mode so far more often. This number
  // is used in combination with the current block size, and thresh_freq_fact
  // to pick a threshold.
  int thresh_mult[MAX_MODES];
  int thresh_mult_sub8x8[MAX_REFS];

533
  int threshes[MAX_SEGMENTS][BLOCK_SIZES_ALL][MAX_MODES];
Jingning Han's avatar
Jingning Han committed
534

535
  int64_t prediction_type_threshes[TOTAL_REFS_PER_FRAME][REFERENCE_MODES];
Jingning Han's avatar
Jingning Han committed
536 537 538 539

  int RDMULT;
} RD_OPT;

Angie Chiang's avatar
Angie Chiang committed
540 541 542 543 544 545 546 547 548
static INLINE void av1_init_rd_stats(RD_STATS *rd_stats) {
#if CONFIG_RD_DEBUG
  int plane;
#endif
  rd_stats->rate = 0;
  rd_stats->dist = 0;
  rd_stats->rdcost = 0;
  rd_stats->sse = 0;
  rd_stats->skip = 1;
549
  rd_stats->zero_rate = 0;
550
  rd_stats->invalid_rate = 0;
551
  rd_stats->ref_rdcost = INT64_MAX;
Yushin Cho's avatar
Yushin Cho committed
552
#if CONFIG_DIST_8X8 && CONFIG_CB4X4
Yushin Cho's avatar
Yushin Cho committed
553 554
  rd_stats->dist_y = 0;
#endif
Angie Chiang's avatar
Angie Chiang committed
555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578
#if CONFIG_RD_DEBUG
  for (plane = 0; plane < MAX_MB_PLANE; ++plane) {
    rd_stats->txb_coeff_cost[plane] = 0;
#if CONFIG_VAR_TX
    {
      int r, c;
      for (r = 0; r < TXB_COEFF_COST_MAP_SIZE; ++r)
        for (c = 0; c < TXB_COEFF_COST_MAP_SIZE; ++c)
          rd_stats->txb_coeff_cost_map[plane][r][c] = 0;
    }
#endif
  }
#endif
}

static INLINE void av1_invalid_rd_stats(RD_STATS *rd_stats) {
#if CONFIG_RD_DEBUG
  int plane;
#endif
  rd_stats->rate = INT_MAX;
  rd_stats->dist = INT64_MAX;
  rd_stats->rdcost = INT64_MAX;
  rd_stats->sse = INT64_MAX;
  rd_stats->skip = 0;
579
  rd_stats->zero_rate = 0;
580
  rd_stats->invalid_rate = 1;
581
  rd_stats->ref_rdcost = INT64_MAX;
Yushin Cho's avatar
Yushin Cho committed
582
#if CONFIG_DIST_8X8 && CONFIG_CB4X4
Yushin Cho's avatar
Yushin Cho committed
583 584
  rd_stats->dist_y = INT64_MAX;
#endif
Angie Chiang's avatar
Angie Chiang committed
585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608
#if CONFIG_RD_DEBUG
  for (plane = 0; plane < MAX_MB_PLANE; ++plane) {
    rd_stats->txb_coeff_cost[plane] = INT_MAX;
#if CONFIG_VAR_TX
    {
      int r, c;
      for (r = 0; r < TXB_COEFF_COST_MAP_SIZE; ++r)
        for (c = 0; c < TXB_COEFF_COST_MAP_SIZE; ++c)
          rd_stats->txb_coeff_cost_map[plane][r][c] = INT_MAX;
    }
#endif
  }
#endif
}

static INLINE void av1_merge_rd_stats(RD_STATS *rd_stats_dst,
                                      const RD_STATS *rd_stats_src) {
#if CONFIG_RD_DEBUG
  int plane;
#endif
  rd_stats_dst->rate += rd_stats_src->rate;
  rd_stats_dst->dist += rd_stats_src->dist;
  rd_stats_dst->sse += rd_stats_src->sse;
  rd_stats_dst->skip &= rd_stats_src->skip;
609
  rd_stats_dst->invalid_rate &= rd_stats_src->invalid_rate;
Yushin Cho's avatar
Yushin Cho committed
610
#if CONFIG_DIST_8X8 && CONFIG_CB4X4
Yushin Cho's avatar
Yushin Cho committed
611 612
  rd_stats_dst->dist_y += rd_stats_src->dist_y;
#endif
Angie Chiang's avatar
Angie Chiang committed
613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632
#if CONFIG_RD_DEBUG
  for (plane = 0; plane < MAX_MB_PLANE; ++plane) {
    rd_stats_dst->txb_coeff_cost[plane] += rd_stats_src->txb_coeff_cost[plane];
#if CONFIG_VAR_TX
    {
      // TODO(angiebird): optimize this part
      int r, c;
      int ref_txb_coeff_cost = 0;
      for (r = 0; r < TXB_COEFF_COST_MAP_SIZE; ++r)
        for (c = 0; c < TXB_COEFF_COST_MAP_SIZE; ++c) {
          rd_stats_dst->txb_coeff_cost_map[plane][r][c] +=
              rd_stats_src->txb_coeff_cost_map[plane][r][c];
          ref_txb_coeff_cost += rd_stats_dst->txb_coeff_cost_map[plane][r][c];
        }
      assert(ref_txb_coeff_cost == rd_stats_dst->txb_coeff_cost[plane]);
    }
#endif
  }
#endif
}
Jingning Han's avatar
Jingning Han committed
633

hui su's avatar
hui su committed
634 635 636 637 638 639 640 641 642 643
static INLINE int av1_get_coeff_token_cost(int token, int eob_val, int is_first,
                                           const int *head_cost_table,
                                           const int *tail_cost_table) {
  if (eob_val == LAST_EOB) return av1_cost_zero(128);
  const int comb_symb = 2 * AOMMIN(token, TWO_TOKEN) - eob_val + is_first;
  int cost = head_cost_table[comb_symb];
  if (token > ONE_TOKEN) cost += tail_cost_table[token - TWO_TOKEN];
  return cost;
}

Jingning Han's avatar
Jingning Han committed
644 645
struct TileInfo;
struct TileDataEnc;
Yaowu Xu's avatar
Yaowu Xu committed
646
struct AV1_COMP;
Jingning Han's avatar
Jingning Han committed
647 648
struct macroblock;

Yaowu Xu's avatar
Yaowu Xu committed
649
int av1_compute_rd_mult(const struct AV1_COMP *cpi, int qindex);
Jingning Han's avatar
Jingning Han committed
650

Yaowu Xu's avatar
Yaowu Xu committed
651
void av1_initialize_rd_consts(struct AV1_COMP *cpi);
Jingning Han's avatar
Jingning Han committed
652

Yaowu Xu's avatar
Yaowu Xu committed
653 654
void av1_initialize_me_consts(const struct AV1_COMP *cpi, MACROBLOCK *x,
                              int qindex);
Jingning Han's avatar
Jingning Han committed
655

Yaowu Xu's avatar
Yaowu Xu committed
656 657
void av1_model_rd_from_var_lapndz(int64_t var, unsigned int n,
                                  unsigned int qstep, int *rate, int64_t *dist);
Jingning Han's avatar
Jingning Han committed
658

659 660
int av1_get_switchable_rate(const AV1_COMMON *const cm, MACROBLOCK *x,
                            const MACROBLOCKD *xd);
Jingning Han's avatar
Jingning Han committed
661

Yaowu Xu's avatar
Yaowu Xu committed
662 663
int av1_raster_block_offset(BLOCK_SIZE plane_bsize, int raster_block,
                            int stride);
Jingning Han's avatar
Jingning Han committed
664

Yaowu Xu's avatar
Yaowu Xu committed
665 666
int16_t *av1_raster_block_offset_int16(BLOCK_SIZE plane_bsize, int raster_block,
                                       int16_t *base);
Jingning Han's avatar
Jingning Han committed
667

Yaowu Xu's avatar
Yaowu Xu committed
668 669
YV12_BUFFER_CONFIG *av1_get_scaled_ref_frame(const struct AV1_COMP *cpi,
                                             int ref_frame);
Jingning Han's avatar
Jingning Han committed
670

Yaowu Xu's avatar
Yaowu Xu committed
671
void av1_init_me_luts(void);
Jingning Han's avatar
Jingning Han committed
672

Yaowu Xu's avatar
Yaowu Xu committed
673 674
void av1_set_mvcost(MACROBLOCK *x, MV_REFERENCE_FRAME ref_frame, int ref,
                    int ref_mv_idx);
675

Yaowu Xu's avatar
Yaowu Xu committed
676 677 678 679
void av1_get_entropy_contexts(BLOCK_SIZE bsize, TX_SIZE tx_size,
                              const struct macroblockd_plane *pd,
                              ENTROPY_CONTEXT t_above[2 * MAX_MIB_SIZE],
                              ENTROPY_CONTEXT t_left[2 * MAX_MIB_SIZE]);
Jingning Han's avatar
Jingning Han committed
680

Yaowu Xu's avatar
Yaowu Xu committed
681
void av1_set_rd_speed_thresholds(struct AV1_COMP *cpi);
Jingning Han's avatar
Jingning Han committed
682

Yaowu Xu's avatar
Yaowu Xu committed
683
void av1_set_rd_speed_thresholds_sub8x8(struct AV1_COMP *cpi);
Jingning Han's avatar
Jingning Han committed
684

Yaowu Xu's avatar
Yaowu Xu committed
685 686 687
void av1_update_rd_thresh_fact(const AV1_COMMON *const cm,
                               int (*fact)[MAX_MODES], int rd_thresh, int bsize,
                               int best_mode_index);
Jingning Han's avatar
Jingning Han committed
688 689 690

static INLINE int rd_less_than_thresh(int64_t best_rd, int thresh,
                                      int thresh_fact) {
691
  return best_rd < ((int64_t)thresh * thresh_fact >> 5) || thresh == INT_MAX;
Jingning Han's avatar
Jingning Han committed
692 693
}

694 695 696
void av1_mv_pred(const struct AV1_COMP *cpi, MACROBLOCK *x,
                 uint8_t *ref_y_buffer, int ref_y_stride, int ref_frame,
                 BLOCK_SIZE block_size);
Jingning Han's avatar
Jingning Han committed
697

698 699 700 701 702
static INLINE void set_error_per_bit(MACROBLOCK *x, int rdmult) {
  x->errorperbit = rdmult >> RD_EPB_SHIFT;
  x->errorperbit += (x->errorperbit == 0);
}

Yaowu Xu's avatar
Yaowu Xu committed
703 704 705 706 707
void av1_setup_pred_block(const MACROBLOCKD *xd,
                          struct buf_2d dst[MAX_MB_PLANE],
                          const YV12_BUFFER_CONFIG *src, int mi_row, int mi_col,
                          const struct scale_factors *scale,
                          const struct scale_factors *scale_uv);
Jingning Han's avatar
Jingning Han committed
708

Yaowu Xu's avatar
Yaowu Xu committed
709 710
int av1_get_intra_cost_penalty(int qindex, int qdelta,
                               aom_bit_depth_t bit_depth);
Jingning Han's avatar
Jingning Han committed
711

712 713 714
void av1_fill_mode_rates(AV1_COMMON *const cm, MACROBLOCK *x,
                         FRAME_CONTEXT *fc);

715 716 717 718
#if CONFIG_LV_MAP
void av1_fill_coeff_costs(MACROBLOCK *x, FRAME_CONTEXT *fc);
#endif

719 720 721
void av1_fill_token_costs_from_cdf(av1_coeff_cost *cost,
                                   coeff_cdf_model (*cdf)[PLANE_TYPES]);

Jingning Han's avatar
Jingning Han committed
722 723 724 725
#ifdef __cplusplus
}  // extern "C"
#endif

Yaowu Xu's avatar
Yaowu Xu committed
726
#endif  // AV1_ENCODER_RD_H_