av1_txfm.h 12.7 KB
Newer Older
Angie Chiang's avatar
Angie Chiang committed
1
/*
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Angie Chiang's avatar
Angie Chiang committed
3
 *
4 5 6 7 8 9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Angie Chiang's avatar
Angie Chiang committed
10
 */
11

Yaowu Xu's avatar
Yaowu Xu committed
12 13
#ifndef AV1_TXFM_H_
#define AV1_TXFM_H_
Angie Chiang's avatar
Angie Chiang committed
14 15

#include <assert.h>
Angie Chiang's avatar
Angie Chiang committed
16 17
#include <math.h>
#include <stdio.h>
Angie Chiang's avatar
Angie Chiang committed
18

19
#include "av1/common/enums.h"
20
#include "av1/common/blockd.h"
Yaowu Xu's avatar
Yaowu Xu committed
21 22
#include "aom/aom_integer.h"
#include "aom_dsp/aom_dsp_common.h"
Angie Chiang's avatar
Angie Chiang committed
23

Angie Chiang's avatar
Angie Chiang committed
24 25 26 27
#ifdef __cplusplus
extern "C" {
#endif

28 29
#define MAX_TXFM_STAGE_NUM 12

Angie Chiang's avatar
Angie Chiang committed
30 31 32 33
static const int cos_bit_min = 10;
static const int cos_bit_max = 16;

// cospi_arr[i][j] = (int)round(cos(M_PI*j/128) * (1<<(cos_bit_min+i)));
34
static const int32_t cospi_arr_data[7][64] = {
clang-format's avatar
clang-format committed
35
  { 1024, 1024, 1023, 1021, 1019, 1016, 1013, 1009, 1004, 999, 993, 987, 980,
clang-format's avatar
clang-format committed
36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57
    972,  964,  955,  946,  936,  926,  915,  903,  891,  878, 865, 851, 837,
    822,  807,  792,  775,  759,  742,  724,  706,  688,  669, 650, 630, 610,
    590,  569,  548,  526,  505,  483,  460,  438,  415,  392, 369, 345, 321,
    297,  273,  249,  224,  200,  175,  150,  125,  100,  75,  50,  25 },
  { 2048, 2047, 2046, 2042, 2038, 2033, 2026, 2018, 2009, 1998, 1987,
    1974, 1960, 1945, 1928, 1911, 1892, 1872, 1851, 1829, 1806, 1782,
    1757, 1730, 1703, 1674, 1645, 1615, 1583, 1551, 1517, 1483, 1448,
    1412, 1375, 1338, 1299, 1260, 1220, 1179, 1138, 1096, 1053, 1009,
    965,  921,  876,  830,  784,  737,  690,  642,  595,  546,  498,
    449,  400,  350,  301,  251,  201,  151,  100,  50 },
  { 4096, 4095, 4091, 4085, 4076, 4065, 4052, 4036, 4017, 3996, 3973,
    3948, 3920, 3889, 3857, 3822, 3784, 3745, 3703, 3659, 3612, 3564,
    3513, 3461, 3406, 3349, 3290, 3229, 3166, 3102, 3035, 2967, 2896,
    2824, 2751, 2675, 2598, 2520, 2440, 2359, 2276, 2191, 2106, 2019,
    1931, 1842, 1751, 1660, 1567, 1474, 1380, 1285, 1189, 1092, 995,
    897,  799,  700,  601,  501,  401,  301,  201,  101 },
  { 8192, 8190, 8182, 8170, 8153, 8130, 8103, 8071, 8035, 7993, 7946,
    7895, 7839, 7779, 7713, 7643, 7568, 7489, 7405, 7317, 7225, 7128,
    7027, 6921, 6811, 6698, 6580, 6458, 6333, 6203, 6070, 5933, 5793,
    5649, 5501, 5351, 5197, 5040, 4880, 4717, 4551, 4383, 4212, 4038,
    3862, 3683, 3503, 3320, 3135, 2948, 2760, 2570, 2378, 2185, 1990,
    1795, 1598, 1401, 1202, 1003, 803,  603,  402,  201 },
clang-format's avatar
clang-format committed
58 59 60
  { 16384, 16379, 16364, 16340, 16305, 16261, 16207, 16143, 16069, 15986, 15893,
    15791, 15679, 15557, 15426, 15286, 15137, 14978, 14811, 14635, 14449, 14256,
    14053, 13842, 13623, 13395, 13160, 12916, 12665, 12406, 12140, 11866, 11585,
clang-format's avatar
clang-format committed
61 62 63
    11297, 11003, 10702, 10394, 10080, 9760,  9434,  9102,  8765,  8423,  8076,
    7723,  7366,  7005,  6639,  6270,  5897,  5520,  5139,  4756,  4370,  3981,
    3590,  3196,  2801,  2404,  2006,  1606,  1205,  804,   402 },
clang-format's avatar
clang-format committed
64 65 66 67
  { 32768, 32758, 32729, 32679, 32610, 32522, 32413, 32286, 32138, 31972, 31786,
    31581, 31357, 31114, 30853, 30572, 30274, 29957, 29622, 29269, 28899, 28511,
    28106, 27684, 27246, 26791, 26320, 25833, 25330, 24812, 24279, 23732, 23170,
    22595, 22006, 21403, 20788, 20160, 19520, 18868, 18205, 17531, 16846, 16151,
clang-format's avatar
clang-format committed
68 69
    15447, 14733, 14010, 13279, 12540, 11793, 11039, 10279, 9512,  8740,  7962,
    7180,  6393,  5602,  4808,  4011,  3212,  2411,  1608,  804 },
clang-format's avatar
clang-format committed
70 71 72 73 74
  { 65536, 65516, 65457, 65358, 65220, 65043, 64827, 64571, 64277, 63944, 63572,
    63162, 62714, 62228, 61705, 61145, 60547, 59914, 59244, 58538, 57798, 57022,
    56212, 55368, 54491, 53581, 52639, 51665, 50660, 49624, 48559, 47464, 46341,
    45190, 44011, 42806, 41576, 40320, 39040, 37736, 36410, 35062, 33692, 32303,
    30893, 29466, 28020, 26558, 25080, 23586, 22078, 20557, 19024, 17479, 15924,
clang-format's avatar
clang-format committed
75
    14359, 12785, 11204, 9616,  8022,  6424,  4821,  3216,  1608 }
clang-format's avatar
clang-format committed
76
};
Angie Chiang's avatar
Angie Chiang committed
77

78 79 80 81
static INLINE const int32_t *cospi_arr(int n) {
  return cospi_arr_data[n - cos_bit_min];
}

Angie Chiang's avatar
Angie Chiang committed
82
static INLINE int32_t round_shift(int32_t value, int bit) {
83
  assert(bit >= 1);
84
  return (value + (1 << (bit - 1))) >> bit;
Angie Chiang's avatar
Angie Chiang committed
85 86 87 88 89 90 91
}

static INLINE void round_shift_array(int32_t *arr, int size, int bit) {
  int i;
  if (bit == 0) {
    return;
  } else {
92 93 94 95 96 97
    if (bit > 0) {
      for (i = 0; i < size; i++) {
        arr[i] = round_shift(arr[i], bit);
      }
    } else {
      for (i = 0; i < size; i++) {
Yaowu Xu's avatar
Yaowu Xu committed
98
        arr[i] = arr[i] * (1 << (-bit));
99
      }
Angie Chiang's avatar
Angie Chiang committed
100 101 102 103 104 105 106 107 108
    }
  }
}

static INLINE int32_t half_btf(int32_t w0, int32_t in0, int32_t w1, int32_t in1,
                               int bit) {
  int32_t result_32 = w0 * in0 + w1 * in1;
#if CONFIG_COEFFICIENT_RANGE_CHECKING
  int64_t result_64 = (int64_t)w0 * (int64_t)in0 + (int64_t)w1 * (int64_t)in1;
109
  if (result_64 < INT32_MIN || result_64 > INT32_MAX) {
Angie Chiang's avatar
Angie Chiang committed
110 111 112 113
    printf("%s %d overflow result_32: %d result_64: %" PRId64
           " w0: %d in0: %d w1: %d in1: "
           "%d\n",
           __FILE__, __LINE__, result_32, result_64, w0, in0, w1, in1);
Angie Chiang's avatar
Angie Chiang committed
114 115 116 117 118 119 120 121 122
    assert(0 && "half_btf overflow");
  }
#endif
  return round_shift(result_32, bit);
}

typedef void (*TxfmFunc)(const int32_t *input, int32_t *output,
                         const int8_t *cos_bit, const int8_t *stage_range);

123 124 125 126 127 128 129 130 131 132
typedef enum TXFM_TYPE {
  TXFM_TYPE_DCT4,
  TXFM_TYPE_DCT8,
  TXFM_TYPE_DCT16,
  TXFM_TYPE_DCT32,
  TXFM_TYPE_DCT64,
  TXFM_TYPE_ADST4,
  TXFM_TYPE_ADST8,
  TXFM_TYPE_ADST16,
  TXFM_TYPE_ADST32,
133 134 135 136
  TXFM_TYPE_IDENTITY4,
  TXFM_TYPE_IDENTITY8,
  TXFM_TYPE_IDENTITY16,
  TXFM_TYPE_IDENTITY32,
137
  TXFM_TYPE_IDENTITY64,
138 139
} TXFM_TYPE;

140
typedef struct TXFM_1D_CFG {
Angie Chiang's avatar
Angie Chiang committed
141
  const int txfm_size;
142
  const int stage_num;
Angie Chiang's avatar
Angie Chiang committed
143

Angie Chiang's avatar
Angie Chiang committed
144
  const int8_t *shift;
145 146 147 148
  const int8_t *stage_range;
  const int8_t *cos_bit;
  const TXFM_TYPE txfm_type;
} TXFM_1D_CFG;
Angie Chiang's avatar
Angie Chiang committed
149

150 151 152
typedef struct TXFM_2D_FLIP_CFG {
  int ud_flip;  // flip upside down
  int lr_flip;  // flip left to right
153 154
  const TXFM_1D_CFG *col_cfg;
  const TXFM_1D_CFG *row_cfg;
155 156
} TXFM_2D_FLIP_CFG;

clang-format's avatar
clang-format committed
157
static INLINE void set_flip_cfg(int tx_type, TXFM_2D_FLIP_CFG *cfg) {
158 159 160 161 162 163 164 165 166
  switch (tx_type) {
    case DCT_DCT:
    case ADST_DCT:
    case DCT_ADST:
    case ADST_ADST:
      cfg->ud_flip = 0;
      cfg->lr_flip = 0;
      break;
#if CONFIG_EXT_TX
167 168 169 170 171 172 173 174
    case IDTX:
    case V_DCT:
    case H_DCT:
    case V_ADST:
    case H_ADST:
      cfg->ud_flip = 0;
      cfg->lr_flip = 0;
      break;
175
    case FLIPADST_DCT:
176
    case FLIPADST_ADST:
177
    case V_FLIPADST:
178 179 180 181
      cfg->ud_flip = 1;
      cfg->lr_flip = 0;
      break;
    case DCT_FLIPADST:
182
    case ADST_FLIPADST:
183
    case H_FLIPADST:
184 185 186 187 188 189 190 191 192 193 194 195 196 197 198
      cfg->ud_flip = 0;
      cfg->lr_flip = 1;
      break;
    case FLIPADST_FLIPADST:
      cfg->ud_flip = 1;
      cfg->lr_flip = 1;
      break;
#endif  // CONFIG_EXT_TX
    default:
      cfg->ud_flip = 0;
      cfg->lr_flip = 0;
      assert(0);
  }
}

199 200 201 202 203 204 205 206 207 208 209 210
#if CONFIG_TXMG
static INLINE int av1_rotate_tx_size(int tx_size) {
  switch (tx_size) {
#if CONFIG_CHROMA_2X2
    case TX_2X2: return TX_2X2;
#endif
    case TX_4X4: return TX_4X4;
    case TX_8X8: return TX_8X8;
    case TX_16X16: return TX_16X16;
    case TX_32X32: return TX_32X32;
#if CONFIG_TX64X64
    case TX_64X64: return TX_64X64;
211 212
    case TX_32X64: return TX_64X32;
    case TX_64X32: return TX_32X64;
213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255
#endif
    case TX_4X8: return TX_8X4;
    case TX_8X4: return TX_4X8;
    case TX_8X16: return TX_16X8;
    case TX_16X8: return TX_8X16;
    case TX_16X32: return TX_32X16;
    case TX_32X16: return TX_16X32;
    case TX_4X16: return TX_16X4;
    case TX_16X4: return TX_4X16;
    case TX_8X32: return TX_32X8;
    case TX_32X8: return TX_8X32;
    default: assert(0); return TX_INVALID;
  }
}

static INLINE int av1_rotate_tx_type(int tx_type) {
  switch (tx_type) {
    case DCT_DCT: return DCT_DCT;
    case ADST_DCT: return DCT_ADST;
    case DCT_ADST: return ADST_DCT;
    case ADST_ADST: return ADST_ADST;
#if CONFIG_EXT_TX
    case FLIPADST_DCT: return DCT_FLIPADST;
    case DCT_FLIPADST: return FLIPADST_DCT;
    case FLIPADST_FLIPADST: return FLIPADST_FLIPADST;
    case ADST_FLIPADST: return FLIPADST_ADST;
    case FLIPADST_ADST: return ADST_FLIPADST;
    case IDTX: return IDTX;
    case V_DCT: return H_DCT;
    case H_DCT: return V_DCT;
    case V_ADST: return H_ADST;
    case H_ADST: return V_ADST;
    case V_FLIPADST: return H_FLIPADST;
    case H_FLIPADST: return V_FLIPADST;
#endif  // CONFIG_EXT_TX
#if CONFIG_MRC_TX
    case MRC_DCT: return MRC_DCT;
#endif  // CONFIG_MRC_TX
    default: assert(0); return TX_TYPES;
  }
}
#endif  // CONFIG_TXMG

256
#if CONFIG_MRC_TX
Sarah Parker's avatar
Sarah Parker committed
257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274
static INLINE int get_mrc_diff_mask_inter(const int16_t *diff, int diff_stride,
                                          uint8_t *mask, int mask_stride,
                                          int width, int height) {
  // placeholder mask generation function
  assert(SIGNAL_MRC_MASK_INTER);
  int n_masked_vals = 0;
  for (int i = 0; i < height; ++i) {
    for (int j = 0; j < width; ++j) {
      mask[i * mask_stride + j] = diff[i * diff_stride + j] > 100 ? 1 : 0;
      n_masked_vals += mask[i * mask_stride + j];
    }
  }
  return n_masked_vals;
}

static INLINE int get_mrc_pred_mask_inter(const uint8_t *pred, int pred_stride,
                                          uint8_t *mask, int mask_stride,
                                          int width, int height) {
275 276 277 278 279 280 281 282 283 284 285
  // placeholder mask generation function
  int n_masked_vals = 0;
  for (int i = 0; i < height; ++i) {
    for (int j = 0; j < width; ++j) {
      mask[i * mask_stride + j] = pred[i * pred_stride + j] > 100 ? 1 : 0;
      n_masked_vals += mask[i * mask_stride + j];
    }
  }
  return n_masked_vals;
}

Sarah Parker's avatar
Sarah Parker committed
286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303
static INLINE int get_mrc_diff_mask_intra(const int16_t *diff, int diff_stride,
                                          uint8_t *mask, int mask_stride,
                                          int width, int height) {
  // placeholder mask generation function
  assert(SIGNAL_MRC_MASK_INTRA);
  int n_masked_vals = 0;
  for (int i = 0; i < height; ++i) {
    for (int j = 0; j < width; ++j) {
      mask[i * mask_stride + j] = diff[i * diff_stride + j] > 100 ? 1 : 0;
      n_masked_vals += mask[i * mask_stride + j];
    }
  }
  return n_masked_vals;
}

static INLINE int get_mrc_pred_mask_intra(const uint8_t *pred, int pred_stride,
                                          uint8_t *mask, int mask_stride,
                                          int width, int height) {
304
  // placeholder mask generation function
305
  int n_masked_vals = 0;
306
  for (int i = 0; i < height; ++i) {
307
    for (int j = 0; j < width; ++j) {
308
      mask[i * mask_stride + j] = pred[i * pred_stride + j] > 100 ? 1 : 0;
309 310
      n_masked_vals += mask[i * mask_stride + j];
    }
311
  }
312 313 314
  return n_masked_vals;
}

Sarah Parker's avatar
Sarah Parker committed
315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333
static INLINE int get_mrc_diff_mask(const int16_t *diff, int diff_stride,
                                    uint8_t *mask, int mask_stride, int width,
                                    int height, int is_inter) {
  if (is_inter) {
    assert(USE_MRC_INTER && "MRC invalid for inter blocks");
    assert(SIGNAL_MRC_MASK_INTER);
    return get_mrc_diff_mask_inter(diff, diff_stride, mask, mask_stride, width,
                                   height);
  } else {
    assert(USE_MRC_INTRA && "MRC invalid for intra blocks");
    assert(SIGNAL_MRC_MASK_INTRA);
    return get_mrc_diff_mask_intra(diff, diff_stride, mask, mask_stride, width,
                                   height);
  }
}

static INLINE int get_mrc_pred_mask(const uint8_t *pred, int pred_stride,
                                    uint8_t *mask, int mask_stride, int width,
                                    int height, int is_inter) {
334 335
  if (is_inter) {
    assert(USE_MRC_INTER && "MRC invalid for inter blocks");
Sarah Parker's avatar
Sarah Parker committed
336 337
    return get_mrc_pred_mask_inter(pred, pred_stride, mask, mask_stride, width,
                                   height);
338 339
  } else {
    assert(USE_MRC_INTRA && "MRC invalid for intra blocks");
Sarah Parker's avatar
Sarah Parker committed
340 341
    return get_mrc_pred_mask_intra(pred, pred_stride, mask, mask_stride, width,
                                   height);
342 343 344
  }
}

345 346
static INLINE int is_valid_mrc_mask(int n_masked_vals, int width, int height) {
  return !(n_masked_vals == 0 || n_masked_vals == (width * height));
347 348 349
}
#endif  // CONFIG_MRC_TX

350 351 352 353 354 355 356
void av1_gen_fwd_stage_range(int8_t *stage_range_col, int8_t *stage_range_row,
                             const TXFM_2D_FLIP_CFG *cfg, int bd);

void av1_gen_inv_stage_range(int8_t *stage_range_col, int8_t *stage_range_row,
                             const TXFM_2D_FLIP_CFG *cfg, int8_t fwd_shift,
                             int bd);

Yaowu Xu's avatar
Yaowu Xu committed
357
TXFM_2D_FLIP_CFG av1_get_fwd_txfm_cfg(int tx_type, int tx_size);
358
#if CONFIG_TX64X64
Yaowu Xu's avatar
Yaowu Xu committed
359
TXFM_2D_FLIP_CFG av1_get_fwd_txfm_64x64_cfg(int tx_type);
360 361 362
TXFM_2D_FLIP_CFG av1_get_fwd_txfm_64x32_cfg(int tx_type);
TXFM_2D_FLIP_CFG av1_get_fwd_txfm_32x64_cfg(int tx_type);
#endif  // CONFIG_TX64X64
Angie Chiang's avatar
Angie Chiang committed
363
TXFM_2D_FLIP_CFG av1_get_inv_txfm_cfg(int tx_type, int tx_size);
364 365 366 367
#ifdef __cplusplus
}
#endif  // __cplusplus

Yaowu Xu's avatar
Yaowu Xu committed
368
#endif  // AV1_TXFM_H_