av1_fwd_txfm2d.c 14.6 KB
Newer Older
Angie Chiang's avatar
Angie Chiang committed
1
/*
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
Angie Chiang's avatar
Angie Chiang committed
3
 *
4 5 6 7 8 9
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
Angie Chiang's avatar
Angie Chiang committed
10 11
 */

12 13
#include <assert.h>

Yaowu Xu's avatar
Yaowu Xu committed
14
#include "./av1_rtcd.h"
15
#include "aom_dsp/txfm_common.h"
16
#include "av1/common/enums.h"
Yaowu Xu's avatar
Yaowu Xu committed
17
#include "av1/common/av1_fwd_txfm1d.h"
18
#include "av1/common/av1_fwd_txfm1d_cfg.h"
Yaowu Xu's avatar
Yaowu Xu committed
19
#include "av1/common/av1_txfm.h"
Angie Chiang's avatar
Angie Chiang committed
20

Yaowu Xu's avatar
Yaowu Xu committed
21
static INLINE TxfmFunc fwd_txfm_type_to_func(TXFM_TYPE txfm_type) {
22
  switch (txfm_type) {
Yaowu Xu's avatar
Yaowu Xu committed
23 24 25 26
    case TXFM_TYPE_DCT4: return av1_fdct4_new;
    case TXFM_TYPE_DCT8: return av1_fdct8_new;
    case TXFM_TYPE_DCT16: return av1_fdct16_new;
    case TXFM_TYPE_DCT32: return av1_fdct32_new;
27 28 29
#if CONFIG_TX64X64
    case TXFM_TYPE_DCT64: return av1_fdct64_new;
#endif  // CONFIG_TX64X64
Yaowu Xu's avatar
Yaowu Xu committed
30 31 32 33
    case TXFM_TYPE_ADST4: return av1_fadst4_new;
    case TXFM_TYPE_ADST8: return av1_fadst8_new;
    case TXFM_TYPE_ADST16: return av1_fadst16_new;
    case TXFM_TYPE_ADST32: return av1_fadst32_new;
34 35 36 37 38 39
#if CONFIG_EXT_TX
    case TXFM_TYPE_IDENTITY4: return av1_fidentity4_c;
    case TXFM_TYPE_IDENTITY8: return av1_fidentity8_c;
    case TXFM_TYPE_IDENTITY16: return av1_fidentity16_c;
    case TXFM_TYPE_IDENTITY32: return av1_fidentity32_c;
#endif  // CONFIG_EXT_TX
clang-format's avatar
clang-format committed
40
    default: assert(0); return NULL;
41 42 43
  }
}

44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68
void av1_gen_fwd_stage_range(int8_t *stage_range_col, int8_t *stage_range_row,
                             const TXFM_2D_FLIP_CFG *cfg, int bd) {
  // Note when assigning txfm_size_col, we use the txfm_size from the
  // row configuration and vice versa. This is intentionally done to
  // accurately perform rectangular transforms. When the transform is
  // rectangular, the number of columns will be the same as the
  // txfm_size stored in the row cfg struct. It will make no difference
  // for square transforms.
  const int txfm_size_col = cfg->row_cfg->txfm_size;
  const int txfm_size_row = cfg->col_cfg->txfm_size;
  // Take the shift from the larger dimension in the rectangular case.
  const int8_t *shift = (txfm_size_col > txfm_size_row) ? cfg->row_cfg->shift
                                                        : cfg->col_cfg->shift;
  // i < MAX_TXFM_STAGE_NUM will mute above array bounds warning
  for (int i = 0; i < cfg->col_cfg->stage_num && i < MAX_TXFM_STAGE_NUM; ++i) {
    stage_range_col[i] = cfg->col_cfg->stage_range[i] + shift[0] + bd + 1;
  }

  // i < MAX_TXFM_STAGE_NUM will mute above array bounds warning
  for (int i = 0; i < cfg->row_cfg->stage_num && i < MAX_TXFM_STAGE_NUM; ++i) {
    stage_range_row[i] =
        cfg->row_cfg->stage_range[i] + shift[0] + shift[1] + bd + 1;
  }
}

Yaowu Xu's avatar
Yaowu Xu committed
69
static INLINE void fwd_txfm2d_c(const int16_t *input, int32_t *output,
70
                                const int stride, const TXFM_2D_FLIP_CFG *cfg,
71
                                int32_t *buf, int bd) {
72
  int c, r;
73 74 75 76 77 78 79 80 81
  // Note when assigning txfm_size_col, we use the txfm_size from the
  // row configuration and vice versa. This is intentionally done to
  // accurately perform rectangular transforms. When the transform is
  // rectangular, the number of columns will be the same as the
  // txfm_size stored in the row cfg struct. It will make no difference
  // for square transforms.
  const int txfm_size_col = cfg->row_cfg->txfm_size;
  const int txfm_size_row = cfg->col_cfg->txfm_size;
  // Take the shift from the larger dimension in the rectangular case.
Sarah Parker's avatar
Sarah Parker committed
82 83
  const int8_t *shift = (txfm_size_col > txfm_size_row) ? cfg->row_cfg->shift
                                                        : cfg->col_cfg->shift;
84 85 86 87 88 89
  int8_t stage_range_col[MAX_TXFM_STAGE_NUM];
  int8_t stage_range_row[MAX_TXFM_STAGE_NUM];
  assert(cfg->col_cfg->stage_num <= MAX_TXFM_STAGE_NUM);
  assert(cfg->row_cfg->stage_num <= MAX_TXFM_STAGE_NUM);
  av1_gen_fwd_stage_range(stage_range_col, stage_range_row, cfg, bd);

90 91 92 93
  const int8_t *cos_bit_col = cfg->col_cfg->cos_bit;
  const int8_t *cos_bit_row = cfg->row_cfg->cos_bit;
  const TxfmFunc txfm_func_col = fwd_txfm_type_to_func(cfg->col_cfg->txfm_type);
  const TxfmFunc txfm_func_row = fwd_txfm_type_to_func(cfg->row_cfg->txfm_type);
Angie Chiang's avatar
Angie Chiang committed
94

95
  // use output buffer as temp buffer
clang-format's avatar
clang-format committed
96
  int32_t *temp_in = output;
97
  int32_t *temp_out = output + txfm_size_row;
Angie Chiang's avatar
Angie Chiang committed
98 99

  // Columns
100
  for (c = 0; c < txfm_size_col; ++c) {
101
    if (cfg->ud_flip == 0) {
102
      for (r = 0; r < txfm_size_row; ++r) temp_in[r] = input[r * stride + c];
103
    } else {
104
      for (r = 0; r < txfm_size_row; ++r)
105
        // flip upside down
106 107 108 109 110 111 112 113
        temp_in[r] = input[(txfm_size_row - r - 1) * stride + c];
    }
    round_shift_array(temp_in, txfm_size_row, -shift[0]);
    // Multiply everything by Sqrt2 on the larger dimension if the
    // transform is rectangular
    if (txfm_size_col > txfm_size_row) {
      for (r = 0; r < txfm_size_row; ++r)
        temp_in[r] = (int32_t)fdct_round_shift(temp_in[r] * Sqrt2);
114
    }
Angie Chiang's avatar
Angie Chiang committed
115
    txfm_func_col(temp_in, temp_out, cos_bit_col, stage_range_col);
116
    round_shift_array(temp_out, txfm_size_row, -shift[1]);
117
    if (cfg->lr_flip == 0) {
118 119
      for (r = 0; r < txfm_size_row; ++r)
        buf[r * txfm_size_col + c] = temp_out[r];
120
    } else {
121
      for (r = 0; r < txfm_size_row; ++r)
122
        // flip from left to right
123
        buf[r * txfm_size_col + (txfm_size_col - c - 1)] = temp_out[r];
124
    }
Angie Chiang's avatar
Angie Chiang committed
125 126 127
  }

  // Rows
128 129 130 131 132 133 134 135 136 137 138
  for (r = 0; r < txfm_size_row; ++r) {
    // Multiply everything by Sqrt2 on the larger dimension if the
    // transform is rectangular
    if (txfm_size_row > txfm_size_col) {
      for (c = 0; c < txfm_size_col; ++c)
        buf[r * txfm_size_col + c] =
            (int32_t)fdct_round_shift(buf[r * txfm_size_col + c] * Sqrt2);
    }
    txfm_func_row(buf + r * txfm_size_col, output + r * txfm_size_col,
                  cos_bit_row, stage_range_row);
    round_shift_array(output + r * txfm_size_col, txfm_size_col, -shift[2]);
Angie Chiang's avatar
Angie Chiang committed
139 140 141
  }
}

142 143
void av1_fwd_txfm2d_4x8_c(const int16_t *input, int32_t *output, int stride,
                          int tx_type, int bd) {
144 145 146 147 148 149 150 151 152 153 154 155
#if CONFIG_TXMG
  int32_t txfm_buf[4 * 8];
  int16_t rinput[4 * 8];
  int tx_size = TX_4X8;
  int rtx_size = av1_rotate_tx_size(tx_size);
  int rtx_type = av1_rotate_tx_type(tx_type);
  int w = tx_size_wide[tx_size];
  int h = tx_size_high[tx_size];
  int rw = h;
  int rh = w;
  transpose_int16(rinput, rw, input, stride, w, h);
  TXFM_2D_FLIP_CFG cfg = av1_get_fwd_txfm_cfg(rtx_type, rtx_size);
156
  fwd_txfm2d_c(rinput, txfm_buf, rw, &cfg, output, bd);
157 158
  transpose_int32(output, w, txfm_buf, rw, rw, rh);
#else
159 160
  int32_t txfm_buf[4 * 8];
  TXFM_2D_FLIP_CFG cfg = av1_get_fwd_txfm_cfg(tx_type, TX_4X8);
161
  fwd_txfm2d_c(input, output, stride, &cfg, txfm_buf, bd);
162
#endif
163 164 165 166 167 168
}

void av1_fwd_txfm2d_8x4_c(const int16_t *input, int32_t *output, int stride,
                          int tx_type, int bd) {
  int32_t txfm_buf[8 * 4];
  TXFM_2D_FLIP_CFG cfg = av1_get_fwd_txfm_cfg(tx_type, TX_8X4);
169
  fwd_txfm2d_c(input, output, stride, &cfg, txfm_buf, bd);
170 171 172 173
}

void av1_fwd_txfm2d_8x16_c(const int16_t *input, int32_t *output, int stride,
                           int tx_type, int bd) {
174 175 176 177 178 179 180 181 182 183 184 185
#if CONFIG_TXMG
  int32_t txfm_buf[8 * 16];
  int16_t rinput[8 * 16];
  int tx_size = TX_8X16;
  int rtx_size = av1_rotate_tx_size(tx_size);
  int rtx_type = av1_rotate_tx_type(tx_type);
  int w = tx_size_wide[tx_size];
  int h = tx_size_high[tx_size];
  int rw = h;
  int rh = w;
  transpose_int16(rinput, rw, input, stride, w, h);
  TXFM_2D_FLIP_CFG cfg = av1_get_fwd_txfm_cfg(rtx_type, rtx_size);
186
  fwd_txfm2d_c(rinput, txfm_buf, rw, &cfg, output, bd);
187 188
  transpose_int32(output, w, txfm_buf, rw, rw, rh);
#else
189 190
  int32_t txfm_buf[8 * 16];
  TXFM_2D_FLIP_CFG cfg = av1_get_fwd_txfm_cfg(tx_type, TX_8X16);
191
  fwd_txfm2d_c(input, output, stride, &cfg, txfm_buf, bd);
192
#endif
193 194 195 196 197 198
}

void av1_fwd_txfm2d_16x8_c(const int16_t *input, int32_t *output, int stride,
                           int tx_type, int bd) {
  int32_t txfm_buf[16 * 8];
  TXFM_2D_FLIP_CFG cfg = av1_get_fwd_txfm_cfg(tx_type, TX_16X8);
199
  fwd_txfm2d_c(input, output, stride, &cfg, txfm_buf, bd);
200 201 202 203
}

void av1_fwd_txfm2d_16x32_c(const int16_t *input, int32_t *output, int stride,
                            int tx_type, int bd) {
204 205 206 207 208 209 210 211 212 213 214 215
#if CONFIG_TXMG
  int32_t txfm_buf[16 * 32];
  int16_t rinput[16 * 32];
  int tx_size = TX_16X32;
  int rtx_size = av1_rotate_tx_size(tx_size);
  int rtx_type = av1_rotate_tx_type(tx_type);
  int w = tx_size_wide[tx_size];
  int h = tx_size_high[tx_size];
  int rw = h;
  int rh = w;
  transpose_int16(rinput, rw, input, stride, w, h);
  TXFM_2D_FLIP_CFG cfg = av1_get_fwd_txfm_cfg(rtx_type, rtx_size);
216
  fwd_txfm2d_c(rinput, txfm_buf, rw, &cfg, output, bd);
217 218
  transpose_int32(output, w, txfm_buf, rw, rw, rh);
#else
219 220
  int32_t txfm_buf[16 * 32];
  TXFM_2D_FLIP_CFG cfg = av1_get_fwd_txfm_cfg(tx_type, TX_16X32);
221
  fwd_txfm2d_c(input, output, stride, &cfg, txfm_buf, bd);
222
#endif
223 224 225 226 227 228
}

void av1_fwd_txfm2d_32x16_c(const int16_t *input, int32_t *output, int stride,
                            int tx_type, int bd) {
  int32_t txfm_buf[32 * 16];
  TXFM_2D_FLIP_CFG cfg = av1_get_fwd_txfm_cfg(tx_type, TX_32X16);
229
  fwd_txfm2d_c(input, output, stride, &cfg, txfm_buf, bd);
230 231
}

Yaowu Xu's avatar
Yaowu Xu committed
232 233
void av1_fwd_txfm2d_4x4_c(const int16_t *input, int32_t *output, int stride,
                          int tx_type, int bd) {
234
  int32_t txfm_buf[4 * 4];
Yaowu Xu's avatar
Yaowu Xu committed
235
  TXFM_2D_FLIP_CFG cfg = av1_get_fwd_txfm_cfg(tx_type, TX_4X4);
236
  fwd_txfm2d_c(input, output, stride, &cfg, txfm_buf, bd);
Angie Chiang's avatar
Angie Chiang committed
237
}
238

Yaowu Xu's avatar
Yaowu Xu committed
239 240
void av1_fwd_txfm2d_8x8_c(const int16_t *input, int32_t *output, int stride,
                          int tx_type, int bd) {
241
  int32_t txfm_buf[8 * 8];
Yaowu Xu's avatar
Yaowu Xu committed
242
  TXFM_2D_FLIP_CFG cfg = av1_get_fwd_txfm_cfg(tx_type, TX_8X8);
243
  fwd_txfm2d_c(input, output, stride, &cfg, txfm_buf, bd);
244 245
}

Yaowu Xu's avatar
Yaowu Xu committed
246 247
void av1_fwd_txfm2d_16x16_c(const int16_t *input, int32_t *output, int stride,
                            int tx_type, int bd) {
248
  int32_t txfm_buf[16 * 16];
Yaowu Xu's avatar
Yaowu Xu committed
249
  TXFM_2D_FLIP_CFG cfg = av1_get_fwd_txfm_cfg(tx_type, TX_16X16);
250
  fwd_txfm2d_c(input, output, stride, &cfg, txfm_buf, bd);
251 252
}

Yaowu Xu's avatar
Yaowu Xu committed
253 254
void av1_fwd_txfm2d_32x32_c(const int16_t *input, int32_t *output, int stride,
                            int tx_type, int bd) {
255
  int32_t txfm_buf[32 * 32];
Yaowu Xu's avatar
Yaowu Xu committed
256
  TXFM_2D_FLIP_CFG cfg = av1_get_fwd_txfm_cfg(tx_type, TX_32X32);
257
  fwd_txfm2d_c(input, output, stride, &cfg, txfm_buf, bd);
258
}
Angie Chiang's avatar
Angie Chiang committed
259

260
#if CONFIG_TX64X64
Yaowu Xu's avatar
Yaowu Xu committed
261 262
void av1_fwd_txfm2d_64x64_c(const int16_t *input, int32_t *output, int stride,
                            int tx_type, int bd) {
263
  int32_t txfm_buf[64 * 64];
Yaowu Xu's avatar
Yaowu Xu committed
264
  TXFM_2D_FLIP_CFG cfg = av1_get_fwd_txfm_64x64_cfg(tx_type);
265
  fwd_txfm2d_c(input, output, stride, &cfg, txfm_buf, bd);
266 267
}

268 269 270 271 272 273 274 275 276 277 278 279 280 281 282
void av1_fwd_txfm2d_32x64_c(const int16_t *input, int32_t *output, int stride,
                            int tx_type, int bd) {
  int32_t txfm_buf[32 * 64];
  TXFM_2D_FLIP_CFG cfg = av1_get_fwd_txfm_32x64_cfg(tx_type);
  fwd_txfm2d_c(input, output, stride, &cfg, txfm_buf, bd);
}

void av1_fwd_txfm2d_64x32_c(const int16_t *input, int32_t *output, int stride,
                            int tx_type, int bd) {
  int32_t txfm_buf[64 * 32];
  TXFM_2D_FLIP_CFG cfg = av1_get_fwd_txfm_64x32_cfg(tx_type);
  fwd_txfm2d_c(input, output, stride, &cfg, txfm_buf, bd);
}
#endif  // CONFIG_TX64X64

283 284
static const TXFM_1D_CFG *fwd_txfm_col_cfg_ls[TX_TYPES_1D][TX_SIZES] = {
  // DCT
285
  {
286
#if CONFIG_CHROMA_2X2
287 288
      NULL,
#endif
289 290 291
      &fwd_txfm_1d_col_cfg_dct_4, &fwd_txfm_1d_col_cfg_dct_8,
      &fwd_txfm_1d_col_cfg_dct_16, &fwd_txfm_1d_col_cfg_dct_32 },
  // ADST
292
  {
293
#if CONFIG_CHROMA_2X2
294 295
      NULL,
#endif
296 297 298 299
      &fwd_txfm_1d_col_cfg_adst_4, &fwd_txfm_1d_col_cfg_adst_8,
      &fwd_txfm_1d_col_cfg_adst_16, &fwd_txfm_1d_col_cfg_adst_32 },
#if CONFIG_EXT_TX
  // FLIPADST
300
  {
301
#if CONFIG_CHROMA_2X2
302 303
      NULL,
#endif
304 305
      &fwd_txfm_1d_col_cfg_adst_4, &fwd_txfm_1d_col_cfg_adst_8,
      &fwd_txfm_1d_col_cfg_adst_16, &fwd_txfm_1d_col_cfg_adst_32 },
306
  // IDENTITY
307
  {
308
#if CONFIG_CHROMA_2X2
309 310
      NULL,
#endif
311 312
      &fwd_txfm_1d_cfg_identity_4, &fwd_txfm_1d_cfg_identity_8,
      &fwd_txfm_1d_cfg_identity_16, &fwd_txfm_1d_cfg_identity_32 },
313 314 315 316 317
#endif  // CONFIG_EXT_TX
};

static const TXFM_1D_CFG *fwd_txfm_row_cfg_ls[TX_TYPES_1D][TX_SIZES] = {
  // DCT
318
  {
319
#if CONFIG_CHROMA_2X2
320 321
      NULL,
#endif
322 323 324
      &fwd_txfm_1d_row_cfg_dct_4, &fwd_txfm_1d_row_cfg_dct_8,
      &fwd_txfm_1d_row_cfg_dct_16, &fwd_txfm_1d_row_cfg_dct_32 },
  // ADST
325
  {
326
#if CONFIG_CHROMA_2X2
327 328
      NULL,
#endif
329 330 331 332
      &fwd_txfm_1d_row_cfg_adst_4, &fwd_txfm_1d_row_cfg_adst_8,
      &fwd_txfm_1d_row_cfg_adst_16, &fwd_txfm_1d_row_cfg_adst_32 },
#if CONFIG_EXT_TX
  // FLIPADST
333
  {
334
#if CONFIG_CHROMA_2X2
335 336
      NULL,
#endif
337 338
      &fwd_txfm_1d_row_cfg_adst_4, &fwd_txfm_1d_row_cfg_adst_8,
      &fwd_txfm_1d_row_cfg_adst_16, &fwd_txfm_1d_row_cfg_adst_32 },
339
  // IDENTITY
340
  {
341
#if CONFIG_CHROMA_2X2
342 343
      NULL,
#endif
344 345
      &fwd_txfm_1d_cfg_identity_4, &fwd_txfm_1d_cfg_identity_8,
      &fwd_txfm_1d_cfg_identity_16, &fwd_txfm_1d_cfg_identity_32 },
346
#endif  // CONFIG_EXT_TX
347
};
348

Yaowu Xu's avatar
Yaowu Xu committed
349
TXFM_2D_FLIP_CFG av1_get_fwd_txfm_cfg(int tx_type, int tx_size) {
350 351
  TXFM_2D_FLIP_CFG cfg;
  set_flip_cfg(tx_type, &cfg);
Sarah Parker's avatar
Sarah Parker committed
352 353 354 355
  const int tx_type_col = vtx_tab[tx_type];
  const int tx_type_row = htx_tab[tx_type];
  const int tx_size_col = txsize_vert_map[tx_size];
  const int tx_size_row = txsize_horz_map[tx_size];
356 357
  cfg.col_cfg = fwd_txfm_col_cfg_ls[tx_type_col][tx_size_col];
  cfg.row_cfg = fwd_txfm_row_cfg_ls[tx_type_row][tx_size_row];
358 359 360
  return cfg;
}

361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393
#if CONFIG_TX64X64
TXFM_2D_FLIP_CFG av1_get_fwd_txfm_32x64_cfg(int tx_type) {
  TXFM_2D_FLIP_CFG cfg;
  const int tx_type_row = htx_tab[tx_type];
  const int tx_size_row = txsize_horz_map[TX_32X64];
  switch (tx_type) {
    case DCT_DCT:
      cfg.col_cfg = &fwd_txfm_1d_col_cfg_dct_64;
      cfg.row_cfg = fwd_txfm_row_cfg_ls[tx_type_row][tx_size_row];
      cfg.ud_flip = 0;
      cfg.lr_flip = 0;
      break;
    default: assert(0);
  }
  return cfg;
}

TXFM_2D_FLIP_CFG av1_get_fwd_txfm_64x32_cfg(int tx_type) {
  TXFM_2D_FLIP_CFG cfg;
  const int tx_type_col = vtx_tab[tx_type];
  const int tx_size_col = txsize_vert_map[TX_64X32];
  switch (tx_type) {
    case DCT_DCT:
      cfg.col_cfg = fwd_txfm_col_cfg_ls[tx_type_col][tx_size_col];
      cfg.row_cfg = &fwd_txfm_1d_row_cfg_dct_64;
      cfg.ud_flip = 0;
      cfg.lr_flip = 0;
      break;
    default: assert(0);
  }
  return cfg;
}

Yaowu Xu's avatar
Yaowu Xu committed
394
TXFM_2D_FLIP_CFG av1_get_fwd_txfm_64x64_cfg(int tx_type) {
395
  TXFM_2D_FLIP_CFG cfg;
396 397
  switch (tx_type) {
    case DCT_DCT:
398 399
      cfg.col_cfg = &fwd_txfm_1d_col_cfg_dct_64;
      cfg.row_cfg = &fwd_txfm_1d_row_cfg_dct_64;
400 401
      cfg.ud_flip = 0;
      cfg.lr_flip = 0;
402 403
      break;
    default:
404 405
      cfg.ud_flip = 0;
      cfg.lr_flip = 0;
406 407 408 409
      assert(0);
  }
  return cfg;
}
410
#endif  // CONFIG_TX64X64