Commit 3f921084 authored by Debargha Mukherjee's avatar Debargha Mukherjee
Browse files

Support 64x16 / 16x64 transform tables

Adds various tables, scan patterns etc. for 16x64 and 64x16
transforms.
Also adds scan tables for previously missing 4:1 transforms
for intra.
Also adds missing CDFs for filterintra with tx64x64.

Change-Id: I8b16e749741f503f13319e7b7b9685128b723956
parent 440a78b8
......@@ -2044,7 +2044,11 @@ static const uint32_t av1_transform_masks[NUM_EDGE_DIRS][TX_SIZES_ALL] = {
4 - 1, // TX_4X16
16 - 1, // TX_16X4
8 - 1, // TX_8X32
32 - 1 // TX_32X8
32 - 1, // TX_32X8
#if CONFIG_TX64X64
16 - 1, // TX_16X64
64 - 1, // TX_64X16
#endif // CONFIG_TX64X64
},
{
4 - 1, // TX_4X4
......@@ -2067,7 +2071,11 @@ static const uint32_t av1_transform_masks[NUM_EDGE_DIRS][TX_SIZES_ALL] = {
16 - 1, // TX_4X16
4 - 1, // TX_16X4
32 - 1, // TX_8X32
8 - 1 // TX_32X8
8 - 1, // TX_32X8
#if CONFIG_TX64X64
64 - 1, // TX_16X64
16 - 1, // TX_64X16
#endif // CONFIG_TX64X64
}
};
......
......@@ -197,7 +197,7 @@ static INLINE TX_SIZE av1_rotate_tx_size(TX_SIZE tx_size) {
case TX_64X64: return TX_64X64;
case TX_32X64: return TX_64X32;
case TX_64X32: return TX_32X64;
#endif
#endif // CONFIG_TX64X64
case TX_4X8: return TX_8X4;
case TX_8X4: return TX_4X8;
case TX_8X16: return TX_16X8;
......@@ -208,6 +208,10 @@ static INLINE TX_SIZE av1_rotate_tx_size(TX_SIZE tx_size) {
case TX_16X4: return TX_4X16;
case TX_8X32: return TX_32X8;
case TX_32X8: return TX_8X32;
#if CONFIG_TX64X64
case TX_16X64: return TX_64X16;
case TX_64X16: return TX_16X64;
#endif // CONFIG_TX64X64
default: assert(0); return TX_INVALID;
}
}
......
This diff is collapsed.
......@@ -102,6 +102,9 @@ const uint16_t band_count_table[TX_SIZES_ALL][8] = {
#endif // CONFIG_TX64X64
{ 1, 2, 3, 4, 11, 64 - 21, 0 }, { 1, 2, 3, 4, 11, 64 - 21, 0 },
{ 1, 2, 3, 4, 11, 256 - 21, 0 }, { 1, 2, 3, 4, 11, 256 - 21, 0 },
#if CONFIG_TX64X64
{ 1, 2, 3, 4, 11, 1024 - 21, 0 }, { 1, 2, 3, 4, 11, 1024 - 21, 0 },
#endif // CONFIG_TX64X64
};
const uint16_t band_cum_count_table[TX_SIZES_ALL][8] = {
......@@ -118,6 +121,9 @@ const uint16_t band_cum_count_table[TX_SIZES_ALL][8] = {
#endif // CONFIG_TX64X64
{ 0, 1, 3, 6, 10, 21, 64, 0 }, { 0, 1, 3, 6, 10, 21, 64, 0 },
{ 0, 1, 3, 6, 10, 21, 256, 0 }, { 0, 1, 3, 6, 10, 21, 256, 0 },
#if CONFIG_TX64X64
{ 0, 1, 3, 6, 10, 21, 1024, 0 }, { 0, 1, 3, 6, 10, 21, 1024, 0 },
#endif // CONFIG_TX64X64
};
const uint8_t av1_coefband_trans_8x8plus[MAX_TX_SQUARE] = {
......
......@@ -349,6 +349,16 @@ static INLINE int get_entropy_context(TX_SIZE tx_size, const ENTROPY_CONTEXT *a,
above_ec = !!*(const uint64_t *)a;
left_ec = !!*(const uint16_t *)l;
break;
#if CONFIG_TX64X64
case TX_16X64:
above_ec = !!*(const uint32_t *)a;
left_ec = !!(*(const uint64_t *)l | *(const uint64_t *)(l + 8));
break;
case TX_64X16:
above_ec = !!(*(const uint64_t *)a | *(const uint64_t *)(a + 8));
left_ec = !!*(const uint32_t *)l;
break;
#endif // CONFIG_TX64X64
default: assert(0 && "Invalid transform size."); break;
}
return combine_entropy_contexts(above_ec, left_ec);
......
......@@ -1484,11 +1484,23 @@ static const aom_cdf_prob default_filter_intra_mode_cdf[2][CDF_SIZE(
{ AOM_CDF5(32768, 32768, 32768, 32768) } };
static const aom_cdf_prob default_filter_intra_cdfs[TX_SIZES_ALL][CDF_SIZE(2)] =
{ { AOM_CDF2(10985) }, { AOM_CDF2(10985) }, { AOM_CDF2(16645) },
{ AOM_CDF2(27378) }, { AOM_CDF2(10985) }, { AOM_CDF2(10985) },
{ AOM_CDF2(15723) }, { AOM_CDF2(12373) }, { AOM_CDF2(27199) },
{ AOM_CDF2(24217) }, { AOM_CDF2(32767) }, { AOM_CDF2(32767) },
{ AOM_CDF2(32767) }, { AOM_CDF2(32767) } };
{
{ AOM_CDF2(10985) }, { AOM_CDF2(10985) }, { AOM_CDF2(16645) },
{ AOM_CDF2(27378) },
#if CONFIG_TX64X64
{ AOM_CDF2(30378) },
#endif // CONFIG_TX64X64
{ AOM_CDF2(10985) }, { AOM_CDF2(10985) }, { AOM_CDF2(15723) },
{ AOM_CDF2(12373) }, { AOM_CDF2(27199) }, { AOM_CDF2(24217) },
#if CONFIG_TX64X64
{ AOM_CDF2(27378) }, { AOM_CDF2(27378) },
#endif // CONFIG_TX64X64
{ AOM_CDF2(16767) }, { AOM_CDF2(16767) }, { AOM_CDF2(27767) },
{ AOM_CDF2(27767) },
#if CONFIG_TX64X64
{ AOM_CDF2(27378) }, { AOM_CDF2(27378) },
#endif // CONFIG_TX64X64
};
#endif // CONFIG_FILTER_INTRA
// FIXME(someone) need real defaults here
......
......@@ -227,13 +227,17 @@ typedef enum ATTRIBUTE_PACKED {
TX_16X32, // 16x32 transform
TX_32X16, // 32x16 transform
#if CONFIG_TX64X64
TX_32X64, // 32x64 transform
TX_64X32, // 64x32 transform
TX_32X64, // 32x64 transform
TX_64X32, // 64x32 transform
#endif // CONFIG_TX64X64
TX_4X16, // 4x16 transform
TX_16X4, // 16x4 transform
TX_8X32, // 8x32 transform
TX_32X8, // 32x8 transform
#if CONFIG_TX64X64
TX_16X64, // 16x64 transform
TX_64X16, // 64x16 transform
#endif // CONFIG_TX64X64
TX_4X16, // 4x16 transform
TX_16X4, // 16x4 transform
TX_8X32, // 8x32 transform
TX_32X8, // 32x8 transform
TX_SIZES_ALL, // Includes rectangular transforms
TX_SIZES = TX_4X8, // Does NOT include rectangular transforms
TX_INVALID = 255 // Invalid transform size
......
......@@ -494,6 +494,9 @@ void aom_qm_init(AV1_COMMON *cm) {
cm->gqmatrix[q][c][t] = cm->gqmatrix[q][c][TX_32X32];
cm->giqmatrix[q][c][t] = cm->giqmatrix[q][c][TX_32X32];
} else {
#if CONFIG_TX64X64
if (t == TX_16X64 || t == TX_64X16) continue;
#endif // CONFIG_TX64X64
assert(current + size <= QM_TOTAL_SIZE);
cm->gqmatrix[q][c][t] = &wt_matrix_ref[q][c >= 1][current];
cm->giqmatrix[q][c][t] = &iwt_matrix_ref[q][c >= 1][current];
......
This diff is collapsed.
......@@ -595,7 +595,6 @@ void av1_quantize_fp_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
p->dequant_QTX, eob_ptr, sc->scan, sc->iscan,
NULL, NULL, qparam->log_scale);
} else {
#if CONFIG_RECT_TX_EXT
if (qparam->tx_size == TX_4X16 || qparam->tx_size == TX_16X4 ||
qparam->tx_size == TX_8X32 || qparam->tx_size == TX_32X8)
av1_quantize_fp_c(coeff_ptr, n_coeffs, skip_block, p->zbin_QTX,
......@@ -603,7 +602,6 @@ void av1_quantize_fp_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr,
p->dequant_QTX, eob_ptr, sc->scan, sc->iscan);
else
#endif // CONFIG_RECT_TX_EXT
av1_quantize_fp(coeff_ptr, n_coeffs, skip_block, p->zbin_QTX,
p->round_fp_QTX, p->quant_fp_QTX,
p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr,
......@@ -611,10 +609,18 @@ void av1_quantize_fp_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
}
break;
case 1:
av1_quantize_fp_32x32(coeff_ptr, n_coeffs, skip_block, p->zbin_QTX,
p->round_fp_QTX, p->quant_fp_QTX,
p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr,
p->dequant_QTX, eob_ptr, sc->scan, sc->iscan);
#if CONFIG_TX64X64
if (qparam->tx_size == TX_16X64 || qparam->tx_size == TX_64X16)
av1_quantize_fp_32x32_c(coeff_ptr, n_coeffs, skip_block, p->zbin_QTX,
p->round_fp_QTX, p->quant_fp_QTX,
p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr,
p->dequant_QTX, eob_ptr, sc->scan, sc->iscan);
else
#endif // CONFIG_RECT_TX_EXT && CONFIG_TX64X64
av1_quantize_fp_32x32(coeff_ptr, n_coeffs, skip_block, p->zbin_QTX,
p->round_fp_QTX, p->quant_fp_QTX,
p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr,
p->dequant_QTX, eob_ptr, sc->scan, sc->iscan);
break;
#if CONFIG_TX64X64
case 2:
......
......@@ -784,6 +784,20 @@ static void get_entropy_contexts_plane(
for (i = 0; i < num_4x4_h; i += 8)
t_left[i] = !!*(const uint64_t *)&left[i];
break;
case TX_16X64:
for (i = 0; i < num_4x4_w; i += 4)
t_above[i] = !!*(const uint32_t *)&above[i];
for (i = 0; i < num_4x4_h; i += 16)
t_left[i] =
!!(*(const uint64_t *)&left[i] | *(const uint64_t *)&left[i + 8]);
break;
case TX_64X16:
for (i = 0; i < num_4x4_w; i += 16)
t_above[i] =
!!(*(const uint64_t *)&above[i] | *(const uint64_t *)&above[i + 8]);
for (i = 0; i < num_4x4_h; i += 4)
t_left[i] = !!*(const uint32_t *)&left[i];
break;
case TX_64X64:
for (i = 0; i < num_4x4_w; i += 16)
t_above[i] =
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment