Commit 48c410a0 authored by Debargha Mukherjee's avatar Debargha Mukherjee

Loopfilter extension to cover 64x64 txfms

Change-Id: I8e02440bfe8e7cb7bb7e92c464981e184367b773
parent 4362b3ec
......@@ -47,6 +47,9 @@ static const uint64_t left_64x64_txform_mask[TX_SIZES] = {
0xffffffffffffffffULL, // TX_8x8
0x5555555555555555ULL, // TX_16x16
0x1111111111111111ULL, // TX_32x32
#if CONFIG_TX64X64
0x0101010101010101ULL, // TX_64x64
#endif // CONFIG_TX64X64
};
// 64 bit masks for above transform size. Each 1 represents a position where
......@@ -74,6 +77,9 @@ static const uint64_t above_64x64_txform_mask[TX_SIZES] = {
0xffffffffffffffffULL, // TX_8x8
0x00ff00ff00ff00ffULL, // TX_16x16
0x000000ff000000ffULL, // TX_32x32
#if CONFIG_TX64X64
0x00000000000000ffULL, // TX_64x64
#endif // CONFIG_TX64X64
};
// 64 bit masks for prediction sizes (left). Each 1 represents a position
......@@ -155,6 +161,9 @@ static const uint16_t left_64x64_txform_mask_uv[TX_SIZES] = {
0xffff, // TX_8x8
0x5555, // TX_16x16
0x1111, // TX_32x32
#if CONFIG_TX64X64
0x0101, // TX_64x64, never used
#endif // CONFIG_TX64X64
};
static const uint16_t above_64x64_txform_mask_uv[TX_SIZES] = {
......@@ -165,6 +174,9 @@ static const uint16_t above_64x64_txform_mask_uv[TX_SIZES] = {
0xffff, // TX_8x8
0x0f0f, // TX_16x16
0x000f, // TX_32x32
#if CONFIG_TX64X64
0x0003, // TX_64x64, never used
#endif // CONFIG_TX64X64
};
// 16 bit left mask to shift and set for each uv prediction size.
......@@ -887,135 +899,150 @@ void av1_setup_mask(AV1_COMMON *const cm, const int mi_row, const int mi_col,
// through the recursive loop structure multiple times.
switch (mip[0]->mbmi.sb_type) {
case BLOCK_64X64: build_masks(lfi_n, mip[0], 0, 0, lfm); break;
case BLOCK_64X32:
build_masks(lfi_n, mip[0], 0, 0, lfm);
case BLOCK_64X32: build_masks(lfi_n, mip[0], 0, 0, lfm);
#if CONFIG_SUPERTX && CONFIG_TX64X64
if (supertx_enabled(&mip->mbmi)) break;
#endif // CONFIG_SUPERTX && CONFIG_TX64X64
mip2 = mip + mode_info_stride * 4;
if (4 >= max_rows) break;
build_masks(lfi_n, mip2[0], 32, 8, lfm);
break;
case BLOCK_32X64:
build_masks(lfi_n, mip[0], 0, 0, lfm);
case BLOCK_32X64: build_masks(lfi_n, mip[0], 0, 0, lfm);
#if CONFIG_SUPERTX && CONFIG_TX64X64
if (supertx_enabled(&mip->mbmi)) break;
#endif // CONFIG_SUPERTX && CONFIG_TX64X64
mip2 = mip + 4;
if (4 >= max_cols) break;
build_masks(lfi_n, mip2[0], 4, 2, lfm);
break;
default:
for (idx_32 = 0; idx_32 < 4; mip += offset_32[idx_32], ++idx_32) {
const int shift_y_32 = shift_32_y[idx_32];
const int shift_uv_32 = shift_32_uv[idx_32];
const int mi_32_col_offset = ((idx_32 & 1) << 2);
const int mi_32_row_offset = ((idx_32 >> 1) << 2);
if (mi_32_col_offset >= max_cols || mi_32_row_offset >= max_rows)
continue;
switch (mip[0]->mbmi.sb_type) {
case BLOCK_32X32:
build_masks(lfi_n, mip[0], shift_y_32, shift_uv_32, lfm);
break;
case BLOCK_32X16:
build_masks(lfi_n, mip[0], shift_y_32, shift_uv_32, lfm);
#if CONFIG_SUPERTX && CONFIG_TX64X64
if (mip->mbmi.tx_size == TX_64X64) {
build_masks(cm, mip, 0, 0, lfm);
} else {
#endif // CONFIG_SUPERTX && CONFIG_TX64X64
for (idx_32 = 0; idx_32 < 4; mip += offset_32[idx_32], ++idx_32) {
const int shift_y_32 = shift_32_y[idx_32];
const int shift_uv_32 = shift_32_uv[idx_32];
const int mi_32_col_offset = ((idx_32 & 1) << 2);
const int mi_32_row_offset = ((idx_32 >> 1) << 2);
if (mi_32_col_offset >= max_cols || mi_32_row_offset >= max_rows)
continue;
switch (mip[0]->mbmi.sb_type) {
case BLOCK_32X32:
build_masks(lfi_n, mip[0], shift_y_32, shift_uv_32, lfm);
break;
case BLOCK_32X16:
build_masks(lfi_n, mip[0], shift_y_32, shift_uv_32, lfm);
#if CONFIG_SUPERTX
if (supertx_enabled(&mip[0]->mbmi)) break;
if (supertx_enabled(&mip[0]->mbmi)) break;
#endif
if (mi_32_row_offset + 2 >= max_rows) continue;
mip2 = mip + mode_info_stride * 2;
build_masks(lfi_n, mip2[0], shift_y_32 + 16, shift_uv_32 + 4, lfm);
break;
case BLOCK_16X32:
build_masks(lfi_n, mip[0], shift_y_32, shift_uv_32, lfm);
if (mi_32_row_offset + 2 >= max_rows) continue;
mip2 = mip + mode_info_stride * 2;
build_masks(lfi_n, mip2[0], shift_y_32 + 16, shift_uv_32 + 4,
lfm);
break;
case BLOCK_16X32:
build_masks(lfi_n, mip[0], shift_y_32, shift_uv_32, lfm);
#if CONFIG_SUPERTX
if (supertx_enabled(&mip[0]->mbmi)) break;
if (supertx_enabled(&mip[0]->mbmi)) break;
#endif
if (mi_32_col_offset + 2 >= max_cols) continue;
mip2 = mip + 2;
build_masks(lfi_n, mip2[0], shift_y_32 + 2, shift_uv_32 + 1, lfm);
break;
default:
#if CONFIG_SUPERTX
if (mip[0]->mbmi.tx_size == TX_32X32) {
build_masks(lfi_n, mip[0], shift_y_32, shift_uv_32, lfm);
if (mi_32_col_offset + 2 >= max_cols) continue;
mip2 = mip + 2;
build_masks(lfi_n, mip2[0], shift_y_32 + 2, shift_uv_32 + 1, lfm);
break;
}
default:
#if CONFIG_SUPERTX
if (mip[0]->mbmi.tx_size == TX_32X32) {
build_masks(lfi_n, mip[0], shift_y_32, shift_uv_32, lfm);
break;
}
#endif
for (idx_16 = 0; idx_16 < 4; mip += offset_16[idx_16], ++idx_16) {
const int shift_y_32_16 = shift_y_32 + shift_16_y[idx_16];
const int shift_uv_32_16 = shift_uv_32 + shift_16_uv[idx_16];
const int mi_16_col_offset =
mi_32_col_offset + ((idx_16 & 1) << 1);
const int mi_16_row_offset =
mi_32_row_offset + ((idx_16 >> 1) << 1);
if (mi_16_col_offset >= max_cols || mi_16_row_offset >= max_rows)
continue;
switch (mip[0]->mbmi.sb_type) {
case BLOCK_16X16:
build_masks(lfi_n, mip[0], shift_y_32_16, shift_uv_32_16,
lfm);
break;
case BLOCK_16X8:
for (idx_16 = 0; idx_16 < 4; mip += offset_16[idx_16], ++idx_16) {
const int shift_y_32_16 = shift_y_32 + shift_16_y[idx_16];
const int shift_uv_32_16 = shift_uv_32 + shift_16_uv[idx_16];
const int mi_16_col_offset =
mi_32_col_offset + ((idx_16 & 1) << 1);
const int mi_16_row_offset =
mi_32_row_offset + ((idx_16 >> 1) << 1);
if (mi_16_col_offset >= max_cols ||
mi_16_row_offset >= max_rows)
continue;
switch (mip[0]->mbmi.sb_type) {
case BLOCK_16X16:
build_masks(lfi_n, mip[0], shift_y_32_16, shift_uv_32_16,
lfm);
break;
case BLOCK_16X8:
#if CONFIG_SUPERTX
if (supertx_enabled(&mip[0]->mbmi)) break;
if (supertx_enabled(&mip[0]->mbmi)) break;
#endif
build_masks(lfi_n, mip[0], shift_y_32_16, shift_uv_32_16,
lfm);
if (mi_16_row_offset + 1 >= max_rows) continue;
mip2 = mip + mode_info_stride;
build_y_mask(lfi_n, mip2[0], shift_y_32_16 + 8,
build_masks(lfi_n, mip[0], shift_y_32_16, shift_uv_32_16,
lfm);
if (mi_16_row_offset + 1 >= max_rows) continue;
mip2 = mip + mode_info_stride;
build_y_mask(lfi_n, mip2[0], shift_y_32_16 + 8,
#if CONFIG_SUPERTX
0,
0,
#endif
lfm);
break;
case BLOCK_8X16:
lfm);
break;
case BLOCK_8X16:
#if CONFIG_SUPERTX
if (supertx_enabled(&mip[0]->mbmi)) break;
if (supertx_enabled(&mip[0]->mbmi)) break;
#endif
build_masks(lfi_n, mip[0], shift_y_32_16, shift_uv_32_16,
lfm);
if (mi_16_col_offset + 1 >= max_cols) continue;
mip2 = mip + 1;
build_y_mask(lfi_n, mip2[0], shift_y_32_16 + 1,
build_masks(lfi_n, mip[0], shift_y_32_16, shift_uv_32_16,
lfm);
if (mi_16_col_offset + 1 >= max_cols) continue;
mip2 = mip + 1;
build_y_mask(lfi_n, mip2[0], shift_y_32_16 + 1,
#if CONFIG_SUPERTX
0,
0,
#endif
lfm);
break;
default: {
const int shift_y_32_16_8_zero = shift_y_32_16 + shift_8_y[0];
lfm);
break;
default: {
const int shift_y_32_16_8_zero =
shift_y_32_16 + shift_8_y[0];
#if CONFIG_SUPERTX
if (mip[0]->mbmi.tx_size == TX_16X16) {
if (mip[0]->mbmi.tx_size == TX_16X16) {
build_masks(lfi_n, mip[0], shift_y_32_16_8_zero,
shift_uv_32_16, lfm);
break;
}
#endif
build_masks(lfi_n, mip[0], shift_y_32_16_8_zero,
shift_uv_32_16, lfm);
break;
}
#endif
build_masks(lfi_n, mip[0], shift_y_32_16_8_zero,
shift_uv_32_16, lfm);
mip += offset[0];
for (idx_8 = 1; idx_8 < 4; mip += offset[idx_8], ++idx_8) {
const int shift_y_32_16_8 =
shift_y_32_16 + shift_8_y[idx_8];
const int mi_8_col_offset =
mi_16_col_offset + ((idx_8 & 1));
const int mi_8_row_offset =
mi_16_row_offset + ((idx_8 >> 1));
if (mi_8_col_offset >= max_cols ||
mi_8_row_offset >= max_rows)
continue;
build_y_mask(lfi_n, mip[0], shift_y_32_16_8,
mip += offset[0];
for (idx_8 = 1; idx_8 < 4; mip += offset[idx_8], ++idx_8) {
const int shift_y_32_16_8 =
shift_y_32_16 + shift_8_y[idx_8];
const int mi_8_col_offset =
mi_16_col_offset + ((idx_8 & 1));
const int mi_8_row_offset =
mi_16_row_offset + ((idx_8 >> 1));
if (mi_8_col_offset >= max_cols ||
mi_8_row_offset >= max_rows)
continue;
build_y_mask(lfi_n, mip[0], shift_y_32_16_8,
#if CONFIG_SUPERTX
supertx_enabled(&mip[0]->mbmi),
supertx_enabled(&mip[0]->mbmi),
#endif
lfm);
lfm);
}
break;
}
break;
}
}
}
break;
break;
}
}
#if CONFIG_SUPERTX && CONFIG_TX64X64
}
#endif // CONFIG_SUPERTX && CONFIG_TX64X64
break;
}
// The largest loopfilter we have is 16x16 so we use the 16x16 mask
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment