Commit 6e4955d4 authored by Jingning Han's avatar Jingning Han
Browse files

Rework loop filter tx size selection

Update and capture the effective transform block size per color
plane.

Change-Id: Ib6e0e7abb3973db6b8d511ee7c9948aaab048788
parent eaddeee1
...@@ -125,6 +125,11 @@ void av1_free_context_buffers(AV1_COMMON *cm) { ...@@ -125,6 +125,11 @@ void av1_free_context_buffers(AV1_COMMON *cm) {
#if CONFIG_VAR_TX #if CONFIG_VAR_TX
aom_free(cm->above_txfm_context); aom_free(cm->above_txfm_context);
cm->above_txfm_context = NULL; cm->above_txfm_context = NULL;
for (i = 0; i < MAX_MB_PLANE; ++i) {
aom_free(cm->top_txfm_context[i]);
cm->top_txfm_context[i] = NULL;
}
#endif #endif
} }
...@@ -170,6 +175,14 @@ int av1_alloc_context_buffers(AV1_COMMON *cm, int width, int height) { ...@@ -170,6 +175,14 @@ int av1_alloc_context_buffers(AV1_COMMON *cm, int width, int height) {
cm->above_txfm_context = (TXFM_CONTEXT *)aom_calloc( cm->above_txfm_context = (TXFM_CONTEXT *)aom_calloc(
aligned_mi_cols << TX_UNIT_WIDE_LOG2, sizeof(*cm->above_txfm_context)); aligned_mi_cols << TX_UNIT_WIDE_LOG2, sizeof(*cm->above_txfm_context));
if (!cm->above_txfm_context) goto fail; if (!cm->above_txfm_context) goto fail;
for (i = 0; i < MAX_MB_PLANE; ++i) {
aom_free(cm->top_txfm_context[i]);
cm->top_txfm_context[i] =
(TXFM_CONTEXT *)aom_calloc(aligned_mi_cols << TX_UNIT_WIDE_LOG2,
sizeof(*cm->top_txfm_context[0]));
if (!cm->top_txfm_context[i]) goto fail;
}
#endif #endif
cm->above_context_alloc_cols = aligned_mi_cols; cm->above_context_alloc_cols = aligned_mi_cols;
......
...@@ -1363,7 +1363,7 @@ typedef struct { ...@@ -1363,7 +1363,7 @@ typedef struct {
// the non420 case). // the non420 case).
// Note: 'row_masks_ptr' and/or 'col_masks_ptr' can be passed NULL. // Note: 'row_masks_ptr' and/or 'col_masks_ptr' can be passed NULL.
static void get_filter_level_and_masks_non420( static void get_filter_level_and_masks_non420(
AV1_COMMON *const cm, const struct macroblockd_plane *const plane, AV1_COMMON *const cm, const struct macroblockd_plane *const plane, int pl,
MODE_INFO **mib, int mi_row, int mi_col, int idx_r, uint8_t *const lfl_r, MODE_INFO **mib, int mi_row, int mi_col, int idx_r, uint8_t *const lfl_r,
unsigned int *const mask_4x4_int_r_ptr, unsigned int *const mask_4x4_int_r_ptr,
unsigned int *const mask_4x4_int_c_ptr, FilterMasks *const row_masks_ptr, unsigned int *const mask_4x4_int_c_ptr, FilterMasks *const row_masks_ptr,
...@@ -1400,10 +1400,6 @@ static void get_filter_level_and_masks_non420( ...@@ -1400,10 +1400,6 @@ static void get_filter_level_and_masks_non420(
(num_4x4_blocks_high_lookup[sb_type] > 1) ? !blk_row : 1; (num_4x4_blocks_high_lookup[sb_type] > 1) ? !blk_row : 1;
const int skip_this_r = skip_this && !block_edge_above; const int skip_this_r = skip_this && !block_edge_above;
#if CONFIG_VAR_TX
const TX_SIZE mb_tx_size = mbmi->inter_tx_size[blk_row][blk_col];
#endif
TX_SIZE tx_size = (plane->plane_type == PLANE_TYPE_UV) TX_SIZE tx_size = (plane->plane_type == PLANE_TYPE_UV)
? get_uv_tx_size(mbmi, plane) ? get_uv_tx_size(mbmi, plane)
: mbmi->tx_size; : mbmi->tx_size;
...@@ -1420,8 +1416,15 @@ static void get_filter_level_and_masks_non420( ...@@ -1420,8 +1416,15 @@ static void get_filter_level_and_masks_non420(
#if CONFIG_VAR_TX #if CONFIG_VAR_TX
if (is_inter_block(mbmi) && !mbmi->skip) { if (is_inter_block(mbmi) && !mbmi->skip) {
const int tx_row_idx =
(blk_row * mi_size_high[BLOCK_8X8] << TX_UNIT_HIGH_LOG2) >> 1;
const int tx_col_idx =
(blk_col * mi_size_wide[BLOCK_8X8] << TX_UNIT_WIDE_LOG2) >> 1;
const BLOCK_SIZE bsize =
AOMMAX(BLOCK_4X4, get_plane_block_size(mbmi->sb_type, plane));
const TX_SIZE mb_tx_size = mbmi->inter_tx_size[tx_row_idx][tx_col_idx];
tx_size = (plane->plane_type == PLANE_TYPE_UV) tx_size = (plane->plane_type == PLANE_TYPE_UV)
? uv_txsize_lookup[sb_type][mb_tx_size][ss_x][ss_y] ? uv_txsize_lookup[bsize][mb_tx_size][0][0]
: mb_tx_size; : mb_tx_size;
} }
#endif #endif
...@@ -1434,18 +1437,29 @@ static void get_filter_level_and_masks_non420( ...@@ -1434,18 +1437,29 @@ static void get_filter_level_and_masks_non420(
#endif #endif
#if CONFIG_VAR_TX #if CONFIG_VAR_TX
TX_SIZE tx_size_r = AOMMIN( TX_SIZE tx_size_r, tx_size_c;
tx_size, cm->above_txfm_context[(mi_col + c) << TX_UNIT_WIDE_LOG2]);
TX_SIZE tx_size_c = const int tx_wide =
AOMMIN(tx_size, cm->left_txfm_context[((mi_row + r) & MAX_MIB_MASK) AOMMIN(tx_size_wide[tx_size],
<< TX_UNIT_HIGH_LOG2]); tx_size_wide[cm->top_txfm_context[pl][(mi_col + idx_c)
<< TX_UNIT_WIDE_LOG2]]);
cm->above_txfm_context[(mi_col + c) << TX_UNIT_WIDE_LOG2] = tx_size; const int tx_high = AOMMIN(
cm->left_txfm_context[((mi_row + r) & MAX_MIB_MASK) << TX_UNIT_HIGH_LOG2] = tx_size_high[tx_size],
tx_size; tx_size_high[cm->left_txfm_context[pl][((mi_row + idx_r) & MAX_MIB_MASK)
<< TX_UNIT_HIGH_LOG2]]);
tx_size_c = get_sqr_tx_size(tx_wide);
tx_size_r = get_sqr_tx_size(tx_high);
memset(cm->top_txfm_context[pl] + ((mi_col + idx_c) << TX_UNIT_WIDE_LOG2),
tx_size, mi_size_wide[BLOCK_8X8] << TX_UNIT_WIDE_LOG2);
memset(cm->left_txfm_context[pl] +
(((mi_row + idx_r) & MAX_MIB_MASK) << TX_UNIT_HIGH_LOG2),
tx_size, mi_size_high[BLOCK_8X8] << TX_UNIT_HIGH_LOG2);
#else #else
TX_SIZE tx_size_c = txsize_horz_map[tx_size]; TX_SIZE tx_size_c = txsize_horz_map[tx_size];
TX_SIZE tx_size_r = txsize_vert_map[tx_size]; TX_SIZE tx_size_r = txsize_vert_map[tx_size];
(void)pl;
#endif // CONFIG_VAR_TX #endif // CONFIG_VAR_TX
if (tx_size_c == TX_32X32) if (tx_size_c == TX_32X32)
...@@ -1530,8 +1544,8 @@ static void get_filter_level_and_masks_non420( ...@@ -1530,8 +1544,8 @@ static void get_filter_level_and_masks_non420(
void av1_filter_block_plane_non420_ver(AV1_COMMON *const cm, void av1_filter_block_plane_non420_ver(AV1_COMMON *const cm,
struct macroblockd_plane *plane, struct macroblockd_plane *plane,
MODE_INFO **mib, int mi_row, MODE_INFO **mib, int mi_row, int mi_col,
int mi_col) { int pl) {
const int ss_y = plane->subsampling_y; const int ss_y = plane->subsampling_y;
const int row_step = mi_size_high[BLOCK_8X8] << ss_y; const int row_step = mi_size_high[BLOCK_8X8] << ss_y;
struct buf_2d *const dst = &plane->dst; struct buf_2d *const dst = &plane->dst;
...@@ -1544,7 +1558,7 @@ void av1_filter_block_plane_non420_ver(AV1_COMMON *const cm, ...@@ -1544,7 +1558,7 @@ void av1_filter_block_plane_non420_ver(AV1_COMMON *const cm,
unsigned int mask_4x4_int; unsigned int mask_4x4_int;
FilterMasks col_masks; FilterMasks col_masks;
const int r = idx_r >> mi_height_log2_lookup[BLOCK_8X8]; const int r = idx_r >> mi_height_log2_lookup[BLOCK_8X8];
get_filter_level_and_masks_non420(cm, plane, mib, mi_row, mi_col, idx_r, get_filter_level_and_masks_non420(cm, plane, pl, mib, mi_row, mi_col, idx_r,
&lfl[r][0], NULL, &mask_4x4_int, NULL, &lfl[r][0], NULL, &mask_4x4_int, NULL,
&col_masks); &col_masks);
...@@ -1579,8 +1593,8 @@ void av1_filter_block_plane_non420_ver(AV1_COMMON *const cm, ...@@ -1579,8 +1593,8 @@ void av1_filter_block_plane_non420_ver(AV1_COMMON *const cm,
void av1_filter_block_plane_non420_hor(AV1_COMMON *const cm, void av1_filter_block_plane_non420_hor(AV1_COMMON *const cm,
struct macroblockd_plane *plane, struct macroblockd_plane *plane,
MODE_INFO **mib, int mi_row, MODE_INFO **mib, int mi_row, int mi_col,
int mi_col) { int pl) {
const int ss_y = plane->subsampling_y; const int ss_y = plane->subsampling_y;
const int row_step = mi_size_high[BLOCK_8X8] << ss_y; const int row_step = mi_size_high[BLOCK_8X8] << ss_y;
struct buf_2d *const dst = &plane->dst; struct buf_2d *const dst = &plane->dst;
...@@ -1592,7 +1606,7 @@ void av1_filter_block_plane_non420_hor(AV1_COMMON *const cm, ...@@ -1592,7 +1606,7 @@ void av1_filter_block_plane_non420_hor(AV1_COMMON *const cm,
for (idx_r = 0; idx_r < cm->mib_size && mi_row + idx_r < cm->mi_rows; for (idx_r = 0; idx_r < cm->mib_size && mi_row + idx_r < cm->mi_rows;
idx_r += row_step) { idx_r += row_step) {
const int r = idx_r >> mi_height_log2_lookup[BLOCK_8X8]; const int r = idx_r >> mi_height_log2_lookup[BLOCK_8X8];
get_filter_level_and_masks_non420(cm, plane, mib, mi_row, mi_col, idx_r, get_filter_level_and_masks_non420(cm, plane, pl, mib, mi_row, mi_col, idx_r,
&lfl[r][0], mask_4x4_int + r, NULL, &lfl[r][0], mask_4x4_int + r, NULL,
row_masks_array + r, NULL); row_masks_array + r, NULL);
} }
...@@ -2192,12 +2206,15 @@ void av1_loop_filter_rows(YV12_BUFFER_CONFIG *frame_buffer, AV1_COMMON *cm, ...@@ -2192,12 +2206,15 @@ void av1_loop_filter_rows(YV12_BUFFER_CONFIG *frame_buffer, AV1_COMMON *cm,
int mi_row, mi_col; int mi_row, mi_col;
#if CONFIG_VAR_TX #if CONFIG_VAR_TX
memset(cm->above_txfm_context, TX_SIZES, cm->mi_cols << TX_UNIT_WIDE_LOG2); for (int i = 0; i < MAX_MB_PLANE; ++i)
memset(cm->top_txfm_context[i], TX_32X32, cm->mi_cols << TX_UNIT_WIDE_LOG2);
#endif // CONFIG_VAR_TX #endif // CONFIG_VAR_TX
for (mi_row = start; mi_row < stop; mi_row += cm->mib_size) { for (mi_row = start; mi_row < stop; mi_row += cm->mib_size) {
MODE_INFO **mi = cm->mi_grid_visible + mi_row * cm->mi_stride; MODE_INFO **mi = cm->mi_grid_visible + mi_row * cm->mi_stride;
#if CONFIG_VAR_TX #if CONFIG_VAR_TX
memset(cm->left_txfm_context, TX_SIZES, MAX_MIB_SIZE << TX_UNIT_WIDE_LOG2); for (int i = 0; i < MAX_MB_PLANE; ++i)
memset(cm->left_txfm_context[i], TX_32X32, MAX_MIB_SIZE
<< TX_UNIT_WIDE_LOG2);
#endif // CONFIG_VAR_TX #endif // CONFIG_VAR_TX
for (mi_col = 0; mi_col < cm->mi_cols; mi_col += cm->mib_size) { for (mi_col = 0; mi_col < cm->mi_cols; mi_col += cm->mib_size) {
int plane; int plane;
...@@ -2206,9 +2223,9 @@ void av1_loop_filter_rows(YV12_BUFFER_CONFIG *frame_buffer, AV1_COMMON *cm, ...@@ -2206,9 +2223,9 @@ void av1_loop_filter_rows(YV12_BUFFER_CONFIG *frame_buffer, AV1_COMMON *cm,
for (plane = 0; plane < num_planes; ++plane) { for (plane = 0; plane < num_planes; ++plane) {
av1_filter_block_plane_non420_ver(cm, &planes[plane], mi + mi_col, av1_filter_block_plane_non420_ver(cm, &planes[plane], mi + mi_col,
mi_row, mi_col); mi_row, mi_col, plane);
av1_filter_block_plane_non420_hor(cm, &planes[plane], mi + mi_col, av1_filter_block_plane_non420_hor(cm, &planes[plane], mi + mi_col,
mi_row, mi_col); mi_row, mi_col, plane);
} }
} }
} }
...@@ -2282,9 +2299,9 @@ void av1_loop_filter_rows(YV12_BUFFER_CONFIG *frame_buffer, AV1_COMMON *cm, ...@@ -2282,9 +2299,9 @@ void av1_loop_filter_rows(YV12_BUFFER_CONFIG *frame_buffer, AV1_COMMON *cm,
break; break;
case LF_PATH_SLOW: case LF_PATH_SLOW:
av1_filter_block_plane_non420_ver(cm, &planes[plane], mi + mi_col, av1_filter_block_plane_non420_ver(cm, &planes[plane], mi + mi_col,
mi_row, mi_col); mi_row, mi_col, plane);
av1_filter_block_plane_non420_hor(cm, &planes[plane], mi + mi_col, av1_filter_block_plane_non420_hor(cm, &planes[plane], mi + mi_col,
mi_row, mi_col); mi_row, mi_col, plane);
break; break;
} }
......
...@@ -115,11 +115,11 @@ void av1_filter_block_plane_ss11_hor(struct AV1Common *const cm, ...@@ -115,11 +115,11 @@ void av1_filter_block_plane_ss11_hor(struct AV1Common *const cm,
void av1_filter_block_plane_non420_ver(struct AV1Common *const cm, void av1_filter_block_plane_non420_ver(struct AV1Common *const cm,
struct macroblockd_plane *plane, struct macroblockd_plane *plane,
MODE_INFO **mi_8x8, int mi_row, MODE_INFO **mi_8x8, int mi_row,
int mi_col); int mi_col, int pl);
void av1_filter_block_plane_non420_hor(struct AV1Common *const cm, void av1_filter_block_plane_non420_hor(struct AV1Common *const cm,
struct macroblockd_plane *plane, struct macroblockd_plane *plane,
MODE_INFO **mi_8x8, int mi_row, MODE_INFO **mi_8x8, int mi_row,
int mi_col); int mi_col, int pl);
void av1_loop_filter_init(struct AV1Common *cm); void av1_loop_filter_init(struct AV1Common *cm);
......
...@@ -386,7 +386,8 @@ typedef struct AV1Common { ...@@ -386,7 +386,8 @@ typedef struct AV1Common {
ENTROPY_CONTEXT *above_context[MAX_MB_PLANE]; ENTROPY_CONTEXT *above_context[MAX_MB_PLANE];
#if CONFIG_VAR_TX #if CONFIG_VAR_TX
TXFM_CONTEXT *above_txfm_context; TXFM_CONTEXT *above_txfm_context;
TXFM_CONTEXT left_txfm_context[2 * MAX_MIB_SIZE]; TXFM_CONTEXT *top_txfm_context[MAX_MB_PLANE];
TXFM_CONTEXT left_txfm_context[MAX_MB_PLANE][2 * MAX_MIB_SIZE];
#endif #endif
int above_context_alloc_cols; int above_context_alloc_cols;
...@@ -928,6 +929,21 @@ static INLINE void txfm_partition_update(TXFM_CONTEXT *above_ctx, ...@@ -928,6 +929,21 @@ static INLINE void txfm_partition_update(TXFM_CONTEXT *above_ctx,
for (i = 0; i < bw; ++i) above_ctx[i] = txw; for (i = 0; i < bw; ++i) above_ctx[i] = txw;
} }
static INLINE TX_SIZE get_sqr_tx_size(int tx_dim) {
TX_SIZE tx_size;
switch (tx_dim) {
#if CONFIG_EXT_PARTITION
case 128:
#endif
case 64:
case 32: tx_size = TX_32X32; break;
case 16: tx_size = TX_16X16; break;
case 8: tx_size = TX_8X8; break;
default: tx_size = TX_4X4;
}
return tx_size;
}
static INLINE int txfm_partition_context(TXFM_CONTEXT *above_ctx, static INLINE int txfm_partition_context(TXFM_CONTEXT *above_ctx,
TXFM_CONTEXT *left_ctx, TXFM_CONTEXT *left_ctx,
BLOCK_SIZE bsize, TX_SIZE tx_size) { BLOCK_SIZE bsize, TX_SIZE tx_size) {
...@@ -935,22 +951,13 @@ static INLINE int txfm_partition_context(TXFM_CONTEXT *above_ctx, ...@@ -935,22 +951,13 @@ static INLINE int txfm_partition_context(TXFM_CONTEXT *above_ctx,
const uint8_t txh = tx_size_high[tx_size]; const uint8_t txh = tx_size_high[tx_size];
const int above = *above_ctx < txw; const int above = *above_ctx < txw;
const int left = *left_ctx < txh; const int left = *left_ctx < txh;
TX_SIZE max_tx_size = max_txsize_lookup[bsize];
int category = TXFM_PARTITION_CONTEXTS - 1; int category = TXFM_PARTITION_CONTEXTS - 1;
// dummy return, not used by others. // dummy return, not used by others.
if (tx_size <= TX_4X4) return 0; if (tx_size <= TX_4X4) return 0;
switch (AOMMAX(block_size_wide[bsize], block_size_high[bsize])) { TX_SIZE max_tx_size =
#if CONFIG_EXT_PARTITION get_sqr_tx_size(AOMMAX(block_size_wide[bsize], block_size_high[bsize]));
case 128:
#endif
case 64:
case 32: max_tx_size = TX_32X32; break;
case 16: max_tx_size = TX_16X16; break;
case 8: max_tx_size = TX_8X8; break;
default: assert(max_tx_size == max_txsize_lookup[bsize]); assert(0);
}
if (max_tx_size >= TX_8X8) { if (max_tx_size >= TX_8X8) {
category = (tx_size != max_tx_size && max_tx_size > TX_8X8) + category = (tx_size != max_tx_size && max_tx_size > TX_8X8) +
......
...@@ -113,7 +113,7 @@ static INLINE void loop_filter_block_plane_ver( ...@@ -113,7 +113,7 @@ static INLINE void loop_filter_block_plane_ver(
break; break;
case LF_PATH_SLOW: case LF_PATH_SLOW:
av1_filter_block_plane_non420_ver(cm, &planes[plane], mi, mi_row, av1_filter_block_plane_non420_ver(cm, &planes[plane], mi, mi_row,
mi_col); mi_col, plane);
break; break;
} }
} }
...@@ -135,7 +135,7 @@ static INLINE void loop_filter_block_plane_hor( ...@@ -135,7 +135,7 @@ static INLINE void loop_filter_block_plane_hor(
break; break;
case LF_PATH_SLOW: case LF_PATH_SLOW:
av1_filter_block_plane_non420_hor(cm, &planes[plane], mi, mi_row, av1_filter_block_plane_non420_hor(cm, &planes[plane], mi, mi_row,
mi_col); mi_col, plane);
break; break;
} }
} }
...@@ -168,7 +168,7 @@ static int loop_filter_ver_row_worker(AV1LfSync *const lf_sync, ...@@ -168,7 +168,7 @@ static int loop_filter_ver_row_worker(AV1LfSync *const lf_sync,
#if CONFIG_EXT_PARTITION_TYPES #if CONFIG_EXT_PARTITION_TYPES
for (plane = 0; plane < num_planes; ++plane) for (plane = 0; plane < num_planes; ++plane)
av1_filter_block_plane_non420_ver(lf_data->cm, &lf_data->planes[plane], av1_filter_block_plane_non420_ver(lf_data->cm, &lf_data->planes[plane],
mi + mi_col, mi_row, mi_col); mi + mi_col, mi_row, mi_col, plane);
#else #else
for (plane = 0; plane < num_planes; ++plane) for (plane = 0; plane < num_planes; ++plane)
...@@ -213,7 +213,7 @@ static int loop_filter_hor_row_worker(AV1LfSync *const lf_sync, ...@@ -213,7 +213,7 @@ static int loop_filter_hor_row_worker(AV1LfSync *const lf_sync,
#if CONFIG_EXT_PARTITION_TYPES #if CONFIG_EXT_PARTITION_TYPES
for (plane = 0; plane < num_planes; ++plane) for (plane = 0; plane < num_planes; ++plane)
av1_filter_block_plane_non420_hor(lf_data->cm, &lf_data->planes[plane], av1_filter_block_plane_non420_hor(lf_data->cm, &lf_data->planes[plane],
mi + mi_col, mi_row, mi_col); mi + mi_col, mi_row, mi_col, plane);
#else #else
for (plane = 0; plane < num_planes; ++plane) for (plane = 0; plane < num_planes; ++plane)
loop_filter_block_plane_hor(lf_data->cm, lf_data->planes, plane, loop_filter_block_plane_hor(lf_data->cm, lf_data->planes, plane,
...@@ -263,9 +263,9 @@ static int loop_filter_row_worker(AV1LfSync *const lf_sync, ...@@ -263,9 +263,9 @@ static int loop_filter_row_worker(AV1LfSync *const lf_sync,
#if CONFIG_EXT_PARTITION_TYPES #if CONFIG_EXT_PARTITION_TYPES
for (plane = 0; plane < num_planes; ++plane) { for (plane = 0; plane < num_planes; ++plane) {
av1_filter_block_plane_non420_ver(lf_data->cm, &lf_data->planes[plane], av1_filter_block_plane_non420_ver(lf_data->cm, &lf_data->planes[plane],
mi + mi_col, mi_row, mi_col); mi + mi_col, mi_row, mi_col, plane);
av1_filter_block_plane_non420_hor(lf_data->cm, &lf_data->planes[plane], av1_filter_block_plane_non420_hor(lf_data->cm, &lf_data->planes[plane],
mi + mi_col, mi_row, mi_col); mi + mi_col, mi_row, mi_col, plane);
} }
#else #else
av1_setup_mask(lf_data->cm, mi_row, mi_col, mi + mi_col, av1_setup_mask(lf_data->cm, mi_row, mi_col, mi + mi_col,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment