Commit b496bba1 authored by Cheng Chen's avatar Cheng Chen Committed by Yaowu Xu

Fix loop filtering bug in non-420 video.

BUG=aomedia:386

Fix the bug by separating mask_4x4_int into mask_4x4_int_r and
mask_4x4_int_c. The horizontal and vertical filtering are now
correctly handled.

Change-Id: I2207a12c83cba3e3003e1e45c7587695ad87a691
parent 16247658
...@@ -1365,7 +1365,8 @@ typedef struct { ...@@ -1365,7 +1365,8 @@ typedef struct {
static void get_filter_level_and_masks_non420( static void get_filter_level_and_masks_non420(
AV1_COMMON *const cm, const struct macroblockd_plane *const plane, AV1_COMMON *const cm, const struct macroblockd_plane *const plane,
MODE_INFO **mib, int mi_row, int mi_col, int idx_r, uint8_t *const lfl_r, MODE_INFO **mib, int mi_row, int mi_col, int idx_r, uint8_t *const lfl_r,
unsigned int *const mask_4x4_int_r, FilterMasks *const row_masks_ptr, unsigned int *const mask_4x4_int_r_ptr,
unsigned int *const mask_4x4_int_c_ptr, FilterMasks *const row_masks_ptr,
FilterMasks *const col_masks_ptr) { FilterMasks *const col_masks_ptr) {
const int ss_x = plane->subsampling_x; const int ss_x = plane->subsampling_x;
const int ss_y = plane->subsampling_y; const int ss_y = plane->subsampling_y;
...@@ -1373,7 +1374,7 @@ static void get_filter_level_and_masks_non420( ...@@ -1373,7 +1374,7 @@ static void get_filter_level_and_masks_non420(
FilterMasks row_masks, col_masks; FilterMasks row_masks, col_masks;
memset(&row_masks, 0, sizeof(row_masks)); memset(&row_masks, 0, sizeof(row_masks));
memset(&col_masks, 0, sizeof(col_masks)); memset(&col_masks, 0, sizeof(col_masks));
*mask_4x4_int_r = 0; unsigned int mask_4x4_int_r = 0, mask_4x4_int_c = 0;
const int r = idx_r >> mi_height_log2_lookup[BLOCK_8X8]; const int r = idx_r >> mi_height_log2_lookup[BLOCK_8X8];
// Determine the vertical edges that need filtering // Determine the vertical edges that need filtering
...@@ -1478,7 +1479,7 @@ static void get_filter_level_and_masks_non420( ...@@ -1478,7 +1479,7 @@ static void get_filter_level_and_masks_non420(
if (!skip_this && tx_size_c < TX_8X8 && !skip_border_4x4_c && if (!skip_this && tx_size_c < TX_8X8 && !skip_border_4x4_c &&
(c_step & tx_size_mask) == 0) (c_step & tx_size_mask) == 0)
*mask_4x4_int_r |= col_mask; mask_4x4_int_c |= col_mask;
} }
if (tx_size_r == TX_32X32) if (tx_size_r == TX_32X32)
...@@ -1512,14 +1513,16 @@ static void get_filter_level_and_masks_non420( ...@@ -1512,14 +1513,16 @@ static void get_filter_level_and_masks_non420(
row_masks.m4x4 |= col_mask; row_masks.m4x4 |= col_mask;
} }
if (!skip_this && tx_size_r < TX_8X8 && !skip_border_4x4_c && if (!skip_this && tx_size_r < TX_8X8 && !skip_border_4x4_r &&
((r >> ss_y) & tx_size_mask) == 0) ((r >> ss_y) & tx_size_mask) == 0)
*mask_4x4_int_r |= col_mask; mask_4x4_int_r |= col_mask;
} }
} }
if (row_masks_ptr) *row_masks_ptr = row_masks; if (row_masks_ptr) *row_masks_ptr = row_masks;
if (col_masks_ptr) *col_masks_ptr = col_masks; if (col_masks_ptr) *col_masks_ptr = col_masks;
if (mask_4x4_int_c_ptr) *mask_4x4_int_c_ptr = mask_4x4_int_c;
if (mask_4x4_int_r_ptr) *mask_4x4_int_r_ptr = mask_4x4_int_r;
} }
void av1_filter_block_plane_non420_ver(AV1_COMMON *const cm, void av1_filter_block_plane_non420_ver(AV1_COMMON *const cm,
...@@ -1539,7 +1542,7 @@ void av1_filter_block_plane_non420_ver(AV1_COMMON *const cm, ...@@ -1539,7 +1542,7 @@ void av1_filter_block_plane_non420_ver(AV1_COMMON *const cm,
FilterMasks col_masks; FilterMasks col_masks;
const int r = idx_r >> mi_height_log2_lookup[BLOCK_8X8]; const int r = idx_r >> mi_height_log2_lookup[BLOCK_8X8];
get_filter_level_and_masks_non420(cm, plane, mib, mi_row, mi_col, idx_r, get_filter_level_and_masks_non420(cm, plane, mib, mi_row, mi_col, idx_r,
&lfl[r][0], &mask_4x4_int, NULL, &lfl[r][0], NULL, &mask_4x4_int, NULL,
&col_masks); &col_masks);
// Disable filtering on the leftmost column or tile boundary // Disable filtering on the leftmost column or tile boundary
...@@ -1587,15 +1590,12 @@ void av1_filter_block_plane_non420_hor(AV1_COMMON *const cm, ...@@ -1587,15 +1590,12 @@ void av1_filter_block_plane_non420_hor(AV1_COMMON *const cm,
idx_r += row_step) { idx_r += row_step) {
const int r = idx_r >> mi_height_log2_lookup[BLOCK_8X8]; const int r = idx_r >> mi_height_log2_lookup[BLOCK_8X8];
get_filter_level_and_masks_non420(cm, plane, mib, mi_row, mi_col, idx_r, get_filter_level_and_masks_non420(cm, plane, mib, mi_row, mi_col, idx_r,
&lfl[r][0], mask_4x4_int + r, &lfl[r][0], mask_4x4_int + r, NULL,
row_masks_array + r, NULL); row_masks_array + r, NULL);
} }
for (idx_r = 0; idx_r < cm->mib_size && mi_row + idx_r < cm->mi_rows; for (idx_r = 0; idx_r < cm->mib_size && mi_row + idx_r < cm->mi_rows;
idx_r += row_step) { idx_r += row_step) {
const int skip_border_4x4_r =
ss_y && mi_row + idx_r >= cm->mi_rows - mi_size_wide[BLOCK_8X8];
const int r = idx_r >> mi_width_log2_lookup[BLOCK_8X8]; const int r = idx_r >> mi_width_log2_lookup[BLOCK_8X8];
const unsigned int mask_4x4_int_r = skip_border_4x4_r ? 0 : mask_4x4_int[r];
FilterMasks row_masks; FilterMasks row_masks;
#if CONFIG_LOOPFILTERING_ACROSS_TILES #if CONFIG_LOOPFILTERING_ACROSS_TILES
...@@ -1616,12 +1616,12 @@ void av1_filter_block_plane_non420_hor(AV1_COMMON *const cm, ...@@ -1616,12 +1616,12 @@ void av1_filter_block_plane_non420_hor(AV1_COMMON *const cm,
if (cm->use_highbitdepth) if (cm->use_highbitdepth)
highbd_filter_selectively_horiz( highbd_filter_selectively_horiz(
CONVERT_TO_SHORTPTR(dst->buf), dst->stride, row_masks.m16x16, CONVERT_TO_SHORTPTR(dst->buf), dst->stride, row_masks.m16x16,
row_masks.m8x8, row_masks.m4x4, mask_4x4_int_r, &cm->lf_info, row_masks.m8x8, row_masks.m4x4, mask_4x4_int[r], &cm->lf_info,
&lfl[r][0], (int)cm->bit_depth); &lfl[r][0], (int)cm->bit_depth);
else else
#endif // CONFIG_HIGHBITDEPTH #endif // CONFIG_HIGHBITDEPTH
filter_selectively_horiz(dst->buf, dst->stride, row_masks.m16x16, filter_selectively_horiz(dst->buf, dst->stride, row_masks.m16x16,
row_masks.m8x8, row_masks.m4x4, mask_4x4_int_r, row_masks.m8x8, row_masks.m4x4, mask_4x4_int[r],
&cm->lf_info, &lfl[r][0]); &cm->lf_info, &lfl[r][0]);
dst->buf += 8 * dst->stride; dst->buf += 8 * dst->stride;
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment