Commit 7cb3498f authored by Yaowu Xu's avatar Yaowu Xu

misc-fix: fix loopfilter uv oddities and bugs

This commit merges the fix for the oddities and bugs in original VP9
loopfilter and subsequent fixes.

Related tracking issues:
https://bugs.chromium.org/p/webm/issues/detail?id=1016
https://bugs.chromium.org/p/webm/issues/detail?id=1280
Original libvpx commit:
034c28b0
Later bug fix commit:
https://aomedia-review.googlesource.com/#/c/4500/

Change-Id: Ie907aa55ee905fe9954a325f4c56c99c3ce26fb3
parent b7ce5234
...@@ -717,11 +717,7 @@ static void build_masks(const loop_filter_info_n *const lfi_n, ...@@ -717,11 +717,7 @@ static void build_masks(const loop_filter_info_n *const lfi_n,
uint64_t *const int_4x4_y = &lfm->int_4x4_y; uint64_t *const int_4x4_y = &lfm->int_4x4_y;
uint16_t *const left_uv = &lfm->left_uv[tx_size_uv]; uint16_t *const left_uv = &lfm->left_uv[tx_size_uv];
uint16_t *const above_uv = &lfm->above_uv[tx_size_uv]; uint16_t *const above_uv = &lfm->above_uv[tx_size_uv];
#if CONFIG_MISC_FIXES
uint16_t *const int_4x4_uv = &lfm->left_int_4x4_uv; uint16_t *const int_4x4_uv = &lfm->left_int_4x4_uv;
#else
uint16_t *const int_4x4_uv = &lfm->int_4x4_uv;
#endif
int i; int i;
// If filter level is 0 we don't loop filter. // If filter level is 0 we don't loop filter.
...@@ -1001,11 +997,7 @@ void av1_setup_mask(AV1_COMMON *const cm, const int mi_row, const int mi_col, ...@@ -1001,11 +997,7 @@ void av1_setup_mask(AV1_COMMON *const cm, const int mi_row, const int mi_col,
lfm->above_uv[i] &= mask_uv; lfm->above_uv[i] &= mask_uv;
} }
lfm->int_4x4_y &= mask_y; lfm->int_4x4_y &= mask_y;
#if CONFIG_MISC_FIXES
lfm->above_int_4x4_uv = lfm->left_int_4x4_uv & mask_uv; lfm->above_int_4x4_uv = lfm->left_int_4x4_uv & mask_uv;
#else
lfm->int_4x4_uv &= mask_uv;
#endif
// We don't apply a wide loop filter on the last uv block row. If set // We don't apply a wide loop filter on the last uv block row. If set
// apply the shorter one instead. // apply the shorter one instead.
...@@ -1017,10 +1009,8 @@ void av1_setup_mask(AV1_COMMON *const cm, const int mi_row, const int mi_col, ...@@ -1017,10 +1009,8 @@ void av1_setup_mask(AV1_COMMON *const cm, const int mi_row, const int mi_col,
lfm->above_uv[TX_8X8] |= lfm->above_uv[TX_16X16] & 0xff00; lfm->above_uv[TX_8X8] |= lfm->above_uv[TX_16X16] & 0xff00;
lfm->above_uv[TX_16X16] &= ~(lfm->above_uv[TX_16X16] & 0xff00); lfm->above_uv[TX_16X16] &= ~(lfm->above_uv[TX_16X16] & 0xff00);
} }
#if CONFIG_MISC_FIXES
} else { } else {
lfm->above_int_4x4_uv = lfm->left_int_4x4_uv; lfm->above_int_4x4_uv = lfm->left_int_4x4_uv;
#endif // CONFIG_MISC_FIXES
} }
if (mi_col + MAX_MIB_SIZE > cm->mi_cols) { if (mi_col + MAX_MIB_SIZE > cm->mi_cols) {
...@@ -1043,11 +1033,7 @@ void av1_setup_mask(AV1_COMMON *const cm, const int mi_row, const int mi_col, ...@@ -1043,11 +1033,7 @@ void av1_setup_mask(AV1_COMMON *const cm, const int mi_row, const int mi_col,
lfm->above_uv[i] &= mask_uv; lfm->above_uv[i] &= mask_uv;
} }
lfm->int_4x4_y &= mask_y; lfm->int_4x4_y &= mask_y;
#if CONFIG_MISC_FIXES
lfm->left_int_4x4_uv &= mask_uv_int; lfm->left_int_4x4_uv &= mask_uv_int;
#else
lfm->int_4x4_uv &= mask_uv_int;
#endif
// We don't apply a wide loop filter on the last uv column. If set // We don't apply a wide loop filter on the last uv column. If set
// apply the shorter one instead. // apply the shorter one instead.
...@@ -1077,11 +1063,7 @@ void av1_setup_mask(AV1_COMMON *const cm, const int mi_row, const int mi_col, ...@@ -1077,11 +1063,7 @@ void av1_setup_mask(AV1_COMMON *const cm, const int mi_row, const int mi_col,
assert(!(lfm->left_uv[TX_16X16] & lfm->left_uv[TX_8X8])); assert(!(lfm->left_uv[TX_16X16] & lfm->left_uv[TX_8X8]));
assert(!(lfm->left_uv[TX_16X16] & lfm->left_uv[TX_4X4])); assert(!(lfm->left_uv[TX_16X16] & lfm->left_uv[TX_4X4]));
assert(!(lfm->left_uv[TX_8X8] & lfm->left_uv[TX_4X4])); assert(!(lfm->left_uv[TX_8X8] & lfm->left_uv[TX_4X4]));
#if CONFIG_MISC_FIXES
assert(!(lfm->left_int_4x4_uv & lfm->left_uv[TX_16X16])); assert(!(lfm->left_int_4x4_uv & lfm->left_uv[TX_16X16]));
#else
assert(!(lfm->int_4x4_uv & lfm->left_uv[TX_16X16]));
#endif
assert(!(lfm->above_y[TX_16X16] & lfm->above_y[TX_8X8])); assert(!(lfm->above_y[TX_16X16] & lfm->above_y[TX_8X8]));
assert(!(lfm->above_y[TX_16X16] & lfm->above_y[TX_4X4])); assert(!(lfm->above_y[TX_16X16] & lfm->above_y[TX_4X4]));
assert(!(lfm->above_y[TX_8X8] & lfm->above_y[TX_4X4])); assert(!(lfm->above_y[TX_8X8] & lfm->above_y[TX_4X4]));
...@@ -1089,11 +1071,7 @@ void av1_setup_mask(AV1_COMMON *const cm, const int mi_row, const int mi_col, ...@@ -1089,11 +1071,7 @@ void av1_setup_mask(AV1_COMMON *const cm, const int mi_row, const int mi_col,
assert(!(lfm->above_uv[TX_16X16] & lfm->above_uv[TX_8X8])); assert(!(lfm->above_uv[TX_16X16] & lfm->above_uv[TX_8X8]));
assert(!(lfm->above_uv[TX_16X16] & lfm->above_uv[TX_4X4])); assert(!(lfm->above_uv[TX_16X16] & lfm->above_uv[TX_4X4]));
assert(!(lfm->above_uv[TX_8X8] & lfm->above_uv[TX_4X4])); assert(!(lfm->above_uv[TX_8X8] & lfm->above_uv[TX_4X4]));
#if CONFIG_MISC_FIXES
assert(!(lfm->above_int_4x4_uv & lfm->above_uv[TX_16X16])); assert(!(lfm->above_int_4x4_uv & lfm->above_uv[TX_16X16]));
#else
assert(!(lfm->int_4x4_uv & lfm->above_uv[TX_16X16]));
#endif
} }
static void filter_selectively_vert( static void filter_selectively_vert(
...@@ -1557,11 +1535,7 @@ void av1_filter_block_plane_ss11_ver(AV1_COMMON *const cm, ...@@ -1557,11 +1535,7 @@ void av1_filter_block_plane_ss11_ver(AV1_COMMON *const cm,
uint16_t mask_16x16 = lfm->left_uv[TX_16X16]; uint16_t mask_16x16 = lfm->left_uv[TX_16X16];
uint16_t mask_8x8 = lfm->left_uv[TX_8X8]; uint16_t mask_8x8 = lfm->left_uv[TX_8X8];
uint16_t mask_4x4 = lfm->left_uv[TX_4X4]; uint16_t mask_4x4 = lfm->left_uv[TX_4X4];
#if CONFIG_MISC_FIXES
uint16_t mask_4x4_int = lfm->left_int_4x4_uv; uint16_t mask_4x4_int = lfm->left_int_4x4_uv;
#else
uint16_t mask_4x4_int = lfm->int_4x4_uv;
#endif
assert(plane->subsampling_x == 1 && plane->subsampling_y == 1); assert(plane->subsampling_x == 1 && plane->subsampling_y == 1);
memset(lfm->lfl_uv, 0, sizeof(lfm->lfl_uv)); memset(lfm->lfl_uv, 0, sizeof(lfm->lfl_uv));
...@@ -1621,11 +1595,7 @@ void av1_filter_block_plane_ss11_hor(AV1_COMMON *const cm, ...@@ -1621,11 +1595,7 @@ void av1_filter_block_plane_ss11_hor(AV1_COMMON *const cm,
uint64_t mask_16x16 = lfm->above_uv[TX_16X16]; uint64_t mask_16x16 = lfm->above_uv[TX_16X16];
uint64_t mask_8x8 = lfm->above_uv[TX_8X8]; uint64_t mask_8x8 = lfm->above_uv[TX_8X8];
uint64_t mask_4x4 = lfm->above_uv[TX_4X4]; uint64_t mask_4x4 = lfm->above_uv[TX_4X4];
#if CONFIG_MISC_FIXES
uint64_t mask_4x4_int = lfm->above_int_4x4_uv; uint64_t mask_4x4_int = lfm->above_int_4x4_uv;
#else
uint64_t mask_4x4_int = lfm->int_4x4_uv;
#endif
assert(plane->subsampling_x == 1 && plane->subsampling_y == 1); assert(plane->subsampling_x == 1 && plane->subsampling_y == 1);
......
...@@ -82,12 +82,8 @@ typedef struct { ...@@ -82,12 +82,8 @@ typedef struct {
uint64_t int_4x4_y; uint64_t int_4x4_y;
uint16_t left_uv[TX_SIZES]; uint16_t left_uv[TX_SIZES];
uint16_t above_uv[TX_SIZES]; uint16_t above_uv[TX_SIZES];
#if CONFIG_MISC_FIXES
uint16_t left_int_4x4_uv; uint16_t left_int_4x4_uv;
uint16_t above_int_4x4_uv; uint16_t above_int_4x4_uv;
#else
uint16_t int_4x4_uv;
#endif
uint8_t lfl_y[64]; uint8_t lfl_y[64];
uint8_t lfl_uv[16]; uint8_t lfl_uv[16];
} LOOP_FILTER_MASK; } LOOP_FILTER_MASK;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment