Commit 5e7d7d3d authored by Yaowu Xu's avatar Yaowu Xu
Browse files

new loop filter functions for macroblock boundaries

The commit adds a new set of loop filter for macroblock edge filtering.
The new loop filter has a mask to detect so-called "flat" regions. The
detection checks 5 pixels of each side of an edge. If the all pixels
have value with +/-1 from the edge pixel on the same side, the region
is treated as a "flat" region. For such case, a 7 tap filter is used
to change 3 pixel values on each side. The 7 taps are:
               [1, 1, 1, 2, 1, 1, 1]/8
The furthest away pixels used as input are +/-5 away from edge. For
non-flat region, we fall back to old filtering. It should be noted
here that the thresholds and filter taps may require more optimization
for best possible results.

Tests on a set of hd clips showed consistent gains:
http://www.corp.google.com/~yaowu/no_crawl/mblpf_hd.html
(avg psnr: .83% glb psnr: .77% ssim: .82%)

Tests on derf set also showed consistent gains:
http://www.corp.google.com/~yaowu/no_crawl/mblpf_derf.html
(avg psnr: .24% glb psnr: .22% ssim: .48%)

Change-Id: I0855b1ff48e79e1175c20b81967137e18b2af352
parent 483b262b
......@@ -226,6 +226,7 @@ EXPERIMENT_LIST="
qimode
uvintra
newnear
newlpf
"
CONFIG_LIST="
external_build
......
......@@ -342,18 +342,26 @@ void vp8_loop_filter_frame
lfi.hev_thr = lfi_n->hev_thr[hev_index];
if (mb_col > 0)
#if CONFIG_NEWLPF
vp8_loop_filter_mbv_c
(y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
#else
LF_INVOKE(&cm->rtcd.loopfilter, normal_mb_v)
(y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
#endif
if (!skip_lf)
LF_INVOKE(&cm->rtcd.loopfilter, normal_b_v)
(y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
/* don't apply across umv border */
if (mb_row > 0)
#if CONFIG_NEWLPF
vp8_loop_filter_mbh_c
(y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
#else
LF_INVOKE(&cm->rtcd.loopfilter, normal_mb_h)
(y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
#endif
if (!skip_lf)
LF_INVOKE(&cm->rtcd.loopfilter, normal_b_h)
(y_ptr, u_ptr, v_ptr, post->y_stride, post->uv_stride, &lfi);
......@@ -454,8 +462,13 @@ void vp8_loop_filter_frame_yonly
lfi.hev_thr = lfi_n->hev_thr[hev_index];
if (mb_col > 0)
#if CONFIG_NEWLPF
vp8_loop_filter_mbv_c
(y_ptr, 0, 0, post->y_stride, 0, &lfi);
#else
LF_INVOKE(&cm->rtcd.loopfilter, normal_mb_v)
(y_ptr, 0, 0, post->y_stride, 0, &lfi);
#endif
if (!skip_lf)
LF_INVOKE(&cm->rtcd.loopfilter, normal_b_v)
......@@ -463,9 +476,13 @@ void vp8_loop_filter_frame_yonly
/* don't apply across umv border */
if (mb_row > 0)
#if CONFIG_NEWLPF
vp8_loop_filter_mbh_c
(y_ptr, 0, 0, post->y_stride, 0, &lfi);
#else
LF_INVOKE(&cm->rtcd.loopfilter, normal_mb_h)
(y_ptr, 0, 0, post->y_stride, 0, &lfi);
#endif
if (!skip_lf)
LF_INVOKE(&cm->rtcd.loopfilter, normal_b_h)
(y_ptr, 0, 0, post->y_stride, 0, &lfi);
......
......@@ -148,7 +148,8 @@ void vp8_loop_filter_vertical_edge_c
do
{
mask = vp8_filter_mask(limit[0], blimit[0],
s[-4], s[-3], s[-2], s[-1], s[0], s[1], s[2], s[3]);
s[-4], s[-3], s[-2], s[-1],
s[0], s[1], s[2], s[3]);
hev = vp8_hevmask(thresh[0], s[-2], s[-1], s[0], s[1]);
......@@ -158,9 +159,100 @@ void vp8_loop_filter_vertical_edge_c
}
while (++i < count * 8);
}
#if CONFIG_NEWLPF
static __inline signed char vp8_flatmask(uc thresh,
uc p4, uc p3, uc p2, uc p1, uc p0,
uc q0, uc q1, uc q2, uc q3, uc q4)
{
signed char flat = 0;
flat |= (abs(p1 - p0) > 1) * -1;
flat |= (abs(q1 - q0) > 1) * -1;
flat |= (abs(p0 - p2) > 1) * -1;
flat |= (abs(q0 - q2) > 1) * -1;
flat |= (abs(p3 - p0) > 1) * -1;
flat |= (abs(q3 - q0) > 1) * -1;
flat |= (abs(p4 - p0) > 1) * -1;
flat |= (abs(q4 - q0) > 1) * -1;
flat = ~flat;
return flat;
}
static __inline void vp8_mbfilter(signed char mask, uc hev, uc flat,
uc *op4, uc *op3, uc *op2, uc *op1, uc *op0,
uc *oq0, uc *oq1, uc *oq2, uc *oq3, uc *oq4)
{
/* use a 7 tap filter [1, 1, 1, 2, 1, 1, 1] for flat line */
if(flat && mask)
{
unsigned char p0, q0;
unsigned char p1, q1;
unsigned char p2, q2;
unsigned char p3, q3;
unsigned char p4, q4;
p4 = *op4;
p3 = *op3;
p2 = *op2;
p1 = *op1;
p0 = *op0;
q0 = *oq0;
q1 = *oq1;
q2 = *oq2;
q3 = *oq3;
q4 = *oq4;
*op2 = ( p4 + p4 + p3 + p2 + p2 + p1 + p0 + q0 + 4)>>3;
*op1 = ( p4 + p3 + p2 + p1 + p1 + p0 + q0 + q1 + 4)>>3;
*op0 = ( p3 + p2 + p1 + p0 + p0 + q0 + q1 + q2 + 4)>>3;
*oq0 = ( p2 + p1 + p0 + q0 + q0 + q1 + q2 + q3 + 4)>>3;
*oq1 = ( p1 + p0 + q0 + q1 + q1 + q2 + q3 + q4 + 4)>>3;
*oq2 = ( p0 + q0 + q1 + q2 + q2 + q3 + q4 + q4 + 4)>>3;
}
else
{
signed char ps0, qs0;
signed char ps1, qs1;
signed char vp8_filter, Filter1, Filter2;
signed char u;
ps1 = (signed char) * op1 ^ 0x80;
ps0 = (signed char) * op0 ^ 0x80;
qs0 = (signed char) * oq0 ^ 0x80;
qs1 = (signed char) * oq1 ^ 0x80;
/* add outer taps if we have high edge variance */
vp8_filter = vp8_signed_char_clamp(ps1 - qs1);
vp8_filter &= hev;
/* inner taps */
vp8_filter = vp8_signed_char_clamp(vp8_filter + 3 * (qs0 - ps0));
vp8_filter &= mask;
Filter1 = vp8_signed_char_clamp(vp8_filter + 4);
Filter2 = vp8_signed_char_clamp(vp8_filter + 3);
Filter1 >>= 3;
Filter2 >>= 3;
u = vp8_signed_char_clamp(qs0 - Filter1);
*oq0 = u ^ 0x80;
u = vp8_signed_char_clamp(ps0 + Filter2);
*op0 = u ^ 0x80;
vp8_filter = Filter1;
/* outer tap adjustments */
vp8_filter += 1;
vp8_filter >>= 1;
vp8_filter &= ~hev;
u = vp8_signed_char_clamp(qs1 - vp8_filter);
*oq1 = u ^ 0x80;
u = vp8_signed_char_clamp(ps1 + vp8_filter);
*op1 = u ^ 0x80;
}
}
#else
static __inline void vp8_mbfilter(signed char mask, uc hev,
uc *op2, uc *op1, uc *op0, uc *oq0, uc *oq1, uc *oq2)
uc *op2, uc *op1, uc *op0,
uc *oq0, uc *oq1, uc *oq2)
{
signed char s, u;
signed char vp8_filter, Filter1, Filter2;
......@@ -213,7 +305,7 @@ static __inline void vp8_mbfilter(signed char mask, uc hev,
s = vp8_signed_char_clamp(ps2 + u);
*op2 = s ^ 0x80;
}
#endif
void vp8_mbloop_filter_horizontal_edge_c
(
unsigned char *s,
......@@ -226,6 +318,9 @@ void vp8_mbloop_filter_horizontal_edge_c
{
signed char hev = 0; /* high edge variance */
signed char mask = 0;
#if CONFIG_NEWLPF
signed char flat = 0;
#endif
int i = 0;
/* loop filter designed to work using chars so that we can make maximum use
......@@ -236,12 +331,21 @@ void vp8_mbloop_filter_horizontal_edge_c
mask = vp8_filter_mask(limit[0], blimit[0],
s[-4*p], s[-3*p], s[-2*p], s[-1*p],
s[0*p], s[1*p], s[2*p], s[3*p]);
s[ 0*p], s[ 1*p], s[ 2*p], s[ 3*p]);
hev = vp8_hevmask(thresh[0], s[-2*p], s[-1*p], s[0*p], s[1*p]);
vp8_mbfilter(mask, hev, s - 3 * p, s - 2 * p, s - 1 * p, s, s + 1 * p, s + 2 * p);
#if CONFIG_NEWLPF
flat = vp8_flatmask(thresh[0],
s[-5*p], s[-4*p], s[-3*p], s[-2*p], s[-1*p],
s[ 0*p], s[ 1*p], s[ 2*p], s[ 3*p], s[ 4*p]);
vp8_mbfilter(mask, hev, flat,
s - 5*p, s - 4*p, s- 3*p, s - 2*p, s - 1*p,
s, s + 1*p, s+ 2*p, s + 3*p, s + 4*p );
#else
vp8_mbfilter(mask, hev,
s - 3*p, s - 2*p, s - 1*p,
s, s + 1*p, s + 2*p);
#endif
++s;
}
while (++i < count * 8);
......@@ -261,18 +365,31 @@ void vp8_mbloop_filter_vertical_edge_c
{
signed char hev = 0; /* high edge variance */
signed char mask = 0;
#if CONFIG_NEWLPF
signed char flat = 0;
#endif
int i = 0;
do
{
mask = vp8_filter_mask(limit[0], blimit[0],
s[-4], s[-3], s[-2], s[-1], s[0], s[1], s[2], s[3]);
s[-4], s[-3], s[-2], s[-1],
s[0], s[1], s[2], s[3]);
hev = vp8_hevmask(thresh[0], s[-2], s[-1], s[0], s[1]);
vp8_mbfilter(mask, hev, s - 3, s - 2, s - 1, s, s + 1, s + 2);
#if CONFIG_NEWLPF
flat = vp8_flatmask(thresh[0],
s[-5],s[-4],s[-3],s[-2],s[-1],
s[ 0],s[ 1],s[ 2],s[ 3],s[ 4]);
vp8_mbfilter(mask, hev, flat,
s - 5, s - 4, s - 3, s - 2, s - 1,
s, s + 1, s + 2, s + 3, s + 4);
#else
vp8_mbfilter(mask, hev,
s - 3, s - 2, s - 1,
s, s + 1, s + 2);
#endif
s += p;
}
while (++i < count * 8);
......@@ -280,7 +397,9 @@ void vp8_mbloop_filter_vertical_edge_c
}
/* should we apply any filter at all ( 11111111 yes, 00000000 no) */
static __inline signed char vp8_simple_filter_mask(uc blimit, uc p1, uc p0, uc q0, uc q1)
static __inline signed char vp8_simple_filter_mask(uc blimit,
uc p1, uc p0,
uc q0, uc q1)
{
/* Why does this cause problems for win32?
* error C2143: syntax error : missing ';' before 'type'
......@@ -290,7 +409,9 @@ static __inline signed char vp8_simple_filter_mask(uc blimit, uc p1, uc p0, uc q
return mask;
}
static __inline void vp8_simple_filter(signed char mask, uc *op1, uc *op0, uc *oq0, uc *oq1)
static __inline void vp8_simple_filter(signed char mask,
uc *op1, uc *op0,
uc *oq0, uc *oq1)
{
signed char vp8_filter, Filter1, Filter2;
signed char p1 = (signed char) * op1 ^ 0x80;
......@@ -327,8 +448,12 @@ void vp8_loop_filter_simple_horizontal_edge_c
do
{
mask = vp8_simple_filter_mask(blimit[0], s[-2*p], s[-1*p], s[0*p], s[1*p]);
vp8_simple_filter(mask, s - 2 * p, s - 1 * p, s, s + 1 * p);
mask = vp8_simple_filter_mask(blimit[0],
s[-2*p], s[-1*p],
s[0*p], s[1*p]);
vp8_simple_filter(mask,
s - 2 * p, s - 1 * p,
s, s + 1 * p);
++s;
}
while (++i < 16);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment