Commit 928b34e8 authored by Adrian Grange's avatar Adrian Grange
Browse files

Fix rounding in ARNR calculation

The rounding of the ARNR filter output prior to
normalization by the filter strength was incorrect
when strength = 0.

In this case 1 << (strength - 1) would not create the
required rounding of 0, rather it would outrange. This
patch fixes this issue.

Change-Id: I771809ba34d6052b17d34c870ea11ff67b418dab
parent 1b7291d5
......@@ -98,6 +98,7 @@ void vp8_temporal_filter_apply_c
unsigned int i, j, k;
int modifier;
int byte = 0;
const int rounding = strength > 0 ? 1 << (strength - 1) : 0;
for (i = 0,k = 0; i < block_size; i++)
{
......@@ -114,7 +115,7 @@ void vp8_temporal_filter_apply_c
*/
modifier *= modifier;
modifier *= 3;
modifier += 1 << (strength - 1);
modifier += rounding;
modifier >>= strength;
if (modifier > 16)
......
......@@ -99,6 +99,7 @@ void vp9_temporal_filter_apply_c(uint8_t *frame1,
unsigned int i, j, k;
int modifier;
int byte = 0;
const int rounding = strength > 0 ? 1 << (strength - 1) : 0;
for (i = 0, k = 0; i < block_size; i++) {
for (j = 0; j < block_size; j++, k++) {
......@@ -111,7 +112,7 @@ void vp9_temporal_filter_apply_c(uint8_t *frame1,
// modifier = (int)roundf(coeff > 16 ? 0 : 16-coeff);
modifier *= modifier;
modifier *= 3;
modifier += 1 << (strength - 1);
modifier += rounding;
modifier >>= strength;
if (modifier > 16)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment