Commit 32d26bc0 authored by Jingning Han's avatar Jingning Han

Avoid right shift negative numbers

This avoids the rounding errors due to the right shift of the
negative numbers that cause the reconstruction coefficient has
higher distortion than the source coefficient.

BUG=aomedia:599

Change-Id: I11ed86bf1d41164dda4398545334a7b4e8e10513
parent 3599435a
......@@ -246,7 +246,7 @@ static int optimize_b_greedy(const AV1_COMMON *cm, MACROBLOCK *mb, int plane,
*/
// compute the distortion for the first candidate
// and the distortion for quantizing to 0.
int dx0 = (-coeff[rc]) * (1 << shift);
int dx0 = abs(coeff[rc]) * (1 << shift);
#if CONFIG_HIGHBITDEPTH
if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
dx0 >>= xd->bd - 8;
......@@ -270,7 +270,9 @@ static int optimize_b_greedy(const AV1_COMMON *cm, MACROBLOCK *mb, int plane,
dx = (dqcoeff[rc] - coeff[rc]) * (1 << shift);
#if CONFIG_HIGHBITDEPTH
if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
dx >>= xd->bd - 8;
int dx_sign = dx < 0 ? 1 : 0;
dx = abs(dx) >> (xd->bd - 8);
if (dx_sign) dx = -dx;
}
#endif // CONFIG_HIGHBITDEPTH
d2 = (int64_t)dx * dx;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment