Commit 9c0e9eac authored by Luc Trudeau's avatar Luc Trudeau

[CFL] Fixed negative rounding in scaled_luma

Since the scaled luma can be negative, ROUND_POWER_OF_TWO_SIGNED must be used.
This changes the behavior from rounding toward -infinity to rounding towards 0.

Results for Subset1 (compared with 35545dd5 with CfL enabled)
  PSNR | PSNR Cb | PSNR Cr | PSNR HVS |    SSIM | MS SSIM | CIEDE 2000
0.0082 | -0.1061 | -0.0119 |  -0.0126 | -0.0011 | -0.0121 |     0.0094

Change-Id: Ie7258a17a199368339d4794fba6b5916e607c95b
parent 5c6744b5
...@@ -14,6 +14,7 @@ ...@@ -14,6 +14,7 @@
#include <assert.h> #include <assert.h>
#include "av1/common/common.h"
#include "av1/common/enums.h" #include "av1/common/enums.h"
// Forward declaration of AV1_COMMON, in order to avoid creating a cyclic // Forward declaration of AV1_COMMON, in order to avoid creating a cyclic
...@@ -66,7 +67,8 @@ typedef struct { ...@@ -66,7 +67,8 @@ typedef struct {
} CFL_CTX; } CFL_CTX;
static INLINE int get_scaled_luma_q0(int alpha_q3, int y_pix, int avg_q3) { static INLINE int get_scaled_luma_q0(int alpha_q3, int y_pix, int avg_q3) {
return (alpha_q3 * ((y_pix << 3) - avg_q3) + 32) >> 6; int scaled_luma_q6 = alpha_q3 * ((y_pix << 3) - avg_q3);
return ROUND_POWER_OF_TWO_SIGNED(scaled_luma_q6, 6);
} }
void cfl_init(CFL_CTX *cfl, AV1_COMMON *cm); void cfl_init(CFL_CTX *cfl, AV1_COMMON *cm);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment