Commit c854bacc authored by David Michael Barr's avatar David Michael Barr

[CFL] Restore missing pair in cfl_alpha_codes

In 84a44dbe, there was a transcription error in the table.
Point { 5, 3 } was duplicated and { 5, 0 } was missing.

While we are here, updated the order and CDF from subset3.

Results on subset1 (compared to cab68ae6 with CfL)
   PSNR | PSNR Cb | PSNR Cr | PSNR HVS |    SSIM | MS SSIM | CIEDE 2000
 0.0416 | -0.2732 | -0.3848 |   0.0385 | 0.0309 |  0.0230 |    -0.0178

Change-Id: I3244245792c5ab99b4149ae5f8a2439d4214ed69
Signed-off-by: default avatarDavid Michael Barr <b@rr-dav.id.au>
parent aa2f1c7a
......@@ -55,8 +55,8 @@ static const double cfl_alpha_mags[CFL_MAGS_SIZE] = {
static const int cfl_alpha_codes[CFL_ALPHABET_SIZE][CFL_PRED_PLANES] = {
// barrbrain's simple 1D quant ordered by subset 3 likelihood
{ 0, 0 }, { 1, 1 }, { 3, 0 }, { 3, 1 }, { 1, 0 }, { 3, 3 },
{ 0, 1 }, { 5, 5 }, { 5, 3 }, { 1, 3 }, { 5, 3 }, { 3, 5 },
{ 0, 0 }, { 1, 1 }, { 3, 0 }, { 3, 3 }, { 1, 0 }, { 3, 1 },
{ 5, 5 }, { 0, 1 }, { 5, 3 }, { 5, 0 }, { 3, 5 }, { 1, 3 },
{ 0, 3 }, { 5, 1 }, { 1, 5 }, { 0, 5 }
};
......
......@@ -3172,11 +3172,10 @@ static const aom_cdf_prob
#if CONFIG_CFL
static const aom_cdf_prob default_cfl_alpha_cdf[CDF_SIZE(CFL_ALPHABET_SIZE)] = {
// decreasing likelihood, after 6 iterations of alpha RDO on subset 3
AOM_ICDF(23928), AOM_ICDF(26267), AOM_ICDF(27319), AOM_ICDF(28330),
AOM_ICDF(29341), AOM_ICDF(30160), AOM_ICDF(30581), AOM_ICDF(30983),
AOM_ICDF(31353), AOM_ICDF(31634), AOM_ICDF(31907), AOM_ICDF(32171),
AOM_ICDF(32407), AOM_ICDF(32558), AOM_ICDF(32669), AOM_ICDF(32768)
AOM_ICDF(20492), AOM_ICDF(24094), AOM_ICDF(25679), AOM_ICDF(27242),
AOM_ICDF(28286), AOM_ICDF(29153), AOM_ICDF(29807), AOM_ICDF(30352),
AOM_ICDF(30866), AOM_ICDF(31295), AOM_ICDF(31703), AOM_ICDF(32046),
AOM_ICDF(32317), AOM_ICDF(32534), AOM_ICDF(32663), AOM_ICDF(32768)
};
#endif
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment