Skip to content

GitLab

Commit ca52fb5f authored by Sarah Parker's avatar Sarah Parker
Browse files

Make new_quant bin sizes uniform 

This gives a -0.05% change in performance over the regular
quantizer. The new_quant bins will remain uniform as we
experiment with different parameters.

Change-Id: Ic92c74fb1183cf1c2460bbcf1e223452f46d8210
parent abdf6556
Hide whitespace changes
Inline Side-by-side
av1/common/quant_common.c
View file @ ca52fb5f
......@@ -42,28 +42,28 @@ static const qprofile_type nuq[QUANT_PROFILES][COEF_BANDS] = {
{ { 64, 128, 128 }, 0 }, // band 5
},
{
{ { 64, 128, 128 }, 4 }, // dc, band 0
{ { 64, 128, 128 }, 6 }, // band 1
{ { 64, 128, 128 }, 8 }, // band 2
{ { 64, 128, 128 }, 10 }, // band 3
{ { 72, 128, 128 }, 12 }, // band 4
{ { 80, 128, 128 }, 14 } // band 5
{ { 64, 128, 128 }, 0 }, // dc, band 0
{ { 64, 128, 128 }, 0 }, // band 1
{ { 64, 128, 128 }, 0 }, // band 2
{ { 64, 128, 128 }, 0 }, // band 3
{ { 64, 128, 128 }, 0 }, // band 4
{ { 64, 128, 128 }, 0 } // band 5
},
{
{ { 64, 128, 128 }, 6 }, // dc, band 0
{ { 64, 128, 128 }, 8 }, // band 1
{ { 64, 128, 128 }, 10 }, // band 2
{ { 64, 128, 128 }, 12 }, // band 3
{ { 72, 128, 128 }, 14 }, // band 4
{ { 80, 128, 128 }, 16 } // band 5
{ { 64, 128, 128 }, 0 }, // dc, band 0
{ { 64, 128, 128 }, 0 }, // band 1
{ { 64, 128, 128 }, 0 }, // band 2
{ { 64, 128, 128 }, 0 }, // band 3
{ { 64, 128, 128 }, 0 }, // band 4
{ { 64, 128, 128 }, 0 } // band 5
},
{
{ { 64, 128, 128 }, 8 }, // dc, band 0
{ { 64, 128, 128 }, 10 }, // band 1
{ { 64, 128, 128 }, 12 }, // band 2
{ { 72, 128, 128 }, 14 }, // band 3
{ { 76, 128, 128 }, 16 }, // band 4
{ { 80, 128, 128 }, 18 } // band 5
{ { 64, 128, 128 }, 0 }, // dc, band 0
{ { 64, 128, 128 }, 0 }, // band 1
{ { 64, 128, 128 }, 0 }, // band 2
{ { 64, 128, 128 }, 0 }, // band 3
{ { 64, 128, 128 }, 0 }, // band 4
{ { 64, 128, 128 }, 0 } // band 5
}
};
 
......
av1/encoder/av1_quantize.c
View file @ ca52fb5f
......@@ -101,7 +101,7 @@ static INLINE int quantize_coeff_fp_nuq(
int i, q;
int tmp = clamp(abs_coeff, INT16_MIN, INT16_MAX);
for (i = 0; i < NUQ_KNOTS; i++) {
if (tmp < cuml_bins_ptr[i]) {
if (tmp <= cuml_bins_ptr[i]) {
q = i;
break;
}
......@@ -131,7 +131,7 @@ static INLINE int quantize_coeff_bigtx_fp_nuq(
int i, q;
int tmp = clamp(abs_coeff, INT16_MIN, INT16_MAX);
for (i = 0; i < NUQ_KNOTS; i++) {
if (tmp < ROUND_POWER_OF_TWO(cuml_bins_ptr[i], logsizeby16)) {
if (tmp <= ROUND_POWER_OF_TWO(cuml_bins_ptr[i], logsizeby16)) {
q = i;
break;
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment