Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
Xiph.Org
aom-rav1e
Commits
a855cefc
Commit
a855cefc
authored
Jan 13, 2018
by
Debargha Mukherjee
Browse files
A small fix for the new-quant b_quantizer
Change-Id: I3698acb824afc5d0f66c2d4a099403f07a373c86
parent
e5267697
Changes
1
Hide whitespace changes
Inline
Side-by-side
av1/encoder/av1_quantize.c
View file @
a855cefc
...
...
@@ -35,7 +35,7 @@ static INLINE int quantize_coeff_nuq(
const
int
abs_coeff
=
(
coeff
^
coeff_sign
)
-
coeff_sign
;
int
q
=
0
;
int
tmp
=
clamp
(
abs_coeff
,
INT16_MIN
,
INT16_MAX
);
if
(
tmp
>=
zbin
)
{
if
(
tmp
>=
AOMMAX
(
zbin
,
cuml_bins_ptr
[
0
])
)
{
tmp
-=
cuml_bins_ptr
[
0
];
q
=
NUQ_KNOTS
+
(((((
tmp
*
quant
)
>>
16
)
+
tmp
)
*
quant_shift
)
>>
16
);
...
...
@@ -60,8 +60,10 @@ static INLINE int quantize_coeff_bigtx_nuq(
const
int
abs_coeff
=
(
coeff
^
coeff_sign
)
-
coeff_sign
;
int
q
=
0
;
int
tmp
=
clamp
(
abs_coeff
,
INT16_MIN
,
INT16_MAX
);
if
(
tmp
>=
zbin_val
)
{
tmp
-=
ROUND_POWER_OF_TWO
(
cuml_bins_ptr
[
0
],
logsizeby16
);
const
int
cuml_bins_ptr_val
=
ROUND_POWER_OF_TWO
(
cuml_bins_ptr
[
0
],
logsizeby16
);
if
(
tmp
>=
AOMMAX
(
zbin_val
,
cuml_bins_ptr_val
))
{
tmp
-=
cuml_bins_ptr_val
;
q
=
NUQ_KNOTS
+
(((((
tmp
*
quant
)
>>
16
)
+
tmp
)
*
quant_shift
)
>>
(
16
-
logsizeby16
));
*
dqcoeff_ptr
=
...
...
@@ -975,7 +977,7 @@ static INLINE int highbd_quantize_coeff_nuq(
const
int
abs_coeff
=
(
coeff
^
coeff_sign
)
-
coeff_sign
;
int
q
=
0
;
int64_t
tmp
=
clamp
(
abs_coeff
,
INT32_MIN
,
INT32_MAX
);
if
(
tmp
>=
zbin
)
{
if
(
tmp
>=
AOMMAX
(
zbin
,
cuml_bins_ptr
[
0
])
)
{
tmp
-=
cuml_bins_ptr
[
0
];
q
=
NUQ_KNOTS
+
(
int
)(((((
tmp
*
quant
)
>>
16
)
+
tmp
)
*
quant_shift
)
>>
16
);
*
dqcoeff_ptr
=
av1_dequant_abscoeff_nuq
(
q
,
dequant
,
dequant_val
,
0
);
...
...
@@ -1045,8 +1047,10 @@ static INLINE int highbd_quantize_coeff_bigtx_nuq(
const
int
abs_coeff
=
(
coeff
^
coeff_sign
)
-
coeff_sign
;
int
q
=
0
;
int64_t
tmp
=
clamp
(
abs_coeff
,
INT32_MIN
,
INT32_MAX
);
if
(
tmp
>=
zbin_val
)
{
tmp
-=
ROUND_POWER_OF_TWO
(
cuml_bins_ptr
[
0
],
logsizeby16
);
const
int
cuml_bins_ptr_val
=
ROUND_POWER_OF_TWO
(
cuml_bins_ptr
[
0
],
logsizeby16
);
if
(
tmp
>=
AOMMAX
(
zbin_val
,
cuml_bins_ptr_val
))
{
tmp
-=
cuml_bins_ptr_val
;
q
=
NUQ_KNOTS
+
(
int
)(((((
tmp
*
quant
)
>>
16
)
+
tmp
)
*
quant_shift
)
>>
(
16
-
logsizeby16
));
*
dqcoeff_ptr
=
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment