Commit 4d34154b authored by Yaowu Xu's avatar Yaowu Xu

Fix IOC warnings

av1_txfm.h: left shift of a negative number
av1/encoder/quantize.c: unsigned int overflow
aom_dsp/entenc.c: unsigned int overflow

Change-Id: I6143e68f7d6e2621f97900808c8ef7ee0ad0c814
parent 4b23059c
...@@ -514,7 +514,7 @@ unsigned char *od_ec_enc_done(od_ec_enc *enc, uint32_t *nbytes) { ...@@ -514,7 +514,7 @@ unsigned char *od_ec_enc_done(od_ec_enc *enc, uint32_t *nbytes) {
unsigned char *out; unsigned char *out;
uint32_t storage; uint32_t storage;
uint16_t *buf; uint16_t *buf;
uint32_t offs; int offs;
uint32_t end_offs; uint32_t end_offs;
int nend_bits; int nend_bits;
od_ec_window m; od_ec_window m;
...@@ -554,7 +554,7 @@ unsigned char *od_ec_enc_done(od_ec_enc *enc, uint32_t *nbytes) { ...@@ -554,7 +554,7 @@ unsigned char *od_ec_enc_done(od_ec_enc *enc, uint32_t *nbytes) {
if (s > 0) { if (s > 0) {
unsigned n; unsigned n;
storage = enc->precarry_storage; storage = enc->precarry_storage;
if (offs + ((s + 7) >> 3) > storage) { if (offs + ((s + 7) >> 3) > (int)storage) {
storage = storage * 2 + ((s + 7) >> 3); storage = storage * 2 + ((s + 7) >> 3);
buf = (uint16_t *)realloc(buf, sizeof(*buf) * storage); buf = (uint16_t *)realloc(buf, sizeof(*buf) * storage);
if (buf == NULL) { if (buf == NULL) {
...@@ -566,7 +566,7 @@ unsigned char *od_ec_enc_done(od_ec_enc *enc, uint32_t *nbytes) { ...@@ -566,7 +566,7 @@ unsigned char *od_ec_enc_done(od_ec_enc *enc, uint32_t *nbytes) {
} }
n = (1 << (c + 16)) - 1; n = (1 << (c + 16)) - 1;
do { do {
OD_ASSERT(offs < storage); OD_ASSERT(offs < (int)storage);
buf[offs++] = (uint16_t)(e >> (c + 16)); buf[offs++] = (uint16_t)(e >> (c + 16));
e &= n; e &= n;
s -= 8; s -= 8;
......
...@@ -81,7 +81,7 @@ static INLINE void round_shift_array(int32_t *arr, int size, int bit) { ...@@ -81,7 +81,7 @@ static INLINE void round_shift_array(int32_t *arr, int size, int bit) {
} }
} else { } else {
for (i = 0; i < size; i++) { for (i = 0; i < size; i++) {
arr[i] = arr[i] << (-bit); arr[i] = arr[i] * (1 << (-bit));
} }
} }
} }
......
...@@ -1464,7 +1464,8 @@ void av1_highbd_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs, ...@@ -1464,7 +1464,8 @@ void av1_highbd_quantize_b_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
#else #else
const uint32_t abs_qcoeff = const uint32_t abs_qcoeff =
(uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> shift); (uint32_t)((tmp2 * quant_shift_ptr[rc != 0]) >> shift);
qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign); qcoeff_ptr[rc] =
(tran_low_t)((int)(abs_qcoeff ^ coeff_sign) - coeff_sign);
dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / scale; dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0] / scale;
#endif // CONFIG_AOM_QM #endif // CONFIG_AOM_QM
if (abs_qcoeff) eob = i; if (abs_qcoeff) eob = i;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment