Commit 4e0b43e0 authored by Jingning Han's avatar Jingning Han
Browse files

Fix potential integer overflow in coeff dequant

This solves a potential integer overflow issue in the coefficient
dequantization. It fixes an enc/dec mismatch issue.

BUG=aomedia:1174

Change-Id: Iea80f7fc2011ed35d2a44e360acd59cd8e85cf42
parent 4d5cf537
......@@ -395,7 +395,7 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd,
*level = NUM_BASE_LEVELS + 1 + br_base + br_offset;
cul_level += *level;
tran_low_t t;
tran_high_t t;
#if CONFIG_NEW_QUANT
dqv_val = &dq_val[pos != 0][0];
t = av1_dequant_abscoeff_nuq(*level, dequant[!!pos], dqv_val);
......@@ -409,7 +409,7 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd,
#endif // !CONFIG_DAALA_TX
#endif // CONFIG_NEW_QUANT
if (signs[pos]) t = -t;
tcoeffs[pos] = t;
tcoeffs[pos] = (tran_low_t)t;
break;
}
if (counts) ++counts->coeff_br[txs_ctx][plane_type][idx][ctx][0];
......@@ -420,7 +420,7 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd,
// decode 0-th order Golomb code
*level = COEFF_BASE_RANGE + 1 + NUM_BASE_LEVELS;
// Save golomb in tcoeffs because adding it to level may incur overflow
tran_low_t t = *level + read_golomb(xd, r, counts);
tran_high_t t = *level + read_golomb(xd, r, counts);
cul_level += t;
#if CONFIG_NEW_QUANT
dqv_val = &dq_val[pos != 0][0];
......@@ -435,7 +435,7 @@ uint8_t av1_read_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCKD *const xd,
#endif // !CONFIG_DAALA_TX
#endif // CONFIG_NEW_QUANT
if (signs[pos]) t = -t;
tcoeffs[pos] = t;
tcoeffs[pos] = (tran_low_t)t;
}
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment