Commit 787b38eb authored by Yaowu Xu's avatar Yaowu Xu

Fix VP8 encoder UBSAN/IOC errors

1. vp8/decoder/dboolhuff.c
2. vp8/decoder/dboolhuff.h
3. vp8/encoder/bitstream.c
4. vp8/encoder/boolhuff.h
5. vp8/encoder/rdopt.c

BUG=https://bugs.chromium.org/p/webm/issues/detail?id=1218

Change-Id: I5d315d63fd7aeaee6f3bd79178e593f3db38a6b1
parent 46ff1072
......@@ -44,7 +44,7 @@ void vp8dx_bool_decoder_fill(BOOL_DECODER *br)
int shift = VP8_BD_VALUE_SIZE - CHAR_BIT - (count + CHAR_BIT);
size_t bytes_left = br->user_buffer_end - bufptr;
size_t bits_left = bytes_left * CHAR_BIT;
int x = (int)(shift + CHAR_BIT - bits_left);
int x = shift + CHAR_BIT - (int)bits_left;
int loop_end = 0;
unsigned char decrypted[sizeof(VP8_BD_VALUE) + 1];
......
......@@ -83,7 +83,7 @@ static int vp8dx_decode_bool(BOOL_DECODER *br, int probability) {
}
{
register unsigned int shift = vp8_norm[range];
register int shift = vp8_norm[range];
range <<= shift;
value <<= shift;
count -= shift;
......
......@@ -163,7 +163,7 @@ void vp8_pack_tokens(vp8_writer *w, const TOKENEXTRA *p, int xcount)
{
const TOKENEXTRA *stop = p + xcount;
unsigned int split;
unsigned int shift;
int shift;
int count = w->count;
unsigned int range = w->range;
unsigned int lowvalue = w->lowvalue;
......
......@@ -65,7 +65,7 @@ static void vp8_encode_bool(BOOL_CODER *br, int bit, int probability)
int count = br->count;
unsigned int range = br->range;
unsigned int lowvalue = br->lowvalue;
register unsigned int shift;
register int shift;
#ifdef VP8_ENTROPY_STATS
#if defined(SECTIONBITS_OUTPUT)
......
......@@ -1899,7 +1899,8 @@ static int calculate_final_rd_costs(int this_rd,
int prob_skip_cost;
prob_skip_cost = vp8_cost_bit(cpi->prob_skip_false, 1);
prob_skip_cost -= vp8_cost_bit(cpi->prob_skip_false, 0);
prob_skip_cost -=
(int)vp8_cost_bit(cpi->prob_skip_false, 0);
rd->rate2 += prob_skip_cost;
*other_cost += prob_skip_cost;
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment