Commit cbb8be76 authored by hui su's avatar hui su

Set block size upper bound for Palette mode

Avoid buffer overflow in case of such new experiments as
128 x 128 superblock size.

Change-Id: Ib775f3925a85fc87227c0ddd9b6a6110a12ef196
parent 91707ac7
......@@ -34,6 +34,7 @@ extern "C" {
#define PALETTE_MAX_SIZE 8
#define PALETTE_BLOCK_SIZES (BLOCK_LARGEST - BLOCK_8X8 + 1)
#define PALETTE_Y_MODE_CONTEXTS 3
#define PALETTE_MAX_BLOCK_SIZE (64 * 64)
struct VP10Common;
......
......@@ -1823,6 +1823,9 @@ static int rd_pick_palette_intra_sby(VP10_COMP *cpi, MACROBLOCK *x,
mbmi->ext_intra_mode_info.use_ext_intra_mode[0] = 0;
#endif // CONFIG_EXT_INTRA
if (rows * cols > PALETTE_MAX_BLOCK_SIZE)
return 0;
for (n = colors > PALETTE_MAX_SIZE ? PALETTE_MAX_SIZE : colors;
n >= 2; --n) {
for (i = 0; i < n; ++i)
......@@ -3552,6 +3555,9 @@ static void rd_pick_palette_intra_sbuv(VP10_COMP *cpi, MACROBLOCK *x,
const uint8_t *const src_u = x->plane[1].src.buf;
const uint8_t *const src_v = x->plane[2].src.buf;
if (rows * cols > PALETTE_MAX_BLOCK_SIZE)
return;
#if CONFIG_EXT_INTRA
mbmi->ext_intra_mode_info.use_ext_intra_mode[1] = 0;
#endif // CONFIG_EXT_INTRA
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment