Commit 6f9cd946 authored by Rupert Swarbrick's avatar Rupert Swarbrick

ext-partition-types: Don't allow 4:1 blocks to use palettes

Since there are no CDFs set up for palettes for 4:1/1:4 blocks, we
should make sure we don't try to use them. Without this patch,
write_palette_mode_info gets called with a bsize of BLOCK_32X8 and
reads (and writes) off the end of the palette_y_size_cdf array.

This patch avoids calling it in this context and adds an assertion to
make sure we don't read off the end of the array in future.

The patch also adds the corresponding logic to rdopt.c.

Change-Id: I4d9aea982d057e305a6b578f35457eada819d38f
parent 15a11220
......@@ -788,6 +788,9 @@ static void read_palette_mode_info(AV1_COMMON *const cm, MACROBLOCKD *const xd,
const BLOCK_SIZE bsize = mbmi->sb_type;
PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info;
assert(bsize >= BLOCK_8X8 && bsize <= BLOCK_LARGEST);
const int block_palette_idx = bsize - BLOCK_8X8;
if (mbmi->mode == DC_PRED) {
int palette_y_mode_ctx = 0;
if (above_mi) {
......@@ -798,12 +801,12 @@ static void read_palette_mode_info(AV1_COMMON *const cm, MACROBLOCKD *const xd,
palette_y_mode_ctx +=
(left_mi->mbmi.palette_mode_info.palette_size[0] > 0);
}
if (aom_read(r, av1_default_palette_y_mode_prob[bsize - BLOCK_8X8]
if (aom_read(r, av1_default_palette_y_mode_prob[block_palette_idx]
[palette_y_mode_ctx],
ACCT_STR)) {
pmi->palette_size[0] =
aom_read_symbol(r,
xd->tile_ctx->palette_y_size_cdf[bsize - BLOCK_8X8],
xd->tile_ctx->palette_y_size_cdf[block_palette_idx],
PALETTE_SIZES, ACCT_STR) +
2;
#if CONFIG_PALETTE_DELTA_ENCODING
......@@ -821,7 +824,7 @@ static void read_palette_mode_info(AV1_COMMON *const cm, MACROBLOCKD *const xd,
ACCT_STR)) {
pmi->palette_size[1] =
aom_read_symbol(r,
xd->tile_ctx->palette_uv_size_cdf[bsize - BLOCK_8X8],
xd->tile_ctx->palette_uv_size_cdf[block_palette_idx],
PALETTE_SIZES, ACCT_STR) +
2;
#if CONFIG_PALETTE_DELTA_ENCODING
......@@ -1210,7 +1213,8 @@ static void read_intra_frame_mode_info(AV1_COMMON *const cm,
#if CONFIG_PALETTE
mbmi->palette_mode_info.palette_size[0] = 0;
mbmi->palette_mode_info.palette_size[1] = 0;
if (bsize >= BLOCK_8X8 && cm->allow_screen_content_tools)
if (bsize >= BLOCK_8X8 && bsize <= BLOCK_LARGEST &&
cm->allow_screen_content_tools)
read_palette_mode_info(cm, xd, r);
#endif // CONFIG_PALETTE
#if CONFIG_FILTER_INTRA
......
......@@ -1533,6 +1533,9 @@ static void write_palette_mode_info(const AV1_COMMON *cm, const MACROBLOCKD *xd,
const BLOCK_SIZE bsize = mbmi->sb_type;
const PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info;
assert(bsize >= BLOCK_8X8 && bsize <= BLOCK_LARGEST);
const int block_palette_idx = bsize - BLOCK_8X8;
if (mbmi->mode == DC_PRED) {
const int n = pmi->palette_size[0];
int palette_y_mode_ctx = 0;
......@@ -1546,10 +1549,10 @@ static void write_palette_mode_info(const AV1_COMMON *cm, const MACROBLOCKD *xd,
}
aom_write(
w, n > 0,
av1_default_palette_y_mode_prob[bsize - BLOCK_8X8][palette_y_mode_ctx]);
av1_default_palette_y_mode_prob[block_palette_idx][palette_y_mode_ctx]);
if (n > 0) {
aom_write_symbol(w, n - PALETTE_MIN_SIZE,
xd->tile_ctx->palette_y_size_cdf[bsize - BLOCK_8X8],
xd->tile_ctx->palette_y_size_cdf[block_palette_idx],
PALETTE_SIZES);
#if CONFIG_PALETTE_DELTA_ENCODING
write_palette_colors_y(xd, pmi, cm->bit_depth, w);
......@@ -1568,7 +1571,7 @@ static void write_palette_mode_info(const AV1_COMMON *cm, const MACROBLOCKD *xd,
aom_write(w, n > 0, av1_default_palette_uv_mode_prob[palette_uv_mode_ctx]);
if (n > 0) {
aom_write_symbol(w, n - PALETTE_MIN_SIZE,
xd->tile_ctx->palette_uv_size_cdf[bsize - BLOCK_8X8],
xd->tile_ctx->palette_uv_size_cdf[block_palette_idx],
PALETTE_SIZES);
#if CONFIG_PALETTE_DELTA_ENCODING
write_palette_colors_uv(xd, pmi, cm->bit_depth, w);
......@@ -2284,7 +2287,8 @@ static void write_mb_modes_kf(AV1_COMMON *cm,
write_intra_angle_info(xd, ec_ctx, w);
#endif // CONFIG_EXT_INTRA
#if CONFIG_PALETTE
if (bsize >= BLOCK_8X8 && cm->allow_screen_content_tools)
if (bsize >= BLOCK_8X8 && bsize <= BLOCK_LARGEST &&
cm->allow_screen_content_tools)
write_palette_mode_info(cm, xd, mi, w);
#endif // CONFIG_PALETTE
#if CONFIG_FILTER_INTRA
......
......@@ -4303,8 +4303,8 @@ static int64_t rd_pick_intra_sby_mode(const AV1_COMP *const cpi, MACROBLOCK *x,
? x->palette_buffer->best_palette_color_map
: NULL;
int palette_y_mode_ctx = 0;
const int try_palette =
cpi->common.allow_screen_content_tools && bsize >= BLOCK_8X8;
const int try_palette = cpi->common.allow_screen_content_tools &&
bsize >= BLOCK_8X8 && bsize <= BLOCK_LARGEST;
#endif // CONFIG_PALETTE
const MODE_INFO *above_mi = xd->above_mi;
const MODE_INFO *left_mi = xd->left_mi;
......@@ -6038,6 +6038,8 @@ static int64_t rd_pick_intra_sbuv_mode(const AV1_COMP *const cpi, MACROBLOCK *x,
#endif // CONFIG_PVQ
#if CONFIG_PALETTE
PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info;
const int try_palette = cpi->common.allow_screen_content_tools &&
bsize >= BLOCK_8X8 && bsize <= BLOCK_LARGEST;
#endif // CONFIG_PALETTE
for (int mode_idx = 0; mode_idx < UV_INTRA_MODES; ++mode_idx) {
......@@ -6099,8 +6101,7 @@ static int64_t rd_pick_intra_sbuv_mode(const AV1_COMP *const cpi, MACROBLOCK *x,
this_rate += av1_cost_bit(cpi->common.fc->filter_intra_probs[1], 0);
#endif // CONFIG_FILTER_INTRA
#if CONFIG_PALETTE
if (cpi->common.allow_screen_content_tools && mbmi->sb_type >= BLOCK_8X8 &&
mode == UV_DC_PRED)
if (try_palette && mode == UV_DC_PRED)
this_rate += av1_cost_bit(
av1_default_palette_uv_mode_prob[pmi->palette_size[0] > 0], 0);
#endif // CONFIG_PALETTE
......@@ -6121,7 +6122,7 @@ static int64_t rd_pick_intra_sbuv_mode(const AV1_COMP *const cpi, MACROBLOCK *x,
}
#if CONFIG_PALETTE
if (cpi->common.allow_screen_content_tools && mbmi->sb_type >= BLOCK_8X8) {
if (try_palette) {
uint8_t *best_palette_color_map = x->palette_buffer->best_palette_color_map;
rd_pick_palette_intra_sbuv(cpi, x,
x->intra_uv_mode_cost[mbmi->mode][UV_DC_PRED],
......@@ -10133,6 +10134,8 @@ static void pick_filter_intra_interframe(
MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
#if CONFIG_PALETTE
PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info;
const int try_palette = cpi->common.allow_screen_content_tools &&
bsize >= BLOCK_8X8 && bsize <= BLOCK_LARGEST;
#endif // CONFIG_PALETTE
int rate2 = 0, rate_y = INT_MAX, skippable = 0, rate_uv, rate_dummy, i;
int dc_mode_index;
......@@ -10201,8 +10204,7 @@ static void pick_filter_intra_interframe(
rate2 = rate_y + intra_mode_cost[mbmi->mode] + rate_uv +
x->intra_uv_mode_cost[mbmi->mode][mbmi->uv_mode];
#if CONFIG_PALETTE
if (cpi->common.allow_screen_content_tools && mbmi->mode == DC_PRED &&
bsize >= BLOCK_8X8)
if (try_palette && mbmi->mode == DC_PRED)
rate2 += av1_cost_bit(
av1_default_palette_y_mode_prob[bsize - BLOCK_8X8][palette_ctx], 0);
#endif // CONFIG_PALETTE
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment