Commit 9bc1d8de authored by hui su's avatar hui su Committed by Hui Su

Fix a conflict between palette and cb4x4

Palette is only enabled for blocks>=8x8

Change-Id: Ia12d42413b0345e5f298ded9e41118622008d233
parent d767bebd
...@@ -2204,6 +2204,7 @@ static void write_tokens_b(AV1_COMP *cpi, const TileInfo *const tile, ...@@ -2204,6 +2204,7 @@ static void write_tokens_b(AV1_COMP *cpi, const TileInfo *const tile,
mbmi->palette_mode_info.palette_size[plane]; mbmi->palette_mode_info.palette_size[plane];
if (palette_size_plane > 0) { if (palette_size_plane > 0) {
int rows, cols; int rows, cols;
assert(mbmi->sb_type >= BLOCK_8X8);
av1_get_block_dimensions(mbmi->sb_type, plane, xd, NULL, NULL, &rows, av1_get_block_dimensions(mbmi->sb_type, plane, xd, NULL, NULL, &rows,
&cols); &cols);
assert(*tok < tok_end); assert(*tok < tok_end);
......
...@@ -3658,6 +3658,8 @@ static int64_t rd_pick_intra_sby_mode(const AV1_COMP *const cpi, MACROBLOCK *x, ...@@ -3658,6 +3658,8 @@ static int64_t rd_pick_intra_sby_mode(const AV1_COMP *const cpi, MACROBLOCK *x,
? x->palette_buffer->best_palette_color_map ? x->palette_buffer->best_palette_color_map
: NULL; : NULL;
int palette_y_mode_ctx = 0; int palette_y_mode_ctx = 0;
const int try_palette =
cpi->common.allow_screen_content_tools && bsize >= BLOCK_8X8;
#endif // CONFIG_PALETTE #endif // CONFIG_PALETTE
const MODE_INFO *above_mi = xd->above_mi; const MODE_INFO *above_mi = xd->above_mi;
const MODE_INFO *left_mi = xd->left_mi; const MODE_INFO *left_mi = xd->left_mi;
...@@ -3754,11 +3756,12 @@ static int64_t rd_pick_intra_sby_mode(const AV1_COMP *const cpi, MACROBLOCK *x, ...@@ -3754,11 +3756,12 @@ static int64_t rd_pick_intra_sby_mode(const AV1_COMP *const cpi, MACROBLOCK *x,
this_rate_tokenonly -= tx_size_cost(cpi, x, bsize, mbmi->tx_size); this_rate_tokenonly -= tx_size_cost(cpi, x, bsize, mbmi->tx_size);
} }
#if CONFIG_PALETTE #if CONFIG_PALETTE
if (cpi->common.allow_screen_content_tools && mbmi->mode == DC_PRED) if (try_palette && mbmi->mode == DC_PRED) {
this_rate += this_rate +=
av1_cost_bit(av1_default_palette_y_mode_prob[bsize - BLOCK_8X8] av1_cost_bit(av1_default_palette_y_mode_prob[bsize - BLOCK_8X8]
[palette_y_mode_ctx], [palette_y_mode_ctx],
0); 0);
}
#endif // CONFIG_PALETTE #endif // CONFIG_PALETTE
#if CONFIG_FILTER_INTRA #if CONFIG_FILTER_INTRA
if (mbmi->mode == DC_PRED) if (mbmi->mode == DC_PRED)
...@@ -3806,7 +3809,7 @@ static int64_t rd_pick_intra_sby_mode(const AV1_COMP *const cpi, MACROBLOCK *x, ...@@ -3806,7 +3809,7 @@ static int64_t rd_pick_intra_sby_mode(const AV1_COMP *const cpi, MACROBLOCK *x,
#endif // CONFIG_PVQ #endif // CONFIG_PVQ
#if CONFIG_PALETTE #if CONFIG_PALETTE
if (cpi->common.allow_screen_content_tools) { if (try_palette) {
rd_pick_palette_intra_sby(cpi, x, bsize, palette_y_mode_ctx, rd_pick_palette_intra_sby(cpi, x, bsize, palette_y_mode_ctx,
bmode_costs[DC_PRED], &best_mbmi, bmode_costs[DC_PRED], &best_mbmi,
best_palette_color_map, &best_rd, &best_model_rd, best_palette_color_map, &best_rd, &best_model_rd,
...@@ -9644,6 +9647,8 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data, ...@@ -9644,6 +9647,8 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
MACROBLOCKD *const xd = &x->e_mbd; MACROBLOCKD *const xd = &x->e_mbd;
MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi; MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
#if CONFIG_PALETTE #if CONFIG_PALETTE
const int try_palette =
cpi->common.allow_screen_content_tools && bsize >= BLOCK_8X8;
PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info; PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info;
#endif // CONFIG_PALETTE #endif // CONFIG_PALETTE
MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext; MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext;
...@@ -9800,7 +9805,7 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data, ...@@ -9800,7 +9805,7 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
#if CONFIG_PALETTE #if CONFIG_PALETTE
av1_zero(pmi_uv); av1_zero(pmi_uv);
if (cm->allow_screen_content_tools) { if (try_palette) {
if (above_mi) if (above_mi)
palette_ctx += (above_mi->mbmi.palette_mode_info.palette_size[0] > 0); palette_ctx += (above_mi->mbmi.palette_mode_info.palette_size[0] > 0);
if (left_mi) if (left_mi)
...@@ -10305,7 +10310,7 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data, ...@@ -10305,7 +10310,7 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
&rate_uv_tokenonly[uv_tx], &dist_uvs[uv_tx], &rate_uv_tokenonly[uv_tx], &dist_uvs[uv_tx],
&skip_uvs[uv_tx], &mode_uv[uv_tx]); &skip_uvs[uv_tx], &mode_uv[uv_tx]);
#if CONFIG_PALETTE #if CONFIG_PALETTE
if (cm->allow_screen_content_tools) pmi_uv[uv_tx] = *pmi; if (try_palette) pmi_uv[uv_tx] = *pmi;
#endif // CONFIG_PALETTE #endif // CONFIG_PALETTE
#if CONFIG_EXT_INTRA #if CONFIG_EXT_INTRA
...@@ -10321,7 +10326,7 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data, ...@@ -10321,7 +10326,7 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
skippable = skippable && skip_uvs[uv_tx]; skippable = skippable && skip_uvs[uv_tx];
mbmi->uv_mode = mode_uv[uv_tx]; mbmi->uv_mode = mode_uv[uv_tx];
#if CONFIG_PALETTE #if CONFIG_PALETTE
if (cm->allow_screen_content_tools) { if (try_palette) {
pmi->palette_size[1] = pmi_uv[uv_tx].palette_size[1]; pmi->palette_size[1] = pmi_uv[uv_tx].palette_size[1];
memcpy(pmi->palette_colors + PALETTE_MAX_SIZE, memcpy(pmi->palette_colors + PALETTE_MAX_SIZE,
pmi_uv[uv_tx].palette_colors + PALETTE_MAX_SIZE, pmi_uv[uv_tx].palette_colors + PALETTE_MAX_SIZE,
...@@ -10351,9 +10356,10 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data, ...@@ -10351,9 +10356,10 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
#endif // CONFIG_CB4X4 #endif // CONFIG_CB4X4
#if CONFIG_PALETTE #if CONFIG_PALETTE
if (cpi->common.allow_screen_content_tools && mbmi->mode == DC_PRED) if (try_palette && mbmi->mode == DC_PRED) {
rate2 += av1_cost_bit( rate2 += av1_cost_bit(
av1_default_palette_y_mode_prob[bsize - BLOCK_8X8][palette_ctx], 0); av1_default_palette_y_mode_prob[bsize - BLOCK_8X8][palette_ctx], 0);
}
#endif // CONFIG_PALETTE #endif // CONFIG_PALETTE
if (!xd->lossless[mbmi->segment_id] && bsize >= BLOCK_8X8) { if (!xd->lossless[mbmi->segment_id] && bsize >= BLOCK_8X8) {
...@@ -11013,7 +11019,7 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data, ...@@ -11013,7 +11019,7 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
#if CONFIG_PALETTE #if CONFIG_PALETTE
// Only try palette mode when the best mode so far is an intra mode. // Only try palette mode when the best mode so far is an intra mode.
if (cm->allow_screen_content_tools && !is_inter_mode(best_mbmode.mode)) { if (try_palette && !is_inter_mode(best_mbmode.mode)) {
int rate2 = 0; int rate2 = 0;
#if CONFIG_SUPERTX #if CONFIG_SUPERTX
int best_rate_nocoef; int best_rate_nocoef;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment