From 9bc1d8de72a258725a3d5f2f1b51bf25fcc280f4 Mon Sep 17 00:00:00 2001
From: hui su <huisu@google.com>
Date: Fri, 24 Mar 2017 12:36:03 -0700
Subject: [PATCH] Fix a conflict between palette and cb4x4

Palette is only enabled for blocks>=8x8

Change-Id: Ia12d42413b0345e5f298ded9e41118622008d233
---
 av1/encoder/bitstream.c |  1 +
 av1/encoder/rdopt.c     | 20 +++++++++++++-------
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index 544c3f140..2d79031c4 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -2204,6 +2204,7 @@ static void write_tokens_b(AV1_COMP *cpi, const TileInfo *const tile,
         mbmi->palette_mode_info.palette_size[plane];
     if (palette_size_plane > 0) {
       int rows, cols;
+      assert(mbmi->sb_type >= BLOCK_8X8);
       av1_get_block_dimensions(mbmi->sb_type, plane, xd, NULL, NULL, &rows,
                                &cols);
       assert(*tok < tok_end);
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 0be894b1e..e6d778a6f 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -3658,6 +3658,8 @@ static int64_t rd_pick_intra_sby_mode(const AV1_COMP *const cpi, MACROBLOCK *x,
           ? x->palette_buffer->best_palette_color_map
           : NULL;
   int palette_y_mode_ctx = 0;
+  const int try_palette =
+      cpi->common.allow_screen_content_tools && bsize >= BLOCK_8X8;
 #endif  // CONFIG_PALETTE
   const MODE_INFO *above_mi = xd->above_mi;
   const MODE_INFO *left_mi = xd->left_mi;
@@ -3754,11 +3756,12 @@ static int64_t rd_pick_intra_sby_mode(const AV1_COMP *const cpi, MACROBLOCK *x,
       this_rate_tokenonly -= tx_size_cost(cpi, x, bsize, mbmi->tx_size);
     }
 #if CONFIG_PALETTE
-    if (cpi->common.allow_screen_content_tools && mbmi->mode == DC_PRED)
+    if (try_palette && mbmi->mode == DC_PRED) {
       this_rate +=
           av1_cost_bit(av1_default_palette_y_mode_prob[bsize - BLOCK_8X8]
                                                       [palette_y_mode_ctx],
                        0);
+    }
 #endif  // CONFIG_PALETTE
 #if CONFIG_FILTER_INTRA
     if (mbmi->mode == DC_PRED)
@@ -3806,7 +3809,7 @@ static int64_t rd_pick_intra_sby_mode(const AV1_COMP *const cpi, MACROBLOCK *x,
 #endif  // CONFIG_PVQ
 
 #if CONFIG_PALETTE
-  if (cpi->common.allow_screen_content_tools) {
+  if (try_palette) {
     rd_pick_palette_intra_sby(cpi, x, bsize, palette_y_mode_ctx,
                               bmode_costs[DC_PRED], &best_mbmi,
                               best_palette_color_map, &best_rd, &best_model_rd,
@@ -9644,6 +9647,8 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
   MACROBLOCKD *const xd = &x->e_mbd;
   MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
 #if CONFIG_PALETTE
+  const int try_palette =
+      cpi->common.allow_screen_content_tools && bsize >= BLOCK_8X8;
   PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info;
 #endif  // CONFIG_PALETTE
   MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext;
@@ -9800,7 +9805,7 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
 
 #if CONFIG_PALETTE
   av1_zero(pmi_uv);
-  if (cm->allow_screen_content_tools) {
+  if (try_palette) {
     if (above_mi)
       palette_ctx += (above_mi->mbmi.palette_mode_info.palette_size[0] > 0);
     if (left_mi)
@@ -10305,7 +10310,7 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
                              &rate_uv_tokenonly[uv_tx], &dist_uvs[uv_tx],
                              &skip_uvs[uv_tx], &mode_uv[uv_tx]);
 #if CONFIG_PALETTE
-        if (cm->allow_screen_content_tools) pmi_uv[uv_tx] = *pmi;
+        if (try_palette) pmi_uv[uv_tx] = *pmi;
 #endif  // CONFIG_PALETTE
 
 #if CONFIG_EXT_INTRA
@@ -10321,7 +10326,7 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
       skippable = skippable && skip_uvs[uv_tx];
       mbmi->uv_mode = mode_uv[uv_tx];
 #if CONFIG_PALETTE
-      if (cm->allow_screen_content_tools) {
+      if (try_palette) {
         pmi->palette_size[1] = pmi_uv[uv_tx].palette_size[1];
         memcpy(pmi->palette_colors + PALETTE_MAX_SIZE,
                pmi_uv[uv_tx].palette_colors + PALETTE_MAX_SIZE,
@@ -10351,9 +10356,10 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
 #endif  // CONFIG_CB4X4
 
 #if CONFIG_PALETTE
-      if (cpi->common.allow_screen_content_tools && mbmi->mode == DC_PRED)
+      if (try_palette && mbmi->mode == DC_PRED) {
         rate2 += av1_cost_bit(
             av1_default_palette_y_mode_prob[bsize - BLOCK_8X8][palette_ctx], 0);
+      }
 #endif  // CONFIG_PALETTE
 
       if (!xd->lossless[mbmi->segment_id] && bsize >= BLOCK_8X8) {
@@ -11013,7 +11019,7 @@ void av1_rd_pick_inter_mode_sb(const AV1_COMP *cpi, TileDataEnc *tile_data,
 
 #if CONFIG_PALETTE
   // Only try palette mode when the best mode so far is an intra mode.
-  if (cm->allow_screen_content_tools && !is_inter_mode(best_mbmode.mode)) {
+  if (try_palette && !is_inter_mode(best_mbmode.mode)) {
     int rate2 = 0;
 #if CONFIG_SUPERTX
     int best_rate_nocoef;
-- 
GitLab