Palette: use uint16_t to store palette colors

Harmonize the low-bitdepth and high-bitdepth.

Change-Id: I533f717ca189a5e577719821ea5aea940e3b6223

av1/common/blockd.h

@@ -266,12 +266,8 @@ typedef int8_t MV_REFERENCE_FRAME;
 typedef struct {
   // Number of base colors for Y (0) and UV (1)
   uint8_t palette_size[2];
-// Value of base colors for Y, U, and V
-#if CONFIG_HIGHBITDEPTH
+  // Value of base colors for Y, U, and V
   uint16_t palette_colors[3 * PALETTE_MAX_SIZE];
-#else
-  uint8_t palette_colors[3 * PALETTE_MAX_SIZE];
-#endif  // CONFIG_HIGHBITDEPTH
   // Only used by encoder to store the color index of the top left pixel.
   // TODO(huisu): move this to encoder
   uint8_t palette_first_color_idx[2];

av1/common/pred_common.c

@@ -165,14 +165,9 @@ int av1_get_palette_cache(const MODE_INFO *above_mi, const MODE_INFO *left_mi,
   int above_idx = plane * PALETTE_MAX_SIZE;
   int left_idx = plane * PALETTE_MAX_SIZE;
   int n = 0;
-#if CONFIG_HIGHBITDEPTH
   const uint16_t *above_colors =
       above_mi->mbmi.palette_mode_info.palette_colors;
   const uint16_t *left_colors = left_mi->mbmi.palette_mode_info.palette_colors;
-#else
-  const uint8_t *above_colors = above_mi->mbmi.palette_mode_info.palette_colors;
-  const uint8_t *left_colors = left_mi->mbmi.palette_mode_info.palette_colors;
-#endif  // CONFIG_HIGHBITDEPTH
   // Merge the sorted lists of base colors from above and left to get
   // combined sorted color cache.
   while (above_n > 0 && left_n > 0) {

av1/common/reconintra.c

@@ -2443,30 +2443,27 @@ static void predict_square_intra_block(const MACROBLOCKD *xd, int wpx, int hpx,
     const int stride = wpx;
     int r, c;
     const uint8_t *const map = xd->plane[plane != 0].color_index_map;
-#if CONFIG_HIGHBITDEPTH
     uint16_t *palette = xd->mi[0]->mbmi.palette_mode_info.palette_colors +
                         plane * PALETTE_MAX_SIZE;
-#else
-    uint8_t *palette = xd->mi[0]->mbmi.palette_mode_info.palette_colors +
-                       plane * PALETTE_MAX_SIZE;
-#endif  // CONFIG_HIGHBITDEPTH
 
 #if CONFIG_HIGHBITDEPTH
     if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH) {
       uint16_t *dst16 = CONVERT_TO_SHORTPTR(dst);
-      for (r = 0; r < bs; ++r)
-        for (c = 0; c < bs; ++c)
+      for (r = 0; r < bs; ++r) {
+        for (c = 0; c < bs; ++c) {
           dst16[r * dst_stride + c] = palette[map[(r + y) * stride + c + x]];
+        }
+      }
     } else {
-      for (r = 0; r < bs; ++r)
-        for (c = 0; c < bs; ++c)
+#endif  // CONFIG_HIGHBITDEPTH
+      for (r = 0; r < bs; ++r) {
+        for (c = 0; c < bs; ++c) {
           dst[r * dst_stride + c] =
-              (uint8_t)(palette[map[(r + y) * stride + c + x]]);
+              (uint8_t)palette[map[(r + y) * stride + c + x]];
+        }
+      }
+#if CONFIG_HIGHBITDEPTH
     }
-#else
-    for (r = 0; r < bs; ++r)
-      for (c = 0; c < bs; ++c)
-        dst[r * dst_stride + c] = palette[map[(r + y) * stride + c + x]];
 #endif  // CONFIG_HIGHBITDEPTH
     return;
   }

av1/decoder/decodemv.c

@@ -646,19 +646,11 @@ static int read_skip(AV1_COMMON *cm, const MACROBLOCKD *xd, int segment_id,
 
 #if CONFIG_PALETTE
 #if CONFIG_PALETTE_DELTA_ENCODING
-#if CONFIG_HIGHBITDEPTH
 static int uint16_compare(const void *a, const void *b) {
   const uint16_t va = *(const uint16_t *)a;
   const uint16_t vb = *(const uint16_t *)b;
   return va - vb;
 }
-#else
-static int uint8_compare(const void *a, const void *b) {
-  const uint8_t va = *(const uint8_t *)a;
-  const uint8_t vb = *(const uint8_t *)b;
-  return va - vb;
-}
-#endif  // CONFIG_HIGHBITDEPTH
 
 static void read_palette_colors_y(MACROBLOCKD *const xd, int bit_depth,
                                   PALETTE_MODE_INFO *const pmi, aom_reader *r) {
@@ -684,11 +676,7 @@ static void read_palette_colors_y(MACROBLOCKD *const xd, int bit_depth,
       }
     }
   }
-#if CONFIG_HIGHBITDEPTH
   qsort(pmi->palette_colors, n, sizeof(pmi->palette_colors[0]), uint16_compare);
-#else
-  qsort(pmi->palette_colors, n, sizeof(pmi->palette_colors[0]), uint8_compare);
-#endif  // CONFIG_HIGHBITDEPTH
 }
 
 static void read_palette_colors_uv(MACROBLOCKD *const xd, int bit_depth,
@@ -717,13 +705,8 @@ static void read_palette_colors_uv(MACROBLOCKD *const xd, int bit_depth,
       }
     }
   }
-#if CONFIG_HIGHBITDEPTH
   qsort(pmi->palette_colors + PALETTE_MAX_SIZE, n,
         sizeof(pmi->palette_colors[0]), uint16_compare);
-#else
-  qsort(pmi->palette_colors + PALETTE_MAX_SIZE, n,
-        sizeof(pmi->palette_colors[0]), uint8_compare);
-#endif  // CONFIG_HIGHBITDEPTH
 
   // V channel colors.
   if (aom_read_bit(r, ACCT_STR)) {  // Delta encoding.

av1/encoder/bitstream.c

@@ -1428,12 +1428,14 @@ static void delta_encode_palette_colors(const int *colors, int num,
                                         int bit_depth, int min_val,
                                         aom_writer *w) {
   if (num <= 0) return;
+  assert(colors[0] < (1 << bit_depth));
   aom_write_literal(w, colors[0], bit_depth);
   if (num == 1) return;
   int max_delta = 0;
   int deltas[PALETTE_MAX_SIZE];
   memset(deltas, 0, sizeof(deltas));
   for (int i = 1; i < num; ++i) {
+    assert(colors[i] < (1 << bit_depth));
     const int delta = colors[i] - colors[i - 1];
     deltas[i - 1] = delta;
     assert(delta >= min_val);
@@ -1441,6 +1443,7 @@ static void delta_encode_palette_colors(const int *colors, int num,
   }
   const int min_bits = bit_depth - 3;
   int bits = AOMMAX(av1_ceil_log2(max_delta + 1 - min_val), min_bits);
+  assert(bits <= bit_depth);
   int range = (1 << bit_depth) - colors[0] - min_val;
   aom_write_literal(w, bits - min_bits, 2);
   for (int i = 0; i < num - 1; ++i) {
@@ -1483,13 +1486,8 @@ static void write_palette_colors_uv(const MACROBLOCKD *const xd,
                                     const PALETTE_MODE_INFO *const pmi,
                                     int bit_depth, aom_writer *w) {
   const int n = pmi->palette_size[1];
-#if CONFIG_HIGHBITDEPTH
   const uint16_t *colors_u = pmi->palette_colors + PALETTE_MAX_SIZE;
   const uint16_t *colors_v = pmi->palette_colors + 2 * PALETTE_MAX_SIZE;
-#else
-  const uint8_t *colors_u = pmi->palette_colors + PALETTE_MAX_SIZE;
-  const uint8_t *colors_v = pmi->palette_colors + 2 * PALETTE_MAX_SIZE;
-#endif  // CONFIG_HIGHBITDEPTH
   // U channel colors.
   const MODE_INFO *const above_mi = xd->above_mi;
   const MODE_INFO *const left_mi = xd->left_mi;
@@ -1516,10 +1514,12 @@ static void write_palette_colors_uv(const MACROBLOCKD *const xd,
       2 + bit_depth + (bits_v + 1) * (n - 1) - zero_count;
   const int rate_using_raw = bit_depth * n;
   if (rate_using_delta < rate_using_raw) {  // delta encoding
+    assert(colors_v[0] < (1 << bit_depth));
     aom_write_bit(w, 1);
     aom_write_literal(w, bits_v - min_bits_v, 2);
     aom_write_literal(w, colors_v[0], bit_depth);
     for (int i = 1; i < n; ++i) {
+      assert(colors_v[i] < (1 << bit_depth));
       if (colors_v[i] == colors_v[i - 1]) {  // No need to signal sign bit.
         aom_write_literal(w, 0, bits_v);
         continue;
@@ -1536,7 +1536,10 @@ static void write_palette_colors_uv(const MACROBLOCKD *const xd,
     }
   } else {  // Transmit raw values.
     aom_write_bit(w, 0);
-    for (int i = 0; i < n; ++i) aom_write_literal(w, colors_v[i], bit_depth);
+    for (int i = 0; i < n; ++i) {
+      assert(colors_v[i] < (1 << bit_depth));
+      aom_write_literal(w, colors_v[i], bit_depth);
+    }
   }
 }
 #endif  // CONFIG_PALETTE_DELTA_ENCODING
@@ -1568,9 +1571,10 @@ static void write_palette_mode_info(const AV1_COMMON *cm, const MACROBLOCKD *xd,
 #if CONFIG_PALETTE_DELTA_ENCODING
       write_palette_colors_y(xd, pmi, cm->bit_depth, w);
 #else
-      int i;
-      for (i = 0; i < n; ++i)
+      for (int i = 0; i < n; ++i) {
+        assert(pmi->palette_colors[i] < (1 << cm->bit_depth));
         aom_write_literal(w, pmi->palette_colors[i], cm->bit_depth);
+      }
 #endif  // CONFIG_PALETTE_DELTA_ENCODING
       write_uniform(w, n, pmi->palette_first_color_idx[0]);
     }
@@ -1587,8 +1591,11 @@ static void write_palette_mode_info(const AV1_COMMON *cm, const MACROBLOCKD *xd,
 #if CONFIG_PALETTE_DELTA_ENCODING
       write_palette_colors_uv(xd, pmi, cm->bit_depth, w);
 #else
-      int i;
-      for (i = 0; i < n; ++i) {
+      for (int i = 0; i < n; ++i) {
+        assert(pmi->palette_colors[PALETTE_MAX_SIZE + i] <
+               (1 << cm->bit_depth));
+        assert(pmi->palette_colors[2 * PALETTE_MAX_SIZE + i] <
+               (1 << cm->bit_depth));
         aom_write_literal(w, pmi->palette_colors[PALETTE_MAX_SIZE + i],
                           cm->bit_depth);
         aom_write_literal(w, pmi->palette_colors[2 * PALETTE_MAX_SIZE + i],

av1/encoder/palette.c

@@ -183,6 +183,7 @@ static int delta_encode_cost(const int *colors, int num, int bit_depth,
     if (delta > max_delta) max_delta = delta;
   }
   int bits_per_delta = AOMMAX(av1_ceil_log2(max_delta + 1 - min_val), min_bits);
+  assert(bits_per_delta <= bit_depth);
   int range = (1 << bit_depth) - colors[0] - min_val;
   for (int i = 0; i < num - 1; ++i) {
     bits_cost += bits_per_delta;
@@ -192,16 +193,11 @@ static int delta_encode_cost(const int *colors, int num, int bit_depth,
   return bits_cost;
 }
 
-int av1_index_color_cache(uint16_t *color_cache, int n_cache,
-                          const void *colors, int n_colors,
+int av1_index_color_cache(const uint16_t *color_cache, int n_cache,
+                          const uint16_t *colors, int n_colors,
                           uint8_t *cache_color_found, int *out_cache_colors) {
-#if CONFIG_HIGHBITDEPTH
-  const uint16_t *colors_in = (const uint16_t *)colors;
-#else
-  const uint8_t *colors_in = (const uint8_t *)colors;
-#endif  // CONFIG_HIGHBITDEPTH
   if (n_cache <= 0) {
-    for (int i = 0; i < n_colors; ++i) out_cache_colors[i] = colors_in[i];
+    for (int i = 0; i < n_colors; ++i) out_cache_colors[i] = colors[i];
     return n_colors;
   }
   memset(cache_color_found, 0, n_cache * sizeof(*cache_color_found));
@@ -210,7 +206,7 @@ int av1_index_color_cache(uint16_t *color_cache, int n_cache,
   memset(in_cache_flags, 0, sizeof(in_cache_flags));
   for (int i = 0; i < n_cache && n_in_cache < n_colors; ++i) {
     for (int j = 0; j < n_colors; ++j) {
-      if (colors_in[j] == color_cache[i]) {
+      if (colors[j] == color_cache[i]) {
         in_cache_flags[j] = 1;
         cache_color_found[i] = 1;
         ++n_in_cache;
@@ -220,7 +216,7 @@ int av1_index_color_cache(uint16_t *color_cache, int n_cache,
   }
   int j = 0;
   for (int i = 0; i < n_colors; ++i)
-    if (!in_cache_flags[i]) out_cache_colors[j++] = colors_in[i];
+    if (!in_cache_flags[i]) out_cache_colors[j++] = colors[i];
   assert(j == n_colors - n_in_cache);
   return j;
 }

av1/encoder/palette.h

@@ -48,8 +48,8 @@ int av1_count_colors_highbd(const uint8_t *src8, int stride, int rows, int cols,
 // Given a color cache and a set of base colors, find if each cache color is
 // present in the base colors, record the binary results in "cache_color_found".
 // Record the colors that are not in the color cache in "out_cache_colors".
-int av1_index_color_cache(uint16_t *color_cache, int n_cache,
-                          const void *colors, int n_colors,
+int av1_index_color_cache(const uint16_t *color_cache, int n_cache,
+                          const uint16_t *colors, int n_colors,
                           uint8_t *cache_color_found, int *out_cache_colors);
 
 // Return the number of bits used to transmit each v palette color delta;