Commit 8265d70c authored by Hui Su's avatar Hui Su

Fix floating point number error in palette RDO

BUG=aomedia:1112

Change-Id: I130bc2087c32bd2d9087bf0dfe5919226f5fafa1
parent 840c66ee
......@@ -2818,6 +2818,12 @@ static void palette_rd_y(const AV1_COMP *const cpi, MACROBLOCK *x,
int64_t *best_model_rd, int *rate, int *rate_tokenonly,
int *rate_overhead, int64_t *distortion,
int *skippable) {
aom_clear_system_state();
#ifndef NDEBUG
for (int i = 0; i < n; ++i) {
assert(!isnan(centroids[i]));
}
#endif // NDEBUG
#if CONFIG_PALETTE_DELTA_ENCODING
optimize_palette_colors(color_cache, n_cache, n, 1, centroids);
#endif // CONFIG_PALETTE_DELTA_ENCODING
......@@ -3004,6 +3010,7 @@ static int rd_pick_palette_intra_sby(const AV1_COMP *const cpi, MACROBLOCK *x,
// K-means clustering.
for (n = AOMMIN(colors, PALETTE_MAX_SIZE); n >= 2; --n) {
aom_clear_system_state();
if (colors == PALETTE_MIN_SIZE) {
// Special case: These colors automatically become the centroids.
assert(colors == n);
......@@ -3016,7 +3023,6 @@ static int rd_pick_palette_intra_sby(const AV1_COMP *const cpi, MACROBLOCK *x,
}
av1_k_means(data, centroids, color_map, rows * cols, n, 1, max_itr);
}
palette_rd_y(cpi, x, mbmi, bsize, palette_ctx, dc_mode_cost, data,
centroids, n,
#if CONFIG_PALETTE_DELTA_ENCODING
......@@ -5219,11 +5225,17 @@ static void rd_pick_palette_intra_sbuv(const AV1_COMP *const cpi, MACROBLOCK *x,
for (n = colors > PALETTE_MAX_SIZE ? PALETTE_MAX_SIZE : colors; n >= 2;
--n) {
aom_clear_system_state();
for (i = 0; i < n; ++i) {
centroids[i * 2] = lb_u + (2 * i + 1) * (ub_u - lb_u) / n / 2;
centroids[i * 2 + 1] = lb_v + (2 * i + 1) * (ub_v - lb_v) / n / 2;
}
av1_k_means(data, centroids, color_map, rows * cols, n, 2, max_itr);
#ifndef NDEBUG
for (i = 0; i < 2 * n; ++i) {
assert(!isnan(centroids[i]));
}
#endif // NDEBUG
#if CONFIG_PALETTE_DELTA_ENCODING
optimize_palette_colors(color_cache, n_cache, n, 2, centroids);
// Sort the U channel colors in ascending order.
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment