Commit 21378b8a authored by Debargha Mukherjee's avatar Debargha Mukherjee Committed by Gerrit Code Review
Browse files

Merge "Fix bilateral filter asan error for highbitdepth" into nextgenv2

parents 005ff815 c57924cb
......@@ -369,6 +369,7 @@ static void loop_bilateral_filter_tile_highbd(uint16_t *data, int tile_idx,
const int tile_height = rst->tile_height >> rst->subsampling_y;
int i, j, subtile_idx;
int h_start, h_end, v_start, v_end;
const int shift = bit_depth - 8;
for (subtile_idx = 0; subtile_idx < BILATERAL_SUBTILES; ++subtile_idx) {
uint16_t *data_p, *tmpdata_p;
......@@ -398,7 +399,7 @@ static void loop_bilateral_filter_tile_highbd(uint16_t *data, int tile_idx,
for (y = -RESTORATION_HALFWIN; y <= RESTORATION_HALFWIN; ++y) {
for (x = -RESTORATION_HALFWIN; x <= RESTORATION_HALFWIN; ++x) {
wt = (int)wx_lut[y + RESTORATION_HALFWIN][x + RESTORATION_HALFWIN] *
(int)wr_lut_[data_p2[x] - data_p[j]];
(int)wr_lut_[(data_p2[x] >> shift) - (data_p[j] >> shift)];
wtsum += (int64_t)wt;
flsum += (int64_t)wt * data_p2[x];
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment