Unverified Commit a9b44d75 authored by Kyle Siefring's avatar Kyle Siefring Committed by GitHub

Fix inter for 12-bit input. (#623)

* Fix inter for 12-bit input.

12-bit is a special case that rounds differently.

See 7.11.3.2. Rounding variable derivation process in the spec.
parent 39b5d478
......@@ -931,6 +931,12 @@ impl PredictionMode {
let max_sample_val = ((1 << bit_depth) - 1) as i32;
let y_filter_idx = if height <= 4 { 4 } else { 0 };
let x_filter_idx = if width <= 4 { 4 } else { 0 };
let shifts = {
let shift_offset = if bit_depth == 12 { 2 } else { 0 };
(3 + shift_offset, 11 - shift_offset)
};
let round_shift =
|n, shift| -> i32 { (n + (1 << (shift - 1))) >> shift };
match (col_frac, row_frac) {
(0, 0) => {
......@@ -962,7 +968,7 @@ impl PredictionMode {
* SUBPEL_FILTERS[y_filter_idx][row_frac as usize][k];
}
let output_index = r * stride + c;
let val = ((sum + 64) >> 7).max(0).min(max_sample_val);
let val = round_shift(sum, 7).max(0).min(max_sample_val);
slice[output_index] = val as u16;
}
}
......@@ -983,7 +989,9 @@ impl PredictionMode {
}
let output_index = r * stride + c;
let val =
((((sum + 4) >> 3) + 8) >> 4).max(0).min(max_sample_val);
round_shift(round_shift(sum, shifts.0), shifts.1 - 7)
.max(0)
.min(max_sample_val);
slice[output_index] = val as u16;
}
}
......@@ -1005,7 +1013,7 @@ impl PredictionMode {
sum += s[r * ref_stride + (c + k)] as i32 * SUBPEL_FILTERS
[x_filter_idx][col_frac as usize][k];
}
let val = (sum + 4) >> 3;
let val = round_shift(sum, shifts.0);
intermediate[8 * r + (c - cg)] = val as i16;
}
}
......@@ -1018,7 +1026,8 @@ impl PredictionMode {
* SUBPEL_FILTERS[y_filter_idx][row_frac as usize][k];
}
let output_index = r * stride + c;
let val = ((sum + 1024) >> 11).max(0).min(max_sample_val);
let val =
round_shift(sum, shifts.1).max(0).min(max_sample_val);
slice[output_index] = val as u16;
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment