From d9295dbbb00a891e7539db91e73571a4b8a2e0f8 Mon Sep 17 00:00:00 2001
From: Monty Montgomery <xiphmont@gmail.com>
Date: Thu, 19 Jul 2018 13:07:01 -0400
Subject: [PATCH] CDEF bounds error fix
Correction to end-of-frame bounds checking on skip flags (can't get
skip flag for blocks of the endge of the frame)
---
src/lib.rs | 100 +++++++++++++++++++++++++++--------------------------
1 file changed, 51 insertions(+), 49 deletions(-)
diff --git a/src/lib.rs b/src/lib.rs
index de874fda..8ec2e2c4 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -2030,58 +2030,60 @@ fn cdef_frame(fi: &FrameInvariants, rec: &mut Frame, bc: &mut BlockContext) {
// Each direction block is 8x8 in y, potentially smaller if subsampled in chroma
for by in 0..8 {
for bx in 0..8 {
- let mut dir = 0;
- let mut var: i32 = 0;
- let skip = bc.at(&sbo.block_offset(bx, by)).skip;
- for p in 0..3 {
- let mut rec_plane = &mut rec.planes[p];
- let mut cdef_plane = &mut cdef_frame.planes[p];
- let xdec = cdef_plane.cfg.xdec;
- let ydec = cdef_plane.cfg.ydec;
- let rec_stride = rec_plane.cfg.stride;
- let rec_po = sbo.plane_offset(&rec_plane.cfg);
- let mut rec_slice = &mut rec_plane.mut_slice(&rec_po);
- let cdef_stride = cdef_plane.cfg.stride;
- let cdef_po = sbo.plane_offset(&cdef_plane.cfg);
- let cdef_slice = &cdef_plane.mut_slice(&cdef_po);
-
- let mut local_pri_strength = 0;
- let mut local_sec_strength = 0;
- let mut local_pri_damping:i32 = cdef_pri_damping + coeff_shift;
- let mut local_sec_damping:i32 = cdef_sec_damping + coeff_shift;
- let mut local_dir:usize = 0;
+ let block_offset = sbo.block_offset(bx, by);
+ if block_offset.x < bc.cols && block_offset.y < bc.rows {
+ let mut dir = 0;
+ let mut var: i32 = 0;
+ let skip = bc.at(&block_offset).skip;
+ for p in 0..3 {
+ let mut rec_plane = &mut rec.planes[p];
+ let mut cdef_plane = &mut cdef_frame.planes[p];
+ let xdec = cdef_plane.cfg.xdec;
+ let ydec = cdef_plane.cfg.ydec;
+ let rec_stride = rec_plane.cfg.stride;
+ let rec_po = sbo.plane_offset(&rec_plane.cfg);
+ let mut rec_slice = &mut rec_plane.mut_slice(&rec_po);
+ let cdef_stride = cdef_plane.cfg.stride;
+ let cdef_po = sbo.plane_offset(&cdef_plane.cfg);
+ let cdef_slice = &cdef_plane.mut_slice(&cdef_po);
- if !skip {
- if p==0 {
- dir = cdef_find_dir(cdef_slice.offset((8*bx>>xdec)+2,(8*by>>ydec)+2),
- cdef_stride, &mut var, coeff_shift);
- local_pri_strength = adjust_strength(cdef_pri_y_strength << coeff_shift, var);
- local_sec_strength = cdef_sec_y_strength << coeff_shift;
- local_dir = if cdef_pri_y_strength != 0 {dir as usize} else {0};
+ let mut local_pri_strength = 0;
+ let mut local_sec_strength = 0;
+ let mut local_pri_damping:i32 = cdef_pri_damping + coeff_shift;
+ let mut local_sec_damping:i32 = cdef_sec_damping + coeff_shift;
+ let mut local_dir:usize = 0;
+
+ if !skip {
+ if p==0 {
+ dir = cdef_find_dir(cdef_slice.offset((8*bx>>xdec)+2,(8*by>>ydec)+2),
+ cdef_stride, &mut var, coeff_shift);
+ local_pri_strength = adjust_strength(cdef_pri_y_strength << coeff_shift, var);
+ local_sec_strength = cdef_sec_y_strength << coeff_shift;
+ local_dir = if cdef_pri_y_strength != 0 {dir as usize} else {0};
} else {
- local_pri_strength = cdef_pri_uv_strength << coeff_shift;
- local_sec_strength = cdef_sec_uv_strength << coeff_shift;
- local_pri_damping -= 1;
- local_sec_damping -= 1;
- local_dir = if cdef_pri_uv_strength != 0 {dir as usize} else {0};
+ local_pri_strength = cdef_pri_uv_strength << coeff_shift;
+ local_sec_strength = cdef_sec_uv_strength << coeff_shift;
+ local_pri_damping -= 1;
+ local_sec_damping -= 1;
+ local_dir = if cdef_pri_uv_strength != 0 {dir as usize} else {0};
+ }
+ }
+ let mut xsize = (fi.padded_w as i32 - 8*bx as i32 >> xdec as i32) - rec_po.x as i32;
+ let mut ysize = (fi.padded_h as i32 - 8*by as i32 >> ydec as i32) - rec_po.y as i32;
+ if xsize > (8>>xdec) {
+ xsize = 8 >> xdec;
+ }
+ if ysize > (8>>ydec) {
+ ysize = 8 >> ydec;
+ }
+ if xsize > 0 && ysize > 0 {
+ cdef_filter_block(rec_slice.offset_as_mutable(8*bx>>xdec,8*by>>ydec), rec_stride as i32,
+ cdef_slice.offset(8*bx>>xdec,8*by>>ydec), cdef_stride as i32,
+ local_pri_strength, local_sec_strength, local_dir,
+ local_pri_damping, local_sec_damping,
+ xsize, ysize,
+ coeff_shift as i32);
}
- }
-
- let mut xsize = (fi.padded_w as i32 - 8*bx as i32 >> xdec as i32) - rec_po.x as i32;
- let mut ysize = (fi.padded_h as i32 - 8*by as i32 >> ydec as i32) - rec_po.y as i32;
- if xsize > (8>>xdec) {
- xsize = 8 >> xdec;
- }
- if ysize > (8>>ydec) {
- ysize = 8 >> ydec;
- }
- if xsize > 0 && ysize > 0 {
- cdef_filter_block(rec_slice.offset_as_mutable(8*bx>>xdec,8*by>>ydec), rec_stride as i32,
- cdef_slice.offset(8*bx>>xdec,8*by>>ydec), cdef_stride as i32,
- local_pri_strength, local_sec_strength, local_dir,
- local_pri_damping, local_sec_damping,
- 8 >> xdec, 8 >> ydec,
- coeff_shift as i32);
}
}
}
--
GitLab