Fix for issue 399 (multiply overflow in cdef_find_dir)

Add guard to not process blocks within a SB that are entirely off the
edge of the frame. The filter function was already guarded against
this possibility, but the find_dir function was not.

src/cdef.rs

@@ -277,34 +277,10 @@ pub fn cdef_frame(fi: &FrameInvariants, rec: &mut Frame, bc: &mut BlockContext)
                             }
                             for p in 0..3 {
                                 let mut rec_plane = &mut rec.planes[p];
+                                let rec_po = sbo.plane_offset(&rec_plane.cfg);
                                 let mut cdef_plane = &mut cdef_frame.planes[p];
                                 let xdec = cdef_plane.cfg.xdec;
                                 let ydec = cdef_plane.cfg.ydec;
-                                let rec_stride = rec_plane.cfg.stride;
-                                let rec_po = sbo.plane_offset(&rec_plane.cfg);
-                                let mut rec_slice = &mut rec_plane.mut_slice(&rec_po);
-                                let cdef_stride = cdef_plane.cfg.stride;
-                                let cdef_po = sbo.plane_offset(&cdef_plane.cfg);
-                                let cdef_slice = &cdef_plane.mut_slice(&cdef_po);
-
-                                let mut local_pri_strength;
-                                let mut local_sec_strength;
-                                let mut local_damping: i32 = cdef_damping + coeff_shift;
-                                let mut local_dir: usize;
-                                
-                                if p==0 {
-                                    dir = cdef_find_dir(cdef_slice.offset((8*bx>>xdec)+2,(8*by>>ydec)+2), 
-                                                        cdef_stride, &mut var, coeff_shift);
-                                    local_pri_strength = adjust_strength(cdef_pri_y_strength << coeff_shift, var);
-                                    local_sec_strength = cdef_sec_y_strength << coeff_shift;
-                                    local_dir = if cdef_pri_y_strength != 0 {dir as usize} else {0};
-                                } else {
-                                    local_pri_strength = cdef_pri_uv_strength << coeff_shift;
-                                    local_sec_strength = cdef_sec_uv_strength << coeff_shift;
-                                    local_damping -= 1;
-                                    local_dir = if cdef_pri_uv_strength != 0 {dir as usize} else {0};
-                                }
-                                
                                 let mut xsize = (fi.padded_w as i32 - 8*bx as i32 >> xdec as i32) - rec_po.x as i32;
                                 let mut ysize = (fi.padded_h as i32 - 8*by as i32 >> ydec as i32) - rec_po.y as i32;
                                 if xsize > (8>>xdec) {
@@ -314,6 +290,30 @@ pub fn cdef_frame(fi: &FrameInvariants, rec: &mut Frame, bc: &mut BlockContext)
                                     ysize = 8 >> ydec;
                                 }
                                 if xsize > 0 && ysize > 0 {
+                                    let rec_stride = rec_plane.cfg.stride;
+                                    let mut rec_slice = &mut rec_plane.mut_slice(&rec_po);
+                                    let cdef_stride = cdef_plane.cfg.stride;
+                                    let cdef_po = sbo.plane_offset(&cdef_plane.cfg);
+                                    let cdef_slice = &cdef_plane.mut_slice(&cdef_po);
+
+                                    let mut local_pri_strength;
+                                    let mut local_sec_strength;
+                                    let mut local_damping: i32 = cdef_damping + coeff_shift;
+                                    let mut local_dir: usize;
+
+                                    if p==0 {
+                                        dir = cdef_find_dir(cdef_slice.offset((8*bx>>xdec)+2,(8*by>>ydec)+2),
+                                                            cdef_stride, &mut var, coeff_shift);
+                                        local_pri_strength = adjust_strength(cdef_pri_y_strength << coeff_shift, var);
+                                        local_sec_strength = cdef_sec_y_strength << coeff_shift;
+                                        local_dir = if cdef_pri_y_strength != 0 {dir as usize} else {0};
+                                    } else {
+                                        local_pri_strength = cdef_pri_uv_strength << coeff_shift;
+                                        local_sec_strength = cdef_sec_uv_strength << coeff_shift;
+                                        local_damping -= 1;
+                                        local_dir = if cdef_pri_uv_strength != 0 {dir as usize} else {0};
+                                    }
+
                                     cdef_filter_block(rec_slice.offset_as_mutable(8*bx>>xdec,8*by>>ydec), rec_stride as i32,
                                                       cdef_slice.offset(8*bx>>xdec,8*by>>ydec), cdef_stride as i32, 
                                                       local_pri_strength, local_sec_strength, local_dir,