diff --git a/src/encoder.rs b/src/encoder.rs
index 095dfb0d459e7db0cfc4d8fe7db2f634ab9fad92..22053235f57f3f7a7786db88ef572f028a5a82c2 100644
--- a/src/encoder.rs
+++ b/src/encoder.rs
@@ -451,19 +451,17 @@ impl FrameState {
 
   pub fn new_with_frame(fi: &FrameInvariants, frame: Arc<Frame>) -> FrameState {
     let rs = RestorationState::new(fi, &frame);
+    let (luma_width, luma_height, luma_xpad, luma_ypad) = (
+      frame.planes[0].cfg.width,
+      frame.planes[0].cfg.height,
+      frame.planes[0].cfg.xpad,
+      frame.planes[0].cfg.ypad
+    );
     FrameState {
       input: frame,
-      input_hres: Plane::new(
-        fi.padded_w/2, fi.padded_h/2,
-        1, 1,
-        (MAX_SB_SIZE + FRAME_MARGIN) / 2, (MAX_SB_SIZE + FRAME_MARGIN) / 2
-      ),
-      input_qres: Plane::new(
-        fi.padded_w/4, fi.padded_h/4,
-        2, 2,
-        (MAX_SB_SIZE + FRAME_MARGIN) / 4, (MAX_SB_SIZE + FRAME_MARGIN) / 4
-      ),
-      rec: Frame::new(fi.padded_w, fi.padded_h, fi.sequence.chroma_sampling),
+      input_hres: Plane::new(luma_width/2, luma_height/2, 1, 1, luma_xpad/2, luma_ypad/2),
+      input_qres: Plane::new(luma_width/4, luma_height/4, 2, 2, luma_xpad/4, luma_ypad/4),
+      rec: Frame::new(luma_width, luma_height, fi.sequence.chroma_sampling),
       qc: Default::default(),
       cdfs: CDFContext::new(0),
       deblock: Default::default(),