diff --git a/ivfenc.c b/ivfenc.c
index 3487b3594a43afe73736333209f16ce92d32b152..9afcf7c69298d29b18aa68b6be836a270ddfc542 100644
--- a/ivfenc.c
+++ b/ivfenc.c
@@ -890,6 +890,8 @@ int main(int argc, const char **argv_)
                 {
                     cfg.g_timebase.num = y4m.fps_d;
                     cfg.g_timebase.den = y4m.fps_n;
+                    /* And don't reset it in the second pass.*/
+                    arg_have_timebase = 1;
                 }
                 arg_use_i420 = 0;
             }