diff --git a/celt/celt_encoder.c b/celt/celt_encoder.c
index d77c80f58f0bcf9b8d782f1927eab6a6b92307b9..f5ff6aee9d5a12b4b3793622eebb8d3179bb42e4 100644
--- a/celt/celt_encoder.c
+++ b/celt/celt_encoder.c
@@ -1551,6 +1551,8 @@ int celt_encode_with_ec(CELTEncoder * OPUS_RESTRICT st, const opus_val16 * pcm,
       surround_masking = DIV32_16(mask_avg,C*st->end) + QCONST16(.0f, DB_SHIFT);
       surround_masking = MIN16(MAX16(surround_masking,-QCONST16(1.5f, DB_SHIFT)), 0);
    }
+   /* Temporal VBR (but not for LFE) */
+   if (!st->lfe)
    {
       opus_val16 follow=-QCONST16(10.0f,DB_SHIFT);
       float frame_avg=0;