diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8535ad5f33240f4b0c32adb441ecebc6a31b5434..eff161ca92970d7abdd237bf20685200fc866b87 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -507,7 +507,9 @@ if(NOT OPUS_DISABLE_INTRINSICS)
     if(OPUS_X86_MAY_HAVE_AVX2)
       add_sources_group(opus celt ${celt_sources_avx2})
       add_sources_group(opus silk ${silk_sources_avx2})
-      add_sources_group(opus silk ${silk_sources_float_avx2})
+      if (NOT OPUS_FIXED_POINT)
+        add_sources_group(opus silk ${silk_sources_float_avx2})
+      endif()
       if (OPUS_DNN)
         add_sources_group(opus lpcnet ${dnn_sources_avx2})
       endif()
@@ -519,7 +521,9 @@ if(NOT OPUS_DISABLE_INTRINSICS)
       endif()
       set_source_files_properties(${celt_sources_avx2} PROPERTIES COMPILE_FLAGS ${AVX2_FLAGS})
       set_source_files_properties(${silk_sources_avx2} PROPERTIES COMPILE_FLAGS ${AVX2_FLAGS})
-      set_source_files_properties(${silk_sources_float_avx2} PROPERTIES COMPILE_FLAGS ${AVX2_FLAGS})
+      if (NOT OPUS_FIXED_POINT)
+        set_source_files_properties(${silk_sources_float_avx2} PROPERTIES COMPILE_FLAGS ${AVX2_FLAGS})
+      endif()
       set_source_files_properties(${dnn_sources_avx2} PROPERTIES COMPILE_FLAGS ${AVX2_FLAGS})
     endif()
     if(OPUS_X86_PRESUME_AVX2)