diff --git a/CMakeLists.txt b/CMakeLists.txt
index db4c39d5..276c65f3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -257,7 +257,11 @@ elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "arm.*")
   set(HAVE_AVX512F FALSE)
   set(HAVE_FMA FALSE)
   set(HAVE_SSE4_1 FALSE)
+
   check_cxx_compiler_flag("-mfpu=neon" HAVE_NEON)
+  if(HAVE_NEON)
+    set(NEON_COMPILE_FLAGS "-mfpu=neon")
+  endif(HAVE_NEON)
 
 else()
 
@@ -271,7 +275,6 @@ else()
 endif(CMAKE_SYSTEM_PROCESSOR MATCHES "x86|x86_64|AMD64|amd64|i386|i686")
 
 if(HAVE_NEON)
-  set(NEON_COMPILE_FLAGS "-mfpu=neon")
   message(STATUS "LTO build is not supported on arm/RBPi.")
   set(ENABLE_LTO FALSE)  # enable LTO cause fatal error on arm/RBPi
 endif()