diff --git a/cpu-all.h b/cpu-all.h
index 25f9cea7a41ae10e7703920085eeff6c30f00ca1..448f2c5bae0afc1a1c74d42471035e3811a20866 100644
--- a/cpu-all.h
+++ b/cpu-all.h
@@ -120,7 +120,8 @@ static inline void tswap64s(uint64_t *s)
    endian ! */
 typedef union {
     float64 d;
-#if defined(WORDS_BIGENDIAN) || (defined(__arm__) && !defined(__VFP_FP__))
+#if defined(WORDS_BIGENDIAN) \
+    || (defined(__arm__) && !defined(__VFP_FP__) && !defined(CONFIG_SOFTFLOAT))
     struct {
         uint32_t upper;
         uint32_t lower;