diff --git a/include/net/xfrm.h b/include/net/xfrm.h
index 43b93d1134edbd3295365fe074d2eb7396f17836..9e3dc7b81a4db0f49b20c32b4c69bc95b62f53ea 100644
--- a/include/net/xfrm.h
+++ b/include/net/xfrm.h
@@ -845,9 +845,9 @@ static inline bool addr_match(const void *token1, const void *token2,
 static inline bool addr4_match(__be32 a1, __be32 a2, u8 prefixlen)
 {
 	/* C99 6.5.7 (3): u32 << 32 is undefined behaviour */
-	if (prefixlen == 0)
+	if (sizeof(long) == 4 && prefixlen == 0)
 		return true;
-	return !((a1 ^ a2) & htonl(0xFFFFFFFFu << (32 - prefixlen)));
+	return !((a1 ^ a2) & htonl(~0UL << (32 - prefixlen)));
 }
 
 static __inline__