diff --git a/arch/x86/include/asm/arch_hweight.h b/arch/x86/include/asm/arch_hweight.h index 9686c3d9ff73a13e15a32b114ad839f899c1391e..259a7c1ef7099bc89b30cdb0ebbd3a311fa88e18 100644 --- a/arch/x86/include/asm/arch_hweight.h +++ b/arch/x86/include/asm/arch_hweight.h @@ -21,7 +21,7 @@ * ARCH_HWEIGHT_CFLAGS in for the respective * compiler switches. */ -static inline unsigned int __arch_hweight32(unsigned int w) +static __always_inline unsigned int __arch_hweight32(unsigned int w) { unsigned int res = 0; @@ -42,20 +42,23 @@ static inline unsigned int __arch_hweight8(unsigned int w) return __arch_hweight32(w & 0xff); } +#ifdef CONFIG_X86_32 static inline unsigned long __arch_hweight64(__u64 w) { - unsigned long res = 0; - -#ifdef CONFIG_X86_32 return __arch_hweight32((u32)w) + __arch_hweight32((u32)(w >> 32)); +} #else +static __always_inline unsigned long __arch_hweight64(__u64 w) +{ + unsigned long res = 0; + asm (ALTERNATIVE("call __sw_hweight64", POPCNT64, X86_FEATURE_POPCNT) : "="REG_OUT (res) : REG_IN (w)); -#endif /* CONFIG_X86_32 */ return res; } +#endif /* CONFIG_X86_32 */ #endif diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h index ea17cca9e6852f982b1f98a5a06f863de7324e61..9653fdb76a4273dc21cecc1ddf87ea3b7576668d 100644 --- a/include/linux/bitmap.h +++ b/include/linux/bitmap.h @@ -295,7 +295,7 @@ static inline int bitmap_full(const unsigned long *src, unsigned int nbits) return find_first_zero_bit(src, nbits) == nbits; } -static inline int bitmap_weight(const unsigned long *src, unsigned int nbits) +static __always_inline int bitmap_weight(const unsigned long *src, unsigned int nbits) { if (small_const_nbits(nbits)) return hweight_long(*src & BITMAP_LAST_WORD_MASK(nbits)); diff --git a/include/linux/bitops.h b/include/linux/bitops.h index 297f5bda4fdf66a798eef2b54b43cf3592e9c005..e63553386ae715d387fadd39c4e53650ce3e226e 100644 --- a/include/linux/bitops.h +++ b/include/linux/bitops.h @@ -57,7 +57,7 @@ extern unsigned long __sw_hweight64(__u64 w); (bit) < (size); \ (bit) = find_next_zero_bit((addr), (size), (bit) + 1)) -static __inline__ int get_bitmask_order(unsigned int count) +static inline int get_bitmask_order(unsigned int count) { int order; @@ -65,7 +65,7 @@ static __inline__ int get_bitmask_order(unsigned int count) return order; /* We could be slightly more clever with -1 here... */ } -static __inline__ int get_count_order(unsigned int count) +static inline int get_count_order(unsigned int count) { int order; @@ -75,7 +75,7 @@ static __inline__ int get_count_order(unsigned int count) return order; } -static inline unsigned long hweight_long(unsigned long w) +static __always_inline unsigned long hweight_long(unsigned long w) { return sizeof(w) == 4 ? hweight32(w) : hweight64(w); } diff --git a/include/linux/jiffies.h b/include/linux/jiffies.h index 535fd3bb1ba889d77afc3d7ad25c093810b5bfd6..1ba48a18c1d77e9fbc497a3f01cdd58bc15459f8 100644 --- a/include/linux/jiffies.h +++ b/include/linux/jiffies.h @@ -351,7 +351,7 @@ static inline unsigned long _msecs_to_jiffies(const unsigned int m) * directly here and from __msecs_to_jiffies() in the case where * constant folding is not possible. */ -static inline unsigned long msecs_to_jiffies(const unsigned int m) +static __always_inline unsigned long msecs_to_jiffies(const unsigned int m) { if (__builtin_constant_p(m)) { if ((int)m < 0) @@ -405,7 +405,7 @@ static inline unsigned long _usecs_to_jiffies(const unsigned int u) * directly here and from __msecs_to_jiffies() in the case where * constant folding is not possible. */ -static inline unsigned long usecs_to_jiffies(const unsigned int u) +static __always_inline unsigned long usecs_to_jiffies(const unsigned int u) { if (__builtin_constant_p(u)) { if (u > jiffies_to_usecs(MAX_JIFFY_OFFSET))