diff --git a/include/linux/rhashtable.h b/include/linux/rhashtable.h index b54e6436547e281c65d0c5bd0e84df08cd6dfaad..882bc0fcea4be62d3e4a213b6a5cb74f2787622f 100644 --- a/include/linux/rhashtable.h +++ b/include/linux/rhashtable.h @@ -316,6 +316,7 @@ static inline struct rhash_lock_head __rcu **rht_bucket_insert( * local_bh. For that we have rht_assign_unlock(). As rcu_assign_pointer() * provides the same release semantics that bit_spin_unlock() provides, * this is safe. + * When we write to a bucket without unlocking, we use rht_assign_locked(). */ static inline void rht_lock(struct bucket_table *tbl, @@ -369,10 +370,12 @@ static inline struct rhash_head *rht_ptr_exclusive( return (void *)(((unsigned long)p) & ~BIT(1)); } -static inline struct rhash_lock_head __rcu *rht_ptr_locked(const - struct rhash_head *p) +static inline void rht_assign_locked(struct rhash_lock_head __rcu **bkt, + struct rhash_head *obj) { - return (void *)(((unsigned long)p) | BIT(1)); + struct rhash_head __rcu **p = (struct rhash_head __rcu **)bkt; + + rcu_assign_pointer(*p, (void *)((unsigned long)obj | BIT(1))); } static inline void rht_assign_unlock(struct bucket_table *tbl, diff --git a/lib/rhashtable.c b/lib/rhashtable.c index 237368ea98c5a18fe6065e61e1b65ed8cd8d20b3..ef5378efdef34f391d327b04bf2c815c475e7be4 100644 --- a/lib/rhashtable.c +++ b/lib/rhashtable.c @@ -259,7 +259,7 @@ static int rhashtable_rehash_one(struct rhashtable *ht, rcu_assign_pointer(*pprev, next); else /* Need to preserved the bit lock. */ - rcu_assign_pointer(*bkt, rht_ptr_locked(next)); + rht_assign_locked(bkt, next); out: return err; @@ -517,7 +517,7 @@ static void *rhashtable_lookup_one(struct rhashtable *ht, rcu_assign_pointer(*pprev, obj); else /* Need to preserve the bit lock */ - rcu_assign_pointer(*bkt, rht_ptr_locked(obj)); + rht_assign_locked(bkt, obj); return NULL; } @@ -570,7 +570,7 @@ static struct bucket_table *rhashtable_insert_one(struct rhashtable *ht, /* bkt is always the head of the list, so it holds * the lock, which we need to preserve */ - rcu_assign_pointer(*bkt, rht_ptr_locked(obj)); + rht_assign_locked(bkt, obj); atomic_inc(&ht->nelems); if (rht_grow_above_75(ht, tbl))