diff --git a/include/net/netfilter/nf_tables.h b/include/net/netfilter/nf_tables.h
index a785699329c9fa512e417f33c50182baa3cfe568..746423332fcb25fe58e15a222c2d429c05c90268 100644
--- a/include/net/netfilter/nf_tables.h
+++ b/include/net/netfilter/nf_tables.h
@@ -258,6 +258,7 @@ void nft_unregister_set(struct nft_set_ops *ops);
  * 	@dtype: data type (verdict or numeric type defined by userspace)
  * 	@size: maximum set size
  * 	@nelems: number of elements
+ * 	@ndeact: number of deactivated elements queued for removal
  * 	@timeout: default timeout value in msecs
  * 	@gc_int: garbage collection interval in msecs
  *	@policy: set parameterization (see enum nft_set_policies)
@@ -275,7 +276,8 @@ struct nft_set {
 	u32				ktype;
 	u32				dtype;
 	u32				size;
-	u32				nelems;
+	atomic_t			nelems;
+	u32				ndeact;
 	u64				timeout;
 	u32				gc_int;
 	u16				policy;
diff --git a/net/netfilter/nf_tables_api.c b/net/netfilter/nf_tables_api.c
index 0dab872e821b6a74f7ffca95f485420d6d9a7e08..27d1bf55a58186959a8255758b66d68abf456c3c 100644
--- a/net/netfilter/nf_tables_api.c
+++ b/net/netfilter/nf_tables_api.c
@@ -3238,9 +3238,6 @@ static int nft_add_set_elem(struct nft_ctx *ctx, struct nft_set *set,
 	u32 flags;
 	int err;
 
-	if (set->size && set->nelems == set->size)
-		return -ENFILE;
-
 	err = nla_parse_nested(nla, NFTA_SET_ELEM_MAX, attr,
 			       nft_set_elem_policy);
 	if (err < 0)
@@ -3391,11 +3388,15 @@ static int nf_tables_newsetelem(struct sock *nlsk, struct sk_buff *skb,
 		return -EBUSY;
 
 	nla_for_each_nested(attr, nla[NFTA_SET_ELEM_LIST_ELEMENTS], rem) {
+		if (set->size &&
+		    !atomic_add_unless(&set->nelems, 1, set->size + set->ndeact))
+			return -ENFILE;
+
 		err = nft_add_set_elem(&ctx, set, attr);
-		if (err < 0)
+		if (err < 0) {
+			atomic_dec(&set->nelems);
 			break;
-
-		set->nelems++;
+		}
 	}
 	return err;
 }
@@ -3477,7 +3478,7 @@ static int nf_tables_delsetelem(struct sock *nlsk, struct sk_buff *skb,
 		if (err < 0)
 			break;
 
-		set->nelems--;
+		set->ndeact++;
 	}
 	return err;
 }
@@ -3810,6 +3811,8 @@ static int nf_tables_commit(struct sk_buff *skb)
 						 &te->elem,
 						 NFT_MSG_DELSETELEM, 0);
 			te->set->ops->remove(te->set, &te->elem);
+			atomic_dec(&te->set->nelems);
+			te->set->ndeact--;
 			break;
 		}
 	}
@@ -3913,16 +3916,16 @@ static int nf_tables_abort(struct sk_buff *skb)
 			nft_trans_destroy(trans);
 			break;
 		case NFT_MSG_NEWSETELEM:
-			nft_trans_elem_set(trans)->nelems--;
 			te = (struct nft_trans_elem *)trans->data;
 
 			te->set->ops->remove(te->set, &te->elem);
+			atomic_dec(&te->set->nelems);
 			break;
 		case NFT_MSG_DELSETELEM:
 			te = (struct nft_trans_elem *)trans->data;
 
-			nft_trans_elem_set(trans)->nelems++;
 			te->set->ops->activate(te->set, &te->elem);
+			te->set->ndeact--;
 
 			nft_trans_destroy(trans);
 			break;
diff --git a/net/netfilter/nft_hash.c b/net/netfilter/nft_hash.c
index 5923ec5472689853103511530c114ca8631fa669..c74e2bf1a1e44dae46b1ceea9e816728ac34f68d 100644
--- a/net/netfilter/nft_hash.c
+++ b/net/netfilter/nft_hash.c
@@ -203,7 +203,7 @@ static void nft_hash_walk(const struct nft_ctx *ctx, const struct nft_set *set,
 
 static void nft_hash_gc(struct work_struct *work)
 {
-	const struct nft_set *set;
+	struct nft_set *set;
 	struct nft_hash_elem *he;
 	struct nft_hash *priv;
 	struct nft_set_gc_batch *gcb = NULL;
@@ -237,6 +237,7 @@ static void nft_hash_gc(struct work_struct *work)
 		if (gcb == NULL)
 			goto out;
 		rhashtable_remove_fast(&priv->ht, &he->node, nft_hash_params);
+		atomic_dec(&set->nelems);
 		nft_set_gc_batch_add(gcb, he);
 	}
 out: