提交 76507f69 编写于 作者: P Patrick McHardy 提交者: David S. Miller

[NETFILTER]: nf_conntrack: use RCU for conntrack hash

Signed-off-by: NPatrick McHardy <kaber@trash.net>
Signed-off-by: NDavid S. Miller <davem@davemloft.net>
上级 7d0742da
...@@ -129,6 +129,8 @@ struct nf_conn ...@@ -129,6 +129,8 @@ struct nf_conn
/* Extensions */ /* Extensions */
struct nf_ct_ext *ext; struct nf_ct_ext *ext;
struct rcu_head rcu;
}; };
static inline struct nf_conn * static inline struct nf_conn *
......
...@@ -39,12 +39,14 @@ struct ct_iter_state { ...@@ -39,12 +39,14 @@ struct ct_iter_state {
static struct hlist_node *ct_get_first(struct seq_file *seq) static struct hlist_node *ct_get_first(struct seq_file *seq)
{ {
struct ct_iter_state *st = seq->private; struct ct_iter_state *st = seq->private;
struct hlist_node *n;
for (st->bucket = 0; for (st->bucket = 0;
st->bucket < nf_conntrack_htable_size; st->bucket < nf_conntrack_htable_size;
st->bucket++) { st->bucket++) {
if (!hlist_empty(&nf_conntrack_hash[st->bucket])) n = rcu_dereference(nf_conntrack_hash[st->bucket].first);
return nf_conntrack_hash[st->bucket].first; if (n)
return n;
} }
return NULL; return NULL;
} }
...@@ -54,11 +56,11 @@ static struct hlist_node *ct_get_next(struct seq_file *seq, ...@@ -54,11 +56,11 @@ static struct hlist_node *ct_get_next(struct seq_file *seq,
{ {
struct ct_iter_state *st = seq->private; struct ct_iter_state *st = seq->private;
head = head->next; head = rcu_dereference(head->next);
while (head == NULL) { while (head == NULL) {
if (++st->bucket >= nf_conntrack_htable_size) if (++st->bucket >= nf_conntrack_htable_size)
return NULL; return NULL;
head = nf_conntrack_hash[st->bucket].first; head = rcu_dereference(nf_conntrack_hash[st->bucket].first);
} }
return head; return head;
} }
...@@ -74,8 +76,9 @@ static struct hlist_node *ct_get_idx(struct seq_file *seq, loff_t pos) ...@@ -74,8 +76,9 @@ static struct hlist_node *ct_get_idx(struct seq_file *seq, loff_t pos)
} }
static void *ct_seq_start(struct seq_file *seq, loff_t *pos) static void *ct_seq_start(struct seq_file *seq, loff_t *pos)
__acquires(RCU)
{ {
read_lock_bh(&nf_conntrack_lock); rcu_read_lock();
return ct_get_idx(seq, *pos); return ct_get_idx(seq, *pos);
} }
...@@ -86,8 +89,9 @@ static void *ct_seq_next(struct seq_file *s, void *v, loff_t *pos) ...@@ -86,8 +89,9 @@ static void *ct_seq_next(struct seq_file *s, void *v, loff_t *pos)
} }
static void ct_seq_stop(struct seq_file *s, void *v) static void ct_seq_stop(struct seq_file *s, void *v)
__releases(RCU)
{ {
read_unlock_bh(&nf_conntrack_lock); rcu_read_unlock();
} }
static int ct_seq_show(struct seq_file *s, void *v) static int ct_seq_show(struct seq_file *s, void *v)
...@@ -226,6 +230,7 @@ static struct hlist_node *ct_expect_get_idx(struct seq_file *seq, loff_t pos) ...@@ -226,6 +230,7 @@ static struct hlist_node *ct_expect_get_idx(struct seq_file *seq, loff_t pos)
} }
static void *exp_seq_start(struct seq_file *seq, loff_t *pos) static void *exp_seq_start(struct seq_file *seq, loff_t *pos)
__acquires(RCU)
{ {
rcu_read_lock(); rcu_read_lock();
return ct_expect_get_idx(seq, *pos); return ct_expect_get_idx(seq, *pos);
...@@ -238,6 +243,7 @@ static void *exp_seq_next(struct seq_file *seq, void *v, loff_t *pos) ...@@ -238,6 +243,7 @@ static void *exp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
} }
static void exp_seq_stop(struct seq_file *seq, void *v) static void exp_seq_stop(struct seq_file *seq, void *v)
__releases(RCU)
{ {
rcu_read_unlock(); rcu_read_unlock();
} }
......
...@@ -166,8 +166,8 @@ static void ...@@ -166,8 +166,8 @@ static void
clean_from_lists(struct nf_conn *ct) clean_from_lists(struct nf_conn *ct)
{ {
pr_debug("clean_from_lists(%p)\n", ct); pr_debug("clean_from_lists(%p)\n", ct);
hlist_del(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnode); hlist_del_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnode);
hlist_del(&ct->tuplehash[IP_CT_DIR_REPLY].hnode); hlist_del_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnode);
/* Destroy all pending expectations */ /* Destroy all pending expectations */
nf_ct_remove_expectations(ct); nf_ct_remove_expectations(ct);
...@@ -253,7 +253,7 @@ __nf_conntrack_find(const struct nf_conntrack_tuple *tuple, ...@@ -253,7 +253,7 @@ __nf_conntrack_find(const struct nf_conntrack_tuple *tuple,
struct hlist_node *n; struct hlist_node *n;
unsigned int hash = hash_conntrack(tuple); unsigned int hash = hash_conntrack(tuple);
hlist_for_each_entry(h, n, &nf_conntrack_hash[hash], hnode) { hlist_for_each_entry_rcu(h, n, &nf_conntrack_hash[hash], hnode) {
if (nf_ct_tuplehash_to_ctrack(h) != ignored_conntrack && if (nf_ct_tuplehash_to_ctrack(h) != ignored_conntrack &&
nf_ct_tuple_equal(tuple, &h->tuple)) { nf_ct_tuple_equal(tuple, &h->tuple)) {
NF_CT_STAT_INC(found); NF_CT_STAT_INC(found);
...@@ -271,12 +271,16 @@ struct nf_conntrack_tuple_hash * ...@@ -271,12 +271,16 @@ struct nf_conntrack_tuple_hash *
nf_conntrack_find_get(const struct nf_conntrack_tuple *tuple) nf_conntrack_find_get(const struct nf_conntrack_tuple *tuple)
{ {
struct nf_conntrack_tuple_hash *h; struct nf_conntrack_tuple_hash *h;
struct nf_conn *ct;
read_lock_bh(&nf_conntrack_lock); rcu_read_lock();
h = __nf_conntrack_find(tuple, NULL); h = __nf_conntrack_find(tuple, NULL);
if (h) if (h) {
atomic_inc(&nf_ct_tuplehash_to_ctrack(h)->ct_general.use); ct = nf_ct_tuplehash_to_ctrack(h);
read_unlock_bh(&nf_conntrack_lock); if (unlikely(!atomic_inc_not_zero(&ct->ct_general.use)))
h = NULL;
}
rcu_read_unlock();
return h; return h;
} }
...@@ -286,10 +290,10 @@ static void __nf_conntrack_hash_insert(struct nf_conn *ct, ...@@ -286,10 +290,10 @@ static void __nf_conntrack_hash_insert(struct nf_conn *ct,
unsigned int hash, unsigned int hash,
unsigned int repl_hash) unsigned int repl_hash)
{ {
hlist_add_head(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnode, hlist_add_head_rcu(&ct->tuplehash[IP_CT_DIR_ORIGINAL].hnode,
&nf_conntrack_hash[hash]); &nf_conntrack_hash[hash]);
hlist_add_head(&ct->tuplehash[IP_CT_DIR_REPLY].hnode, hlist_add_head_rcu(&ct->tuplehash[IP_CT_DIR_REPLY].hnode,
&nf_conntrack_hash[repl_hash]); &nf_conntrack_hash[repl_hash]);
} }
void nf_conntrack_hash_insert(struct nf_conn *ct) void nf_conntrack_hash_insert(struct nf_conn *ct)
...@@ -392,9 +396,9 @@ nf_conntrack_tuple_taken(const struct nf_conntrack_tuple *tuple, ...@@ -392,9 +396,9 @@ nf_conntrack_tuple_taken(const struct nf_conntrack_tuple *tuple,
{ {
struct nf_conntrack_tuple_hash *h; struct nf_conntrack_tuple_hash *h;
read_lock_bh(&nf_conntrack_lock); rcu_read_lock();
h = __nf_conntrack_find(tuple, ignored_conntrack); h = __nf_conntrack_find(tuple, ignored_conntrack);
read_unlock_bh(&nf_conntrack_lock); rcu_read_unlock();
return h != NULL; return h != NULL;
} }
...@@ -413,21 +417,23 @@ static int early_drop(unsigned int hash) ...@@ -413,21 +417,23 @@ static int early_drop(unsigned int hash)
unsigned int i, cnt = 0; unsigned int i, cnt = 0;
int dropped = 0; int dropped = 0;
read_lock_bh(&nf_conntrack_lock); rcu_read_lock();
for (i = 0; i < nf_conntrack_htable_size; i++) { for (i = 0; i < nf_conntrack_htable_size; i++) {
hlist_for_each_entry(h, n, &nf_conntrack_hash[hash], hnode) { hlist_for_each_entry_rcu(h, n, &nf_conntrack_hash[hash],
hnode) {
tmp = nf_ct_tuplehash_to_ctrack(h); tmp = nf_ct_tuplehash_to_ctrack(h);
if (!test_bit(IPS_ASSURED_BIT, &tmp->status)) if (!test_bit(IPS_ASSURED_BIT, &tmp->status))
ct = tmp; ct = tmp;
cnt++; cnt++;
} }
if (ct && unlikely(!atomic_inc_not_zero(&ct->ct_general.use)))
ct = NULL;
if (ct || cnt >= NF_CT_EVICTION_RANGE) if (ct || cnt >= NF_CT_EVICTION_RANGE)
break; break;
hash = (hash + 1) % nf_conntrack_htable_size; hash = (hash + 1) % nf_conntrack_htable_size;
} }
if (ct) rcu_read_unlock();
atomic_inc(&ct->ct_general.use);
read_unlock_bh(&nf_conntrack_lock);
if (!ct) if (!ct)
return dropped; return dropped;
...@@ -480,17 +486,25 @@ struct nf_conn *nf_conntrack_alloc(const struct nf_conntrack_tuple *orig, ...@@ -480,17 +486,25 @@ struct nf_conn *nf_conntrack_alloc(const struct nf_conntrack_tuple *orig,
/* Don't set timer yet: wait for confirmation */ /* Don't set timer yet: wait for confirmation */
setup_timer(&conntrack->timeout, death_by_timeout, setup_timer(&conntrack->timeout, death_by_timeout,
(unsigned long)conntrack); (unsigned long)conntrack);
INIT_RCU_HEAD(&conntrack->rcu);
return conntrack; return conntrack;
} }
EXPORT_SYMBOL_GPL(nf_conntrack_alloc); EXPORT_SYMBOL_GPL(nf_conntrack_alloc);
void nf_conntrack_free(struct nf_conn *conntrack) static void nf_conntrack_free_rcu(struct rcu_head *head)
{ {
nf_ct_ext_free(conntrack); struct nf_conn *ct = container_of(head, struct nf_conn, rcu);
kmem_cache_free(nf_conntrack_cachep, conntrack);
nf_ct_ext_free(ct);
kmem_cache_free(nf_conntrack_cachep, ct);
atomic_dec(&nf_conntrack_count); atomic_dec(&nf_conntrack_count);
} }
void nf_conntrack_free(struct nf_conn *conntrack)
{
call_rcu(&conntrack->rcu, nf_conntrack_free_rcu);
}
EXPORT_SYMBOL_GPL(nf_conntrack_free); EXPORT_SYMBOL_GPL(nf_conntrack_free);
/* Allocate a new conntrack: we return -ENOMEM if classification /* Allocate a new conntrack: we return -ENOMEM if classification
...@@ -1036,12 +1050,17 @@ int nf_conntrack_set_hashsize(const char *val, struct kernel_param *kp) ...@@ -1036,12 +1050,17 @@ int nf_conntrack_set_hashsize(const char *val, struct kernel_param *kp)
* use a newrandom seed */ * use a newrandom seed */
get_random_bytes(&rnd, 4); get_random_bytes(&rnd, 4);
/* Lookups in the old hash might happen in parallel, which means we
* might get false negatives during connection lookup. New connections
* created because of a false negative won't make it into the hash
* though since that required taking the lock.
*/
write_lock_bh(&nf_conntrack_lock); write_lock_bh(&nf_conntrack_lock);
for (i = 0; i < nf_conntrack_htable_size; i++) { for (i = 0; i < nf_conntrack_htable_size; i++) {
while (!hlist_empty(&nf_conntrack_hash[i])) { while (!hlist_empty(&nf_conntrack_hash[i])) {
h = hlist_entry(nf_conntrack_hash[i].first, h = hlist_entry(nf_conntrack_hash[i].first,
struct nf_conntrack_tuple_hash, hnode); struct nf_conntrack_tuple_hash, hnode);
hlist_del(&h->hnode); hlist_del_rcu(&h->hnode);
bucket = __hash_conntrack(&h->tuple, hashsize, rnd); bucket = __hash_conntrack(&h->tuple, hashsize, rnd);
hlist_add_head(&h->hnode, &hash[bucket]); hlist_add_head(&h->hnode, &hash[bucket]);
} }
......
...@@ -545,12 +545,12 @@ ctnetlink_dump_table(struct sk_buff *skb, struct netlink_callback *cb) ...@@ -545,12 +545,12 @@ ctnetlink_dump_table(struct sk_buff *skb, struct netlink_callback *cb)
struct nfgenmsg *nfmsg = NLMSG_DATA(cb->nlh); struct nfgenmsg *nfmsg = NLMSG_DATA(cb->nlh);
u_int8_t l3proto = nfmsg->nfgen_family; u_int8_t l3proto = nfmsg->nfgen_family;
read_lock_bh(&nf_conntrack_lock); rcu_read_lock();
last = (struct nf_conn *)cb->args[1]; last = (struct nf_conn *)cb->args[1];
for (; cb->args[0] < nf_conntrack_htable_size; cb->args[0]++) { for (; cb->args[0] < nf_conntrack_htable_size; cb->args[0]++) {
restart: restart:
hlist_for_each_entry(h, n, &nf_conntrack_hash[cb->args[0]], hlist_for_each_entry_rcu(h, n, &nf_conntrack_hash[cb->args[0]],
hnode) { hnode) {
if (NF_CT_DIRECTION(h) != IP_CT_DIR_ORIGINAL) if (NF_CT_DIRECTION(h) != IP_CT_DIR_ORIGINAL)
continue; continue;
ct = nf_ct_tuplehash_to_ctrack(h); ct = nf_ct_tuplehash_to_ctrack(h);
...@@ -568,7 +568,8 @@ ctnetlink_dump_table(struct sk_buff *skb, struct netlink_callback *cb) ...@@ -568,7 +568,8 @@ ctnetlink_dump_table(struct sk_buff *skb, struct netlink_callback *cb)
cb->nlh->nlmsg_seq, cb->nlh->nlmsg_seq,
IPCTNL_MSG_CT_NEW, IPCTNL_MSG_CT_NEW,
1, ct) < 0) { 1, ct) < 0) {
nf_conntrack_get(&ct->ct_general); if (!atomic_inc_not_zero(&ct->ct_general.use))
continue;
cb->args[1] = (unsigned long)ct; cb->args[1] = (unsigned long)ct;
goto out; goto out;
} }
...@@ -584,7 +585,7 @@ ctnetlink_dump_table(struct sk_buff *skb, struct netlink_callback *cb) ...@@ -584,7 +585,7 @@ ctnetlink_dump_table(struct sk_buff *skb, struct netlink_callback *cb)
} }
} }
out: out:
read_unlock_bh(&nf_conntrack_lock); rcu_read_unlock();
if (last) if (last)
nf_ct_put(last); nf_ct_put(last);
......
...@@ -58,12 +58,14 @@ struct ct_iter_state { ...@@ -58,12 +58,14 @@ struct ct_iter_state {
static struct hlist_node *ct_get_first(struct seq_file *seq) static struct hlist_node *ct_get_first(struct seq_file *seq)
{ {
struct ct_iter_state *st = seq->private; struct ct_iter_state *st = seq->private;
struct hlist_node *n;
for (st->bucket = 0; for (st->bucket = 0;
st->bucket < nf_conntrack_htable_size; st->bucket < nf_conntrack_htable_size;
st->bucket++) { st->bucket++) {
if (!hlist_empty(&nf_conntrack_hash[st->bucket])) n = rcu_dereference(nf_conntrack_hash[st->bucket].first);
return nf_conntrack_hash[st->bucket].first; if (n)
return n;
} }
return NULL; return NULL;
} }
...@@ -73,11 +75,11 @@ static struct hlist_node *ct_get_next(struct seq_file *seq, ...@@ -73,11 +75,11 @@ static struct hlist_node *ct_get_next(struct seq_file *seq,
{ {
struct ct_iter_state *st = seq->private; struct ct_iter_state *st = seq->private;
head = head->next; head = rcu_dereference(head->next);
while (head == NULL) { while (head == NULL) {
if (++st->bucket >= nf_conntrack_htable_size) if (++st->bucket >= nf_conntrack_htable_size)
return NULL; return NULL;
head = nf_conntrack_hash[st->bucket].first; head = rcu_dereference(nf_conntrack_hash[st->bucket].first);
} }
return head; return head;
} }
...@@ -93,9 +95,9 @@ static struct hlist_node *ct_get_idx(struct seq_file *seq, loff_t pos) ...@@ -93,9 +95,9 @@ static struct hlist_node *ct_get_idx(struct seq_file *seq, loff_t pos)
} }
static void *ct_seq_start(struct seq_file *seq, loff_t *pos) static void *ct_seq_start(struct seq_file *seq, loff_t *pos)
__acquires(nf_conntrack_lock) __acquires(RCU)
{ {
read_lock_bh(&nf_conntrack_lock); rcu_read_lock();
return ct_get_idx(seq, *pos); return ct_get_idx(seq, *pos);
} }
...@@ -106,9 +108,9 @@ static void *ct_seq_next(struct seq_file *s, void *v, loff_t *pos) ...@@ -106,9 +108,9 @@ static void *ct_seq_next(struct seq_file *s, void *v, loff_t *pos)
} }
static void ct_seq_stop(struct seq_file *s, void *v) static void ct_seq_stop(struct seq_file *s, void *v)
__releases(nf_conntrack_lock) __releases(RCU)
{ {
read_unlock_bh(&nf_conntrack_lock); rcu_read_unlock();
} }
/* return 0 on success, 1 in case of error */ /* return 0 on success, 1 in case of error */
......
...@@ -120,7 +120,7 @@ static int count_them(struct xt_connlimit_data *data, ...@@ -120,7 +120,7 @@ static int count_them(struct xt_connlimit_data *data,
else else
hash = &data->iphash[connlimit_iphash(addr->ip & mask->ip)]; hash = &data->iphash[connlimit_iphash(addr->ip & mask->ip)];
read_lock_bh(&nf_conntrack_lock); rcu_read_lock();
/* check the saved connections */ /* check the saved connections */
list_for_each_entry_safe(conn, tmp, hash, list) { list_for_each_entry_safe(conn, tmp, hash, list) {
...@@ -163,7 +163,7 @@ static int count_them(struct xt_connlimit_data *data, ...@@ -163,7 +163,7 @@ static int count_them(struct xt_connlimit_data *data,
++matches; ++matches;
} }
read_unlock_bh(&nf_conntrack_lock); rcu_read_unlock();
if (addit) { if (addit) {
/* save the new connection in our list */ /* save the new connection in our list */
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册