datapath.c 55.9 KB
Newer Older
1
/*
2
 * Copyright (c) 2007-2014 Nicira, Inc.
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of version 2 of the GNU General Public
 * License as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
 * 02110-1301, USA
 */

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <linux/init.h>
#include <linux/module.h>
#include <linux/if_arp.h>
#include <linux/if_vlan.h>
#include <linux/in.h>
#include <linux/ip.h>
#include <linux/jhash.h>
#include <linux/delay.h>
#include <linux/time.h>
#include <linux/etherdevice.h>
#include <linux/genetlink.h>
#include <linux/kernel.h>
#include <linux/kthread.h>
#include <linux/mutex.h>
#include <linux/percpu.h>
#include <linux/rcupdate.h>
#include <linux/tcp.h>
#include <linux/udp.h>
#include <linux/ethtool.h>
#include <linux/wait.h>
#include <asm/div64.h>
#include <linux/highmem.h>
#include <linux/netfilter_bridge.h>
#include <linux/netfilter_ipv4.h>
#include <linux/inetdevice.h>
#include <linux/list.h>
#include <linux/openvswitch.h>
#include <linux/rculist.h>
#include <linux/dmi.h>
#include <net/genetlink.h>
51 52
#include <net/net_namespace.h>
#include <net/netns/generic.h>
53 54 55

#include "datapath.h"
#include "flow.h"
56
#include "flow_table.h"
57
#include "flow_netlink.h"
58
#include "vport-internal_dev.h"
59
#include "vport-netdev.h"
60

61
int ovs_net_id __read_mostly;
62
EXPORT_SYMBOL_GPL(ovs_net_id);
63

64 65 66 67
static struct genl_family dp_packet_genl_family;
static struct genl_family dp_flow_genl_family;
static struct genl_family dp_datapath_genl_family;

68 69
static const struct nla_policy flow_policy[];

70 71
static const struct genl_multicast_group ovs_dp_flow_multicast_group = {
	.name = OVS_FLOW_MCGROUP,
72 73
};

74 75
static const struct genl_multicast_group ovs_dp_datapath_multicast_group = {
	.name = OVS_DATAPATH_MCGROUP,
76 77
};

78 79
static const struct genl_multicast_group ovs_dp_vport_multicast_group = {
	.name = OVS_VPORT_MCGROUP,
80 81
};

82 83
/* Check if need to build a reply message.
 * OVS userspace sets the NLM_F_ECHO flag if it needs the reply. */
84 85
static bool ovs_must_notify(struct genl_family *family, struct genl_info *info,
			    unsigned int group)
86 87
{
	return info->nlhdr->nlmsg_flags & NLM_F_ECHO ||
88
	       genl_has_listeners(family, genl_info_net(info), group);
89 90
}

91
static void ovs_notify(struct genl_family *family,
92
		       struct sk_buff *skb, struct genl_info *info)
93
{
94
	genl_notify(family, skb, genl_info_net(info), info->snd_portid,
95
		    0, info->nlhdr, GFP_KERNEL);
96 97
}

98 99 100
/**
 * DOC: Locking:
 *
101 102 103 104
 * All writes e.g. Writes to device state (add/remove datapath, port, set
 * operations on vports, etc.), Writes to other state (flow table
 * modifications, set miscellaneous datapath parameters, etc.) are protected
 * by ovs_lock.
105 106 107 108 109 110
 *
 * Reads are protected by RCU.
 *
 * There are a few special cases (mostly stats) that have their own
 * synchronization but they nest under all of above and don't interact with
 * each other.
111 112
 *
 * The RTNL lock nests inside ovs_mutex.
113 114
 */

115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134
static DEFINE_MUTEX(ovs_mutex);

void ovs_lock(void)
{
	mutex_lock(&ovs_mutex);
}

void ovs_unlock(void)
{
	mutex_unlock(&ovs_mutex);
}

#ifdef CONFIG_LOCKDEP
int lockdep_ovsl_is_held(void)
{
	if (debug_locks)
		return lockdep_is_held(&ovs_mutex);
	else
		return 1;
}
135
EXPORT_SYMBOL_GPL(lockdep_ovsl_is_held);
136 137
#endif

138
static struct vport *new_vport(const struct vport_parms *);
139
static int queue_gso_packets(struct datapath *dp, struct sk_buff *,
140
			     const struct sw_flow_key *,
141
			     const struct dp_upcall_info *);
142
static int queue_userspace_packet(struct datapath *dp, struct sk_buff *,
143
				  const struct sw_flow_key *,
144 145
				  const struct dp_upcall_info *);

146 147
/* Must be called with rcu_read_lock. */
static struct datapath *get_dp_rcu(struct net *net, int dp_ifindex)
148
{
149
	struct net_device *dev = dev_get_by_index_rcu(net, dp_ifindex);
150 151 152 153

	if (dev) {
		struct vport *vport = ovs_internal_dev_get_vport(dev);
		if (vport)
154
			return vport->dp;
155
	}
156 157 158 159 160 161 162 163 164 165 166 167 168 169

	return NULL;
}

/* The caller must hold either ovs_mutex or rcu_read_lock to keep the
 * returned dp pointer valid.
 */
static inline struct datapath *get_dp(struct net *net, int dp_ifindex)
{
	struct datapath *dp;

	WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_ovsl_is_held());
	rcu_read_lock();
	dp = get_dp_rcu(net, dp_ifindex);
170 171 172 173 174
	rcu_read_unlock();

	return dp;
}

175
/* Must be called with rcu_read_lock or ovs_mutex. */
176
const char *ovs_dp_name(const struct datapath *dp)
177
{
178
	struct vport *vport = ovs_vport_ovsl_rcu(dp, OVSP_LOCAL);
179 180 181
	return vport->ops->get_name(vport);
}

182
static int get_dpifindex(const struct datapath *dp)
183 184 185 186 187 188
{
	struct vport *local;
	int ifindex;

	rcu_read_lock();

189
	local = ovs_vport_rcu(dp, OVSP_LOCAL);
190
	if (local)
191
		ifindex = netdev_vport_priv(local)->dev->ifindex;
192 193 194 195 196 197 198 199 200 201 202 203
	else
		ifindex = 0;

	rcu_read_unlock();

	return ifindex;
}

static void destroy_dp_rcu(struct rcu_head *rcu)
{
	struct datapath *dp = container_of(rcu, struct datapath, rcu);

204
	ovs_flow_tbl_destroy(&dp->table);
205
	free_percpu(dp->stats_percpu);
206
	kfree(dp->ports);
207 208 209
	kfree(dp);
}

210 211 212 213 214 215
static struct hlist_head *vport_hash_bucket(const struct datapath *dp,
					    u16 port_no)
{
	return &dp->ports[port_no & (DP_VPORT_HASH_BUCKETS - 1)];
}

J
Jarno Rajahalme 已提交
216
/* Called with ovs_mutex or RCU read lock. */
217 218 219 220 221 222
struct vport *ovs_lookup_vport(const struct datapath *dp, u16 port_no)
{
	struct vport *vport;
	struct hlist_head *head;

	head = vport_hash_bucket(dp, port_no);
223
	hlist_for_each_entry_rcu(vport, head, dp_hash_node) {
224 225 226 227 228 229
		if (vport->port_no == port_no)
			return vport;
	}
	return NULL;
}

230
/* Called with ovs_mutex. */
231 232 233 234 235 236 237
static struct vport *new_vport(const struct vport_parms *parms)
{
	struct vport *vport;

	vport = ovs_vport_add(parms);
	if (!IS_ERR(vport)) {
		struct datapath *dp = parms->dp;
238
		struct hlist_head *head = vport_hash_bucket(dp, vport->port_no);
239

240
		hlist_add_head_rcu(&vport->dp_hash_node, head);
241 242 243 244 245 246
	}
	return vport;
}

void ovs_dp_detach_port(struct vport *p)
{
247
	ASSERT_OVSL();
248 249

	/* First drop references to device. */
250
	hlist_del_rcu(&p->dp_hash_node);
251 252 253 254 255 256

	/* Then destroy it. */
	ovs_vport_del(p);
}

/* Must be called with rcu_read_lock. */
257
void ovs_dp_process_packet(struct sk_buff *skb, struct sw_flow_key *key)
258
{
259
	const struct vport *p = OVS_CB(skb)->input_vport;
260 261
	struct datapath *dp = p->dp;
	struct sw_flow *flow;
262
	struct sw_flow_actions *sf_acts;
263 264
	struct dp_stats_percpu *stats;
	u64 *stats_counter;
265
	u32 n_mask_hit;
266

267
	stats = this_cpu_ptr(dp->stats_percpu);
268 269

	/* Look up flow. */
270
	flow = ovs_flow_tbl_lookup_stats(&dp->table, key, &n_mask_hit);
271 272
	if (unlikely(!flow)) {
		struct dp_upcall_info upcall;
273
		int error;
274 275 276

		upcall.cmd = OVS_PACKET_CMD_MISS;
		upcall.userdata = NULL;
277
		upcall.portid = ovs_vport_find_upcall_portid(p, skb);
278
		upcall.egress_tun_info = NULL;
279
		error = ovs_dp_upcall(dp, skb, key, &upcall);
280 281 282 283
		if (unlikely(error))
			kfree_skb(skb);
		else
			consume_skb(skb);
284 285 286 287
		stats_counter = &stats->n_missed;
		goto out;
	}

288 289 290
	ovs_flow_stats_update(flow, key->tp.flags, skb);
	sf_acts = rcu_dereference(flow->sf_acts);
	ovs_execute_actions(dp, skb, sf_acts, key);
291

292
	stats_counter = &stats->n_hit;
293 294 295

out:
	/* Update datapath statistics. */
296
	u64_stats_update_begin(&stats->syncp);
297
	(*stats_counter)++;
298
	stats->n_mask_hit += n_mask_hit;
299
	u64_stats_update_end(&stats->syncp);
300 301 302
}

int ovs_dp_upcall(struct datapath *dp, struct sk_buff *skb,
303
		  const struct sw_flow_key *key,
304
		  const struct dp_upcall_info *upcall_info)
305 306 307 308
{
	struct dp_stats_percpu *stats;
	int err;

309
	if (upcall_info->portid == 0) {
310 311 312 313 314
		err = -ENOTCONN;
		goto err;
	}

	if (!skb_is_gso(skb))
315
		err = queue_userspace_packet(dp, skb, key, upcall_info);
316
	else
317
		err = queue_gso_packets(dp, skb, key, upcall_info);
318 319 320 321 322 323
	if (err)
		goto err;

	return 0;

err:
324
	stats = this_cpu_ptr(dp->stats_percpu);
325

326
	u64_stats_update_begin(&stats->syncp);
327
	stats->n_lost++;
328
	u64_stats_update_end(&stats->syncp);
329 330 331 332

	return err;
}

333
static int queue_gso_packets(struct datapath *dp, struct sk_buff *skb,
334
			     const struct sw_flow_key *key,
335 336
			     const struct dp_upcall_info *upcall_info)
{
337
	unsigned short gso_type = skb_shinfo(skb)->gso_type;
338 339
	struct sw_flow_key later_key;
	struct sk_buff *segs, *nskb;
340
	struct ovs_skb_cb ovs_cb;
341 342
	int err;

343
	ovs_cb = *OVS_CB(skb);
344
	segs = __skb_gso_segment(skb, NETIF_F_SG, false);
345
	*OVS_CB(skb) = ovs_cb;
346 347
	if (IS_ERR(segs))
		return PTR_ERR(segs);
348 349
	if (segs == NULL)
		return -EINVAL;
350

351 352 353 354 355 356 357 358 359
	if (gso_type & SKB_GSO_UDP) {
		/* The initial flow key extracted by ovs_flow_key_extract()
		 * in this case is for a first fragment, so we need to
		 * properly mark later fragments.
		 */
		later_key = *key;
		later_key.ip.frag = OVS_FRAG_TYPE_LATER;
	}

360 361 362
	/* Queue all of the segments. */
	skb = segs;
	do {
363 364 365 366 367
		*OVS_CB(skb) = ovs_cb;
		if (gso_type & SKB_GSO_UDP && skb != segs)
			key = &later_key;

		err = queue_userspace_packet(dp, skb, key, upcall_info);
368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384
		if (err)
			break;

	} while ((skb = skb->next));

	/* Free all of the segments. */
	skb = segs;
	do {
		nskb = skb->next;
		if (err)
			kfree_skb(skb);
		else
			consume_skb(skb);
	} while ((skb = nskb));
	return err;
}

385
static size_t upcall_msg_size(const struct dp_upcall_info *upcall_info,
386
			      unsigned int hdrlen)
387 388
{
	size_t size = NLMSG_ALIGN(sizeof(struct ovs_header))
389
		+ nla_total_size(hdrlen) /* OVS_PACKET_ATTR_PACKET */
390
		+ nla_total_size(ovs_key_attr_size()); /* OVS_PACKET_ATTR_KEY */
391 392

	/* OVS_PACKET_ATTR_USERDATA */
393 394 395 396 397 398
	if (upcall_info->userdata)
		size += NLA_ALIGN(upcall_info->userdata->nla_len);

	/* OVS_PACKET_ATTR_EGRESS_TUN_KEY */
	if (upcall_info->egress_tun_info)
		size += nla_total_size(ovs_tun_key_attr_size());
399 400 401 402

	return size;
}

403
static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb,
404
				  const struct sw_flow_key *key,
405 406 407 408
				  const struct dp_upcall_info *upcall_info)
{
	struct ovs_header *upcall;
	struct sk_buff *nskb = NULL;
L
Li RongQing 已提交
409
	struct sk_buff *user_skb = NULL; /* to be queued to userspace */
410
	struct nlattr *nla;
411
	struct genl_info info = {
412
		.dst_sk = ovs_dp_get_net(dp)->genl_sock,
413 414 415
		.snd_portid = upcall_info->portid,
	};
	size_t len;
416
	unsigned int hlen;
417 418 419 420 421
	int err, dp_ifindex;

	dp_ifindex = get_dpifindex(dp);
	if (!dp_ifindex)
		return -ENODEV;
422

423
	if (skb_vlan_tag_present(skb)) {
424 425 426 427
		nskb = skb_clone(skb, GFP_ATOMIC);
		if (!nskb)
			return -ENOMEM;

428
		nskb = __vlan_hwaccel_push_inside(nskb);
429
		if (!nskb)
430 431 432 433 434 435 436 437 438 439
			return -ENOMEM;

		skb = nskb;
	}

	if (nla_attr_size(skb->len) > USHRT_MAX) {
		err = -EFBIG;
		goto out;
	}

440 441 442 443 444 445 446 447 448 449 450 451 452 453
	/* Complete checksum if needed */
	if (skb->ip_summed == CHECKSUM_PARTIAL &&
	    (err = skb_checksum_help(skb)))
		goto out;

	/* Older versions of OVS user space enforce alignment of the last
	 * Netlink attribute to NLA_ALIGNTO which would require extensive
	 * padding logic. Only perform zerocopy if padding is not required.
	 */
	if (dp->user_features & OVS_DP_F_UNALIGNED)
		hlen = skb_zerocopy_headlen(skb);
	else
		hlen = skb->len;

454
	len = upcall_msg_size(upcall_info, hlen);
455
	user_skb = genlmsg_new_unicast(len, &info, GFP_ATOMIC);
456 457 458 459 460 461 462 463 464
	if (!user_skb) {
		err = -ENOMEM;
		goto out;
	}

	upcall = genlmsg_put(user_skb, 0, 0, &dp_packet_genl_family,
			     0, upcall_info->cmd);
	upcall->dp_ifindex = dp_ifindex;

465
	err = ovs_nla_put_key(key, key, OVS_PACKET_ATTR_KEY, false, user_skb);
466
	BUG_ON(err);
467 468

	if (upcall_info->userdata)
469 470 471
		__nla_put(user_skb, OVS_PACKET_ATTR_USERDATA,
			  nla_len(upcall_info->userdata),
			  nla_data(upcall_info->userdata));
472

473 474 475 476 477 478 479 480
	if (upcall_info->egress_tun_info) {
		nla = nla_nest_start(user_skb, OVS_PACKET_ATTR_EGRESS_TUN_KEY);
		err = ovs_nla_put_egress_tunnel_key(user_skb,
						    upcall_info->egress_tun_info);
		BUG_ON(err);
		nla_nest_end(user_skb, nla);
	}

481 482 483 484 485 486 487
	/* Only reserve room for attribute header, packet data is added
	 * in skb_zerocopy() */
	if (!(nla = nla_reserve(user_skb, OVS_PACKET_ATTR_PACKET, 0))) {
		err = -ENOBUFS;
		goto out;
	}
	nla->nla_len = nla_attr_size(skb->len);
488

489 490 491
	err = skb_zerocopy(user_skb, skb, skb->len, hlen);
	if (err)
		goto out;
492

493 494 495 496 497 498 499 500
	/* Pad OVS_PACKET_ATTR_PACKET if linear copy was performed */
	if (!(dp->user_features & OVS_DP_F_UNALIGNED)) {
		size_t plen = NLA_ALIGN(user_skb->len) - user_skb->len;

		if (plen > 0)
			memset(skb_put(user_skb, plen), 0, plen);
	}

501
	((struct nlmsghdr *) user_skb->data)->nlmsg_len = user_skb->len;
502

503
	err = genlmsg_unicast(ovs_dp_get_net(dp), user_skb, upcall_info->portid);
L
Li RongQing 已提交
504
	user_skb = NULL;
505
out:
506 507
	if (err)
		skb_tx_error(skb);
L
Li RongQing 已提交
508
	kfree_skb(user_skb);
509 510 511 512 513 514 515 516 517 518 519
	kfree_skb(nskb);
	return err;
}

static int ovs_packet_cmd_execute(struct sk_buff *skb, struct genl_info *info)
{
	struct ovs_header *ovs_header = info->userhdr;
	struct nlattr **a = info->attrs;
	struct sw_flow_actions *acts;
	struct sk_buff *packet;
	struct sw_flow *flow;
520
	struct sw_flow_actions *sf_acts;
521 522
	struct datapath *dp;
	struct ethhdr *eth;
523
	struct vport *input_vport;
524 525
	int len;
	int err;
526
	bool log = !a[OVS_PACKET_ATTR_PROBE];
527 528 529

	err = -EINVAL;
	if (!a[OVS_PACKET_ATTR_PACKET] || !a[OVS_PACKET_ATTR_KEY] ||
530
	    !a[OVS_PACKET_ATTR_ACTIONS])
531 532 533 534 535 536 537 538 539
		goto err;

	len = nla_len(a[OVS_PACKET_ATTR_PACKET]);
	packet = __dev_alloc_skb(NET_IP_ALIGN + len, GFP_KERNEL);
	err = -ENOMEM;
	if (!packet)
		goto err;
	skb_reserve(packet, NET_IP_ALIGN);

540
	nla_memcpy(__skb_put(packet, len), a[OVS_PACKET_ATTR_PACKET], len);
541 542 543 544 545 546 547

	skb_reset_mac_header(packet);
	eth = eth_hdr(packet);

	/* Normally, setting the skb 'protocol' field would be handled by a
	 * call to eth_type_trans(), but it assumes there's a sending
	 * device, which we may not have. */
S
Simon Horman 已提交
548
	if (ntohs(eth->h_proto) >= ETH_P_802_3_MIN)
549 550 551 552 553
		packet->protocol = eth->h_proto;
	else
		packet->protocol = htons(ETH_P_802_2);

	/* Build an sw_flow for sending this packet. */
554
	flow = ovs_flow_alloc();
555 556 557 558
	err = PTR_ERR(flow);
	if (IS_ERR(flow))
		goto err_kfree_skb;

559
	err = ovs_flow_key_extract_userspace(a[OVS_PACKET_ATTR_KEY], packet,
560
					     &flow->key, log);
561 562 563
	if (err)
		goto err_flow_free;

564
	err = ovs_nla_copy_actions(a[OVS_PACKET_ATTR_ACTIONS],
565
				   &flow->key, &acts, log);
566 567
	if (err)
		goto err_flow_free;
568

569 570
	rcu_assign_pointer(flow->sf_acts, acts);
	OVS_CB(packet)->egress_tun_info = NULL;
571
	packet->priority = flow->key.phy.priority;
572
	packet->mark = flow->key.phy.skb_mark;
573 574

	rcu_read_lock();
575
	dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex);
576 577 578 579
	err = -ENODEV;
	if (!dp)
		goto err_unlock;

580 581 582 583 584 585 586 587
	input_vport = ovs_vport_rcu(dp, flow->key.phy.in_port);
	if (!input_vport)
		input_vport = ovs_vport_rcu(dp, OVSP_LOCAL);

	if (!input_vport)
		goto err_unlock;

	OVS_CB(packet)->input_vport = input_vport;
588
	sf_acts = rcu_dereference(flow->sf_acts);
589

590
	local_bh_disable();
591
	err = ovs_execute_actions(dp, packet, sf_acts, &flow->key);
592 593 594
	local_bh_enable();
	rcu_read_unlock();

595
	ovs_flow_free(flow, false);
596 597 598 599 600
	return err;

err_unlock:
	rcu_read_unlock();
err_flow_free:
601
	ovs_flow_free(flow, false);
602 603 604 605 606 607 608
err_kfree_skb:
	kfree_skb(packet);
err:
	return err;
}

static const struct nla_policy packet_policy[OVS_PACKET_ATTR_MAX + 1] = {
609
	[OVS_PACKET_ATTR_PACKET] = { .len = ETH_HLEN },
610 611
	[OVS_PACKET_ATTR_KEY] = { .type = NLA_NESTED },
	[OVS_PACKET_ATTR_ACTIONS] = { .type = NLA_NESTED },
612
	[OVS_PACKET_ATTR_PROBE] = { .type = NLA_FLAG },
613 614
};

615
static const struct genl_ops dp_packet_genl_ops[] = {
616 617 618 619 620 621 622
	{ .cmd = OVS_PACKET_CMD_EXECUTE,
	  .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
	  .policy = packet_policy,
	  .doit = ovs_packet_cmd_execute
	}
};

623 624 625 626 627 628 629 630 631 632 633 634
static struct genl_family dp_packet_genl_family = {
	.id = GENL_ID_GENERATE,
	.hdrsize = sizeof(struct ovs_header),
	.name = OVS_PACKET_FAMILY,
	.version = OVS_PACKET_VERSION,
	.maxattr = OVS_PACKET_ATTR_MAX,
	.netnsok = true,
	.parallel_ops = true,
	.ops = dp_packet_genl_ops,
	.n_ops = ARRAY_SIZE(dp_packet_genl_ops),
};

635
static void get_dp_stats(const struct datapath *dp, struct ovs_dp_stats *stats,
636
			 struct ovs_dp_megaflow_stats *mega_stats)
637 638 639
{
	int i;

640 641
	memset(mega_stats, 0, sizeof(*mega_stats));

642
	stats->n_flows = ovs_flow_tbl_count(&dp->table);
643
	mega_stats->n_masks = ovs_flow_tbl_num_masks(&dp->table);
644 645

	stats->n_hit = stats->n_missed = stats->n_lost = 0;
646

647 648 649 650 651 652 653 654
	for_each_possible_cpu(i) {
		const struct dp_stats_percpu *percpu_stats;
		struct dp_stats_percpu local_stats;
		unsigned int start;

		percpu_stats = per_cpu_ptr(dp->stats_percpu, i);

		do {
655
			start = u64_stats_fetch_begin_irq(&percpu_stats->syncp);
656
			local_stats = *percpu_stats;
657
		} while (u64_stats_fetch_retry_irq(&percpu_stats->syncp, start));
658 659 660 661

		stats->n_hit += local_stats.n_hit;
		stats->n_missed += local_stats.n_missed;
		stats->n_lost += local_stats.n_lost;
662
		mega_stats->n_mask_hit += local_stats.n_mask_hit;
663 664 665
	}
}

666 667 668 669 670 671 672 673 674 675 676 677
static bool should_fill_key(const struct sw_flow_id *sfid, uint32_t ufid_flags)
{
	return ovs_identifier_is_ufid(sfid) &&
	       !(ufid_flags & OVS_UFID_F_OMIT_KEY);
}

static bool should_fill_mask(uint32_t ufid_flags)
{
	return !(ufid_flags & OVS_UFID_F_OMIT_MASK);
}

static bool should_fill_actions(uint32_t ufid_flags)
678
{
679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704
	return !(ufid_flags & OVS_UFID_F_OMIT_ACTIONS);
}

static size_t ovs_flow_cmd_msg_size(const struct sw_flow_actions *acts,
				    const struct sw_flow_id *sfid,
				    uint32_t ufid_flags)
{
	size_t len = NLMSG_ALIGN(sizeof(struct ovs_header));

	/* OVS_FLOW_ATTR_UFID */
	if (sfid && ovs_identifier_is_ufid(sfid))
		len += nla_total_size(sfid->ufid_len);

	/* OVS_FLOW_ATTR_KEY */
	if (!sfid || should_fill_key(sfid, ufid_flags))
		len += nla_total_size(ovs_key_attr_size());

	/* OVS_FLOW_ATTR_MASK */
	if (should_fill_mask(ufid_flags))
		len += nla_total_size(ovs_key_attr_size());

	/* OVS_FLOW_ATTR_ACTIONS */
	if (should_fill_actions(ufid_flags))
		len += nla_total_size(acts->actions_len);

	return len
705 706
		+ nla_total_size(sizeof(struct ovs_flow_stats)) /* OVS_FLOW_ATTR_STATS */
		+ nla_total_size(1) /* OVS_FLOW_ATTR_TCP_FLAGS */
707
		+ nla_total_size(8); /* OVS_FLOW_ATTR_USED */
708 709
}

710 711 712 713 714 715 716
/* Called with ovs_mutex or RCU read lock. */
static int ovs_flow_cmd_fill_stats(const struct sw_flow *flow,
				   struct sk_buff *skb)
{
	struct ovs_flow_stats stats;
	__be16 tcp_flags;
	unsigned long used;
717

718
	ovs_flow_stats_get(flow, &stats, &used, &tcp_flags);
719

720 721
	if (used &&
	    nla_put_u64(skb, OVS_FLOW_ATTR_USED, ovs_flow_used_time(used)))
722
		return -EMSGSIZE;
723

724
	if (stats.n_packets &&
725
	    nla_put(skb, OVS_FLOW_ATTR_STATS, sizeof(struct ovs_flow_stats), &stats))
726
		return -EMSGSIZE;
727

728 729
	if ((u8)ntohs(tcp_flags) &&
	     nla_put_u8(skb, OVS_FLOW_ATTR_TCP_FLAGS, (u8)ntohs(tcp_flags)))
730 731 732 733 734 735 736 737 738 739 740
		return -EMSGSIZE;

	return 0;
}

/* Called with ovs_mutex or RCU read lock. */
static int ovs_flow_cmd_fill_actions(const struct sw_flow *flow,
				     struct sk_buff *skb, int skb_orig_len)
{
	struct nlattr *start;
	int err;
741 742 743 744 745 746 747 748 749 750 751

	/* If OVS_FLOW_ATTR_ACTIONS doesn't fit, skip dumping the actions if
	 * this is the first flow to be dumped into 'skb'.  This is unusual for
	 * Netlink but individual action lists can be longer than
	 * NLMSG_GOODSIZE and thus entirely undumpable if we didn't do this.
	 * The userspace caller can always fetch the actions separately if it
	 * really wants them.  (Most userspace callers in fact don't care.)
	 *
	 * This can only fail for dump operations because the skb is always
	 * properly sized for single flows.
	 */
752 753
	start = nla_nest_start(skb, OVS_FLOW_ATTR_ACTIONS);
	if (start) {
754 755
		const struct sw_flow_actions *sf_acts;

756
		sf_acts = rcu_dereference_ovsl(flow->sf_acts);
757 758
		err = ovs_nla_put_actions(sf_acts->actions,
					  sf_acts->actions_len, skb);
759

760 761 762 763
		if (!err)
			nla_nest_end(skb, start);
		else {
			if (skb_orig_len)
764
				return err;
765 766 767

			nla_nest_cancel(skb, start);
		}
768 769 770 771 772 773 774 775 776 777
	} else if (skb_orig_len) {
		return -EMSGSIZE;
	}

	return 0;
}

/* Called with ovs_mutex or RCU read lock. */
static int ovs_flow_cmd_fill_info(const struct sw_flow *flow, int dp_ifindex,
				  struct sk_buff *skb, u32 portid,
778
				  u32 seq, u32 flags, u8 cmd, u32 ufid_flags)
779 780 781 782 783 784 785 786 787 788 789 790
{
	const int skb_orig_len = skb->len;
	struct ovs_header *ovs_header;
	int err;

	ovs_header = genlmsg_put(skb, portid, seq, &dp_flow_genl_family,
				 flags, cmd);
	if (!ovs_header)
		return -EMSGSIZE;

	ovs_header->dp_ifindex = dp_ifindex;

791
	err = ovs_nla_put_identifier(flow, skb);
792 793 794
	if (err)
		goto error;

795 796 797 798 799 800 801 802 803 804 805
	if (should_fill_key(&flow->id, ufid_flags)) {
		err = ovs_nla_put_masked_key(flow, skb);
		if (err)
			goto error;
	}

	if (should_fill_mask(ufid_flags)) {
		err = ovs_nla_put_mask(flow, skb);
		if (err)
			goto error;
	}
806 807 808 809 810

	err = ovs_flow_cmd_fill_stats(flow, skb);
	if (err)
		goto error;

811 812 813 814 815
	if (should_fill_actions(ufid_flags)) {
		err = ovs_flow_cmd_fill_actions(flow, skb, skb_orig_len);
		if (err)
			goto error;
	}
816

817 818
	genlmsg_end(skb, ovs_header);
	return 0;
819 820 821 822 823 824

error:
	genlmsg_cancel(skb, ovs_header);
	return err;
}

825 826
/* May not be called with RCU read lock. */
static struct sk_buff *ovs_flow_cmd_alloc_info(const struct sw_flow_actions *acts,
827
					       const struct sw_flow_id *sfid,
828
					       struct genl_info *info,
829 830
					       bool always,
					       uint32_t ufid_flags)
831
{
832
	struct sk_buff *skb;
833
	size_t len;
834

835
	if (!always && !ovs_must_notify(&dp_flow_genl_family, info, 0))
836 837
		return NULL;

838 839
	len = ovs_flow_cmd_msg_size(acts, sfid, ufid_flags);
	skb = genlmsg_new_unicast(len, info, GFP_KERNEL);
840 841 842 843
	if (!skb)
		return ERR_PTR(-ENOMEM);

	return skb;
844 845
}

846 847 848 849
/* Called with ovs_mutex. */
static struct sk_buff *ovs_flow_cmd_build_info(const struct sw_flow *flow,
					       int dp_ifindex,
					       struct genl_info *info, u8 cmd,
850
					       bool always, u32 ufid_flags)
851 852 853 854
{
	struct sk_buff *skb;
	int retval;

855 856
	skb = ovs_flow_cmd_alloc_info(ovsl_dereference(flow->sf_acts),
				      &flow->id, info, always, ufid_flags);
857
	if (IS_ERR_OR_NULL(skb))
858
		return skb;
859

860 861
	retval = ovs_flow_cmd_fill_info(flow, dp_ifindex, skb,
					info->snd_portid, info->snd_seq, 0,
862
					cmd, ufid_flags);
863 864 865 866
	BUG_ON(retval < 0);
	return skb;
}

867
static int ovs_flow_cmd_new(struct sk_buff *skb, struct genl_info *info)
868 869 870
{
	struct nlattr **a = info->attrs;
	struct ovs_header *ovs_header = info->userhdr;
871
	struct sw_flow *flow = NULL, *new_flow;
872
	struct sw_flow_mask mask;
873 874
	struct sk_buff *reply;
	struct datapath *dp;
875
	struct sw_flow_key key;
876
	struct sw_flow_actions *acts;
877
	struct sw_flow_match match;
878
	u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
879
	int error;
880
	bool log = !a[OVS_FLOW_ATTR_PROBE];
881

882
	/* Must have key and actions. */
883
	error = -EINVAL;
884
	if (!a[OVS_FLOW_ATTR_KEY]) {
885
		OVS_NLERR(log, "Flow key attr not present in new flow.");
886
		goto error;
887 888
	}
	if (!a[OVS_FLOW_ATTR_ACTIONS]) {
889
		OVS_NLERR(log, "Flow actions attr not present in new flow.");
890
		goto error;
891
	}
892

893 894 895 896 897 898 899 900 901 902
	/* Most of the time we need to allocate a new flow, do it before
	 * locking.
	 */
	new_flow = ovs_flow_alloc();
	if (IS_ERR(new_flow)) {
		error = PTR_ERR(new_flow);
		goto error;
	}

	/* Extract key. */
903
	ovs_match_init(&match, &key, &mask);
904 905
	error = ovs_nla_get_match(&match, a[OVS_FLOW_ATTR_KEY],
				  a[OVS_FLOW_ATTR_MASK], log);
906
	if (error)
907
		goto err_kfree_flow;
908

909 910 911 912 913 914 915
	ovs_flow_mask_key(&new_flow->key, &key, &mask);

	/* Extract flow identifier. */
	error = ovs_nla_get_identifier(&new_flow->id, a[OVS_FLOW_ATTR_UFID],
				       &key, log);
	if (error)
		goto err_kfree_flow;
916

917 918
	/* Validate actions. */
	error = ovs_nla_copy_actions(a[OVS_FLOW_ATTR_ACTIONS], &new_flow->key,
919
				     &acts, log);
920
	if (error) {
921
		OVS_NLERR(log, "Flow actions may not be safe on all matching packets.");
922
		goto err_kfree_flow;
923 924
	}

925 926
	reply = ovs_flow_cmd_alloc_info(acts, &new_flow->id, info, false,
					ufid_flags);
927 928 929
	if (IS_ERR(reply)) {
		error = PTR_ERR(reply);
		goto err_kfree_acts;
930 931
	}

932
	ovs_lock();
933
	dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
934 935
	if (unlikely(!dp)) {
		error = -ENODEV;
936
		goto err_unlock_ovs;
937
	}
938

939
	/* Check if this is a duplicate flow */
940 941 942 943
	if (ovs_identifier_is_ufid(&new_flow->id))
		flow = ovs_flow_tbl_lookup_ufid(&dp->table, &new_flow->id);
	if (!flow)
		flow = ovs_flow_tbl_lookup(&dp->table, &key);
944 945
	if (likely(!flow)) {
		rcu_assign_pointer(new_flow->sf_acts, acts);
946 947

		/* Put flow in bucket. */
948 949
		error = ovs_flow_tbl_insert(&dp->table, new_flow, &mask);
		if (unlikely(error)) {
950
			acts = NULL;
951 952 953 954 955 956 957 958
			goto err_unlock_ovs;
		}

		if (unlikely(reply)) {
			error = ovs_flow_cmd_fill_info(new_flow,
						       ovs_header->dp_ifindex,
						       reply, info->snd_portid,
						       info->snd_seq, 0,
959 960
						       OVS_FLOW_CMD_NEW,
						       ufid_flags);
961
			BUG_ON(error < 0);
962
		}
963
		ovs_unlock();
964
	} else {
965 966
		struct sw_flow_actions *old_acts;

967 968 969 970 971 972
		/* Bail out if we're not allowed to modify an existing flow.
		 * We accept NLM_F_CREATE in place of the intended NLM_F_EXCL
		 * because Generic Netlink treats the latter as a dump
		 * request.  We also accept NLM_F_EXCL in case that bug ever
		 * gets fixed.
		 */
973 974 975
		if (unlikely(info->nlhdr->nlmsg_flags & (NLM_F_CREATE
							 | NLM_F_EXCL))) {
			error = -EEXIST;
976
			goto err_unlock_ovs;
977
		}
978 979 980 981 982 983 984 985 986
		/* The flow identifier has to be the same for flow updates.
		 * Look for any overlapping flow.
		 */
		if (unlikely(!ovs_flow_cmp(flow, &match))) {
			if (ovs_identifier_is_key(&flow->id))
				flow = ovs_flow_tbl_lookup_exact(&dp->table,
								 &match);
			else /* UFID matches but key is different */
				flow = NULL;
987 988 989 990
			if (!flow) {
				error = -ENOENT;
				goto err_unlock_ovs;
			}
991
		}
992 993 994 995
		/* Update actions. */
		old_acts = ovsl_dereference(flow->sf_acts);
		rcu_assign_pointer(flow->sf_acts, acts);

996 997 998 999 1000
		if (unlikely(reply)) {
			error = ovs_flow_cmd_fill_info(flow,
						       ovs_header->dp_ifindex,
						       reply, info->snd_portid,
						       info->snd_seq, 0,
1001 1002
						       OVS_FLOW_CMD_NEW,
						       ufid_flags);
1003 1004 1005
			BUG_ON(error < 0);
		}
		ovs_unlock();
1006

1007 1008
		ovs_nla_free_flow_actions(old_acts);
		ovs_flow_free(new_flow, false);
1009
	}
1010 1011 1012

	if (reply)
		ovs_notify(&dp_flow_genl_family, reply, info);
1013 1014 1015 1016
	return 0;

err_unlock_ovs:
	ovs_unlock();
1017 1018
	kfree_skb(reply);
err_kfree_acts:
1019
	kfree(acts);
1020 1021
err_kfree_flow:
	ovs_flow_free(new_flow, false);
1022 1023 1024
error:
	return error;
}
1025

1026
/* Factor out action copy to avoid "Wframe-larger-than=1024" warning. */
1027 1028
static struct sw_flow_actions *get_flow_actions(const struct nlattr *a,
						const struct sw_flow_key *key,
1029 1030
						const struct sw_flow_mask *mask,
						bool log)
1031 1032 1033 1034 1035 1036
{
	struct sw_flow_actions *acts;
	struct sw_flow_key masked_key;
	int error;

	ovs_flow_mask_key(&masked_key, key, mask);
1037
	error = ovs_nla_copy_actions(a, &masked_key, &acts, log);
1038
	if (error) {
1039 1040
		OVS_NLERR(log,
			  "Actions may not be safe on all matching packets");
1041 1042 1043 1044 1045 1046
		return ERR_PTR(error);
	}

	return acts;
}

1047 1048 1049 1050
static int ovs_flow_cmd_set(struct sk_buff *skb, struct genl_info *info)
{
	struct nlattr **a = info->attrs;
	struct ovs_header *ovs_header = info->userhdr;
1051
	struct sw_flow_key key;
1052 1053 1054 1055
	struct sw_flow *flow;
	struct sw_flow_mask mask;
	struct sk_buff *reply = NULL;
	struct datapath *dp;
1056
	struct sw_flow_actions *old_acts = NULL, *acts = NULL;
1057
	struct sw_flow_match match;
1058 1059
	struct sw_flow_id sfid;
	u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1060
	int error;
1061
	bool log = !a[OVS_FLOW_ATTR_PROBE];
1062
	bool ufid_present;
1063 1064 1065

	/* Extract key. */
	error = -EINVAL;
1066
	if (!a[OVS_FLOW_ATTR_KEY]) {
1067
		OVS_NLERR(log, "Flow key attribute not present in set flow.");
1068
		goto error;
1069
	}
1070

1071
	ufid_present = ovs_nla_get_ufid(&sfid, a[OVS_FLOW_ATTR_UFID], log);
1072
	ovs_match_init(&match, &key, &mask);
1073 1074
	error = ovs_nla_get_match(&match, a[OVS_FLOW_ATTR_KEY],
				  a[OVS_FLOW_ATTR_MASK], log);
1075 1076 1077 1078 1079
	if (error)
		goto error;

	/* Validate actions. */
	if (a[OVS_FLOW_ATTR_ACTIONS]) {
1080 1081
		acts = get_flow_actions(a[OVS_FLOW_ATTR_ACTIONS], &key, &mask,
					log);
1082 1083
		if (IS_ERR(acts)) {
			error = PTR_ERR(acts);
1084
			goto error;
1085 1086
		}

1087
		/* Can allocate before locking if have acts. */
1088 1089
		reply = ovs_flow_cmd_alloc_info(acts, &sfid, info, false,
						ufid_flags);
1090 1091 1092
		if (IS_ERR(reply)) {
			error = PTR_ERR(reply);
			goto err_kfree_acts;
1093
		}
1094
	}
1095

1096 1097
	ovs_lock();
	dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1098 1099
	if (unlikely(!dp)) {
		error = -ENODEV;
1100
		goto err_unlock_ovs;
1101
	}
1102
	/* Check that the flow exists. */
1103 1104 1105 1106
	if (ufid_present)
		flow = ovs_flow_tbl_lookup_ufid(&dp->table, &sfid);
	else
		flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
1107 1108
	if (unlikely(!flow)) {
		error = -ENOENT;
1109
		goto err_unlock_ovs;
1110
	}
1111

1112
	/* Update actions, if present. */
1113
	if (likely(acts)) {
1114 1115
		old_acts = ovsl_dereference(flow->sf_acts);
		rcu_assign_pointer(flow->sf_acts, acts);
1116 1117 1118 1119 1120 1121

		if (unlikely(reply)) {
			error = ovs_flow_cmd_fill_info(flow,
						       ovs_header->dp_ifindex,
						       reply, info->snd_portid,
						       info->snd_seq, 0,
1122 1123
						       OVS_FLOW_CMD_NEW,
						       ufid_flags);
1124 1125 1126 1127 1128
			BUG_ON(error < 0);
		}
	} else {
		/* Could not alloc without acts before locking. */
		reply = ovs_flow_cmd_build_info(flow, ovs_header->dp_ifindex,
1129 1130 1131
						info, OVS_FLOW_CMD_NEW, false,
						ufid_flags);

1132 1133 1134 1135
		if (unlikely(IS_ERR(reply))) {
			error = PTR_ERR(reply);
			goto err_unlock_ovs;
		}
1136
	}
1137 1138 1139 1140

	/* Clear stats. */
	if (a[OVS_FLOW_ATTR_CLEAR])
		ovs_flow_stats_clear(flow);
1141
	ovs_unlock();
1142

1143 1144 1145 1146
	if (reply)
		ovs_notify(&dp_flow_genl_family, reply, info);
	if (old_acts)
		ovs_nla_free_flow_actions(old_acts);
1147

1148 1149
	return 0;

1150 1151
err_unlock_ovs:
	ovs_unlock();
1152 1153
	kfree_skb(reply);
err_kfree_acts:
1154
	kfree(acts);
1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166
error:
	return error;
}

static int ovs_flow_cmd_get(struct sk_buff *skb, struct genl_info *info)
{
	struct nlattr **a = info->attrs;
	struct ovs_header *ovs_header = info->userhdr;
	struct sw_flow_key key;
	struct sk_buff *reply;
	struct sw_flow *flow;
	struct datapath *dp;
1167
	struct sw_flow_match match;
1168 1169 1170
	struct sw_flow_id ufid;
	u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
	int err = 0;
1171
	bool log = !a[OVS_FLOW_ATTR_PROBE];
1172
	bool ufid_present;
1173

1174 1175 1176 1177 1178 1179
	ufid_present = ovs_nla_get_ufid(&ufid, a[OVS_FLOW_ATTR_UFID], log);
	if (a[OVS_FLOW_ATTR_KEY]) {
		ovs_match_init(&match, &key, NULL);
		err = ovs_nla_get_match(&match, a[OVS_FLOW_ATTR_KEY], NULL,
					log);
	} else if (!ufid_present) {
1180 1181
		OVS_NLERR(log,
			  "Flow get message rejected, Key attribute missing.");
1182
		err = -EINVAL;
1183
	}
1184 1185 1186
	if (err)
		return err;

1187
	ovs_lock();
1188
	dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1189 1190 1191 1192
	if (!dp) {
		err = -ENODEV;
		goto unlock;
	}
1193

1194 1195 1196 1197
	if (ufid_present)
		flow = ovs_flow_tbl_lookup_ufid(&dp->table, &ufid);
	else
		flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
1198
	if (!flow) {
1199 1200 1201
		err = -ENOENT;
		goto unlock;
	}
1202

1203
	reply = ovs_flow_cmd_build_info(flow, ovs_header->dp_ifindex, info,
1204
					OVS_FLOW_CMD_NEW, true, ufid_flags);
1205 1206 1207 1208
	if (IS_ERR(reply)) {
		err = PTR_ERR(reply);
		goto unlock;
	}
1209

1210
	ovs_unlock();
1211
	return genlmsg_reply(reply, info);
1212 1213 1214
unlock:
	ovs_unlock();
	return err;
1215 1216 1217 1218 1219 1220 1221 1222
}

static int ovs_flow_cmd_del(struct sk_buff *skb, struct genl_info *info)
{
	struct nlattr **a = info->attrs;
	struct ovs_header *ovs_header = info->userhdr;
	struct sw_flow_key key;
	struct sk_buff *reply;
1223
	struct sw_flow *flow = NULL;
1224
	struct datapath *dp;
1225
	struct sw_flow_match match;
1226 1227
	struct sw_flow_id ufid;
	u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1228
	int err;
1229
	bool log = !a[OVS_FLOW_ATTR_PROBE];
1230
	bool ufid_present;
1231

1232 1233
	ufid_present = ovs_nla_get_ufid(&ufid, a[OVS_FLOW_ATTR_UFID], log);
	if (a[OVS_FLOW_ATTR_KEY]) {
1234
		ovs_match_init(&match, &key, NULL);
1235 1236
		err = ovs_nla_get_match(&match, a[OVS_FLOW_ATTR_KEY], NULL,
					log);
1237 1238 1239 1240
		if (unlikely(err))
			return err;
	}

1241
	ovs_lock();
1242
	dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1243
	if (unlikely(!dp)) {
1244 1245 1246
		err = -ENODEV;
		goto unlock;
	}
1247

1248
	if (unlikely(!a[OVS_FLOW_ATTR_KEY] && !ufid_present)) {
1249
		err = ovs_flow_tbl_flush(&dp->table);
1250 1251
		goto unlock;
	}
1252

1253 1254 1255 1256
	if (ufid_present)
		flow = ovs_flow_tbl_lookup_ufid(&dp->table, &ufid);
	else
		flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
1257
	if (unlikely(!flow)) {
1258 1259 1260
		err = -ENOENT;
		goto unlock;
	}
1261

1262
	ovs_flow_tbl_remove(&dp->table, flow);
1263
	ovs_unlock();
1264

1265
	reply = ovs_flow_cmd_alloc_info((const struct sw_flow_actions __force *) flow->sf_acts,
1266
					&flow->id, info, false, ufid_flags);
1267 1268 1269 1270 1271 1272
	if (likely(reply)) {
		if (likely(!IS_ERR(reply))) {
			rcu_read_lock();	/*To keep RCU checker happy. */
			err = ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex,
						     reply, info->snd_portid,
						     info->snd_seq, 0,
1273 1274
						     OVS_FLOW_CMD_DEL,
						     ufid_flags);
1275 1276 1277 1278 1279 1280 1281
			rcu_read_unlock();
			BUG_ON(err < 0);

			ovs_notify(&dp_flow_genl_family, reply, info);
		} else {
			netlink_set_err(sock_net(skb->sk)->genl_sock, 0, 0, PTR_ERR(reply));
		}
1282
	}
1283

1284
	ovs_flow_free(flow, true);
1285
	return 0;
1286 1287 1288
unlock:
	ovs_unlock();
	return err;
1289 1290 1291 1292
}

static int ovs_flow_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
{
1293
	struct nlattr *a[__OVS_FLOW_ATTR_MAX];
1294
	struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
1295
	struct table_instance *ti;
1296
	struct datapath *dp;
1297 1298 1299 1300 1301 1302 1303 1304
	u32 ufid_flags;
	int err;

	err = genlmsg_parse(cb->nlh, &dp_flow_genl_family, a,
			    OVS_FLOW_ATTR_MAX, flow_policy);
	if (err)
		return err;
	ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1305

1306
	rcu_read_lock();
1307
	dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex);
1308
	if (!dp) {
1309
		rcu_read_unlock();
1310
		return -ENODEV;
1311
	}
1312

1313
	ti = rcu_dereference(dp->table.ti);
1314 1315 1316 1317 1318 1319
	for (;;) {
		struct sw_flow *flow;
		u32 bucket, obj;

		bucket = cb->args[0];
		obj = cb->args[1];
1320
		flow = ovs_flow_tbl_dump_next(ti, &bucket, &obj);
1321 1322 1323
		if (!flow)
			break;

1324
		if (ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex, skb,
1325
					   NETLINK_CB(cb->skb).portid,
1326
					   cb->nlh->nlmsg_seq, NLM_F_MULTI,
1327
					   OVS_FLOW_CMD_NEW, ufid_flags) < 0)
1328 1329 1330 1331 1332
			break;

		cb->args[0] = bucket;
		cb->args[1] = obj;
	}
1333
	rcu_read_unlock();
1334 1335 1336
	return skb->len;
}

1337 1338
static const struct nla_policy flow_policy[OVS_FLOW_ATTR_MAX + 1] = {
	[OVS_FLOW_ATTR_KEY] = { .type = NLA_NESTED },
1339
	[OVS_FLOW_ATTR_MASK] = { .type = NLA_NESTED },
1340 1341
	[OVS_FLOW_ATTR_ACTIONS] = { .type = NLA_NESTED },
	[OVS_FLOW_ATTR_CLEAR] = { .type = NLA_FLAG },
1342
	[OVS_FLOW_ATTR_PROBE] = { .type = NLA_FLAG },
1343 1344
	[OVS_FLOW_ATTR_UFID] = { .type = NLA_UNSPEC, .len = 1 },
	[OVS_FLOW_ATTR_UFID_FLAGS] = { .type = NLA_U32 },
1345 1346
};

1347
static const struct genl_ops dp_flow_genl_ops[] = {
1348 1349 1350
	{ .cmd = OVS_FLOW_CMD_NEW,
	  .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
	  .policy = flow_policy,
1351
	  .doit = ovs_flow_cmd_new
1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366
	},
	{ .cmd = OVS_FLOW_CMD_DEL,
	  .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
	  .policy = flow_policy,
	  .doit = ovs_flow_cmd_del
	},
	{ .cmd = OVS_FLOW_CMD_GET,
	  .flags = 0,		    /* OK for unprivileged users. */
	  .policy = flow_policy,
	  .doit = ovs_flow_cmd_get,
	  .dumpit = ovs_flow_cmd_dump
	},
	{ .cmd = OVS_FLOW_CMD_SET,
	  .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
	  .policy = flow_policy,
1367
	  .doit = ovs_flow_cmd_set,
1368 1369 1370
	},
};

1371
static struct genl_family dp_flow_genl_family = {
1372 1373
	.id = GENL_ID_GENERATE,
	.hdrsize = sizeof(struct ovs_header),
1374 1375 1376
	.name = OVS_FLOW_FAMILY,
	.version = OVS_FLOW_VERSION,
	.maxattr = OVS_FLOW_ATTR_MAX,
1377 1378
	.netnsok = true,
	.parallel_ops = true,
1379 1380 1381 1382
	.ops = dp_flow_genl_ops,
	.n_ops = ARRAY_SIZE(dp_flow_genl_ops),
	.mcgrps = &ovs_dp_flow_multicast_group,
	.n_mcgrps = 1,
1383 1384
};

1385 1386 1387 1388 1389 1390
static size_t ovs_dp_cmd_msg_size(void)
{
	size_t msgsize = NLMSG_ALIGN(sizeof(struct ovs_header));

	msgsize += nla_total_size(IFNAMSIZ);
	msgsize += nla_total_size(sizeof(struct ovs_dp_stats));
1391
	msgsize += nla_total_size(sizeof(struct ovs_dp_megaflow_stats));
1392
	msgsize += nla_total_size(sizeof(u32)); /* OVS_DP_ATTR_USER_FEATURES */
1393 1394 1395 1396

	return msgsize;
}

1397
/* Called with ovs_mutex. */
1398
static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb,
1399
				u32 portid, u32 seq, u32 flags, u8 cmd)
1400 1401 1402
{
	struct ovs_header *ovs_header;
	struct ovs_dp_stats dp_stats;
1403
	struct ovs_dp_megaflow_stats dp_megaflow_stats;
1404 1405
	int err;

1406
	ovs_header = genlmsg_put(skb, portid, seq, &dp_datapath_genl_family,
1407 1408 1409 1410 1411 1412 1413 1414 1415 1416
				   flags, cmd);
	if (!ovs_header)
		goto error;

	ovs_header->dp_ifindex = get_dpifindex(dp);

	err = nla_put_string(skb, OVS_DP_ATTR_NAME, ovs_dp_name(dp));
	if (err)
		goto nla_put_failure;

1417 1418 1419 1420 1421 1422 1423 1424
	get_dp_stats(dp, &dp_stats, &dp_megaflow_stats);
	if (nla_put(skb, OVS_DP_ATTR_STATS, sizeof(struct ovs_dp_stats),
			&dp_stats))
		goto nla_put_failure;

	if (nla_put(skb, OVS_DP_ATTR_MEGAFLOW_STATS,
			sizeof(struct ovs_dp_megaflow_stats),
			&dp_megaflow_stats))
1425
		goto nla_put_failure;
1426

1427 1428 1429
	if (nla_put_u32(skb, OVS_DP_ATTR_USER_FEATURES, dp->user_features))
		goto nla_put_failure;

1430 1431
	genlmsg_end(skb, ovs_header);
	return 0;
1432 1433 1434 1435 1436 1437 1438

nla_put_failure:
	genlmsg_cancel(skb, ovs_header);
error:
	return -EMSGSIZE;
}

1439
static struct sk_buff *ovs_dp_cmd_alloc_info(struct genl_info *info)
1440
{
1441
	return genlmsg_new_unicast(ovs_dp_cmd_msg_size(), info, GFP_KERNEL);
1442 1443
}

J
Jarno Rajahalme 已提交
1444
/* Called with rcu_read_lock or ovs_mutex. */
1445
static struct datapath *lookup_datapath(struct net *net,
1446
					const struct ovs_header *ovs_header,
1447 1448 1449 1450 1451
					struct nlattr *a[OVS_DP_ATTR_MAX + 1])
{
	struct datapath *dp;

	if (!a[OVS_DP_ATTR_NAME])
1452
		dp = get_dp(net, ovs_header->dp_ifindex);
1453 1454 1455
	else {
		struct vport *vport;

1456
		vport = ovs_vport_locate(net, nla_data(a[OVS_DP_ATTR_NAME]));
1457 1458 1459 1460 1461
		dp = vport && vport->port_no == OVSP_LOCAL ? vport->dp : NULL;
	}
	return dp ? dp : ERR_PTR(-ENODEV);
}

1462 1463 1464 1465 1466
static void ovs_dp_reset_user_features(struct sk_buff *skb, struct genl_info *info)
{
	struct datapath *dp;

	dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1467
	if (IS_ERR(dp))
1468 1469 1470 1471 1472 1473
		return;

	WARN(dp->user_features, "Dropping previously announced user features\n");
	dp->user_features = 0;
}

1474
static void ovs_dp_change(struct datapath *dp, struct nlattr *a[])
1475 1476 1477 1478 1479
{
	if (a[OVS_DP_ATTR_USER_FEATURES])
		dp->user_features = nla_get_u32(a[OVS_DP_ATTR_USER_FEATURES]);
}

1480 1481 1482 1483 1484 1485 1486
static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info)
{
	struct nlattr **a = info->attrs;
	struct vport_parms parms;
	struct sk_buff *reply;
	struct datapath *dp;
	struct vport *vport;
1487
	struct ovs_net *ovs_net;
1488
	int err, i;
1489 1490 1491 1492 1493

	err = -EINVAL;
	if (!a[OVS_DP_ATTR_NAME] || !a[OVS_DP_ATTR_UPCALL_PID])
		goto err;

1494 1495 1496
	reply = ovs_dp_cmd_alloc_info(info);
	if (!reply)
		return -ENOMEM;
1497 1498 1499 1500

	err = -ENOMEM;
	dp = kzalloc(sizeof(*dp), GFP_KERNEL);
	if (dp == NULL)
1501
		goto err_free_reply;
1502

1503
	ovs_dp_set_net(dp, sock_net(skb->sk));
1504 1505

	/* Allocate table. */
1506 1507
	err = ovs_flow_tbl_init(&dp->table);
	if (err)
1508 1509
		goto err_free_dp;

1510
	dp->stats_percpu = netdev_alloc_pcpu_stats(struct dp_stats_percpu);
1511 1512 1513 1514 1515
	if (!dp->stats_percpu) {
		err = -ENOMEM;
		goto err_destroy_table;
	}

1516
	dp->ports = kmalloc(DP_VPORT_HASH_BUCKETS * sizeof(struct hlist_head),
1517
			    GFP_KERNEL);
1518 1519 1520 1521 1522 1523 1524 1525
	if (!dp->ports) {
		err = -ENOMEM;
		goto err_destroy_percpu;
	}

	for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++)
		INIT_HLIST_HEAD(&dp->ports[i]);

1526 1527 1528 1529 1530 1531
	/* Set up our datapath device. */
	parms.name = nla_data(a[OVS_DP_ATTR_NAME]);
	parms.type = OVS_VPORT_TYPE_INTERNAL;
	parms.options = NULL;
	parms.dp = dp;
	parms.port_no = OVSP_LOCAL;
1532
	parms.upcall_portids = a[OVS_DP_ATTR_UPCALL_PID];
1533

1534 1535
	ovs_dp_change(dp, a);

1536 1537 1538
	/* So far only local changes have been made, now need the lock. */
	ovs_lock();

1539 1540 1541 1542 1543 1544
	vport = new_vport(&parms);
	if (IS_ERR(vport)) {
		err = PTR_ERR(vport);
		if (err == -EBUSY)
			err = -EEXIST;

1545 1546 1547 1548 1549 1550 1551 1552 1553
		if (err == -EEXIST) {
			/* An outdated user space instance that does not understand
			 * the concept of user_features has attempted to create a new
			 * datapath and is likely to reuse it. Drop all user features.
			 */
			if (info->genlhdr->version < OVS_DP_VER_FEATURES)
				ovs_dp_reset_user_features(skb, info);
		}

1554
		goto err_destroy_ports_array;
1555 1556
	}

1557 1558 1559
	err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
				   info->snd_seq, 0, OVS_DP_CMD_NEW);
	BUG_ON(err < 0);
1560

1561
	ovs_net = net_generic(ovs_dp_get_net(dp), ovs_net_id);
1562
	list_add_tail_rcu(&dp->list_node, &ovs_net->dps);
1563 1564

	ovs_unlock();
1565

1566
	ovs_notify(&dp_datapath_genl_family, reply, info);
1567 1568
	return 0;

1569
err_destroy_ports_array:
1570
	ovs_unlock();
1571
	kfree(dp->ports);
1572 1573 1574
err_destroy_percpu:
	free_percpu(dp->stats_percpu);
err_destroy_table:
1575
	ovs_flow_tbl_destroy(&dp->table);
1576 1577
err_free_dp:
	kfree(dp);
1578 1579
err_free_reply:
	kfree_skb(reply);
1580 1581 1582 1583
err:
	return err;
}

1584
/* Called with ovs_mutex. */
1585
static void __dp_destroy(struct datapath *dp)
1586
{
1587
	int i;
1588

1589 1590
	for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
		struct vport *vport;
1591
		struct hlist_node *n;
1592

1593
		hlist_for_each_entry_safe(vport, n, &dp->ports[i], dp_hash_node)
1594 1595 1596
			if (vport->port_no != OVSP_LOCAL)
				ovs_dp_detach_port(vport);
	}
1597

1598
	list_del_rcu(&dp->list_node);
1599

1600
	/* OVSP_LOCAL is datapath internal port. We need to make sure that
1601
	 * all ports in datapath are destroyed first before freeing datapath.
1602
	 */
1603
	ovs_dp_detach_port(ovs_vport_ovsl(dp, OVSP_LOCAL));
1604

1605
	/* RCU destroy the flow table */
1606
	call_rcu(&dp->rcu, destroy_dp_rcu);
1607 1608 1609 1610 1611 1612 1613 1614
}

static int ovs_dp_cmd_del(struct sk_buff *skb, struct genl_info *info)
{
	struct sk_buff *reply;
	struct datapath *dp;
	int err;

1615 1616 1617 1618
	reply = ovs_dp_cmd_alloc_info(info);
	if (!reply)
		return -ENOMEM;

1619
	ovs_lock();
1620 1621 1622
	dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
	err = PTR_ERR(dp);
	if (IS_ERR(dp))
1623
		goto err_unlock_free;
1624

1625 1626 1627
	err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
				   info->snd_seq, 0, OVS_DP_CMD_DEL);
	BUG_ON(err < 0);
1628 1629

	__dp_destroy(dp);
1630
	ovs_unlock();
1631

1632
	ovs_notify(&dp_datapath_genl_family, reply, info);
1633 1634

	return 0;
1635 1636

err_unlock_free:
1637
	ovs_unlock();
1638
	kfree_skb(reply);
1639
	return err;
1640 1641 1642 1643 1644 1645 1646 1647
}

static int ovs_dp_cmd_set(struct sk_buff *skb, struct genl_info *info)
{
	struct sk_buff *reply;
	struct datapath *dp;
	int err;

1648 1649 1650 1651
	reply = ovs_dp_cmd_alloc_info(info);
	if (!reply)
		return -ENOMEM;

1652
	ovs_lock();
1653
	dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1654
	err = PTR_ERR(dp);
1655
	if (IS_ERR(dp))
1656
		goto err_unlock_free;
1657

1658 1659
	ovs_dp_change(dp, info->attrs);

1660 1661 1662
	err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
				   info->snd_seq, 0, OVS_DP_CMD_NEW);
	BUG_ON(err < 0);
1663

1664
	ovs_unlock();
1665
	ovs_notify(&dp_datapath_genl_family, reply, info);
1666 1667

	return 0;
1668 1669

err_unlock_free:
1670
	ovs_unlock();
1671
	kfree_skb(reply);
1672
	return err;
1673 1674 1675 1676 1677 1678
}

static int ovs_dp_cmd_get(struct sk_buff *skb, struct genl_info *info)
{
	struct sk_buff *reply;
	struct datapath *dp;
1679
	int err;
1680

1681 1682 1683 1684
	reply = ovs_dp_cmd_alloc_info(info);
	if (!reply)
		return -ENOMEM;

1685
	ovs_lock();
1686
	dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1687 1688
	if (IS_ERR(dp)) {
		err = PTR_ERR(dp);
1689
		goto err_unlock_free;
1690
	}
1691 1692 1693
	err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
				   info->snd_seq, 0, OVS_DP_CMD_NEW);
	BUG_ON(err < 0);
1694
	ovs_unlock();
1695 1696

	return genlmsg_reply(reply, info);
1697

1698
err_unlock_free:
1699
	ovs_unlock();
1700
	kfree_skb(reply);
1701
	return err;
1702 1703 1704 1705
}

static int ovs_dp_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
{
1706
	struct ovs_net *ovs_net = net_generic(sock_net(skb->sk), ovs_net_id);
1707 1708 1709 1710
	struct datapath *dp;
	int skip = cb->args[0];
	int i = 0;

1711 1712
	ovs_lock();
	list_for_each_entry(dp, &ovs_net->dps, list_node) {
1713
		if (i >= skip &&
1714
		    ovs_dp_cmd_fill_info(dp, skb, NETLINK_CB(cb->skb).portid,
1715 1716 1717 1718 1719
					 cb->nlh->nlmsg_seq, NLM_F_MULTI,
					 OVS_DP_CMD_NEW) < 0)
			break;
		i++;
	}
1720
	ovs_unlock();
1721 1722 1723 1724 1725 1726

	cb->args[0] = i;

	return skb->len;
}

1727 1728 1729 1730 1731 1732
static const struct nla_policy datapath_policy[OVS_DP_ATTR_MAX + 1] = {
	[OVS_DP_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
	[OVS_DP_ATTR_UPCALL_PID] = { .type = NLA_U32 },
	[OVS_DP_ATTR_USER_FEATURES] = { .type = NLA_U32 },
};

1733
static const struct genl_ops dp_datapath_genl_ops[] = {
1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756
	{ .cmd = OVS_DP_CMD_NEW,
	  .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
	  .policy = datapath_policy,
	  .doit = ovs_dp_cmd_new
	},
	{ .cmd = OVS_DP_CMD_DEL,
	  .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
	  .policy = datapath_policy,
	  .doit = ovs_dp_cmd_del
	},
	{ .cmd = OVS_DP_CMD_GET,
	  .flags = 0,		    /* OK for unprivileged users. */
	  .policy = datapath_policy,
	  .doit = ovs_dp_cmd_get,
	  .dumpit = ovs_dp_cmd_dump
	},
	{ .cmd = OVS_DP_CMD_SET,
	  .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
	  .policy = datapath_policy,
	  .doit = ovs_dp_cmd_set,
	},
};

1757
static struct genl_family dp_datapath_genl_family = {
1758 1759
	.id = GENL_ID_GENERATE,
	.hdrsize = sizeof(struct ovs_header),
1760 1761 1762
	.name = OVS_DATAPATH_FAMILY,
	.version = OVS_DATAPATH_VERSION,
	.maxattr = OVS_DP_ATTR_MAX,
1763 1764
	.netnsok = true,
	.parallel_ops = true,
1765 1766 1767 1768
	.ops = dp_datapath_genl_ops,
	.n_ops = ARRAY_SIZE(dp_datapath_genl_ops),
	.mcgrps = &ovs_dp_datapath_multicast_group,
	.n_mcgrps = 1,
1769 1770
};

1771
/* Called with ovs_mutex or RCU read lock. */
1772
static int ovs_vport_cmd_fill_info(struct vport *vport, struct sk_buff *skb,
1773
				   u32 portid, u32 seq, u32 flags, u8 cmd)
1774 1775 1776 1777 1778
{
	struct ovs_header *ovs_header;
	struct ovs_vport_stats vport_stats;
	int err;

1779
	ovs_header = genlmsg_put(skb, portid, seq, &dp_vport_genl_family,
1780 1781 1782 1783 1784 1785
				 flags, cmd);
	if (!ovs_header)
		return -EMSGSIZE;

	ovs_header->dp_ifindex = get_dpifindex(vport->dp);

1786 1787
	if (nla_put_u32(skb, OVS_VPORT_ATTR_PORT_NO, vport->port_no) ||
	    nla_put_u32(skb, OVS_VPORT_ATTR_TYPE, vport->ops->type) ||
1788 1789
	    nla_put_string(skb, OVS_VPORT_ATTR_NAME,
			   vport->ops->get_name(vport)))
1790
		goto nla_put_failure;
1791 1792

	ovs_vport_get_stats(vport, &vport_stats);
1793 1794 1795
	if (nla_put(skb, OVS_VPORT_ATTR_STATS, sizeof(struct ovs_vport_stats),
		    &vport_stats))
		goto nla_put_failure;
1796

1797 1798 1799
	if (ovs_vport_get_upcall_portids(vport, skb))
		goto nla_put_failure;

1800 1801 1802 1803
	err = ovs_vport_get_options(vport, skb);
	if (err == -EMSGSIZE)
		goto error;

1804 1805
	genlmsg_end(skb, ovs_header);
	return 0;
1806 1807 1808 1809 1810 1811 1812 1813

nla_put_failure:
	err = -EMSGSIZE;
error:
	genlmsg_cancel(skb, ovs_header);
	return err;
}

1814 1815 1816 1817 1818 1819
static struct sk_buff *ovs_vport_cmd_alloc_info(void)
{
	return nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
}

/* Called with ovs_mutex, only via ovs_dp_notify_wq(). */
1820
struct sk_buff *ovs_vport_cmd_build_info(struct vport *vport, u32 portid,
1821 1822 1823 1824 1825 1826 1827 1828 1829
					 u32 seq, u8 cmd)
{
	struct sk_buff *skb;
	int retval;

	skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_ATOMIC);
	if (!skb)
		return ERR_PTR(-ENOMEM);

1830
	retval = ovs_vport_cmd_fill_info(vport, skb, portid, seq, 0, cmd);
1831 1832
	BUG_ON(retval < 0);

1833 1834 1835
	return skb;
}

1836
/* Called with ovs_mutex or RCU read lock. */
1837
static struct vport *lookup_vport(struct net *net,
1838
				  const struct ovs_header *ovs_header,
1839 1840 1841 1842 1843 1844
				  struct nlattr *a[OVS_VPORT_ATTR_MAX + 1])
{
	struct datapath *dp;
	struct vport *vport;

	if (a[OVS_VPORT_ATTR_NAME]) {
1845
		vport = ovs_vport_locate(net, nla_data(a[OVS_VPORT_ATTR_NAME]));
1846 1847
		if (!vport)
			return ERR_PTR(-ENODEV);
1848 1849 1850
		if (ovs_header->dp_ifindex &&
		    ovs_header->dp_ifindex != get_dpifindex(vport->dp))
			return ERR_PTR(-ENODEV);
1851 1852 1853 1854 1855 1856 1857
		return vport;
	} else if (a[OVS_VPORT_ATTR_PORT_NO]) {
		u32 port_no = nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]);

		if (port_no >= DP_MAX_PORTS)
			return ERR_PTR(-EFBIG);

1858
		dp = get_dp(net, ovs_header->dp_ifindex);
1859 1860 1861
		if (!dp)
			return ERR_PTR(-ENODEV);

1862
		vport = ovs_vport_ovsl_rcu(dp, port_no);
1863
		if (!vport)
1864
			return ERR_PTR(-ENODEV);
1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882
		return vport;
	} else
		return ERR_PTR(-EINVAL);
}

static int ovs_vport_cmd_new(struct sk_buff *skb, struct genl_info *info)
{
	struct nlattr **a = info->attrs;
	struct ovs_header *ovs_header = info->userhdr;
	struct vport_parms parms;
	struct sk_buff *reply;
	struct vport *vport;
	struct datapath *dp;
	u32 port_no;
	int err;

	if (!a[OVS_VPORT_ATTR_NAME] || !a[OVS_VPORT_ATTR_TYPE] ||
	    !a[OVS_VPORT_ATTR_UPCALL_PID])
1883 1884 1885 1886 1887 1888 1889 1890 1891 1892
		return -EINVAL;

	port_no = a[OVS_VPORT_ATTR_PORT_NO]
		? nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]) : 0;
	if (port_no >= DP_MAX_PORTS)
		return -EFBIG;

	reply = ovs_vport_cmd_alloc_info();
	if (!reply)
		return -ENOMEM;
1893

1894
	ovs_lock();
1895
restart:
1896
	dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
1897 1898
	err = -ENODEV;
	if (!dp)
1899
		goto exit_unlock_free;
1900

1901
	if (port_no) {
1902
		vport = ovs_vport_ovsl(dp, port_no);
1903 1904
		err = -EBUSY;
		if (vport)
1905
			goto exit_unlock_free;
1906 1907 1908 1909
	} else {
		for (port_no = 1; ; port_no++) {
			if (port_no >= DP_MAX_PORTS) {
				err = -EFBIG;
1910
				goto exit_unlock_free;
1911
			}
1912
			vport = ovs_vport_ovsl(dp, port_no);
1913 1914 1915 1916 1917 1918 1919 1920 1921 1922
			if (!vport)
				break;
		}
	}

	parms.name = nla_data(a[OVS_VPORT_ATTR_NAME]);
	parms.type = nla_get_u32(a[OVS_VPORT_ATTR_TYPE]);
	parms.options = a[OVS_VPORT_ATTR_OPTIONS];
	parms.dp = dp;
	parms.port_no = port_no;
1923
	parms.upcall_portids = a[OVS_VPORT_ATTR_UPCALL_PID];
1924 1925 1926

	vport = new_vport(&parms);
	err = PTR_ERR(vport);
1927 1928 1929
	if (IS_ERR(vport)) {
		if (err == -EAGAIN)
			goto restart;
1930
		goto exit_unlock_free;
1931
	}
1932

1933 1934 1935 1936
	err = ovs_vport_cmd_fill_info(vport, reply, info->snd_portid,
				      info->snd_seq, 0, OVS_VPORT_CMD_NEW);
	BUG_ON(err < 0);
	ovs_unlock();
1937

1938
	ovs_notify(&dp_vport_genl_family, reply, info);
1939
	return 0;
1940

1941
exit_unlock_free:
1942
	ovs_unlock();
1943
	kfree_skb(reply);
1944 1945 1946 1947 1948 1949 1950 1951 1952 1953
	return err;
}

static int ovs_vport_cmd_set(struct sk_buff *skb, struct genl_info *info)
{
	struct nlattr **a = info->attrs;
	struct sk_buff *reply;
	struct vport *vport;
	int err;

1954 1955 1956 1957
	reply = ovs_vport_cmd_alloc_info();
	if (!reply)
		return -ENOMEM;

1958
	ovs_lock();
1959
	vport = lookup_vport(sock_net(skb->sk), info->userhdr, a);
1960 1961
	err = PTR_ERR(vport);
	if (IS_ERR(vport))
1962
		goto exit_unlock_free;
1963 1964

	if (a[OVS_VPORT_ATTR_TYPE] &&
1965
	    nla_get_u32(a[OVS_VPORT_ATTR_TYPE]) != vport->ops->type) {
1966
		err = -EINVAL;
1967
		goto exit_unlock_free;
1968 1969
	}

1970
	if (a[OVS_VPORT_ATTR_OPTIONS]) {
1971
		err = ovs_vport_set_options(vport, a[OVS_VPORT_ATTR_OPTIONS]);
1972
		if (err)
1973
			goto exit_unlock_free;
1974
	}
1975

1976 1977 1978 1979 1980 1981 1982 1983

	if (a[OVS_VPORT_ATTR_UPCALL_PID]) {
		struct nlattr *ids = a[OVS_VPORT_ATTR_UPCALL_PID];

		err = ovs_vport_set_upcall_portids(vport, ids);
		if (err)
			goto exit_unlock_free;
	}
1984

1985 1986 1987
	err = ovs_vport_cmd_fill_info(vport, reply, info->snd_portid,
				      info->snd_seq, 0, OVS_VPORT_CMD_NEW);
	BUG_ON(err < 0);
1988

1989
	ovs_unlock();
1990
	ovs_notify(&dp_vport_genl_family, reply, info);
1991
	return 0;
1992

1993
exit_unlock_free:
1994
	ovs_unlock();
1995
	kfree_skb(reply);
1996 1997 1998 1999 2000 2001 2002 2003 2004 2005
	return err;
}

static int ovs_vport_cmd_del(struct sk_buff *skb, struct genl_info *info)
{
	struct nlattr **a = info->attrs;
	struct sk_buff *reply;
	struct vport *vport;
	int err;

2006 2007 2008 2009
	reply = ovs_vport_cmd_alloc_info();
	if (!reply)
		return -ENOMEM;

2010
	ovs_lock();
2011
	vport = lookup_vport(sock_net(skb->sk), info->userhdr, a);
2012 2013
	err = PTR_ERR(vport);
	if (IS_ERR(vport))
2014
		goto exit_unlock_free;
2015 2016 2017

	if (vport->port_no == OVSP_LOCAL) {
		err = -EINVAL;
2018
		goto exit_unlock_free;
2019 2020
	}

2021 2022 2023
	err = ovs_vport_cmd_fill_info(vport, reply, info->snd_portid,
				      info->snd_seq, 0, OVS_VPORT_CMD_DEL);
	BUG_ON(err < 0);
2024
	ovs_dp_detach_port(vport);
2025
	ovs_unlock();
2026

2027
	ovs_notify(&dp_vport_genl_family, reply, info);
2028
	return 0;
2029

2030
exit_unlock_free:
2031
	ovs_unlock();
2032
	kfree_skb(reply);
2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043
	return err;
}

static int ovs_vport_cmd_get(struct sk_buff *skb, struct genl_info *info)
{
	struct nlattr **a = info->attrs;
	struct ovs_header *ovs_header = info->userhdr;
	struct sk_buff *reply;
	struct vport *vport;
	int err;

2044 2045 2046 2047
	reply = ovs_vport_cmd_alloc_info();
	if (!reply)
		return -ENOMEM;

2048
	rcu_read_lock();
2049
	vport = lookup_vport(sock_net(skb->sk), ovs_header, a);
2050 2051
	err = PTR_ERR(vport);
	if (IS_ERR(vport))
2052 2053 2054 2055
		goto exit_unlock_free;
	err = ovs_vport_cmd_fill_info(vport, reply, info->snd_portid,
				      info->snd_seq, 0, OVS_VPORT_CMD_NEW);
	BUG_ON(err < 0);
2056 2057 2058 2059
	rcu_read_unlock();

	return genlmsg_reply(reply, info);

2060
exit_unlock_free:
2061
	rcu_read_unlock();
2062
	kfree_skb(reply);
2063 2064 2065 2066 2067 2068 2069
	return err;
}

static int ovs_vport_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
{
	struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
	struct datapath *dp;
2070 2071
	int bucket = cb->args[0], skip = cb->args[1];
	int i, j = 0;
2072

J
Jarno Rajahalme 已提交
2073
	rcu_read_lock();
2074
	dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex);
J
Jarno Rajahalme 已提交
2075 2076
	if (!dp) {
		rcu_read_unlock();
2077
		return -ENODEV;
J
Jarno Rajahalme 已提交
2078
	}
2079
	for (i = bucket; i < DP_VPORT_HASH_BUCKETS; i++) {
2080
		struct vport *vport;
2081 2082

		j = 0;
2083
		hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node) {
2084 2085
			if (j >= skip &&
			    ovs_vport_cmd_fill_info(vport, skb,
2086
						    NETLINK_CB(cb->skb).portid,
2087 2088 2089 2090 2091 2092 2093 2094
						    cb->nlh->nlmsg_seq,
						    NLM_F_MULTI,
						    OVS_VPORT_CMD_NEW) < 0)
				goto out;

			j++;
		}
		skip = 0;
2095
	}
2096
out:
2097 2098
	rcu_read_unlock();

2099 2100
	cb->args[0] = i;
	cb->args[1] = j;
2101

2102
	return skb->len;
2103 2104
}

2105 2106 2107 2108 2109 2110 2111 2112 2113
static const struct nla_policy vport_policy[OVS_VPORT_ATTR_MAX + 1] = {
	[OVS_VPORT_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
	[OVS_VPORT_ATTR_STATS] = { .len = sizeof(struct ovs_vport_stats) },
	[OVS_VPORT_ATTR_PORT_NO] = { .type = NLA_U32 },
	[OVS_VPORT_ATTR_TYPE] = { .type = NLA_U32 },
	[OVS_VPORT_ATTR_UPCALL_PID] = { .type = NLA_U32 },
	[OVS_VPORT_ATTR_OPTIONS] = { .type = NLA_NESTED },
};

2114
static const struct genl_ops dp_vport_genl_ops[] = {
2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137
	{ .cmd = OVS_VPORT_CMD_NEW,
	  .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
	  .policy = vport_policy,
	  .doit = ovs_vport_cmd_new
	},
	{ .cmd = OVS_VPORT_CMD_DEL,
	  .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
	  .policy = vport_policy,
	  .doit = ovs_vport_cmd_del
	},
	{ .cmd = OVS_VPORT_CMD_GET,
	  .flags = 0,		    /* OK for unprivileged users. */
	  .policy = vport_policy,
	  .doit = ovs_vport_cmd_get,
	  .dumpit = ovs_vport_cmd_dump
	},
	{ .cmd = OVS_VPORT_CMD_SET,
	  .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
	  .policy = vport_policy,
	  .doit = ovs_vport_cmd_set,
	},
};

2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149
struct genl_family dp_vport_genl_family = {
	.id = GENL_ID_GENERATE,
	.hdrsize = sizeof(struct ovs_header),
	.name = OVS_VPORT_FAMILY,
	.version = OVS_VPORT_VERSION,
	.maxattr = OVS_VPORT_ATTR_MAX,
	.netnsok = true,
	.parallel_ops = true,
	.ops = dp_vport_genl_ops,
	.n_ops = ARRAY_SIZE(dp_vport_genl_ops),
	.mcgrps = &ovs_dp_vport_multicast_group,
	.n_mcgrps = 1,
2150 2151
};

2152 2153 2154 2155 2156
static struct genl_family * const dp_genl_families[] = {
	&dp_datapath_genl_family,
	&dp_vport_genl_family,
	&dp_flow_genl_family,
	&dp_packet_genl_family,
2157 2158 2159 2160 2161 2162 2163
};

static void dp_unregister_genl(int n_families)
{
	int i;

	for (i = 0; i < n_families; i++)
2164
		genl_unregister_family(dp_genl_families[i]);
2165 2166 2167 2168 2169 2170 2171 2172 2173
}

static int dp_register_genl(void)
{
	int err;
	int i;

	for (i = 0; i < ARRAY_SIZE(dp_genl_families); i++) {

2174
		err = genl_register_family(dp_genl_families[i]);
2175 2176 2177 2178 2179 2180 2181
		if (err)
			goto error;
	}

	return 0;

error:
2182
	dp_unregister_genl(i);
2183 2184 2185
	return err;
}

2186 2187 2188 2189 2190
static int __net_init ovs_init_net(struct net *net)
{
	struct ovs_net *ovs_net = net_generic(net, ovs_net_id);

	INIT_LIST_HEAD(&ovs_net->dps);
2191
	INIT_WORK(&ovs_net->dp_notify_work, ovs_dp_notify_wq);
2192 2193 2194
	return 0;
}

P
Pravin B Shelar 已提交
2195 2196
static void __net_exit list_vports_from_net(struct net *net, struct net *dnet,
					    struct list_head *head)
2197
{
2198
	struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
P
Pravin B Shelar 已提交
2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227
	struct datapath *dp;

	list_for_each_entry(dp, &ovs_net->dps, list_node) {
		int i;

		for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
			struct vport *vport;

			hlist_for_each_entry(vport, &dp->ports[i], dp_hash_node) {
				struct netdev_vport *netdev_vport;

				if (vport->ops->type != OVS_VPORT_TYPE_INTERNAL)
					continue;

				netdev_vport = netdev_vport_priv(vport);
				if (dev_net(netdev_vport->dev) == dnet)
					list_add(&vport->detach_list, head);
			}
		}
	}
}

static void __net_exit ovs_exit_net(struct net *dnet)
{
	struct datapath *dp, *dp_next;
	struct ovs_net *ovs_net = net_generic(dnet, ovs_net_id);
	struct vport *vport, *vport_next;
	struct net *net;
	LIST_HEAD(head);
2228

2229
	ovs_lock();
2230 2231
	list_for_each_entry_safe(dp, dp_next, &ovs_net->dps, list_node)
		__dp_destroy(dp);
P
Pravin B Shelar 已提交
2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243

	rtnl_lock();
	for_each_net(net)
		list_vports_from_net(net, dnet, &head);
	rtnl_unlock();

	/* Detach all vports from given namespace. */
	list_for_each_entry_safe(vport, vport_next, &head, detach_list) {
		list_del(&vport->detach_list);
		ovs_dp_detach_port(vport);
	}

2244 2245 2246
	ovs_unlock();

	cancel_work_sync(&ovs_net->dp_notify_work);
2247 2248 2249 2250 2251 2252 2253 2254 2255
}

static struct pernet_operations ovs_net_ops = {
	.init = ovs_init_net,
	.exit = ovs_exit_net,
	.id   = &ovs_net_id,
	.size = sizeof(struct ovs_net),
};

2256 2257 2258 2259
static int __init dp_init(void)
{
	int err;

2260
	BUILD_BUG_ON(sizeof(struct ovs_skb_cb) > FIELD_SIZEOF(struct sk_buff, cb));
2261 2262 2263

	pr_info("Open vSwitch switching datapath\n");

2264
	err = action_fifos_init();
2265 2266 2267
	if (err)
		goto error;

2268 2269 2270 2271
	err = ovs_internal_dev_rtnl_link_register();
	if (err)
		goto error_action_fifos_exit;

2272 2273 2274 2275
	err = ovs_flow_init();
	if (err)
		goto error_unreg_rtnl_link;

2276 2277 2278 2279
	err = ovs_vport_init();
	if (err)
		goto error_flow_exit;

2280
	err = register_pernet_device(&ovs_net_ops);
2281 2282 2283
	if (err)
		goto error_vport_exit;

2284 2285 2286 2287
	err = register_netdevice_notifier(&ovs_dp_device_notifier);
	if (err)
		goto error_netns_exit;

2288 2289 2290 2291
	err = ovs_netdev_init();
	if (err)
		goto error_unreg_notifier;

2292 2293
	err = dp_register_genl();
	if (err < 0)
2294
		goto error_unreg_netdev;
2295 2296 2297

	return 0;

2298 2299
error_unreg_netdev:
	ovs_netdev_exit();
2300 2301
error_unreg_notifier:
	unregister_netdevice_notifier(&ovs_dp_device_notifier);
2302 2303
error_netns_exit:
	unregister_pernet_device(&ovs_net_ops);
2304 2305 2306 2307
error_vport_exit:
	ovs_vport_exit();
error_flow_exit:
	ovs_flow_exit();
2308 2309
error_unreg_rtnl_link:
	ovs_internal_dev_rtnl_link_unregister();
2310 2311
error_action_fifos_exit:
	action_fifos_exit();
2312 2313 2314 2315 2316 2317 2318
error:
	return err;
}

static void dp_cleanup(void)
{
	dp_unregister_genl(ARRAY_SIZE(dp_genl_families));
2319
	ovs_netdev_exit();
2320
	unregister_netdevice_notifier(&ovs_dp_device_notifier);
2321 2322
	unregister_pernet_device(&ovs_net_ops);
	rcu_barrier();
2323 2324
	ovs_vport_exit();
	ovs_flow_exit();
2325
	ovs_internal_dev_rtnl_link_unregister();
2326
	action_fifos_exit();
2327 2328 2329 2330 2331 2332 2333
}

module_init(dp_init);
module_exit(dp_cleanup);

MODULE_DESCRIPTION("Open vSwitch switching datapath");
MODULE_LICENSE("GPL");