diff --git a/drivers/net/Kconfig b/drivers/net/Kconfig index a993cbeb9e0c84326a63369226b14ed008702a13..823bc2fd201ff022f28f08800694fab19ae3c9e8 100644 --- a/drivers/net/Kconfig +++ b/drivers/net/Kconfig @@ -135,6 +135,7 @@ config MACVTAP tristate "MAC-VLAN based tap driver" depends on MACVLAN depends on INET + select TAP help This adds a specialized tap character device driver that is based on the MAC-VLAN network interface, called macvtap. A macvtap device @@ -165,6 +166,19 @@ config IPVLAN To compile this driver as a module, choose M here: the module will be called ipvlan. +config IPVTAP + tristate "IP-VLAN based tap driver" + depends on IPVLAN + depends on INET + select TAP + ---help--- + This adds a specialized tap character device driver that is based + on the IP-VLAN network interface, called ipvtap. An ipvtap device + can be added in the same way as a ipvlan device, using 'type + ipvtap', and then be accessed through the tap user space interface. + + To compile this driver as a module, choose M here: the module + will be called ipvtap. config VXLAN tristate "Virtual eXtensible Local Area Network (VXLAN)" @@ -287,6 +301,12 @@ config TUN If you don't know what to use this for, you don't need it. +config TAP + tristate + ---help--- + This option is selected by any driver implementing tap user space + interface for a virtual interface to re-use core tap functionality. + config TUN_VNET_CROSS_LE bool "Support for cross-endian vnet headers on little-endian kernels" default n diff --git a/drivers/net/Makefile b/drivers/net/Makefile index 7336cbd3ef5d94d4d28a6e2913e0e8439b205bf3..98ed4d96987c87fda074a219a234cbf8e48f2b9a 100644 --- a/drivers/net/Makefile +++ b/drivers/net/Makefile @@ -7,6 +7,7 @@ # obj-$(CONFIG_BONDING) += bonding/ obj-$(CONFIG_IPVLAN) += ipvlan/ +obj-$(CONFIG_IPVTAP) += ipvlan/ obj-$(CONFIG_DUMMY) += dummy.o obj-$(CONFIG_EQUALIZER) += eql.o obj-$(CONFIG_IFB) += ifb.o @@ -21,6 +22,7 @@ obj-$(CONFIG_PHYLIB) += phy/ obj-$(CONFIG_RIONET) += rionet.o obj-$(CONFIG_NET_TEAM) += team/ obj-$(CONFIG_TUN) += tun.o +obj-$(CONFIG_TAP) += tap.o obj-$(CONFIG_VETH) += veth.o obj-$(CONFIG_VIRTIO_NET) += virtio_net.o obj-$(CONFIG_VXLAN) += vxlan.o diff --git a/drivers/net/ipvlan/Makefile b/drivers/net/ipvlan/Makefile index df79910192d6c92b489c6c71f2f5fbdc795a7782..8a2c64dc964105d378b56be5a99477599e011b57 100644 --- a/drivers/net/ipvlan/Makefile +++ b/drivers/net/ipvlan/Makefile @@ -3,5 +3,6 @@ # obj-$(CONFIG_IPVLAN) += ipvlan.o +obj-$(CONFIG_IPVTAP) += ipvtap.o ipvlan-objs := ipvlan_core.o ipvlan_main.o diff --git a/drivers/net/ipvlan/ipvlan.h b/drivers/net/ipvlan/ipvlan.h index 406ae4ff0ae891aa6bd1522bbb367699f16109b1..800a46c8d26c25f74244ee30872097982ebc775e 100644 --- a/drivers/net/ipvlan/ipvlan.h +++ b/drivers/net/ipvlan/ipvlan.h @@ -135,4 +135,11 @@ struct sk_buff *ipvlan_l3_rcv(struct net_device *dev, struct sk_buff *skb, u16 proto); unsigned int ipvlan_nf_input(void *priv, struct sk_buff *skb, const struct nf_hook_state *state); +void ipvlan_count_rx(const struct ipvl_dev *ipvlan, + unsigned int len, bool success, bool mcast); +int ipvlan_link_new(struct net *src_net, struct net_device *dev, + struct nlattr *tb[], struct nlattr *data[]); +void ipvlan_link_delete(struct net_device *dev, struct list_head *head); +void ipvlan_link_setup(struct net_device *dev); +int ipvlan_link_register(struct rtnl_link_ops *ops); #endif /* __IPVLAN_H */ diff --git a/drivers/net/ipvlan/ipvlan_core.c b/drivers/net/ipvlan/ipvlan_core.c index 8ae335d73d3886a0f7a91f55a428059120288de6..1f3295e274d0f5fbb36211b71023320327a86f12 100644 --- a/drivers/net/ipvlan/ipvlan_core.c +++ b/drivers/net/ipvlan/ipvlan_core.c @@ -16,7 +16,7 @@ void ipvlan_init_secret(void) net_get_random_once(&ipvlan_jhash_secret, sizeof(ipvlan_jhash_secret)); } -static void ipvlan_count_rx(const struct ipvl_dev *ipvlan, +void ipvlan_count_rx(const struct ipvl_dev *ipvlan, unsigned int len, bool success, bool mcast) { if (likely(success)) { @@ -33,6 +33,7 @@ static void ipvlan_count_rx(const struct ipvl_dev *ipvlan, this_cpu_inc(ipvlan->pcpu_stats->rx_errs); } } +EXPORT_SYMBOL_GPL(ipvlan_count_rx); static u8 ipvlan_get_v6_hash(const void *iaddr) { diff --git a/drivers/net/ipvlan/ipvlan_main.c b/drivers/net/ipvlan/ipvlan_main.c index 95b18f4602cf1d9653fd740c56f8644d764189a4..aa8575ccbce392426ab1ceb65ecadee5175dcb1b 100644 --- a/drivers/net/ipvlan/ipvlan_main.c +++ b/drivers/net/ipvlan/ipvlan_main.c @@ -496,8 +496,8 @@ static int ipvlan_nl_fillinfo(struct sk_buff *skb, return ret; } -static int ipvlan_link_new(struct net *src_net, struct net_device *dev, - struct nlattr *tb[], struct nlattr *data[]) +int ipvlan_link_new(struct net *src_net, struct net_device *dev, + struct nlattr *tb[], struct nlattr *data[]) { struct ipvl_dev *ipvlan = netdev_priv(dev); struct ipvl_port *port; @@ -594,8 +594,9 @@ static int ipvlan_link_new(struct net *src_net, struct net_device *dev, ipvlan_port_destroy(phy_dev); return err; } +EXPORT_SYMBOL_GPL(ipvlan_link_new); -static void ipvlan_link_delete(struct net_device *dev, struct list_head *head) +void ipvlan_link_delete(struct net_device *dev, struct list_head *head) { struct ipvl_dev *ipvlan = netdev_priv(dev); struct ipvl_addr *addr, *next; @@ -611,8 +612,9 @@ static void ipvlan_link_delete(struct net_device *dev, struct list_head *head) unregister_netdevice_queue(dev, head); netdev_upper_dev_unlink(ipvlan->phy_dev, dev); } +EXPORT_SYMBOL_GPL(ipvlan_link_delete); -static void ipvlan_link_setup(struct net_device *dev) +void ipvlan_link_setup(struct net_device *dev) { ether_setup(dev); @@ -623,6 +625,7 @@ static void ipvlan_link_setup(struct net_device *dev) dev->header_ops = &ipvlan_header_ops; dev->ethtool_ops = &ipvlan_ethtool_ops; } +EXPORT_SYMBOL_GPL(ipvlan_link_setup); static const struct nla_policy ipvlan_nl_policy[IFLA_IPVLAN_MAX + 1] = { @@ -633,22 +636,22 @@ static struct rtnl_link_ops ipvlan_link_ops = { .kind = "ipvlan", .priv_size = sizeof(struct ipvl_dev), - .get_size = ipvlan_nl_getsize, - .policy = ipvlan_nl_policy, - .validate = ipvlan_nl_validate, - .fill_info = ipvlan_nl_fillinfo, - .changelink = ipvlan_nl_changelink, - .maxtype = IFLA_IPVLAN_MAX, - .setup = ipvlan_link_setup, .newlink = ipvlan_link_new, .dellink = ipvlan_link_delete, }; -static int ipvlan_link_register(struct rtnl_link_ops *ops) +int ipvlan_link_register(struct rtnl_link_ops *ops) { + ops->get_size = ipvlan_nl_getsize; + ops->policy = ipvlan_nl_policy; + ops->validate = ipvlan_nl_validate; + ops->fill_info = ipvlan_nl_fillinfo; + ops->changelink = ipvlan_nl_changelink; + ops->maxtype = IFLA_IPVLAN_MAX; return rtnl_link_register(ops); } +EXPORT_SYMBOL_GPL(ipvlan_link_register); static int ipvlan_device_event(struct notifier_block *unused, unsigned long event, void *ptr) diff --git a/drivers/net/ipvlan/ipvtap.c b/drivers/net/ipvlan/ipvtap.c new file mode 100644 index 0000000000000000000000000000000000000000..2b713b63b62c2d3edfce1128ffd417444f566c51 --- /dev/null +++ b/drivers/net/ipvlan/ipvtap.c @@ -0,0 +1,241 @@ +#include +#include "ipvlan.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#define TUN_OFFLOADS (NETIF_F_HW_CSUM | NETIF_F_TSO_ECN | NETIF_F_TSO | \ + NETIF_F_TSO6 | NETIF_F_UFO) + +static dev_t ipvtap_major; +static struct cdev ipvtap_cdev; + +static const void *ipvtap_net_namespace(struct device *d) +{ + struct net_device *dev = to_net_dev(d->parent); + return dev_net(dev); +} + +static struct class ipvtap_class = { + .name = "ipvtap", + .owner = THIS_MODULE, + .ns_type = &net_ns_type_operations, + .namespace = ipvtap_net_namespace, +}; + +struct ipvtap_dev { + struct ipvl_dev vlan; + struct tap_dev tap; +}; + +static void ipvtap_count_tx_dropped(struct tap_dev *tap) +{ + struct ipvtap_dev *vlantap = container_of(tap, struct ipvtap_dev, tap); + struct ipvl_dev *vlan = &vlantap->vlan; + + this_cpu_inc(vlan->pcpu_stats->tx_drps); +} + +static void ipvtap_count_rx_dropped(struct tap_dev *tap) +{ + struct ipvtap_dev *vlantap = container_of(tap, struct ipvtap_dev, tap); + struct ipvl_dev *vlan = &vlantap->vlan; + + ipvlan_count_rx(vlan, 0, 0, 0); +} + +static void ipvtap_update_features(struct tap_dev *tap, + netdev_features_t features) +{ + struct ipvtap_dev *vlantap = container_of(tap, struct ipvtap_dev, tap); + struct ipvl_dev *vlan = &vlantap->vlan; + + vlan->sfeatures = features; + netdev_update_features(vlan->dev); +} + +static int ipvtap_newlink(struct net *src_net, + struct net_device *dev, + struct nlattr *tb[], + struct nlattr *data[]) +{ + struct ipvtap_dev *vlantap = netdev_priv(dev); + int err; + + INIT_LIST_HEAD(&vlantap->tap.queue_list); + + /* Since macvlan supports all offloads by default, make + * tap support all offloads also. + */ + vlantap->tap.tap_features = TUN_OFFLOADS; + vlantap->tap.count_tx_dropped = ipvtap_count_tx_dropped; + vlantap->tap.update_features = ipvtap_update_features; + vlantap->tap.count_rx_dropped = ipvtap_count_rx_dropped; + + err = netdev_rx_handler_register(dev, tap_handle_frame, &vlantap->tap); + if (err) + return err; + + /* Don't put anything that may fail after macvlan_common_newlink + * because we can't undo what it does. + */ + err = ipvlan_link_new(src_net, dev, tb, data); + if (err) { + netdev_rx_handler_unregister(dev); + return err; + } + + vlantap->tap.dev = vlantap->vlan.dev; + + return err; +} + +static void ipvtap_dellink(struct net_device *dev, + struct list_head *head) +{ + struct ipvtap_dev *vlan = netdev_priv(dev); + + netdev_rx_handler_unregister(dev); + tap_del_queues(&vlan->tap); + ipvlan_link_delete(dev, head); +} + +static void ipvtap_setup(struct net_device *dev) +{ + ipvlan_link_setup(dev); + dev->tx_queue_len = TUN_READQ_SIZE; + dev->priv_flags &= ~IFF_NO_QUEUE; +} + +static struct rtnl_link_ops ipvtap_link_ops __read_mostly = { + .kind = "ipvtap", + .setup = ipvtap_setup, + .newlink = ipvtap_newlink, + .dellink = ipvtap_dellink, + .priv_size = sizeof(struct ipvtap_dev), +}; + +static int ipvtap_device_event(struct notifier_block *unused, + unsigned long event, void *ptr) +{ + struct net_device *dev = netdev_notifier_info_to_dev(ptr); + struct ipvtap_dev *vlantap; + struct device *classdev; + dev_t devt; + int err; + char tap_name[IFNAMSIZ]; + + if (dev->rtnl_link_ops != &ipvtap_link_ops) + return NOTIFY_DONE; + + snprintf(tap_name, IFNAMSIZ, "tap%d", dev->ifindex); + vlantap = netdev_priv(dev); + + switch (event) { + case NETDEV_REGISTER: + /* Create the device node here after the network device has + * been registered but before register_netdevice has + * finished running. + */ + err = tap_get_minor(ipvtap_major, &vlantap->tap); + if (err) + return notifier_from_errno(err); + + devt = MKDEV(MAJOR(ipvtap_major), vlantap->tap.minor); + classdev = device_create(&ipvtap_class, &dev->dev, devt, + dev, tap_name); + if (IS_ERR(classdev)) { + tap_free_minor(ipvtap_major, &vlantap->tap); + return notifier_from_errno(PTR_ERR(classdev)); + } + err = sysfs_create_link(&dev->dev.kobj, &classdev->kobj, + tap_name); + if (err) + return notifier_from_errno(err); + break; + case NETDEV_UNREGISTER: + /* vlan->minor == 0 if NETDEV_REGISTER above failed */ + if (vlantap->tap.minor == 0) + break; + sysfs_remove_link(&dev->dev.kobj, tap_name); + devt = MKDEV(MAJOR(ipvtap_major), vlantap->tap.minor); + device_destroy(&ipvtap_class, devt); + tap_free_minor(ipvtap_major, &vlantap->tap); + break; + case NETDEV_CHANGE_TX_QUEUE_LEN: + if (tap_queue_resize(&vlantap->tap)) + return NOTIFY_BAD; + break; + } + + return NOTIFY_DONE; +} + +static struct notifier_block ipvtap_notifier_block __read_mostly = { + .notifier_call = ipvtap_device_event, +}; + +static int ipvtap_init(void) +{ + int err; + + err = tap_create_cdev(&ipvtap_cdev, &ipvtap_major, "ipvtap"); + + if (err) + goto out1; + + err = class_register(&ipvtap_class); + if (err) + goto out2; + + err = register_netdevice_notifier(&ipvtap_notifier_block); + if (err) + goto out3; + + err = ipvlan_link_register(&ipvtap_link_ops); + if (err) + goto out4; + + return 0; + +out4: + unregister_netdevice_notifier(&ipvtap_notifier_block); +out3: + class_unregister(&ipvtap_class); +out2: + tap_destroy_cdev(ipvtap_major, &ipvtap_cdev); +out1: + return err; +} +module_init(ipvtap_init); + +static void ipvtap_exit(void) +{ + rtnl_link_unregister(&ipvtap_link_ops); + unregister_netdevice_notifier(&ipvtap_notifier_block); + class_unregister(&ipvtap_class); + tap_destroy_cdev(ipvtap_major, &ipvtap_cdev); +} +module_exit(ipvtap_exit); +MODULE_ALIAS_RTNL_LINK("ipvtap"); +MODULE_AUTHOR("Sainath Grandhi "); +MODULE_LICENSE("GPL"); diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c index cbfc1be23a0e33c9b45f86798f459fa35bb4e85e..9261722960a719a8e6d46f4ea5b39f500a0817e8 100644 --- a/drivers/net/macvlan.c +++ b/drivers/net/macvlan.c @@ -1525,7 +1525,6 @@ static const struct nla_policy macvlan_policy[IFLA_MACVLAN_MAX + 1] = { int macvlan_link_register(struct rtnl_link_ops *ops) { /* common fields */ - ops->priv_size = sizeof(struct macvlan_dev); ops->validate = macvlan_validate; ops->maxtype = IFLA_MACVLAN_MAX; ops->policy = macvlan_policy; @@ -1548,6 +1547,7 @@ static struct rtnl_link_ops macvlan_link_ops = { .newlink = macvlan_newlink, .dellink = macvlan_dellink, .get_link_net = macvlan_get_link_net, + .priv_size = sizeof(struct macvlan_dev), }; static int macvlan_device_event(struct notifier_block *unused, diff --git a/drivers/net/macvtap.c b/drivers/net/macvtap.c index c27011bbe30c52d2eb892ab0d86f8cf3d6f4deb9..a4bfc10b61ddd7872dfa89634f20478be1e2f29a 100644 --- a/drivers/net/macvtap.c +++ b/drivers/net/macvtap.c @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -23,114 +24,16 @@ #include #include -/* - * A macvtap queue is the central object of this driver, it connects - * an open character device to a macvlan interface. There can be - * multiple queues on one interface, which map back to queues - * implemented in hardware on the underlying device. - * - * macvtap_proto is used to allocate queues through the sock allocation - * mechanism. - * - */ -struct macvtap_queue { - struct sock sk; - struct socket sock; - struct socket_wq wq; - int vnet_hdr_sz; - struct macvlan_dev __rcu *vlan; - struct file *file; - unsigned int flags; - u16 queue_index; - bool enabled; - struct list_head next; - struct skb_array skb_array; -}; - -#define MACVTAP_FEATURES (IFF_VNET_HDR | IFF_MULTI_QUEUE) - -#define MACVTAP_VNET_LE 0x80000000 -#define MACVTAP_VNET_BE 0x40000000 - -#ifdef CONFIG_TUN_VNET_CROSS_LE -static inline bool macvtap_legacy_is_little_endian(struct macvtap_queue *q) -{ - return q->flags & MACVTAP_VNET_BE ? false : - virtio_legacy_is_little_endian(); -} - -static long macvtap_get_vnet_be(struct macvtap_queue *q, int __user *sp) -{ - int s = !!(q->flags & MACVTAP_VNET_BE); - - if (put_user(s, sp)) - return -EFAULT; - - return 0; -} - -static long macvtap_set_vnet_be(struct macvtap_queue *q, int __user *sp) -{ - int s; - - if (get_user(s, sp)) - return -EFAULT; - - if (s) - q->flags |= MACVTAP_VNET_BE; - else - q->flags &= ~MACVTAP_VNET_BE; - - return 0; -} -#else -static inline bool macvtap_legacy_is_little_endian(struct macvtap_queue *q) -{ - return virtio_legacy_is_little_endian(); -} - -static long macvtap_get_vnet_be(struct macvtap_queue *q, int __user *argp) -{ - return -EINVAL; -} - -static long macvtap_set_vnet_be(struct macvtap_queue *q, int __user *argp) -{ - return -EINVAL; -} -#endif /* CONFIG_TUN_VNET_CROSS_LE */ - -static inline bool macvtap_is_little_endian(struct macvtap_queue *q) -{ - return q->flags & MACVTAP_VNET_LE || - macvtap_legacy_is_little_endian(q); -} - -static inline u16 macvtap16_to_cpu(struct macvtap_queue *q, __virtio16 val) -{ - return __virtio16_to_cpu(macvtap_is_little_endian(q), val); -} - -static inline __virtio16 cpu_to_macvtap16(struct macvtap_queue *q, u16 val) -{ - return __cpu_to_virtio16(macvtap_is_little_endian(q), val); -} - -static struct proto macvtap_proto = { - .name = "macvtap", - .owner = THIS_MODULE, - .obj_size = sizeof (struct macvtap_queue), +struct macvtap_dev { + struct macvlan_dev vlan; + struct tap_dev tap; }; /* * Variables for dealing with macvtaps device numbers. */ static dev_t macvtap_major; -#define MACVTAP_NUM_DEVS (1U << MINORBITS) -static DEFINE_MUTEX(minor_lock); -static DEFINE_IDR(minor_idr); -#define GOODCOPY_LEN 128 static const void *macvtap_net_namespace(struct device *d) { struct net_device *dev = to_net_dev(d->parent); @@ -145,328 +48,33 @@ static struct class macvtap_class = { }; static struct cdev macvtap_cdev; -static const struct proto_ops macvtap_socket_ops; - #define TUN_OFFLOADS (NETIF_F_HW_CSUM | NETIF_F_TSO_ECN | NETIF_F_TSO | \ NETIF_F_TSO6 | NETIF_F_UFO) -#define RX_OFFLOADS (NETIF_F_GRO | NETIF_F_LRO) -#define TAP_FEATURES (NETIF_F_GSO | NETIF_F_SG | NETIF_F_FRAGLIST) - -static struct macvlan_dev *macvtap_get_vlan_rcu(const struct net_device *dev) -{ - return rcu_dereference(dev->rx_handler_data); -} - -/* - * RCU usage: - * The macvtap_queue and the macvlan_dev are loosely coupled, the - * pointers from one to the other can only be read while rcu_read_lock - * or rtnl is held. - * - * Both the file and the macvlan_dev hold a reference on the macvtap_queue - * through sock_hold(&q->sk). When the macvlan_dev goes away first, - * q->vlan becomes inaccessible. When the files gets closed, - * macvtap_get_queue() fails. - * - * There may still be references to the struct sock inside of the - * queue from outbound SKBs, but these never reference back to the - * file or the dev. The data structure is freed through __sk_free - * when both our references and any pending SKBs are gone. - */ - -static int macvtap_enable_queue(struct net_device *dev, struct file *file, - struct macvtap_queue *q) -{ - struct macvlan_dev *vlan = netdev_priv(dev); - int err = -EINVAL; - - ASSERT_RTNL(); - - if (q->enabled) - goto out; - - err = 0; - rcu_assign_pointer(vlan->taps[vlan->numvtaps], q); - q->queue_index = vlan->numvtaps; - q->enabled = true; - - vlan->numvtaps++; -out: - return err; -} - -/* Requires RTNL */ -static int macvtap_set_queue(struct net_device *dev, struct file *file, - struct macvtap_queue *q) -{ - struct macvlan_dev *vlan = netdev_priv(dev); - - if (vlan->numqueues == MAX_MACVTAP_QUEUES) - return -EBUSY; - - rcu_assign_pointer(q->vlan, vlan); - rcu_assign_pointer(vlan->taps[vlan->numvtaps], q); - sock_hold(&q->sk); - - q->file = file; - q->queue_index = vlan->numvtaps; - q->enabled = true; - file->private_data = q; - list_add_tail(&q->next, &vlan->queue_list); - - vlan->numvtaps++; - vlan->numqueues++; - - return 0; -} - -static int macvtap_disable_queue(struct macvtap_queue *q) -{ - struct macvlan_dev *vlan; - struct macvtap_queue *nq; - - ASSERT_RTNL(); - if (!q->enabled) - return -EINVAL; - - vlan = rtnl_dereference(q->vlan); - - if (vlan) { - int index = q->queue_index; - BUG_ON(index >= vlan->numvtaps); - nq = rtnl_dereference(vlan->taps[vlan->numvtaps - 1]); - nq->queue_index = index; - - rcu_assign_pointer(vlan->taps[index], nq); - RCU_INIT_POINTER(vlan->taps[vlan->numvtaps - 1], NULL); - q->enabled = false; - - vlan->numvtaps--; - } - - return 0; -} - -/* - * The file owning the queue got closed, give up both - * the reference that the files holds as well as the - * one from the macvlan_dev if that still exists. - * - * Using the spinlock makes sure that we don't get - * to the queue again after destroying it. - */ -static void macvtap_put_queue(struct macvtap_queue *q) -{ - struct macvlan_dev *vlan; - - rtnl_lock(); - vlan = rtnl_dereference(q->vlan); - - if (vlan) { - if (q->enabled) - BUG_ON(macvtap_disable_queue(q)); - - vlan->numqueues--; - RCU_INIT_POINTER(q->vlan, NULL); - sock_put(&q->sk); - list_del_init(&q->next); - } - - rtnl_unlock(); - - synchronize_rcu(); - sock_put(&q->sk); -} - -/* - * Select a queue based on the rxq of the device on which this packet - * arrived. If the incoming device is not mq, calculate a flow hash - * to select a queue. If all fails, find the first available queue. - * Cache vlan->numvtaps since it can become zero during the execution - * of this function. - */ -static struct macvtap_queue *macvtap_get_queue(struct net_device *dev, - struct sk_buff *skb) -{ - struct macvlan_dev *vlan = netdev_priv(dev); - struct macvtap_queue *tap = NULL; - /* Access to taps array is protected by rcu, but access to numvtaps - * isn't. Below we use it to lookup a queue, but treat it as a hint - * and validate that the result isn't NULL - in case we are - * racing against queue removal. - */ - int numvtaps = ACCESS_ONCE(vlan->numvtaps); - __u32 rxq; - - if (!numvtaps) - goto out; - - if (numvtaps == 1) - goto single; - - /* Check if we can use flow to select a queue */ - rxq = skb_get_hash(skb); - if (rxq) { - tap = rcu_dereference(vlan->taps[rxq % numvtaps]); - goto out; - } - - if (likely(skb_rx_queue_recorded(skb))) { - rxq = skb_get_rx_queue(skb); - - while (unlikely(rxq >= numvtaps)) - rxq -= numvtaps; - tap = rcu_dereference(vlan->taps[rxq]); - goto out; - } - -single: - tap = rcu_dereference(vlan->taps[0]); -out: - return tap; -} - -/* - * The net_device is going away, give up the reference - * that it holds on all queues and safely set the pointer - * from the queues to NULL. - */ -static void macvtap_del_queues(struct net_device *dev) +static void macvtap_count_tx_dropped(struct tap_dev *tap) { - struct macvlan_dev *vlan = netdev_priv(dev); - struct macvtap_queue *q, *tmp; + struct macvtap_dev *vlantap = container_of(tap, struct macvtap_dev, tap); + struct macvlan_dev *vlan = &vlantap->vlan; - ASSERT_RTNL(); - list_for_each_entry_safe(q, tmp, &vlan->queue_list, next) { - list_del_init(&q->next); - RCU_INIT_POINTER(q->vlan, NULL); - if (q->enabled) - vlan->numvtaps--; - vlan->numqueues--; - sock_put(&q->sk); - } - BUG_ON(vlan->numvtaps); - BUG_ON(vlan->numqueues); - /* guarantee that any future macvtap_set_queue will fail */ - vlan->numvtaps = MAX_MACVTAP_QUEUES; + this_cpu_inc(vlan->pcpu_stats->tx_dropped); } -static rx_handler_result_t macvtap_handle_frame(struct sk_buff **pskb) +static void macvtap_count_rx_dropped(struct tap_dev *tap) { - struct sk_buff *skb = *pskb; - struct net_device *dev = skb->dev; - struct macvlan_dev *vlan; - struct macvtap_queue *q; - netdev_features_t features = TAP_FEATURES; - - vlan = macvtap_get_vlan_rcu(dev); - if (!vlan) - return RX_HANDLER_PASS; - - q = macvtap_get_queue(dev, skb); - if (!q) - return RX_HANDLER_PASS; - - if (__skb_array_full(&q->skb_array)) - goto drop; - - skb_push(skb, ETH_HLEN); - - /* Apply the forward feature mask so that we perform segmentation - * according to users wishes. This only works if VNET_HDR is - * enabled. - */ - if (q->flags & IFF_VNET_HDR) - features |= vlan->tap_features; - if (netif_needs_gso(skb, features)) { - struct sk_buff *segs = __skb_gso_segment(skb, features, false); - - if (IS_ERR(segs)) - goto drop; + struct macvtap_dev *vlantap = container_of(tap, struct macvtap_dev, tap); + struct macvlan_dev *vlan = &vlantap->vlan; - if (!segs) { - if (skb_array_produce(&q->skb_array, skb)) - goto drop; - goto wake_up; - } - - consume_skb(skb); - while (segs) { - struct sk_buff *nskb = segs->next; - - segs->next = NULL; - if (skb_array_produce(&q->skb_array, segs)) { - kfree_skb(segs); - kfree_skb_list(nskb); - break; - } - segs = nskb; - } - } else { - /* If we receive a partial checksum and the tap side - * doesn't support checksum offload, compute the checksum. - * Note: it doesn't matter which checksum feature to - * check, we either support them all or none. - */ - if (skb->ip_summed == CHECKSUM_PARTIAL && - !(features & NETIF_F_CSUM_MASK) && - skb_checksum_help(skb)) - goto drop; - if (skb_array_produce(&q->skb_array, skb)) - goto drop; - } - -wake_up: - wake_up_interruptible_poll(sk_sleep(&q->sk), POLLIN | POLLRDNORM | POLLRDBAND); - return RX_HANDLER_CONSUMED; - -drop: - /* Count errors/drops only here, thus don't care about args. */ macvlan_count_rx(vlan, 0, 0, 0); - kfree_skb(skb); - return RX_HANDLER_CONSUMED; -} - -static int macvtap_get_minor(struct macvlan_dev *vlan) -{ - int retval = -ENOMEM; - - mutex_lock(&minor_lock); - retval = idr_alloc(&minor_idr, vlan, 1, MACVTAP_NUM_DEVS, GFP_KERNEL); - if (retval >= 0) { - vlan->minor = retval; - } else if (retval == -ENOSPC) { - netdev_err(vlan->dev, "Too many macvtap devices\n"); - retval = -EINVAL; - } - mutex_unlock(&minor_lock); - return retval < 0 ? retval : 0; -} - -static void macvtap_free_minor(struct macvlan_dev *vlan) -{ - mutex_lock(&minor_lock); - if (vlan->minor) { - idr_remove(&minor_idr, vlan->minor); - vlan->minor = 0; - } - mutex_unlock(&minor_lock); } -static struct net_device *dev_get_by_macvtap_minor(int minor) +static void macvtap_update_features(struct tap_dev *tap, + netdev_features_t features) { - struct net_device *dev = NULL; - struct macvlan_dev *vlan; + struct macvtap_dev *vlantap = container_of(tap, struct macvtap_dev, tap); + struct macvlan_dev *vlan = &vlantap->vlan; - mutex_lock(&minor_lock); - vlan = idr_find(&minor_idr, minor); - if (vlan) { - dev = vlan->dev; - dev_hold(dev); - } - mutex_unlock(&minor_lock); - return dev; + vlan->set_features = features; + netdev_update_features(vlan->dev); } static int macvtap_newlink(struct net *src_net, @@ -474,17 +82,24 @@ static int macvtap_newlink(struct net *src_net, struct nlattr *tb[], struct nlattr *data[]) { - struct macvlan_dev *vlan = netdev_priv(dev); + struct macvtap_dev *vlantap = netdev_priv(dev); int err; - INIT_LIST_HEAD(&vlan->queue_list); + INIT_LIST_HEAD(&vlantap->tap.queue_list); /* Since macvlan supports all offloads by default, make * tap support all offloads also. */ - vlan->tap_features = TUN_OFFLOADS; + vlantap->tap.tap_features = TUN_OFFLOADS; + + /* Register callbacks for rx/tx drops accounting and updating + * net_device features + */ + vlantap->tap.count_tx_dropped = macvtap_count_tx_dropped; + vlantap->tap.count_rx_dropped = macvtap_count_rx_dropped; + vlantap->tap.update_features = macvtap_update_features; - err = netdev_rx_handler_register(dev, macvtap_handle_frame, vlan); + err = netdev_rx_handler_register(dev, tap_handle_frame, &vlantap->tap); if (err) return err; @@ -497,14 +112,18 @@ static int macvtap_newlink(struct net *src_net, return err; } + vlantap->tap.dev = vlantap->vlan.dev; + return 0; } static void macvtap_dellink(struct net_device *dev, struct list_head *head) { + struct macvtap_dev *vlantap = netdev_priv(dev); + netdev_rx_handler_unregister(dev); - macvtap_del_queues(dev); + tap_del_queues(&vlantap->tap); macvlan_dellink(dev, head); } @@ -519,749 +138,14 @@ static struct rtnl_link_ops macvtap_link_ops __read_mostly = { .setup = macvtap_setup, .newlink = macvtap_newlink, .dellink = macvtap_dellink, + .priv_size = sizeof(struct macvtap_dev), }; - -static void macvtap_sock_write_space(struct sock *sk) -{ - wait_queue_head_t *wqueue; - - if (!sock_writeable(sk) || - !test_and_clear_bit(SOCKWQ_ASYNC_NOSPACE, &sk->sk_socket->flags)) - return; - - wqueue = sk_sleep(sk); - if (wqueue && waitqueue_active(wqueue)) - wake_up_interruptible_poll(wqueue, POLLOUT | POLLWRNORM | POLLWRBAND); -} - -static void macvtap_sock_destruct(struct sock *sk) -{ - struct macvtap_queue *q = container_of(sk, struct macvtap_queue, sk); - - skb_array_cleanup(&q->skb_array); -} - -static int macvtap_open(struct inode *inode, struct file *file) -{ - struct net *net = current->nsproxy->net_ns; - struct net_device *dev; - struct macvtap_queue *q; - int err = -ENODEV; - - rtnl_lock(); - dev = dev_get_by_macvtap_minor(iminor(inode)); - if (!dev) - goto err; - - err = -ENOMEM; - q = (struct macvtap_queue *)sk_alloc(net, AF_UNSPEC, GFP_KERNEL, - &macvtap_proto, 0); - if (!q) - goto err; - - RCU_INIT_POINTER(q->sock.wq, &q->wq); - init_waitqueue_head(&q->wq.wait); - q->sock.type = SOCK_RAW; - q->sock.state = SS_CONNECTED; - q->sock.file = file; - q->sock.ops = &macvtap_socket_ops; - sock_init_data(&q->sock, &q->sk); - q->sk.sk_write_space = macvtap_sock_write_space; - q->sk.sk_destruct = macvtap_sock_destruct; - q->flags = IFF_VNET_HDR | IFF_NO_PI | IFF_TAP; - q->vnet_hdr_sz = sizeof(struct virtio_net_hdr); - - /* - * so far only KVM virtio_net uses macvtap, enable zero copy between - * guest kernel and host kernel when lower device supports zerocopy - * - * The macvlan supports zerocopy iff the lower device supports zero - * copy so we don't have to look at the lower device directly. - */ - if ((dev->features & NETIF_F_HIGHDMA) && (dev->features & NETIF_F_SG)) - sock_set_flag(&q->sk, SOCK_ZEROCOPY); - - err = -ENOMEM; - if (skb_array_init(&q->skb_array, dev->tx_queue_len, GFP_KERNEL)) - goto err_array; - - err = macvtap_set_queue(dev, file, q); - if (err) - goto err_queue; - - dev_put(dev); - - rtnl_unlock(); - return err; - -err_queue: - skb_array_cleanup(&q->skb_array); -err_array: - sock_put(&q->sk); -err: - if (dev) - dev_put(dev); - - rtnl_unlock(); - return err; -} - -static int macvtap_release(struct inode *inode, struct file *file) -{ - struct macvtap_queue *q = file->private_data; - macvtap_put_queue(q); - return 0; -} - -static unsigned int macvtap_poll(struct file *file, poll_table * wait) -{ - struct macvtap_queue *q = file->private_data; - unsigned int mask = POLLERR; - - if (!q) - goto out; - - mask = 0; - poll_wait(file, &q->wq.wait, wait); - - if (!skb_array_empty(&q->skb_array)) - mask |= POLLIN | POLLRDNORM; - - if (sock_writeable(&q->sk) || - (!test_and_set_bit(SOCKWQ_ASYNC_NOSPACE, &q->sock.flags) && - sock_writeable(&q->sk))) - mask |= POLLOUT | POLLWRNORM; - -out: - return mask; -} - -static inline struct sk_buff *macvtap_alloc_skb(struct sock *sk, size_t prepad, - size_t len, size_t linear, - int noblock, int *err) -{ - struct sk_buff *skb; - - /* Under a page? Don't bother with paged skb. */ - if (prepad + len < PAGE_SIZE || !linear) - linear = len; - - skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock, - err, 0); - if (!skb) - return NULL; - - skb_reserve(skb, prepad); - skb_put(skb, linear); - skb->data_len = len - linear; - skb->len += len - linear; - - return skb; -} - -/* Neighbour code has some assumptions on HH_DATA_MOD alignment */ -#define MACVTAP_RESERVE HH_DATA_OFF(ETH_HLEN) - -/* Get packet from user space buffer */ -static ssize_t macvtap_get_user(struct macvtap_queue *q, struct msghdr *m, - struct iov_iter *from, int noblock) -{ - int good_linear = SKB_MAX_HEAD(MACVTAP_RESERVE); - struct sk_buff *skb; - struct macvlan_dev *vlan; - unsigned long total_len = iov_iter_count(from); - unsigned long len = total_len; - int err; - struct virtio_net_hdr vnet_hdr = { 0 }; - int vnet_hdr_len = 0; - int copylen = 0; - int depth; - bool zerocopy = false; - size_t linear; - - if (q->flags & IFF_VNET_HDR) { - vnet_hdr_len = READ_ONCE(q->vnet_hdr_sz); - - err = -EINVAL; - if (len < vnet_hdr_len) - goto err; - len -= vnet_hdr_len; - - err = -EFAULT; - if (!copy_from_iter_full(&vnet_hdr, sizeof(vnet_hdr), from)) - goto err; - iov_iter_advance(from, vnet_hdr_len - sizeof(vnet_hdr)); - if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) && - macvtap16_to_cpu(q, vnet_hdr.csum_start) + - macvtap16_to_cpu(q, vnet_hdr.csum_offset) + 2 > - macvtap16_to_cpu(q, vnet_hdr.hdr_len)) - vnet_hdr.hdr_len = cpu_to_macvtap16(q, - macvtap16_to_cpu(q, vnet_hdr.csum_start) + - macvtap16_to_cpu(q, vnet_hdr.csum_offset) + 2); - err = -EINVAL; - if (macvtap16_to_cpu(q, vnet_hdr.hdr_len) > len) - goto err; - } - - err = -EINVAL; - if (unlikely(len < ETH_HLEN)) - goto err; - - if (m && m->msg_control && sock_flag(&q->sk, SOCK_ZEROCOPY)) { - struct iov_iter i; - - copylen = vnet_hdr.hdr_len ? - macvtap16_to_cpu(q, vnet_hdr.hdr_len) : GOODCOPY_LEN; - if (copylen > good_linear) - copylen = good_linear; - else if (copylen < ETH_HLEN) - copylen = ETH_HLEN; - linear = copylen; - i = *from; - iov_iter_advance(&i, copylen); - if (iov_iter_npages(&i, INT_MAX) <= MAX_SKB_FRAGS) - zerocopy = true; - } - - if (!zerocopy) { - copylen = len; - linear = macvtap16_to_cpu(q, vnet_hdr.hdr_len); - if (linear > good_linear) - linear = good_linear; - else if (linear < ETH_HLEN) - linear = ETH_HLEN; - } - - skb = macvtap_alloc_skb(&q->sk, MACVTAP_RESERVE, copylen, - linear, noblock, &err); - if (!skb) - goto err; - - if (zerocopy) - err = zerocopy_sg_from_iter(skb, from); - else - err = skb_copy_datagram_from_iter(skb, 0, from, len); - - if (err) - goto err_kfree; - - skb_set_network_header(skb, ETH_HLEN); - skb_reset_mac_header(skb); - skb->protocol = eth_hdr(skb)->h_proto; - - if (vnet_hdr_len) { - err = virtio_net_hdr_to_skb(skb, &vnet_hdr, - macvtap_is_little_endian(q)); - if (err) - goto err_kfree; - } - - skb_probe_transport_header(skb, ETH_HLEN); - - /* Move network header to the right position for VLAN tagged packets */ - if ((skb->protocol == htons(ETH_P_8021Q) || - skb->protocol == htons(ETH_P_8021AD)) && - __vlan_get_protocol(skb, skb->protocol, &depth) != 0) - skb_set_network_header(skb, depth); - - rcu_read_lock(); - vlan = rcu_dereference(q->vlan); - /* copy skb_ubuf_info for callback when skb has no error */ - if (zerocopy) { - skb_shinfo(skb)->destructor_arg = m->msg_control; - skb_shinfo(skb)->tx_flags |= SKBTX_DEV_ZEROCOPY; - skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG; - } else if (m && m->msg_control) { - struct ubuf_info *uarg = m->msg_control; - uarg->callback(uarg, false); - } - - if (vlan) { - skb->dev = vlan->dev; - dev_queue_xmit(skb); - } else { - kfree_skb(skb); - } - rcu_read_unlock(); - - return total_len; - -err_kfree: - kfree_skb(skb); - -err: - rcu_read_lock(); - vlan = rcu_dereference(q->vlan); - if (vlan) - this_cpu_inc(vlan->pcpu_stats->tx_dropped); - rcu_read_unlock(); - - return err; -} - -static ssize_t macvtap_write_iter(struct kiocb *iocb, struct iov_iter *from) -{ - struct file *file = iocb->ki_filp; - struct macvtap_queue *q = file->private_data; - - return macvtap_get_user(q, NULL, from, file->f_flags & O_NONBLOCK); -} - -/* Put packet to the user space buffer */ -static ssize_t macvtap_put_user(struct macvtap_queue *q, - const struct sk_buff *skb, - struct iov_iter *iter) -{ - int ret; - int vnet_hdr_len = 0; - int vlan_offset = 0; - int total; - - if (q->flags & IFF_VNET_HDR) { - struct virtio_net_hdr vnet_hdr; - vnet_hdr_len = READ_ONCE(q->vnet_hdr_sz); - if (iov_iter_count(iter) < vnet_hdr_len) - return -EINVAL; - - if (virtio_net_hdr_from_skb(skb, &vnet_hdr, - macvtap_is_little_endian(q), true)) - BUG(); - - if (copy_to_iter(&vnet_hdr, sizeof(vnet_hdr), iter) != - sizeof(vnet_hdr)) - return -EFAULT; - - iov_iter_advance(iter, vnet_hdr_len - sizeof(vnet_hdr)); - } - total = vnet_hdr_len; - total += skb->len; - - if (skb_vlan_tag_present(skb)) { - struct { - __be16 h_vlan_proto; - __be16 h_vlan_TCI; - } veth; - veth.h_vlan_proto = skb->vlan_proto; - veth.h_vlan_TCI = htons(skb_vlan_tag_get(skb)); - - vlan_offset = offsetof(struct vlan_ethhdr, h_vlan_proto); - total += VLAN_HLEN; - - ret = skb_copy_datagram_iter(skb, 0, iter, vlan_offset); - if (ret || !iov_iter_count(iter)) - goto done; - - ret = copy_to_iter(&veth, sizeof(veth), iter); - if (ret != sizeof(veth) || !iov_iter_count(iter)) - goto done; - } - - ret = skb_copy_datagram_iter(skb, vlan_offset, iter, - skb->len - vlan_offset); - -done: - return ret ? ret : total; -} - -static ssize_t macvtap_do_read(struct macvtap_queue *q, - struct iov_iter *to, - int noblock) -{ - DEFINE_WAIT(wait); - struct sk_buff *skb; - ssize_t ret = 0; - - if (!iov_iter_count(to)) - return 0; - - while (1) { - if (!noblock) - prepare_to_wait(sk_sleep(&q->sk), &wait, - TASK_INTERRUPTIBLE); - - /* Read frames from the queue */ - skb = skb_array_consume(&q->skb_array); - if (skb) - break; - if (noblock) { - ret = -EAGAIN; - break; - } - if (signal_pending(current)) { - ret = -ERESTARTSYS; - break; - } - /* Nothing to read, let's sleep */ - schedule(); - } - if (!noblock) - finish_wait(sk_sleep(&q->sk), &wait); - - if (skb) { - ret = macvtap_put_user(q, skb, to); - if (unlikely(ret < 0)) - kfree_skb(skb); - else - consume_skb(skb); - } - return ret; -} - -static ssize_t macvtap_read_iter(struct kiocb *iocb, struct iov_iter *to) -{ - struct file *file = iocb->ki_filp; - struct macvtap_queue *q = file->private_data; - ssize_t len = iov_iter_count(to), ret; - - ret = macvtap_do_read(q, to, file->f_flags & O_NONBLOCK); - ret = min_t(ssize_t, ret, len); - if (ret > 0) - iocb->ki_pos = ret; - return ret; -} - -static struct macvlan_dev *macvtap_get_vlan(struct macvtap_queue *q) -{ - struct macvlan_dev *vlan; - - ASSERT_RTNL(); - vlan = rtnl_dereference(q->vlan); - if (vlan) - dev_hold(vlan->dev); - - return vlan; -} - -static void macvtap_put_vlan(struct macvlan_dev *vlan) -{ - dev_put(vlan->dev); -} - -static int macvtap_ioctl_set_queue(struct file *file, unsigned int flags) -{ - struct macvtap_queue *q = file->private_data; - struct macvlan_dev *vlan; - int ret; - - vlan = macvtap_get_vlan(q); - if (!vlan) - return -EINVAL; - - if (flags & IFF_ATTACH_QUEUE) - ret = macvtap_enable_queue(vlan->dev, file, q); - else if (flags & IFF_DETACH_QUEUE) - ret = macvtap_disable_queue(q); - else - ret = -EINVAL; - - macvtap_put_vlan(vlan); - return ret; -} - -static int set_offload(struct macvtap_queue *q, unsigned long arg) -{ - struct macvlan_dev *vlan; - netdev_features_t features; - netdev_features_t feature_mask = 0; - - vlan = rtnl_dereference(q->vlan); - if (!vlan) - return -ENOLINK; - - features = vlan->dev->features; - - if (arg & TUN_F_CSUM) { - feature_mask = NETIF_F_HW_CSUM; - - if (arg & (TUN_F_TSO4 | TUN_F_TSO6)) { - if (arg & TUN_F_TSO_ECN) - feature_mask |= NETIF_F_TSO_ECN; - if (arg & TUN_F_TSO4) - feature_mask |= NETIF_F_TSO; - if (arg & TUN_F_TSO6) - feature_mask |= NETIF_F_TSO6; - } - - if (arg & TUN_F_UFO) - feature_mask |= NETIF_F_UFO; - } - - /* tun/tap driver inverts the usage for TSO offloads, where - * setting the TSO bit means that the userspace wants to - * accept TSO frames and turning it off means that user space - * does not support TSO. - * For macvtap, we have to invert it to mean the same thing. - * When user space turns off TSO, we turn off GSO/LRO so that - * user-space will not receive TSO frames. - */ - if (feature_mask & (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_UFO)) - features |= RX_OFFLOADS; - else - features &= ~RX_OFFLOADS; - - /* tap_features are the same as features on tun/tap and - * reflect user expectations. - */ - vlan->tap_features = feature_mask; - vlan->set_features = features; - netdev_update_features(vlan->dev); - - return 0; -} - -/* - * provide compatibility with generic tun/tap interface - */ -static long macvtap_ioctl(struct file *file, unsigned int cmd, - unsigned long arg) -{ - struct macvtap_queue *q = file->private_data; - struct macvlan_dev *vlan; - void __user *argp = (void __user *)arg; - struct ifreq __user *ifr = argp; - unsigned int __user *up = argp; - unsigned short u; - int __user *sp = argp; - struct sockaddr sa; - int s; - int ret; - - switch (cmd) { - case TUNSETIFF: - /* ignore the name, just look at flags */ - if (get_user(u, &ifr->ifr_flags)) - return -EFAULT; - - ret = 0; - if ((u & ~MACVTAP_FEATURES) != (IFF_NO_PI | IFF_TAP)) - ret = -EINVAL; - else - q->flags = (q->flags & ~MACVTAP_FEATURES) | u; - - return ret; - - case TUNGETIFF: - rtnl_lock(); - vlan = macvtap_get_vlan(q); - if (!vlan) { - rtnl_unlock(); - return -ENOLINK; - } - - ret = 0; - u = q->flags; - if (copy_to_user(&ifr->ifr_name, vlan->dev->name, IFNAMSIZ) || - put_user(u, &ifr->ifr_flags)) - ret = -EFAULT; - macvtap_put_vlan(vlan); - rtnl_unlock(); - return ret; - - case TUNSETQUEUE: - if (get_user(u, &ifr->ifr_flags)) - return -EFAULT; - rtnl_lock(); - ret = macvtap_ioctl_set_queue(file, u); - rtnl_unlock(); - return ret; - - case TUNGETFEATURES: - if (put_user(IFF_TAP | IFF_NO_PI | MACVTAP_FEATURES, up)) - return -EFAULT; - return 0; - - case TUNSETSNDBUF: - if (get_user(s, sp)) - return -EFAULT; - - q->sk.sk_sndbuf = s; - return 0; - - case TUNGETVNETHDRSZ: - s = q->vnet_hdr_sz; - if (put_user(s, sp)) - return -EFAULT; - return 0; - - case TUNSETVNETHDRSZ: - if (get_user(s, sp)) - return -EFAULT; - if (s < (int)sizeof(struct virtio_net_hdr)) - return -EINVAL; - - q->vnet_hdr_sz = s; - return 0; - - case TUNGETVNETLE: - s = !!(q->flags & MACVTAP_VNET_LE); - if (put_user(s, sp)) - return -EFAULT; - return 0; - - case TUNSETVNETLE: - if (get_user(s, sp)) - return -EFAULT; - if (s) - q->flags |= MACVTAP_VNET_LE; - else - q->flags &= ~MACVTAP_VNET_LE; - return 0; - - case TUNGETVNETBE: - return macvtap_get_vnet_be(q, sp); - - case TUNSETVNETBE: - return macvtap_set_vnet_be(q, sp); - - case TUNSETOFFLOAD: - /* let the user check for future flags */ - if (arg & ~(TUN_F_CSUM | TUN_F_TSO4 | TUN_F_TSO6 | - TUN_F_TSO_ECN | TUN_F_UFO)) - return -EINVAL; - - rtnl_lock(); - ret = set_offload(q, arg); - rtnl_unlock(); - return ret; - - case SIOCGIFHWADDR: - rtnl_lock(); - vlan = macvtap_get_vlan(q); - if (!vlan) { - rtnl_unlock(); - return -ENOLINK; - } - ret = 0; - u = vlan->dev->type; - if (copy_to_user(&ifr->ifr_name, vlan->dev->name, IFNAMSIZ) || - copy_to_user(&ifr->ifr_hwaddr.sa_data, vlan->dev->dev_addr, ETH_ALEN) || - put_user(u, &ifr->ifr_hwaddr.sa_family)) - ret = -EFAULT; - macvtap_put_vlan(vlan); - rtnl_unlock(); - return ret; - - case SIOCSIFHWADDR: - if (copy_from_user(&sa, &ifr->ifr_hwaddr, sizeof(sa))) - return -EFAULT; - rtnl_lock(); - vlan = macvtap_get_vlan(q); - if (!vlan) { - rtnl_unlock(); - return -ENOLINK; - } - ret = dev_set_mac_address(vlan->dev, &sa); - macvtap_put_vlan(vlan); - rtnl_unlock(); - return ret; - - default: - return -EINVAL; - } -} - -#ifdef CONFIG_COMPAT -static long macvtap_compat_ioctl(struct file *file, unsigned int cmd, - unsigned long arg) -{ - return macvtap_ioctl(file, cmd, (unsigned long)compat_ptr(arg)); -} -#endif - -static const struct file_operations macvtap_fops = { - .owner = THIS_MODULE, - .open = macvtap_open, - .release = macvtap_release, - .read_iter = macvtap_read_iter, - .write_iter = macvtap_write_iter, - .poll = macvtap_poll, - .llseek = no_llseek, - .unlocked_ioctl = macvtap_ioctl, -#ifdef CONFIG_COMPAT - .compat_ioctl = macvtap_compat_ioctl, -#endif -}; - -static int macvtap_sendmsg(struct socket *sock, struct msghdr *m, - size_t total_len) -{ - struct macvtap_queue *q = container_of(sock, struct macvtap_queue, sock); - return macvtap_get_user(q, m, &m->msg_iter, m->msg_flags & MSG_DONTWAIT); -} - -static int macvtap_recvmsg(struct socket *sock, struct msghdr *m, - size_t total_len, int flags) -{ - struct macvtap_queue *q = container_of(sock, struct macvtap_queue, sock); - int ret; - if (flags & ~(MSG_DONTWAIT|MSG_TRUNC)) - return -EINVAL; - ret = macvtap_do_read(q, &m->msg_iter, flags & MSG_DONTWAIT); - if (ret > total_len) { - m->msg_flags |= MSG_TRUNC; - ret = flags & MSG_TRUNC ? ret : total_len; - } - return ret; -} - -static int macvtap_peek_len(struct socket *sock) -{ - struct macvtap_queue *q = container_of(sock, struct macvtap_queue, - sock); - return skb_array_peek_len(&q->skb_array); -} - -/* Ops structure to mimic raw sockets with tun */ -static const struct proto_ops macvtap_socket_ops = { - .sendmsg = macvtap_sendmsg, - .recvmsg = macvtap_recvmsg, - .peek_len = macvtap_peek_len, -}; - -/* Get an underlying socket object from tun file. Returns error unless file is - * attached to a device. The returned object works like a packet socket, it - * can be used for sock_sendmsg/sock_recvmsg. The caller is responsible for - * holding a reference to the file for as long as the socket is in use. */ -struct socket *macvtap_get_socket(struct file *file) -{ - struct macvtap_queue *q; - if (file->f_op != &macvtap_fops) - return ERR_PTR(-EINVAL); - q = file->private_data; - if (!q) - return ERR_PTR(-EBADFD); - return &q->sock; -} -EXPORT_SYMBOL_GPL(macvtap_get_socket); - -static int macvtap_queue_resize(struct macvlan_dev *vlan) -{ - struct net_device *dev = vlan->dev; - struct macvtap_queue *q; - struct skb_array **arrays; - int n = vlan->numqueues; - int ret, i = 0; - - arrays = kmalloc(sizeof *arrays * n, GFP_KERNEL); - if (!arrays) - return -ENOMEM; - - list_for_each_entry(q, &vlan->queue_list, next) - arrays[i++] = &q->skb_array; - - ret = skb_array_resize_multiple(arrays, n, - dev->tx_queue_len, GFP_KERNEL); - - kfree(arrays); - return ret; -} - static int macvtap_device_event(struct notifier_block *unused, unsigned long event, void *ptr) { struct net_device *dev = netdev_notifier_info_to_dev(ptr); - struct macvlan_dev *vlan; + struct macvtap_dev *vlantap; struct device *classdev; dev_t devt; int err; @@ -1271,7 +155,7 @@ static int macvtap_device_event(struct notifier_block *unused, return NOTIFY_DONE; snprintf(tap_name, IFNAMSIZ, "tap%d", dev->ifindex); - vlan = netdev_priv(dev); + vlantap = netdev_priv(dev); switch (event) { case NETDEV_REGISTER: @@ -1279,15 +163,15 @@ static int macvtap_device_event(struct notifier_block *unused, * been registered but before register_netdevice has * finished running. */ - err = macvtap_get_minor(vlan); + err = tap_get_minor(macvtap_major, &vlantap->tap); if (err) return notifier_from_errno(err); - devt = MKDEV(MAJOR(macvtap_major), vlan->minor); + devt = MKDEV(MAJOR(macvtap_major), vlantap->tap.minor); classdev = device_create(&macvtap_class, &dev->dev, devt, dev, tap_name); if (IS_ERR(classdev)) { - macvtap_free_minor(vlan); + tap_free_minor(macvtap_major, &vlantap->tap); return notifier_from_errno(PTR_ERR(classdev)); } err = sysfs_create_link(&dev->dev.kobj, &classdev->kobj, @@ -1297,15 +181,15 @@ static int macvtap_device_event(struct notifier_block *unused, break; case NETDEV_UNREGISTER: /* vlan->minor == 0 if NETDEV_REGISTER above failed */ - if (vlan->minor == 0) + if (vlantap->tap.minor == 0) break; sysfs_remove_link(&dev->dev.kobj, tap_name); - devt = MKDEV(MAJOR(macvtap_major), vlan->minor); + devt = MKDEV(MAJOR(macvtap_major), vlantap->tap.minor); device_destroy(&macvtap_class, devt); - macvtap_free_minor(vlan); + tap_free_minor(macvtap_major, &vlantap->tap); break; case NETDEV_CHANGE_TX_QUEUE_LEN: - if (macvtap_queue_resize(vlan)) + if (tap_queue_resize(&vlantap->tap)) return NOTIFY_BAD; break; } @@ -1321,38 +205,31 @@ static int macvtap_init(void) { int err; - err = alloc_chrdev_region(&macvtap_major, 0, - MACVTAP_NUM_DEVS, "macvtap"); - if (err) - goto out1; + err = tap_create_cdev(&macvtap_cdev, &macvtap_major, "macvtap"); - cdev_init(&macvtap_cdev, &macvtap_fops); - err = cdev_add(&macvtap_cdev, macvtap_major, MACVTAP_NUM_DEVS); if (err) - goto out2; + goto out1; err = class_register(&macvtap_class); if (err) - goto out3; + goto out2; err = register_netdevice_notifier(&macvtap_notifier_block); if (err) - goto out4; + goto out3; err = macvlan_link_register(&macvtap_link_ops); if (err) - goto out5; + goto out4; return 0; -out5: - unregister_netdevice_notifier(&macvtap_notifier_block); out4: - class_unregister(&macvtap_class); + unregister_netdevice_notifier(&macvtap_notifier_block); out3: - cdev_del(&macvtap_cdev); + class_unregister(&macvtap_class); out2: - unregister_chrdev_region(macvtap_major, MACVTAP_NUM_DEVS); + tap_destroy_cdev(macvtap_major, &macvtap_cdev); out1: return err; } @@ -1363,9 +240,7 @@ static void macvtap_exit(void) rtnl_link_unregister(&macvtap_link_ops); unregister_netdevice_notifier(&macvtap_notifier_block); class_unregister(&macvtap_class); - cdev_del(&macvtap_cdev); - unregister_chrdev_region(macvtap_major, MACVTAP_NUM_DEVS); - idr_destroy(&minor_idr); + tap_destroy_cdev(macvtap_major, &macvtap_cdev); } module_exit(macvtap_exit); diff --git a/drivers/net/tap.c b/drivers/net/tap.c new file mode 100644 index 0000000000000000000000000000000000000000..35b55a2fa1a1540d742bf8b18981a630fddf9bb2 --- /dev/null +++ b/drivers/net/tap.c @@ -0,0 +1,1285 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#define TAP_IFFEATURES (IFF_VNET_HDR | IFF_MULTI_QUEUE) + +#define TAP_VNET_LE 0x80000000 +#define TAP_VNET_BE 0x40000000 + +#ifdef CONFIG_TUN_VNET_CROSS_LE +static inline bool tap_legacy_is_little_endian(struct tap_queue *q) +{ + return q->flags & TAP_VNET_BE ? false : + virtio_legacy_is_little_endian(); +} + +static long tap_get_vnet_be(struct tap_queue *q, int __user *sp) +{ + int s = !!(q->flags & TAP_VNET_BE); + + if (put_user(s, sp)) + return -EFAULT; + + return 0; +} + +static long tap_set_vnet_be(struct tap_queue *q, int __user *sp) +{ + int s; + + if (get_user(s, sp)) + return -EFAULT; + + if (s) + q->flags |= TAP_VNET_BE; + else + q->flags &= ~TAP_VNET_BE; + + return 0; +} +#else +static inline bool tap_legacy_is_little_endian(struct tap_queue *q) +{ + return virtio_legacy_is_little_endian(); +} + +static long tap_get_vnet_be(struct tap_queue *q, int __user *argp) +{ + return -EINVAL; +} + +static long tap_set_vnet_be(struct tap_queue *q, int __user *argp) +{ + return -EINVAL; +} +#endif /* CONFIG_TUN_VNET_CROSS_LE */ + +static inline bool tap_is_little_endian(struct tap_queue *q) +{ + return q->flags & TAP_VNET_LE || + tap_legacy_is_little_endian(q); +} + +static inline u16 tap16_to_cpu(struct tap_queue *q, __virtio16 val) +{ + return __virtio16_to_cpu(tap_is_little_endian(q), val); +} + +static inline __virtio16 cpu_to_tap16(struct tap_queue *q, u16 val) +{ + return __cpu_to_virtio16(tap_is_little_endian(q), val); +} + +static struct proto tap_proto = { + .name = "tap", + .owner = THIS_MODULE, + .obj_size = sizeof(struct tap_queue), +}; + +#define TAP_NUM_DEVS (1U << MINORBITS) + +static LIST_HEAD(major_list); + +struct major_info { + struct rcu_head rcu; + dev_t major; + struct idr minor_idr; + struct mutex minor_lock; + const char *device_name; + struct list_head next; +}; + +#define GOODCOPY_LEN 128 + +static const struct proto_ops tap_socket_ops; + +#define RX_OFFLOADS (NETIF_F_GRO | NETIF_F_LRO) +#define TAP_FEATURES (NETIF_F_GSO | NETIF_F_SG | NETIF_F_FRAGLIST) + +static struct tap_dev *tap_dev_get_rcu(const struct net_device *dev) +{ + return rcu_dereference(dev->rx_handler_data); +} + +/* + * RCU usage: + * The tap_queue and the macvlan_dev are loosely coupled, the + * pointers from one to the other can only be read while rcu_read_lock + * or rtnl is held. + * + * Both the file and the macvlan_dev hold a reference on the tap_queue + * through sock_hold(&q->sk). When the macvlan_dev goes away first, + * q->vlan becomes inaccessible. When the files gets closed, + * tap_get_queue() fails. + * + * There may still be references to the struct sock inside of the + * queue from outbound SKBs, but these never reference back to the + * file or the dev. The data structure is freed through __sk_free + * when both our references and any pending SKBs are gone. + */ + +static int tap_enable_queue(struct tap_dev *tap, struct file *file, + struct tap_queue *q) +{ + int err = -EINVAL; + + ASSERT_RTNL(); + + if (q->enabled) + goto out; + + err = 0; + rcu_assign_pointer(tap->taps[tap->numvtaps], q); + q->queue_index = tap->numvtaps; + q->enabled = true; + + tap->numvtaps++; +out: + return err; +} + +/* Requires RTNL */ +static int tap_set_queue(struct tap_dev *tap, struct file *file, + struct tap_queue *q) +{ + if (tap->numqueues == MAX_TAP_QUEUES) + return -EBUSY; + + rcu_assign_pointer(q->tap, tap); + rcu_assign_pointer(tap->taps[tap->numvtaps], q); + sock_hold(&q->sk); + + q->file = file; + q->queue_index = tap->numvtaps; + q->enabled = true; + file->private_data = q; + list_add_tail(&q->next, &tap->queue_list); + + tap->numvtaps++; + tap->numqueues++; + + return 0; +} + +static int tap_disable_queue(struct tap_queue *q) +{ + struct tap_dev *tap; + struct tap_queue *nq; + + ASSERT_RTNL(); + if (!q->enabled) + return -EINVAL; + + tap = rtnl_dereference(q->tap); + + if (tap) { + int index = q->queue_index; + BUG_ON(index >= tap->numvtaps); + nq = rtnl_dereference(tap->taps[tap->numvtaps - 1]); + nq->queue_index = index; + + rcu_assign_pointer(tap->taps[index], nq); + RCU_INIT_POINTER(tap->taps[tap->numvtaps - 1], NULL); + q->enabled = false; + + tap->numvtaps--; + } + + return 0; +} + +/* + * The file owning the queue got closed, give up both + * the reference that the files holds as well as the + * one from the macvlan_dev if that still exists. + * + * Using the spinlock makes sure that we don't get + * to the queue again after destroying it. + */ +static void tap_put_queue(struct tap_queue *q) +{ + struct tap_dev *tap; + + rtnl_lock(); + tap = rtnl_dereference(q->tap); + + if (tap) { + if (q->enabled) + BUG_ON(tap_disable_queue(q)); + + tap->numqueues--; + RCU_INIT_POINTER(q->tap, NULL); + sock_put(&q->sk); + list_del_init(&q->next); + } + + rtnl_unlock(); + + synchronize_rcu(); + sock_put(&q->sk); +} + +/* + * Select a queue based on the rxq of the device on which this packet + * arrived. If the incoming device is not mq, calculate a flow hash + * to select a queue. If all fails, find the first available queue. + * Cache vlan->numvtaps since it can become zero during the execution + * of this function. + */ +static struct tap_queue *tap_get_queue(struct tap_dev *tap, + struct sk_buff *skb) +{ + struct tap_queue *queue = NULL; + /* Access to taps array is protected by rcu, but access to numvtaps + * isn't. Below we use it to lookup a queue, but treat it as a hint + * and validate that the result isn't NULL - in case we are + * racing against queue removal. + */ + int numvtaps = ACCESS_ONCE(tap->numvtaps); + __u32 rxq; + + if (!numvtaps) + goto out; + + if (numvtaps == 1) + goto single; + + /* Check if we can use flow to select a queue */ + rxq = skb_get_hash(skb); + if (rxq) { + queue = rcu_dereference(tap->taps[rxq % numvtaps]); + goto out; + } + + if (likely(skb_rx_queue_recorded(skb))) { + rxq = skb_get_rx_queue(skb); + + while (unlikely(rxq >= numvtaps)) + rxq -= numvtaps; + + queue = rcu_dereference(tap->taps[rxq]); + goto out; + } + +single: + queue = rcu_dereference(tap->taps[0]); +out: + return queue; +} + +/* + * The net_device is going away, give up the reference + * that it holds on all queues and safely set the pointer + * from the queues to NULL. + */ +void tap_del_queues(struct tap_dev *tap) +{ + struct tap_queue *q, *tmp; + + ASSERT_RTNL(); + list_for_each_entry_safe(q, tmp, &tap->queue_list, next) { + list_del_init(&q->next); + RCU_INIT_POINTER(q->tap, NULL); + if (q->enabled) + tap->numvtaps--; + tap->numqueues--; + sock_put(&q->sk); + } + BUG_ON(tap->numvtaps); + BUG_ON(tap->numqueues); + /* guarantee that any future tap_set_queue will fail */ + tap->numvtaps = MAX_TAP_QUEUES; +} +EXPORT_SYMBOL_GPL(tap_del_queues); + +rx_handler_result_t tap_handle_frame(struct sk_buff **pskb) +{ + struct sk_buff *skb = *pskb; + struct net_device *dev = skb->dev; + struct tap_dev *tap; + struct tap_queue *q; + netdev_features_t features = TAP_FEATURES; + + tap = tap_dev_get_rcu(dev); + if (!tap) + return RX_HANDLER_PASS; + + q = tap_get_queue(tap, skb); + if (!q) + return RX_HANDLER_PASS; + + if (__skb_array_full(&q->skb_array)) + goto drop; + + skb_push(skb, ETH_HLEN); + + /* Apply the forward feature mask so that we perform segmentation + * according to users wishes. This only works if VNET_HDR is + * enabled. + */ + if (q->flags & IFF_VNET_HDR) + features |= tap->tap_features; + if (netif_needs_gso(skb, features)) { + struct sk_buff *segs = __skb_gso_segment(skb, features, false); + + if (IS_ERR(segs)) + goto drop; + + if (!segs) { + if (skb_array_produce(&q->skb_array, skb)) + goto drop; + goto wake_up; + } + + consume_skb(skb); + while (segs) { + struct sk_buff *nskb = segs->next; + + segs->next = NULL; + if (skb_array_produce(&q->skb_array, segs)) { + kfree_skb(segs); + kfree_skb_list(nskb); + break; + } + segs = nskb; + } + } else { + /* If we receive a partial checksum and the tap side + * doesn't support checksum offload, compute the checksum. + * Note: it doesn't matter which checksum feature to + * check, we either support them all or none. + */ + if (skb->ip_summed == CHECKSUM_PARTIAL && + !(features & NETIF_F_CSUM_MASK) && + skb_checksum_help(skb)) + goto drop; + if (skb_array_produce(&q->skb_array, skb)) + goto drop; + } + +wake_up: + wake_up_interruptible_poll(sk_sleep(&q->sk), POLLIN | POLLRDNORM | POLLRDBAND); + return RX_HANDLER_CONSUMED; + +drop: + /* Count errors/drops only here, thus don't care about args. */ + if (tap->count_rx_dropped) + tap->count_rx_dropped(tap); + kfree_skb(skb); + return RX_HANDLER_CONSUMED; +} +EXPORT_SYMBOL_GPL(tap_handle_frame); + +static struct major_info *tap_get_major(int major) +{ + struct major_info *tap_major; + + list_for_each_entry_rcu(tap_major, &major_list, next) { + if (tap_major->major == major) + return tap_major; + } + + return NULL; +} + +int tap_get_minor(dev_t major, struct tap_dev *tap) +{ + int retval = -ENOMEM; + struct major_info *tap_major; + + rcu_read_lock(); + tap_major = tap_get_major(MAJOR(major)); + if (!tap_major) { + retval = -EINVAL; + goto unlock; + } + + mutex_lock(&tap_major->minor_lock); + retval = idr_alloc(&tap_major->minor_idr, tap, 1, TAP_NUM_DEVS, GFP_KERNEL); + if (retval >= 0) { + tap->minor = retval; + } else if (retval == -ENOSPC) { + netdev_err(tap->dev, "Too many tap devices\n"); + retval = -EINVAL; + } + mutex_unlock(&tap_major->minor_lock); + +unlock: + rcu_read_unlock(); + return retval < 0 ? retval : 0; +} +EXPORT_SYMBOL_GPL(tap_get_minor); + +void tap_free_minor(dev_t major, struct tap_dev *tap) +{ + struct major_info *tap_major; + + rcu_read_lock(); + tap_major = tap_get_major(MAJOR(major)); + if (!tap_major) { + goto unlock; + } + + mutex_lock(&tap_major->minor_lock); + if (tap->minor) { + idr_remove(&tap_major->minor_idr, tap->minor); + tap->minor = 0; + } + mutex_unlock(&tap_major->minor_lock); + +unlock: + rcu_read_unlock(); +} +EXPORT_SYMBOL_GPL(tap_free_minor); + +static struct tap_dev *dev_get_by_tap_file(int major, int minor) +{ + struct net_device *dev = NULL; + struct tap_dev *tap; + struct major_info *tap_major; + + rcu_read_lock(); + tap_major = tap_get_major(major); + if (!tap_major) { + tap = NULL; + goto unlock; + } + + mutex_lock(&tap_major->minor_lock); + tap = idr_find(&tap_major->minor_idr, minor); + if (tap) { + dev = tap->dev; + dev_hold(dev); + } + mutex_unlock(&tap_major->minor_lock); + +unlock: + rcu_read_unlock(); + return tap; +} + +static void tap_sock_write_space(struct sock *sk) +{ + wait_queue_head_t *wqueue; + + if (!sock_writeable(sk) || + !test_and_clear_bit(SOCKWQ_ASYNC_NOSPACE, &sk->sk_socket->flags)) + return; + + wqueue = sk_sleep(sk); + if (wqueue && waitqueue_active(wqueue)) + wake_up_interruptible_poll(wqueue, POLLOUT | POLLWRNORM | POLLWRBAND); +} + +static void tap_sock_destruct(struct sock *sk) +{ + struct tap_queue *q = container_of(sk, struct tap_queue, sk); + + skb_array_cleanup(&q->skb_array); +} + +static int tap_open(struct inode *inode, struct file *file) +{ + struct net *net = current->nsproxy->net_ns; + struct tap_dev *tap; + struct tap_queue *q; + int err = -ENODEV; + + rtnl_lock(); + tap = dev_get_by_tap_file(imajor(inode), iminor(inode)); + if (!tap) + goto err; + + err = -ENOMEM; + q = (struct tap_queue *)sk_alloc(net, AF_UNSPEC, GFP_KERNEL, + &tap_proto, 0); + if (!q) + goto err; + + RCU_INIT_POINTER(q->sock.wq, &q->wq); + init_waitqueue_head(&q->wq.wait); + q->sock.type = SOCK_RAW; + q->sock.state = SS_CONNECTED; + q->sock.file = file; + q->sock.ops = &tap_socket_ops; + sock_init_data(&q->sock, &q->sk); + q->sk.sk_write_space = tap_sock_write_space; + q->sk.sk_destruct = tap_sock_destruct; + q->flags = IFF_VNET_HDR | IFF_NO_PI | IFF_TAP; + q->vnet_hdr_sz = sizeof(struct virtio_net_hdr); + + /* + * so far only KVM virtio_net uses tap, enable zero copy between + * guest kernel and host kernel when lower device supports zerocopy + * + * The macvlan supports zerocopy iff the lower device supports zero + * copy so we don't have to look at the lower device directly. + */ + if ((tap->dev->features & NETIF_F_HIGHDMA) && (tap->dev->features & NETIF_F_SG)) + sock_set_flag(&q->sk, SOCK_ZEROCOPY); + + err = -ENOMEM; + if (skb_array_init(&q->skb_array, tap->dev->tx_queue_len, GFP_KERNEL)) + goto err_array; + + err = tap_set_queue(tap, file, q); + if (err) + goto err_queue; + + dev_put(tap->dev); + + rtnl_unlock(); + return err; + +err_queue: + skb_array_cleanup(&q->skb_array); +err_array: + sock_put(&q->sk); +err: + if (tap) + dev_put(tap->dev); + + rtnl_unlock(); + return err; +} + +static int tap_release(struct inode *inode, struct file *file) +{ + struct tap_queue *q = file->private_data; + tap_put_queue(q); + return 0; +} + +static unsigned int tap_poll(struct file *file, poll_table *wait) +{ + struct tap_queue *q = file->private_data; + unsigned int mask = POLLERR; + + if (!q) + goto out; + + mask = 0; + poll_wait(file, &q->wq.wait, wait); + + if (!skb_array_empty(&q->skb_array)) + mask |= POLLIN | POLLRDNORM; + + if (sock_writeable(&q->sk) || + (!test_and_set_bit(SOCKWQ_ASYNC_NOSPACE, &q->sock.flags) && + sock_writeable(&q->sk))) + mask |= POLLOUT | POLLWRNORM; + +out: + return mask; +} + +static inline struct sk_buff *tap_alloc_skb(struct sock *sk, size_t prepad, + size_t len, size_t linear, + int noblock, int *err) +{ + struct sk_buff *skb; + + /* Under a page? Don't bother with paged skb. */ + if (prepad + len < PAGE_SIZE || !linear) + linear = len; + + skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock, + err, 0); + if (!skb) + return NULL; + + skb_reserve(skb, prepad); + skb_put(skb, linear); + skb->data_len = len - linear; + skb->len += len - linear; + + return skb; +} + +/* Neighbour code has some assumptions on HH_DATA_MOD alignment */ +#define TAP_RESERVE HH_DATA_OFF(ETH_HLEN) + +/* Get packet from user space buffer */ +static ssize_t tap_get_user(struct tap_queue *q, struct msghdr *m, + struct iov_iter *from, int noblock) +{ + int good_linear = SKB_MAX_HEAD(TAP_RESERVE); + struct sk_buff *skb; + struct tap_dev *tap; + unsigned long total_len = iov_iter_count(from); + unsigned long len = total_len; + int err; + struct virtio_net_hdr vnet_hdr = { 0 }; + int vnet_hdr_len = 0; + int copylen = 0; + int depth; + bool zerocopy = false; + size_t linear; + + if (q->flags & IFF_VNET_HDR) { + vnet_hdr_len = READ_ONCE(q->vnet_hdr_sz); + + err = -EINVAL; + if (len < vnet_hdr_len) + goto err; + len -= vnet_hdr_len; + + err = -EFAULT; + if (!copy_from_iter_full(&vnet_hdr, sizeof(vnet_hdr), from)) + goto err; + iov_iter_advance(from, vnet_hdr_len - sizeof(vnet_hdr)); + if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) && + tap16_to_cpu(q, vnet_hdr.csum_start) + + tap16_to_cpu(q, vnet_hdr.csum_offset) + 2 > + tap16_to_cpu(q, vnet_hdr.hdr_len)) + vnet_hdr.hdr_len = cpu_to_tap16(q, + tap16_to_cpu(q, vnet_hdr.csum_start) + + tap16_to_cpu(q, vnet_hdr.csum_offset) + 2); + err = -EINVAL; + if (tap16_to_cpu(q, vnet_hdr.hdr_len) > len) + goto err; + } + + err = -EINVAL; + if (unlikely(len < ETH_HLEN)) + goto err; + + if (m && m->msg_control && sock_flag(&q->sk, SOCK_ZEROCOPY)) { + struct iov_iter i; + + copylen = vnet_hdr.hdr_len ? + tap16_to_cpu(q, vnet_hdr.hdr_len) : GOODCOPY_LEN; + if (copylen > good_linear) + copylen = good_linear; + else if (copylen < ETH_HLEN) + copylen = ETH_HLEN; + linear = copylen; + i = *from; + iov_iter_advance(&i, copylen); + if (iov_iter_npages(&i, INT_MAX) <= MAX_SKB_FRAGS) + zerocopy = true; + } + + if (!zerocopy) { + copylen = len; + linear = tap16_to_cpu(q, vnet_hdr.hdr_len); + if (linear > good_linear) + linear = good_linear; + else if (linear < ETH_HLEN) + linear = ETH_HLEN; + } + + skb = tap_alloc_skb(&q->sk, TAP_RESERVE, copylen, + linear, noblock, &err); + if (!skb) + goto err; + + if (zerocopy) + err = zerocopy_sg_from_iter(skb, from); + else + err = skb_copy_datagram_from_iter(skb, 0, from, len); + + if (err) + goto err_kfree; + + skb_set_network_header(skb, ETH_HLEN); + skb_reset_mac_header(skb); + skb->protocol = eth_hdr(skb)->h_proto; + + if (vnet_hdr_len) { + err = virtio_net_hdr_to_skb(skb, &vnet_hdr, + tap_is_little_endian(q)); + if (err) + goto err_kfree; + } + + skb_probe_transport_header(skb, ETH_HLEN); + + /* Move network header to the right position for VLAN tagged packets */ + if ((skb->protocol == htons(ETH_P_8021Q) || + skb->protocol == htons(ETH_P_8021AD)) && + __vlan_get_protocol(skb, skb->protocol, &depth) != 0) + skb_set_network_header(skb, depth); + + rcu_read_lock(); + tap = rcu_dereference(q->tap); + /* copy skb_ubuf_info for callback when skb has no error */ + if (zerocopy) { + skb_shinfo(skb)->destructor_arg = m->msg_control; + skb_shinfo(skb)->tx_flags |= SKBTX_DEV_ZEROCOPY; + skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG; + } else if (m && m->msg_control) { + struct ubuf_info *uarg = m->msg_control; + uarg->callback(uarg, false); + } + + if (tap) { + skb->dev = tap->dev; + dev_queue_xmit(skb); + } else { + kfree_skb(skb); + } + rcu_read_unlock(); + + return total_len; + +err_kfree: + kfree_skb(skb); + +err: + rcu_read_lock(); + tap = rcu_dereference(q->tap); + if (tap && tap->count_tx_dropped) + tap->count_tx_dropped(tap); + rcu_read_unlock(); + + return err; +} + +static ssize_t tap_write_iter(struct kiocb *iocb, struct iov_iter *from) +{ + struct file *file = iocb->ki_filp; + struct tap_queue *q = file->private_data; + + return tap_get_user(q, NULL, from, file->f_flags & O_NONBLOCK); +} + +/* Put packet to the user space buffer */ +static ssize_t tap_put_user(struct tap_queue *q, + const struct sk_buff *skb, + struct iov_iter *iter) +{ + int ret; + int vnet_hdr_len = 0; + int vlan_offset = 0; + int total; + + if (q->flags & IFF_VNET_HDR) { + struct virtio_net_hdr vnet_hdr; + vnet_hdr_len = READ_ONCE(q->vnet_hdr_sz); + if (iov_iter_count(iter) < vnet_hdr_len) + return -EINVAL; + + if (virtio_net_hdr_from_skb(skb, &vnet_hdr, + tap_is_little_endian(q), true)) + BUG(); + + if (copy_to_iter(&vnet_hdr, sizeof(vnet_hdr), iter) != + sizeof(vnet_hdr)) + return -EFAULT; + + iov_iter_advance(iter, vnet_hdr_len - sizeof(vnet_hdr)); + } + total = vnet_hdr_len; + total += skb->len; + + if (skb_vlan_tag_present(skb)) { + struct { + __be16 h_vlan_proto; + __be16 h_vlan_TCI; + } veth; + veth.h_vlan_proto = skb->vlan_proto; + veth.h_vlan_TCI = htons(skb_vlan_tag_get(skb)); + + vlan_offset = offsetof(struct vlan_ethhdr, h_vlan_proto); + total += VLAN_HLEN; + + ret = skb_copy_datagram_iter(skb, 0, iter, vlan_offset); + if (ret || !iov_iter_count(iter)) + goto done; + + ret = copy_to_iter(&veth, sizeof(veth), iter); + if (ret != sizeof(veth) || !iov_iter_count(iter)) + goto done; + } + + ret = skb_copy_datagram_iter(skb, vlan_offset, iter, + skb->len - vlan_offset); + +done: + return ret ? ret : total; +} + +static ssize_t tap_do_read(struct tap_queue *q, + struct iov_iter *to, + int noblock) +{ + DEFINE_WAIT(wait); + struct sk_buff *skb; + ssize_t ret = 0; + + if (!iov_iter_count(to)) + return 0; + + while (1) { + if (!noblock) + prepare_to_wait(sk_sleep(&q->sk), &wait, + TASK_INTERRUPTIBLE); + + /* Read frames from the queue */ + skb = skb_array_consume(&q->skb_array); + if (skb) + break; + if (noblock) { + ret = -EAGAIN; + break; + } + if (signal_pending(current)) { + ret = -ERESTARTSYS; + break; + } + /* Nothing to read, let's sleep */ + schedule(); + } + if (!noblock) + finish_wait(sk_sleep(&q->sk), &wait); + + if (skb) { + ret = tap_put_user(q, skb, to); + if (unlikely(ret < 0)) + kfree_skb(skb); + else + consume_skb(skb); + } + return ret; +} + +static ssize_t tap_read_iter(struct kiocb *iocb, struct iov_iter *to) +{ + struct file *file = iocb->ki_filp; + struct tap_queue *q = file->private_data; + ssize_t len = iov_iter_count(to), ret; + + ret = tap_do_read(q, to, file->f_flags & O_NONBLOCK); + ret = min_t(ssize_t, ret, len); + if (ret > 0) + iocb->ki_pos = ret; + return ret; +} + +static struct tap_dev *tap_get_tap_dev(struct tap_queue *q) +{ + struct tap_dev *tap; + + ASSERT_RTNL(); + tap = rtnl_dereference(q->tap); + if (tap) + dev_hold(tap->dev); + + return tap; +} + +static void tap_put_tap_dev(struct tap_dev *tap) +{ + dev_put(tap->dev); +} + +static int tap_ioctl_set_queue(struct file *file, unsigned int flags) +{ + struct tap_queue *q = file->private_data; + struct tap_dev *tap; + int ret; + + tap = tap_get_tap_dev(q); + if (!tap) + return -EINVAL; + + if (flags & IFF_ATTACH_QUEUE) + ret = tap_enable_queue(tap, file, q); + else if (flags & IFF_DETACH_QUEUE) + ret = tap_disable_queue(q); + else + ret = -EINVAL; + + tap_put_tap_dev(tap); + return ret; +} + +static int set_offload(struct tap_queue *q, unsigned long arg) +{ + struct tap_dev *tap; + netdev_features_t features; + netdev_features_t feature_mask = 0; + + tap = rtnl_dereference(q->tap); + if (!tap) + return -ENOLINK; + + features = tap->dev->features; + + if (arg & TUN_F_CSUM) { + feature_mask = NETIF_F_HW_CSUM; + + if (arg & (TUN_F_TSO4 | TUN_F_TSO6)) { + if (arg & TUN_F_TSO_ECN) + feature_mask |= NETIF_F_TSO_ECN; + if (arg & TUN_F_TSO4) + feature_mask |= NETIF_F_TSO; + if (arg & TUN_F_TSO6) + feature_mask |= NETIF_F_TSO6; + } + + if (arg & TUN_F_UFO) + feature_mask |= NETIF_F_UFO; + } + + /* tun/tap driver inverts the usage for TSO offloads, where + * setting the TSO bit means that the userspace wants to + * accept TSO frames and turning it off means that user space + * does not support TSO. + * For tap, we have to invert it to mean the same thing. + * When user space turns off TSO, we turn off GSO/LRO so that + * user-space will not receive TSO frames. + */ + if (feature_mask & (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_UFO)) + features |= RX_OFFLOADS; + else + features &= ~RX_OFFLOADS; + + /* tap_features are the same as features on tun/tap and + * reflect user expectations. + */ + tap->tap_features = feature_mask; + if (tap->update_features) + tap->update_features(tap, features); + + return 0; +} + +/* + * provide compatibility with generic tun/tap interface + */ +static long tap_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + struct tap_queue *q = file->private_data; + struct tap_dev *tap; + void __user *argp = (void __user *)arg; + struct ifreq __user *ifr = argp; + unsigned int __user *up = argp; + unsigned short u; + int __user *sp = argp; + struct sockaddr sa; + int s; + int ret; + + switch (cmd) { + case TUNSETIFF: + /* ignore the name, just look at flags */ + if (get_user(u, &ifr->ifr_flags)) + return -EFAULT; + + ret = 0; + if ((u & ~TAP_IFFEATURES) != (IFF_NO_PI | IFF_TAP)) + ret = -EINVAL; + else + q->flags = (q->flags & ~TAP_IFFEATURES) | u; + + return ret; + + case TUNGETIFF: + rtnl_lock(); + tap = tap_get_tap_dev(q); + if (!tap) { + rtnl_unlock(); + return -ENOLINK; + } + + ret = 0; + u = q->flags; + if (copy_to_user(&ifr->ifr_name, tap->dev->name, IFNAMSIZ) || + put_user(u, &ifr->ifr_flags)) + ret = -EFAULT; + tap_put_tap_dev(tap); + rtnl_unlock(); + return ret; + + case TUNSETQUEUE: + if (get_user(u, &ifr->ifr_flags)) + return -EFAULT; + rtnl_lock(); + ret = tap_ioctl_set_queue(file, u); + rtnl_unlock(); + return ret; + + case TUNGETFEATURES: + if (put_user(IFF_TAP | IFF_NO_PI | TAP_IFFEATURES, up)) + return -EFAULT; + return 0; + + case TUNSETSNDBUF: + if (get_user(s, sp)) + return -EFAULT; + + q->sk.sk_sndbuf = s; + return 0; + + case TUNGETVNETHDRSZ: + s = q->vnet_hdr_sz; + if (put_user(s, sp)) + return -EFAULT; + return 0; + + case TUNSETVNETHDRSZ: + if (get_user(s, sp)) + return -EFAULT; + if (s < (int)sizeof(struct virtio_net_hdr)) + return -EINVAL; + + q->vnet_hdr_sz = s; + return 0; + + case TUNGETVNETLE: + s = !!(q->flags & TAP_VNET_LE); + if (put_user(s, sp)) + return -EFAULT; + return 0; + + case TUNSETVNETLE: + if (get_user(s, sp)) + return -EFAULT; + if (s) + q->flags |= TAP_VNET_LE; + else + q->flags &= ~TAP_VNET_LE; + return 0; + + case TUNGETVNETBE: + return tap_get_vnet_be(q, sp); + + case TUNSETVNETBE: + return tap_set_vnet_be(q, sp); + + case TUNSETOFFLOAD: + /* let the user check for future flags */ + if (arg & ~(TUN_F_CSUM | TUN_F_TSO4 | TUN_F_TSO6 | + TUN_F_TSO_ECN | TUN_F_UFO)) + return -EINVAL; + + rtnl_lock(); + ret = set_offload(q, arg); + rtnl_unlock(); + return ret; + + case SIOCGIFHWADDR: + rtnl_lock(); + tap = tap_get_tap_dev(q); + if (!tap) { + rtnl_unlock(); + return -ENOLINK; + } + ret = 0; + u = tap->dev->type; + if (copy_to_user(&ifr->ifr_name, tap->dev->name, IFNAMSIZ) || + copy_to_user(&ifr->ifr_hwaddr.sa_data, tap->dev->dev_addr, ETH_ALEN) || + put_user(u, &ifr->ifr_hwaddr.sa_family)) + ret = -EFAULT; + tap_put_tap_dev(tap); + rtnl_unlock(); + return ret; + + case SIOCSIFHWADDR: + if (copy_from_user(&sa, &ifr->ifr_hwaddr, sizeof(sa))) + return -EFAULT; + rtnl_lock(); + tap = tap_get_tap_dev(q); + if (!tap) { + rtnl_unlock(); + return -ENOLINK; + } + ret = dev_set_mac_address(tap->dev, &sa); + tap_put_tap_dev(tap); + rtnl_unlock(); + return ret; + + default: + return -EINVAL; + } +} + +#ifdef CONFIG_COMPAT +static long tap_compat_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + return tap_ioctl(file, cmd, (unsigned long)compat_ptr(arg)); +} +#endif + +const struct file_operations tap_fops = { + .owner = THIS_MODULE, + .open = tap_open, + .release = tap_release, + .read_iter = tap_read_iter, + .write_iter = tap_write_iter, + .poll = tap_poll, + .llseek = no_llseek, + .unlocked_ioctl = tap_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = tap_compat_ioctl, +#endif +}; + +static int tap_sendmsg(struct socket *sock, struct msghdr *m, + size_t total_len) +{ + struct tap_queue *q = container_of(sock, struct tap_queue, sock); + return tap_get_user(q, m, &m->msg_iter, m->msg_flags & MSG_DONTWAIT); +} + +static int tap_recvmsg(struct socket *sock, struct msghdr *m, + size_t total_len, int flags) +{ + struct tap_queue *q = container_of(sock, struct tap_queue, sock); + int ret; + if (flags & ~(MSG_DONTWAIT|MSG_TRUNC)) + return -EINVAL; + ret = tap_do_read(q, &m->msg_iter, flags & MSG_DONTWAIT); + if (ret > total_len) { + m->msg_flags |= MSG_TRUNC; + ret = flags & MSG_TRUNC ? ret : total_len; + } + return ret; +} + +static int tap_peek_len(struct socket *sock) +{ + struct tap_queue *q = container_of(sock, struct tap_queue, + sock); + return skb_array_peek_len(&q->skb_array); +} + +/* Ops structure to mimic raw sockets with tun */ +static const struct proto_ops tap_socket_ops = { + .sendmsg = tap_sendmsg, + .recvmsg = tap_recvmsg, + .peek_len = tap_peek_len, +}; + +/* Get an underlying socket object from tun file. Returns error unless file is + * attached to a device. The returned object works like a packet socket, it + * can be used for sock_sendmsg/sock_recvmsg. The caller is responsible for + * holding a reference to the file for as long as the socket is in use. */ +struct socket *tap_get_socket(struct file *file) +{ + struct tap_queue *q; + if (file->f_op != &tap_fops) + return ERR_PTR(-EINVAL); + q = file->private_data; + if (!q) + return ERR_PTR(-EBADFD); + return &q->sock; +} +EXPORT_SYMBOL_GPL(tap_get_socket); + +int tap_queue_resize(struct tap_dev *tap) +{ + struct net_device *dev = tap->dev; + struct tap_queue *q; + struct skb_array **arrays; + int n = tap->numqueues; + int ret, i = 0; + + arrays = kmalloc(sizeof *arrays * n, GFP_KERNEL); + if (!arrays) + return -ENOMEM; + + list_for_each_entry(q, &tap->queue_list, next) + arrays[i++] = &q->skb_array; + + ret = skb_array_resize_multiple(arrays, n, + dev->tx_queue_len, GFP_KERNEL); + + kfree(arrays); + return ret; +} +EXPORT_SYMBOL_GPL(tap_queue_resize); + +static int tap_list_add(dev_t major, const char *device_name) +{ + struct major_info *tap_major; + + tap_major = kzalloc(sizeof(*tap_major), GFP_ATOMIC); + if (!tap_major) + return -ENOMEM; + + tap_major->major = MAJOR(major); + + idr_init(&tap_major->minor_idr); + mutex_init(&tap_major->minor_lock); + + tap_major->device_name = device_name; + + list_add_tail_rcu(&tap_major->next, &major_list); + return 0; +} + +int tap_create_cdev(struct cdev *tap_cdev, + dev_t *tap_major, const char *device_name) +{ + int err; + + err = alloc_chrdev_region(tap_major, 0, TAP_NUM_DEVS, device_name); + if (err) + goto out1; + + cdev_init(tap_cdev, &tap_fops); + err = cdev_add(tap_cdev, *tap_major, TAP_NUM_DEVS); + if (err) + goto out2; + + err = tap_list_add(*tap_major, device_name); + if (err) + goto out3; + + return 0; + +out3: + cdev_del(tap_cdev); +out2: + unregister_chrdev_region(*tap_major, TAP_NUM_DEVS); +out1: + return err; +} +EXPORT_SYMBOL_GPL(tap_create_cdev); + +void tap_destroy_cdev(dev_t major, struct cdev *tap_cdev) +{ + struct major_info *tap_major, *tmp; + + cdev_del(tap_cdev); + unregister_chrdev_region(major, TAP_NUM_DEVS); + list_for_each_entry_safe(tap_major, tmp, &major_list, next) { + if (tap_major->major == MAJOR(major)) { + idr_destroy(&tap_major->minor_idr); + list_del_rcu(&tap_major->next); + kfree_rcu(tap_major, rcu); + } + } +} +EXPORT_SYMBOL_GPL(tap_destroy_cdev); + +MODULE_AUTHOR("Arnd Bergmann "); +MODULE_AUTHOR("Sainath Grandhi "); +MODULE_LICENSE("GPL"); diff --git a/drivers/vhost/Kconfig b/drivers/vhost/Kconfig index 40764ecad9ce16724535b9867bb6a1aa7758403b..cfdecea5078f4095546adcee397213cd0f2b67b1 100644 --- a/drivers/vhost/Kconfig +++ b/drivers/vhost/Kconfig @@ -1,6 +1,6 @@ config VHOST_NET tristate "Host kernel accelerator for virtio net" - depends on NET && EVENTFD && (TUN || !TUN) && (MACVTAP || !MACVTAP) + depends on NET && EVENTFD && (TUN || !TUN) && (TAP || !TAP) select VHOST ---help--- This kernel module can be loaded in host kernel to accelerate diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c index c42e9c30513417815ccc72a47ed298b11c0d5211..2fe35354f20e5ea82509fc9859435029a5a2dfcc 100644 --- a/drivers/vhost/net.c +++ b/drivers/vhost/net.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -960,7 +961,7 @@ static struct socket *get_tap_socket(int fd) sock = tun_get_socket(file); if (!IS_ERR(sock)) return sock; - sock = macvtap_get_socket(file); + sock = tap_get_socket(file); if (IS_ERR(sock)) fput(file); return sock; diff --git a/include/linux/if_macvlan.h b/include/linux/if_macvlan.h index a4ccc3122f9389ce93dda50d388057cfcd1b1bbb..c9ec1343d1879cc463ffe50afcee143623350b5d 100644 --- a/include/linux/if_macvlan.h +++ b/include/linux/if_macvlan.h @@ -9,19 +9,6 @@ #include #include -#if IS_ENABLED(CONFIG_MACVTAP) -struct socket *macvtap_get_socket(struct file *); -#else -#include -#include -struct file; -struct socket; -static inline struct socket *macvtap_get_socket(struct file *f) -{ - return ERR_PTR(-EINVAL); -} -#endif /* CONFIG_MACVTAP */ - struct macvlan_port; struct macvtap_queue; @@ -29,7 +16,7 @@ struct macvtap_queue; * Maximum times a macvtap device can be opened. This can be used to * configure the number of receive queue, e.g. for multiqueue virtio. */ -#define MAX_MACVTAP_QUEUES 256 +#define MAX_TAP_QUEUES 256 #define MACVLAN_MC_FILTER_BITS 8 #define MACVLAN_MC_FILTER_SZ (1 << MACVLAN_MC_FILTER_BITS) @@ -49,7 +36,7 @@ struct macvlan_dev { enum macvlan_mode mode; u16 flags; /* This array tracks active taps. */ - struct macvtap_queue __rcu *taps[MAX_MACVTAP_QUEUES]; + struct tap_queue __rcu *taps[MAX_TAP_QUEUES]; /* This list tracks all taps (both enabled and disabled) */ struct list_head queue_list; int numvtaps; diff --git a/include/linux/if_tap.h b/include/linux/if_tap.h new file mode 100644 index 0000000000000000000000000000000000000000..3482c3c2037d8c35a3825bc1b16ff573c9d60905 --- /dev/null +++ b/include/linux/if_tap.h @@ -0,0 +1,75 @@ +#ifndef _LINUX_IF_TAP_H_ +#define _LINUX_IF_TAP_H_ + +#if IS_ENABLED(CONFIG_TAP) +struct socket *tap_get_socket(struct file *); +#else +#include +#include +struct file; +struct socket; +static inline struct socket *tap_get_socket(struct file *f) +{ + return ERR_PTR(-EINVAL); +} +#endif /* CONFIG_TAP */ + +#include +#include + +#define MAX_TAP_QUEUES 256 + +struct tap_queue; + +struct tap_dev { + struct net_device *dev; + u16 flags; + /* This array tracks active taps. */ + struct tap_queue __rcu *taps[MAX_TAP_QUEUES]; + /* This list tracks all taps (both enabled and disabled) */ + struct list_head queue_list; + int numvtaps; + int numqueues; + netdev_features_t tap_features; + int minor; + + void (*update_features)(struct tap_dev *tap, netdev_features_t features); + void (*count_tx_dropped)(struct tap_dev *tap); + void (*count_rx_dropped)(struct tap_dev *tap); +}; + +/* + * A tap queue is the central object of tap module, it connects + * an open character device to virtual interface. There can be + * multiple queues on one interface, which map back to queues + * implemented in hardware on the underlying device. + * + * tap_proto is used to allocate queues through the sock allocation + * mechanism. + * + */ + +struct tap_queue { + struct sock sk; + struct socket sock; + struct socket_wq wq; + int vnet_hdr_sz; + struct tap_dev __rcu *tap; + struct file *file; + unsigned int flags; + u16 queue_index; + bool enabled; + struct list_head next; + struct skb_array skb_array; +}; + +rx_handler_result_t tap_handle_frame(struct sk_buff **pskb); +void tap_del_queues(struct tap_dev *tap); +int tap_get_minor(dev_t major, struct tap_dev *tap); +void tap_free_minor(dev_t major, struct tap_dev *tap); +int tap_queue_resize(struct tap_dev *tap); +int tap_create_cdev(struct cdev *tap_cdev, + dev_t *tap_major, const char *device_name); +void tap_destroy_cdev(dev_t major, struct cdev *tap_cdev); + +#endif /*_LINUX_IF_TAP_H_*/