diff --git a/drivers/net/ethernet/sun/sunvnet.c b/drivers/net/ethernet/sun/sunvnet.c index 3652afd3ec78d9e395a7902d9fe3cc6da570b43a..c390a2784546545536833a18de19bf414b6aebd5 100644 --- a/drivers/net/ethernet/sun/sunvnet.c +++ b/drivers/net/ethernet/sun/sunvnet.c @@ -311,9 +311,7 @@ static int vnet_rx_one(struct vnet_port *port, unsigned int len, dev->stats.rx_packets++; dev->stats.rx_bytes += len; - - netif_rx(skb); - + napi_gro_receive(&port->napi, skb); return 0; out_free_skb: @@ -430,6 +428,7 @@ static int vnet_walk_rx_one(struct vnet_port *port, struct vio_driver_state *vio = &port->vio; int err; + BUG_ON(desc == NULL); if (IS_ERR(desc)) return PTR_ERR(desc); @@ -456,10 +455,11 @@ static int vnet_walk_rx_one(struct vnet_port *port, } static int vnet_walk_rx(struct vnet_port *port, struct vio_dring_state *dr, - u32 start, u32 end) + u32 start, u32 end, int *npkts, int budget) { struct vio_driver_state *vio = &port->vio; int ack_start = -1, ack_end = -1; + bool send_ack = true; end = (end == (u32) -1) ? prev_idx(start, dr) : next_idx(end, dr); @@ -471,6 +471,7 @@ static int vnet_walk_rx(struct vnet_port *port, struct vio_dring_state *dr, return err; if (err != 0) break; + (*npkts)++; if (ack_start == -1) ack_start = start; ack_end = start; @@ -482,13 +483,26 @@ static int vnet_walk_rx(struct vnet_port *port, struct vio_dring_state *dr, return err; ack_start = -1; } + if ((*npkts) >= budget) { + send_ack = false; + break; + } } if (unlikely(ack_start == -1)) ack_start = ack_end = prev_idx(start, dr); - return vnet_send_ack(port, dr, ack_start, ack_end, VIO_DRING_STOPPED); + if (send_ack) { + port->napi_resume = false; + return vnet_send_ack(port, dr, ack_start, ack_end, + VIO_DRING_STOPPED); + } else { + port->napi_resume = true; + port->napi_stop_idx = ack_end; + return 1; + } } -static int vnet_rx(struct vnet_port *port, void *msgbuf) +static int vnet_rx(struct vnet_port *port, void *msgbuf, int *npkts, + int budget) { struct vio_dring_data *pkt = msgbuf; struct vio_dring_state *dr = &port->vio.drings[VIO_DRIVER_RX_RING]; @@ -505,11 +519,13 @@ static int vnet_rx(struct vnet_port *port, void *msgbuf) return 0; } - dr->rcv_nxt++; + if (!port->napi_resume) + dr->rcv_nxt++; /* XXX Validate pkt->start_idx and pkt->end_idx XXX */ - return vnet_walk_rx(port, dr, pkt->start_idx, pkt->end_idx); + return vnet_walk_rx(port, dr, pkt->start_idx, pkt->end_idx, + npkts, budget); } static int idx_is_pending(struct vio_dring_state *dr, u32 end) @@ -542,9 +558,12 @@ static int vnet_ack(struct vnet_port *port, void *msgbuf) if (unlikely(!idx_is_pending(dr, end))) return 0; + vp = port->vp; + dev = vp->dev; /* sync for race conditions with vnet_start_xmit() and tell xmit it * is time to send a trigger. */ + netif_tx_lock(dev); dr->cons = next_idx(end, dr); desc = vio_dring_entry(dr, dr->cons); if (desc->hdr.state == VIO_DESC_READY && port->start_cons) { @@ -559,10 +578,8 @@ static int vnet_ack(struct vnet_port *port, void *msgbuf) } else { port->start_cons = true; } + netif_tx_unlock(dev); - - vp = port->vp; - dev = vp->dev; if (unlikely(netif_queue_stopped(dev) && vnet_tx_dring_avail(dr) >= VNET_TX_WAKEUP_THRESH(dr))) return 1; @@ -591,9 +608,8 @@ static int handle_mcast(struct vnet_port *port, void *msgbuf) return 0; } -static void maybe_tx_wakeup(unsigned long param) +static void maybe_tx_wakeup(struct vnet *vp) { - struct vnet *vp = (struct vnet *)param; struct net_device *dev = vp->dev; netif_tx_lock(dev); @@ -601,7 +617,8 @@ static void maybe_tx_wakeup(unsigned long param) struct vnet_port *port; int wake = 1; - list_for_each_entry(port, &vp->port_list, list) { + rcu_read_lock(); + list_for_each_entry_rcu(port, &vp->port_list, list) { struct vio_dring_state *dr; dr = &port->vio.drings[VIO_DRIVER_TX_RING]; @@ -611,38 +628,50 @@ static void maybe_tx_wakeup(unsigned long param) break; } } + rcu_read_unlock(); if (wake) netif_wake_queue(dev); } netif_tx_unlock(dev); } -static void vnet_event(void *arg, int event) +static inline bool port_is_up(struct vnet_port *vnet) +{ + struct vio_driver_state *vio = &vnet->vio; + + return !!(vio->hs_state & VIO_HS_COMPLETE); +} + +static int vnet_event_napi(struct vnet_port *port, int budget) { - struct vnet_port *port = arg; struct vio_driver_state *vio = &port->vio; - unsigned long flags; int tx_wakeup, err; + int npkts = 0; + int event = (port->rx_event & LDC_EVENT_RESET); - spin_lock_irqsave(&vio->lock, flags); - +ldc_ctrl: if (unlikely(event == LDC_EVENT_RESET || event == LDC_EVENT_UP)) { vio_link_state_change(vio, event); - spin_unlock_irqrestore(&vio->lock, flags); if (event == LDC_EVENT_RESET) { port->rmtu = 0; vio_port_up(vio); } - return; + port->rx_event = 0; + return 0; } + /* We may have multiple LDC events in rx_event. Unroll send_events() */ + event = (port->rx_event & LDC_EVENT_UP); + port->rx_event &= ~(LDC_EVENT_RESET|LDC_EVENT_UP); + if (event == LDC_EVENT_UP) + goto ldc_ctrl; + event = port->rx_event; + if (!(event & LDC_EVENT_DATA_READY)) + return 0; - if (unlikely(event != LDC_EVENT_DATA_READY)) { - pr_warn("Unexpected LDC event %d\n", event); - spin_unlock_irqrestore(&vio->lock, flags); - return; - } + /* we dont expect any other bits than RESET, UP, DATA_READY */ + BUG_ON(event != LDC_EVENT_DATA_READY); tx_wakeup = err = 0; while (1) { @@ -651,6 +680,21 @@ static void vnet_event(void *arg, int event) u64 raw[8]; } msgbuf; + if (port->napi_resume) { + struct vio_dring_data *pkt = + (struct vio_dring_data *)&msgbuf; + struct vio_dring_state *dr = + &port->vio.drings[VIO_DRIVER_RX_RING]; + + pkt->tag.type = VIO_TYPE_DATA; + pkt->tag.stype = VIO_SUBTYPE_INFO; + pkt->tag.stype_env = VIO_DRING_DATA; + pkt->seq = dr->rcv_nxt; + pkt->start_idx = next_idx(port->napi_stop_idx, dr); + pkt->end_idx = -1; + goto napi_resume; + } +ldc_read: err = ldc_read(vio->lp, &msgbuf, sizeof(msgbuf)); if (unlikely(err < 0)) { if (err == -ECONNRESET) @@ -667,10 +711,22 @@ static void vnet_event(void *arg, int event) err = vio_validate_sid(vio, &msgbuf.tag); if (err < 0) break; - +napi_resume: if (likely(msgbuf.tag.type == VIO_TYPE_DATA)) { if (msgbuf.tag.stype == VIO_SUBTYPE_INFO) { - err = vnet_rx(port, &msgbuf); + if (!port_is_up(port)) { + /* failures like handshake_failure() + * may have cleaned up dring, but + * NAPI polling may bring us here. + */ + err = -ECONNRESET; + break; + } + err = vnet_rx(port, &msgbuf, &npkts, budget); + if (npkts >= budget) + break; + if (npkts == 0 && err != -ECONNRESET) + goto ldc_read; } else if (msgbuf.tag.stype == VIO_SUBTYPE_ACK) { err = vnet_ack(port, &msgbuf); if (err > 0) @@ -691,15 +747,33 @@ static void vnet_event(void *arg, int event) if (err == -ECONNRESET) break; } - spin_unlock(&vio->lock); - /* Kick off a tasklet to wake the queue. We cannot call - * maybe_tx_wakeup directly here because we could deadlock on - * netif_tx_lock() with dev_watchdog() - */ if (unlikely(tx_wakeup && err != -ECONNRESET)) - tasklet_schedule(&port->vp->vnet_tx_wakeup); + maybe_tx_wakeup(port->vp); + return npkts; +} + +static int vnet_poll(struct napi_struct *napi, int budget) +{ + struct vnet_port *port = container_of(napi, struct vnet_port, napi); + struct vio_driver_state *vio = &port->vio; + int processed = vnet_event_napi(port, budget); + + if (processed < budget) { + napi_complete(napi); + vio_set_intr(vio->vdev->rx_ino, HV_INTR_ENABLED); + } + return processed; +} + +static void vnet_event(void *arg, int event) +{ + struct vnet_port *port = arg; + struct vio_driver_state *vio = &port->vio; + + port->rx_event |= event; + vio_set_intr(vio->vdev->rx_ino, HV_INTR_DISABLED); + napi_schedule(&port->napi); - local_irq_restore(flags); } static int __vnet_tx_trigger(struct vnet_port *port, u32 start) @@ -746,26 +820,19 @@ static int __vnet_tx_trigger(struct vnet_port *port, u32 start) return err; } -static inline bool port_is_up(struct vnet_port *vnet) -{ - struct vio_driver_state *vio = &vnet->vio; - - return !!(vio->hs_state & VIO_HS_COMPLETE); -} - struct vnet_port *__tx_port_find(struct vnet *vp, struct sk_buff *skb) { unsigned int hash = vnet_hashfn(skb->data); struct hlist_head *hp = &vp->port_hash[hash]; struct vnet_port *port; - hlist_for_each_entry(port, hp, hash) { + hlist_for_each_entry_rcu(port, hp, hash) { if (!port_is_up(port)) continue; if (ether_addr_equal(port->raddr, skb->data)) return port; } - list_for_each_entry(port, &vp->port_list, list) { + list_for_each_entry_rcu(port, &vp->port_list, list) { if (!port->switch_port) continue; if (!port_is_up(port)) @@ -775,18 +842,6 @@ struct vnet_port *__tx_port_find(struct vnet *vp, struct sk_buff *skb) return NULL; } -struct vnet_port *tx_port_find(struct vnet *vp, struct sk_buff *skb) -{ - struct vnet_port *ret; - unsigned long flags; - - spin_lock_irqsave(&vp->lock, flags); - ret = __tx_port_find(vp, skb); - spin_unlock_irqrestore(&vp->lock, flags); - - return ret; -} - static struct sk_buff *vnet_clean_tx_ring(struct vnet_port *port, unsigned *pending) { @@ -847,11 +902,10 @@ static void vnet_clean_timer_expire(unsigned long port0) struct vnet_port *port = (struct vnet_port *)port0; struct sk_buff *freeskbs; unsigned pending; - unsigned long flags; - spin_lock_irqsave(&port->vio.lock, flags); + netif_tx_lock(port->vp->dev); freeskbs = vnet_clean_tx_ring(port, &pending); - spin_unlock_irqrestore(&port->vio.lock, flags); + netif_tx_unlock(port->vp->dev); vnet_free_skbs(freeskbs); @@ -901,10 +955,9 @@ static inline struct sk_buff *vnet_skb_shape(struct sk_buff *skb, void **pstart, static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev) { struct vnet *vp = netdev_priv(dev); - struct vnet_port *port = tx_port_find(vp, skb); + struct vnet_port *port = NULL; struct vio_dring_state *dr; struct vio_net_desc *d; - unsigned long flags; unsigned int len; struct sk_buff *freeskbs = NULL; int i, err, txi; @@ -912,14 +965,15 @@ static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev) int nlen = 0; unsigned pending = 0; - if (unlikely(!port)) - goto out_dropped; - skb = vnet_skb_shape(skb, &start, &nlen); - if (unlikely(!skb)) goto out_dropped; + rcu_read_lock(); + port = __tx_port_find(vp, skb); + if (unlikely(!port)) + goto out_dropped; + if (skb->len > port->rmtu) { unsigned long localmtu = port->rmtu - ETH_HLEN; @@ -937,6 +991,7 @@ static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev) fl4.saddr = ip_hdr(skb)->saddr; rt = ip_route_output_key(dev_net(dev), &fl4); + rcu_read_unlock(); if (!IS_ERR(rt)) { skb_dst_set(skb, &rt->dst); icmp_send(skb, ICMP_DEST_UNREACH, @@ -951,8 +1006,6 @@ static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev) goto out_dropped; } - spin_lock_irqsave(&port->vio.lock, flags); - dr = &port->vio.drings[VIO_DRIVER_TX_RING]; if (unlikely(vnet_tx_dring_avail(dr) < 1)) { if (!netif_queue_stopped(dev)) { @@ -962,7 +1015,7 @@ static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev) netdev_err(dev, "BUG! Tx Ring full when queue awake!\n"); dev->stats.tx_errors++; } - spin_unlock_irqrestore(&port->vio.lock, flags); + rcu_read_unlock(); return NETDEV_TX_BUSY; } @@ -986,7 +1039,7 @@ static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev) (LDC_MAP_SHADOW | LDC_MAP_DIRECT | LDC_MAP_RW)); if (err < 0) { netdev_info(dev, "tx buffer map error %d\n", err); - goto out_dropped_unlock; + goto out_dropped; } port->tx_bufs[txi].ncookies = err; @@ -1039,7 +1092,7 @@ static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev) netdev_info(dev, "TX trigger error %d\n", err); d->hdr.state = VIO_DESC_FREE; dev->stats.tx_carrier_errors++; - goto out_dropped_unlock; + goto out_dropped; } ldc_start_done: @@ -1055,26 +1108,24 @@ static int vnet_start_xmit(struct sk_buff *skb, struct net_device *dev) netif_wake_queue(dev); } - spin_unlock_irqrestore(&port->vio.lock, flags); + (void)mod_timer(&port->clean_timer, jiffies + VNET_CLEAN_TIMEOUT); + rcu_read_unlock(); vnet_free_skbs(freeskbs); - (void)mod_timer(&port->clean_timer, jiffies + VNET_CLEAN_TIMEOUT); - return NETDEV_TX_OK; -out_dropped_unlock: - spin_unlock_irqrestore(&port->vio.lock, flags); - out_dropped: - if (skb) - dev_kfree_skb(skb); - vnet_free_skbs(freeskbs); if (pending) (void)mod_timer(&port->clean_timer, jiffies + VNET_CLEAN_TIMEOUT); else if (port) del_timer(&port->clean_timer); + if (port) + rcu_read_unlock(); + if (skb) + dev_kfree_skb(skb); + vnet_free_skbs(freeskbs); dev->stats.tx_dropped++; return NETDEV_TX_OK; } @@ -1204,18 +1255,17 @@ static void vnet_set_rx_mode(struct net_device *dev) { struct vnet *vp = netdev_priv(dev); struct vnet_port *port; - unsigned long flags; - spin_lock_irqsave(&vp->lock, flags); - if (!list_empty(&vp->port_list)) { - port = list_entry(vp->port_list.next, struct vnet_port, list); + rcu_read_lock(); + list_for_each_entry_rcu(port, &vp->port_list, list) { if (port->switch_port) { __update_mc_list(vp, dev); __send_mc_list(vp, port); + break; } } - spin_unlock_irqrestore(&vp->lock, flags); + rcu_read_unlock(); } static int vnet_change_mtu(struct net_device *dev, int new_mtu) @@ -1342,6 +1392,21 @@ static int vnet_port_alloc_tx_bufs(struct vnet_port *port) return err; } +#ifdef CONFIG_NET_POLL_CONTROLLER +static void vnet_poll_controller(struct net_device *dev) +{ + struct vnet *vp = netdev_priv(dev); + struct vnet_port *port; + unsigned long flags; + + spin_lock_irqsave(&vp->lock, flags); + if (!list_empty(&vp->port_list)) { + port = list_entry(vp->port_list.next, struct vnet_port, list); + napi_schedule(&port->napi); + } + spin_unlock_irqrestore(&vp->lock, flags); +} +#endif static LIST_HEAD(vnet_list); static DEFINE_MUTEX(vnet_list_mutex); @@ -1354,6 +1419,9 @@ static const struct net_device_ops vnet_ops = { .ndo_tx_timeout = vnet_tx_timeout, .ndo_change_mtu = vnet_change_mtu, .ndo_start_xmit = vnet_start_xmit, +#ifdef CONFIG_NET_POLL_CONTROLLER + .ndo_poll_controller = vnet_poll_controller, +#endif }; static struct vnet *vnet_new(const u64 *local_mac) @@ -1374,7 +1442,6 @@ static struct vnet *vnet_new(const u64 *local_mac) vp = netdev_priv(dev); spin_lock_init(&vp->lock); - tasklet_init(&vp->vnet_tx_wakeup, maybe_tx_wakeup, (unsigned long)vp); vp->dev = dev; INIT_LIST_HEAD(&vp->port_list); @@ -1434,7 +1501,6 @@ static void vnet_cleanup(void) vp = list_first_entry(&vnet_list, struct vnet, list); list_del(&vp->list); dev = vp->dev; - tasklet_kill(&vp->vnet_tx_wakeup); /* vio_unregister_driver() should have cleaned up port_list */ BUG_ON(!list_empty(&vp->port_list)); unregister_netdev(dev); @@ -1536,6 +1602,8 @@ static int vnet_port_probe(struct vio_dev *vdev, const struct vio_device_id *id) if (err) goto err_out_free_port; + netif_napi_add(port->vp->dev, &port->napi, vnet_poll, NAPI_POLL_WEIGHT); + err = vnet_port_alloc_tx_bufs(port); if (err) goto err_out_free_ldc; @@ -1550,10 +1618,11 @@ static int vnet_port_probe(struct vio_dev *vdev, const struct vio_device_id *id) spin_lock_irqsave(&vp->lock, flags); if (switch_port) - list_add(&port->list, &vp->port_list); + list_add_rcu(&port->list, &vp->port_list); else - list_add_tail(&port->list, &vp->port_list); - hlist_add_head(&port->hash, &vp->port_hash[vnet_hashfn(port->raddr)]); + list_add_tail_rcu(&port->list, &vp->port_list); + hlist_add_head_rcu(&port->hash, + &vp->port_hash[vnet_hashfn(port->raddr)]); spin_unlock_irqrestore(&vp->lock, flags); dev_set_drvdata(&vdev->dev, port); @@ -1564,6 +1633,7 @@ static int vnet_port_probe(struct vio_dev *vdev, const struct vio_device_id *id) setup_timer(&port->clean_timer, vnet_clean_timer_expire, (unsigned long)port); + napi_enable(&port->napi); vio_port_up(&port->vio); mdesc_release(hp); @@ -1571,6 +1641,7 @@ static int vnet_port_probe(struct vio_dev *vdev, const struct vio_device_id *id) return 0; err_out_free_ldc: + netif_napi_del(&port->napi); vio_ldc_free(&port->vio); err_out_free_port: @@ -1586,17 +1657,17 @@ static int vnet_port_remove(struct vio_dev *vdev) struct vnet_port *port = dev_get_drvdata(&vdev->dev); if (port) { - struct vnet *vp = port->vp; - unsigned long flags; del_timer_sync(&port->vio.timer); - del_timer_sync(&port->clean_timer); - spin_lock_irqsave(&vp->lock, flags); - list_del(&port->list); - hlist_del(&port->hash); - spin_unlock_irqrestore(&vp->lock, flags); + napi_disable(&port->napi); + + list_del_rcu(&port->list); + hlist_del_rcu(&port->hash); + synchronize_rcu(); + del_timer_sync(&port->clean_timer); + netif_napi_del(&port->napi); vnet_port_free_tx_bufs(port); vio_ldc_free(&port->vio); diff --git a/drivers/net/ethernet/sun/sunvnet.h b/drivers/net/ethernet/sun/sunvnet.h index c9110454261905bc60861eb34a7bfe7c268f0b9f..c8a862e471dd13b3e982d8ddba6b880d0f9ade52 100644 --- a/drivers/net/ethernet/sun/sunvnet.h +++ b/drivers/net/ethernet/sun/sunvnet.h @@ -56,6 +56,11 @@ struct vnet_port { struct timer_list clean_timer; u64 rmtu; + + struct napi_struct napi; + u32 napi_stop_idx; + bool napi_resume; + int rx_event; }; static inline struct vnet_port *to_vnet_port(struct vio_driver_state *vio) @@ -97,7 +102,6 @@ struct vnet { struct list_head list; u64 local_mac; - struct tasklet_struct vnet_tx_wakeup; }; #endif /* _SUNVNET_H */