提交 679abf38 编写于 作者: D Davidlohr Bueso 提交者: Linus Torvalds

fs/eventpoll.c: loosen irq safety in ep_poll()

Similar to other calls, ep_poll() is not called with interrupts disabled,
and we can therefore avoid the irq save/restore dance and just disable
local irqs.  In fact, the call should never be called in irq context at
all, considering that the only path is

epoll_wait(2) -> do_epoll_wait() -> ep_poll().

When running on a 2 socket 40-core (ht) IvyBridge a common pipe based
epoll_wait(2) microbenchmark, the following performance improvements are
seen:

    # threads       vanilla         dirty
	 1          1805587	    2106412
	 2          1854064	    2090762
	 4          1805484	    2017436
	 8          1751222	    1974475
	 16         1725299	    1962104
	 32         1378463	    1571233
	 64          787368	     900784

Which is a pretty constantly near 15%.

Also add a lockdep check such that we detect any mischief before
deadlocking.

Link: http://lkml.kernel.org/r/20180727053432.16679-2-dave@stgolabs.netSigned-off-by: NDavidlohr Bueso <dbueso@suse.de>
Reviewed-by: NAndrew Morton <akpm@linux-foundation.org>
Cc: Al Viro <viro@zeniv.linux.org.uk>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Jason Baron <jbaron@akamai.com>
Signed-off-by: NAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: NLinus Torvalds <torvalds@linux-foundation.org>
上级 514056d5
...@@ -1746,11 +1746,12 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events, ...@@ -1746,11 +1746,12 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events,
int maxevents, long timeout) int maxevents, long timeout)
{ {
int res = 0, eavail, timed_out = 0; int res = 0, eavail, timed_out = 0;
unsigned long flags;
u64 slack = 0; u64 slack = 0;
wait_queue_entry_t wait; wait_queue_entry_t wait;
ktime_t expires, *to = NULL; ktime_t expires, *to = NULL;
lockdep_assert_irqs_enabled();
if (timeout > 0) { if (timeout > 0) {
struct timespec64 end_time = ep_set_mstimeout(timeout); struct timespec64 end_time = ep_set_mstimeout(timeout);
...@@ -1763,7 +1764,7 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events, ...@@ -1763,7 +1764,7 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events,
* caller specified a non blocking operation. * caller specified a non blocking operation.
*/ */
timed_out = 1; timed_out = 1;
spin_lock_irqsave(&ep->wq.lock, flags); spin_lock_irq(&ep->wq.lock);
goto check_events; goto check_events;
} }
...@@ -1772,7 +1773,7 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events, ...@@ -1772,7 +1773,7 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events,
if (!ep_events_available(ep)) if (!ep_events_available(ep))
ep_busy_loop(ep, timed_out); ep_busy_loop(ep, timed_out);
spin_lock_irqsave(&ep->wq.lock, flags); spin_lock_irq(&ep->wq.lock);
if (!ep_events_available(ep)) { if (!ep_events_available(ep)) {
/* /*
...@@ -1814,11 +1815,11 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events, ...@@ -1814,11 +1815,11 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events,
break; break;
} }
spin_unlock_irqrestore(&ep->wq.lock, flags); spin_unlock_irq(&ep->wq.lock);
if (!schedule_hrtimeout_range(to, slack, HRTIMER_MODE_ABS)) if (!schedule_hrtimeout_range(to, slack, HRTIMER_MODE_ABS))
timed_out = 1; timed_out = 1;
spin_lock_irqsave(&ep->wq.lock, flags); spin_lock_irq(&ep->wq.lock);
} }
__remove_wait_queue(&ep->wq, &wait); __remove_wait_queue(&ep->wq, &wait);
...@@ -1828,7 +1829,7 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events, ...@@ -1828,7 +1829,7 @@ static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events,
/* Is it worth to try to dig for events ? */ /* Is it worth to try to dig for events ? */
eavail = ep_events_available(ep); eavail = ep_events_available(ep);
spin_unlock_irqrestore(&ep->wq.lock, flags); spin_unlock_irq(&ep->wq.lock);
/* /*
* Try to transfer events to user space. In case we get 0 events and * Try to transfer events to user space. In case we get 0 events and
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册