diff --git a/net/core/dev.c b/net/core/dev.c
index 988e42912e722c07739cf82f2f5b04c1310cd7c2..cdcb9cbedf41fe496fa38a559a342b58757884a3 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -2432,8 +2432,10 @@ static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
 			return NET_RX_SUCCESS;
 		}
 
-		/* Schedule NAPI for backlog device */
-		if (napi_schedule_prep(&sd->backlog)) {
+		/* Schedule NAPI for backlog device
+		 * We can use non atomic operation since we own the queue lock
+		 */
+		if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) {
 			if (!rps_ipi_queued(sd))
 				____napi_schedule(sd, &sd->backlog);
 		}