diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index 43e72e6b24ae43e5939352eabd05f5df255a8d72..02c33ff13bdc6e95f56fe2a0d2eb000cfceffb6c 100644
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -1302,7 +1302,8 @@ static void udp_rmem_release(struct sock *sk, int size, int partial,
 	if (likely(partial)) {
 		up->forward_deficit += size;
 		size = up->forward_deficit;
-		if (size < (sk->sk_rcvbuf >> 2))
+		if (size < (sk->sk_rcvbuf >> 2) &&
+		    !skb_queue_empty(&up->reader_queue))
 			return;
 	} else {
 		size += up->forward_deficit;