diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c
index 94cacae76aca41e6e7feb7575c7999a414145c49..4c11b810a4477c2f657ba6a58cec47809547eee0 100644
--- a/net/ipv4/ip_output.c
+++ b/net/ipv4/ip_output.c
@@ -1090,7 +1090,8 @@ static int __ip_append_data(struct sock *sk,
 		length -= copy;
 	}
 
-	refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc);
+	if (wmem_alloc_delta)
+		refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc);
 	return 0;
 
 error_efault:
diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c
index 66a768b7b8fb796670bf69fc451264a72d153228..b8ee50e94af384b1a9b404845291e6967517bf88 100644
--- a/net/ipv6/ip6_output.c
+++ b/net/ipv6/ip6_output.c
@@ -1545,7 +1545,8 @@ static int __ip6_append_data(struct sock *sk,
 		length -= copy;
 	}
 
-	refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc);
+	if (wmem_alloc_delta)
+		refcount_add(wmem_alloc_delta, &sk->sk_wmem_alloc);
 	return 0;
 
 error_efault: