diff --git a/net/ipv4/esp4.c b/net/ipv4/esp4.c
index 97689012b357b9fd826f3093def4df9a475d2274..12a43a5369a54df3b31630a5b0f13e3e1441cbeb 100644
--- a/net/ipv4/esp4.c
+++ b/net/ipv4/esp4.c
@@ -325,7 +325,7 @@ int esp_output_head(struct xfrm_state *x, struct sk_buff *skb, struct esp_info *
 			skb->len += tailen;
 			skb->data_len += tailen;
 			skb->truesize += tailen;
-			if (sk)
+			if (sk && sk_fullsock(sk))
 				refcount_add(tailen, &sk->sk_wmem_alloc);
 
 			goto out;
diff --git a/net/ipv6/esp6.c b/net/ipv6/esp6.c
index 88a7579c23bdb3ae432a126bb0781493bc8d60b7..a7d996148eeda138f205839840c7d092aef820af 100644
--- a/net/ipv6/esp6.c
+++ b/net/ipv6/esp6.c
@@ -293,7 +293,7 @@ int esp6_output_head(struct xfrm_state *x, struct sk_buff *skb, struct esp_info
 			skb->len += tailen;
 			skb->data_len += tailen;
 			skb->truesize += tailen;
-			if (sk)
+			if (sk && sk_fullsock(sk))
 				refcount_add(tailen, &sk->sk_wmem_alloc);
 
 			goto out;