diff --git a/net/ipv4/fou.c b/net/ipv4/fou.c
index ff069f6597ace6302b46add285c787942645aac2..c8db62718e5f97a51feeba3177fd375a0c745519 100644
--- a/net/ipv4/fou.c
+++ b/net/ipv4/fou.c
@@ -410,7 +410,8 @@ static void fou_release(struct fou *fou)
 	struct socket *sock = fou->sock;
 	struct sock *sk = sock->sk;
 
-	udp_del_offload(&fou->udp_offloads);
+	if (sk->sk_family == AF_INET)
+		udp_del_offload(&fou->udp_offloads);
 
 	list_del(&fou->list);
 
@@ -528,7 +529,6 @@ static int fou_destroy(struct net *net, struct fou_cfg *cfg)
 	spin_lock(&fou_lock);
 	list_for_each_entry(fou, &fou_list, list) {
 		if (fou->port == port) {
-			udp_del_offload(&fou->udp_offloads);
 			fou_release(fou);
 			err = 0;
 			break;