diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c
index 20c44e2e1e4e1e813a937a75d8f65a880324867f..396b3f1e7cc0371b74abc40c34b77e76cd4e772f 100644
--- a/net/packet/af_packet.c
+++ b/net/packet/af_packet.c
@@ -1519,10 +1519,10 @@ static void __fanout_unlink(struct sock *sk, struct packet_sock *po)
 
 static bool match_fanout_group(struct packet_type *ptype, struct sock *sk)
 {
-	if (ptype->af_packet_priv == (void *)((struct packet_sock *)sk)->fanout)
-		return true;
+	if (sk->sk_family != PF_PACKET)
+		return false;
 
-	return false;
+	return ptype->af_packet_priv == pkt_sk(sk)->fanout;
 }
 
 static void fanout_init_data(struct packet_fanout *f)