diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c
index cac0273ec447930d281074f5cd06985ff118e385..e9969a2fc8462116a5b8cb256e1e8cbc97a7fd3c 100644
--- a/security/selinux/hooks.c
+++ b/security/selinux/hooks.c
@@ -3619,7 +3619,9 @@ static void selinux_sock_graft(struct sock* sk, struct socket *parent)
 	struct inode_security_struct *isec = SOCK_INODE(parent)->i_security;
 	struct sk_security_struct *sksec = sk->sk_security;
 
-	isec->sid = sksec->sid;
+	if (sk->sk_family == PF_INET || sk->sk_family == PF_INET6 ||
+	    sk->sk_family == PF_UNIX)
+		isec->sid = sksec->sid;
 
 	selinux_netlbl_sock_graft(sk, parent);
 }