diff --git a/net/key/af_key.c b/net/key/af_key.c
index 9bba7ac5fee0d30a986c49ed0b43e664b056d975..7470e367272b6059751eca446c7fc1f0d08edc13 100644
--- a/net/key/af_key.c
+++ b/net/key/af_key.c
@@ -3030,6 +3030,9 @@ static int key_notify_sa_expire(struct xfrm_state *x, struct km_event *c)
 
 static int pfkey_send_notify(struct xfrm_state *x, struct km_event *c)
 {
+	if (atomic_read(&pfkey_socks_nr) == 0)
+		return 0;
+
 	switch (c->event) {
 	case XFRM_MSG_EXPIRE:
 		return key_notify_sa_expire(x, c);