diff --git a/net/bluetooth/smp.c b/net/bluetooth/smp.c
index 2566a3e43bb5087327c6e93d1e12f09c156a4c93..641ce8b69d2a430d5c63e0b231671043fd4d1bc6 100644
--- a/net/bluetooth/smp.c
+++ b/net/bluetooth/smp.c
@@ -638,7 +638,7 @@ int smp_user_confirm_reply(struct hci_conn *hcon, u16 mgmt_op, __le32 passkey)
 
 	BT_DBG("");
 
-	if (!conn)
+	if (!conn || !test_bit(HCI_CONN_LE_SMP_PEND, &hcon->flags))
 		return -ENOTCONN;
 
 	smp = conn->smp_chan;