diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index 69bb0892e8876ca1b56521086070908f7c96ef54..f383a4f50ab01289069365f89dfdfcfbbfb56a20 100644
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -932,6 +932,7 @@ static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
 	mutex_unlock(&lock);
 	if (ret) {
 		ret = -ENODEV;
+		cma_exch(conn_id, CMA_DESTROYING);
 		cma_release_remove(conn_id);
 		rdma_destroy_id(&conn_id->id);
 		goto out;