diff --git a/drivers/infiniband/core/restrack.c b/drivers/infiniband/core/restrack.c
index e1d9934d6e81427a6d4a578385239df9846ab588..4cad0cd9aa0c904049136f265e25e36e4a558009 100644
--- a/drivers/infiniband/core/restrack.c
+++ b/drivers/infiniband/core/restrack.c
@@ -113,13 +113,15 @@ void rdma_restrack_add(struct rdma_restrack_entry *res)
 	if (!dev)
 		return;
 
+	if (res->type != RDMA_RESTRACK_CM_ID || !res_is_user(res))
+		res->task = NULL;
+
 	if (res_is_user(res)) {
 		if (!res->task)
 			rdma_restrack_set_task(res, current);
 		res->kern_name = NULL;
 	} else {
 		set_kern_name(res);
-		res->task = NULL;
 	}
 
 	kref_init(&res->kref);