diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c
index 064454aee86322c510f4d76f27fc2da4c656669f..1c59e474973667be74cc5ae14d30654a074ee290 100644
--- a/drivers/infiniband/hw/mlx4/main.c
+++ b/drivers/infiniband/hw/mlx4/main.c
@@ -2692,6 +2692,8 @@ static void do_slave_init(struct mlx4_ib_dev *ibdev, int slave, int do_init)
 		spin_lock_irqsave(&ibdev->sriov.going_down_lock, flags);
 		if (!ibdev->sriov.is_going_down)
 			queue_work(ibdev->sriov.demux[i].ud_wq, &dm[i]->work);
+		else
+			kfree(dm[i]);
 		spin_unlock_irqrestore(&ibdev->sriov.going_down_lock, flags);
 	}
 out: