diff --git a/drivers/usb/chipidea/ci_hdrc_imx.c b/drivers/usb/chipidea/ci_hdrc_imx.c
index cc53e63c9ce9390b4f86e018b40bf8d7ae1fcedd..74d998d9b45b28001e2f15f6026640ecb6e07085 100644
--- a/drivers/usb/chipidea/ci_hdrc_imx.c
+++ b/drivers/usb/chipidea/ci_hdrc_imx.c
@@ -27,57 +27,48 @@ struct ci_hdrc_imx_data {
 	struct usb_phy *phy;
 	struct platform_device *ci_pdev;
 	struct clk *clk;
+	struct imx_usbmisc_data *usbmisc_data;
 };
 
-static const struct usbmisc_ops *usbmisc_ops;
-
 /* Common functions shared by usbmisc drivers */
 
-int usbmisc_set_ops(const struct usbmisc_ops *ops)
-{
-	if (usbmisc_ops)
-		return -EBUSY;
-
-	usbmisc_ops = ops;
-
-	return 0;
-}
-EXPORT_SYMBOL_GPL(usbmisc_set_ops);
-
-void usbmisc_unset_ops(const struct usbmisc_ops *ops)
-{
-	usbmisc_ops = NULL;
-}
-EXPORT_SYMBOL_GPL(usbmisc_unset_ops);
-
-int usbmisc_get_init_data(struct device *dev, struct usbmisc_usb_device *usbdev)
+static struct imx_usbmisc_data *usbmisc_get_init_data(struct device *dev)
 {
 	struct device_node *np = dev->of_node;
 	struct of_phandle_args args;
+	struct imx_usbmisc_data *data;
 	int ret;
 
-	usbdev->dev = dev;
+	/*
+	 * In case the fsl,usbmisc property is not present this device doesn't
+	 * need usbmisc. Return NULL (which is no error here)
+	 */
+	if (!of_get_property(np, "fsl,usbmisc", NULL))
+		return NULL;
+
+	data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL);
+	if (!data)
+		return ERR_PTR(-ENOMEM);
 
 	ret = of_parse_phandle_with_args(np, "fsl,usbmisc", "#index-cells",
 					0, &args);
 	if (ret) {
 		dev_err(dev, "Failed to parse property fsl,usbmisc, errno %d\n",
 			ret);
-		memset(usbdev, 0, sizeof(*usbdev));
-		return ret;
+		return ERR_PTR(ret);
 	}
-	usbdev->index = args.args[0];
+
+	data->index = args.args[0];
 	of_node_put(args.np);
 
 	if (of_find_property(np, "disable-over-current", NULL))
-		usbdev->disable_oc = 1;
+		data->disable_oc = 1;
 
 	if (of_find_property(np, "external-vbus-divider", NULL))
-		usbdev->evdo = 1;
+		data->evdo = 1;
 
-	return 0;
+	return data;
 }
-EXPORT_SYMBOL_GPL(usbmisc_get_init_data);
 
 /* End of common functions shared by usbmisc drivers*/
 
@@ -92,16 +83,16 @@ static int ci_hdrc_imx_probe(struct platform_device *pdev)
 	};
 	int ret;
 
-	if (of_find_property(pdev->dev.of_node, "fsl,usbmisc", NULL)
-		&& !usbmisc_ops)
-		return -EPROBE_DEFER;
-
 	data = devm_kzalloc(&pdev->dev, sizeof(*data), GFP_KERNEL);
 	if (!data) {
 		dev_err(&pdev->dev, "Failed to allocate ci_hdrc-imx data!\n");
 		return -ENOMEM;
 	}
 
+	data->usbmisc_data = usbmisc_get_init_data(&pdev->dev);
+	if (IS_ERR(data->usbmisc_data))
+		return PTR_ERR(data->usbmisc_data);
+
 	data->clk = devm_clk_get(&pdev->dev, NULL);
 	if (IS_ERR(data->clk)) {
 		dev_err(&pdev->dev,
@@ -135,11 +126,11 @@ static int ci_hdrc_imx_probe(struct platform_device *pdev)
 	if (!pdev->dev.coherent_dma_mask)
 		pdev->dev.coherent_dma_mask = DMA_BIT_MASK(32);
 
-	if (usbmisc_ops && usbmisc_ops->init) {
-		ret = usbmisc_ops->init(&pdev->dev);
+	if (data->usbmisc_data) {
+		ret = imx_usbmisc_init(data->usbmisc_data);
 		if (ret) {
-			dev_err(&pdev->dev,
-				"usbmisc init failed, ret=%d\n", ret);
+			dev_err(&pdev->dev, "usbmisc init failed, ret=%d\n",
+					ret);
 			goto err_clk;
 		}
 	}
@@ -155,11 +146,11 @@ static int ci_hdrc_imx_probe(struct platform_device *pdev)
 		goto err_clk;
 	}
 
-	if (usbmisc_ops && usbmisc_ops->post) {
-		ret = usbmisc_ops->post(&pdev->dev);
+	if (data->usbmisc_data) {
+		ret = imx_usbmisc_init_post(data->usbmisc_data);
 		if (ret) {
-			dev_err(&pdev->dev,
-				"usbmisc post failed, ret=%d\n", ret);
+			dev_err(&pdev->dev, "usbmisc post failed, ret=%d\n",
+					ret);
 			goto disable_device;
 		}
 	}
diff --git a/drivers/usb/chipidea/ci_hdrc_imx.h b/drivers/usb/chipidea/ci_hdrc_imx.h
index 550bfa4576202c6f6f8f9c3f73ab8921971fd234..c7271590dd0a8af9c8e7f79508d5cf94be79ca75 100644
--- a/drivers/usb/chipidea/ci_hdrc_imx.h
+++ b/drivers/usb/chipidea/ci_hdrc_imx.h
@@ -9,23 +9,12 @@
  * http://www.gnu.org/copyleft/gpl.html
  */
 
-/* Used to set SoC specific callbacks */
-struct usbmisc_ops {
-	/* It's called once when probe a usb device */
-	int (*init)(struct device *dev);
-	/* It's called once after adding a usb device */
-	int (*post)(struct device *dev);
-};
-
-struct usbmisc_usb_device {
-	struct device *dev; /* usb controller device */
+struct imx_usbmisc_data {
 	int index;
 
 	unsigned int disable_oc:1; /* over current detect disabled */
 	unsigned int evdo:1; /* set external vbus divider option */
 };
 
-int usbmisc_set_ops(const struct usbmisc_ops *ops);
-void usbmisc_unset_ops(const struct usbmisc_ops *ops);
-int
-usbmisc_get_init_data(struct device *dev, struct usbmisc_usb_device *usbdev);
+int imx_usbmisc_init(struct imx_usbmisc_data *);
+int imx_usbmisc_init_post(struct imx_usbmisc_data *);
diff --git a/drivers/usb/chipidea/usbmisc_imx.c b/drivers/usb/chipidea/usbmisc_imx.c
index ac5a4615520011089be9958c76bf702c1a1b38c3..8a1094b1182f9bc625dd18f26b5e97d852e83b1f 100644
--- a/drivers/usb/chipidea/usbmisc_imx.c
+++ b/drivers/usb/chipidea/usbmisc_imx.c
@@ -18,8 +18,6 @@
 
 #include "ci_hdrc_imx.h"
 
-#define USB_DEV_MAX 4
-
 #define MX25_USB_PHY_CTRL_OFFSET	0x08
 #define MX25_BM_EXTERNAL_VBUS_DIVIDER	BIT(23)
 
@@ -32,51 +30,34 @@
 
 #define MX6_BM_OVER_CUR_DIS		BIT(7)
 
+struct usbmisc_ops {
+	/* It's called once when probe a usb device */
+	int (*init)(struct imx_usbmisc_data *data);
+	/* It's called once after adding a usb device */
+	int (*post)(struct imx_usbmisc_data *data);
+};
+
 struct imx_usbmisc {
 	void __iomem *base;
 	spinlock_t lock;
 	struct clk *clk;
-	struct usbmisc_usb_device usbdev[USB_DEV_MAX];
 	const struct usbmisc_ops *ops;
 };
 
 static struct imx_usbmisc *usbmisc;
 
-static struct usbmisc_usb_device *get_usbdev(struct device *dev)
-{
-	int i, ret;
-
-	for (i = 0; i < USB_DEV_MAX; i++) {
-		if (usbmisc->usbdev[i].dev == dev)
-			return &usbmisc->usbdev[i];
-		else if (!usbmisc->usbdev[i].dev)
-			break;
-	}
-
-	if (i >= USB_DEV_MAX)
-		return ERR_PTR(-EBUSY);
-
-	ret = usbmisc_get_init_data(dev, &usbmisc->usbdev[i]);
-	if (ret)
-		return ERR_PTR(ret);
-
-	return &usbmisc->usbdev[i];
-}
-
-static int usbmisc_imx25_post(struct device *dev)
+static int usbmisc_imx25_post(struct imx_usbmisc_data *data)
 {
-	struct usbmisc_usb_device *usbdev;
 	void __iomem *reg;
 	unsigned long flags;
 	u32 val;
 
-	usbdev = get_usbdev(dev);
-	if (IS_ERR(usbdev))
-		return PTR_ERR(usbdev);
+	if (data->index > 2)
+		return -EINVAL;
 
 	reg = usbmisc->base + MX25_USB_PHY_CTRL_OFFSET;
 
-	if (usbdev->evdo) {
+	if (data->evdo) {
 		spin_lock_irqsave(&usbmisc->lock, flags);
 		val = readl(reg);
 		writel(val | MX25_BM_EXTERNAL_VBUS_DIVIDER, reg);
@@ -87,20 +68,18 @@ static int usbmisc_imx25_post(struct device *dev)
 	return 0;
 }
 
-static int usbmisc_imx53_init(struct device *dev)
+static int usbmisc_imx53_init(struct imx_usbmisc_data *data)
 {
-	struct usbmisc_usb_device *usbdev;
 	void __iomem *reg = NULL;
 	unsigned long flags;
 	u32 val = 0;
 
-	usbdev = get_usbdev(dev);
-	if (IS_ERR(usbdev))
-		return PTR_ERR(usbdev);
+	if (data->index > 3)
+		return -EINVAL;
 
-	if (usbdev->disable_oc) {
+	if (data->disable_oc) {
 		spin_lock_irqsave(&usbmisc->lock, flags);
-		switch (usbdev->index) {
+		switch (data->index) {
 		case 0:
 			reg = usbmisc->base + MX53_USB_OTG_PHY_CTRL_0_OFFSET;
 			val = readl(reg) | MX53_BM_OVER_CUR_DIS_OTG;
@@ -126,22 +105,19 @@ static int usbmisc_imx53_init(struct device *dev)
 	return 0;
 }
 
-static int usbmisc_imx6q_init(struct device *dev)
+static int usbmisc_imx6q_init(struct imx_usbmisc_data *data)
 {
-
-	struct usbmisc_usb_device *usbdev;
 	unsigned long flags;
 	u32 reg;
 
-	usbdev = get_usbdev(dev);
-	if (IS_ERR(usbdev))
-		return PTR_ERR(usbdev);
+	if (data->index > 3)
+		return -EINVAL;
 
-	if (usbdev->disable_oc) {
+	if (data->disable_oc) {
 		spin_lock_irqsave(&usbmisc->lock, flags);
-		reg = readl(usbmisc->base + usbdev->index * 4);
+		reg = readl(usbmisc->base + data->index * 4);
 		writel(reg | MX6_BM_OVER_CUR_DIS,
-			usbmisc->base + usbdev->index * 4);
+			usbmisc->base + data->index * 4);
 		spin_unlock_irqrestore(&usbmisc->lock, flags);
 	}
 
@@ -160,6 +136,26 @@ static const struct usbmisc_ops imx6q_usbmisc_ops = {
 	.init = usbmisc_imx6q_init,
 };
 
+int imx_usbmisc_init(struct imx_usbmisc_data *data)
+{
+	if (!usbmisc)
+		return -EPROBE_DEFER;
+	if (!usbmisc->ops->init)
+		return 0;
+	return usbmisc->ops->init(data);
+}
+EXPORT_SYMBOL_GPL(imx_usbmisc_init);
+
+int imx_usbmisc_init_post(struct imx_usbmisc_data *data)
+{
+	if (!usbmisc)
+		return -EPROBE_DEFER;
+	if (!usbmisc->ops->post)
+		return 0;
+	return usbmisc->ops->post(data);
+}
+EXPORT_SYMBOL_GPL(imx_usbmisc_init_post);
+
 static const struct of_device_id usbmisc_imx_dt_ids[] = {
 	{
 		.compatible = "fsl,imx25-usbmisc",
@@ -216,19 +212,12 @@ static int usbmisc_imx_probe(struct platform_device *pdev)
 		of_match_device(usbmisc_imx_dt_ids, &pdev->dev);
 	data->ops = (const struct usbmisc_ops *)tmp_dev->data;
 	usbmisc = data;
-	ret = usbmisc_set_ops(data->ops);
-	if (ret) {
-		usbmisc = NULL;
-		clk_disable_unprepare(data->clk);
-		return ret;
-	}
 
 	return 0;
 }
 
 static int usbmisc_imx_remove(struct platform_device *pdev)
 {
-	usbmisc_unset_ops(usbmisc->ops);
 	clk_disable_unprepare(usbmisc->clk);
 	usbmisc = NULL;
 	return 0;