diff --git a/drivers/usb/gadget/lpc32xx_udc.c b/drivers/usb/gadget/lpc32xx_udc.c
index 14783278390085f36297b2a1c8a98f471605e8be..b943d8cdfbf75d056eb50cba2899ace0fbdb55a5 100644
--- a/drivers/usb/gadget/lpc32xx_udc.c
+++ b/drivers/usb/gadget/lpc32xx_udc.c
@@ -2946,7 +2946,6 @@ static int lpc32xx_start(struct usb_gadget *gadget,
 	}
 
 	udc->driver = driver;
-	udc->gadget.dev.driver = &driver->driver;
 	udc->gadget.dev.of_node = udc->dev->of_node;
 	udc->enabled = 1;
 	udc->selfpowered = 1;
@@ -2995,7 +2994,6 @@ static int lpc32xx_stop(struct usb_gadget *gadget,
 	}
 
 	udc->enabled = 0;
-	udc->gadget.dev.driver = NULL;
 	udc->driver = NULL;
 
 	return 0;