diff --git a/drivers/usb/gadget/pch_udc.c b/drivers/usb/gadget/pch_udc.c
index c1db902ebb7fa6fe1f1269ddf58071ee1ee95abc..79cf2966f6341167f138df9433c9fbba7c1ba081 100644
--- a/drivers/usb/gadget/pch_udc.c
+++ b/drivers/usb/gadget/pch_udc.c
@@ -2988,7 +2988,6 @@ static int pch_udc_start(struct usb_gadget *g,
 
 	driver->driver.bus = NULL;
 	dev->driver = driver;
-	dev->gadget.dev.driver = &driver->driver;
 
 	/* get ready for ep0 traffic */
 	pch_udc_setup_ep0(dev);
@@ -3009,7 +3008,6 @@ static int pch_udc_stop(struct usb_gadget *g,
 	pch_udc_disable_interrupts(dev, UDC_DEVINT_MSK);
 
 	/* Assures that there are no pending requests with this driver */
-	dev->gadget.dev.driver = NULL;
 	dev->driver = NULL;
 	dev->connected = 0;