diff --git a/drivers/usb/renesas_usbhs/mod_gadget.c b/drivers/usb/renesas_usbhs/mod_gadget.c
index 6a3afa9b764ce7d84c7705e5ef7fea2f9b7fb11e..883b0120b45421d0bb10a103c9776a249b7028d3 100644
--- a/drivers/usb/renesas_usbhs/mod_gadget.c
+++ b/drivers/usb/renesas_usbhs/mod_gadget.c
@@ -845,7 +845,6 @@ static int usbhsg_gadget_start(struct usb_gadget *gadget,
 
 	/* first hook up the driver ... */
 	gpriv->driver = driver;
-	gpriv->gadget.dev.driver = &driver->driver;
 
 	return usbhsg_try_start(priv, USBHSG_STATUS_REGISTERD);
 }
@@ -861,7 +860,6 @@ static int usbhsg_gadget_stop(struct usb_gadget *gadget,
 		return -EINVAL;
 
 	usbhsg_try_stop(priv, USBHSG_STATUS_REGISTERD);
-	gpriv->gadget.dev.driver = NULL;
 	gpriv->driver = NULL;
 
 	return 0;