diff --git a/drivers/usb/gadget/ci_udc.c b/drivers/usb/gadget/ci_udc.c
index cdbdbcc5cac1caf357ee881070001aaf333bdd6e..cdb8f6fb3d09bd6fcb927bb0d9f571f710084ca2 100644
--- a/drivers/usb/gadget/ci_udc.c
+++ b/drivers/usb/gadget/ci_udc.c
@@ -1053,6 +1053,13 @@ int usb_gadget_unregister_driver(struct usb_gadget_driver *driver)
 	free(controller.items_mem);
 	free(controller.epts);
 
+#if CONFIG_IS_ENABLED(DM_USB)
+	usb_remove_ehci_gadget(&controller.ctrl);
+#else
+	usb_lowlevel_stop(0);
+	controller.ctrl = NULL;
+#endif
+
 	return 0;
 }
 
diff --git a/drivers/usb/host/usb-uclass.c b/drivers/usb/host/usb-uclass.c
index e5dda79b94c830aec37348d1a6d8138eaca73be8..8773824e05737fa577ff09aedec24b0b8e986633 100644
--- a/drivers/usb/host/usb-uclass.c
+++ b/drivers/usb/host/usb-uclass.c
@@ -411,6 +411,24 @@ int usb_setup_ehci_gadget(struct ehci_ctrl **ctlrp)
 	return 0;
 }
 
+int usb_remove_ehci_gadget(struct ehci_ctrl **ctlrp)
+{
+	struct udevice *dev;
+	int ret;
+
+	/* Find the old device and remove it */
+	ret = uclass_find_device_by_seq(UCLASS_USB, 0, true, &dev);
+	if (ret)
+		return ret;
+	ret = device_remove(dev, DM_REMOVE_NORMAL);
+	if (ret)
+		return ret;
+
+	*ctlrp = NULL;
+
+	return 0;
+}
+
 /* returns 0 if no match, 1 if match */
 static int usb_match_device(const struct usb_device_descriptor *desc,
 			    const struct usb_device_id *id)
diff --git a/include/usb.h b/include/usb.h
index fa9e09607e2c614ea06265ea67f51b6c040e22d3..5a7af882fb60109b21166cf98b691306578f87a3 100644
--- a/include/usb.h
+++ b/include/usb.h
@@ -921,6 +921,15 @@ struct ehci_ctrl;
  */
 int usb_setup_ehci_gadget(struct ehci_ctrl **ctlrp);
 
+/**
+ * usb_remove_ehci_gadget() - Remove a gadget USB device
+ *
+ * TODO(sjg@chromium.org): Tidy this up when USB gadgets can use driver model
+ *
+ * This provides a way to tell a controller to remove a USB device
+ */
+int usb_remove_ehci_gadget(struct ehci_ctrl **ctlrp);
+
 /**
  * usb_stor_reset() - Prepare to scan USB storage devices
  *