diff --git a/drivers/usb/gadget/udc/amd5536udc.c b/drivers/usb/gadget/udc/amd5536udc.c
index 3f85044be196486790ca365c33b3b1a72385a1d7..6223b1be029f9f39fecc969a6b408760f0a84e58 100644
--- a/drivers/usb/gadget/udc/amd5536udc.c
+++ b/drivers/usb/gadget/udc/amd5536udc.c
@@ -3127,7 +3127,8 @@ static void udc_pci_remove(struct pci_dev *pdev)
 
 	usb_del_gadget_udc(&udc->gadget);
 	/* gadget driver must not be registered */
-	BUG_ON(dev->driver != NULL);
+	if (WARN_ON(dev->driver))
+		return;
 
 	/* dma pool cleanup */
 	if (dev->data_requests)