diff --git a/drivers/crypto/hisilicon/migration/acc_vf_migration.c b/drivers/crypto/hisilicon/migration/acc_vf_migration.c
index e4164aed173e4ea71a2b54ba000b6705d5b1144c..da8b80937137f8f7c5b1feb6933b09e41b403731 100644
--- a/drivers/crypto/hisilicon/migration/acc_vf_migration.c
+++ b/drivers/crypto/hisilicon/migration/acc_vf_migration.c
@@ -1739,6 +1739,8 @@ static void acc_vf_remove(void *vendor_data)
 static struct vfio_pci_vendor_driver_ops  sec_vf_mig_ops = {
 	.owner		= THIS_MODULE,
 	.name		= "hisi_sec2",
+	.vendor		= PCI_VENDOR_ID_HUAWEI,
+	.device		= PCI_DEVICE_ID_HUAWEI_SEC_VF,
 	.probe		= acc_vf_probe,
 	.remove		= acc_vf_remove,
 	.device_ops	= &acc_vf_device_ops_node,
@@ -1747,6 +1749,8 @@ static struct vfio_pci_vendor_driver_ops  sec_vf_mig_ops = {
 static struct vfio_pci_vendor_driver_ops  hpre_vf_mig_ops = {
 	.owner		= THIS_MODULE,
 	.name		= "hisi_hpre",
+	.vendor		= PCI_VENDOR_ID_HUAWEI,
+	.device		= PCI_DEVICE_ID_HUAWEI_HPRE_VF,
 	.probe		= acc_vf_probe,
 	.remove		= acc_vf_remove,
 	.device_ops	= &acc_vf_device_ops_node,
@@ -1755,6 +1759,8 @@ static struct vfio_pci_vendor_driver_ops  hpre_vf_mig_ops = {
 static struct vfio_pci_vendor_driver_ops  zip_vf_mig_ops = {
 	.owner		= THIS_MODULE,
 	.name		= "hisi_zip",
+	.vendor		= PCI_VENDOR_ID_HUAWEI,
+	.device		= PCI_DEVICE_ID_HUAWEI_ZIP_VF,
 	.probe		= acc_vf_probe,
 	.remove		= acc_vf_remove,
 	.device_ops	= &acc_vf_device_ops_node,
diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c
index 6d7ae4e3b98318f7edaabc020822098098612dbc..2382222b096051c4053d86c0ab3dc0104f25305f 100644
--- a/drivers/vfio/pci/vfio_pci.c
+++ b/drivers/vfio/pci/vfio_pci.c
@@ -2077,6 +2077,10 @@ static int probe_vendor_drivers(struct vfio_pci_device *vdev)
 	list_for_each_entry(driver, &vfio_pci.vendor_drivers_list, next) {
 		void *data;
 
+		if (vdev->pdev->vendor != driver->ops->vendor ||
+		    vdev->pdev->device != driver->ops->device)
+			continue;
+
 		if (!try_module_get(driver->ops->owner))
 			continue;
 
diff --git a/include/linux/vfio.h b/include/linux/vfio.h
index 9a2217f13753a1186b2e9e569cb60484a4f47e19..e8841446a2eba81d3b6b73f9ca8536b807645a2d 100644
--- a/include/linux/vfio.h
+++ b/include/linux/vfio.h
@@ -253,6 +253,9 @@ extern int vfio_pci_set_vendor_regions(void *device_data,
 struct vfio_pci_vendor_driver_ops {
 	char			*name;
 	struct module		*owner;
+	/* Used to match device */
+	unsigned short		vendor;
+	unsigned short		device;
 	void			*(*probe)(struct pci_dev *pdev);
 	void			(*remove)(void *vendor_data);
 	struct vfio_device_ops *device_ops;