From 16c9890383fd3f2bc43720a169971aeedb643aa1 Mon Sep 17 00:00:00 2001 From: Daniel Henrique Barboza Date: Thu, 29 Aug 2019 16:18:58 -0300 Subject: [PATCH] virpcimock.c: mock /dev/vfio This patch adds mock of the /dev/vfio path, needed for proper implementation of the support for multifunction/multiple devices per iommu groups. To do that, the existing bind and unbind operations were adapted to operate with the mocked filesystem as well. Signed-off-by: Daniel Henrique Barboza Signed-off-by: Michal Privoznik Reviewed-by: Michal Privoznik --- tests/virpcimock.c | 157 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 141 insertions(+), 16 deletions(-) diff --git a/tests/virpcimock.c b/tests/virpcimock.c index a5045ed97c..9f6ca24737 100644 --- a/tests/virpcimock.c +++ b/tests/virpcimock.c @@ -105,6 +105,11 @@ struct pciDriver { size_t len; /* @len is used for both @vendor and @device */ }; +struct pciIommuGroup { + int iommu; + size_t nDevicesBoundToVFIO; /* Indicates the devices in the group */ +}; + struct pciDeviceAddress { unsigned int domain; unsigned int bus; @@ -133,6 +138,9 @@ size_t nPCIDevices = 0; struct pciDriver **pciDrivers = NULL; size_t nPCIDrivers = 0; +struct pciIommuGroup **pciIommuGroups = NULL; +size_t npciIommuGroups = 0; + struct fdCallback *callbacks = NULL; size_t nCallbacks = 0; @@ -254,6 +262,15 @@ getrealpath(char **newpath, errno = ENOMEM; return -1; } + } else if (STRPREFIX(path, "/sys/kernel/") || + STRPREFIX(path, "/dev/vfio/")) { + + if (virAsprintfQuiet(newpath, "%s/%s", + fakerootdir, + path) < 0) { + errno = ENOMEM; + return -1; + } } else { if (VIR_STRDUP_QUIET(*newpath, path) < 0) return -1; @@ -389,8 +406,10 @@ static void pci_device_create_iommu(const struct pciDevice *dev, const char *devid) { + struct pciIommuGroup *iommuGroup; VIR_AUTOFREE(char *) iommuPath = NULL; char tmp[256]; + size_t i; if (virAsprintfQuiet(&iommuPath, "%s/sys/kernel/iommu_groups/%d/devices/", fakerootdir, dev->iommuGroup) < 0) @@ -406,6 +425,24 @@ pci_device_create_iommu(const struct pciDevice *dev, } make_symlink(iommuPath, devid, tmp); + + /* pci_device_create_iommu can be called more than one for the + * same iommuGroup. Bail out here if the iommuGroup was already + * created beforehand. */ + for (i = 0; i < npciIommuGroups; i++) { + if (pciIommuGroups[i]->iommu == dev->iommuGroup) + return; + } + + if (VIR_ALLOC_QUIET(iommuGroup) < 0) + ABORT_OOM(); + + iommuGroup->iommu = dev->iommuGroup; + iommuGroup->nDevicesBoundToVFIO = 0; /* No device bound to VFIO by default */ + + if (VIR_APPEND_ELEMENT_QUIET(pciIommuGroups, npciIommuGroups, + iommuGroup) < 0) + ABORT_OOM(); } @@ -558,6 +595,74 @@ pci_device_autobind(struct pciDevice *dev) return pci_driver_bind(driver, dev); } +static int +pci_vfio_release_iommu(struct pciDevice *device) +{ + VIR_AUTOFREE(char *) vfiopath = NULL; + size_t i = 0; + + for (i = 0; i < npciIommuGroups; i++) { + if (device->iommuGroup != pciIommuGroups[i]->iommu) + continue; + + if (pciIommuGroups[i]->nDevicesBoundToVFIO == 0) { + errno = EXDEV; + return -1; + } + + pciIommuGroups[i]->nDevicesBoundToVFIO--; + + if (!pciIommuGroups[i]->nDevicesBoundToVFIO) { + if (virAsprintfQuiet(&vfiopath, "%s/dev/vfio/%d", + fakerootdir, + device->iommuGroup) < 0) { + errno = ENOMEM; + return -1; + } + + if (unlink(vfiopath) < 0) + return -1; + } + break; + } + + return 0; +} + +static int +pci_vfio_lock_iommu(struct pciDevice *device) +{ + VIR_AUTOFREE(char *) vfiopath = NULL; + int ret = -1; + size_t i = 0; + int fd = -1; + + for (i = 0; i < npciIommuGroups; i++) { + if (device->iommuGroup != pciIommuGroups[i]->iommu) + continue; + + if (pciIommuGroups[i]->nDevicesBoundToVFIO == 0) { + if (virAsprintfQuiet(&vfiopath, "%s/dev/vfio/%d", + fakerootdir, + device->iommuGroup) < 0) { + errno = ENOMEM; + goto cleanup; + } + if ((fd = real_open(vfiopath, O_CREAT)) < 0) + goto cleanup; + + } + + pciIommuGroups[i]->nDevicesBoundToVFIO++; + break; + } + + ret = 0; + cleanup: + if (fd != -1) + real_close(fd); + return ret; +} /* * PCI Driver functions @@ -719,6 +824,10 @@ pci_driver_bind(struct pciDriver *driver, if (symlink(devpath, driverpath) < 0) return -1; + if (STREQ(driver->name, "vfio-pci") && + pci_vfio_lock_iommu(dev) < 0) + return -1; + dev->driver = driver; return 0; } @@ -749,6 +858,10 @@ pci_driver_unbind(struct pciDriver *driver, unlink(driverpath) < 0) return -1; + if (STREQ(driver->name, "vfio-pci") && + pci_vfio_release_iommu(dev) < 0) + return -1; + dev->driver = NULL; return 0; } @@ -865,6 +978,16 @@ init_env(void) make_dir(tmp, "drivers"); make_file(tmp, "drivers_probe", NULL, -1); + /* Create /dev/vfio/ dir and /dev/vfio/vfio file */ + VIR_FREE(tmp); + if (virAsprintfQuiet(&tmp, "%s/dev/vfio", fakerootdir) < 0) + ABORT_OOM(); + + if (virFileMakePath(tmp) < 0) + ABORT("Unable to create: %s", tmp); + + make_file(tmp, "vfio", NULL, -1); + # define MAKE_PCI_DRIVER(name, ...) \ pci_driver_new(name, __VA_ARGS__, -1, -1) @@ -872,29 +995,31 @@ init_env(void) MAKE_PCI_DRIVER("i915", 0x8086, 0x0046, 0x8086, 0x0047); MAKE_PCI_DRIVER("vfio-pci", -1, -1); -# define MAKE_PCI_DEVICE(Id, Vendor, Device, ...) \ +# define MAKE_PCI_DEVICE(Id, Vendor, Device, IommuGroup, ...) \ do { \ struct pciDevice dev = {.vendor = Vendor, \ - .device = Device, __VA_ARGS__}; \ + .device = Device, \ + .iommuGroup = IommuGroup, __VA_ARGS__}; \ if (pci_address_parse(&dev.addr, Id) < 0) \ ABORT("Unable to parse PCI address " Id); \ pci_device_new_from_stub(&dev); \ } while (0) - MAKE_PCI_DEVICE("0000:00:00.0", 0x8086, 0x0044); - MAKE_PCI_DEVICE("0000:00:01.0", 0x8086, 0x0044); - MAKE_PCI_DEVICE("0000:00:02.0", 0x8086, 0x0046); - MAKE_PCI_DEVICE("0000:00:03.0", 0x8086, 0x0048); - MAKE_PCI_DEVICE("0001:00:00.0", 0x1014, 0x03b9, .klass = 0x060400); - MAKE_PCI_DEVICE("0001:01:00.0", 0x8086, 0x105e, .iommuGroup = 0); - MAKE_PCI_DEVICE("0001:01:00.1", 0x8086, 0x105e, .iommuGroup = 0); - MAKE_PCI_DEVICE("0005:80:00.0", 0x10b5, 0x8112, .klass = 0x060400); - MAKE_PCI_DEVICE("0005:90:01.0", 0x1033, 0x0035, .iommuGroup = 1); - MAKE_PCI_DEVICE("0005:90:01.1", 0x1033, 0x0035, .iommuGroup = 1); - MAKE_PCI_DEVICE("0005:90:01.2", 0x1033, 0x00e0, .iommuGroup = 1); - MAKE_PCI_DEVICE("0000:0a:01.0", 0x8086, 0x0047); - MAKE_PCI_DEVICE("0000:0a:02.0", 0x8286, 0x0048); - MAKE_PCI_DEVICE("0000:0a:03.0", 0x8386, 0x0048); + MAKE_PCI_DEVICE("0000:00:00.0", 0x8086, 0x0044, 0); + MAKE_PCI_DEVICE("0000:00:01.0", 0x8086, 0x0044, 1); + MAKE_PCI_DEVICE("0000:00:02.0", 0x8086, 0x0046, 2); + MAKE_PCI_DEVICE("0000:00:03.0", 0x8086, 0x0048, 3); + MAKE_PCI_DEVICE("0001:00:00.0", 0x1014, 0x03b9, 4, .klass = 0x060400); + MAKE_PCI_DEVICE("0001:01:00.0", 0x8086, 0x105e, 5); + MAKE_PCI_DEVICE("0001:01:00.1", 0x8086, 0x105e, 5); + MAKE_PCI_DEVICE("0005:80:00.0", 0x10b5, 0x8112, 6, .klass = 0x060400); + MAKE_PCI_DEVICE("0005:90:01.0", 0x1033, 0x0035, 7); + MAKE_PCI_DEVICE("0005:90:01.1", 0x1033, 0x0035, 7); + MAKE_PCI_DEVICE("0005:90:01.2", 0x1033, 0x00e0, 7); + MAKE_PCI_DEVICE("0005:90:01.3", 0x1033, 0x00e0, 7); + MAKE_PCI_DEVICE("0000:0a:01.0", 0x8086, 0x0047, 8); + MAKE_PCI_DEVICE("0000:0a:02.0", 0x8286, 0x0048, 8); + MAKE_PCI_DEVICE("0000:0a:03.0", 0x8386, 0x0048, 8); } -- GitLab