From 0702f48ef485d6b893a1633886a1b92851a2101f Mon Sep 17 00:00:00 2001 From: Laine Stump Date: Mon, 19 Sep 2016 14:38:47 -0400 Subject: [PATCH] qemu: auto-add pcie-root-port/dmi-to-pci-bridge controllers as needed Previously libvirt would only add pci-bridge devices automatically when an address was requested for a device that required a legacy PCI slot and none was available. This patch expands that support to dmi-to-pci-bridge (which is needed in order to add a pci-bridge on a machine with a pcie-root), and pcie-root-port (which is needed to add a hotpluggable PCIe device). It does *not* automatically add pcie-switch-upstream-ports or pcie-switch-downstream-ports (and currently there are no plans for that). Given the existing code to auto-add pci-bridge devices, automatically adding pcie-root-ports is fairly straightforward. The dmi-to-pci-bridge support is a bit tricky though, for a few reasons: 1) Although the only reason to add a dmi-to-pci-bridge is so that there is a reasonable place to plug in a pci-bridge controller, most of the time it's not the presence of a pci-bridge *in the config* that triggers the requirement to add a dmi-to-pci-bridge. Rather, it is the presence of a legacy-PCI device in the config, which triggers auto-add of a pci-bridge, which triggers auto-add of a dmi-to-pci-bridge (this is handled in virDomainPCIAddressSetGrow() - if there's a request to add a pci-bridge we'll check if there is a suitable bus to plug it into; if not, we first add a dmi-to-pci-bridge). 2) Once there is already a single dmi-to-pci-bridge on the system, there won't be a need for any more, even if it's full, as long as there is a pci-bridge with an open slot - you can also plug pci-bridges into existing pci-bridges. So we have to make sure we don't add a dmi-to-pci-bridge unless there aren't any dmi-to-pci-bridges *or* any pci-bridges. 3) Although it is strongly discouraged, it is legal for a pci-bridge to be directly plugged into pcie-root, and we don't want to auto-add a dmi-to-pci-bridge if there is already a pci-bridge that's been forced directly into pcie-root. Although libvirt will now automatically create a dmi-to-pci-bridge when it's needed, the code still remains for now that forces a dmi-to-pci-bridge on all domains with pcie-root (in qemuDomainDefAddDefaultDevices()). That will be removed in a future patch. For now, the pcie-root-ports are added one to a slot, which is a bit wasteful and means it will fail after 31 total PCIe devices (30 if there are also some PCI devices), but helps keep the changeset down for this patch. A future patch will have 8 pcie-root-ports sharing the functions on a single slot. --- src/conf/domain_addr.c | 154 ++++++++++++++++-- src/qemu/qemu_domain_address.c | 54 ++++-- .../qemuxml2argv-q35-pcie-autoadd.args | 57 +++++++ .../qemuxml2argv-q35-pcie-autoadd.xml | 51 ++++++ tests/qemuxml2argvtest.c | 24 +++ .../qemuxml2xmlout-q35-pcie-autoadd.xml | 147 +++++++++++++++++ tests/qemuxml2xmltest.c | 25 ++- 7 files changed, 478 insertions(+), 34 deletions(-) create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-q35-pcie-autoadd.args create mode 100644 tests/qemuxml2argvdata/qemuxml2argv-q35-pcie-autoadd.xml create mode 100644 tests/qemuxml2xmloutdata/qemuxml2xmlout-q35-pcie-autoadd.xml diff --git a/src/conf/domain_addr.c b/src/conf/domain_addr.c index d3a2f6f23e..4d3eabc545 100644 --- a/src/conf/domain_addr.c +++ b/src/conf/domain_addr.c @@ -82,6 +82,36 @@ virDomainPCIControllerModelToConnectType(virDomainControllerModelPCI model) return 0; } + +static int +virDomainPCIControllerConnectTypeToModel(virDomainPCIConnectFlags flags) +{ + if (flags & VIR_PCI_CONNECT_TYPE_PCIE_ROOT_PORT) + return VIR_DOMAIN_CONTROLLER_MODEL_PCIE_ROOT_PORT; + + if (flags & VIR_PCI_CONNECT_TYPE_PCIE_SWITCH_UPSTREAM_PORT) + return VIR_DOMAIN_CONTROLLER_MODEL_PCIE_SWITCH_UPSTREAM_PORT; + + if (flags & VIR_PCI_CONNECT_TYPE_PCIE_SWITCH_DOWNSTREAM_PORT) + return VIR_DOMAIN_CONTROLLER_MODEL_PCIE_SWITCH_DOWNSTREAM_PORT; + + if (flags & VIR_PCI_CONNECT_TYPE_DMI_TO_PCI_BRIDGE) + return VIR_DOMAIN_CONTROLLER_MODEL_DMI_TO_PCI_BRIDGE; + + if (flags & VIR_PCI_CONNECT_TYPE_PCI_EXPANDER_BUS) + return VIR_DOMAIN_CONTROLLER_MODEL_PCI_EXPANDER_BUS; + + if (flags & VIR_PCI_CONNECT_TYPE_PCIE_EXPANDER_BUS) + return VIR_DOMAIN_CONTROLLER_MODEL_PCIE_EXPANDER_BUS; + + if (flags & VIR_PCI_CONNECT_TYPE_PCI_BRIDGE) + return VIR_DOMAIN_CONTROLLER_MODEL_PCI_BRIDGE; + + /* some connect types don't correspond to a controller model */ + return -1; +} + + bool virDomainPCIAddressFlagsCompatible(virPCIDeviceAddressPtr addr, const char *addrStr, @@ -334,10 +364,7 @@ virDomainPCIAddressBusSetModel(virDomainPCIAddressBusPtr bus, } -/* Ensure addr fits in the address set, by expanding it if needed. - * This will only grow if the flags say that we need a normal - * hot-pluggable PCI slot. If we need a different type of slot, it - * will fail. +/* Ensure addr fits in the address set, by expanding it if needed * * Return value: * -1 = OOM @@ -351,32 +378,125 @@ virDomainPCIAddressSetGrow(virDomainPCIAddressSetPtr addrs, { int add; size_t i; + int model; + bool needDMIToPCIBridge = false; add = addr->bus - addrs->nbuses + 1; - i = addrs->nbuses; if (add <= 0) return 0; - /* auto-grow only works when we're adding plain PCI devices */ - if (!(flags & VIR_PCI_CONNECT_TYPE_PCI_DEVICE)) { - virReportError(VIR_ERR_INTERNAL_ERROR, "%s", - _("Cannot automatically add a new PCI bus for a " - "device requiring a slot other than standard PCI.")); + /* remember that the flags aren't for the type of controller that + * we want to add, they are the type of *device* that we want to + * plug in, and this function must decide on the appropriate + * controller to add in order to give us a slot for that device. + */ + + if (flags & VIR_PCI_CONNECT_TYPE_PCI_DEVICE) { + model = VIR_DOMAIN_CONTROLLER_MODEL_PCI_BRIDGE; + + /* if there aren't yet any buses that will accept a + * pci-bridge, and the caller is asking for one, we'll need to + * add a dmi-to-pci-bridge first. + */ + needDMIToPCIBridge = true; + for (i = 0; i < addrs->nbuses; i++) { + if (addrs->buses[i].flags & VIR_PCI_CONNECT_TYPE_PCI_BRIDGE) { + needDMIToPCIBridge = false; + break; + } + } + if (needDMIToPCIBridge && add == 1) { + /* We need to add a single pci-bridge to provide the bus + * our legacy PCI device will be plugged into; however, we + * have also determined that there isn't yet any proper + * place to connect that pci-bridge we're about to add (on + * a system with pcie-root, that "proper place" would be a + * dmi-to-pci-bridge". So, to give the pci-bridge a place + * to connect, we increase the count of buses to add, + * while also incrementing the bus number in the address + * for the device (since the pci-bridge will now be at an + * index 1 higher than the caller had anticipated). + */ + add++; + addr->bus++; + } + } else if (flags & VIR_PCI_CONNECT_TYPE_PCI_BRIDGE && + addrs->buses[0].model == VIR_DOMAIN_CONTROLLER_MODEL_PCIE_ROOT) { + /* NB: if the root bus is pci-root, and we couldn't find an + * open place to connect a pci-bridge, then there is nothing + * we can do (since the only way to gain a new slot that + * accepts a pci-bridge is to add *a pci-bridge* (which is the + * reason we're here in the first place!) + */ + model = VIR_DOMAIN_CONTROLLER_MODEL_DMI_TO_PCI_BRIDGE; + } else if (flags & (VIR_PCI_CONNECT_TYPE_PCIE_DEVICE | + VIR_PCI_CONNECT_TYPE_PCIE_SWITCH_UPSTREAM_PORT)) { + model = VIR_DOMAIN_CONTROLLER_MODEL_PCIE_ROOT_PORT; + } else { + /* The types of devices that we can't auto-add a controller for: + * + * VIR_CONNECT_TYPE_DMI_TO_PCI_BRIDGE & + * VIR_PCI_CONNECT_TYPE_ROOT_PORT - these can only plug into + * pcie-root or pcie-expander-bus. By definition there is + * only 1 pcie-root, and we don't support auto-adding + * pcie-expander-bus (because it is intended for NUMA usage, + * and we can't automatically decide which numa node to + * associate it with) + * + * VIR_CONNECT_TYPE_PCIE_SWITCH_DOWNSTREAM_PORT - we ndon't + * support this, because it can only plug into an + * upstream-port, and the upstream port might need a + * root-port; supporting this extra layer needlessly + * complicates the code, and upstream/downstream ports are + * outside the scope of our "automatic-bus-expansion" model + * anyway. + * + * VIR_CONNECT_TYPE_PCI[E]_EXPANDER_BUS - these were created + * to support guest awareness of the NUMA node placement of + * devices on the host, and are also outside the scope of our + * "automatic-bus-expansion". + * + * VIR_PCI_CONNECT_TYPE_PCI_BRIDGE (when the root bus is + * pci-root) - see the comment above in the case that handles + * adding a slot for pci-bridge to a guest with pcie-root. + * + */ + int existingContModel = virDomainPCIControllerConnectTypeToModel(flags); + + if (existingContModel >= 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("a PCI slot is needed to connect a PCI controller " + "model='%s', but none is available, and it " + "cannot be automatically added"), + virDomainControllerModelPCITypeToString(existingContModel)); + } else { + virReportError(VIR_ERR_INTERNAL_ERROR, + _("Cannot automatically add a new PCI bus for a " + "device with connect flags %.2x"), flags); + } return -1; } + i = addrs->nbuses; + if (VIR_EXPAND_N(addrs->buses, addrs->nbuses, add) < 0) return -1; - for (; i < addrs->nbuses; i++) { - /* Any time we auto-add a bus, we will want a multi-slot - * bus. Currently the only type of bus we will auto-add is a - * pci-bridge, which is hot-pluggable and provides standard - * PCI slots. + if (needDMIToPCIBridge) { + /* first of the new buses is dmi-to-pci-bridge, the + * rest are of the requested type */ - virDomainPCIAddressBusSetModel(&addrs->buses[i], - VIR_DOMAIN_CONTROLLER_MODEL_PCI_BRIDGE); + if (virDomainPCIAddressBusSetModel(&addrs->buses[i++], + VIR_DOMAIN_CONTROLLER_MODEL_DMI_TO_PCI_BRIDGE) < 0) { + return -1; + } } + + for (; i < addrs->nbuses; i++) { + if (virDomainPCIAddressBusSetModel(&addrs->buses[i], model) < 0) + return -1; + } + return add; } diff --git a/src/qemu/qemu_domain_address.c b/src/qemu/qemu_domain_address.c index 4920a91fc6..b260164875 100644 --- a/src/qemu/qemu_domain_address.c +++ b/src/qemu/qemu_domain_address.c @@ -911,27 +911,14 @@ qemuDomainPCIAddressSetCreate(virDomainDefPtr def, { virDomainPCIAddressSetPtr addrs; size_t i; + bool hasPCIeRoot = false; + virDomainControllerModelPCI defaultModel; if ((addrs = virDomainPCIAddressSetAlloc(nbuses)) == NULL) return NULL; addrs->dryRun = dryRun; - /* As a safety measure, set default model='pci-root' for first pci - * controller and 'pci-bridge' for all subsequent. After setting - * those defaults, then scan the config and set the actual model - * for all addrs[idx]->bus that already have a corresponding - * controller in the config. - * - */ - if (nbuses > 0) - virDomainPCIAddressBusSetModel(&addrs->buses[0], - VIR_DOMAIN_CONTROLLER_MODEL_PCI_ROOT); - for (i = 1; i < nbuses; i++) { - virDomainPCIAddressBusSetModel(&addrs->buses[i], - VIR_DOMAIN_CONTROLLER_MODEL_PCI_BRIDGE); - } - for (i = 0; i < def->ncontrollers; i++) { virDomainControllerDefPtr cont = def->controllers[i]; size_t idx = cont->idx; @@ -948,7 +935,42 @@ qemuDomainPCIAddressSetCreate(virDomainDefPtr def, if (virDomainPCIAddressBusSetModel(&addrs->buses[idx], cont->model) < 0) goto error; - } + + if (cont->model == VIR_DOMAIN_CONTROLLER_MODEL_PCIE_ROOT) + hasPCIeRoot = true; + } + + if (nbuses > 0 && !addrs->buses[0].model) { + /* This is just here to replicate a safety measure already in + * an older version of this code. In practice, the root bus + * should have already been added at index 0 prior to + * assigning addresses to devices. + */ + if (virDomainPCIAddressBusSetModel(&addrs->buses[0], + VIR_DOMAIN_CONTROLLER_MODEL_PCI_ROOT) < 0) + goto error; + } + + /* Now fill in a reasonable model for all the buses in the set + * that don't yet have a corresponding controller in the domain + * config. + */ + if (hasPCIeRoot) + defaultModel = VIR_DOMAIN_CONTROLLER_MODEL_PCIE_ROOT_PORT; + else + defaultModel = VIR_DOMAIN_CONTROLLER_MODEL_PCI_BRIDGE; + + for (i = 1; i < addrs->nbuses; i++) { + + if (addrs->buses[i].model) + continue; + + if (virDomainPCIAddressBusSetModel(&addrs->buses[i], defaultModel) < 0) + goto error; + + VIR_DEBUG("Auto-adding ", + virDomainControllerModelPCITypeToString(defaultModel), i); + } if (virDomainDeviceInfoIterate(def, qemuDomainCollectPCIAddress, addrs) < 0) goto error; diff --git a/tests/qemuxml2argvdata/qemuxml2argv-q35-pcie-autoadd.args b/tests/qemuxml2argvdata/qemuxml2argv-q35-pcie-autoadd.args new file mode 100644 index 0000000000..70c759e310 --- /dev/null +++ b/tests/qemuxml2argvdata/qemuxml2argv-q35-pcie-autoadd.args @@ -0,0 +1,57 @@ +LC_ALL=C \ +PATH=/bin \ +HOME=/home/test \ +USER=test \ +LOGNAME=test \ +QEMU_AUDIO_DRV=none \ +/usr/libexec/qemu-kvm \ +-name q35-test \ +-S \ +-M q35 \ +-m 2048 \ +-smp 2,sockets=2,cores=1,threads=1 \ +-uuid 11dbdcdd-4c3b-482b-8903-9bdb8c0a2774 \ +-nographic \ +-nodefaults \ +-monitor unix:/tmp/lib/domain--1-q35-test/monitor.sock,server,nowait \ +-no-acpi \ +-boot c \ +-device i82801b11-bridge,id=pci.1,bus=pcie.0,addr=0x1e \ +-device ioh3420,port=0x10,chassis=2,id=pci.2,bus=pcie.0,addr=0x2 \ +-device ioh3420,port=0x18,chassis=3,id=pci.3,bus=pcie.0,addr=0x3 \ +-device ioh3420,port=0x20,chassis=4,id=pci.4,bus=pcie.0,addr=0x4 \ +-device ioh3420,port=0x28,chassis=5,id=pci.5,bus=pcie.0,addr=0x5 \ +-device ioh3420,port=0x30,chassis=6,id=pci.6,bus=pcie.0,addr=0x6 \ +-device ioh3420,port=0x38,chassis=7,id=pci.7,bus=pcie.0,addr=0x7 \ +-device ioh3420,port=0x40,chassis=8,id=pci.8,bus=pcie.0,addr=0x8 \ +-device ioh3420,port=0x48,chassis=9,id=pci.9,bus=pcie.0,addr=0x9 \ +-device ioh3420,port=0x50,chassis=10,id=pci.10,bus=pcie.0,addr=0xa \ +-device ioh3420,port=0x58,chassis=11,id=pci.11,bus=pcie.0,addr=0xb \ +-device ioh3420,port=0x60,chassis=12,id=pci.12,bus=pcie.0,addr=0xc \ +-device ioh3420,port=0x68,chassis=13,id=pci.13,bus=pcie.0,addr=0xd \ +-device ioh3420,port=0x70,chassis=14,id=pci.14,bus=pcie.0,addr=0xe \ +-device nec-usb-xhci,id=usb,bus=pci.7,addr=0x0 \ +-device virtio-scsi-pci,id=scsi0,bus=pci.6,addr=0x0 \ +-device virtio-serial-pci,id=virtio-serial0,bus=pci.5,addr=0x0 \ +-drive file=/dev/HostVG/QEMUGuest1,format=raw,if=none,id=drive-virtio-disk1 \ +-device virtio-blk-pci,bus=pci.8,addr=0x0,drive=drive-virtio-disk1,\ +id=virtio-disk1 \ +-fsdev local,security_model=passthrough,id=fsdev-fs0,path=/export/to/guest \ +-device virtio-9p-pci,id=fs0,fsdev=fsdev-fs0,mount_tag=/import/from/host,\ +bus=pci.2,addr=0x0 \ +-netdev user,id=hostnet0 \ +-device virtio-net-pci,netdev=hostnet0,id=net0,mac=00:11:22:33:44:55,bus=pci.3,\ +addr=0x0 \ +-netdev user,id=hostnet1 \ +-device e1000e,netdev=hostnet1,id=net1,mac=00:11:22:33:44:66,bus=pci.4,\ +addr=0x0 \ +-device virtio-input-host-pci,id=input0,evdev=/dev/input/event1234,bus=pci.11,\ +addr=0x0 \ +-device virtio-mouse-pci,id=input1,bus=pci.12,addr=0x0 \ +-device virtio-keyboard-pci,id=input2,bus=pci.13,addr=0x0 \ +-device virtio-tablet-pci,id=input3,bus=pci.14,addr=0x0 \ +-device virtio-gpu-pci,id=video0,bus=pcie.0,addr=0x1 \ +-device virtio-balloon-pci,id=balloon0,bus=pci.9,addr=0x0 \ +-object rng-random,id=objrng0,filename=/dev/urandom \ +-device virtio-rng-pci,rng=objrng0,id=rng0,max-bytes=123,period=1234,\ +bus=pci.10,addr=0x0 diff --git a/tests/qemuxml2argvdata/qemuxml2argv-q35-pcie-autoadd.xml b/tests/qemuxml2argvdata/qemuxml2argv-q35-pcie-autoadd.xml new file mode 100644 index 0000000000..06cdf61107 --- /dev/null +++ b/tests/qemuxml2argvdata/qemuxml2argv-q35-pcie-autoadd.xml @@ -0,0 +1,51 @@ + + q35-test + 11dbdcdd-4c3b-482b-8903-9bdb8c0a2774 + 2097152 + 2097152 + 2 + + hvm + + + + destroy + restart + destroy + + /usr/libexec/qemu-kvm + + + + + + + + + + + + + + + + + + + + + + + + /dev/urandom + + + + + + + + + diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c index 2c3d2582eb..6f2069f30e 100644 --- a/tests/qemuxml2argvtest.c +++ b/tests/qemuxml2argvtest.c @@ -1820,6 +1820,30 @@ mymain(void) QEMU_CAPS_ICH9_USB_EHCI1, QEMU_CAPS_NEC_USB_XHCI, QEMU_CAPS_DEVICE_VIDEO_PRIMARY); + /* same as q35-pcie, but all PCI controllers are added automatically */ + DO_TEST("q35-pcie-autoadd", + QEMU_CAPS_VIRTIO_PCI_DISABLE_LEGACY, + QEMU_CAPS_DEVICE_VIRTIO_RNG, + QEMU_CAPS_OBJECT_RNG_RANDOM, + QEMU_CAPS_NETDEV, + QEMU_CAPS_DEVICE_VIRTIO_NET, + QEMU_CAPS_DEVICE_VIRTIO_GPU, + QEMU_CAPS_VIRTIO_GPU_VIRGL, + QEMU_CAPS_VIRTIO_KEYBOARD, + QEMU_CAPS_VIRTIO_MOUSE, + QEMU_CAPS_VIRTIO_TABLET, + QEMU_CAPS_VIRTIO_INPUT_HOST, + QEMU_CAPS_VIRTIO_SCSI, + QEMU_CAPS_FSDEV, + QEMU_CAPS_FSDEV_WRITEOUT, + QEMU_CAPS_DEVICE_PCI_BRIDGE, + QEMU_CAPS_DEVICE_DMI_TO_PCI_BRIDGE, + QEMU_CAPS_DEVICE_IOH3420, + QEMU_CAPS_ICH9_AHCI, + QEMU_CAPS_PCI_MULTIFUNCTION, + QEMU_CAPS_ICH9_USB_EHCI1, + QEMU_CAPS_NEC_USB_XHCI, + QEMU_CAPS_DEVICE_VIDEO_PRIMARY); DO_TEST("pcie-root-port", QEMU_CAPS_DEVICE_PCI_BRIDGE, QEMU_CAPS_DEVICE_DMI_TO_PCI_BRIDGE, diff --git a/tests/qemuxml2xmloutdata/qemuxml2xmlout-q35-pcie-autoadd.xml b/tests/qemuxml2xmloutdata/qemuxml2xmlout-q35-pcie-autoadd.xml new file mode 100644 index 0000000000..b27dbe7cb3 --- /dev/null +++ b/tests/qemuxml2xmloutdata/qemuxml2xmlout-q35-pcie-autoadd.xml @@ -0,0 +1,147 @@ + + q35-test + 11dbdcdd-4c3b-482b-8903-9bdb8c0a2774 + 2097152 + 2097152 + 2 + + hvm + + + + destroy + restart + destroy + + /usr/libexec/qemu-kvm + + + +
+ + +
+ + +
+ + +
+ + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + + +
+ + + +
+ + +
+ + +
+ + +
+ + + +