qemu: specify domain in host-side PCI addresses when needed/supported

This uses the new QEMU_CAPS_HOST_PCI_MULTIDOMAIN capability when
present, for -devivce pci-assign, -device vfio-pci, and -pcidevice.

While creating tests for this new functionality, I noticed that the
xmls for two existing tests had erroneously specified an
until-now-ignored domain="0x0002", so I corrected those two tests, and
also added two failure tests to be sure that we alert users who
attempt to use a non-zero domain with a qemu that doesn't support it.

src/qemu/qemu_command.c

@@ -5522,7 +5522,18 @@ qemuBuildPCIHostdevDevStr(virDomainDefPtr def,
         break;
     }
 
-    virBufferAsprintf(&buf, ",host=%.2x:%.2x.%.1x",
+    virBufferAddLit(&buf, ",host=");
+    if (dev->source.subsys.u.pci.addr.domain) {
+        if (!virQEMUCapsGet(qemuCaps, QEMU_CAPS_HOST_PCI_MULTIDOMAIN)) {
+            virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                           _("non-zero domain='%.4x' in host device PCI address "
+                             "not supported in this QEMU binary"),
+                           dev->source.subsys.u.pci.addr.domain);
+            goto error;
+        }
+        virBufferAsprintf(&buf, "%.4x:", dev->source.subsys.u.pci.addr.domain);
+    }
+    virBufferAsprintf(&buf, "%.2x:%.2x.%.1x",
                       dev->source.subsys.u.pci.addr.bus,
                       dev->source.subsys.u.pci.addr.slot,
                       dev->source.subsys.u.pci.addr.function);
@@ -5548,14 +5559,31 @@ qemuBuildPCIHostdevDevStr(virDomainDefPtr def,
 
 
 char *
-qemuBuildPCIHostdevPCIDevStr(virDomainHostdevDefPtr dev)
+qemuBuildPCIHostdevPCIDevStr(virDomainHostdevDefPtr dev,
+                             virQEMUCapsPtr qemuCaps)
 {
     char *ret = NULL;
 
-    ignore_value(virAsprintf(&ret, "host=%.2x:%.2x.%.1x",
-                             dev->source.subsys.u.pci.addr.bus,
-                             dev->source.subsys.u.pci.addr.slot,
-                             dev->source.subsys.u.pci.addr.function));
+    if (dev->source.subsys.u.pci.addr.domain) {
+        if (!virQEMUCapsGet(qemuCaps, QEMU_CAPS_HOST_PCI_MULTIDOMAIN)) {
+            virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                           _("non-zero domain='%.4x' in host device PCI address "
+                             "not supported in this QEMU binary"),
+                           dev->source.subsys.u.pci.addr.domain);
+            goto cleanup;
+        }
+        ignore_value(virAsprintf(&ret, "host=%.4x:%.2x:%.2x.%.1x",
+                                 dev->source.subsys.u.pci.addr.domain,
+                                 dev->source.subsys.u.pci.addr.bus,
+                                 dev->source.subsys.u.pci.addr.slot,
+                                 dev->source.subsys.u.pci.addr.function));
+    } else {
+        ignore_value(virAsprintf(&ret, "host=%.2x:%.2x.%.1x",
+                                 dev->source.subsys.u.pci.addr.bus,
+                                 dev->source.subsys.u.pci.addr.slot,
+                                 dev->source.subsys.u.pci.addr.function));
+    }
+ cleanup:
     return ret;
 }
 
@@ -9493,7 +9521,7 @@ qemuBuildCommandLine(virConnectPtr conn,
                 VIR_FREE(devstr);
             } else if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_PCIDEVICE)) {
                 virCommandAddArg(cmd, "-pcidevice");
-                if (!(devstr = qemuBuildPCIHostdevPCIDevStr(hostdev)))
+                if (!(devstr = qemuBuildPCIHostdevPCIDevStr(hostdev, qemuCaps)))
                     goto error;
                 virCommandAddArg(cmd, devstr);
                 VIR_FREE(devstr);

src/qemu/qemu_command.h

@@ -152,7 +152,8 @@ char * qemuBuildSoundDevStr(virDomainDefPtr domainDef,
                             virQEMUCapsPtr qemuCaps);
 
 /* Legacy, pre device support */
-char * qemuBuildPCIHostdevPCIDevStr(virDomainHostdevDefPtr dev);
+char * qemuBuildPCIHostdevPCIDevStr(virDomainHostdevDefPtr dev,
+                                    virQEMUCapsPtr qemuCaps);
 /* Current, best practice */
 char * qemuBuildPCIHostdevDevStr(virDomainDefPtr def,
                                  virDomainHostdevDefPtr dev,

src/qemu/qemu_hotplug.c

@@ -1258,16 +1258,23 @@ qemuDomainAttachHostPCIDevice(virQEMUDriverPtr driver,
                                          configfd, configfd_name);
         qemuDomainObjExitMonitor(driver, vm);
     } else {
-        virDevicePCIAddress guestAddr = hostdev->info->addr.pci;
+        virDevicePCIAddressPtr guestAddr = &hostdev->info->addr.pci;
+        virDevicePCIAddressPtr hostAddr = &hostdev->source.subsys.u.pci.addr;
+
+        if (hostAddr->domain &&
+            !virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_HOST_PCI_MULTIDOMAIN)) {
+            virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
+                           _("non-zero domain='%.4x' in host device "
+                             "PCI address not supported in this QEMU binary"),
+                           hostAddr->domain);
+            goto error;
+        }
 
         qemuDomainObjEnterMonitor(driver, vm);
-        ret = qemuMonitorAddPCIHostDevice(priv->mon,
-                                          &hostdev->source.subsys.u.pci.addr,
-                                          &guestAddr);
+        ret = qemuMonitorAddPCIHostDevice(priv->mon, hostAddr, guestAddr);
         qemuDomainObjExitMonitor(driver, vm);
 
         hostdev->info->type = VIR_DOMAIN_DEVICE_ADDRESS_TYPE_PCI;
-        memcpy(&hostdev->info->addr.pci, &guestAddr, sizeof(guestAddr));
     }
     virDomainAuditHostdev(vm, hostdev, "attach", ret == 0);
     if (ret < 0)

src/qemu/qemu_monitor_text.c

@@ -1826,10 +1826,20 @@ int qemuMonitorTextAddPCIHostDevice(qemuMonitorPtr mon,
 
     memset(guestAddr, 0, sizeof(*guestAddr));
 
-    /* XXX hostAddr->domain */
-    if (virAsprintf(&cmd, "pci_add pci_addr=auto host host=%.2x:%.2x.%.1x",
-                    hostAddr->bus, hostAddr->slot, hostAddr->function) < 0)
-        goto cleanup;
+    if (hostAddr->domain) {
+        /* if domain > 0, the caller has already verified that this qemu
+         * supports specifying domain in pci_add command
+         */
+        if (virAsprintf(&cmd,
+                        "pci_add pci_addr=auto host host=%.4x:%.2x:%.2x.%.1x",
+                        hostAddr->domain, hostAddr->bus,
+                        hostAddr->slot, hostAddr->function) < 0)
+            goto cleanup;
+    } else {
+        if (virAsprintf(&cmd, "pci_add pci_addr=auto host host=%.2x:%.2x.%.1x",
+                        hostAddr->bus, hostAddr->slot, hostAddr->function) < 0)
+            goto cleanup;
+    }
 
     if (qemuMonitorHMPCommand(mon, cmd, &reply) < 0)
         goto cleanup;

tests/qemuxml2argvdata/qemuxml2argv-hostdev-vfio-multidomain.args

+LC_ALL=C PATH=/bin HOME=/home/test USER=test LOGNAME=test QEMU_AUDIO_DRV=none \
+/usr/bin/qemu -S -M \
+pc -m 214 -smp 1 -nographic -nodefconfig -nodefaults -monitor \
+unix:/tmp/test-monitor,server,nowait -no-acpi -boot c -usb -hda \
+/dev/HostVG/QEMUGuest2 -device vfio-pci,host=55aa:20:0f.3,id=hostdev0,\
+bus=pci.0,addr=0x3 -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x4

tests/qemuxml2argvdata/qemuxml2argv-hostdev-vfio-multidomain.xml

+<domain type='qemu'>
+  <name>QEMUGuest2</name>
+  <uuid>c7a5fdbd-edaf-9466-926a-d65c16db1809</uuid>
+  <memory unit='KiB'>219100</memory>
+  <currentMemory unit='KiB'>219100</currentMemory>
+  <vcpu placement='static'>1</vcpu>
+  <os>
+    <type arch='i686' machine='pc'>hvm</type>
+    <boot dev='hd'/>
+  </os>
+  <clock offset='utc'/>
+  <on_poweroff>destroy</on_poweroff>
+  <on_reboot>restart</on_reboot>
+  <on_crash>destroy</on_crash>
+  <devices>
+    <emulator>/usr/bin/qemu</emulator>
+    <disk type='block' device='disk'>
+      <source dev='/dev/HostVG/QEMUGuest2'/>
+      <target dev='hda' bus='ide'/>
+      <address type='drive' controller='0' bus='0' target='0' unit='0'/>
+    </disk>
+    <controller type='usb' index='0'/>
+    <controller type='ide' index='0'/>
+    <controller type='pci' index='0' model='pci-root'/>
+    <hostdev mode='subsystem' type='pci' managed='yes'>
+      <driver name='vfio'/>
+      <source>
+        <address domain='0x55aa' bus='32' slot='15' function='3'/>
+      </source>
+    </hostdev>
+    <memballoon model='virtio'/>
+  </devices>
+</domain>

tests/qemuxml2argvdata/qemuxml2argv-net-hostdev-multidomain.args

+LC_ALL=C PATH=/bin HOME=/home/test USER=test LOGNAME=test QEMU_AUDIO_DRV=none \
+/usr/bin/qemu -S \
+-M pc -m 214 -smp 1 -nographic -nodefconfig -nodefaults -monitor \
+unix:/tmp/test-monitor,server,nowait -no-acpi -boot c \
+-usb -hda /dev/HostVG/QEMUGuest1 \
+-device pci-assign,host=2424:21:1c.6,id=hostdev0,bus=pci.0,addr=0x3 \
+-device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x4

tests/qemuxml2argvdata/qemuxml2argv-net-hostdev-multidomain.xml

+<domain type='qemu'>
+  <name>QEMUGuest1</name>
+  <uuid>c7a5fdbd-edaf-9455-926a-d65c16db1809</uuid>
+  <memory unit='KiB'>219136</memory>
+  <currentMemory unit='KiB'>219136</currentMemory>
+  <vcpu placement='static'>1</vcpu>
+  <os>
+    <type arch='i686' machine='pc'>hvm</type>
+    <boot dev='hd'/>
+  </os>
+  <clock offset='utc'/>
+  <on_poweroff>destroy</on_poweroff>
+  <on_reboot>restart</on_reboot>
+  <on_crash>destroy</on_crash>
+  <devices>
+    <emulator>/usr/bin/qemu</emulator>
+    <disk type='block' device='disk'>
+      <source dev='/dev/HostVG/QEMUGuest1'/>
+      <target dev='hda' bus='ide'/>
+      <address type='drive' controller='0' bus='0' target='0' unit='0'/>
+    </disk>
+    <controller type='usb' index='0'/>
+    <controller type='ide' index='0'/>
+    <controller type='pci' index='0' model='pci-root'/>
+    <interface type='hostdev' managed='yes'>
+      <mac address='00:11:22:33:44:55'/>
+      <source>
+        <address type='pci' domain='0x2424' bus='0x21' slot='0x1c' function='0x6'/>
+      </source>
+      <vlan>
+        <tag id='42'/>
+      </vlan>
+      <virtualport type='802.1Qbg'>
+        <parameters managerid='11' typeid='1193047' typeidversion='2' instanceid='09b11c53-8b5c-4eeb-8f00-d84eaa0aaa4f'/>
+      </virtualport>
+      <model type='rtl8139'/>
+    </interface>
+    <memballoon model='virtio'/>
+  </devices>
+</domain>

tests/qemuxml2argvdata/qemuxml2argv-net-hostdev-vfio-multidomain.args

+LC_ALL=C PATH=/bin HOME=/home/test USER=test LOGNAME=test QEMU_AUDIO_DRV=none \
+/usr/bin/qemu -S \
+-M pc -m 214 -smp 1 -nographic -nodefconfig -nodefaults -monitor \
+unix:/tmp/test-monitor,server,nowait -no-acpi -boot c \
+-usb -hda /dev/HostVG/QEMUGuest1 \
+-device vfio-pci,host=0021:de:1f.1,id=hostdev0,bus=pci.0,addr=0x3 \
+-device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x4

tests/qemuxml2argvdata/qemuxml2argv-net-hostdev-vfio-multidomain.xml

+<domain type='qemu'>
+  <name>QEMUGuest1</name>
+  <uuid>c7a5fdbd-edaf-9455-926a-d65c16db1809</uuid>
+  <memory unit='KiB'>219136</memory>
+  <currentMemory unit='KiB'>219136</currentMemory>
+  <vcpu placement='static'>1</vcpu>
+  <os>
+    <type arch='i686' machine='pc'>hvm</type>
+    <boot dev='hd'/>
+  </os>
+  <clock offset='utc'/>
+  <on_poweroff>destroy</on_poweroff>
+  <on_reboot>restart</on_reboot>
+  <on_crash>destroy</on_crash>
+  <devices>
+    <emulator>/usr/bin/qemu</emulator>
+    <disk type='block' device='disk'>
+      <source dev='/dev/HostVG/QEMUGuest1'/>
+      <target dev='hda' bus='ide'/>
+      <address type='drive' controller='0' bus='0' target='0' unit='0'/>
+    </disk>
+    <controller type='usb' index='0'/>
+    <controller type='ide' index='0'/>
+    <controller type='pci' index='0' model='pci-root'/>
+    <interface type='hostdev' managed='yes'>
+      <mac address='00:11:22:33:44:55'/>
+      <driver name='vfio'/>
+      <source>
+        <address type='pci' domain='0x0021' bus='0xde' slot='0x1f' function='0x1'/>
+      </source>
+      <vlan>
+        <tag id='42'/>
+      </vlan>
+      <virtualport type='802.1Qbg'>
+        <parameters managerid='11' typeid='1193047' typeidversion='2' instanceid='09b11c53-8b5c-4eeb-8f00-d84eaa0aaa4f'/>
+      </virtualport>
+      <model type='rtl8139'/>
+    </interface>
+    <memballoon model='virtio'/>
+  </devices>
+</domain>

tests/qemuxml2argvdata/qemuxml2argv-net-hostdev-vfio.xml

@@ -26,7 +26,7 @@
       <mac address='00:11:22:33:44:55'/>
       <driver name='vfio'/>
       <source>
-        <address type='pci' domain='0x0002' bus='0x03' slot='0x07' function='0x1'/>
+        <address type='pci' domain='0x0000' bus='0x03' slot='0x07' function='0x1'/>
       </source>
       <vlan>
         <tag id='42'/>

tests/qemuxml2argvdata/qemuxml2argv-net-hostdev.xml

@@ -25,7 +25,7 @@
     <interface type='hostdev' managed='yes'>
       <mac address='00:11:22:33:44:55'/>
       <source>
-        <address type='pci' domain='0x0002' bus='0x03' slot='0x07' function='0x1'/>
+        <address type='pci' domain='0x0000' bus='0x03' slot='0x07' function='0x1'/>
       </source>
       <vlan>
         <tag id='42'/>

tests/qemuxml2argvtest.c

@@ -942,9 +942,21 @@ mymain(void)
     DO_TEST("net-mcast", NONE);
     DO_TEST("net-hostdev",
             QEMU_CAPS_PCIDEVICE, QEMU_CAPS_DEVICE, QEMU_CAPS_NODEFCONFIG);
+    DO_TEST("net-hostdev-multidomain",
+            QEMU_CAPS_PCIDEVICE, QEMU_CAPS_DEVICE, QEMU_CAPS_NODEFCONFIG,
+            QEMU_CAPS_HOST_PCI_MULTIDOMAIN);
+    DO_TEST_FAILURE("net-hostdev-multidomain",
+                    QEMU_CAPS_PCIDEVICE, QEMU_CAPS_DEVICE,
+                    QEMU_CAPS_NODEFCONFIG);
     DO_TEST("net-hostdev-vfio",
             QEMU_CAPS_PCIDEVICE, QEMU_CAPS_DEVICE, QEMU_CAPS_NODEFCONFIG,
             QEMU_CAPS_DEVICE_VFIO_PCI);
+    DO_TEST("net-hostdev-vfio-multidomain",
+            QEMU_CAPS_PCIDEVICE, QEMU_CAPS_DEVICE, QEMU_CAPS_NODEFCONFIG,
+            QEMU_CAPS_DEVICE_VFIO_PCI, QEMU_CAPS_HOST_PCI_MULTIDOMAIN);
+    DO_TEST_FAILURE("net-hostdev-vfio-multidomain",
+                    QEMU_CAPS_PCIDEVICE, QEMU_CAPS_DEVICE,
+                    QEMU_CAPS_NODEFCONFIG, QEMU_CAPS_DEVICE_VFIO_PCI);
 
     DO_TEST("serial-vc", NONE);
     DO_TEST("serial-pty", NONE);
@@ -1121,6 +1133,12 @@ mymain(void)
     DO_TEST("hostdev-vfio",
             QEMU_CAPS_PCIDEVICE, QEMU_CAPS_DEVICE, QEMU_CAPS_NODEFCONFIG,
             QEMU_CAPS_DEVICE_VFIO_PCI);
+    DO_TEST("hostdev-vfio-multidomain",
+            QEMU_CAPS_PCIDEVICE, QEMU_CAPS_DEVICE, QEMU_CAPS_NODEFCONFIG,
+            QEMU_CAPS_DEVICE_VFIO_PCI, QEMU_CAPS_HOST_PCI_MULTIDOMAIN);
+    DO_TEST_FAILURE("hostdev-vfio-multidomain",
+                    QEMU_CAPS_PCIDEVICE, QEMU_CAPS_DEVICE,
+                    QEMU_CAPS_NODEFCONFIG, QEMU_CAPS_DEVICE_VFIO_PCI);
     DO_TEST("pci-rom",
             QEMU_CAPS_PCIDEVICE, QEMU_CAPS_DEVICE, QEMU_CAPS_NODEFCONFIG,
             QEMU_CAPS_PCI_ROMBAR);