qemu: set qemu process' RLIMIT_MEMLOCK when VFIO is used

VFIO requires all of the guest's memory and IO space to be lockable in
RAM. The domain's max_balloon is the maximum amount of memory the
domain can have (in KiB). We add a generous 1GiB to that for IO space
(still much better than KVM device assignment, where the KVM module
actually *ignores* the process limits and locks everything anyway),
and convert from KiB to bytes.

In the case of hotplug, we are changing the limit for the already
existing qemu process (prlimit() is used under the hood), and for
regular commandline additions of vfio devices, we schedule a call to
setrlimit() that will happen after the qemu process is forked.

src/qemu/qemu_command.c

@@ -7906,13 +7906,24 @@ qemuBuildCommandLine(virConnectPtr conn,
         if (hostdev->mode == VIR_DOMAIN_HOSTDEV_MODE_SUBSYS &&
             hostdev->source.subsys.type == VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI) {
 
-            if ((hostdev->source.subsys.u.pci.backend
-                 == VIR_DOMAIN_HOSTDEV_PCI_BACKEND_TYPE_VFIO) &&
-                !virQEMUCapsGet(qemuCaps, QEMU_CAPS_DEVICE_VFIO_PCI)) {
-                virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
-                               _("VFIO PCI device assignment is not supported by "
-                                 "this version of qemu"));
-                goto error;
+            if (hostdev->source.subsys.u.pci.backend
+                == VIR_DOMAIN_HOSTDEV_PCI_BACKEND_TYPE_VFIO) {
+                unsigned long long memKB;
+
+                if (!virQEMUCapsGet(qemuCaps, QEMU_CAPS_DEVICE_VFIO_PCI)) {
+                    virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+                                   _("VFIO PCI device assignment is not "
+                                     "supported by this version of qemu"));
+                    goto error;
+                }
+                /* VFIO requires all of the guest's memory to be
+                 * locked resident, plus some amount for IO
+                 * space. Alex Williamson suggested adding 1GiB for IO
+                 * space just to be safe (some finer tuning might be
+                 * nice, though).
+                 */
+                memKB = def->mem.max_balloon + (1024 * 1024);
+                virCommandSetMaxMemLock(cmd, memKB * 1024);
             }
 
             if (virQEMUCapsGet(qemuCaps, QEMU_CAPS_DEVICE)) {

src/qemu/qemu_hotplug.c

@@ -38,6 +38,7 @@
 #include "viralloc.h"
 #include "virpci.h"
 #include "virfile.h"
+#include "virprocess.h"
 #include "qemu_cgroup.h"
 #include "locking/domain_lock.h"
 #include "network/bridge_driver.h"
@@ -999,13 +1000,25 @@ int qemuDomainAttachHostPciDevice(virQEMUDriverPtr driver,
                                      &hostdev, 1) < 0)
         return -1;
 
-    if ((hostdev->source.subsys.u.pci.backend
-         == VIR_DOMAIN_HOSTDEV_PCI_BACKEND_TYPE_VFIO) &&
-        !virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_DEVICE_VFIO_PCI)) {
-        virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
-                       _("VFIO PCI device assignment is not supported by "
-                         "this version of qemu"));
-        goto error;
+    if (hostdev->source.subsys.u.pci.backend
+        == VIR_DOMAIN_HOSTDEV_PCI_BACKEND_TYPE_VFIO) {
+        unsigned long long memKB;
+
+        if (!virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_DEVICE_VFIO_PCI)) {
+            virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+                           _("VFIO PCI device assignment is not "
+                             "supported by this version of qemu"));
+            goto error;
+        }
+        /* VFIO requires all of the guest's memory to be locked
+         * resident, plus some amount for IO space. Alex Williamson
+         * suggested adding 1GiB for IO space just to be safe (some
+         * finer tuning might be nice, though).
+         * In this case, the guest's memory may already be locked, but
+         * it doesn't hurt to "change" the limit to the same value.
+         */
+        memKB = vm->def->mem.max_balloon + (1024 * 1024);
+        virProcessSetMaxMemLock(vm->pid, memKB * 1024);
     }
 
     if (virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_DEVICE)) {