Pull schedular affinity code out into a separate module

* src/Makefile.am: Add processinfo.h/processinfo.c
* src/util/processinfo.c, src/util/processinfo.h: Module providing
  APIs for getting/setting process CPU affinity
* src/qemu/qemu_driver.c: Switch over to new APIs for schedular
  affinity
* src/libvirt_private.syms: Export virProcessInfoSetAffinity
  and virProcessInfoGetAffinity to internal drivers

po/POTFILES.in

@@ -52,6 +52,7 @@ src/util/conf.c
 src/util/iptables.c
 src/util/logging.c
 src/util/pci.c
+src/util/processinfo.c
 src/util/storage_file.c
 src/util/util.c
 src/util/uuid.c

src/Makefile.am

@@ -55,6 +55,7 @@ UTIL_SOURCES =							\
 		util/logging.c util/logging.h			\
 		util/memory.c util/memory.h			\
 		util/pci.c util/pci.h				\
+		util/processinfo.c util/processinfo.h		\
 		util/hostusb.c util/hostusb.h			\
 		util/network.c util/network.h			\
 		util/qparams.c util/qparams.h			\

src/libvirt_private.syms

@@ -374,6 +374,11 @@ pciDeviceListUnlock;
 pciDeviceListSteal;
 
 
+# processinfo.h
+virProcessInfoSetAffinity;
+virProcessInfoGetAffinity;
+
+
 # qparams.h
 qparam_get_query;
 qparam_query_parse;

src/qemu/qemu_driver.c

@@ -47,10 +47,6 @@
 #include <sys/ioctl.h>
 #include <sys/un.h>
 
-#if HAVE_SCHED_H
-#include <sched.h>
-#endif
-
 #include "virterror_internal.h"
 #include "logging.h"
 #include "datatypes.h"
@@ -72,6 +68,7 @@
 #include "node_device_conf.h"
 #include "pci.h"
 #include "hostusb.h"
+#include "processinfo.h"
 #include "security/security_driver.h"
 #include "cgroup.h"
 #include "libvirt_internal.h"
@@ -1362,11 +1359,11 @@ qemudInitCpus(virConnectPtr conn,
               struct qemud_driver *driver,
               virDomainObjPtr vm,
               const char *migrateFrom) {
-#if HAVE_SCHED_GETAFFINITY
-    cpu_set_t mask;
     int i, hostcpus, maxcpu = QEMUD_CPUMASK_LEN;
     virNodeInfo nodeinfo;
     qemuDomainObjPrivatePtr priv = vm->privateData;
+    unsigned char *cpumap;
+    int cpumaplen;
 
     if (nodeGetInfo(conn, &nodeinfo) < 0)
         return -1;
@@ -1377,25 +1374,37 @@ qemudInitCpus(virConnectPtr conn,
     if (maxcpu > hostcpus)
         maxcpu = hostcpus;
 
-    CPU_ZERO(&mask);
+    cpumaplen = VIR_CPU_MAPLEN(maxcpu);
+    if (VIR_ALLOC_N(cpumap, cpumaplen) < 0) {
+        virReportOOMError(conn);
+        return -1;
+    }
+
     if (vm->def->cpumask) {
-        for (i = 0 ; i < maxcpu ; i++)
+        /* XXX why don't we keep 'cpumask' in the libvirt cpumap
+         * format to start with ?!?! */
+        for (i = 0 ; i < maxcpu && i < vm->def->cpumasklen ; i++)
             if (vm->def->cpumask[i])
-                CPU_SET(i, &mask);
+                VIR_USE_CPU(cpumap, i);
     } else {
+        /* You may think this is redundant, but we can't assume libvirtd
+         * itself is running on all pCPUs, so we need to explicitly set
+         * the spawned QEMU instance to all pCPUs if no map is given in
+         * its config file */
         for (i = 0 ; i < maxcpu ; i++)
-            CPU_SET(i, &mask);
+            VIR_USE_CPU(cpumap, i);
     }
 
+    /* The XML config only gives a per-VM affinity, so we apply
+     * the same mapping to all vCPUs */
     for (i = 0 ; i < vm->nvcpupids ; i++) {
-        if (sched_setaffinity(vm->vcpupids[i],
-                              sizeof(mask), &mask) < 0) {
-            virReportSystemError(conn, errno, "%s",
-                                 _("failed to set CPU affinity"));
+        if (virProcessInfoSetAffinity(vm->vcpupids[i],
+                                      cpumap, cpumaplen, maxcpu) < 0) {
+            VIR_FREE(cpumap);
             return -1;
         }
     }
-#endif /* HAVE_SCHED_GETAFFINITY */
+    VIR_FREE(cpumap);
 
     /* XXX This resume doesn't really belong here. Move it up to caller */
     if (migrateFrom == NULL) {
@@ -3660,7 +3669,6 @@ cleanup:
 }
 
 
-#if HAVE_SCHED_GETAFFINITY
 static int
 qemudDomainPinVcpu(virDomainPtr dom,
                    unsigned int vcpu,
@@ -3668,8 +3676,7 @@ qemudDomainPinVcpu(virDomainPtr dom,
                    int maplen) {
     struct qemud_driver *driver = dom->conn->privateData;
     virDomainObjPtr vm;
-    cpu_set_t mask;
-    int i, maxcpu, hostcpus;
+    int maxcpu, hostcpus;
     virNodeInfo nodeinfo;
     int ret = -1;
 
@@ -3706,18 +3713,10 @@ qemudDomainPinVcpu(virDomainPtr dom,
     if (maxcpu > hostcpus)
         maxcpu = hostcpus;
 
-    CPU_ZERO(&mask);
-    for (i = 0 ; i < maxcpu ; i++) {
-        if (VIR_CPU_USABLE(cpumap, maplen, 0, i))
-            CPU_SET(i, &mask);
-    }
-
     if (vm->vcpupids != NULL) {
-        if (sched_setaffinity(vm->vcpupids[vcpu], sizeof(mask), &mask) < 0) {
-            virReportSystemError(dom->conn, errno, "%s",
-                                 _("cannot set affinity"));
+        if (virProcessInfoSetAffinity(vm->vcpupids[vcpu],
+                                      cpumap, maplen, maxcpu) < 0)
             goto cleanup;
-        }
     } else {
         qemudReportError(dom->conn, dom, NULL, VIR_ERR_NO_SUPPORT,
                          "%s", _("cpu affinity is not supported"));
@@ -3797,20 +3796,12 @@ qemudDomainGetVcpus(virDomainPtr dom,
             memset(cpumaps, 0, maplen * maxinfo);
             if (vm->vcpupids != NULL) {
                 for (v = 0 ; v < maxinfo ; v++) {
-                    cpu_set_t mask;
                     unsigned char *cpumap = VIR_GET_CPUMAP(cpumaps, maplen, v);
-                    CPU_ZERO(&mask);
 
-                    if (sched_getaffinity(vm->vcpupids[v], sizeof(mask), &mask) < 0) {
-                        virReportSystemError(dom->conn, errno, "%s",
-                                             _("cannot get affinity"));
+                    if (virProcessInfoGetAffinity(vm->vcpupids[v],
+                                                  cpumap, maplen, maxcpu) < 0)
                         goto cleanup;
                 }
-
-                    for (i = 0 ; i < maxcpu ; i++)
-                        if (CPU_ISSET(i, &mask))
-                            VIR_USE_CPU(cpumap, i);
-                }
             } else {
                 qemudReportError(dom->conn, dom, NULL, VIR_ERR_NO_SUPPORT,
                                  "%s", _("cpu affinity is not available"));
@@ -3825,7 +3816,6 @@ cleanup:
         virDomainObjUnlock(vm);
     return ret;
 }
-#endif /* HAVE_SCHED_GETAFFINITY */
 
 
 static int qemudDomainGetMaxVcpus(virDomainPtr dom) {
@@ -7513,13 +7503,8 @@ static virDriver qemuDriver = {
     qemudDomainRestore, /* domainRestore */
     qemudDomainCoreDump, /* domainCoreDump */
     qemudDomainSetVcpus, /* domainSetVcpus */
-#if HAVE_SCHED_GETAFFINITY
     qemudDomainPinVcpu, /* domainPinVcpu */
     qemudDomainGetVcpus, /* domainGetVcpus */
-#else
-    NULL, /* domainPinVcpu */
-    NULL, /* domainGetVcpus */
-#endif
     qemudDomainGetMaxVcpus, /* domainGetMaxVcpus */
     qemudDomainGetSecurityLabel, /* domainGetSecurityLabel */
     qemudNodeGetSecurityModel, /* nodeGetSecurityModel */

src/util/processinfo.c

+/*
+ * Copyright (C) 2009 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307  USA
+ *
+ * Authors:
+ *     Daniel P. Berrange <berrange@redhat.com>
+ */
+
+#include <config.h>
+
+#if HAVE_SCHED_H
+#include <sched.h>
+#endif
+
+#include "processinfo.h"
+#include "virterror_internal.h"
+
+#define VIR_FROM_THIS VIR_FROM_NONE
+
+#if HAVE_SCHED_GETAFFINITY
+
+int virProcessInfoSetAffinity(pid_t pid,
+                              const unsigned char *map,
+                              size_t maplen,
+                              int maxcpu)
+{
+    int i;
+    cpu_set_t mask;
+
+    CPU_ZERO(&mask);
+    for (i = 0 ; i < maxcpu ; i++) {
+        if (VIR_CPU_USABLE(map, maplen, 0, i))
+            CPU_SET(i, &mask);
+    }
+
+    if (sched_setaffinity(pid, sizeof(mask), &mask) < 0) {
+        virReportSystemError(NULL, errno,
+                             _("cannot set CPU affinity on process %d"), pid);
+        return -1;
+    }
+
+    return 0;
+}
+
+int virProcessInfoGetAffinity(pid_t pid,
+                              unsigned char *map,
+                              size_t maplen ATTRIBUTE_UNUSED,
+                              int maxcpu)
+{
+    int i;
+    cpu_set_t mask;
+
+    CPU_ZERO(&mask);
+    if (sched_getaffinity(pid, sizeof(mask), &mask) < 0) {
+        virReportSystemError(NULL, errno,
+                             _("cannot set CPU affinity on process %d"), pid);
+        return -1;
+    }
+
+    for (i = 0 ; i < maxcpu ; i++)
+        if (CPU_ISSET(i, &mask))
+            VIR_USE_CPU(map, i);
+
+    return 0;
+}
+
+#else /* HAVE_SCHED_GETAFFINITY */
+
+int virProcessInfoSetAffinity(pid_t pid ATTRIBUTE_UNUSED,
+                              unsigned char *map ATTRIBUTE_UNUSED,
+                              size_t maplen ATTRIBUTE_UNUSED,
+                              int maxcpu ATTRIBUTE_UNUSED)
+{
+    virReportSystemError(NULL, ENOSYS, "%s",
+                         _("Process CPU affinity is not supported on this platform"));
+    return -1;
+}
+
+int virProcessInfoGetAffinity(pid_t pid ATTRIBUTE_UNUSED,
+                              unsigned char *map ATTRIBUTE_UNUSED,
+                              size_t maplen ATTRIBUTE_UNUSED,
+                              int maxcpu ATTRIBUTE_UNUSED)
+{
+    virReportSystemError(NULL, ENOSYS, "%s",
+                         _("Process CPU affinity is not supported on this platform"));
+    return -1;
+}
+#endif /* HAVE_SCHED_GETAFFINITY */

src/util/processinfo.h

+/*
+ * Copyright (C) 2009 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307  USA
+ *
+ * Authors:
+ *     Daniel P. Berrange <berrange@redhat.com>
+ */
+
+#ifndef __VIR_PROCESSINFO_H__
+#define __VIR_PROCESSINFO_H__
+
+#include "internal.h"
+
+int virProcessInfoSetAffinity(pid_t pid,
+                              const unsigned char *map,
+                              size_t maplen,
+                              int maxcpu);
+
+int virProcessInfoGetAffinity(pid_t pid,
+                              unsigned char *map,
+                              size_t maplen,
+                              int maxcpu);
+
+#endif /* __VIR_PROCESSINFO_H__ */