From 37f415da42e8bf8fb3862a83c15c305a12ad7cf0 Mon Sep 17 00:00:00 2001 From: "Daniel P. Berrange" Date: Mon, 16 Nov 2009 15:22:34 +0000 Subject: [PATCH] Pull schedular affinity code out into a separate module * src/Makefile.am: Add processinfo.h/processinfo.c * src/util/processinfo.c, src/util/processinfo.h: Module providing APIs for getting/setting process CPU affinity * src/qemu/qemu_driver.c: Switch over to new APIs for schedular affinity * src/libvirt_private.syms: Export virProcessInfoSetAffinity and virProcessInfoGetAffinity to internal drivers --- po/POTFILES.in | 1 + src/Makefile.am | 1 + src/libvirt_private.syms | 5 ++ src/qemu/qemu_driver.c | 73 +++++++++++----------------- src/util/processinfo.c | 101 +++++++++++++++++++++++++++++++++++++++ src/util/processinfo.h | 37 ++++++++++++++ 6 files changed, 174 insertions(+), 44 deletions(-) create mode 100644 src/util/processinfo.c create mode 100644 src/util/processinfo.h diff --git a/po/POTFILES.in b/po/POTFILES.in index 266f70c735..116aa87098 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -52,6 +52,7 @@ src/util/conf.c src/util/iptables.c src/util/logging.c src/util/pci.c +src/util/processinfo.c src/util/storage_file.c src/util/util.c src/util/uuid.c diff --git a/src/Makefile.am b/src/Makefile.am index 26be8e2306..9a3c9c80fe 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -55,6 +55,7 @@ UTIL_SOURCES = \ util/logging.c util/logging.h \ util/memory.c util/memory.h \ util/pci.c util/pci.h \ + util/processinfo.c util/processinfo.h \ util/hostusb.c util/hostusb.h \ util/network.c util/network.h \ util/qparams.c util/qparams.h \ diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index c473d499c5..e880c2e791 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -374,6 +374,11 @@ pciDeviceListUnlock; pciDeviceListSteal; +# processinfo.h +virProcessInfoSetAffinity; +virProcessInfoGetAffinity; + + # qparams.h qparam_get_query; qparam_query_parse; diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index a4a87ac683..2c5086b558 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -47,10 +47,6 @@ #include #include -#if HAVE_SCHED_H -#include -#endif - #include "virterror_internal.h" #include "logging.h" #include "datatypes.h" @@ -72,6 +68,7 @@ #include "node_device_conf.h" #include "pci.h" #include "hostusb.h" +#include "processinfo.h" #include "security/security_driver.h" #include "cgroup.h" #include "libvirt_internal.h" @@ -1362,11 +1359,11 @@ qemudInitCpus(virConnectPtr conn, struct qemud_driver *driver, virDomainObjPtr vm, const char *migrateFrom) { -#if HAVE_SCHED_GETAFFINITY - cpu_set_t mask; int i, hostcpus, maxcpu = QEMUD_CPUMASK_LEN; virNodeInfo nodeinfo; qemuDomainObjPrivatePtr priv = vm->privateData; + unsigned char *cpumap; + int cpumaplen; if (nodeGetInfo(conn, &nodeinfo) < 0) return -1; @@ -1377,25 +1374,37 @@ qemudInitCpus(virConnectPtr conn, if (maxcpu > hostcpus) maxcpu = hostcpus; - CPU_ZERO(&mask); + cpumaplen = VIR_CPU_MAPLEN(maxcpu); + if (VIR_ALLOC_N(cpumap, cpumaplen) < 0) { + virReportOOMError(conn); + return -1; + } + if (vm->def->cpumask) { - for (i = 0 ; i < maxcpu ; i++) + /* XXX why don't we keep 'cpumask' in the libvirt cpumap + * format to start with ?!?! */ + for (i = 0 ; i < maxcpu && i < vm->def->cpumasklen ; i++) if (vm->def->cpumask[i]) - CPU_SET(i, &mask); + VIR_USE_CPU(cpumap, i); } else { + /* You may think this is redundant, but we can't assume libvirtd + * itself is running on all pCPUs, so we need to explicitly set + * the spawned QEMU instance to all pCPUs if no map is given in + * its config file */ for (i = 0 ; i < maxcpu ; i++) - CPU_SET(i, &mask); + VIR_USE_CPU(cpumap, i); } + /* The XML config only gives a per-VM affinity, so we apply + * the same mapping to all vCPUs */ for (i = 0 ; i < vm->nvcpupids ; i++) { - if (sched_setaffinity(vm->vcpupids[i], - sizeof(mask), &mask) < 0) { - virReportSystemError(conn, errno, "%s", - _("failed to set CPU affinity")); + if (virProcessInfoSetAffinity(vm->vcpupids[i], + cpumap, cpumaplen, maxcpu) < 0) { + VIR_FREE(cpumap); return -1; } } -#endif /* HAVE_SCHED_GETAFFINITY */ + VIR_FREE(cpumap); /* XXX This resume doesn't really belong here. Move it up to caller */ if (migrateFrom == NULL) { @@ -3660,7 +3669,6 @@ cleanup: } -#if HAVE_SCHED_GETAFFINITY static int qemudDomainPinVcpu(virDomainPtr dom, unsigned int vcpu, @@ -3668,8 +3676,7 @@ qemudDomainPinVcpu(virDomainPtr dom, int maplen) { struct qemud_driver *driver = dom->conn->privateData; virDomainObjPtr vm; - cpu_set_t mask; - int i, maxcpu, hostcpus; + int maxcpu, hostcpus; virNodeInfo nodeinfo; int ret = -1; @@ -3706,18 +3713,10 @@ qemudDomainPinVcpu(virDomainPtr dom, if (maxcpu > hostcpus) maxcpu = hostcpus; - CPU_ZERO(&mask); - for (i = 0 ; i < maxcpu ; i++) { - if (VIR_CPU_USABLE(cpumap, maplen, 0, i)) - CPU_SET(i, &mask); - } - if (vm->vcpupids != NULL) { - if (sched_setaffinity(vm->vcpupids[vcpu], sizeof(mask), &mask) < 0) { - virReportSystemError(dom->conn, errno, "%s", - _("cannot set affinity")); + if (virProcessInfoSetAffinity(vm->vcpupids[vcpu], + cpumap, maplen, maxcpu) < 0) goto cleanup; - } } else { qemudReportError(dom->conn, dom, NULL, VIR_ERR_NO_SUPPORT, "%s", _("cpu affinity is not supported")); @@ -3797,19 +3796,11 @@ qemudDomainGetVcpus(virDomainPtr dom, memset(cpumaps, 0, maplen * maxinfo); if (vm->vcpupids != NULL) { for (v = 0 ; v < maxinfo ; v++) { - cpu_set_t mask; unsigned char *cpumap = VIR_GET_CPUMAP(cpumaps, maplen, v); - CPU_ZERO(&mask); - if (sched_getaffinity(vm->vcpupids[v], sizeof(mask), &mask) < 0) { - virReportSystemError(dom->conn, errno, "%s", - _("cannot get affinity")); + if (virProcessInfoGetAffinity(vm->vcpupids[v], + cpumap, maplen, maxcpu) < 0) goto cleanup; - } - - for (i = 0 ; i < maxcpu ; i++) - if (CPU_ISSET(i, &mask)) - VIR_USE_CPU(cpumap, i); } } else { qemudReportError(dom->conn, dom, NULL, VIR_ERR_NO_SUPPORT, @@ -3825,7 +3816,6 @@ cleanup: virDomainObjUnlock(vm); return ret; } -#endif /* HAVE_SCHED_GETAFFINITY */ static int qemudDomainGetMaxVcpus(virDomainPtr dom) { @@ -7513,13 +7503,8 @@ static virDriver qemuDriver = { qemudDomainRestore, /* domainRestore */ qemudDomainCoreDump, /* domainCoreDump */ qemudDomainSetVcpus, /* domainSetVcpus */ -#if HAVE_SCHED_GETAFFINITY qemudDomainPinVcpu, /* domainPinVcpu */ qemudDomainGetVcpus, /* domainGetVcpus */ -#else - NULL, /* domainPinVcpu */ - NULL, /* domainGetVcpus */ -#endif qemudDomainGetMaxVcpus, /* domainGetMaxVcpus */ qemudDomainGetSecurityLabel, /* domainGetSecurityLabel */ qemudNodeGetSecurityModel, /* nodeGetSecurityModel */ diff --git a/src/util/processinfo.c b/src/util/processinfo.c new file mode 100644 index 0000000000..aaffd8855a --- /dev/null +++ b/src/util/processinfo.c @@ -0,0 +1,101 @@ +/* + * Copyright (C) 2009 Red Hat, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Authors: + * Daniel P. Berrange + */ + +#include + +#if HAVE_SCHED_H +#include +#endif + +#include "processinfo.h" +#include "virterror_internal.h" + +#define VIR_FROM_THIS VIR_FROM_NONE + +#if HAVE_SCHED_GETAFFINITY + +int virProcessInfoSetAffinity(pid_t pid, + const unsigned char *map, + size_t maplen, + int maxcpu) +{ + int i; + cpu_set_t mask; + + CPU_ZERO(&mask); + for (i = 0 ; i < maxcpu ; i++) { + if (VIR_CPU_USABLE(map, maplen, 0, i)) + CPU_SET(i, &mask); + } + + if (sched_setaffinity(pid, sizeof(mask), &mask) < 0) { + virReportSystemError(NULL, errno, + _("cannot set CPU affinity on process %d"), pid); + return -1; + } + + return 0; +} + +int virProcessInfoGetAffinity(pid_t pid, + unsigned char *map, + size_t maplen ATTRIBUTE_UNUSED, + int maxcpu) +{ + int i; + cpu_set_t mask; + + CPU_ZERO(&mask); + if (sched_getaffinity(pid, sizeof(mask), &mask) < 0) { + virReportSystemError(NULL, errno, + _("cannot set CPU affinity on process %d"), pid); + return -1; + } + + for (i = 0 ; i < maxcpu ; i++) + if (CPU_ISSET(i, &mask)) + VIR_USE_CPU(map, i); + + return 0; +} + +#else /* HAVE_SCHED_GETAFFINITY */ + +int virProcessInfoSetAffinity(pid_t pid ATTRIBUTE_UNUSED, + unsigned char *map ATTRIBUTE_UNUSED, + size_t maplen ATTRIBUTE_UNUSED, + int maxcpu ATTRIBUTE_UNUSED) +{ + virReportSystemError(NULL, ENOSYS, "%s", + _("Process CPU affinity is not supported on this platform")); + return -1; +} + +int virProcessInfoGetAffinity(pid_t pid ATTRIBUTE_UNUSED, + unsigned char *map ATTRIBUTE_UNUSED, + size_t maplen ATTRIBUTE_UNUSED, + int maxcpu ATTRIBUTE_UNUSED) +{ + virReportSystemError(NULL, ENOSYS, "%s", + _("Process CPU affinity is not supported on this platform")); + return -1; +} +#endif /* HAVE_SCHED_GETAFFINITY */ diff --git a/src/util/processinfo.h b/src/util/processinfo.h new file mode 100644 index 0000000000..17800bd56b --- /dev/null +++ b/src/util/processinfo.h @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2009 Red Hat, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * + * Authors: + * Daniel P. Berrange + */ + +#ifndef __VIR_PROCESSINFO_H__ +#define __VIR_PROCESSINFO_H__ + +#include "internal.h" + +int virProcessInfoSetAffinity(pid_t pid, + const unsigned char *map, + size_t maplen, + int maxcpu); + +int virProcessInfoGetAffinity(pid_t pid, + unsigned char *map, + size_t maplen, + int maxcpu); + +#endif /* __VIR_PROCESSINFO_H__ */ -- GitLab