diff --git a/src/qemu/qemu.conf b/src/qemu/qemu.conf index 18ce2a84904d635516c0adaa8e3822a309906aa5..a079b93d20199751d3703e96bed7ada913751d31 100644 --- a/src/qemu/qemu.conf +++ b/src/qemu/qemu.conf @@ -330,7 +330,14 @@ # unspecified here, determination of a host mount point in /proc/mounts # will be attempted. Specifying an explicit mount overrides detection # of the same in /proc/mounts. Setting the mount point to "" will -# disable guest hugepage backing. +# disable guest hugepage backing. If desired, multiple mount points can +# be specified at once, separated by comma and enclosed in square +# brackets, for example: +# +# hugetlbfs_mount = ["/dev/hugepages2M", "/dev/hugepages1G"] +# +# The size of huge page served by specific mount point is determined by +# libvirt at the daemon startup. # # NB, within this mount point, guests will create memory backing files # in a location of $MOUNTPOINT/libvirt/qemu diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c index beb8ca80daccbc191bc0ecfed29ff9673d4e2862..2f788e956e39086bb95e8ed5c491a0da19d6255a 100644 --- a/src/qemu/qemu_command.c +++ b/src/qemu/qemu_command.c @@ -7384,14 +7384,12 @@ qemuBuildCommandLine(virConnectPtr conn, def->mem.max_balloon = VIR_DIV_UP(def->mem.max_balloon, 1024) * 1024; virCommandAddArgFormat(cmd, "%llu", def->mem.max_balloon / 1024); if (def->mem.hugepage_backed) { - if (!cfg->hugetlbfsMount) { - virReportError(VIR_ERR_INTERNAL_ERROR, - "%s", _("hugetlbfs filesystem is not mounted")); - goto error; - } - if (!cfg->hugepagePath) { + char *mem_path; + + if (!cfg->nhugetlbfs) { virReportError(VIR_ERR_INTERNAL_ERROR, - "%s", _("hugepages are disabled by administrator config")); + "%s", _("hugetlbfs filesystem is not mounted " + "or disabled by administrator config")); goto error; } if (!virQEMUCapsGet(qemuCaps, QEMU_CAPS_MEM_PATH)) { @@ -7400,8 +7398,14 @@ qemuBuildCommandLine(virConnectPtr conn, def->emulator); goto error; } + + if (!(mem_path = qemuGetDefaultHugepath(cfg->hugetlbfs, + cfg->nhugetlbfs))) + goto error; + virCommandAddArgList(cmd, "-mem-prealloc", "-mem-path", - cfg->hugepagePath, NULL); + mem_path, NULL); + VIR_FREE(mem_path); } if (def->mem.locked && !virQEMUCapsGet(qemuCaps, QEMU_CAPS_MLOCK)) { diff --git a/src/qemu/qemu_conf.c b/src/qemu/qemu_conf.c index 9636a87097a1ad62cd7c73bcaa572b5d618e8445..6bfa48ed964f28c4807ce4a15b8e8ce24c3cd73e 100644 --- a/src/qemu/qemu_conf.c +++ b/src/qemu/qemu_conf.c @@ -230,19 +230,17 @@ virQEMUDriverConfigPtr virQEMUDriverConfigNew(bool privileged) cfg->migrationPortMin = QEMU_MIGRATION_PORT_MIN; cfg->migrationPortMax = QEMU_MIGRATION_PORT_MAX; -#if defined HAVE_MNTENT_H && defined HAVE_GETMNTENT_R - /* For privileged driver, try and find hugepage mount automatically. + /* For privileged driver, try and find hugetlbfs mounts automatically. * Non-privileged driver requires admin to create a dir for the - * user, chown it, and then let user configure it manually */ + * user, chown it, and then let user configure it manually. */ if (privileged && - !(cfg->hugetlbfsMount = virFileFindMountPoint("hugetlbfs"))) { - if (errno != ENOENT) { - virReportSystemError(errno, "%s", - _("unable to find hugetlbfs mountpoint")); + virFileFindHugeTLBFS(&cfg->hugetlbfs, &cfg->nhugetlbfs) < 0) { + /* This however is not implemented on all platforms. */ + virErrorPtr err = virGetLastError(); + if (err && err->code != VIR_ERR_NO_SUPPORT) goto error; - } } -#endif + if (VIR_STRDUP(cfg->bridgeHelperName, "/usr/libexec/qemu-bridge-helper") < 0) goto error; @@ -293,8 +291,11 @@ static void virQEMUDriverConfigDispose(void *obj) VIR_FREE(cfg->spicePassword); VIR_FREE(cfg->spiceSASLdir); - VIR_FREE(cfg->hugetlbfsMount); - VIR_FREE(cfg->hugepagePath); + while (cfg->nhugetlbfs) { + cfg->nhugetlbfs--; + VIR_FREE(cfg->hugetlbfs[cfg->nhugetlbfs].mnt_dir); + } + VIR_FREE(cfg->hugetlbfs); VIR_FREE(cfg->bridgeHelperName); VIR_FREE(cfg->saveImageFormat); @@ -307,6 +308,26 @@ static void virQEMUDriverConfigDispose(void *obj) } +static int +virQEMUDriverConfigHugeTLBFSInit(virHugeTLBFSPtr hugetlbfs, + const char *path, + bool deflt) +{ + int ret = -1; + + if (VIR_STRDUP(hugetlbfs->mnt_dir, path) < 0) + goto cleanup; + + if (virFileGetHugepageSize(path, &hugetlbfs->size) < 0) + goto cleanup; + + hugetlbfs->deflt = deflt; + ret = 0; + cleanup: + return ret; +} + + int virQEMUDriverConfigLoadFile(virQEMUDriverConfigPtr cfg, const char *filename) { @@ -555,7 +576,59 @@ int virQEMUDriverConfigLoadFile(virQEMUDriverConfigPtr cfg, GET_VALUE_BOOL("auto_dump_bypass_cache", cfg->autoDumpBypassCache); GET_VALUE_BOOL("auto_start_bypass_cache", cfg->autoStartBypassCache); - GET_VALUE_STR("hugetlbfs_mount", cfg->hugetlbfsMount); + /* Some crazy backcompat. Back in the old days, this was just a pure + * string. We must continue supporting it. These days however, this may be + * an array of strings. */ + p = virConfGetValue(conf, "hugetlbfs_mount"); + if (p) { + /* There already might be something autodetected. Avoid leaking it. */ + while (cfg->nhugetlbfs) { + cfg->nhugetlbfs--; + VIR_FREE(cfg->hugetlbfs[cfg->nhugetlbfs].mnt_dir); + } + VIR_FREE(cfg->hugetlbfs); + + if (p->type == VIR_CONF_LIST) { + size_t len = 0; + virConfValuePtr pp = p->list; + + /* Calc length and check items */ + while (pp) { + if (pp->type != VIR_CONF_STRING) { + virReportError(VIR_ERR_CONF_SYNTAX, "%s", + _("hugetlbfs_mount must be a list of strings")); + goto cleanup; + } + len++; + pp = pp->next; + } + + if (len && VIR_ALLOC_N(cfg->hugetlbfs, len) < 0) + goto cleanup; + cfg->nhugetlbfs = len; + + pp = p->list; + len = 0; + while (pp) { + if (virQEMUDriverConfigHugeTLBFSInit(&cfg->hugetlbfs[len], + pp->str, !len) < 0) + goto cleanup; + len++; + pp = pp->next; + } + } else { + CHECK_TYPE("hugetlbfs_mount", VIR_CONF_STRING); + if (STRNEQ(p->str, "")) { + if (VIR_ALLOC_N(cfg->hugetlbfs, 1) < 0) + goto cleanup; + cfg->nhugetlbfs = 1; + if (virQEMUDriverConfigHugeTLBFSInit(&cfg->hugetlbfs[0], + p->str, true) < 0) + goto cleanup; + } + } + } + GET_VALUE_STR("bridge_helper", cfg->bridgeHelperName); GET_VALUE_BOOL("mac_filter", cfg->macFilter); @@ -1406,3 +1479,30 @@ qemuTranslateSnapshotDiskSourcePool(virConnectPtr conn ATTRIBUTE_UNUSED, _("Snapshots are not yet supported with 'pool' volumes")); return -1; } + +char * +qemuGetHugepagePath(virHugeTLBFSPtr hugepage) +{ + char *ret; + + if (virAsprintf(&ret, "%s/libvirt/qemu", hugepage->mnt_dir) < 0) + return NULL; + + return ret; +} + +char * +qemuGetDefaultHugepath(virHugeTLBFSPtr hugetlbfs, + size_t nhugetlbfs) +{ + size_t i; + + for (i = 0; i < nhugetlbfs; i++) + if (hugetlbfs[i].deflt) + break; + + if (i == nhugetlbfs) + i = 0; + + return qemuGetHugepagePath(&hugetlbfs[i]); +} diff --git a/src/qemu/qemu_conf.h b/src/qemu/qemu_conf.h index 78b08e55c88ba0bda3788eec670ad752c59bb575..c3c9d6c5ae875e1c89666e02a6ccff1ef96f5339 100644 --- a/src/qemu/qemu_conf.h +++ b/src/qemu/qemu_conf.h @@ -45,6 +45,7 @@ # include "qemu_capabilities.h" # include "virclosecallbacks.h" # include "virhostdev.h" +# include "virfile.h" # ifdef CPU_SETSIZE /* Linux */ # define QEMUD_CPUMASK_LEN CPU_SETSIZE @@ -126,8 +127,9 @@ struct _virQEMUDriverConfig { int webSocketPortMin; int webSocketPortMax; - char *hugetlbfsMount; - char *hugepagePath; + virHugeTLBFSPtr hugetlbfs; + size_t nhugetlbfs; + char *bridgeHelperName; bool macFilter; @@ -311,4 +313,7 @@ int qemuTranslateDiskSourcePool(virConnectPtr conn, int qemuTranslateSnapshotDiskSourcePool(virConnectPtr conn, virDomainSnapshotDiskDefPtr def); +char * qemuGetHugepagePath(virHugeTLBFSPtr hugepage); +char * qemuGetDefaultHugepath(virHugeTLBFSPtr hugetlbfs, + size_t nhugetlbfs); #endif /* __QEMUD_CONF_H */ diff --git a/src/qemu/qemu_driver.c b/src/qemu/qemu_driver.c index 704ba3918eb049ddec6e07a1421765d8616789cf..c80753947391ed019229df32b0c1b975f95cb79a 100644 --- a/src/qemu/qemu_driver.c +++ b/src/qemu/qemu_driver.c @@ -627,11 +627,11 @@ qemuStateInitialize(bool privileged, char *driverConf = NULL; virConnectPtr conn = NULL; char ebuf[1024]; - char *membase = NULL; - char *mempath = NULL; virQEMUDriverConfigPtr cfg; uid_t run_uid = -1; gid_t run_gid = -1; + char *hugepagePath = NULL; + size_t i; if (VIR_ALLOC(qemu_driver) < 0) return -1; @@ -803,37 +803,33 @@ qemuStateInitialize(bool privileged, /* If hugetlbfs is present, then we need to create a sub-directory within * it, since we can't assume the root mount point has permissions that - * will let our spawned QEMU instances use it. - * - * NB the check for '/', since user may config "" to disable hugepages - * even when mounted - */ - if (cfg->hugetlbfsMount && - cfg->hugetlbfsMount[0] == '/') { - if (virAsprintf(&membase, "%s/libvirt", - cfg->hugetlbfsMount) < 0 || - virAsprintf(&mempath, "%s/qemu", membase) < 0) + * will let our spawned QEMU instances use it. */ + for (i = 0; i < cfg->nhugetlbfs; i++) { + hugepagePath = qemuGetHugepagePath(&cfg->hugetlbfs[i]); + + if (!hugepagePath) goto error; - if (virFileMakePath(mempath) < 0) { + if (virFileMakePath(hugepagePath) < 0) { virReportSystemError(errno, - _("unable to create hugepage path %s"), mempath); + _("unable to create hugepage path %s"), + hugepagePath); goto error; } if (cfg->privileged) { - if (virFileUpdatePerm(membase, 0, S_IXGRP | S_IXOTH) < 0) + if (virFileUpdatePerm(cfg->hugetlbfs[i].mnt_dir, + 0, S_IXGRP | S_IXOTH) < 0) goto error; - if (chown(mempath, cfg->user, cfg->group) < 0) { + if (chown(hugepagePath, cfg->user, cfg->group) < 0) { virReportSystemError(errno, _("unable to set ownership on %s to %d:%d"), - mempath, (int) cfg->user, + hugepagePath, + (int) cfg->user, (int) cfg->group); goto error; } } - VIR_FREE(membase); - - cfg->hugepagePath = mempath; + VIR_FREE(hugepagePath); } if (!(qemu_driver->closeCallbacks = virCloseCallbacksNew())) @@ -894,8 +890,7 @@ qemuStateInitialize(bool privileged, error: virObjectUnref(conn); VIR_FREE(driverConf); - VIR_FREE(membase); - VIR_FREE(mempath); + VIR_FREE(hugepagePath); qemuStateCleanup(); return -1; } diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c index 3d1073205626f35bd0ad2b4163ea805602172838..7626cef2aec1eb529f414c5198e5efcb1197b6ee 100644 --- a/src/qemu/qemu_process.c +++ b/src/qemu/qemu_process.c @@ -3791,12 +3791,21 @@ int qemuProcessStart(virConnectPtr conn, } virDomainAuditSecurityLabel(vm, true); - if (cfg->hugepagePath && vm->def->mem.hugepage_backed) { - if (virSecurityManagerSetHugepages(driver->securityManager, - vm->def, cfg->hugepagePath) < 0) { - virReportError(VIR_ERR_INTERNAL_ERROR, - "%s", _("Unable to set huge path in security driver")); - goto cleanup; + if (vm->def->mem.hugepage_backed) { + for (i = 0; i < cfg->nhugetlbfs; i++) { + char *hugepagePath = qemuGetHugepagePath(&cfg->hugetlbfs[i]); + + if (!hugepagePath) + goto cleanup; + + if (virSecurityManagerSetHugepages(driver->securityManager, + vm->def, hugepagePath) < 0) { + virReportError(VIR_ERR_INTERNAL_ERROR, + "%s", _("Unable to set huge path in security driver")); + VIR_FREE(hugepagePath); + goto cleanup; + } + VIR_FREE(hugepagePath); } } diff --git a/tests/qemuxml2argvtest.c b/tests/qemuxml2argvtest.c index 115cd37681144a3ac3bb9708beca0a2e2dc0ecdd..a86f779967443e894814c96ef74af00e5863fbc4 100644 --- a/tests/qemuxml2argvtest.c +++ b/tests/qemuxml2argvtest.c @@ -524,12 +524,14 @@ mymain(void) VIR_FREE(driver.config->stateDir); if (VIR_STRDUP_QUIET(driver.config->stateDir, "/nowhere") < 0) return EXIT_FAILURE; - VIR_FREE(driver.config->hugetlbfsMount); - if (VIR_STRDUP_QUIET(driver.config->hugetlbfsMount, "/dev/hugepages") < 0) + VIR_FREE(driver.config->hugetlbfs); + if (VIR_ALLOC_N(driver.config->hugetlbfs, 1) < 0) return EXIT_FAILURE; - VIR_FREE(driver.config->hugepagePath); - if (VIR_STRDUP_QUIET(driver.config->hugepagePath, "/dev/hugepages/libvirt/qemu") < 0) + driver.config->nhugetlbfs = 1; + if (VIR_STRDUP(driver.config->hugetlbfs[0].mnt_dir, "/dev/hugepages") < 0) return EXIT_FAILURE; + driver.config->hugetlbfs[0].size = 2048; + driver.config->hugetlbfs[0].deflt = true; driver.config->spiceTLS = 1; if (VIR_STRDUP_QUIET(driver.config->spicePassword, "123456") < 0) return EXIT_FAILURE;