From aa8604dd453eb0ef08c57c7495c120611ec31c1a Mon Sep 17 00:00:00 2001 From: "Daniel P. Berrange" Date: Thu, 28 Mar 2013 18:08:39 +0000 Subject: [PATCH] Add a new virCgroupNewPartition for setting up resource partitions A resource partition is an absolute cgroup path, ignoring the current process placement. Expose a virCgroupNewPartition API for constructing such cgroups Signed-off-by: Daniel P. Berrange --- src/libvirt_private.syms | 4 +- src/lxc/lxc_cgroup.c | 2 +- src/qemu/qemu_cgroup.c | 2 +- src/util/vircgroup.c | 147 +++++++++++++++++++++++++++++++--- src/util/vircgroup.h | 20 ++++- tests/vircgrouptest.c | 165 ++++++++++++++++++++++++++++++++++++++- 6 files changed, 320 insertions(+), 20 deletions(-) diff --git a/src/libvirt_private.syms b/src/libvirt_private.syms index 0d8582e50f..a8e2cb6ad4 100644 --- a/src/libvirt_private.syms +++ b/src/libvirt_private.syms @@ -1122,9 +1122,11 @@ virCgroupKill; virCgroupKillPainfully; virCgroupKillRecursive; virCgroupMoveTask; -virCgroupNewDomain; +virCgroupNewDomainDriver; +virCgroupNewDomainPartition; virCgroupNewDriver; virCgroupNewEmulator; +virCgroupNewPartition; virCgroupNewSelf; virCgroupNewVcpu; virCgroupPathOfController; diff --git a/src/lxc/lxc_cgroup.c b/src/lxc/lxc_cgroup.c index 7d1432b7fe..72940bde00 100644 --- a/src/lxc/lxc_cgroup.c +++ b/src/lxc/lxc_cgroup.c @@ -536,7 +536,7 @@ virCgroupPtr virLXCCgroupCreate(virDomainDefPtr def) goto cleanup; } - rc = virCgroupNewDomain(driver, def->name, true, &cgroup); + rc = virCgroupNewDomainDriver(driver, def->name, true, &cgroup); if (rc != 0) { virReportSystemError(-rc, _("Unable to create cgroup for domain %s"), diff --git a/src/qemu/qemu_cgroup.c b/src/qemu/qemu_cgroup.c index dbd2210d31..e6af69c467 100644 --- a/src/qemu/qemu_cgroup.c +++ b/src/qemu/qemu_cgroup.c @@ -243,7 +243,7 @@ int qemuInitCgroup(virQEMUDriverPtr driver, goto cleanup; } - rc = virCgroupNewDomain(driverGroup, vm->def->name, true, &priv->cgroup); + rc = virCgroupNewDomainDriver(driverGroup, vm->def->name, true, &priv->cgroup); if (rc != 0) { virReportSystemError(-rc, _("Unable to create cgroup for %s"), diff --git a/src/util/vircgroup.c b/src/util/vircgroup.c index cabe2fe612..470b8a17b3 100644 --- a/src/util/vircgroup.c +++ b/src/util/vircgroup.c @@ -1056,6 +1056,77 @@ cleanup: return rc; } + +#if defined HAVE_MNTENT_H && defined HAVE_GETMNTENT_R +/** + * virCgroupNewPartition: + * @path: path for the partition + * @create: true to create the cgroup tree + * @controllers: mask of controllers to create + * + * Creates a new cgroup to represent the resource + * partition path identified by @name. + * + * Returns 0 on success, -errno on failure + */ +int virCgroupNewPartition(const char *path, + bool create, + int controllers, + virCgroupPtr *group) +{ + int rc; + char *parentPath = NULL; + virCgroupPtr parent = NULL; + VIR_DEBUG("path=%s create=%d controllers=%x", + path, create, controllers); + + if (path[0] != '/') + return -EINVAL; + + rc = virCgroupNew(path, NULL, controllers, group); + if (rc != 0) + goto cleanup; + + if (STRNEQ(path, "/")) { + char *tmp; + if (!(parentPath = strdup(path))) { + rc = -ENOMEM; + goto cleanup; + } + + tmp = strrchr(parentPath, '/'); + tmp++; + *tmp = '\0'; + + rc = virCgroupNew(parentPath, NULL, controllers, &parent); + if (rc != 0) + goto cleanup; + + rc = virCgroupMakeGroup(parent, *group, create, VIR_CGROUP_NONE); + if (rc != 0) { + virCgroupRemove(*group); + goto cleanup; + } + } + +cleanup: + if (rc != 0) + virCgroupFree(group); + virCgroupFree(&parent); + VIR_FREE(parentPath); + return rc; +} +#else +int virCgroupNewPartition(const char *path ATTRIBUTE_UNUSED, + bool create ATTRIBUTE_UNUSED, + int controllers ATTRIBUTE_UNUSED, + virCgroupPtr *group ATTRIBUTE_UNUSED) +{ + /* Claim no support */ + return -ENXIO; +} +#endif + /** * virCgroupNewDriver: * @@ -1127,7 +1198,7 @@ int virCgroupNewSelf(virCgroupPtr *group ATTRIBUTE_UNUSED) #endif /** - * virCgroupNewDomain: + * virCgroupNewDomainDriver: * * @driver: group for driver owning the domain * @name: name of the domain @@ -1136,10 +1207,10 @@ int virCgroupNewSelf(virCgroupPtr *group ATTRIBUTE_UNUSED) * Returns 0 on success */ #if defined HAVE_MNTENT_H && defined HAVE_GETMNTENT_R -int virCgroupNewDomain(virCgroupPtr driver, - const char *name, - bool create, - virCgroupPtr *group) +int virCgroupNewDomainDriver(virCgroupPtr driver, + const char *name, + bool create, + virCgroupPtr *group) { int rc; @@ -1166,10 +1237,68 @@ int virCgroupNewDomain(virCgroupPtr driver, return rc; } #else -int virCgroupNewDomain(virCgroupPtr driver ATTRIBUTE_UNUSED, - const char *name ATTRIBUTE_UNUSED, - bool create ATTRIBUTE_UNUSED, - virCgroupPtr *group ATTRIBUTE_UNUSED) +int virCgroupNewDomainDriver(virCgroupPtr driver ATTRIBUTE_UNUSED, + const char *name ATTRIBUTE_UNUSED, + bool create ATTRIBUTE_UNUSED, + virCgroupPtr *group ATTRIBUTE_UNUSED) +{ + return -ENXIO; +} +#endif + +/** + * virCgroupNewDomainPartition: + * + * @partition: partition holding the domain + * @driver: name of the driver + * @name: name of the domain + * @group: Pointer to returned virCgroupPtr + * + * Returns 0 on success + */ +#if defined HAVE_MNTENT_H && defined HAVE_GETMNTENT_R +int virCgroupNewDomainPartition(virCgroupPtr partition, + const char *driver, + const char *name, + bool create, + virCgroupPtr *group) +{ + int rc; + char *dirname = NULL; + + if (virAsprintf(&dirname, "%s.%s.libvirt", + name, driver) < 0) + return -ENOMEM; + + rc = virCgroupNew(dirname, partition, -1, group); + + if (rc == 0) { + /* + * Create a cgroup with memory.use_hierarchy enabled to + * surely account memory usage of lxc with ns subsystem + * enabled. (To be exact, memory and ns subsystems are + * enabled at the same time.) + * + * The reason why doing it here, not a upper group, say + * a group for driver, is to avoid overhead to track + * cumulative usage that we don't need. + */ + rc = virCgroupMakeGroup(partition, *group, create, VIR_CGROUP_MEM_HIERACHY); + if (rc != 0) { + virCgroupRemove(*group); + virCgroupFree(group); + } + } + + VIR_FREE(dirname); + return rc; +} +#else +int virCgroupNewDomainPartition(virCgroupPtr partition ATTRIBUTE_UNUSED, + const char *driver ATTRIBUTE_UNUSED, + const char *name ATTRIBUTE_UNUSED, + bool create ATTRIBUTE_UNUSED, + virCgroupPtr *group ATTRIBUTE_UNUSED) { return -ENXIO; } diff --git a/src/util/vircgroup.h b/src/util/vircgroup.h index 91143e2936..33f86a6103 100644 --- a/src/util/vircgroup.h +++ b/src/util/vircgroup.h @@ -44,6 +44,12 @@ enum { VIR_ENUM_DECL(virCgroupController); +int virCgroupNewPartition(const char *path, + bool create, + int controllers, + virCgroupPtr *group) + ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(4); + int virCgroupNewDriver(const char *name, bool privileged, bool create, @@ -54,10 +60,16 @@ int virCgroupNewDriver(const char *name, int virCgroupNewSelf(virCgroupPtr *group) ATTRIBUTE_NONNULL(1); -int virCgroupNewDomain(virCgroupPtr driver, - const char *name, - bool create, - virCgroupPtr *group) +int virCgroupNewDomainDriver(virCgroupPtr driver, + const char *name, + bool create, + virCgroupPtr *group) + ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) ATTRIBUTE_NONNULL(4); +int virCgroupNewDomainPartition(virCgroupPtr partition, + const char *driver, + const char *name, + bool create, + virCgroupPtr *group) ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) ATTRIBUTE_NONNULL(4); int virCgroupNewVcpu(virCgroupPtr domain, diff --git a/tests/vircgrouptest.c b/tests/vircgrouptest.c index f16a7d25be..c70441bf83 100644 --- a/tests/vircgrouptest.c +++ b/tests/vircgrouptest.c @@ -185,7 +185,7 @@ cleanup: } -static int testCgroupNewForDomain(const void *args ATTRIBUTE_UNUSED) +static int testCgroupNewForDriverDomain(const void *args ATTRIBUTE_UNUSED) { virCgroupPtr drivercgroup = NULL; virCgroupPtr domaincgroup = NULL; @@ -206,7 +206,7 @@ static int testCgroupNewForDomain(const void *args ATTRIBUTE_UNUSED) goto cleanup; } - if ((rv = virCgroupNewDomain(drivercgroup, "wibble", true, &domaincgroup)) != 0) { + if ((rv = virCgroupNewDomainDriver(drivercgroup, "wibble", true, &domaincgroup)) != 0) { fprintf(stderr, "Cannot create LXC cgroup: %d\n", -rv); goto cleanup; } @@ -220,9 +220,156 @@ cleanup: } -# define FAKESYSFSDIRTEMPLATE abs_builddir "/fakesysfsdir-XXXXXX" +static int testCgroupNewForPartition(const void *args ATTRIBUTE_UNUSED) +{ + virCgroupPtr cgroup = NULL; + int ret = -1; + int rv; + const char *placementSmall[VIR_CGROUP_CONTROLLER_LAST] = { + [VIR_CGROUP_CONTROLLER_CPU] = "/virtualmachines", + [VIR_CGROUP_CONTROLLER_CPUACCT] = "/virtualmachines", + [VIR_CGROUP_CONTROLLER_CPUSET] = NULL, + [VIR_CGROUP_CONTROLLER_MEMORY] = "/virtualmachines", + [VIR_CGROUP_CONTROLLER_DEVICES] = NULL, + [VIR_CGROUP_CONTROLLER_FREEZER] = NULL, + [VIR_CGROUP_CONTROLLER_BLKIO] = NULL, + }; + const char *placementFull[VIR_CGROUP_CONTROLLER_LAST] = { + [VIR_CGROUP_CONTROLLER_CPU] = "/virtualmachines", + [VIR_CGROUP_CONTROLLER_CPUACCT] = "/virtualmachines", + [VIR_CGROUP_CONTROLLER_CPUSET] = "/virtualmachines", + [VIR_CGROUP_CONTROLLER_MEMORY] = "/virtualmachines", + [VIR_CGROUP_CONTROLLER_DEVICES] = NULL, + [VIR_CGROUP_CONTROLLER_FREEZER] = "/virtualmachines", + [VIR_CGROUP_CONTROLLER_BLKIO] = "/virtualmachines", + }; + + if ((rv = virCgroupNewPartition("/virtualmachines", false, -1, &cgroup)) != -ENOENT) { + fprintf(stderr, "Unexpected found /virtualmachines cgroup: %d\n", -rv); + goto cleanup; + } + + /* Asking for impossible combination since CPU is co-mounted */ + if ((rv = virCgroupNewPartition("/virtualmachines", true, + (1 << VIR_CGROUP_CONTROLLER_CPU), + &cgroup)) != -EINVAL) { + fprintf(stderr, "Should not have created /virtualmachines cgroup: %d\n", -rv); + goto cleanup; + } + + /* Asking for impossible combination since devices is not mounted */ + if ((rv = virCgroupNewPartition("/virtualmachines", true, + (1 << VIR_CGROUP_CONTROLLER_DEVICES), + &cgroup)) != -ENOENT) { + fprintf(stderr, "Should not have created /virtualmachines cgroup: %d\n", -rv); + goto cleanup; + } + + /* Asking for small combination since devices is not mounted */ + if ((rv = virCgroupNewPartition("/virtualmachines", true, + (1 << VIR_CGROUP_CONTROLLER_CPU) | + (1 << VIR_CGROUP_CONTROLLER_CPUACCT) | + (1 << VIR_CGROUP_CONTROLLER_MEMORY), + &cgroup)) != 0) { + fprintf(stderr, "Cannot create /virtualmachines cgroup: %d\n", -rv); + goto cleanup; + } + ret = validateCgroup(cgroup, "/virtualmachines", mountsSmall, placementSmall); + virCgroupFree(&cgroup); + + if ((rv = virCgroupNewPartition("/virtualmachines", true, -1, &cgroup)) != 0) { + fprintf(stderr, "Cannot create /virtualmachines cgroup: %d\n", -rv); + goto cleanup; + } + ret = validateCgroup(cgroup, "/virtualmachines", mountsFull, placementFull); + +cleanup: + virCgroupFree(&cgroup); + return ret; +} + +static int testCgroupNewForPartitionNested(const void *args ATTRIBUTE_UNUSED) +{ + virCgroupPtr cgroup = NULL; + int ret = -1; + int rv; + const char *placementFull[VIR_CGROUP_CONTROLLER_LAST] = { + [VIR_CGROUP_CONTROLLER_CPU] = "/users/berrange", + [VIR_CGROUP_CONTROLLER_CPUACCT] = "/users/berrange", + [VIR_CGROUP_CONTROLLER_CPUSET] = "/users/berrange", + [VIR_CGROUP_CONTROLLER_MEMORY] = "/users/berrange", + [VIR_CGROUP_CONTROLLER_DEVICES] = NULL, + [VIR_CGROUP_CONTROLLER_FREEZER] = "/users/berrange", + [VIR_CGROUP_CONTROLLER_BLKIO] = "/users/berrange", + }; + if ((rv = virCgroupNewPartition("/users/berrange", false, -1, &cgroup)) != -ENOENT) { + fprintf(stderr, "Unexpected found /users/berrange cgroup: %d\n", -rv); + goto cleanup; + } + + /* Should not work, since we require /users to be pre-created */ + if ((rv = virCgroupNewPartition("/users/berrange", true, -1, &cgroup)) != -ENOENT) { + fprintf(stderr, "Unexpected created /users/berrange cgroup: %d\n", -rv); + goto cleanup; + } + + if ((rv = virCgroupNewPartition("/users", true, -1, &cgroup)) != 0) { + fprintf(stderr, "Failed to create /users cgroup: %d\n", -rv); + goto cleanup; + } + + /* Should now work */ + if ((rv = virCgroupNewPartition("/users/berrange", true, -1, &cgroup)) != 0) { + fprintf(stderr, "Failed to create /users/berrange cgroup: %d\n", -rv); + goto cleanup; + } + + ret = validateCgroup(cgroup, "/users/berrange", mountsFull, placementFull); + +cleanup: + virCgroupFree(&cgroup); + return ret; +} + + + +static int testCgroupNewForPartitionDomain(const void *args ATTRIBUTE_UNUSED) +{ + virCgroupPtr partitioncgroup = NULL; + virCgroupPtr domaincgroup = NULL; + int ret = -1; + int rv; + const char *placement[VIR_CGROUP_CONTROLLER_LAST] = { + [VIR_CGROUP_CONTROLLER_CPU] = "/production/foo.lxc.libvirt", + [VIR_CGROUP_CONTROLLER_CPUACCT] = "/production/foo.lxc.libvirt", + [VIR_CGROUP_CONTROLLER_CPUSET] = "/production/foo.lxc.libvirt", + [VIR_CGROUP_CONTROLLER_MEMORY] = "/production/foo.lxc.libvirt", + [VIR_CGROUP_CONTROLLER_DEVICES] = NULL, + [VIR_CGROUP_CONTROLLER_FREEZER] = "/production/foo.lxc.libvirt", + [VIR_CGROUP_CONTROLLER_BLKIO] = "/production/foo.lxc.libvirt", + }; + + if ((rv = virCgroupNewPartition("/production", true, -1, &partitioncgroup)) != 0) { + fprintf(stderr, "Failed to create /production cgroup: %d\n", -rv); + goto cleanup; + } + + if ((rv = virCgroupNewDomainPartition(partitioncgroup, "lxc", "foo", true, &domaincgroup)) != 0) { + fprintf(stderr, "Cannot create LXC cgroup: %d\n", -rv); + goto cleanup; + } + + ret = validateCgroup(domaincgroup, "/production/foo.lxc.libvirt", mountsFull, placement); + +cleanup: + virCgroupFree(&partitioncgroup); + virCgroupFree(&domaincgroup); + return ret; +} + +# define FAKESYSFSDIRTEMPLATE abs_builddir "/fakesysfsdir-XXXXXX" static int mymain(void) @@ -248,9 +395,19 @@ mymain(void) if (virtTestRun("New cgroup for driver", 1, testCgroupNewForDriver, NULL) < 0) ret = -1; - if (virtTestRun("New cgroup for domain", 1, testCgroupNewForDomain, NULL) < 0) + if (virtTestRun("New cgroup for domain driver", 1, testCgroupNewForDriverDomain, NULL) < 0) + ret = -1; + + if (virtTestRun("New cgroup for partition", 1, testCgroupNewForPartition, NULL) < 0) ret = -1; + if (virtTestRun("New cgroup for partition nested", 1, testCgroupNewForPartitionNested, NULL) < 0) + ret = -1; + + if (virtTestRun("New cgroup for domain partition", 1, testCgroupNewForPartitionDomain, NULL) < 0) + ret = -1; + + if (getenv("LIBVIRT_SKIP_CLEANUP") == NULL) virFileDeleteTree(fakesysfsdir); -- GitLab