Add a new virCgroupNewPartition for setting up resource partitions

A resource partition is an absolute cgroup path, ignoring the
current process placement. Expose a virCgroupNewPartition API
for constructing such cgroups
Signed-off-by: NDaniel P. Berrange <berrange@redhat.com>

src/libvirt_private.syms

@@ -1122,9 +1122,11 @@ virCgroupKill;
 virCgroupKillPainfully;
 virCgroupKillRecursive;
 virCgroupMoveTask;
-virCgroupNewDomain;
+virCgroupNewDomainDriver;
+virCgroupNewDomainPartition;
 virCgroupNewDriver;
 virCgroupNewEmulator;
+virCgroupNewPartition;
 virCgroupNewSelf;
 virCgroupNewVcpu;
 virCgroupPathOfController;

src/lxc/lxc_cgroup.c

@@ -536,7 +536,7 @@ virCgroupPtr virLXCCgroupCreate(virDomainDefPtr def)
         goto cleanup;
     }
 
-    rc = virCgroupNewDomain(driver, def->name, true, &cgroup);
+    rc = virCgroupNewDomainDriver(driver, def->name, true, &cgroup);
     if (rc != 0) {
         virReportSystemError(-rc,
                              _("Unable to create cgroup for domain %s"),

src/qemu/qemu_cgroup.c

@@ -243,7 +243,7 @@ int qemuInitCgroup(virQEMUDriverPtr driver,
         goto cleanup;
     }
 
-    rc = virCgroupNewDomain(driverGroup, vm->def->name, true, &priv->cgroup);
+    rc = virCgroupNewDomainDriver(driverGroup, vm->def->name, true, &priv->cgroup);
     if (rc != 0) {
         virReportSystemError(-rc,
                              _("Unable to create cgroup for %s"),

src/util/vircgroup.c

@@ -1056,6 +1056,77 @@ cleanup:
     return rc;
 }
 
+
+#if defined HAVE_MNTENT_H && defined HAVE_GETMNTENT_R
+/**
+ * virCgroupNewPartition:
+ * @path: path for the partition
+ * @create: true to create the cgroup tree
+ * @controllers: mask of controllers to create
+ *
+ * Creates a new cgroup to represent the resource
+ * partition path identified by @name.
+ *
+ * Returns 0 on success, -errno on failure
+ */
+int virCgroupNewPartition(const char *path,
+                          bool create,
+                          int controllers,
+                          virCgroupPtr *group)
+{
+    int rc;
+    char *parentPath = NULL;
+    virCgroupPtr parent = NULL;
+    VIR_DEBUG("path=%s create=%d controllers=%x",
+              path, create, controllers);
+
+    if (path[0] != '/')
+        return -EINVAL;
+
+    rc = virCgroupNew(path, NULL, controllers, group);
+    if (rc != 0)
+        goto cleanup;
+
+    if (STRNEQ(path, "/")) {
+        char *tmp;
+        if (!(parentPath = strdup(path))) {
+            rc = -ENOMEM;
+            goto cleanup;
+        }
+
+        tmp = strrchr(parentPath, '/');
+        tmp++;
+        *tmp = '\0';
+
+        rc = virCgroupNew(parentPath, NULL, controllers, &parent);
+        if (rc != 0)
+            goto cleanup;
+
+        rc = virCgroupMakeGroup(parent, *group, create, VIR_CGROUP_NONE);
+        if (rc != 0) {
+            virCgroupRemove(*group);
+            goto cleanup;
+        }
+    }
+
+cleanup:
+    if (rc != 0)
+        virCgroupFree(group);
+    virCgroupFree(&parent);
+    VIR_FREE(parentPath);
+    return rc;
+}
+#else
+int virCgroupNewPartition(const char *path ATTRIBUTE_UNUSED,
+                          bool create ATTRIBUTE_UNUSED,
+                          int controllers ATTRIBUTE_UNUSED,
+                          virCgroupPtr *group ATTRIBUTE_UNUSED)
+{
+    /* Claim no support */
+    return -ENXIO;
+}
+#endif
+
 /**
  * virCgroupNewDriver:
  *
@@ -1127,7 +1198,7 @@ int virCgroupNewSelf(virCgroupPtr *group ATTRIBUTE_UNUSED)
 #endif
 
 /**
- * virCgroupNewDomain:
+ * virCgroupNewDomainDriver:
  *
  * @driver: group for driver owning the domain
  * @name: name of the domain
@@ -1136,10 +1207,10 @@ int virCgroupNewSelf(virCgroupPtr *group ATTRIBUTE_UNUSED)
  * Returns 0 on success
  */
 #if defined HAVE_MNTENT_H && defined HAVE_GETMNTENT_R
-int virCgroupNewDomain(virCgroupPtr driver,
-                       const char *name,
-                       bool create,
-                       virCgroupPtr *group)
+int virCgroupNewDomainDriver(virCgroupPtr driver,
+                             const char *name,
+                             bool create,
+                             virCgroupPtr *group)
 {
     int rc;
 
@@ -1166,10 +1237,68 @@ int virCgroupNewDomain(virCgroupPtr driver,
     return rc;
 }
 #else
-int virCgroupNewDomain(virCgroupPtr driver ATTRIBUTE_UNUSED,
-                       const char *name ATTRIBUTE_UNUSED,
-                       bool create ATTRIBUTE_UNUSED,
-                       virCgroupPtr *group ATTRIBUTE_UNUSED)
+int virCgroupNewDomainDriver(virCgroupPtr driver ATTRIBUTE_UNUSED,
+                             const char *name ATTRIBUTE_UNUSED,
+                             bool create ATTRIBUTE_UNUSED,
+                             virCgroupPtr *group ATTRIBUTE_UNUSED)
+{
+    return -ENXIO;
+}
+#endif
+
+/**
+ * virCgroupNewDomainPartition:
+ *
+ * @partition: partition holding the domain
+ * @driver: name of the driver
+ * @name: name of the domain
+ * @group: Pointer to returned virCgroupPtr
+ *
+ * Returns 0 on success
+ */
+#if defined HAVE_MNTENT_H && defined HAVE_GETMNTENT_R
+int virCgroupNewDomainPartition(virCgroupPtr partition,
+                                const char *driver,
+                                const char *name,
+                                bool create,
+                                virCgroupPtr *group)
+{
+    int rc;
+    char *dirname = NULL;
+
+    if (virAsprintf(&dirname, "%s.%s.libvirt",
+                    name, driver) < 0)
+        return -ENOMEM;
+
+    rc = virCgroupNew(dirname, partition, -1, group);
+
+    if (rc == 0) {
+        /*
+         * Create a cgroup with memory.use_hierarchy enabled to
+         * surely account memory usage of lxc with ns subsystem
+         * enabled. (To be exact, memory and ns subsystems are
+         * enabled at the same time.)
+         *
+         * The reason why doing it here, not a upper group, say
+         * a group for driver, is to avoid overhead to track
+         * cumulative usage that we don't need.
+         */
+        rc = virCgroupMakeGroup(partition, *group, create, VIR_CGROUP_MEM_HIERACHY);
+        if (rc != 0) {
+            virCgroupRemove(*group);
+            virCgroupFree(group);
+        }
+    }
+
+    VIR_FREE(dirname);
+    return rc;
+}
+#else
+int virCgroupNewDomainPartition(virCgroupPtr partition ATTRIBUTE_UNUSED,
+                                const char *driver ATTRIBUTE_UNUSED,
+                                const char *name ATTRIBUTE_UNUSED,
+                                bool create ATTRIBUTE_UNUSED,
+                                virCgroupPtr *group ATTRIBUTE_UNUSED)
 {
     return -ENXIO;
 }

src/util/vircgroup.h

@@ -44,6 +44,12 @@ enum {
 
 VIR_ENUM_DECL(virCgroupController);
 
+int virCgroupNewPartition(const char *path,
+                          bool create,
+                          int controllers,
+                          virCgroupPtr *group)
+    ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(4);
+
 int virCgroupNewDriver(const char *name,
                        bool privileged,
                        bool create,
@@ -54,10 +60,16 @@ int virCgroupNewDriver(const char *name,
 int virCgroupNewSelf(virCgroupPtr *group)
     ATTRIBUTE_NONNULL(1);
 
-int virCgroupNewDomain(virCgroupPtr driver,
-                       const char *name,
-                       bool create,
-                       virCgroupPtr *group)
+int virCgroupNewDomainDriver(virCgroupPtr driver,
+                             const char *name,
+                             bool create,
+                             virCgroupPtr *group)
+    ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) ATTRIBUTE_NONNULL(4);
+int virCgroupNewDomainPartition(virCgroupPtr partition,
+                                const char *driver,
+                                const char *name,
+                                bool create,
+                                virCgroupPtr *group)
     ATTRIBUTE_NONNULL(1) ATTRIBUTE_NONNULL(2) ATTRIBUTE_NONNULL(4);
 
 int virCgroupNewVcpu(virCgroupPtr domain,

tests/vircgrouptest.c

@@ -185,7 +185,7 @@ cleanup:
 }
 
 
-static int testCgroupNewForDomain(const void *args ATTRIBUTE_UNUSED)
+static int testCgroupNewForDriverDomain(const void *args ATTRIBUTE_UNUSED)
 {
     virCgroupPtr drivercgroup = NULL;
     virCgroupPtr domaincgroup = NULL;
@@ -206,7 +206,7 @@ static int testCgroupNewForDomain(const void *args ATTRIBUTE_UNUSED)
         goto cleanup;
     }
 
-    if ((rv = virCgroupNewDomain(drivercgroup, "wibble", true, &domaincgroup)) != 0) {
+    if ((rv = virCgroupNewDomainDriver(drivercgroup, "wibble", true, &domaincgroup)) != 0) {
         fprintf(stderr, "Cannot create LXC cgroup: %d\n", -rv);
         goto cleanup;
     }
@@ -220,9 +220,156 @@ cleanup:
 }
 
 
-# define FAKESYSFSDIRTEMPLATE abs_builddir "/fakesysfsdir-XXXXXX"
+static int testCgroupNewForPartition(const void *args ATTRIBUTE_UNUSED)
+{
+    virCgroupPtr cgroup = NULL;
+    int ret = -1;
+    int rv;
+    const char *placementSmall[VIR_CGROUP_CONTROLLER_LAST] = {
+        [VIR_CGROUP_CONTROLLER_CPU] = "/virtualmachines",
+        [VIR_CGROUP_CONTROLLER_CPUACCT] = "/virtualmachines",
+        [VIR_CGROUP_CONTROLLER_CPUSET] = NULL,
+        [VIR_CGROUP_CONTROLLER_MEMORY] = "/virtualmachines",
+        [VIR_CGROUP_CONTROLLER_DEVICES] = NULL,
+        [VIR_CGROUP_CONTROLLER_FREEZER] = NULL,
+        [VIR_CGROUP_CONTROLLER_BLKIO] = NULL,
+    };
+    const char *placementFull[VIR_CGROUP_CONTROLLER_LAST] = {
+        [VIR_CGROUP_CONTROLLER_CPU] = "/virtualmachines",
+        [VIR_CGROUP_CONTROLLER_CPUACCT] = "/virtualmachines",
+        [VIR_CGROUP_CONTROLLER_CPUSET] = "/virtualmachines",
+        [VIR_CGROUP_CONTROLLER_MEMORY] = "/virtualmachines",
+        [VIR_CGROUP_CONTROLLER_DEVICES] = NULL,
+        [VIR_CGROUP_CONTROLLER_FREEZER] = "/virtualmachines",
+        [VIR_CGROUP_CONTROLLER_BLKIO] = "/virtualmachines",
+    };
+
+    if ((rv = virCgroupNewPartition("/virtualmachines", false, -1, &cgroup)) != -ENOENT) {
+        fprintf(stderr, "Unexpected found /virtualmachines cgroup: %d\n", -rv);
+        goto cleanup;
+    }
+
+    /* Asking for impossible combination since CPU is co-mounted */
+    if ((rv = virCgroupNewPartition("/virtualmachines", true,
+                                    (1 << VIR_CGROUP_CONTROLLER_CPU),
+                                    &cgroup)) != -EINVAL) {
+        fprintf(stderr, "Should not have created /virtualmachines cgroup: %d\n", -rv);
+        goto cleanup;
+    }
+
+    /* Asking for impossible combination since devices is not mounted */
+    if ((rv = virCgroupNewPartition("/virtualmachines", true,
+                                    (1 << VIR_CGROUP_CONTROLLER_DEVICES),
+                                    &cgroup)) != -ENOENT) {
+        fprintf(stderr, "Should not have created /virtualmachines cgroup: %d\n", -rv);
+        goto cleanup;
+    }
+
+    /* Asking for small combination since devices is not mounted */
+    if ((rv = virCgroupNewPartition("/virtualmachines", true,
+                                    (1 << VIR_CGROUP_CONTROLLER_CPU) |
+                                    (1 << VIR_CGROUP_CONTROLLER_CPUACCT) |
+                                    (1 << VIR_CGROUP_CONTROLLER_MEMORY),
+                                    &cgroup)) != 0) {
+        fprintf(stderr, "Cannot create /virtualmachines cgroup: %d\n", -rv);
+        goto cleanup;
+    }
+    ret = validateCgroup(cgroup, "/virtualmachines", mountsSmall, placementSmall);
+    virCgroupFree(&cgroup);
+
+    if ((rv = virCgroupNewPartition("/virtualmachines", true, -1, &cgroup)) != 0) {
+        fprintf(stderr, "Cannot create /virtualmachines cgroup: %d\n", -rv);
+        goto cleanup;
+    }
+    ret = validateCgroup(cgroup, "/virtualmachines", mountsFull, placementFull);
+
+cleanup:
+    virCgroupFree(&cgroup);
+    return ret;
+}
+
 
+static int testCgroupNewForPartitionNested(const void *args ATTRIBUTE_UNUSED)
+{
+    virCgroupPtr cgroup = NULL;
+    int ret = -1;
+    int rv;
+    const char *placementFull[VIR_CGROUP_CONTROLLER_LAST] = {
+        [VIR_CGROUP_CONTROLLER_CPU] = "/users/berrange",
+        [VIR_CGROUP_CONTROLLER_CPUACCT] = "/users/berrange",
+        [VIR_CGROUP_CONTROLLER_CPUSET] = "/users/berrange",
+        [VIR_CGROUP_CONTROLLER_MEMORY] = "/users/berrange",
+        [VIR_CGROUP_CONTROLLER_DEVICES] = NULL,
+        [VIR_CGROUP_CONTROLLER_FREEZER] = "/users/berrange",
+        [VIR_CGROUP_CONTROLLER_BLKIO] = "/users/berrange",
+    };
 
+    if ((rv = virCgroupNewPartition("/users/berrange", false, -1, &cgroup)) != -ENOENT) {
+        fprintf(stderr, "Unexpected found /users/berrange cgroup: %d\n", -rv);
+        goto cleanup;
+    }
+
+    /* Should not work, since we require /users to be pre-created */
+    if ((rv = virCgroupNewPartition("/users/berrange", true, -1, &cgroup)) != -ENOENT) {
+        fprintf(stderr, "Unexpected created /users/berrange cgroup: %d\n", -rv);
+        goto cleanup;
+    }
+
+    if ((rv = virCgroupNewPartition("/users", true, -1, &cgroup)) != 0) {
+        fprintf(stderr, "Failed to create /users cgroup: %d\n", -rv);
+        goto cleanup;
+    }
+
+    /* Should now work */
+    if ((rv = virCgroupNewPartition("/users/berrange", true, -1, &cgroup)) != 0) {
+        fprintf(stderr, "Failed to create /users/berrange cgroup: %d\n", -rv);
+        goto cleanup;
+    }
+
+    ret = validateCgroup(cgroup, "/users/berrange", mountsFull, placementFull);
+
+cleanup:
+    virCgroupFree(&cgroup);
+    return ret;
+}
+
+
+
+static int testCgroupNewForPartitionDomain(const void *args ATTRIBUTE_UNUSED)
+{
+    virCgroupPtr partitioncgroup = NULL;
+    virCgroupPtr domaincgroup = NULL;
+    int ret = -1;
+    int rv;
+    const char *placement[VIR_CGROUP_CONTROLLER_LAST] = {
+        [VIR_CGROUP_CONTROLLER_CPU] = "/production/foo.lxc.libvirt",
+        [VIR_CGROUP_CONTROLLER_CPUACCT] = "/production/foo.lxc.libvirt",
+        [VIR_CGROUP_CONTROLLER_CPUSET] = "/production/foo.lxc.libvirt",
+        [VIR_CGROUP_CONTROLLER_MEMORY] = "/production/foo.lxc.libvirt",
+        [VIR_CGROUP_CONTROLLER_DEVICES] = NULL,
+        [VIR_CGROUP_CONTROLLER_FREEZER] = "/production/foo.lxc.libvirt",
+        [VIR_CGROUP_CONTROLLER_BLKIO] = "/production/foo.lxc.libvirt",
+    };
+
+    if ((rv = virCgroupNewPartition("/production", true, -1, &partitioncgroup)) != 0) {
+        fprintf(stderr, "Failed to create /production cgroup: %d\n", -rv);
+        goto cleanup;
+    }
+
+    if ((rv = virCgroupNewDomainPartition(partitioncgroup, "lxc", "foo", true, &domaincgroup)) != 0) {
+        fprintf(stderr, "Cannot create LXC cgroup: %d\n", -rv);
+        goto cleanup;
+    }
+
+    ret = validateCgroup(domaincgroup, "/production/foo.lxc.libvirt", mountsFull, placement);
+
+cleanup:
+    virCgroupFree(&partitioncgroup);
+    virCgroupFree(&domaincgroup);
+    return ret;
+}
+
+# define FAKESYSFSDIRTEMPLATE abs_builddir "/fakesysfsdir-XXXXXX"
 
 static int
 mymain(void)
@@ -248,9 +395,19 @@ mymain(void)
     if (virtTestRun("New cgroup for driver", 1, testCgroupNewForDriver, NULL) < 0)
         ret = -1;
 
-    if (virtTestRun("New cgroup for domain", 1, testCgroupNewForDomain, NULL) < 0)
+    if (virtTestRun("New cgroup for domain driver", 1, testCgroupNewForDriverDomain, NULL) < 0)
+        ret = -1;
+
+    if (virtTestRun("New cgroup for partition", 1, testCgroupNewForPartition, NULL) < 0)
         ret = -1;
 
+    if (virtTestRun("New cgroup for partition nested", 1, testCgroupNewForPartitionNested, NULL) < 0)
+        ret = -1;
+
+    if (virtTestRun("New cgroup for domain partition", 1, testCgroupNewForPartitionDomain, NULL) < 0)
+        ret = -1;
+
+
     if (getenv("LIBVIRT_SKIP_CLEANUP") == NULL)
         virFileDeleteTree(fakesysfsdir);