Track symlinks for co-mounted cgroup controllers

If a cgroup controller is co-mounted with another, eg

   /sys/fs/cgroup/cpu,cpuacct

Then it is a requirement that there exist symlinks at

   /sys/fs/cgroup/cpu
   /sys/fs/cgroup/cpuacct

pointing to the real mount point. Add support to virCgroupPtr
to detect and track these symlinks
Signed-off-by: NDaniel P. Berrange <berrange@redhat.com>

src/util/vircgroup.c

@@ -75,6 +75,7 @@ void virCgroupFree(virCgroupPtr *group)
 
     for (i = 0 ; i < VIR_CGROUP_CONTROLLER_LAST ; i++) {
         VIR_FREE((*group)->controllers[i].mountPoint);
+        VIR_FREE((*group)->controllers[i].linkPoint);
         VIR_FREE((*group)->controllers[i].placement);
     }
 
@@ -114,6 +115,14 @@ static int virCgroupCopyMounts(virCgroupPtr group,
 
         if (!group->controllers[i].mountPoint)
             return -ENOMEM;
+
+        if (parent->controllers[i].linkPoint) {
+            group->controllers[i].linkPoint =
+                strdup(parent->controllers[i].linkPoint);
+
+            if (!group->controllers[i].linkPoint)
+                return -ENOMEM;
+        }
     }
     return 0;
 }
@@ -157,9 +166,46 @@ static int virCgroupDetectMounts(virCgroupPtr group)
                  * first entry only
                  */
                 if (typelen == len && STREQLEN(typestr, tmp, len) &&
-                    !group->controllers[i].mountPoint &&
-                    !(group->controllers[i].mountPoint = strdup(entry.mnt_dir)))
-                    goto no_memory;
+                    !group->controllers[i].mountPoint) {
+                    char *linksrc;
+                    struct stat sb;
+                    char *tmp2;
+
+                    if (!(group->controllers[i].mountPoint = strdup(entry.mnt_dir)))
+                        goto no_memory;
+
+                    tmp2 = strrchr(entry.mnt_dir, '/');
+                    if (!tmp2) {
+                        errno = EINVAL;
+                        goto error;
+                    }
+                    *tmp2 = '\0';
+                    /* If it is a co-mount it has a filename like "cpu,cpuacct"
+                     * and we must identify the symlink path */
+                    if (strchr(tmp2 + 1, ',')) {
+                        if (virAsprintf(&linksrc, "%s/%s",
+                                        entry.mnt_dir, typestr) < 0)
+                            goto no_memory;
+                        *tmp2 = '/';
+
+                        if (lstat(linksrc, &sb) < 0) {
+                            if (errno == ENOENT) {
+                                VIR_WARN("Controller %s co-mounted at %s is missing symlink at %s",
+                                         typestr, entry.mnt_dir, linksrc);
+                                VIR_FREE(linksrc);
+                            } else {
+                                goto error;
+                            }
+                        } else {
+                            if (!S_ISLNK(sb.st_mode)) {
+                                VIR_WARN("Expecting a symlink at %s for controller %s",
+                                         linksrc, typestr);
+                            } else {
+                                group->controllers[i].linkPoint = linksrc;
+                            }
+                        }
+                    }
+                }
                 tmp = next;
             }
         }
@@ -170,8 +216,10 @@ static int virCgroupDetectMounts(virCgroupPtr group)
     return 0;
 
 no_memory:
+    errno = ENOMEM;
+error:
     VIR_FORCE_FCLOSE(mounts);
-    return -ENOMEM;
+    return -errno;
 }
 
 

src/util/vircgrouppriv.h

@@ -34,6 +34,11 @@
 struct virCgroupController {
     int type;
     char *mountPoint;
+    /* If mountPoint holds several controllers co-mounted,
+     * then linkPoint is path of the symlink to the mountPoint
+     * for just the one controller
+     */
+    char *linkPoint;
     char *placement;
 };
 

tests/vircgroupmock.c

@@ -34,6 +34,8 @@
 static int (*realopen)(const char *path, int flags, ...);
 static FILE *(*realfopen)(const char *path, const char *mode);
 static int (*realaccess)(const char *path, int mode);
+static int (*reallstat)(const char *path, struct stat *sb);
+static int (*real__lxstat)(int ver, const char *path, struct stat *sb);
 static int (*realmkdir)(const char *path, mode_t mode);
 static char *fakesysfsdir;
 
@@ -310,8 +312,18 @@ static void init_syms(void)
         }                                                               \
     } while (0)
 
+# define LOAD_SYM_ALT(name1, name2)                                     \
+    do {                                                                \
+        if (!(real ## name1 = dlsym(RTLD_NEXT, #name1)) &&              \
+            !(real ## name2 = dlsym(RTLD_NEXT, #name2))) {              \
+            fprintf(stderr, "Cannot find real '%s' or '%s' symbol\n", #name1, #name2); \
+            abort();                                                    \
+        }                                                               \
+    } while (0)
+
     LOAD_SYM(fopen);
     LOAD_SYM(access);
+    LOAD_SYM_ALT(lstat, __lxstat);
     LOAD_SYM(mkdir);
     LOAD_SYM(open);
 }
@@ -395,6 +407,52 @@ int access(const char *path, int mode)
     return ret;
 }
 
+int __lxstat(int ver, const char *path, struct stat *sb)
+{
+    int ret;
+
+    init_syms();
+
+    if (STRPREFIX(path, SYSFS_PREFIX)) {
+        init_sysfs();
+        char *newpath;
+        if (asprintf(&newpath, "%s/%s",
+                     fakesysfsdir,
+                     path + strlen(SYSFS_PREFIX)) < 0) {
+            errno = ENOMEM;
+            return -1;
+        }
+        ret = real__lxstat(ver, newpath, sb);
+        free(newpath);
+    } else {
+        ret = real__lxstat(ver, path, sb);
+    }
+    return ret;
+}
+
+int lstat(const char *path, struct stat *sb)
+{
+    int ret;
+
+    init_syms();
+
+    if (STRPREFIX(path, SYSFS_PREFIX)) {
+        init_sysfs();
+        char *newpath;
+        if (asprintf(&newpath, "%s/%s",
+                     fakesysfsdir,
+                     path + strlen(SYSFS_PREFIX)) < 0) {
+            errno = ENOMEM;
+            return -1;
+        }
+        ret = reallstat(newpath, sb);
+        free(newpath);
+    } else {
+        ret = reallstat(path, sb);
+    }
+    return ret;
+}
+
 int mkdir(const char *path, mode_t mode)
 {
     int ret;

tests/vircgrouptest.c

@@ -38,6 +38,7 @@
 static int validateCgroup(virCgroupPtr cgroup,
                           const char *expectPath,
                           const char **expectMountPoint,
+                          const char **expectLinkPoint,
                           const char **expectPlacement)
 {
     int i;
@@ -57,6 +58,14 @@ static int validateCgroup(virCgroupPtr cgroup,
                     virCgroupControllerTypeToString(i));
             return -1;
         }
+        if (STRNEQ_NULLABLE(expectLinkPoint[i],
+                            cgroup->controllers[i].linkPoint)) {
+            fprintf(stderr, "Wrong link '%s', expected '%s' for '%s'\n",
+                    cgroup->controllers[i].linkPoint,
+                    expectLinkPoint[i],
+                    virCgroupControllerTypeToString(i));
+            return -1;
+        }
         if (STRNEQ_NULLABLE(expectPlacement[i],
                             cgroup->controllers[i].placement)) {
             fprintf(stderr, "Wrong placement '%s', expected '%s' for '%s'\n",
@@ -89,6 +98,17 @@ const char *mountsFull[VIR_CGROUP_CONTROLLER_LAST] = {
     [VIR_CGROUP_CONTROLLER_BLKIO] = "/not/really/sys/fs/cgroup/blkio",
 };
 
+const char *links[VIR_CGROUP_CONTROLLER_LAST] = {
+    [VIR_CGROUP_CONTROLLER_CPU] = "/not/really/sys/fs/cgroup/cpu",
+    [VIR_CGROUP_CONTROLLER_CPUACCT] = "/not/really/sys/fs/cgroup/cpuacct",
+    [VIR_CGROUP_CONTROLLER_CPUSET] = NULL,
+    [VIR_CGROUP_CONTROLLER_MEMORY] = NULL,
+    [VIR_CGROUP_CONTROLLER_DEVICES] = NULL,
+    [VIR_CGROUP_CONTROLLER_FREEZER] = NULL,
+    [VIR_CGROUP_CONTROLLER_BLKIO] = NULL,
+};
+
+
 static int testCgroupNewForSelf(const void *args ATTRIBUTE_UNUSED)
 {
     virCgroupPtr cgroup = NULL;
@@ -108,7 +128,7 @@ static int testCgroupNewForSelf(const void *args ATTRIBUTE_UNUSED)
         goto cleanup;
     }
 
-    ret = validateCgroup(cgroup, "", mountsFull, placement);
+    ret = validateCgroup(cgroup, "", mountsFull, links, placement);
 
 cleanup:
     virCgroupFree(&cgroup);
@@ -170,14 +190,14 @@ static int testCgroupNewForDriver(const void *args ATTRIBUTE_UNUSED)
         fprintf(stderr, "Cannot create LXC cgroup: %d\n", -rv);
         goto cleanup;
     }
-    ret = validateCgroup(cgroup, "libvirt/lxc", mountsSmall, placementSmall);
+    ret = validateCgroup(cgroup, "libvirt/lxc", mountsSmall, links, placementSmall);
     virCgroupFree(&cgroup);
 
     if ((rv = virCgroupNewDriver("lxc", true, -1, &cgroup)) != 0) {
         fprintf(stderr, "Cannot create LXC cgroup: %d\n", -rv);
         goto cleanup;
     }
-    ret = validateCgroup(cgroup, "libvirt/lxc", mountsFull, placementFull);
+    ret = validateCgroup(cgroup, "libvirt/lxc", mountsFull, links, placementFull);
 
 cleanup:
     virCgroupFree(&cgroup);
@@ -211,7 +231,7 @@ static int testCgroupNewForDriverDomain(const void *args ATTRIBUTE_UNUSED)
         goto cleanup;
     }
 
-    ret = validateCgroup(domaincgroup, "libvirt/lxc/wibble", mountsFull, placement);
+    ret = validateCgroup(domaincgroup, "libvirt/lxc/wibble", mountsFull, links, placement);
 
 cleanup:
     virCgroupFree(&drivercgroup);
@@ -274,14 +294,14 @@ static int testCgroupNewForPartition(const void *args ATTRIBUTE_UNUSED)
         fprintf(stderr, "Cannot create /virtualmachines cgroup: %d\n", -rv);
         goto cleanup;
     }
-    ret = validateCgroup(cgroup, "/virtualmachines", mountsSmall, placementSmall);
+    ret = validateCgroup(cgroup, "/virtualmachines", mountsSmall, links, placementSmall);
     virCgroupFree(&cgroup);
 
     if ((rv = virCgroupNewPartition("/virtualmachines", true, -1, &cgroup)) != 0) {
         fprintf(stderr, "Cannot create /virtualmachines cgroup: %d\n", -rv);
         goto cleanup;
     }
-    ret = validateCgroup(cgroup, "/virtualmachines", mountsFull, placementFull);
+    ret = validateCgroup(cgroup, "/virtualmachines", mountsFull, links, placementFull);
 
 cleanup:
     virCgroupFree(&cgroup);
@@ -326,7 +346,7 @@ static int testCgroupNewForPartitionNested(const void *args ATTRIBUTE_UNUSED)
         goto cleanup;
     }
 
-    ret = validateCgroup(cgroup, "/users/berrange", mountsFull, placementFull);
+    ret = validateCgroup(cgroup, "/users/berrange", mountsFull, links, placementFull);
 
 cleanup:
     virCgroupFree(&cgroup);
@@ -361,7 +381,7 @@ static int testCgroupNewForPartitionDomain(const void *args ATTRIBUTE_UNUSED)
         goto cleanup;
     }
 
-    ret = validateCgroup(domaincgroup, "/production/foo.lxc.libvirt", mountsFull, placement);
+    ret = validateCgroup(domaincgroup, "/production/foo.lxc.libvirt", mountsFull, links, placement);
 
 cleanup:
     virCgroupFree(&partitioncgroup);