LXC: Wire up the virDomainCreate{XML}WithFiles methods

Wire up the new virDomainCreate{XML}WithFiles methods in the
LXC driver, so that FDs get passed down to the init process.

The lxc_container code needs to do a little dance in order
to renumber the file descriptors it receives into linear
order, starting from STDERR_FILENO + 1.
Signed-off-by: NDaniel P. Berrange <berrange@redhat.com>

src/lxc/lxc_container.c

@@ -103,8 +103,10 @@ struct __lxc_child_argv {
     size_t nveths;
     char **veths;
     int monitor;
-    char **ttyPaths;
+    size_t npassFDs;
+    int *passFDs;
     size_t nttyPaths;
+    char **ttyPaths;
     int handshakefd;
 };
 
@@ -217,20 +219,28 @@ static virCommandPtr lxcContainerBuildInitCmd(virDomainDefPtr vmDef)
 }
 
 /**
- * lxcContainerSetStdio:
+ * lxcContainerSetupFDs:
  * @control: control FD from parent
  * @ttyfd: FD of tty to set as the container console
+ * @npassFDs: number of extra FDs
+ * @passFDs: list of extra FDs
  *
- * Sets the given tty as the primary conosole for the container as well as
- * stdout, stdin and stderr.
+ * Setup file descriptors in the container. @ttyfd is set to be
+ * the container's stdin, stdout & stderr. Any FDs included in
+ * @passFDs, will be dup()'d such that they start from stderr+1
+ * with no gaps.
  *
  * Returns 0 on success or -1 in case of error
  */
-static int lxcContainerSetStdio(int control, int ttyfd, int handshakefd)
+static int lxcContainerSetupFDs(int *ttyfd,
+                                size_t npassFDs, int *passFDs)
 {
     int rc = -1;
     int open_max;
     int fd;
+    int last_fd;
+    size_t i;
+    size_t j;
 
     if (setsid() < 0) {
         virReportSystemError(errno, "%s",
@@ -238,44 +248,99 @@ static int lxcContainerSetStdio(int control, int ttyfd, int handshakefd)
         goto cleanup;
     }
 
-    if (ioctl(ttyfd, TIOCSCTTY, NULL) < 0) {
+    if (ioctl(*ttyfd, TIOCSCTTY, NULL) < 0) {
         virReportSystemError(errno, "%s",
                              _("ioctl(TIOCSTTY) failed"));
         goto cleanup;
     }
 
-    /* Just in case someone forget to set FD_CLOEXEC, explicitly
-     * close all FDs before executing the container */
-    open_max = sysconf(_SC_OPEN_MAX);
-    if (open_max < 0) {
+    if (dup2(*ttyfd, STDIN_FILENO) < 0) {
         virReportSystemError(errno, "%s",
-                             _("sysconf(_SC_OPEN_MAX) failed"));
+                             _("dup2(stdin) failed"));
         goto cleanup;
     }
-    for (fd = 0; fd < open_max; fd++)
-        if (fd != ttyfd && fd != control && fd != handshakefd) {
-            int tmpfd = fd;
-            VIR_MASS_CLOSE(tmpfd);
-        }
 
-    if (dup2(ttyfd, 0) < 0) {
+    if (dup2(*ttyfd, STDOUT_FILENO) < 0) {
         virReportSystemError(errno, "%s",
-                             _("dup2(stdin) failed"));
+                             _("dup2(stdout) failed"));
         goto cleanup;
     }
 
-    if (dup2(ttyfd, 1) < 0) {
+    if (dup2(*ttyfd, STDERR_FILENO) < 0) {
         virReportSystemError(errno, "%s",
-                             _("dup2(stdout) failed"));
+                             _("dup2(stderr) failed"));
         goto cleanup;
     }
 
-    if (dup2(ttyfd, 2) < 0) {
+    VIR_FORCE_CLOSE(*ttyfd);
+
+    /* Any FDs in @passFDs need to be moved around so that
+     * they are numbered, without gaps, starting from
+     * STDERR_FILENO + 1
+     */
+    for (i = 0; i < npassFDs; i++) {
+        int wantfd;
+
+        wantfd = STDERR_FILENO + i + 1;
+        VIR_DEBUG("Pass %d onto %d", passFDs[i], wantfd);
+
+        /* If we already have desired FD number, life
+         * is easy. Nothing needs renumbering */
+        if (passFDs[i] == wantfd)
+            continue;
+
+        /*
+         * Lets check to see if any later FDs are occupying
+         * our desired FD number. If so, we must move them
+         * out of the way
+         */
+        for (j = i + 1; j < npassFDs; j++) {
+            if (passFDs[j] == wantfd) {
+                VIR_DEBUG("Clash %zu", j);
+                int newfd = dup(passFDs[j]);
+                if (newfd < 0) {
+                    virReportSystemError(errno,
+                                         _("Cannot move fd %d out of the way"),
+                                         passFDs[j]);
+                    goto cleanup;
+                }
+                /* We're intentionally not closing the
+                 * old value of passFDs[j], because we
+                 * don't want later iterations of the
+                 * loop to take it back. dup2() will
+                 * cause it to be closed shortly anyway
+                 */
+                VIR_DEBUG("Moved clash onto %d", newfd);
+                passFDs[j] = newfd;
+            }
+        }
+
+        /* Finally we can move into our desired FD number */
+        if (dup2(passFDs[i], wantfd) < 0) {
+            virReportSystemError(errno,
+                                 _("Cannot duplicate fd %d onto fd %d"),
+                                 passFDs[i], wantfd);
+            goto cleanup;
+        }
+        VIR_FORCE_CLOSE(passFDs[i]);
+    }
+
+    last_fd = STDERR_FILENO + npassFDs;
+
+    /* Just in case someone forget to set FD_CLOEXEC, explicitly
+     * close all remaining FDs before executing the container */
+    open_max = sysconf(_SC_OPEN_MAX);
+    if (open_max < 0) {
         virReportSystemError(errno, "%s",
-                             _("dup2(stderr) failed"));
+                             _("sysconf(_SC_OPEN_MAX) failed"));
         goto cleanup;
     }
 
+    for (fd = last_fd + 1; fd < open_max; fd++) {
+        int tmpfd = fd;
+        VIR_MASS_CLOSE(tmpfd);
+    }
+
     rc = 0;
 
 cleanup:
@@ -1677,9 +1742,11 @@ static int lxcContainerChild(void *data)
     if (virSecurityManagerSetProcessLabel(argv->securityDriver, vmDef) < 0)
         goto cleanup;
 
-    if (lxcContainerSetStdio(argv->monitor, ttyfd, argv->handshakefd) < 0) {
+    VIR_FORCE_CLOSE(argv->handshakefd);
+    VIR_FORCE_CLOSE(argv->monitor);
+    if (lxcContainerSetupFDs(&ttyfd,
+                             argv->npassFDs, argv->passFDs) < 0)
         goto cleanup;
-    }
 
     ret = 0;
 cleanup:
@@ -1762,18 +1829,29 @@ int lxcContainerStart(virDomainDefPtr def,
                       virSecurityManagerPtr securityDriver,
                       size_t nveths,
                       char **veths,
+                      size_t npassFDs,
+                      int *passFDs,
                       int control,
                       int handshakefd,
-                      char **ttyPaths,
-                      size_t nttyPaths)
+                      size_t nttyPaths,
+                      char **ttyPaths)
 {
     pid_t pid;
     int cflags;
     int stacksize = getpagesize() * 4;
     char *stack, *stacktop;
-    lxc_child_argv_t args = { def, securityDriver,
-                              nveths, veths, control,
-                              ttyPaths, nttyPaths, handshakefd};
+    lxc_child_argv_t args = {
+        .config = def,
+        .securityDriver = securityDriver,
+        .nveths = nveths,
+        .veths = veths,
+        .npassFDs = npassFDs,
+        .passFDs = passFDs,
+        .monitor = control,
+        .nttyPaths = nttyPaths,
+        .ttyPaths = ttyPaths,
+        .handshakefd = handshakefd
+    };
 
     /* allocate a stack for the container */
     if (VIR_ALLOC_N(stack, stacksize) < 0)

src/lxc/lxc_container.h

@@ -56,10 +56,12 @@ int lxcContainerStart(virDomainDefPtr def,
                       virSecurityManagerPtr securityDriver,
                       size_t nveths,
                       char **veths,
+                      size_t npassFDs,
+                      int *passFDs,
                       int control,
                       int handshakefd,
-                      char **ttyPaths,
-                      size_t nttyPaths);
+                      size_t nttyPaths,
+                      char **ttyPaths);
 
 int lxcContainerAvailable(int features);
 

src/lxc/lxc_controller.c

@@ -108,6 +108,9 @@ struct _virLXCController {
     size_t nveths;
     char **veths;
 
+    size_t npassFDs;
+    int *passFDs;
+
     size_t nconsoles;
     virLXCControllerConsolePtr consoles;
     char *devptmx;
@@ -253,6 +256,10 @@ static void virLXCControllerFree(virLXCControllerPtr ctrl)
         VIR_FREE(ctrl->veths[i]);
     VIR_FREE(ctrl->veths);
 
+    for (i = 0; i < ctrl->npassFDs; i++)
+        VIR_FORCE_CLOSE(ctrl->passFDs[i]);
+    VIR_FREE(ctrl->passFDs);
+
     for (i = 0; i < ctrl->nconsoles; i++)
         virLXCControllerConsoleClose(&(ctrl->consoles[i]));
     VIR_FREE(ctrl->consoles);
@@ -2135,14 +2142,19 @@ virLXCControllerRun(virLXCControllerPtr ctrl)
                                            ctrl->securityManager,
                                            ctrl->nveths,
                                            ctrl->veths,
+                                           ctrl->npassFDs,
+                                           ctrl->passFDs,
                                            control[1],
                                            containerhandshake[1],
-                                           containerTTYPaths,
-                                           ctrl->nconsoles)) < 0)
+                                           ctrl->nconsoles,
+                                           containerTTYPaths)) < 0)
         goto cleanup;
     VIR_FORCE_CLOSE(control[1]);
     VIR_FORCE_CLOSE(containerhandshake[1]);
 
+    for (i = 0; i < ctrl->npassFDs; i++)
+        VIR_FORCE_CLOSE(ctrl->passFDs[i]);
+
     if (virLXCControllerSetupUserns(ctrl) < 0)
         goto cleanup;
 
@@ -2209,6 +2221,7 @@ int main(int argc, char *argv[])
         { "name",   1, NULL, 'n' },
         { "veth",   1, NULL, 'v' },
         { "console", 1, NULL, 'c' },
+        { "passfd", 1, NULL, 'p' },
         { "handshakefd", 1, NULL, 's' },
         { "security", 1, NULL, 'S' },
         { "help", 0, NULL, 'h' },
@@ -2216,6 +2229,8 @@ int main(int argc, char *argv[])
     };
     int *ttyFDs = NULL;
     size_t nttyFDs = 0;
+    int *passFDs = NULL;
+    size_t npassFDs = 0;
     virLXCControllerPtr ctrl = NULL;
     size_t i;
     const char *securityDriver = "none";
@@ -2233,7 +2248,7 @@ int main(int argc, char *argv[])
     while (1) {
         int c;
 
-        c = getopt_long(argc, argv, "dn:v:m:c:s:h:S:",
+        c = getopt_long(argc, argv, "dn:v:p:m:c:s:h:S:",
                        options, NULL);
 
         if (c == -1)
@@ -2265,6 +2280,15 @@ int main(int argc, char *argv[])
             }
             break;
 
+        case 'p':
+            if (VIR_REALLOC_N(passFDs, npassFDs + 1) < 0)
+                goto cleanup;
+            if (virStrToLong_i(optarg, NULL, 10, &passFDs[npassFDs++]) < 0) {
+                fprintf(stderr, "malformed --passfd argument '%s'", optarg);
+                goto cleanup;
+            }
+            break;
+
         case 's':
             if (virStrToLong_i(optarg, NULL, 10, &handshakeFd) < 0) {
                 fprintf(stderr, "malformed --handshakefd argument '%s'",
@@ -2337,6 +2361,9 @@ int main(int argc, char *argv[])
     ctrl->veths = veths;
     ctrl->nveths = nveths;
 
+    ctrl->passFDs = passFDs;
+    ctrl->npassFDs = npassFDs;
+
     for (i = 0; i < nttyFDs; i++) {
         if (virLXCControllerAddConsole(ctrl, ttyFDs[i]) < 0)
             goto cleanup;
@@ -2402,6 +2429,9 @@ cleanup:
     for (i = 0; i < nttyFDs; i++)
         VIR_FORCE_CLOSE(ttyFDs[i]);
     VIR_FREE(ttyFDs);
+    for (i = 0; i < npassFDs; i++)
+        VIR_FORCE_CLOSE(passFDs[i]);
+    VIR_FREE(passFDs);
 
     virLXCControllerFree(ctrl);
 

src/lxc/lxc_driver.c

@@ -1021,7 +1021,10 @@ cleanup:
  *
  * Returns 0 on success or -1 in case of error
  */
-static int lxcDomainCreateWithFlags(virDomainPtr dom, unsigned int flags)
+static int lxcDomainCreateWithFiles(virDomainPtr dom,
+                                    unsigned int nfiles,
+                                    int *files,
+                                    unsigned int flags)
 {
     virLXCDriverPtr driver = dom->conn->privateData;
     virDomainObjPtr vm;
@@ -1040,7 +1043,7 @@ static int lxcDomainCreateWithFlags(virDomainPtr dom, unsigned int flags)
         goto cleanup;
     }
 
-    if (virDomainCreateWithFlagsEnsureACL(dom->conn, vm->def) < 0)
+    if (virDomainCreateWithFilesEnsureACL(dom->conn, vm->def) < 0)
         goto cleanup;
 
     if ((vm->def->nets != NULL) && !(driver->have_netns)) {
@@ -1056,6 +1059,7 @@ static int lxcDomainCreateWithFlags(virDomainPtr dom, unsigned int flags)
     }
 
     ret = virLXCProcessStart(dom->conn, driver, vm,
+                             nfiles, files,
                              (flags & VIR_DOMAIN_START_AUTODESTROY),
                              VIR_DOMAIN_RUNNING_BOOTED);
 
@@ -1087,7 +1091,21 @@ cleanup:
  */
 static int lxcDomainCreate(virDomainPtr dom)
 {
-    return lxcDomainCreateWithFlags(dom, 0);
+    return lxcDomainCreateWithFiles(dom, 0, NULL, 0);
+}
+
+/**
+ * lxcDomainCreateWithFlags:
+ * @dom: domain to start
+ *
+ * Looks up domain and starts it.
+ *
+ * Returns 0 on success or -1 in case of error
+ */
+static int lxcDomainCreateWithFlags(virDomainPtr dom,
+                                    unsigned int flags)
+{
+    return lxcDomainCreateWithFiles(dom, 0, NULL, flags);
 }
 
 /**
@@ -1101,9 +1119,11 @@ static int lxcDomainCreate(virDomainPtr dom)
  * Returns 0 on success or -1 in case of error
  */
 static virDomainPtr
-lxcDomainCreateXML(virConnectPtr conn,
-                   const char *xml,
-                   unsigned int flags) {
+lxcDomainCreateXMLWithFiles(virConnectPtr conn,
+                            const char *xml,
+                            unsigned int nfiles,
+                            int *files,
+                            unsigned int flags) {
     virLXCDriverPtr driver = conn->privateData;
     virDomainObjPtr vm = NULL;
     virDomainDefPtr def;
@@ -1118,7 +1138,7 @@ lxcDomainCreateXML(virConnectPtr conn,
                                         VIR_DOMAIN_XML_INACTIVE)))
         goto cleanup;
 
-    if (virDomainCreateXMLEnsureACL(conn, def) < 0)
+    if (virDomainCreateXMLWithFilesEnsureACL(conn, def) < 0)
         goto cleanup;
 
     if (virSecurityManagerVerify(driver->securityManager, def) < 0)
@@ -1139,6 +1159,7 @@ lxcDomainCreateXML(virConnectPtr conn,
     def = NULL;
 
     if (virLXCProcessStart(conn, driver, vm,
+                           nfiles, files,
                            (flags & VIR_DOMAIN_START_AUTODESTROY),
                            VIR_DOMAIN_RUNNING_BOOTED) < 0) {
         virDomainAuditStart(vm, "booted", false);
@@ -1167,6 +1188,14 @@ cleanup:
 }
 
 
+static virDomainPtr
+lxcDomainCreateXML(virConnectPtr conn,
+                   const char *xml,
+                   unsigned int flags) {
+    return lxcDomainCreateXMLWithFiles(conn, xml, 0, NULL,  flags);
+}
+
+
 static int lxcDomainGetSecurityLabel(virDomainPtr dom, virSecurityLabelPtr seclabel)
 {
     virLXCDriverPtr driver = dom->conn->privateData;
@@ -4849,6 +4878,7 @@ static virDriver lxcDriver = {
     .connectNumOfDomains = lxcConnectNumOfDomains, /* 0.4.2 */
     .connectListAllDomains = lxcConnectListAllDomains, /* 0.9.13 */
     .domainCreateXML = lxcDomainCreateXML, /* 0.4.4 */
+    .domainCreateXMLWithFiles = lxcDomainCreateXMLWithFiles, /* 1.1.1 */
     .domainLookupByID = lxcDomainLookupByID, /* 0.4.2 */
     .domainLookupByUUID = lxcDomainLookupByUUID, /* 0.4.2 */
     .domainLookupByName = lxcDomainLookupByName, /* 0.4.2 */
@@ -4873,6 +4903,7 @@ static virDriver lxcDriver = {
     .connectNumOfDefinedDomains = lxcConnectNumOfDefinedDomains, /* 0.4.2 */
     .domainCreate = lxcDomainCreate, /* 0.4.4 */
     .domainCreateWithFlags = lxcDomainCreateWithFlags, /* 0.8.2 */
+    .domainCreateWithFiles = lxcDomainCreateWithFiles, /* 1.1.1 */
     .domainDefineXML = lxcDomainDefineXML, /* 0.4.2 */
     .domainUndefine = lxcDomainUndefine, /* 0.4.2 */
     .domainUndefineFlags = lxcDomainUndefineFlags, /* 0.9.4 */

src/lxc/lxc_process.c

@@ -192,7 +192,8 @@ virLXCProcessReboot(virLXCDriverPtr driver,
     vm->newDef = NULL;
     virLXCProcessStop(driver, vm, VIR_DOMAIN_SHUTOFF_SHUTDOWN);
     vm->newDef = savedDef;
-    if (virLXCProcessStart(conn, driver, vm, autodestroy, reason) < 0) {
+    if (virLXCProcessStart(conn, driver, vm,
+                           0, NULL, autodestroy, reason) < 0) {
         VIR_WARN("Unable to handle reboot of vm %s",
                  vm->def->name);
         goto cleanup;
@@ -803,6 +804,8 @@ virLXCProcessBuildControllerCmd(virLXCDriverPtr driver,
                                 char **veths,
                                 int *ttyFDs,
                                 size_t nttyFDs,
+                                int *files,
+                                size_t nfiles,
                                 int handshakefd)
 {
     size_t i;
@@ -853,6 +856,12 @@ virLXCProcessBuildControllerCmd(virLXCDriverPtr driver,
         virCommandPreserveFD(cmd, ttyFDs[i]);
     }
 
+    for (i = 0; i < nfiles; i++) {
+        virCommandAddArg(cmd, "--passfd");
+        virCommandAddArgFormat(cmd, "%d", files[i]);
+        virCommandPreserveFD(cmd, files[i]);
+    }
+
     virCommandAddArgPair(cmd, "--security",
                          virSecurityManagerGetModel(driver->securityManager));
 
@@ -1024,6 +1033,7 @@ error:
 int virLXCProcessStart(virConnectPtr conn,
                        virLXCDriverPtr  driver,
                        virDomainObjPtr vm,
+                       unsigned int nfiles, int *files,
                        bool autoDestroy,
                        virDomainRunningReason reason)
 {
@@ -1189,6 +1199,7 @@ int virLXCProcessStart(virConnectPtr conn,
                                                 vm,
                                                 nveths, veths,
                                                 ttyFDs, nttyFDs,
+                                                files, nfiles,
                                                 handshakefds[1])))
         goto cleanup;
     virCommandSetOutputFD(cmd, &logfd);
@@ -1382,7 +1393,8 @@ virLXCProcessAutostartDomain(virDomainObjPtr vm,
     virObjectLock(vm);
     if (vm->autostart &&
         !virDomainObjIsActive(vm)) {
-        ret = virLXCProcessStart(data->conn, data->driver, vm, false,
+        ret = virLXCProcessStart(data->conn, data->driver, vm,
+                                 0, NULL, false,
                                  VIR_DOMAIN_RUNNING_BOOTED);
         virDomainAuditStart(vm, "booted", ret >= 0);
         if (ret < 0) {

src/lxc/lxc_process.h

@@ -27,6 +27,7 @@
 int virLXCProcessStart(virConnectPtr conn,
                        virLXCDriverPtr  driver,
                        virDomainObjPtr vm,
+                       unsigned int nfiles, int *files,
                        bool autoDestroy,
                        virDomainRunningReason reason);
 int virLXCProcessStop(virLXCDriverPtr driver,