Add support for multiple consoles in LXC

Currently the LXC controller only supports setup of a single
text console. This is wired up to the container init's stdio,
as well as /dev/console and /dev/tty1. Extending support for
multiple consoles, means wiring up additional PTYs to /dev/tty2,
/dev/tty3, etc, etc. The LXC controller is passed multiple open
file handles, one for each console requested.

* src/lxc/lxc_container.c, src/lxc/lxc_container.h: Wire up
  all the /dev/ttyN links required to symlink to /dev/pts/NN
* src/lxc/lxc_container.h: Open more container side /dev/pts/NN
  devices, and adapt event loop to handle I/O from all consoles
* src/lxc/lxc_driver.c: Setup multiple host side PTYs

src/lxc/lxc_container.c

@@ -94,7 +94,8 @@ struct __lxc_child_argv {
     unsigned int nveths;
     char **veths;
     int monitor;
-    char *ttyPath;
+    char **ttyPaths;
+    size_t nttyPaths;
     int handshakefd;
 };
 
@@ -526,9 +527,9 @@ static int lxcContainerMountDevFS(virDomainFSDefPtr root)
     return rc;
 }
 
-static int lxcContainerPopulateDevices(void)
+static int lxcContainerPopulateDevices(char **ttyPaths, size_t nttyPaths)
 {
-    int i;
+    size_t i;
     const struct {
         int maj;
         int min;
@@ -570,21 +571,28 @@ static int lxcContainerPopulateDevices(void)
         }
     }
 
-    /* XXX we should allow multiple consoles per container
-     * for tty2, tty3, etc, but the domain XML does not
-     * handle this yet
-     */
-    if (symlink("/dev/pts/0", "/dev/tty1") < 0) {
-        virReportSystemError(errno, "%s",
-                             _("Failed to symlink /dev/pts/0 to /dev/tty1"));
-        return -1;
-    }
-    if (symlink("/dev/pts/0", "/dev/console") < 0) {
-        virReportSystemError(errno, "%s",
-                             _("Failed to symlink /dev/pts/0 to /dev/console"));
-        return -1;
+    for (i = 0 ; i < nttyPaths ; i++) {
+        char *tty;
+        if (virAsprintf(&tty, "/dev/tty%zu", i+1) < 0) {
+            virReportOOMError();
+            return -1;
+        }
+        if (symlink(ttyPaths[i], tty) < 0) {
+            VIR_FREE(tty);
+            virReportSystemError(errno,
+                                 _("Failed to symlink %s to %s"),
+                                 ttyPaths[i], tty);
+            return -1;
+        }
+        VIR_FREE(tty);
+        if (i == 0 &&
+            symlink(ttyPaths[i], "/dev/console") < 0) {
+            virReportSystemError(errno,
+                                 _("Failed to symlink %s to /dev/console"),
+                                 ttyPaths[i]);
+            return -1;
+        }
     }
-
     return 0;
 }
 
@@ -1043,7 +1051,9 @@ cleanup:
  * this is based on this thread http://lkml.org/lkml/2008/3/5/29
  */
 static int lxcContainerSetupPivotRoot(virDomainDefPtr vmDef,
-                                      virDomainFSDefPtr root)
+                                      virDomainFSDefPtr root,
+                                      char **ttyPaths,
+                                      size_t nttyPaths)
 {
     /* Gives us a private root, leaving all parent OS mounts on /.oldroot */
     if (lxcContainerPivotRoot(root) < 0)
@@ -1058,7 +1068,7 @@ static int lxcContainerSetupPivotRoot(virDomainDefPtr vmDef,
         return -1;
 
     /* Populates device nodes in /dev/ */
-    if (lxcContainerPopulateDevices() < 0)
+    if (lxcContainerPopulateDevices(ttyPaths, nttyPaths) < 0)
         return -1;
 
     /* Sets up any non-root mounts from guest config */
@@ -1102,10 +1112,12 @@ static int lxcContainerSetupExtraMounts(virDomainDefPtr vmDef)
 }
 
 static int lxcContainerSetupMounts(virDomainDefPtr vmDef,
-                                   virDomainFSDefPtr root)
+                                   virDomainFSDefPtr root,
+                                   char **ttyPaths,
+                                   size_t nttyPaths)
 {
     if (root)
-        return lxcContainerSetupPivotRoot(vmDef, root);
+        return lxcContainerSetupPivotRoot(vmDef, root, ttyPaths, nttyPaths);
     else
         return lxcContainerSetupExtraMounts(vmDef);
 }
@@ -1189,17 +1201,25 @@ static int lxcContainerChild( void *data )
 
     root = virDomainGetRootFilesystem(vmDef);
 
-    if (root) {
-        if (virAsprintf(&ttyPath, "%s%s", root->src, argv->ttyPath) < 0) {
-            virReportOOMError();
-            goto cleanup;
+    if (argv->nttyPaths) {
+        if (root) {
+            if (virAsprintf(&ttyPath, "%s%s", root->src, argv->ttyPaths[0]) < 0) {
+                virReportOOMError();
+                goto cleanup;
+            }
+        } else {
+            if (!(ttyPath = strdup(argv->ttyPaths[0]))) {
+                virReportOOMError();
+                goto cleanup;
+            }
         }
     } else {
-        if (!(ttyPath = strdup(argv->ttyPath))) {
+        if (!(ttyPath = strdup("/dev/null"))) {
             virReportOOMError();
             goto cleanup;
         }
     }
+
     VIR_DEBUG("Container TTY path: %s", ttyPath);
 
     ttyfd = open(ttyPath, O_RDWR|O_NOCTTY);
@@ -1210,7 +1230,7 @@ static int lxcContainerChild( void *data )
         goto cleanup;
     }
 
-    if (lxcContainerSetupMounts(vmDef, root) < 0)
+    if (lxcContainerSetupMounts(vmDef, root, argv->ttyPaths, argv->nttyPaths) < 0)
         goto cleanup;
 
     if (!virFileExists(vmDef->os.init)) {
@@ -1314,14 +1334,15 @@ int lxcContainerStart(virDomainDefPtr def,
                       char **veths,
                       int control,
                       int handshakefd,
-                      char *ttyPath)
+                      char **ttyPaths,
+                      size_t nttyPaths)
 {
     pid_t pid;
     int cflags;
     int stacksize = getpagesize() * 4;
     char *stack, *stacktop;
-    lxc_child_argv_t args = { def, nveths, veths, control, ttyPath,
-                              handshakefd};
+    lxc_child_argv_t args = { def, nveths, veths, control,
+                              ttyPaths, nttyPaths, handshakefd};
 
     /* allocate a stack for the container */
     if (VIR_ALLOC_N(stack, stacksize) < 0) {

src/lxc/lxc_container.h

@@ -53,7 +53,8 @@ int lxcContainerStart(virDomainDefPtr def,
                       char **veths,
                       int control,
                       int handshakefd,
-                      char *ttyPath);
+                      char **ttyPaths,
+                      size_t nttyPaths);
 
 int lxcContainerAvailable(int features);
 

src/lxc/lxc_controller.c

@@ -797,20 +797,19 @@ error:
  */
 static int lxcControllerMain(int serverFd,
                              int clientFd,
-                             int hostFd,
-                             int contFd,
+                             int *hostFds,
+                             int *contFds,
+                             size_t nFds,
                              pid_t container)
 {
-    struct lxcConsole console = {
-        .hostFd = hostFd,
-        .contFd = contFd,
-    };
+    struct lxcConsole *consoles;
     struct lxcMonitor monitor = {
         .serverFd = serverFd,
         .clientFd = clientFd,
     };
     virErrorPtr err;
     int rc = -1;
+    size_t i;
 
     if (virMutexInit(&lock) < 0)
         goto cleanup2;
@@ -837,8 +836,8 @@ static int lxcControllerMain(int serverFd,
         goto cleanup;
     }
 
-    VIR_DEBUG("serverFd=%d clientFd=%d hostFd=%d contFd=%d",
-              serverFd, clientFd, hostFd, contFd);
+    VIR_DEBUG("serverFd=%d clientFd=%d",
+              serverFd, clientFd);
     virResetLastError();
 
     if ((monitor.serverWatch = virEventAddHandle(monitor.serverFd,
@@ -862,24 +861,34 @@ static int lxcControllerMain(int serverFd,
         goto cleanup;
     }
 
-    if ((console.hostWatch = virEventAddHandle(console.hostFd,
-                                               VIR_EVENT_HANDLE_READABLE,
-                                               lxcConsoleIO,
-                                               &console,
-                                               NULL)) < 0) {
-        lxcError(VIR_ERR_INTERNAL_ERROR, "%s",
-                 _("Unable to watch host console PTY"));
+    if (VIR_ALLOC_N(consoles, nFds) < 0) {
+        virReportOOMError();
         goto cleanup;
     }
 
-    if ((console.contWatch = virEventAddHandle(console.contFd,
-                                               VIR_EVENT_HANDLE_READABLE,
-                                               lxcConsoleIO,
-                                               &console,
-                                               NULL)) < 0) {
-        lxcError(VIR_ERR_INTERNAL_ERROR, "%s",
-                 _("Unable to watch host console PTY"));
-        goto cleanup;
+    for (i = 0 ; i < nFds ; i++) {
+        consoles[i].hostFd = hostFds[i];
+        consoles[i].contFd = contFds[i];
+
+        if ((consoles[i].hostWatch = virEventAddHandle(consoles[i].hostFd,
+                                                       VIR_EVENT_HANDLE_READABLE,
+                                                       lxcConsoleIO,
+                                                       &consoles[i],
+                                                       NULL)) < 0) {
+            lxcError(VIR_ERR_INTERNAL_ERROR, "%s",
+                     _("Unable to watch host console PTY"));
+            goto cleanup;
+        }
+
+        if ((consoles[i].contWatch = virEventAddHandle(consoles[i].contFd,
+                                                       VIR_EVENT_HANDLE_READABLE,
+                                                       lxcConsoleIO,
+                                                       &consoles[i],
+                                                       NULL)) < 0) {
+            lxcError(VIR_ERR_INTERNAL_ERROR, "%s",
+                     _("Unable to watch host console PTY"));
+            goto cleanup;
+        }
     }
 
     virMutexLock(&lock);
@@ -899,10 +908,9 @@ cleanup:
     virMutexDestroy(&lock);
     signal(SIGCHLD, SIG_DFL);
 cleanup2:
-    VIR_FORCE_CLOSE(console.hostFd);
-    VIR_FORCE_CLOSE(console.contFd);
     VIR_FORCE_CLOSE(monitor.serverFd);
     VIR_FORCE_CLOSE(monitor.clientFd);
+    VIR_FREE(consoles);
     return rc;
 }
 
@@ -1027,14 +1035,15 @@ lxcControllerRun(virDomainDefPtr def,
                  char **veths,
                  int monitor,
                  int client,
-                 int appPty,
+                 int *ttyFDs,
+                 size_t nttyFDs,
                  int handshakefd)
 {
     int rc = -1;
     int control[2] = { -1, -1};
     int containerhandshake[2] = { -1, -1 };
-    int containerPty = -1;
-    char *containerPtyPath = NULL;
+    int *containerTtyFDs = NULL;
+    char **containerTtyPaths = NULL;
     pid_t container = -1;
     virDomainFSDefPtr root;
     char *devpts = NULL;
@@ -1043,6 +1052,15 @@ lxcControllerRun(virDomainDefPtr def,
     int *loopDevs = NULL;
     size_t i;
 
+    if (VIR_ALLOC_N(containerTtyFDs, nttyFDs) < 0) {
+        virReportOOMError();
+        goto cleanup;
+    }
+    if (VIR_ALLOC_N(containerTtyPaths, nttyFDs) < 0) {
+        virReportOOMError();
+        goto cleanup;
+    }
+
     if (socketpair(PF_UNIX, SOCK_STREAM, 0, control) < 0) {
         virReportSystemError(errno, "%s",
                              _("sockpair failed"));
@@ -1133,26 +1151,36 @@ lxcControllerRun(virDomainDefPtr def,
             VIR_WARN("Kernel does not support private devpts, using shared devpts");
             VIR_FREE(devptmx);
         }
-    }
-
-    if (devptmx) {
-        VIR_DEBUG("Opening tty on private %s", devptmx);
-        if (lxcCreateTty(devptmx, &containerPty, &containerPtyPath) < 0) {
-            virReportSystemError(errno, "%s",
-                                 _("Failed to allocate tty"));
-            goto cleanup;
-        }
     } else {
-        VIR_DEBUG("Opening tty on shared /dev/ptmx");
-        if (virFileOpenTty(&containerPty,
-                           &containerPtyPath,
-                           0) < 0) {
-            virReportSystemError(errno, "%s",
-                                 _("Failed to allocate tty"));
+        if (nttyFDs != -1) {
+            lxcError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
+                     _("Expected exactly one TTY fd"));
             goto cleanup;
         }
     }
 
+    for (i = 0 ; i < nttyFDs ; i++) {
+        if (devptmx) {
+            VIR_DEBUG("Opening tty on private %s", devptmx);
+            if (lxcCreateTty(devptmx,
+                             &containerTtyFDs[i],
+                             &containerTtyPaths[i]) < 0) {
+                virReportSystemError(errno, "%s",
+                                     _("Failed to allocate tty"));
+                goto cleanup;
+            }
+        } else {
+            VIR_DEBUG("Opening tty on shared /dev/ptmx");
+            if (virFileOpenTty(&containerTtyFDs[i],
+                               &containerTtyPaths[i],
+                               0) < 0) {
+                virReportSystemError(errno, "%s",
+                                     _("Failed to allocate tty"));
+                goto cleanup;
+            }
+        }
+    }
+
     if (lxcSetPersonality(def) < 0)
         goto cleanup;
 
@@ -1161,7 +1189,8 @@ lxcControllerRun(virDomainDefPtr def,
                                        veths,
                                        control[1],
                                        containerhandshake[1],
-                                       containerPtyPath)) < 0)
+                                       containerTtyPaths,
+                                       nttyFDs)) < 0)
         goto cleanup;
     VIR_FORCE_CLOSE(control[1]);
     VIR_FORCE_CLOSE(containerhandshake[1]);
@@ -1200,33 +1229,41 @@ lxcControllerRun(virDomainDefPtr def,
     VIR_FORCE_CLOSE(handshakefd);
 
     if (virSetBlocking(monitor, false) < 0 ||
-        virSetBlocking(client, false) < 0 ||
-        virSetBlocking(appPty, false) < 0 ||
-        virSetBlocking(containerPty, false) < 0) {
+        virSetBlocking(client, false) < 0) {
         virReportSystemError(errno, "%s",
                              _("Unable to set file descriptor non blocking"));
         goto cleanup;
     }
+    for (i = 0 ; i < nttyFDs ; i++) {
+        if (virSetBlocking(ttyFDs[i], false) < 0 ||
+            virSetBlocking(containerTtyFDs[i], false) < 0) {
+            virReportSystemError(errno, "%s",
+                                 _("Unable to set file descriptor non blocking"));
+            goto cleanup;
+        }
+    }
 
-    rc = lxcControllerMain(monitor, client, appPty, containerPty, container);
-    monitor = client = appPty = containerPty = -1;
+    rc = lxcControllerMain(monitor, client, ttyFDs, containerTtyFDs, nttyFDs, container);
+    monitor = client = -1;
 
 cleanup:
     VIR_FREE(devptmx);
     VIR_FREE(devpts);
     VIR_FORCE_CLOSE(control[0]);
     VIR_FORCE_CLOSE(control[1]);
-    VIR_FREE(containerPtyPath);
-    VIR_FORCE_CLOSE(containerPty);
     VIR_FORCE_CLOSE(handshakefd);
     VIR_FORCE_CLOSE(containerhandshake[0]);
     VIR_FORCE_CLOSE(containerhandshake[1]);
 
-    if (loopDevs) {
-        for (i = 0 ; i < nloopDevs ; i++)
-            VIR_FORCE_CLOSE(loopDevs[i]);
-    }
+    for (i = 0 ; i < nttyFDs ; i++)
+        VIR_FREE(containerTtyPaths[i]);
+    VIR_FREE(containerTtyPaths);
+    for (i = 0 ; i < nttyFDs ; i++)
+        VIR_FORCE_CLOSE(containerTtyFDs[i]);
+    VIR_FREE(containerTtyFDs);
 
+    for (i = 0 ; i < nloopDevs ; i++)
+        VIR_FORCE_CLOSE(loopDevs[i]);
     VIR_FREE(loopDevs);
 
     if (container > 1) {
@@ -1250,7 +1287,6 @@ int main(int argc, char *argv[])
     int nveths = 0;
     char **veths = NULL;
     int monitor = -1;
-    int appPty = -1;
     int handshakefd = -1;
     int bg = 0;
     virCapsPtr caps = NULL;
@@ -1266,6 +1302,8 @@ int main(int argc, char *argv[])
         { "help", 0, NULL, 'h' },
         { 0, 0, 0, 0 },
     };
+    int *ttyFDs = NULL;
+    size_t nttyFDs = 0;
 
     if (setlocale(LC_ALL, "") == NULL ||
         bindtextdomain(PACKAGE, LOCALEDIR) == NULL ||
@@ -1307,7 +1345,11 @@ int main(int argc, char *argv[])
             break;
 
         case 'c':
-            if (virStrToLong_i(optarg, NULL, 10, &appPty) < 0) {
+            if (VIR_REALLOC_N(ttyFDs, nttyFDs + 1) < 0) {
+                virReportOOMError();
+                goto cleanup;
+            }
+            if (virStrToLong_i(optarg, NULL, 10, &ttyFDs[nttyFDs++]) < 0) {
                 fprintf(stderr, "malformed --console argument '%s'", optarg);
                 goto cleanup;
             }
@@ -1345,11 +1387,6 @@ int main(int argc, char *argv[])
         goto cleanup;
     }
 
-    if (appPty < 0) {
-        fprintf(stderr, "%s: missing --console argument for container PTY\n", argv[0]);
-        goto cleanup;
-    }
-
     if (handshakefd < 0) {
         fprintf(stderr, "%s: missing --handshake argument for container PTY\n",
                 argv[0]);
@@ -1429,8 +1466,8 @@ int main(int argc, char *argv[])
         goto cleanup;
     }
 
-    rc = lxcControllerRun(def, nveths, veths, monitor, client, appPty,
-                          handshakefd);
+    rc = lxcControllerRun(def, nveths, veths, monitor, client,
+                          ttyFDs, nttyFDs, handshakefd);
 
 
 cleanup:

src/lxc/lxc_driver.c

@@ -1447,11 +1447,12 @@ lxcBuildControllerCmd(lxc_driver_t *driver,
                       virDomainObjPtr vm,
                       int nveths,
                       char **veths,
-                      int appPty,
+                      int *ttyFDs,
+                      size_t nttyFDs,
                       int logfile,
                       int handshakefd)
 {
-    int i;
+    size_t i;
     char *filterstr;
     char *outputstr;
     virCommandPtr cmd;
@@ -1492,8 +1493,12 @@ lxcBuildControllerCmd(lxc_driver_t *driver,
                                virLogGetDefaultPriority());
     }
 
-    virCommandAddArgList(cmd, "--name", vm->def->name, "--console", NULL);
-    virCommandAddArgFormat(cmd, "%d", appPty);
+    virCommandAddArgList(cmd, "--name", vm->def->name, NULL);
+    for (i = 0 ; i < nttyFDs ; i++) {
+        virCommandAddArg(cmd, "--console");
+        virCommandAddArgFormat(cmd, "%d", ttyFDs[i]);
+        virCommandPreserveFD(cmd, ttyFDs[i]);
+    }
     virCommandAddArg(cmd, "--handshake");
     virCommandAddArgFormat(cmd, "%d", handshakefd);
     virCommandAddArg(cmd, "--background");
@@ -1518,7 +1523,6 @@ lxcBuildControllerCmd(lxc_driver_t *driver,
             goto cleanup;
     }
 
-    virCommandPreserveFD(cmd, appPty);
     virCommandPreserveFD(cmd, handshakefd);
     virCommandSetOutputFD(cmd, &logfile);
     virCommandSetErrorFD(cmd, &logfile);
@@ -1621,9 +1625,9 @@ static int lxcVmStart(virConnectPtr conn,
                       virDomainRunningReason reason)
 {
     int rc = -1, r;
-    unsigned int i;
-    int parentTty;
-    char *parentTtyPath = NULL;
+    size_t nttyFDs = 0;
+    int *ttyFDs = NULL;
+    size_t i;
     char *logfile = NULL;
     int logfd = -1;
     unsigned int nveths = 0;
@@ -1674,26 +1678,34 @@ static int lxcVmStart(virConnectPtr conn,
         return -1;
     }
 
-    /* open parent tty */
-    if (virFileOpenTty(&parentTty, &parentTtyPath, 1) < 0) {
-        virReportSystemError(errno, "%s",
-                             _("Failed to allocate tty"));
+    /* Here we open all the PTYs we need on the host OS side.
+     * The LXC controller will open the guest OS side PTYs
+     * and forward I/O between them.
+     */
+    nttyFDs = vm->def->nconsoles;
+    if (VIR_ALLOC_N(ttyFDs, nttyFDs) < 0) {
+        virReportOOMError();
         goto cleanup;
     }
-    if (vm->def->nconsoles) {
-        if (vm->def->nconsoles > 1) {
+    for (i = 0 ; i < vm->def->nconsoles ; i++)
+        ttyFDs[i] = -1;
+
+    for (i = 0 ; i < vm->def->nconsoles ; i++) {
+        char *ttyPath;
+        if (vm->def->consoles[i]->source.type != VIR_DOMAIN_CHR_TYPE_PTY) {
             lxcError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
-                     _("Only one console supported"));
+                     _("Only PTY console types are supported"));
             goto cleanup;
         }
-        if (vm->def->consoles[0]->source.type == VIR_DOMAIN_CHR_TYPE_PTY) {
-            VIR_FREE(vm->def->consoles[0]->source.data.file.path);
-            vm->def->consoles[0]->source.data.file.path = parentTtyPath;
-        } else {
-            VIR_FREE(parentTtyPath);
+
+        if (virFileOpenTty(&ttyFDs[i], &ttyPath, 1) < 0) {
+            virReportSystemError(errno, "%s",
+                                 _("Failed to allocate tty"));
+            goto cleanup;
         }
-    } else {
-        VIR_FREE(parentTtyPath);
+
+        VIR_FREE(vm->def->consoles[i]->source.data.file.path);
+        vm->def->consoles[i]->source.data.file.path = ttyPath;
     }
 
     if (lxcSetupInterfaces(conn, vm->def, &nveths, &veths) != 0)
@@ -1720,7 +1732,8 @@ static int lxcVmStart(virConnectPtr conn,
     if (!(cmd = lxcBuildControllerCmd(driver,
                                       vm,
                                       nveths, veths,
-                                      parentTty, logfd, handshakefds[1])))
+                                      ttyFDs, nttyFDs,
+                                      logfd, handshakefds[1])))
         goto cleanup;
 
     /* Log timestamp */
@@ -1822,7 +1835,8 @@ cleanup:
         VIR_FORCE_CLOSE(priv->monitor);
         virDomainConfVMNWFilterTeardown(vm);
     }
-    VIR_FORCE_CLOSE(parentTty);
+    for (i = 0 ; i < nttyFDs ; i++)
+        VIR_FORCE_CLOSE(ttyFDs[i]);
     VIR_FORCE_CLOSE(handshakefds[0]);
     VIR_FORCE_CLOSE(handshakefds[1]);
     VIR_FREE(logfile);