Make LXC I/O controller process a parent of the container process

ChangeLog

+Wed Aug 13 11:48:36 BST 2008 Daniel Berrange <berrange@redhat.com>
+
+	* configure.in: Add check for termios.h
+	* src/util.h, src/util.c: Add virFileOpenTty and helpers
+	for creating/deleting/reading PID files
+	* src/lxc_conf.h, src/lxc_conf.c, src/lxc_container.c,
+	src/lxc_container.h, src/lxc_controller.c,
+	src/lxc_controller.h, src/lxc_driver.c: Re-arrange
+	container launch process so that the I/O helper is
+	a direct parent of the container process. Daemonize
+	container so it survives restarts of libvirtd.
+
+Wed Aug 13 11:23:36 BST 2008 Daniel Berrange <berrange@redhat.com>
+
+	Re-arrange code between LXC driver files
+	* src/lxc_container.c, src/lxc_container.h,
+	src/lxc_controller.h, src/lxc_container.c,
+	src/lxc_driver.c: Move code for I/O handling into
+	a seprate lxc_controller module, and move code for
+	creating containers into lcx_container module.
+
 Wed Aug 13 10:55:36 BST 2008 Daniel Berrange <berrange@redhat.com>
 
 	* src/lxc_conf.h, src/lxc_conf.c, src/lxc_container.h,

configure.in

@@ -68,7 +68,7 @@ dnl Availability of various common functions (non-fatal if missing).
 AC_CHECK_FUNCS([cfmakeraw regexec uname sched_getaffinity])
 
 dnl Availability of various common headers (non-fatal if missing).
-AC_CHECK_HEADERS([pwd.h paths.h sys/syslimits.h sys/utsname.h sys/wait.h winsock2.h sched.h])
+AC_CHECK_HEADERS([pwd.h paths.h sys/syslimits.h sys/utsname.h sys/wait.h winsock2.h sched.h termios.h])
 
 dnl Where are the XDR functions?
 dnl If portablexdr is installed, prefer that.

src/lxc_conf.c

@@ -833,25 +833,24 @@ static lxc_vm_t * lxcLoadConfig(lxc_driver_t *driver,
     strncpy(vm->configFileBase, file, PATH_MAX);
     vm->configFile[PATH_MAX-1] = '\0';
 
-    if (lxcLoadTtyPid(driver, vm) < 0) {
-        DEBUG0("failed to load tty pid");
-    }
-
     return vm;
 }
 
 int lxcLoadDriverConfig(lxc_driver_t *driver)
 {
     /* Set the container configuration directory */
-    driver->configDir = strdup(SYSCONF_DIR "/libvirt/lxc");
-    if (NULL == driver->configDir) {
-        lxcError(NULL, NULL, VIR_ERR_NO_MEMORY, "configDir");
-        return -1;
-    }
-
-    driver->stateDir = strdup(LOCAL_STATE_DIR "/run/libvirt/lxc");
+    if ((driver->configDir = strdup(SYSCONF_DIR "/libvirt/lxc")) == NULL)
+        goto no_memory;
+    if ((driver->stateDir = strdup(LOCAL_STATE_DIR "/run/libvirt/lxc")) == NULL)
+        goto no_memory;
+    if ((driver->logDir = strdup(LOCAL_STATE_DIR "/log/libvirt/lxc")) == NULL)
+        goto no_memory;
 
     return 0;
+
+no_memory:
+    lxcError(NULL, NULL, VIR_ERR_NO_MEMORY, "configDir");
+    return -1;
 }
 
 int lxcLoadContainerConfigFile(lxc_driver_t *driver,
@@ -1012,9 +1011,7 @@ void lxcFreeVMDef(lxc_vm_def_t *vmdef)
     curNet = vmdef->nets;
     while (curNet) {
         nextNet = curNet->next;
-        printf("Freeing %s:%s\n", curNet->parentVeth, curNet->containerVeth);
         VIR_FREE(curNet->parentVeth);
-        VIR_FREE(curNet->containerVeth);
         VIR_FREE(curNet->txName);
         VIR_FREE(curNet);
         curNet = nextNet;
@@ -1106,176 +1103,4 @@ int lxcDeleteConfig(virConnectPtr conn,
     return 0;
 }
 
-/**
- * lxcStoreTtyPid:
- * @driver: pointer to driver
- * @vm: Ptr to VM
- *
- * Stores the pid of the tty forward process contained in vm->pid
- * LOCAL_STATE_DIR/run/libvirt/lxc/{container_name}.pid
- *
- * Returns 0 on success or -1 in case of error
- */
-int lxcStoreTtyPid(const lxc_driver_t *driver, lxc_vm_t *vm)
-{
-    int rc = -1;
-    int fd;
-    FILE *file = NULL;
-
-    if (vm->ttyPidFile[0] == 0x00) {
-        if ((rc = virFileMakePath(driver->stateDir))) {
-            lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR,
-                     _("cannot create lxc state directory %s: %s"),
-                     driver->stateDir, strerror(rc));
-            goto error_out;
-        }
-
-        if (virFileBuildPath(driver->stateDir, vm->def->name, ".pid",
-                             vm->ttyPidFile, PATH_MAX) < 0) {
-            lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR,
-                     _("cannot construct tty pid file path"));
-            goto error_out;
-        }
-    }
-
-    if ((fd = open(vm->ttyPidFile,
-                   O_WRONLY | O_CREAT | O_TRUNC,
-                   S_IRUSR | S_IWUSR)) < 0) {
-        lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR,
-                 _("cannot create tty pid file %s: %s"),
-                 vm->ttyPidFile, strerror(errno));
-        goto error_out;
-    }
-
-    if (!(file = fdopen(fd, "w"))) {
-        lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR,
-                 _("cannot fdopen tty pid file %s: %s"),
-                 vm->ttyPidFile, strerror(errno));
-
-        if (close(fd) < 0) {
-            lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR,
-                     _("failed to close tty pid file %s: %s"),
-                     vm->ttyPidFile, strerror(errno));
-        }
-
-        goto error_out;
-    }
-
-    if (fprintf(file, "%d", vm->pid) < 0) {
-        lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR,
-                 _("cannot write tty pid file %s: %s"),
-                 vm->ttyPidFile, strerror(errno));
-
-        goto fclose_error_out;
-    }
-
-    rc = 0;
-
-fclose_error_out:
-    if (fclose(file) < 0) {
-        lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR,
-                 _("failed to close tty pid file %s: %s"),
-                 vm->ttyPidFile, strerror(errno));
-    }
-
-error_out:
-    return rc;
-}
-
-/**
- * lxcLoadTtyPid:
- * @driver: pointer to driver
- * @vm: Ptr to VM
- *
- * Loads the pid of the tty forward process from the pid file.
- * LOCAL_STATE_DIR/run/libvirt/lxc/{container_name}.pid
- *
- * Returns
- * > 0 - pid of tty process
- *   0 - no tty pid file
- *  -1 - error
- */
-int lxcLoadTtyPid(const lxc_driver_t *driver, lxc_vm_t *vm)
-{
-    int rc = -1;
-    FILE *file;
-
-    if (vm->ttyPidFile[0] == 0x00) {
-        if ((rc = virFileMakePath(driver->stateDir))) {
-            lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR,
-                     _("cannot create lxc state directory %s: %s"),
-                     driver->stateDir, strerror(rc));
-            goto cleanup;
-        }
-
-        if (virFileBuildPath(driver->stateDir, vm->def->name, ".pid",
-                             vm->ttyPidFile, PATH_MAX) < 0) {
-            lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR,
-                     _("cannot construct tty pid file path"));
-            goto cleanup;
-        }
-    }
-
-    if (!(file = fopen(vm->ttyPidFile, "r"))) {
-        if (ENOENT == errno) {
-            rc = 0;
-            goto cleanup;
-        }
-
-        lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR,
-                 _("cannot open tty pid file %s: %s"),
-                 vm->ttyPidFile, strerror(errno));
-        goto cleanup;
-    }
-
-    if (fscanf(file, "%d", &(vm->pid)) < 0) {
-        lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR,
-                 _("cannot read tty pid file %s: %s"),
-                 vm->ttyPidFile, strerror(errno));
-        goto cleanup;
-    }
-
-    if (fclose(file) < 0) {
-        lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR,
-                 _("failed to close tty pid file %s: %s"),
-                 vm->ttyPidFile, strerror(errno));
-        goto cleanup;
-    }
-
-    rc = vm->pid;
-
- cleanup:
-    return rc;
-}
-
-/**
- * lxcDeleteTtyPid:
- * @vm: Ptr to VM
- *
- * Unlinks the tty pid file for the vm
- * LOCAL_STATE_DIR/run/libvirt/lxc/{container_name}.pid
- *
- * Returns on 0 success or -1 in case of error
- */
-int lxcDeleteTtyPidFile(const lxc_vm_t *vm)
-{
-    if (vm->ttyPidFile[0] == 0x00) {
-        goto no_file;
-    }
-
-    if (unlink(vm->ttyPidFile) < 0) {
-        if (errno == ENOENT) {
-            goto no_file;
-        }
-
-        lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR,
-                 _("cannot remove ttyPidFile %s: %s"), vm->ttyPidFile,
-                 strerror(errno));
-        return -1;
-    }
-
-no_file:
-    return 0;
-}
-
 #endif /* WITH_LXC */

src/lxc_conf.h

@@ -46,7 +46,6 @@ typedef struct __lxc_net_def lxc_net_def_t;
 struct __lxc_net_def {
     int type;
     char *parentVeth;       /* veth device in parent namespace */
-    char *containerVeth;    /* veth device in container namespace */
     char *txName;           /* bridge or network name */
 
     lxc_net_def_t *next;
@@ -87,12 +86,11 @@ typedef struct __lxc_vm lxc_vm_t;
 struct __lxc_vm {
     int pid;
     int state;
+    int monitor;
 
     char configFile[PATH_MAX];
     char configFileBase[PATH_MAX];
 
-    char ttyPidFile[PATH_MAX];
-
     lxc_vm_def_t *def;
 
     lxc_vm_t *next;
@@ -103,8 +101,9 @@ struct __lxc_driver {
     lxc_vm_t *vms;
     int nactivevms;
     int ninactivevms;
-    char* configDir;
-    char* stateDir;
+    char *configDir;
+    char *stateDir;
+    char *logDir;
     int have_netns;
 };
 
@@ -154,9 +153,6 @@ int lxcDeleteConfig(virConnectPtr conn,
                     lxc_driver_t *driver,
                     const char *configFile,
                     const char *name);
-int lxcStoreTtyPid(const lxc_driver_t *driver, lxc_vm_t *vm);
-int lxcLoadTtyPid(const lxc_driver_t *driver, lxc_vm_t *vm);
-int lxcDeleteTtyPidFile(const lxc_vm_t *vm);
 
 void lxcError(virConnectPtr conn,
               virDomainPtr dom,

src/lxc_container.c

@@ -69,6 +69,8 @@ typedef char lxc_message_t;
 typedef struct __lxc_child_argv lxc_child_argv_t;
 struct __lxc_child_argv {
     lxc_vm_def_t *config;
+    unsigned int nveths;
+    char **veths;
     int monitor;
     char *ttyPath;
 };
@@ -171,8 +173,7 @@ error_out:
  *
  * Returns 0 on success or -1 in case of error
  */
-int lxcContainerSendContinue(virConnectPtr conn,
-                             int control)
+int lxcContainerSendContinue(int control)
 {
     int rc = -1;
     lxc_message_t msg = LXC_CONTINUE_MSG;
@@ -180,7 +181,7 @@ int lxcContainerSendContinue(virConnectPtr conn,
 
     writeCount = safewrite(control, &msg, sizeof(msg));
     if (writeCount != sizeof(msg)) {
-        lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR,
+        lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR,
                  _("unable to send container continue message: %s"),
                  strerror(errno));
         goto error_out;
@@ -230,21 +231,22 @@ static int lxcContainerWaitForContinue(int control)
  *
  * Returns 0 on success or nonzero in case of error
  */
-static int lxcContainerEnableInterfaces(const lxc_vm_def_t *def)
+static int lxcContainerEnableInterfaces(unsigned int nveths,
+                                        char **veths)
 {
     int rc = 0;
-    const lxc_net_def_t *net;
+    unsigned int i;
 
-    for (net = def->nets; net; net = net->next) {
-        DEBUG("Enabling %s", net->containerVeth);
-        rc =  vethInterfaceUpOrDown(net->containerVeth, 1);
+    for (i = 0 ; i < nveths ; i++) {
+        DEBUG("Enabling %s", veths[i]);
+        rc =  vethInterfaceUpOrDown(veths[i], 1);
         if (0 != rc) {
             goto error_out;
         }
     }
 
     /* enable lo device only if there were other net devices */
-    if (def->nets)
+    if (veths)
         rc = vethInterfaceUpOrDown("lo", 1);
 
 error_out:
@@ -311,7 +313,7 @@ static int lxcContainerChild( void *data )
         return -1;
 
     /* enable interfaces */
-    if (lxcContainerEnableInterfaces(vmDef) < 0)
+    if (lxcContainerEnableInterfaces(argv->nveths, argv->veths) < 0)
         return -1;
 
     /* this function will only return if an error occured */
@@ -320,7 +322,6 @@ static int lxcContainerChild( void *data )
 
 /**
  * lxcContainerStart:
- * @conn: pointer to connection
  * @driver: pointer to driver structure
  * @vm: pointer to virtual machine structure
  *
@@ -328,8 +329,9 @@ static int lxcContainerChild( void *data )
  *
  * Returns PID of container on success or -1 in case of error
  */
-int lxcContainerStart(virConnectPtr conn,
-                      lxc_vm_def_t *def,
+int lxcContainerStart(lxc_vm_def_t *def,
+                      unsigned int nveths,
+                      char **veths,
                       int control,
                       char *ttyPath)
 {
@@ -337,12 +339,11 @@ int lxcContainerStart(virConnectPtr conn,
     int flags;
     int stacksize = getpagesize() * 4;
     char *stack, *stacktop;
-    lxc_child_argv_t args = { def, control, ttyPath };
+    lxc_child_argv_t args = { def, nveths, veths, control, ttyPath };
 
     /* allocate a stack for the container */
     if (VIR_ALLOC_N(stack, stacksize) < 0) {
-        lxcError(conn, NULL, VIR_ERR_NO_MEMORY,
-                 _("unable to allocate container stack"));
+        lxcError(NULL, NULL, VIR_ERR_NO_MEMORY, NULL);
         return -1;
     }
     stacktop = stack + stacksize;
@@ -357,7 +358,7 @@ int lxcContainerStart(virConnectPtr conn,
     DEBUG("clone() returned, %d", pid);
 
     if (pid < 0) {
-        lxcError(conn, NULL, VIR_ERR_INTERNAL_ERROR,
+        lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR,
                  _("clone() failed, %s"), strerror(errno));
         return -1;
     }

src/lxc_container.h

@@ -32,11 +32,11 @@ enum {
     LXC_CONTAINER_FEATURE_NET = (1 << 0),
 };
 
-int lxcContainerSendContinue(virConnectPtr conn,
-                             int control);
+int lxcContainerSendContinue(int control);
 
-int lxcContainerStart(virConnectPtr conn,
-                      lxc_vm_def_t *def,
+int lxcContainerStart(lxc_vm_def_t *def,
+                      unsigned int nveths,
+                      char **veths,
                       int control,
                       char *ttyPath);
 

src/lxc_controller.c

@@ -26,16 +26,27 @@
 #ifdef WITH_LXC
 
 #include <sys/epoll.h>
+#include <sys/wait.h>
+#include <sys/socket.h>
 #include <unistd.h>
+#include <paths.h>
+#include <fcntl.h>
+#include <signal.h>
 
 #include "internal.h"
 #include "util.h"
 
 #include "lxc_conf.h"
+#include "lxc_container.h"
 #include "lxc_controller.h"
+#include "veth.h"
+#include "memory.h"
+#include "util.h"
 
 
 #define DEBUG(fmt,...) VIR_DEBUG(__FILE__, fmt, __VA_ARGS__)
+#define DEBUG0(msg) VIR_DEBUG(__FILE__, "%s", msg)
+
 
 /**
  * lxcFdForward:
@@ -91,7 +102,10 @@ typedef struct _lxcTtyForwardFd_t {
  *
  * Returns 0 on success or -1 in case of error
  */
-int lxcControllerMain(int appPty, int contPty)
+static int lxcControllerMain(int monitor,
+                             int client,
+                             int appPty,
+                             int contPty)
 {
     int rc = -1;
     int epollFd;
@@ -120,41 +134,77 @@ int lxcControllerMain(int appPty, int contPty)
     memset(&epollEvent, 0x00, sizeof(epollEvent));
     epollEvent.events = EPOLLIN|EPOLLET;    /* edge triggered */
     epollEvent.data.fd = appPty;
-    epollEvent.data.u32 = 0;                /* fdArray position */
     if (0 > epoll_ctl(epollFd, EPOLL_CTL_ADD, appPty, &epollEvent)) {
         lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR,
                  _("epoll_ctl(appPty) failed: %s"), strerror(errno));
         goto cleanup;
     }
     epollEvent.data.fd = contPty;
-    epollEvent.data.u32 = 1;                /* fdArray position */
     if (0 > epoll_ctl(epollFd, EPOLL_CTL_ADD, contPty, &epollEvent)) {
         lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR,
                  _("epoll_ctl(contPty) failed: %s"), strerror(errno));
         goto cleanup;
     }
 
+    epollEvent.events = EPOLLIN;
+    epollEvent.data.fd = monitor;
+    if (0 > epoll_ctl(epollFd, EPOLL_CTL_ADD, monitor, &epollEvent)) {
+        lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR,
+                 _("epoll_ctl(contPty) failed: %s"), strerror(errno));
+        goto cleanup;
+    }
+
+    epollEvent.events = EPOLLHUP;
+    epollEvent.data.fd = client;
+    if (0 > epoll_ctl(epollFd, EPOLL_CTL_ADD, client, &epollEvent)) {
+        lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR,
+                 _("epoll_ctl(contPty) failed: %s"), strerror(errno));
+        goto cleanup;
+    }
+
     while (1) {
         /* if active fd's, return if no events, else wait forever */
         timeout = (numActive > 0) ? 0 : -1;
         numEvents = epoll_wait(epollFd, &epollEvent, 1, timeout);
-        if (0 < numEvents) {
-            if (epollEvent.events & EPOLLIN) {
-                curFdOff = epollEvent.data.u32;
-                if (!fdArray[curFdOff].active) {
-                    fdArray[curFdOff].active = 1;
-                    ++numActive;
+        if (numEvents > 0) {
+            if (epollEvent.data.fd == monitor) {
+                int fd = accept(monitor, NULL, 0);
+                if (client != -1) { /* Already connected, so kick new one out */
+                    close(fd);
+                    continue;
                 }
-
-            } else if (epollEvent.events & EPOLLHUP) {
-                DEBUG("EPOLLHUP from fd %d", epollEvent.data.fd);
-                continue;
+                client = fd;
+                epollEvent.events = EPOLLHUP;
+                epollEvent.data.fd = client;
+                if (0 > epoll_ctl(epollFd, EPOLL_CTL_ADD, client, &epollEvent)) {
+                    lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR,
+                             _("epoll_ctl(contPty) failed: %s"), strerror(errno));
+                    goto cleanup;
+                }
+            } else if (client != -1 && epollEvent.data.fd == client) {
+                if (0 > epoll_ctl(epollFd, EPOLL_CTL_DEL, client, &epollEvent)) {
+                    lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR,
+                             _("epoll_ctl(contPty) failed: %s"), strerror(errno));
+                    goto cleanup;
+                }
+                close(client);
+                client = -1;
             } else {
-                lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR,
-                         _("error event %d"), epollEvent.events);
-                goto cleanup;
+                if (epollEvent.events & EPOLLIN) {
+                    curFdOff = epollEvent.data.fd == appPty ? 0 : 1;
+                    if (!fdArray[curFdOff].active) {
+                        fdArray[curFdOff].active = 1;
+                        ++numActive;
+                    }
+                } else if (epollEvent.events & EPOLLHUP) {
+                    DEBUG("EPOLLHUP from fd %d", epollEvent.data.fd);
+                    continue;
+                } else {
+                    lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR,
+                             _("error event %d"), epollEvent.events);
+                    goto cleanup;
+                }
             }
-
         } else if (0 == numEvents) {
             if (2 == numActive) {
                 /* both fds active, toggle between the two */
@@ -202,4 +252,255 @@ cleanup:
     return rc;
 }
 
+
+
+/**
+ * lxcControllerMoveInterfaces
+ * @nveths: number of interfaces
+ * @veths: interface names
+ * @container: pid of container
+ *
+ * Moves network interfaces into a container's namespace
+ *
+ * Returns 0 on success or -1 in case of error
+ */
+static int lxcControllerMoveInterfaces(unsigned int nveths,
+                                       char **veths,
+                                       pid_t container)
+{
+    unsigned int i;
+    for (i = 0 ; i < nveths ; i++)
+        if (moveInterfaceToNetNs(veths[i], container) < 0) {
+            lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR,
+                     _("failed to move interface %s to ns %d"),
+                     veths[i], container);
+            return -1;
+        }
+
+    return 0;
+}
+
+
+/**
+ * lxcCleanupInterfaces:
+ * @conn: pointer to connection
+ * @vm: pointer to virtual machine structure
+ *
+ * Cleans up the container interfaces by deleting the veth device pairs.
+ *
+ * Returns 0 on success or -1 in case of error
+ */
+static int lxcControllerCleanupInterfaces(unsigned int nveths,
+                                          char **veths)
+{
+    unsigned int i;
+    for (i = 0 ; i < nveths ; i++)
+        if (vethDelete(veths[i]) < 0)
+            lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR,
+                     _("failed to delete veth: %s"), veths[i]);
+            /* will continue to try to cleanup any other interfaces */
+
+    return 0;
+}
+
+
+static int
+lxcControllerRun(const char *stateDir,
+                 lxc_vm_def_t *def,
+                 unsigned int nveths,
+                 char **veths,
+                 int monitor,
+                 int client,
+                 int appPty)
+{
+    int rc = -1;
+    int control[2] = { -1, -1};
+    int containerPty;
+    char *containerPtyPath;
+    pid_t container = -1;
+
+    if (socketpair(PF_UNIX, SOCK_STREAM, 0, control) < 0) {
+        lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR,
+                 _("sockpair failed: %s"), strerror(errno));
+        goto cleanup;
+    }
+
+    if (virFileOpenTty(&containerPty,
+                       &containerPtyPath,
+                       0) < 0) {
+        lxcError(NULL, NULL, VIR_ERR_INTERNAL_ERROR,
+                 _("failed to allocate tty: %s"), strerror(errno));
+        goto cleanup;
+    }
+
+    if ((container = lxcContainerStart(def,
+                                       nveths,
+                                       veths,
+                                       control[1],
+                                       containerPtyPath)) < 0)
+        goto cleanup;
+    close(control[1]);
+    control[1] = -1;
+
+    if (lxcControllerMoveInterfaces(nveths, veths, container) < 0)
+        goto cleanup;
+
+    if (lxcContainerSendContinue(control[0]) < 0)
+        goto cleanup;
+
+    rc = lxcControllerMain(monitor, client, appPty, containerPty);
+
+cleanup:
+    if (control[0] != -1)
+        close(control[0]);
+    if (control[1] != -1)
+        close(control[1]);
+    VIR_FREE(containerPtyPath);
+    if (containerPty != -1)
+        close(containerPty);
+
+    kill(container, SIGTERM);
+    waitpid(container, NULL, 0);
+    lxcControllerCleanupInterfaces(nveths, veths);
+    virFileDeletePid(stateDir, def->name);
+    return rc;
+}
+
+
+int lxcControllerStart(const char *stateDir,
+                       lxc_vm_def_t *def,
+                       unsigned int nveths,
+                       char **veths,
+                       int monitor,
+                       int appPty,
+                       int logfd)
+{
+    pid_t pid;
+    int rc;
+    int status, null;
+    int open_max, i;
+    int client;
+    struct sigaction sig_action;
+
+    if ((pid = fork()) < 0)
+        return -1;
+
+    if (pid > 0) {
+        /* Original caller waits for first child to exit */
+        while (1) {
+            rc = waitpid(pid, &status, 0);
+            if (rc < 0) {
+                if (errno == EINTR)
+                    continue;
+                return -1;
+            }
+            if (rc != pid) {
+                fprintf(stderr,
+                        _("Unexpected pid %d != %d from waitpid\n"),
+                        rc, pid);
+                return -1;
+            }
+            if (WIFEXITED(status) &&
+                WEXITSTATUS(status) == 0)
+                return 0;
+            else {
+                fprintf(stderr,
+                        _("Unexpected status %d from pid %d\n"),
+                        status, pid);
+                return -1;
+            }
+        }
+    }
+
+    /* First child is running here */
+
+    /* Clobber all libvirtd's signal handlers so they
+     * don't affect us
+     */
+    sig_action.sa_handler = SIG_DFL;
+    sig_action.sa_flags = 0;
+    sigemptyset(&sig_action.sa_mask);
+
+    sigaction(SIGHUP, &sig_action, NULL);
+    sigaction(SIGINT, &sig_action, NULL);
+    sigaction(SIGQUIT, &sig_action, NULL);
+    sigaction(SIGTERM, &sig_action, NULL);
+    sigaction(SIGCHLD, &sig_action, NULL);
+
+    sig_action.sa_handler = SIG_IGN;
+    sigaction(SIGPIPE, &sig_action, NULL);
+
+
+    /* Don't hold onto any cwd we inherit from libvirtd either */
+    if (chdir("/") < 0) {
+        fprintf(stderr, _("Unable to change to root dir: %s\n"),
+                strerror(errno));
+        _exit(-1);
+    }
+
+    if (setsid() < 0) {
+        fprintf(stderr, _("Unable to become session leader: %s\n"),
+                strerror(errno));
+        _exit(-1);
+    }
+
+    if ((null = open(_PATH_DEVNULL, O_RDONLY)) < 0) {
+        fprintf(stderr, _("Unable to open %s: %s\n"),
+                _PATH_DEVNULL, strerror(errno));
+        _exit(-1);
+    }
+
+    open_max = sysconf (_SC_OPEN_MAX);
+    for (i = 0; i < open_max; i++)
+        if (i != appPty &&
+            i != monitor &&
+            i != logfd &&
+            i != null)
+            close(i);
+
+    if (dup2(null, STDIN_FILENO) < 0 ||
+        dup2(logfd, STDOUT_FILENO) < 0 ||
+        dup2(logfd, STDERR_FILENO) < 0) {
+        fprintf(stderr, _("Unable to redirect stdio: %s\n"),
+                strerror(errno));
+        _exit(-1);
+    }
+
+    close(null);
+    close(logfd);
+
+    /* Now fork the real controller process */
+    if ((pid = fork()) < 0) {
+        fprintf(stderr, _("Unable to fork controller: %s\n"),
+                strerror(errno));
+        _exit(-1);
+    }
+
+    if (pid > 0) {
+        if ((rc = virFileWritePid(stateDir, def->name, pid)) != 0) {
+            fprintf(stderr, _("Unable to write pid file: %s\n"),
+                    strerror(rc));
+            _exit(-1);
+        }
+        /* First child now exits, allowing originall caller to
+         * complete their waitpid & continue */
+        _exit(0);
+    }
+
+    /* This is real controller running finally... */
+
+    /* Accept initial client which is the libvirtd daemon */
+    if ((client = accept(monitor, NULL, 0))) {
+        fprintf(stderr, _("Failed connection from LXC driver: %s\n"),
+                strerror(errno));
+        _exit(-1);
+    }
+
+    /* Controlling libvirtd LXC driver now knows
+       what our PID is, and is able to cleanup after
+       us from now on */
+    _exit(lxcControllerRun(stateDir, def, nveths, veths, monitor, client, appPty));
+}
+
+
 #endif

src/lxc_controller.h

@@ -26,7 +26,15 @@
 
 #ifdef WITH_LXC
 
-int lxcControllerMain(int appPty, int contPty);
+#include "lxc_conf.h"
+
+int lxcControllerStart(const char *stateDir,
+                       lxc_vm_def_t *def,
+                       unsigned int nveths,
+                       char **veths,
+                       int monitor,
+                       int appPty,
+                       int logfd);
 
 #endif /* WITH_LXC */
 

src/util.c

@@ -37,6 +37,9 @@
 #include <sys/wait.h>
 #endif
 #include <string.h>
+#if HAVE_TERMIOS_H
+#include <termios.h>
+#endif
 #include "c-ctype.h"
 
 #ifdef HAVE_PATHS_H
@@ -472,6 +475,169 @@ int virFileBuildPath(const char *dir,
     return 0;
 }
 
+
+#ifdef __linux__
+int virFileOpenTty(int *ttymaster,
+                   char **ttyName,
+                   int rawmode)
+{
+    int rc = -1;
+
+    if ((*ttymaster = posix_openpt(O_RDWR|O_NOCTTY|O_NONBLOCK)) < 0)
+        goto cleanup;
+
+    if (unlockpt(*ttymaster) < 0)
+        goto cleanup;
+
+    if (grantpt(*ttymaster) < 0)
+        goto cleanup;
+
+    if (rawmode) {
+        struct termios ttyAttr;
+        if (tcgetattr(*ttymaster, &ttyAttr) < 0)
+            goto cleanup;
+
+        cfmakeraw(&ttyAttr);
+
+        if (tcsetattr(*ttymaster, TCSADRAIN, &ttyAttr) < 0)
+            goto cleanup;
+    }
+
+    if (ttyName) {
+        char tempTtyName[PATH_MAX];
+        if (ptsname_r(*ttymaster, tempTtyName, sizeof(tempTtyName)) < 0)
+            goto cleanup;
+
+        if ((*ttyName = strdup(tempTtyName)) == NULL) {
+            errno = ENOMEM;
+            goto cleanup;
+        }
+    }
+
+    rc = 0;
+
+cleanup:
+    if (rc != 0 &&
+        *ttymaster != -1) {
+        close(*ttymaster);
+    }
+
+    return rc;
+
+}
+#else
+int virFileOpenTty(int *ttymaster ATTRIBUTE_UNUSED,
+                   char **ttyName ATTRIBUTE_UNUSED,
+                   int rawmode ATTRIBUTE_UNUSED)
+{
+    return -1;
+}
+#endif
+
+
+int virFileWritePid(const char *dir,
+                    const char *name,
+                    pid_t pid)
+{
+    int rc;
+    int fd;
+    FILE *file = NULL;
+    char *pidfile = NULL;
+
+    if ((rc = virFileMakePath(dir)))
+        goto cleanup;
+
+    if (asprintf(&pidfile, "%s/%s.pid", dir, name) < 0) {
+        rc = ENOMEM;
+        goto cleanup;
+    }
+
+    if ((fd = open(pidfile,
+                   O_WRONLY | O_CREAT | O_TRUNC,
+                   S_IRUSR | S_IWUSR)) < 0) {
+        rc = errno;
+        goto cleanup;
+    }
+
+    if (!(file = fdopen(fd, "w"))) {
+        rc = errno;
+        close(fd);
+        goto cleanup;
+    }
+
+    if (fprintf(file, "%d", pid) < 0) {
+        rc = errno;
+        goto cleanup;
+    }
+
+    rc = 0;
+
+cleanup:
+    if (file &&
+        fclose(file) < 0) {
+        rc = errno;
+    }
+
+    VIR_FREE(pidfile);
+    return rc;
+}
+
+int virFileReadPid(const char *dir,
+                   const char *name,
+                   pid_t *pid)
+{
+    int rc;
+    FILE *file;
+    char *pidfile = NULL;
+    *pid = 0;
+    if (asprintf(&pidfile, "%s/%s.pid", dir, name) < 0) {
+        rc = ENOMEM;
+        goto cleanup;
+    }
+
+    if (!(file = fopen(pidfile, "r"))) {
+        rc = errno;
+        goto cleanup;
+    }
+
+    if (fscanf(file, "%d", pid) != 1) {
+        rc = EINVAL;
+        goto cleanup;
+    }
+
+    if (fclose(file) < 0) {
+        rc = errno;
+        goto cleanup;
+    }
+
+    rc = 0;
+
+ cleanup:
+    VIR_FREE(pidfile);
+    return rc;
+}
+
+int virFileDeletePid(const char *dir,
+                     const char *name)
+{
+    int rc = 0;
+    char *pidfile = NULL;
+
+    if (asprintf(&pidfile, "%s/%s.pid", dir, name) < 0) {
+        rc = errno;
+        goto cleanup;
+    }
+
+    if (unlink(pidfile) < 0 && errno != ENOENT)
+        rc = errno;
+
+cleanup:
+    VIR_FREE(pidfile);
+    return rc;
+}
+
+
+
 /* Like strtol, but produce an "int" result, and check more carefully.
    Return 0 upon success;  return -1 to indicate failure.
    When END_PTR is NULL, the byte after the final valid digit must be NUL.

src/util.h

@@ -58,6 +58,18 @@ int virFileBuildPath(const char *dir,
                      char *buf,
                      unsigned int buflen);
 
+int virFileOpenTty(int *ttymaster,
+                   char **ttyName,
+                   int rawmode);
+
+int virFileWritePid(const char *dir,
+                    const char *name,
+                    pid_t pid);
+int virFileReadPid(const char *dir,
+                   const char *name,
+                   pid_t *pid);
+int virFileDeletePid(const char *dir,
+                     const char *name);
 
 int __virStrToLong_i(char const *s,
                      char **end_ptr,