Run an RPC protocol over the LXC controller monitor

This defines a new RPC protocol to be used between the LXC
controller and the libvirtd LXC driver. There is only a
single RPC message defined thus far, an asynchronous "EXIT"
event that is emitted just before the LXC controller process
exits. This provides the LXC driver with details about how
the container shutdown - normally, or abnormally (crashed),
thus allowing the driver to emit better libvirt events.

Emitting the event in the LXC controller requires a few
little tricks with the RPC service. Simply calling the
virNetServiceClientSendMessage does not work, since this
merely queues the message for asynchronous processing.
In addition the main event loop is no longer running at
the point the event is emitted, so no I/O is processed.

Thus after invoking virNetServiceClientSendMessage it is
necessary to mark the client as being in "delayed close"
mode. Then the event loop is run again, until the client
completes its close - this happens only after the queued
message has been fully transmitted. The final complexity
is that it is not safe to run virNetServerQuit() from the
client close callback, since that is invoked from a
context where the server is locked. Thus a zero-second
timer is used to trigger shutdown of the event loop,
causing the controller to finally exit.

* src/Makefile.am: Add rules for generating RPC protocol
  files and dispatch methods
* src/lxc/lxc_controller.c: Emit an RPC event immediately
  before exiting
* src/lxc/lxc_domain.h: Record the shutdown reason
  given by the controller
* src/lxc/lxc_monitor.c, src/lxc/lxc_monitor.h: Register
  RPC program and event handler. Add callback to let
  driver receive EXIT event.
* src/lxc/lxc_process.c: Use monitor exit event to decide
  what kind of domain event to emit
* src/lxc/lxc_protocol.x: Define wire protocol for LXC
  controller monitor.
Signed-off-by: NDaniel P. Berrange <berrange@redhat.com>

.gitignore

@@ -105,6 +105,10 @@
 /src/locking/qemu-sanlock.conf
 /src/locking/test_libvirt_sanlock.aug
 /src/lxc/test_libvirtd_lxc.aug
+/src/lxc/lxc_controller_dispatch.h
+/src/lxc/lxc_monitor_dispatch.h
+/src/lxc/lxc_protocol.c
+/src/lxc/lxc_protocol.h
 /src/qemu/test_libvirtd_qemu.aug
 /src/remote/*_client_bodies.h
 /src/remote/*_protocol.[ch]

src/Makefile.am

@@ -387,7 +387,47 @@ if WITH_XEN_INOTIFY
 XEN_DRIVER_SOURCES += xen/xen_inotify.c xen/xen_inotify.h
 endif
 
+LXC_PROTOCOL_GENERATED = \
+	$(srcdir)/lxc/lxc_protocol.h \
+	$(srcdir)/lxc/lxc_protocol.c \
+	$(NULL)
+
+LXC_MONITOR_GENERATED = \
+	$(srcdir)/lxc/lxc_monitor_dispatch.h \
+	$(NULL)
+
+LXC_CONTROLLER_GENERATED = \
+	$(srcdir)/lxc/lxc_controller_dispatch.h \
+	$(NULL)
+
+LXC_GENERATED = \
+	$(LXC_PROTOCOL_GENERATED) \
+	$(LXC_MONITOR_GENERATED) \
+	$(LXC_CONTROLLER_GENERATED) \
+	$(NULL)
+
+LXC_PROTOCOL = $(srcdir)/lxc/lxc_protocol.x
+
+$(srcdir)/lxc/lxc_monitor_dispatch.h: $(srcdir)/rpc/gendispatch.pl \
+		$(LXC_PROTOCOL)
+	$(AM_V_GEN)$(PERL) -w $(srcdir)/rpc/gendispatch.pl \
+	  -k virLXCProtocol VIR_LXC_PROTOCOL $(LXC_PROTOCOL) > $@
+
+$(srcdir)/lxc/lxc_controller_dispatch.h: $(srcdir)/rpc/gendispatch.pl \
+		$(REMOTE_PROTOCOL)
+	$(AM_V_GEN)$(PERL) -w $(srcdir)/rpc/gendispatch.pl \
+	  -b virLXCProtocol VIR_LXC_PROTOCOL $(LXC_PROTOCOL) > $@
+
+EXTRA_DIST += \
+	$(LXC_PROTOCOL) \
+	$(LXC_GENERATED) \
+	$(NULL)
+
+BUILT_SOURCES += $(LXC_GENERATED)
+
 LXC_DRIVER_SOURCES =						\
+		$(LXC_PROTOCOL_GENERATED)			\
+		$(LXC_MONITOR_GENERATED)			\
 		lxc/lxc_conf.c lxc/lxc_conf.h			\
 		lxc/lxc_container.c lxc/lxc_container.h		\
 		lxc/lxc_cgroup.c lxc/lxc_cgroup.h		\
@@ -397,6 +437,8 @@ LXC_DRIVER_SOURCES =						\
 		lxc/lxc_driver.c lxc/lxc_driver.h
 
 LXC_CONTROLLER_SOURCES =					\
+		$(LXC_PROTOCOL_GENERATED)			\
+		$(LXC_CONTROLLER_GENERATED)			\
 		lxc/lxc_conf.c lxc/lxc_conf.h			\
 		lxc/lxc_container.c lxc/lxc_container.h		\
 		lxc/lxc_cgroup.c lxc/lxc_cgroup.h		\

src/lxc/lxc_controller.c

@@ -59,6 +59,7 @@
 #include "lxc_conf.h"
 #include "lxc_container.h"
 #include "lxc_cgroup.h"
+#include "lxc_protocol.h"
 #include "virnetdev.h"
 #include "virnetdevveth.h"
 #include "memory.h"
@@ -121,10 +122,25 @@ struct _virLXCController {
 
     /* Server socket */
     virNetServerPtr server;
+    virNetServerClientPtr client;
+    virNetServerProgramPtr prog;
+    bool inShutdown;
+    int timerShutdown;
 };
 
+#include "lxc_controller_dispatch.h"
+
 static void virLXCControllerFree(virLXCControllerPtr ctrl);
 
+static void virLXCControllerQuitTimer(int timer ATTRIBUTE_UNUSED, void *opaque)
+{
+    virLXCControllerPtr ctrl = opaque;
+
+    VIR_DEBUG("Triggering event loop quit");
+    virNetServerQuit(ctrl->server);
+}
+
+
 static virLXCControllerPtr virLXCControllerNew(const char *name)
 {
     virLXCControllerPtr ctrl = NULL;
@@ -134,6 +150,8 @@ static virLXCControllerPtr virLXCControllerNew(const char *name)
     if (VIR_ALLOC(ctrl) < 0)
         goto no_memory;
 
+    ctrl->timerShutdown = -1;
+
     if (!(ctrl->name = strdup(name)))
         goto no_memory;
 
@@ -150,6 +168,11 @@ static virLXCControllerPtr virLXCControllerNew(const char *name)
                                            0)) == NULL)
         goto error;
 
+    if ((ctrl->timerShutdown = virEventAddTimeout(-1,
+                                                  virLXCControllerQuitTimer, ctrl,
+                                                  NULL)) < 0)
+        goto error;
+
 cleanup:
     VIR_FREE(configFile);
     virCapabilitiesFree(caps);
@@ -238,6 +261,9 @@ static void virLXCControllerFree(virLXCControllerPtr ctrl)
     virDomainDefFree(ctrl->def);
     VIR_FREE(ctrl->name);
 
+    if (ctrl->timerShutdown != -1)
+        virEventRemoveTimeout(ctrl->timerShutdown);
+
     virNetServerFree(ctrl->server);
 
     VIR_FREE(ctrl);
@@ -539,12 +565,28 @@ static int virLXCControllerSetupResourceLimits(virLXCControllerPtr ctrl)
 }
 
 
+static void virLXCControllerClientCloseHook(virNetServerClientPtr client)
+{
+    virLXCControllerPtr ctrl = virNetServerClientGetPrivateData(client);
+
+    VIR_DEBUG("Client %p has closed", client);
+    if (ctrl->client == client)
+        ctrl->client = NULL;
+    if (ctrl->inShutdown) {
+        VIR_DEBUG("Arm timer to quit event loop");
+        virEventUpdateTimeout(ctrl->timerShutdown, 0);
+    }
+}
+
 static int virLXCControllerClientHook(virNetServerPtr server ATTRIBUTE_UNUSED,
                                       virNetServerClientPtr client,
                                       void *opaque)
 {
     virLXCControllerPtr ctrl = opaque;
     virNetServerClientSetPrivateData(client, ctrl, NULL);
+    virNetServerClientSetCloseHook(client, virLXCControllerClientCloseHook);
+    VIR_DEBUG("Got new client %p", client);
+    ctrl->client = client;
     return 0;
 }
 
@@ -581,6 +623,12 @@ static int virLXCControllerSetupServer(virLXCControllerPtr ctrl)
     virNetServerServiceFree(svc);
     svc = NULL;
 
+    if (!(ctrl->prog = virNetServerProgramNew(VIR_LXC_PROTOCOL_PROGRAM,
+                                              VIR_LXC_PROTOCOL_PROGRAM_VERSION,
+                                              virLXCProtocolProcs,
+                                              virLXCProtocolNProcs)))
+        goto error;
+
     virNetServerUpdateServices(ctrl->server, true);
     VIR_FREE(sockpath);
     return 0;
@@ -1219,6 +1267,79 @@ virLXCControllerSetupConsoles(virLXCControllerPtr ctrl,
 }
 
 
+static void
+virLXCControllerEventSend(virLXCControllerPtr ctrl,
+                          int procnr,
+                          xdrproc_t proc,
+                          void *data)
+{
+    virNetMessagePtr msg;
+
+    if (!ctrl->client)
+        return;
+
+    VIR_DEBUG("Send event %d client=%p", procnr, ctrl->client);
+    if (!(msg = virNetMessageNew(false)))
+        goto error;
+
+    msg->header.prog = virNetServerProgramGetID(ctrl->prog);
+    msg->header.vers = virNetServerProgramGetVersion(ctrl->prog);
+    msg->header.proc = procnr;
+    msg->header.type = VIR_NET_MESSAGE;
+    msg->header.serial = 1;
+    msg->header.status = VIR_NET_OK;
+
+    if (virNetMessageEncodeHeader(msg) < 0)
+        goto error;
+
+    if (virNetMessageEncodePayload(msg, proc, data) < 0)
+        goto error;
+
+    VIR_DEBUG("Queue event %d %zu", procnr, msg->bufferLength);
+    virNetServerClientSendMessage(ctrl->client, msg);
+
+    xdr_free(proc, data);
+    return;
+
+error:
+    virNetMessageFree(msg);
+    xdr_free(proc, data);
+}
+
+
+static int
+virLXCControllerEventSendExit(virLXCControllerPtr ctrl,
+                              int exitstatus)
+{
+    virLXCProtocolExitEventMsg msg;
+
+    VIR_DEBUG("Exit status %d", exitstatus);
+    memset(&msg, 0, sizeof(msg));
+    switch (exitstatus) {
+    case 0:
+        msg.status = VIR_LXC_PROTOCOL_EXIT_STATUS_SHUTDOWN;
+        break;
+    default:
+        msg.status = VIR_LXC_PROTOCOL_EXIT_STATUS_ERROR;
+        break;
+    }
+
+    virLXCControllerEventSend(ctrl,
+                              VIR_LXC_PROTOCOL_PROC_EXIT_EVENT,
+                              (xdrproc_t)xdr_virLXCProtocolExitEventMsg,
+                              (void*)&msg);
+
+    if (ctrl->client) {
+        VIR_DEBUG("Waiting for client to complete dispatch");
+        ctrl->inShutdown = true;
+        virNetServerClientDelayedClose(ctrl->client);
+        virNetServerRun(ctrl->server);
+    }
+    VIR_DEBUG("Client has gone away");
+    return 0;
+}
+
+
 static int
 virLXCControllerRun(virLXCControllerPtr ctrl)
 {
@@ -1306,6 +1427,8 @@ virLXCControllerRun(virLXCControllerPtr ctrl)
 
     rc = virLXCControllerMain(ctrl);
 
+    virLXCControllerEventSendExit(ctrl, rc);
+
 cleanup:
     VIR_FORCE_CLOSE(control[0]);
     VIR_FORCE_CLOSE(control[1]);
@@ -1527,5 +1650,5 @@ cleanup:
 
     virLXCControllerFree(ctrl);
 
-    return rc ? EXIT_FAILURE : EXIT_SUCCESS;
+    return rc < 0? EXIT_FAILURE : EXIT_SUCCESS;
 }

src/lxc/lxc_domain.h

@@ -32,6 +32,7 @@ typedef virLXCDomainObjPrivate *virLXCDomainObjPrivatePtr;
 struct _virLXCDomainObjPrivate {
     virLXCMonitorPtr monitor;
     bool doneStopEvent;
+    int stopReason;
 };
 
 void virLXCDomainSetPrivateDataHooks(virCapsPtr caps);

src/lxc/lxc_monitor.c

@@ -22,6 +22,8 @@
 
 #include "lxc_monitor.h"
 #include "lxc_conf.h"
+#include "lxc_protocol.h"
+#include "lxc_monitor_dispatch.h"
 
 #include "memory.h"
 
@@ -41,9 +43,35 @@ struct _virLXCMonitor {
     virLXCMonitorCallbacksPtr cb;
 
     virNetClientPtr client;
+    virNetClientProgramPtr program;
 };
 
 static void virLXCMonitorFree(virLXCMonitorPtr mon);
+static void
+virLXCMonitorHandleEventExit(virNetClientProgramPtr prog,
+                             virNetClientPtr client,
+                             void *evdata, void *opaque);
+
+static virNetClientProgramEvent virLXCProtocolEvents[] = {
+    { VIR_LXC_PROTOCOL_PROC_EXIT_EVENT,
+      virLXCMonitorHandleEventExit,
+      sizeof(virLXCProtocolExitEventMsg),
+      (xdrproc_t)xdr_virLXCProtocolExitEventMsg },
+};
+
+
+static void
+virLXCMonitorHandleEventExit(virNetClientProgramPtr prog ATTRIBUTE_UNUSED,
+                             virNetClientPtr client ATTRIBUTE_UNUSED,
+                             void *evdata, void *opaque)
+{
+    virLXCMonitorPtr mon = opaque;
+    virLXCProtocolExitEventMsg *msg = evdata;
+
+    VIR_DEBUG("Event exit %d", msg->status);
+    if (mon->cb->exitNotify)
+        mon->cb->exitNotify(mon, msg->status, mon->vm);
+}
 
 
 static void virLXCMonitorEOFNotify(virNetClientPtr client ATTRIBUTE_UNUSED,
@@ -54,6 +82,7 @@ static void virLXCMonitorEOFNotify(virNetClientPtr client ATTRIBUTE_UNUSED,
     virLXCMonitorCallbackEOFNotify eofNotify;
     virDomainObjPtr vm;
 
+    VIR_DEBUG("EOF notify");
     virLXCMonitorLock(mon);
     eofNotify = mon->cb->eofNotify;
     vm = mon->vm;
@@ -101,6 +130,17 @@ virLXCMonitorPtr virLXCMonitorNew(virDomainObjPtr vm,
         goto error;
 
 
+    if (!(mon->program = virNetClientProgramNew(VIR_LXC_PROTOCOL_PROGRAM,
+                                                VIR_LXC_PROTOCOL_PROGRAM_VERSION,
+                                                virLXCProtocolEvents,
+                                                ARRAY_CARDINALITY(virLXCProtocolEvents),
+                                                mon)))
+        goto error;
+
+    if (virNetClientAddProgram(mon->client,
+                               mon->program) < 0)
+        goto error;
+
     mon->vm = vm;
     mon->cb = cb;
 
@@ -130,6 +170,7 @@ static void virLXCMonitorFree(virLXCMonitorPtr mon)
     if (mon->cb && mon->cb->destroy)
         (mon->cb->destroy)(mon, mon->vm);
     virMutexDestroy(&mon->lock);
+    virNetClientProgramFree(mon->program);
     VIR_FREE(mon);
 }
 

src/lxc/lxc_monitor.h

@@ -22,6 +22,7 @@
 # define __LXC_MONITOR_H__
 
 # include "domain_conf.h"
+# include "lxc_protocol.h"
 
 typedef struct _virLXCMonitor virLXCMonitor;
 typedef virLXCMonitor *virLXCMonitorPtr;
@@ -34,9 +35,14 @@ typedef void (*virLXCMonitorCallbackDestroy)(virLXCMonitorPtr mon,
 typedef void (*virLXCMonitorCallbackEOFNotify)(virLXCMonitorPtr mon,
                                                virDomainObjPtr vm);
 
+typedef void (*virLXCMonitorCallbackExitNotify)(virLXCMonitorPtr mon,
+                                                virLXCProtocolExitStatus status,
+                                                virDomainObjPtr vm);
+
 struct _virLXCMonitorCallbacks {
     virLXCMonitorCallbackDestroy destroy;
     virLXCMonitorCallbackEOFNotify eofNotify;
+    virLXCMonitorCallbackExitNotify exitNotify;
 };
 
 virLXCMonitorPtr virLXCMonitorNew(virDomainObjPtr vm,

src/lxc/lxc_process.c

@@ -167,6 +167,9 @@ static void virLXCProcessCleanup(virLXCDriverPtr driver,
     virLXCDomainObjPrivatePtr priv = vm->privateData;
     virNetDevVPortProfilePtr vport = NULL;
 
+    VIR_DEBUG("Stopping VM name=%s pid=%d reason=%d",
+              vm->def->name, (int)vm->pid, (int)reason);
+
     /* now that we know it's stopped call the hook if present */
     if (virHookPresent(VIR_HOOK_DRIVER_LXC)) {
         char *xml = virDomainDefFormat(vm->def, 0);
@@ -509,7 +512,7 @@ static void virLXCProcessMonitorEOFNotify(virLXCMonitorPtr mon ATTRIBUTE_UNUSED,
     if (!priv->doneStopEvent) {
         event = virDomainEventNewFromObj(vm,
                                          VIR_DOMAIN_EVENT_STOPPED,
-                                         VIR_DOMAIN_EVENT_STOPPED_SHUTDOWN);
+                                         priv->stopReason);
         virDomainAuditStop(vm, "shutdown");
     } else {
         VIR_DEBUG("Stop event has already been sent");
@@ -528,10 +531,31 @@ static void virLXCProcessMonitorEOFNotify(virLXCMonitorPtr mon ATTRIBUTE_UNUSED,
     }
 }
 
+static void virLXCProcessMonitorExitNotify(virLXCMonitorPtr mon ATTRIBUTE_UNUSED,
+                                           virLXCProtocolExitStatus status,
+                                           virDomainObjPtr vm)
+{
+    virLXCDomainObjPrivatePtr priv = vm->privateData;
+
+    switch (status) {
+    case VIR_LXC_PROTOCOL_EXIT_STATUS_SHUTDOWN:
+        priv->stopReason = VIR_DOMAIN_EVENT_STOPPED_SHUTDOWN;
+        break;
+    case VIR_LXC_PROTOCOL_EXIT_STATUS_ERROR:
+        priv->stopReason = VIR_DOMAIN_EVENT_STOPPED_FAILED;
+        break;
+    default:
+        priv->stopReason = VIR_DOMAIN_EVENT_STOPPED_FAILED;
+        break;
+    }
+    VIR_DEBUG("Domain shutoff reason %d (from status %d)",
+              priv->stopReason, status);
+}
 
 static virLXCMonitorCallbacks monitorCallbacks = {
     .eofNotify = virLXCProcessMonitorEOFNotify,
     .destroy = virLXCProcessMonitorDestroy,
+    .exitNotify = virLXCProcessMonitorExitNotify,
 };
 
 
@@ -573,6 +597,8 @@ int virLXCProcessStop(virLXCDriverPtr driver,
     virCgroupPtr group = NULL;
     int rc;
 
+    VIR_DEBUG("Stopping VM name=%s pid=%d reason=%d",
+              vm->def->name, (int)vm->pid, (int)reason);
     if (vm->pid <= 0) {
         virReportError(VIR_ERR_INTERNAL_ERROR,
                        _("Invalid PID %d for container"), vm->pid);
@@ -1006,6 +1032,7 @@ int virLXCProcessStart(virConnectPtr conn,
         goto cleanup;
     }
 
+    priv->stopReason = VIR_DOMAIN_EVENT_STOPPED_FAILED;
     vm->def->id = vm->pid;
     virDomainObjSetState(vm, VIR_DOMAIN_RUNNING, reason);
     priv->doneStopEvent = false;

src/lxc/lxc_protocol.x

+/* -*- c -*-
+ * Define wire protocol for communication between the
+ * LXC driver in libvirtd, and the LXC controller in
+ * the libvirt_lxc helper program.
+ */
+
+enum virLXCProtocolExitStatus {
+    VIR_LXC_PROTOCOL_EXIT_STATUS_ERROR,
+    VIR_LXC_PROTOCOL_EXIT_STATUS_SHUTDOWN
+};
+
+struct virLXCProtocolExitEventMsg {
+    enum virLXCProtocolExitStatus status;
+};
+
+const VIR_LXC_PROTOCOL_PROGRAM = 0x12341234;
+const VIR_LXC_PROTOCOL_PROGRAM_VERSION = 1;
+
+enum virLXCProtocolProcedure {
+    VIR_LXC_PROTOCOL_PROC_EXIT_EVENT = 1 /* skipgen skipgen */
+};