diff --git a/src/rpc/virnetserver.c b/src/rpc/virnetserver.c
index 2b9dd4d695f8efaf10f4cc3d449e37c8e4cfbc0d..c93c3f8a39a06705d29dd3be8335b7c735948d46 100644
--- a/src/rpc/virnetserver.c
+++ b/src/rpc/virnetserver.c
@@ -131,8 +131,6 @@ static void virNetServerHandleJob(void *jobOpaque, void *opaque)
     virNetServerProgramPtr prog = NULL;
     size_t i;
 
-    virNetServerClientRef(job->client);
-
     virNetServerLock(srv);
     VIR_DEBUG("server=%p client=%p message=%p",
               srv, job->client, job->msg);
diff --git a/src/rpc/virnetserverclient.c b/src/rpc/virnetserverclient.c
index 317d59cf559ab92bfe302a4459f5e708eb3d40e4..3c0dba8d65f9f20e7a454a852ef082c1fbf8fea9 100644
--- a/src/rpc/virnetserverclient.c
+++ b/src/rpc/virnetserverclient.c
@@ -763,10 +763,12 @@ readmore:
 
         /* Send off to for normal dispatch to workers */
         if (msg) {
+            client->refs++;
             if (!client->dispatchFunc ||
                 client->dispatchFunc(client, msg, client->dispatchOpaque) < 0) {
                 virNetMessageFree(msg);
                 client->wantClose = true;
+                client->refs--;
                 return;
             }
         }