diff --git a/src/qemu/qemu_process.c b/src/qemu/qemu_process.c
index c8c188a10ab107acb6e43bdbdee8667b4f229f75..42da617db2869fd8ae1dc9868af6dafd35547aaa 100644
--- a/src/qemu/qemu_process.c
+++ b/src/qemu/qemu_process.c
@@ -129,7 +129,8 @@ qemuProcessHandleAgentEOF(qemuAgentPtr agent,
     virDomainObjLock(vm);
 
     priv = vm->privateData;
-    priv->agent = NULL;
+    if (priv->agent == agent)
+        priv->agent = NULL;
 
     virDomainObjUnlock(vm);
     qemuDriverUnlock(driver);
@@ -167,16 +168,9 @@ qemuProcessHandleAgentError(qemuAgentPtr agent ATTRIBUTE_UNUSED,
 static void qemuProcessHandleAgentDestroy(qemuAgentPtr agent,
                                           virDomainObjPtr vm)
 {
-    qemuDomainObjPrivatePtr priv;
-
     VIR_DEBUG("Received destroy agent=%p vm=%p", agent, vm);
 
-    virDomainObjLock(vm);
-    priv = vm->privateData;
-    if (priv->agent == agent)
-        priv->agent = NULL;
-    if (virObjectUnref(vm))
-        virDomainObjUnlock(vm);
+    virObjectUnref(vm);
 }