diff --git a/hw/virtio-pci.c b/hw/virtio-pci.c
index 3ab974727628bc56254b78add9a9f5d2e01ec623..125eded9caded51ce07fbfd9aca0a42dfaf03e20 100644
--- a/hw/virtio-pci.c
+++ b/hw/virtio-pci.c
@@ -160,7 +160,7 @@ static int virtio_pci_load_queue(void * opaque, int n, QEMUFile *f)
 }
 
 static int virtio_pci_set_host_notifier_internal(VirtIOPCIProxy *proxy,
-                                                 int n, bool assign)
+                                                 int n, bool assign, bool set_handler)
 {
     VirtQueue *vq = virtio_get_queue(proxy->vdev, n);
     EventNotifier *notifier = virtio_queue_get_host_notifier(vq);
@@ -173,13 +173,13 @@ static int virtio_pci_set_host_notifier_internal(VirtIOPCIProxy *proxy,
                          __func__, r);
             return r;
         }
-        virtio_queue_set_host_notifier_fd_handler(vq, true);
+        virtio_queue_set_host_notifier_fd_handler(vq, true, set_handler);
         memory_region_add_eventfd(&proxy->bar, VIRTIO_PCI_QUEUE_NOTIFY, 2,
                                   true, n, notifier);
     } else {
         memory_region_del_eventfd(&proxy->bar, VIRTIO_PCI_QUEUE_NOTIFY, 2,
                                   true, n, notifier);
-        virtio_queue_set_host_notifier_fd_handler(vq, false);
+        virtio_queue_set_host_notifier_fd_handler(vq, false, false);
         event_notifier_cleanup(notifier);
     }
     return r;
@@ -200,7 +200,7 @@ static void virtio_pci_start_ioeventfd(VirtIOPCIProxy *proxy)
             continue;
         }
 
-        r = virtio_pci_set_host_notifier_internal(proxy, n, true);
+        r = virtio_pci_set_host_notifier_internal(proxy, n, true, true);
         if (r < 0) {
             goto assign_error;
         }
@@ -214,7 +214,7 @@ assign_error:
             continue;
         }
 
-        r = virtio_pci_set_host_notifier_internal(proxy, n, false);
+        r = virtio_pci_set_host_notifier_internal(proxy, n, false, false);
         assert(r >= 0);
     }
     proxy->ioeventfd_started = false;
@@ -235,7 +235,7 @@ static void virtio_pci_stop_ioeventfd(VirtIOPCIProxy *proxy)
             continue;
         }
 
-        r = virtio_pci_set_host_notifier_internal(proxy, n, false);
+        r = virtio_pci_set_host_notifier_internal(proxy, n, false, false);
         assert(r >= 0);
     }
     proxy->ioeventfd_started = false;
@@ -683,7 +683,7 @@ static int virtio_pci_set_host_notifier(void *opaque, int n, bool assign)
      * currently only stops on status change away from ok,
      * reset, vmstop and such. If we do add code to start here,
      * need to check vmstate, device state etc. */
-    return virtio_pci_set_host_notifier_internal(proxy, n, assign);
+    return virtio_pci_set_host_notifier_internal(proxy, n, assign, false);
 }
 
 static void virtio_pci_vmstate_change(void *opaque, bool running)
diff --git a/hw/virtio.c b/hw/virtio.c
index d146f86f132eb25a469ba36ed239059ef86213b3..209c763751bd0f9686f78357d19a866feb221b24 100644
--- a/hw/virtio.c
+++ b/hw/virtio.c
@@ -1021,13 +1021,16 @@ static void virtio_queue_host_notifier_read(EventNotifier *n)
     }
 }
 
-void virtio_queue_set_host_notifier_fd_handler(VirtQueue *vq, bool assign)
+void virtio_queue_set_host_notifier_fd_handler(VirtQueue *vq, bool assign,
+                                               bool set_handler)
 {
-    if (assign) {
+    if (assign && set_handler) {
         event_notifier_set_handler(&vq->host_notifier,
                                    virtio_queue_host_notifier_read);
     } else {
         event_notifier_set_handler(&vq->host_notifier, NULL);
+    }
+    if (!assign) {
         /* Test and clear notifier before after disabling event,
          * in case poll callback didn't have time to run. */
         virtio_queue_host_notifier_read(&vq->host_notifier);
diff --git a/hw/virtio.h b/hw/virtio.h
index f8b5535db1af96b9a43c479215face893d07b2de..7a4f564529eae37290425d520ca40c137b38f1b3 100644
--- a/hw/virtio.h
+++ b/hw/virtio.h
@@ -233,7 +233,8 @@ EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq);
 void virtio_queue_set_guest_notifier_fd_handler(VirtQueue *vq, bool assign,
                                                 bool with_irqfd);
 EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq);
-void virtio_queue_set_host_notifier_fd_handler(VirtQueue *vq, bool assign);
+void virtio_queue_set_host_notifier_fd_handler(VirtQueue *vq, bool assign,
+                                               bool set_handler);
 void virtio_queue_notify_vq(VirtQueue *vq);
 void virtio_irq(VirtQueue *vq);
 #endif