diff --git a/hw/virtio/vhost.c b/hw/virtio/vhost.c
index 501a5f4a78274529a720c370910a9d6b2ebc8f29..131f1643b2a45a3b315a000ed17d9ac1b3d830c3 100644
--- a/hw/virtio/vhost.c
+++ b/hw/virtio/vhost.c
@@ -1196,6 +1196,7 @@ int vhost_dev_enable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev)
         goto fail;
     }
 
+    virtio_device_stop_ioeventfd(vdev);
     for (i = 0; i < hdev->nvqs; ++i) {
         r = virtio_bus_set_host_notifier(VIRTIO_BUS(qbus), hdev->vq_index + i,
                                          true);
@@ -1215,6 +1216,7 @@ fail_vq:
         }
         assert (e >= 0);
     }
+    virtio_device_start_ioeventfd(vdev);
 fail:
     return r;
 }
@@ -1237,6 +1239,7 @@ void vhost_dev_disable_notifiers(struct vhost_dev *hdev, VirtIODevice *vdev)
         }
         assert (r >= 0);
     }
+    virtio_device_start_ioeventfd(vdev);
 }
 
 /* Test and clear event pending status.
diff --git a/hw/virtio/virtio-bus.c b/hw/virtio/virtio-bus.c
index a619445c3a1a9c488742fe46641cc56d0086fbea..b0e45442d92953fc3ba4106c406f659cd61d9f18 100644
--- a/hw/virtio/virtio-bus.c
+++ b/hw/virtio/virtio-bus.c
@@ -190,7 +190,7 @@ int virtio_bus_start_ioeventfd(VirtioBusState *bus)
     if (!k->ioeventfd_assign || !k->ioeventfd_enabled(proxy)) {
         return -ENOSYS;
     }
-    if (bus->ioeventfd_started || bus->ioeventfd_disabled) {
+    if (bus->ioeventfd_started) {
         return 0;
     }
     r = vdc->start_ioeventfd(vdev);
@@ -226,8 +226,8 @@ bool virtio_bus_ioeventfd_enabled(VirtioBusState *bus)
 }
 
 /*
- * This function switches from/to the generic ioeventfd handler.
- * assign==false means 'use generic ioeventfd handler'.
+ * This function switches ioeventfd on/off in the device.
+ * The caller must set or clear the handlers for the EventNotifier.
  */
 int virtio_bus_set_host_notifier(VirtioBusState *bus, int n, bool assign)
 {
@@ -237,19 +237,12 @@ int virtio_bus_set_host_notifier(VirtioBusState *bus, int n, bool assign)
     if (!k->ioeventfd_assign) {
         return -ENOSYS;
     }
-    bus->ioeventfd_disabled = assign;
     if (assign) {
-        /*
-         * Stop using the generic ioeventfd, we are doing eventfd handling
-         * ourselves below
-         *
-         * FIXME: We should just switch the handler and not deassign the
-         * ioeventfd.
-         * Otherwise, there's a window where we don't have an
-         * ioeventfd and we may end up with a notification where
-         * we don't expect one.
-         */
-        virtio_bus_stop_ioeventfd(bus);
+        assert(!bus->ioeventfd_started);
+    } else {
+        if (!bus->ioeventfd_started) {
+            return 0;
+        }
     }
     return set_host_notifier_internal(proxy, bus, n, assign);
 }
diff --git a/include/hw/virtio/virtio-bus.h b/include/hw/virtio/virtio-bus.h
index af6b5c4368ad7a2f89471cb548e75d507b28370c..cbdf745c8452bc865e637bd68bb31a0f677b4260 100644
--- a/include/hw/virtio/virtio-bus.h
+++ b/include/hw/virtio/virtio-bus.h
@@ -93,12 +93,6 @@ typedef struct VirtioBusClass {
 struct VirtioBusState {
     BusState parent_obj;
 
-    /*
-     * Set if the default ioeventfd handlers are disabled by vhost
-     * or dataplane.
-     */
-    bool ioeventfd_disabled;
-
     /*
      * Set if ioeventfd has been started.
      */