diff --git a/src/qemu/qemu_hotplug.c b/src/qemu/qemu_hotplug.c
index 6eb483c4151d4f4da314b0377d605d1ab85db563..0d9a3aa460fbe1b4b44d39659f6c3d4bea91a57b 100644
--- a/src/qemu/qemu_hotplug.c
+++ b/src/qemu/qemu_hotplug.c
@@ -1135,6 +1135,7 @@ int qemuDomainAttachHostPciDevice(virQEMUDriverPtr driver,
     int configfd = -1;
     char *configfd_name = NULL;
     bool releaseaddr = false;
+    bool teardowncgroup = false;
     int backend = hostdev->source.subsys.u.pci.backend;
 
     if (VIR_REALLOC_N(vm->def->hostdevs, vm->def->nhostdevs + 1) < 0)
@@ -1171,6 +1172,10 @@ int qemuDomainAttachHostPciDevice(virQEMUDriverPtr driver,
         break;
     }
 
+    if (qemuSetupHostdevCGroup(vm, hostdev) < 0)
+        goto error;
+    teardowncgroup = true;
+
     if (virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_DEVICE)) {
         if (qemuAssignDeviceHostdevAlias(vm->def, hostdev, -1) < 0)
             goto error;
@@ -1226,6 +1231,9 @@ int qemuDomainAttachHostPciDevice(virQEMUDriverPtr driver,
     return 0;
 
 error:
+    if (teardowncgroup && qemuTeardownHostdevCgroup(vm, hostdev) < 0)
+        VIR_WARN("Unable to remove host device cgroup ACL on hotplug fail");
+
     if (releaseaddr)
         qemuDomainReleaseDeviceAddress(vm, hostdev->info, NULL);
 
@@ -1418,6 +1426,7 @@ int qemuDomainAttachHostUsbDevice(virQEMUDriverPtr driver,
     virUSBDevicePtr usb = NULL;
     char *devstr = NULL;
     bool added = false;
+    bool teardowncgroup = false;
     int ret = -1;
 
     if (qemuFindHostdevUSBDevice(hostdev, true, &usb) < 0)
@@ -1435,6 +1444,10 @@ int qemuDomainAttachHostUsbDevice(virQEMUDriverPtr driver,
     added = true;
     virUSBDeviceListSteal(list, usb);
 
+    if (qemuSetupHostdevCGroup(vm, hostdev) < 0)
+        goto cleanup;
+    teardowncgroup = true;
+
     if (virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_DEVICE)) {
         if (qemuAssignDeviceHostdevAlias(vm->def, hostdev, -1) < 0)
             goto cleanup;
@@ -1461,6 +1474,10 @@ int qemuDomainAttachHostUsbDevice(virQEMUDriverPtr driver,
 
     ret = 0;
 cleanup:
+    if (ret < 0 &&
+        teardowncgroup &&
+        qemuTeardownHostdevCgroup(vm, hostdev) < 0)
+        VIR_WARN("Unable to remove host device cgroup ACL on hotplug fail");
     if (added)
         virUSBDeviceListSteal(driver->activeUsbHostdevs, usb);
     virUSBDeviceFree(usb);
@@ -1478,6 +1495,7 @@ qemuDomainAttachHostScsiDevice(virQEMUDriverPtr driver,
     qemuDomainObjPrivatePtr priv = vm->privateData;
     char *devstr = NULL;
     char *drvstr = NULL;
+    bool teardowncgroup = false;
 
     if (!virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_DRIVE) ||
         !virQEMUCapsGet(priv->qemuCaps, QEMU_CAPS_DEVICE) ||
@@ -1498,6 +1516,10 @@ qemuDomainAttachHostScsiDevice(virQEMUDriverPtr driver,
         return -1;
     }
 
+    if (qemuSetupHostdevCGroup(vm, hostdev) < 0)
+        goto cleanup;
+    teardowncgroup = true;
+
     if (qemuAssignDeviceHostdevAlias(vm->def, hostdev, -1) < 0)
         goto cleanup;
 
@@ -1535,8 +1557,11 @@ qemuDomainAttachHostScsiDevice(virQEMUDriverPtr driver,
 
     ret = 0;
 cleanup:
-    if (ret < 0)
+    if (ret < 0) {
         qemuDomainReAttachHostScsiDevices(driver, vm->def->name, &hostdev, 1);
+        if (teardowncgroup && qemuTeardownHostdevCgroup(vm, hostdev) < 0)
+            VIR_WARN("Unable to remove host device cgroup ACL on hotplug fail");
+    }
     VIR_FREE(drvstr);
     VIR_FREE(devstr);
     return ret;
@@ -1553,12 +1578,9 @@ int qemuDomainAttachHostDevice(virQEMUDriverPtr driver,
         return -1;
     }
 
-    if (qemuSetupHostdevCGroup(vm, hostdev) < 0)
-        return -1;
-
     if (virSecurityManagerSetHostdevLabel(driver->securityManager,
                                           vm->def, hostdev, NULL) < 0)
-        goto cleanup;
+        return -1;
 
     switch (hostdev->source.subsys.type) {
     case VIR_DOMAIN_HOSTDEV_SUBSYS_TYPE_PCI:
@@ -1592,10 +1614,6 @@ error:
     if (virSecurityManagerRestoreHostdevLabel(driver->securityManager,
                                               vm->def, hostdev, NULL) < 0)
         VIR_WARN("Unable to restore host device labelling on hotplug fail");
-
-cleanup:
-    if (qemuTeardownHostdevCgroup(vm, hostdev) < 0)
-        VIR_WARN("Unable to remove host device cgroup ACL on hotplug fail");
     return -1;
 }