qemuDomainAttachDeviceMknodRecursive: Support file mount points

https://bugzilla.redhat.com/show_bug.cgi?id=1462060

Just like in the previous commit, when attaching a file based
device which has its source living under /dev (that is not a
device rather than a regular file), calling mknod() is no help.
We need to:

1) bind mount device to some temporary location
2) enter the namespace
3) move the mount point to desired place
4) umount it in the parent namespace from the temporary location

At the same time, the check in qemuDomainNamespaceSetupDisk makes
no longer sense. Therefore remove it.
Signed-off-by: NMichal Privoznik <mprivozn@redhat.com>
Reviewed-by: NJohn Ferlan <jferlan@redhat.com>

src/qemu/qemu_domain.c

@@ -8544,6 +8544,7 @@ qemuDomainAttachDeviceMknodHelper(pid_t pid ATTRIBUTE_UNUSED,
     bool delDevice = false;
     bool isLink = S_ISLNK(data->sb.st_mode);
     bool isDev = S_ISCHR(data->sb.st_mode) || S_ISBLK(data->sb.st_mode);
+    bool isReg = S_ISREG(data->sb.st_mode);
 
     qemuSecurityPostFork(data->driver->securityManager);
 
@@ -8582,6 +8583,23 @@ qemuDomainAttachDeviceMknodHelper(pid_t pid ATTRIBUTE_UNUSED,
         } else {
             delDevice = true;
         }
+    } else if (isReg) {
+        /* We are not cleaning up disks on virDomainDetachDevice
+         * because disk might be still in use by different disk
+         * as its backing chain. This might however clash here.
+         * Therefore do the cleanup here. */
+        if (umount(data->file) < 0 &&
+            errno != ENOENT) {
+            virReportSystemError(errno,
+                                 _("Unable to umount %s"),
+                                 data->file);
+            goto cleanup;
+        }
+        if (virFileTouch(data->file, data->sb.st_mode) < 0)
+            goto cleanup;
+        delDevice = true;
+        /* Just create the file here so that code below sets
+         * proper owner and mode. Move the mount only after that. */
     } else {
         virReportError(VIR_ERR_OPERATION_UNSUPPORTED,
                        _("unsupported device type %s 0%o"),
@@ -8596,6 +8614,15 @@ qemuDomainAttachDeviceMknodHelper(pid_t pid ATTRIBUTE_UNUSED,
         goto cleanup;
     }
 
+    /* Symlinks don't have mode */
+    if (!isLink &&
+        chmod(data->file, data->sb.st_mode) < 0) {
+        virReportSystemError(errno,
+                             _("Failed to set permissions for device %s"),
+                             data->file);
+        goto cleanup;
+    }
+
     /* Symlinks don't have ACLs. */
     if (!isLink &&
         virFileSetACLs(data->file, data->acl) < 0 &&
@@ -8619,6 +8646,11 @@ qemuDomainAttachDeviceMknodHelper(pid_t pid ATTRIBUTE_UNUSED,
     }
 #endif
 
+    /* Finish mount process started earlier. */
+    if (isReg &&
+        virFileMoveMount(data->target, data->file) < 0)
+        goto cleanup;
+
     ret = 0;
  cleanup:
     if (ret < 0 && delDevice)
@@ -8639,10 +8671,12 @@ qemuDomainAttachDeviceMknodRecursive(virQEMUDriverPtr driver,
                                      size_t ndevMountsPath,
                                      unsigned int ttl)
 {
+    virQEMUDriverConfigPtr cfg = NULL;
     struct qemuDomainAttachDeviceMknodData data;
     int ret = -1;
     char *target = NULL;
     bool isLink;
+    bool isReg;
 
     if (!ttl) {
         virReportSystemError(ELOOP,
@@ -8664,8 +8698,18 @@ qemuDomainAttachDeviceMknodRecursive(virQEMUDriverPtr driver,
     }
 
     isLink = S_ISLNK(data.sb.st_mode);
+    isReg = S_ISREG(data.sb.st_mode);
 
-    if (isLink) {
+    if (isReg && STRPREFIX(file, DEVPREFIX)) {
+        cfg = virQEMUDriverGetConfig(driver);
+        if (!(target = qemuDomainGetPreservedMountPath(cfg, vm, file)))
+            goto cleanup;
+
+        if (virFileBindMountDevice(file, target) < 0)
+            goto cleanup;
+
+        data.target = target;
+    } else if (isLink) {
         if (virFileReadLink(file, &target) < 0) {
             virReportSystemError(errno,
                                  _("unable to resolve symlink %s"),
@@ -8752,7 +8796,10 @@ qemuDomainAttachDeviceMknodRecursive(virQEMUDriverPtr driver,
     freecon(data.tcon);
 #endif
     virFileFreeACLs(&data.acl);
+    if (isReg && target)
+        umount(target);
     VIR_FREE(target);
+    virObjectUnref(cfg);
     return ret;
 }
 
@@ -8830,7 +8877,6 @@ qemuDomainNamespaceSetupDisk(virQEMUDriverPtr driver,
     char **devMountsPath = NULL;
     size_t ndevMountsPath = 0;
     virStorageSourcePtr next;
-    struct stat sb;
     int ret = -1;
 
     if (!qemuDomainNamespaceEnabled(vm, QEMU_DOMAIN_NS_MOUNT))
@@ -8849,15 +8895,6 @@ qemuDomainNamespaceSetupDisk(virQEMUDriverPtr driver,
             continue;
         }
 
-        if (stat(next->path, &sb) < 0) {
-            virReportSystemError(errno,
-                                 _("Unable to access %s"), next->path);
-            goto cleanup;
-        }
-
-        if (!S_ISBLK(sb.st_mode))
-            continue;
-
         if (qemuDomainAttachDeviceMknod(driver,
                                         vm,
                                         next->path,