diff --git a/src/security/security_dac.c b/src/security/security_dac.c
index 3c21dbbddb3af6d29be4f234dba9c03b4475d15d..300c383dd5d82b2bb1dc64d296040a90af3a1a05 100644
--- a/src/security/security_dac.c
+++ b/src/security/security_dac.c
@@ -458,10 +458,11 @@ virSecurityDACRecallLabel(virSecurityDACDataPtr priv ATTRIBUTE_UNUSED,
 {
     char *label;
     int ret = -1;
+    int rv;
 
-    if (virSecurityGetRememberedLabel(SECURITY_DAC_NAME,
-                                      path, &label) < 0)
-        goto cleanup;
+    rv = virSecurityGetRememberedLabel(SECURITY_DAC_NAME, path, &label);
+    if (rv < 0)
+        return rv;
 
     if (!label)
         return 1;
@@ -760,7 +761,9 @@ virSecurityDACSetOwnership(virSecurityManagerPtr mgr,
         }
 
         refcount = virSecurityDACRememberLabel(priv, path, sb.st_uid, sb.st_gid);
-        if (refcount < 0) {
+        if (refcount == -2) {
+            /* Not supported. Don't error though. */
+        } else if (refcount < 0) {
             return -1;
         } else if (refcount > 1) {
             /* Refcount is greater than 1 which means that there
@@ -827,10 +830,13 @@ virSecurityDACRestoreFileLabelInternal(virSecurityManagerPtr mgr,
 
     if (recall && path) {
         rv = virSecurityDACRecallLabel(priv, path, &uid, &gid);
-        if (rv < 0)
+        if (rv == -2) {
+            /* Not supported. Don't error though. */
+        } else if (rv < 0) {
             return -1;
-        if (rv > 0)
+        } else if (rv > 0) {
             return 0;
+        }
     }
 
     VIR_INFO("Restoring DAC user and group on '%s' to %ld:%ld",
diff --git a/src/security/security_selinux.c b/src/security/security_selinux.c
index ec8d407351eed46e24b6de12bcb46e7f21ec9e5f..ff54d47e23831380c699271f50641940a38fc151 100644
--- a/src/security/security_selinux.c
+++ b/src/security/security_selinux.c
@@ -207,9 +207,11 @@ static int
 virSecuritySELinuxRecallLabel(const char *path,
                               security_context_t *con)
 {
-    if (virSecurityGetRememberedLabel(SECURITY_SELINUX_NAME,
-                                      path, con) < 0)
-        return -1;
+    int rv;
+
+    rv = virSecurityGetRememberedLabel(SECURITY_SELINUX_NAME, path, con);
+    if (rv < 0)
+        return rv;
 
     if (!*con)
         return 1;
@@ -1337,7 +1339,9 @@ virSecuritySELinuxSetFileconHelper(virSecurityManagerPtr mgr,
 
         if (econ) {
             refcount = virSecuritySELinuxRememberLabel(path, econ);
-            if (refcount < 0) {
+            if (refcount == -2) {
+                /* Not supported. Don't error though. */
+            } else if (refcount < 0) {
                 goto cleanup;
             } else if (refcount > 1) {
                 /* Refcount is greater than 1 which means that there
@@ -1485,13 +1489,18 @@ virSecuritySELinuxRestoreFileLabel(virSecurityManagerPtr mgr,
     }
 
     if (recall) {
-        if ((rc = virSecuritySELinuxRecallLabel(newpath, &fcon)) < 0) {
+        rc = virSecuritySELinuxRecallLabel(newpath, &fcon);
+        if (rc == -2) {
+            /* Not supported. Lookup the default label below. */
+        } else if (rc < 0) {
             goto cleanup;
         } else if (rc > 0) {
             ret = 0;
             goto cleanup;
         }
-    } else {
+    }
+
+    if (!recall || rc == -2) {
         if (stat(newpath, &buf) != 0) {
             VIR_WARN("cannot stat %s: %s", newpath,
                      virStrerror(errno, ebuf, sizeof(ebuf)));
diff --git a/src/security/security_util.c b/src/security/security_util.c
index f09a18a6231152cc484e2ace0f5ecc83fabf34a5..3c24d7cded2991dd24b060972dc82fc0b8942bcb 100644
--- a/src/security/security_util.c
+++ b/src/security/security_util.c
@@ -105,6 +105,7 @@ virSecurityGetRefCountAttrName(const char *name ATTRIBUTE_UNUSED)
  * zero) and returns zero.
  *
  * Returns: 0 on success,
+ *         -2 if underlying file system doesn't support XATTRs,
  *         -1 otherwise (with error reported)
  */
 int
@@ -125,7 +126,7 @@ virSecurityGetRememberedLabel(const char *name,
 
     if (virFileGetXAttrQuiet(path, ref_name, &value) < 0) {
         if (errno == ENOSYS || errno == ENODATA || errno == ENOTSUP) {
-            ret = 0;
+            ret = -2;
         } else {
             virReportSystemError(errno,
                                  _("Unable to get XATTR %s on %s"),
@@ -192,6 +193,7 @@ virSecurityGetRememberedLabel(const char *name,
  * See also virSecurityGetRememberedLabel.
  *
  * Returns: the new refcount value on success,
+ *         -2 if underlying file system doesn't support XATTRs,
  *         -1 otherwise (with error reported)
  */
 int
@@ -210,7 +212,7 @@ virSecuritySetRememberedLabel(const char *name,
 
     if (virFileGetXAttrQuiet(path, ref_name, &value) < 0) {
         if (errno == ENOSYS || errno == ENOTSUP) {
-            ret = 0;
+            ret = -2;
             goto cleanup;
         } else if (errno != ENODATA) {
             virReportSystemError(errno,