diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c
index d41e467b4aa2d527226d82bb2ec3c7d8b2445931..1c50f7c4f5acb48502e34fbaf3bf6e9ea2737795 100644
--- a/fs/btrfs/inode.c
+++ b/fs/btrfs/inode.c
@@ -3200,9 +3200,10 @@ static int btrfs_rmdir(struct inode *dir, struct dentry *dentry)
 	struct btrfs_trans_handle *trans;
 	unsigned long nr = 0;
 
-	if (inode->i_size > BTRFS_EMPTY_DIR_SIZE ||
-	    btrfs_ino(inode) == BTRFS_FIRST_FREE_OBJECTID)
+	if (inode->i_size > BTRFS_EMPTY_DIR_SIZE)
 		return -ENOTEMPTY;
+	if (btrfs_ino(inode) == BTRFS_FIRST_FREE_OBJECTID)
+		return -EPERM;
 
 	trans = __unlink_start_trans(dir, dentry);
 	if (IS_ERR(trans))