alinux: fuse: add sysfs api to flush processing queue requests

The failover of fuse userspace daemon will reuse the existing fuse conn,
without unmounting it, during daemon crashing and recovery procedure.
But some requests might be in process in the daemon before sending out reply,
when the crash happens. This will stuck the application since it will
never get the reply after the failover.

We add the sysfs api to flush these requests, after the daemon crash, before
recovery. It is easy to reproduce the issue in the fuse userspace daemon,
just exit after receiving the request and before sending the reply back.
The application will hang up in some read/write operation, before
echo 1 > /sys/fs/fuse/connection/xxx/flush. The flush operation will make
the io fail and return the error to the application.
Signed-off-by: NMa Jie Yue <majieyue@linux.alibaba.com>
Reviewed-by: NJoseph Qi <joseph.qi@linux.alibaba.com>
Signed-off-by: NJoseph Qi <joseph.qi@linux.alibaba.com>
Reviewed-by: NXiaoguang Wang <xiaoguang.wang@linux.alibaba.com>
Acked-by: NCaspar Zhang <caspar@linux.alibaba.com>

fs/fuse/control.c

@@ -41,6 +41,18 @@ static ssize_t fuse_conn_abort_write(struct file *file, const char __user *buf,
 	return count;
 }
 
+static ssize_t fuse_conn_flush_write(struct file *file, const char __user *buf,
+					 size_t count, loff_t *ppos)
+{
+	struct fuse_conn *fc = fuse_ctl_file_conn_get(file);
+
+	if (fc) {
+		fuse_flush_pq(fc);
+		fuse_conn_put(fc);
+	}
+	return count;
+}
+
 static ssize_t fuse_conn_waiting_read(struct file *file, char __user *buf,
 				      size_t len, loff_t *ppos)
 {
@@ -176,6 +188,12 @@ static const struct file_operations fuse_ctl_abort_ops = {
 	.llseek = no_llseek,
 };
 
+static const struct file_operations fuse_ctl_flush_ops = {
+	.open = nonseekable_open,
+	.write = fuse_conn_flush_write,
+	.llseek = no_llseek,
+};
+
 static const struct file_operations fuse_ctl_waiting_ops = {
 	.open = nonseekable_open,
 	.read = fuse_conn_waiting_read,
@@ -260,6 +278,8 @@ int fuse_ctl_add_conn(struct fuse_conn *fc)
 				 NULL, &fuse_ctl_waiting_ops) ||
 	    !fuse_ctl_add_dentry(parent, fc, "abort", S_IFREG | 0200, 1,
 				 NULL, &fuse_ctl_abort_ops) ||
+	    !fuse_ctl_add_dentry(parent, fc, "flush", S_IFREG | 0200, 1,
+				 NULL, &fuse_ctl_flush_ops) ||
 	    !fuse_ctl_add_dentry(parent, fc, "max_background", S_IFREG | 0600,
 				 1, NULL, &fuse_conn_max_background_ops) ||
 	    !fuse_ctl_add_dentry(parent, fc, "congestion_threshold",

fs/fuse/dev.c

@@ -2208,6 +2208,41 @@ int fuse_dev_release(struct inode *inode, struct file *file)
 }
 EXPORT_SYMBOL_GPL(fuse_dev_release);
 
+/*
+ * Flush all pending processing requests.
+ *
+ * The failover procedure will reuse the fuse_conn after usersapce crash and
+ * recovery. But the reqs in the processing queue will never get the reply,
+ * making the application stuck forever.
+ *
+ * So we need to flush these reqs by sysfs api. We only flush the reqs in
+ * the processing queue, because these reqs have been sent to userspace.
+ * Firstly we need to dequeue the req from the processing queue, and secondly
+ * we need to call request_end to finish it.
+ */
+void fuse_flush_pq(struct fuse_conn *fc)
+{
+	struct fuse_dev *fud;
+	LIST_HEAD(to_end);
+
+	spin_lock(&fc->lock);
+	if (!fc->connected) {
+		spin_unlock(&fc->lock);
+		return;
+	}
+	list_for_each_entry(fud, &fc->devices, entry) {
+		struct fuse_pqueue *fpq = &fud->pq;
+
+		spin_lock(&fpq->lock);
+		WARN_ON(!list_empty(&fpq->io));
+		list_splice_init(&fpq->processing, &to_end);
+		spin_unlock(&fpq->lock);
+	}
+	spin_unlock(&fc->lock);
+
+	end_requests(fc, &to_end);
+}
+
 static int fuse_dev_fasync(int fd, struct file *file, int on)
 {
 	struct fuse_dev *fud = fuse_get_dev(file);

fs/fuse/fuse_i.h

@@ -38,7 +38,7 @@
 #define FUSE_NAME_MAX 1024
 
 /** Number of dentries for each connection in the control filesystem */
-#define FUSE_CTL_NUM_DENTRIES 5
+#define FUSE_CTL_NUM_DENTRIES 6
 
 /** Number of page pointers embedded in fuse_req */
 #define FUSE_REQ_INLINE_PAGES 1
@@ -867,6 +867,9 @@ void fuse_request_send_background_locked(struct fuse_conn *fc,
 void fuse_abort_conn(struct fuse_conn *fc, bool is_abort);
 void fuse_wait_aborted(struct fuse_conn *fc);
 
+/* Flush all requests in processing queue */
+void fuse_flush_pq(struct fuse_conn *fc);
+
 /**
  * Invalidate inode attributes
  */