dev.c 53.3 KB
Newer Older
M
Miklos Szeredi 已提交
1 2
/*
  FUSE: Filesystem in Userspace
M
Miklos Szeredi 已提交
3
  Copyright (C) 2001-2008  Miklos Szeredi <miklos@szeredi.hu>
M
Miklos Szeredi 已提交
4 5 6 7 8 9 10 11 12 13

  This program can be distributed under the terms of the GNU GPL.
  See the file COPYING.
*/

#include "fuse_i.h"

#include <linux/init.h>
#include <linux/module.h>
#include <linux/poll.h>
14
#include <linux/sched/signal.h>
M
Miklos Szeredi 已提交
15 16 17 18 19
#include <linux/uio.h>
#include <linux/miscdevice.h>
#include <linux/pagemap.h>
#include <linux/file.h>
#include <linux/slab.h>
20
#include <linux/pipe_fs_i.h>
21 22
#include <linux/swap.h>
#include <linux/splice.h>
23
#include <linux/sched.h>
M
Miklos Szeredi 已提交
24 25

MODULE_ALIAS_MISCDEV(FUSE_MINOR);
26
MODULE_ALIAS("devname:fuse");
M
Miklos Szeredi 已提交
27

28
static struct kmem_cache *fuse_req_cachep;
M
Miklos Szeredi 已提交
29

30
static struct fuse_dev *fuse_get_dev(struct file *file)
M
Miklos Szeredi 已提交
31
{
M
Miklos Szeredi 已提交
32 33 34 35
	/*
	 * Lockless access is OK, because file->private data is set
	 * once during mount and is valid until the file is released.
	 */
36
	return READ_ONCE(file->private_data);
M
Miklos Szeredi 已提交
37 38
}

39
static void fuse_request_init(struct fuse_req *req, struct page **pages,
40
			      struct fuse_page_desc *page_descs,
41
			      unsigned npages)
M
Miklos Szeredi 已提交
42 43
{
	memset(req, 0, sizeof(*req));
44
	memset(pages, 0, sizeof(*pages) * npages);
45
	memset(page_descs, 0, sizeof(*page_descs) * npages);
M
Miklos Szeredi 已提交
46
	INIT_LIST_HEAD(&req->list);
47
	INIT_LIST_HEAD(&req->intr_entry);
M
Miklos Szeredi 已提交
48
	init_waitqueue_head(&req->waitq);
49
	refcount_set(&req->count, 1);
50
	req->pages = pages;
51
	req->page_descs = page_descs;
52
	req->max_pages = npages;
M
Miklos Szeredi 已提交
53
	__set_bit(FR_PENDING, &req->flags);
M
Miklos Szeredi 已提交
54 55
}

56
static struct fuse_req *__fuse_request_alloc(unsigned npages, gfp_t flags)
M
Miklos Szeredi 已提交
57
{
58 59 60
	struct fuse_req *req = kmem_cache_alloc(fuse_req_cachep, flags);
	if (req) {
		struct page **pages;
61
		struct fuse_page_desc *page_descs;
62

63
		if (npages <= FUSE_REQ_INLINE_PAGES) {
64
			pages = req->inline_pages;
65 66
			page_descs = req->inline_page_descs;
		} else {
67 68 69 70 71 72
			pages = kmalloc_array(npages, sizeof(struct page *),
					      flags);
			page_descs =
				kmalloc_array(npages,
					      sizeof(struct fuse_page_desc),
					      flags);
73
		}
74

75 76 77
		if (!pages || !page_descs) {
			kfree(pages);
			kfree(page_descs);
78 79 80 81
			kmem_cache_free(fuse_req_cachep, req);
			return NULL;
		}

82
		fuse_request_init(req, pages, page_descs, npages);
83
	}
M
Miklos Szeredi 已提交
84 85
	return req;
}
86 87 88 89 90

struct fuse_req *fuse_request_alloc(unsigned npages)
{
	return __fuse_request_alloc(npages, GFP_KERNEL);
}
91
EXPORT_SYMBOL_GPL(fuse_request_alloc);
M
Miklos Szeredi 已提交
92

93
struct fuse_req *fuse_request_alloc_nofs(unsigned npages)
M
Miklos Szeredi 已提交
94
{
95
	return __fuse_request_alloc(npages, GFP_NOFS);
M
Miklos Szeredi 已提交
96 97
}

M
Miklos Szeredi 已提交
98 99
void fuse_request_free(struct fuse_req *req)
{
100
	if (req->pages != req->inline_pages) {
101
		kfree(req->pages);
102 103
		kfree(req->page_descs);
	}
M
Miklos Szeredi 已提交
104 105 106
	kmem_cache_free(fuse_req_cachep, req);
}

107
void __fuse_get_request(struct fuse_req *req)
M
Miklos Szeredi 已提交
108
{
109
	refcount_inc(&req->count);
M
Miklos Szeredi 已提交
110 111 112 113 114
}

/* Must be called with > 1 refcount */
static void __fuse_put_request(struct fuse_req *req)
{
115
	refcount_dec(&req->count);
M
Miklos Szeredi 已提交
116 117
}

118 119 120 121 122 123 124
void fuse_set_initialized(struct fuse_conn *fc)
{
	/* Make sure stores before this are seen on another CPU */
	smp_wmb();
	fc->initialized = 1;
}

125 126 127 128 129
static bool fuse_block_alloc(struct fuse_conn *fc, bool for_background)
{
	return !fc->initialized || (for_background && fc->blocked);
}

130 131
static void fuse_drop_waiting(struct fuse_conn *fc)
{
132 133 134 135 136 137 138
	/*
	 * lockess check of fc->connected is okay, because atomic_dec_and_test()
	 * provides a memory barrier mached with the one in fuse_wait_aborted()
	 * to ensure no wake-up is missed.
	 */
	if (atomic_dec_and_test(&fc->num_waiting) &&
	    !READ_ONCE(fc->connected)) {
139 140 141 142 143
		/* wake up aborters */
		wake_up_all(&fc->blocked_waitq);
	}
}

144 145
static struct fuse_req *__fuse_get_req(struct fuse_conn *fc, unsigned npages,
				       bool for_background)
M
Miklos Szeredi 已提交
146
{
147 148
	struct fuse_req *req;
	int err;
149
	atomic_inc(&fc->num_waiting);
150 151 152

	if (fuse_block_alloc(fc, for_background)) {
		err = -EINTR;
A
Al Viro 已提交
153 154
		if (wait_event_killable_exclusive(fc->blocked_waitq,
				!fuse_block_alloc(fc, for_background)))
155 156
			goto out;
	}
157 158
	/* Matches smp_wmb() in fuse_set_initialized() */
	smp_rmb();
159

160 161 162 163
	err = -ENOTCONN;
	if (!fc->connected)
		goto out;

M
Miklos Szeredi 已提交
164 165 166 167
	err = -ECONNREFUSED;
	if (fc->conn_error)
		goto out;

M
Maxim Patlasov 已提交
168
	req = fuse_request_alloc(npages);
169
	err = -ENOMEM;
170 171 172
	if (!req) {
		if (for_background)
			wake_up(&fc->blocked_waitq);
173
		goto out;
174
	}
M
Miklos Szeredi 已提交
175

176 177
	req->in.h.uid = from_kuid(fc->user_ns, current_fsuid());
	req->in.h.gid = from_kgid(fc->user_ns, current_fsgid());
178 179
	req->in.h.pid = pid_nr_ns(task_pid(current), fc->pid_ns);

M
Miklos Szeredi 已提交
180 181 182 183
	__set_bit(FR_WAITING, &req->flags);
	if (for_background)
		__set_bit(FR_BACKGROUND, &req->flags);

184 185 186 187 188
	if (unlikely(req->in.h.uid == ((uid_t)-1) ||
		     req->in.h.gid == ((gid_t)-1))) {
		fuse_put_request(fc, req);
		return ERR_PTR(-EOVERFLOW);
	}
M
Miklos Szeredi 已提交
189
	return req;
190 191

 out:
192
	fuse_drop_waiting(fc);
193
	return ERR_PTR(err);
M
Miklos Szeredi 已提交
194
}
195 196 197 198 199

struct fuse_req *fuse_get_req(struct fuse_conn *fc, unsigned npages)
{
	return __fuse_get_req(fc, npages, false);
}
200
EXPORT_SYMBOL_GPL(fuse_get_req);
M
Miklos Szeredi 已提交
201

202 203 204 205 206 207 208
struct fuse_req *fuse_get_req_for_background(struct fuse_conn *fc,
					     unsigned npages)
{
	return __fuse_get_req(fc, npages, true);
}
EXPORT_SYMBOL_GPL(fuse_get_req_for_background);

209 210 211 212 213 214 215 216 217 218 219 220
/*
 * Return request in fuse_file->reserved_req.  However that may
 * currently be in use.  If that is the case, wait for it to become
 * available.
 */
static struct fuse_req *get_reserved_req(struct fuse_conn *fc,
					 struct file *file)
{
	struct fuse_req *req = NULL;
	struct fuse_file *ff = file->private_data;

	do {
221
		wait_event(fc->reserved_req_waitq, ff->reserved_req);
222 223 224 225
		spin_lock(&fc->lock);
		if (ff->reserved_req) {
			req = ff->reserved_req;
			ff->reserved_req = NULL;
A
Al Viro 已提交
226
			req->stolen_file = get_file(file);
227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242
		}
		spin_unlock(&fc->lock);
	} while (!req);

	return req;
}

/*
 * Put stolen request back into fuse_file->reserved_req
 */
static void put_reserved_req(struct fuse_conn *fc, struct fuse_req *req)
{
	struct file *file = req->stolen_file;
	struct fuse_file *ff = file->private_data;

	spin_lock(&fc->lock);
243
	fuse_request_init(req, req->pages, req->page_descs, req->max_pages);
244 245
	BUG_ON(ff->reserved_req);
	ff->reserved_req = req;
246
	wake_up_all(&fc->reserved_req_waitq);
247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263
	spin_unlock(&fc->lock);
	fput(file);
}

/*
 * Gets a requests for a file operation, always succeeds
 *
 * This is used for sending the FLUSH request, which must get to
 * userspace, due to POSIX locks which may need to be unlocked.
 *
 * If allocation fails due to OOM, use the reserved request in
 * fuse_file.
 *
 * This is very unlikely to deadlock accidentally, since the
 * filesystem should not have it's own file open.  If deadlock is
 * intentional, it can still be broken by "aborting" the filesystem.
 */
M
Maxim Patlasov 已提交
264 265
struct fuse_req *fuse_get_req_nofail_nopages(struct fuse_conn *fc,
					     struct file *file)
266 267 268 269
{
	struct fuse_req *req;

	atomic_inc(&fc->num_waiting);
270
	wait_event(fc->blocked_waitq, fc->initialized);
271 272
	/* Matches smp_wmb() in fuse_set_initialized() */
	smp_rmb();
M
Maxim Patlasov 已提交
273
	req = fuse_request_alloc(0);
274 275 276
	if (!req)
		req = get_reserved_req(fc, file);

277 278
	req->in.h.uid = from_kuid_munged(fc->user_ns, current_fsuid());
	req->in.h.gid = from_kgid_munged(fc->user_ns, current_fsgid());
279 280
	req->in.h.pid = pid_nr_ns(task_pid(current), fc->pid_ns);

M
Miklos Szeredi 已提交
281 282
	__set_bit(FR_WAITING, &req->flags);
	__clear_bit(FR_BACKGROUND, &req->flags);
283 284 285
	return req;
}

M
Miklos Szeredi 已提交
286
void fuse_put_request(struct fuse_conn *fc, struct fuse_req *req)
287
{
288
	if (refcount_dec_and_test(&req->count)) {
M
Miklos Szeredi 已提交
289
		if (test_bit(FR_BACKGROUND, &req->flags)) {
290 291 292 293 294 295 296 297 298 299
			/*
			 * We get here in the unlikely case that a background
			 * request was allocated but not sent
			 */
			spin_lock(&fc->lock);
			if (!fc->blocked)
				wake_up(&fc->blocked_waitq);
			spin_unlock(&fc->lock);
		}

M
Miklos Szeredi 已提交
300 301
		if (test_bit(FR_WAITING, &req->flags)) {
			__clear_bit(FR_WAITING, &req->flags);
302
			fuse_drop_waiting(fc);
M
Miklos Szeredi 已提交
303
		}
304 305 306 307 308

		if (req->stolen_file)
			put_reserved_req(fc, req);
		else
			fuse_request_free(req);
309 310
	}
}
311
EXPORT_SYMBOL_GPL(fuse_put_request);
312

313 314 315 316 317 318 319 320 321 322 323
static unsigned len_args(unsigned numargs, struct fuse_arg *args)
{
	unsigned nbytes = 0;
	unsigned i;

	for (i = 0; i < numargs; i++)
		nbytes += args[i].size;

	return nbytes;
}

M
Miklos Szeredi 已提交
324
static u64 fuse_get_unique(struct fuse_iqueue *fiq)
325
{
M
Miklos Szeredi 已提交
326
	return ++fiq->reqctr;
327 328
}

M
Miklos Szeredi 已提交
329
static void queue_request(struct fuse_iqueue *fiq, struct fuse_req *req)
330 331 332
{
	req->in.h.len = sizeof(struct fuse_in_header) +
		len_args(req->in.numargs, (struct fuse_arg *) req->in.args);
M
Miklos Szeredi 已提交
333
	list_add_tail(&req->list, &fiq->pending);
334
	wake_up(&fiq->waitq);
M
Miklos Szeredi 已提交
335
	kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
336 337
}

338 339 340
void fuse_queue_forget(struct fuse_conn *fc, struct fuse_forget_link *forget,
		       u64 nodeid, u64 nlookup)
{
M
Miklos Szeredi 已提交
341 342
	struct fuse_iqueue *fiq = &fc->iq;

343 344
	forget->forget_one.nodeid = nodeid;
	forget->forget_one.nlookup = nlookup;
345

346
	spin_lock(&fiq->lock);
347
	if (fiq->connected) {
M
Miklos Szeredi 已提交
348 349
		fiq->forget_list_tail->next = forget;
		fiq->forget_list_tail = forget;
350
		wake_up(&fiq->waitq);
M
Miklos Szeredi 已提交
351
		kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
M
Miklos Szeredi 已提交
352 353 354
	} else {
		kfree(forget);
	}
355
	spin_unlock(&fiq->lock);
356 357
}

358 359
static void flush_bg_queue(struct fuse_conn *fc)
{
360
	while (fc->active_background < fc->max_background &&
361 362
	       !list_empty(&fc->bg_queue)) {
		struct fuse_req *req;
M
Miklos Szeredi 已提交
363
		struct fuse_iqueue *fiq = &fc->iq;
364 365 366 367

		req = list_entry(fc->bg_queue.next, struct fuse_req, list);
		list_del(&req->list);
		fc->active_background++;
368
		spin_lock(&fiq->lock);
M
Miklos Szeredi 已提交
369 370
		req->in.h.unique = fuse_get_unique(fiq);
		queue_request(fiq, req);
371
		spin_unlock(&fiq->lock);
372 373 374
	}
}

M
Miklos Szeredi 已提交
375 376
/*
 * This function is called when a request is finished.  Either a reply
377
 * has arrived or it was aborted (and not yet sent) or some error
M
Miklos Szeredi 已提交
378
 * occurred during communication with userspace, or the device file
379 380 381
 * was closed.  The requester thread is woken up (if still waiting),
 * the 'end' callback is called if given, else the reference to the
 * request is released
M
Miklos Szeredi 已提交
382 383 384
 */
static void request_end(struct fuse_conn *fc, struct fuse_req *req)
{
M
Miklos Szeredi 已提交
385
	struct fuse_iqueue *fiq = &fc->iq;
M
Miklos Szeredi 已提交
386

387
	if (test_and_set_bit(FR_FINISHED, &req->flags))
388
		goto put_request;
M
Miklos Szeredi 已提交
389

390
	spin_lock(&fiq->lock);
M
Miklos Szeredi 已提交
391
	list_del_init(&req->intr_entry);
392
	spin_unlock(&fiq->lock);
M
Miklos Szeredi 已提交
393 394
	WARN_ON(test_bit(FR_PENDING, &req->flags));
	WARN_ON(test_bit(FR_SENT, &req->flags));
M
Miklos Szeredi 已提交
395
	if (test_bit(FR_BACKGROUND, &req->flags)) {
396
		spin_lock(&fc->lock);
M
Miklos Szeredi 已提交
397
		clear_bit(FR_BACKGROUND, &req->flags);
M
Miklos Szeredi 已提交
398
		if (fc->num_background == fc->max_background) {
399
			fc->blocked = 0;
400
			wake_up(&fc->blocked_waitq);
M
Miklos Szeredi 已提交
401 402 403 404 405 406 407 408 409 410
		} else if (!fc->blocked) {
			/*
			 * Wake up next waiter, if any.  It's okay to use
			 * waitqueue_active(), as we've already synced up
			 * fc->blocked with waiters with the wake_up() call
			 * above.
			 */
			if (waitqueue_active(&fc->blocked_waitq))
				wake_up(&fc->blocked_waitq);
		}
411

412
		if (fc->num_background == fc->congestion_threshold && fc->sb) {
413 414
			clear_bdi_congested(fc->sb->s_bdi, BLK_RW_SYNC);
			clear_bdi_congested(fc->sb->s_bdi, BLK_RW_ASYNC);
415
		}
416
		fc->num_background--;
417 418
		fc->active_background--;
		flush_bg_queue(fc);
419
		spin_unlock(&fc->lock);
M
Miklos Szeredi 已提交
420
	}
421
	wake_up(&req->waitq);
M
Miklos Szeredi 已提交
422 423
	if (req->end)
		req->end(fc, req);
424
put_request:
425
	fuse_put_request(fc, req);
M
Miklos Szeredi 已提交
426 427
}

M
Miklos Szeredi 已提交
428
static void queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req)
429
{
430
	spin_lock(&fiq->lock);
431
	if (test_bit(FR_FINISHED, &req->flags)) {
432
		spin_unlock(&fiq->lock);
433 434
		return;
	}
435 436
	if (list_empty(&req->intr_entry)) {
		list_add_tail(&req->intr_entry, &fiq->interrupts);
437
		wake_up(&fiq->waitq);
438
	}
439
	spin_unlock(&fiq->lock);
M
Miklos Szeredi 已提交
440
	kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
441 442
}

443
static void request_wait_answer(struct fuse_conn *fc, struct fuse_req *req)
M
Miklos Szeredi 已提交
444
{
M
Miklos Szeredi 已提交
445
	struct fuse_iqueue *fiq = &fc->iq;
446 447
	int err;

448 449
	if (!fc->no_interrupt) {
		/* Any signal may interrupt this */
450
		err = wait_event_interruptible(req->waitq,
M
Miklos Szeredi 已提交
451
					test_bit(FR_FINISHED, &req->flags));
452
		if (!err)
453 454
			return;

M
Miklos Szeredi 已提交
455
		set_bit(FR_INTERRUPTED, &req->flags);
456 457
		/* matches barrier in fuse_dev_do_read() */
		smp_mb__after_atomic();
M
Miklos Szeredi 已提交
458
		if (test_bit(FR_SENT, &req->flags))
M
Miklos Szeredi 已提交
459
			queue_interrupt(fiq, req);
460 461
	}

M
Miklos Szeredi 已提交
462
	if (!test_bit(FR_FORCE, &req->flags)) {
463
		/* Only fatal signals may interrupt this */
A
Al Viro 已提交
464
		err = wait_event_killable(req->waitq,
M
Miklos Szeredi 已提交
465
					test_bit(FR_FINISHED, &req->flags));
466
		if (!err)
M
Miklos Szeredi 已提交
467 468
			return;

469
		spin_lock(&fiq->lock);
M
Miklos Szeredi 已提交
470
		/* Request is not yet in userspace, bail out */
M
Miklos Szeredi 已提交
471
		if (test_bit(FR_PENDING, &req->flags)) {
M
Miklos Szeredi 已提交
472
			list_del(&req->list);
473
			spin_unlock(&fiq->lock);
M
Miklos Szeredi 已提交
474 475 476 477
			__fuse_put_request(req);
			req->out.h.error = -EINTR;
			return;
		}
478
		spin_unlock(&fiq->lock);
479
	}
M
Miklos Szeredi 已提交
480

M
Miklos Szeredi 已提交
481 482 483 484
	/*
	 * Either request is already in userspace, or it was forced.
	 * Wait it out.
	 */
M
Miklos Szeredi 已提交
485
	wait_event(req->waitq, test_bit(FR_FINISHED, &req->flags));
M
Miklos Szeredi 已提交
486 487
}

488
static void __fuse_request_send(struct fuse_conn *fc, struct fuse_req *req)
M
Miklos Szeredi 已提交
489
{
490 491
	struct fuse_iqueue *fiq = &fc->iq;

M
Miklos Szeredi 已提交
492
	BUG_ON(test_bit(FR_BACKGROUND, &req->flags));
493
	spin_lock(&fiq->lock);
494
	if (!fiq->connected) {
495
		spin_unlock(&fiq->lock);
M
Miklos Szeredi 已提交
496
		req->out.h.error = -ENOTCONN;
497
	} else {
M
Miklos Szeredi 已提交
498 499
		req->in.h.unique = fuse_get_unique(fiq);
		queue_request(fiq, req);
M
Miklos Szeredi 已提交
500 501 502
		/* acquire extra reference, since request is still needed
		   after request_end() */
		__fuse_get_request(req);
503
		spin_unlock(&fiq->lock);
M
Miklos Szeredi 已提交
504

505
		request_wait_answer(fc, req);
506 507
		/* Pairs with smp_wmb() in request_end() */
		smp_rmb();
M
Miklos Szeredi 已提交
508 509
	}
}
510 511 512

void fuse_request_send(struct fuse_conn *fc, struct fuse_req *req)
{
M
Miklos Szeredi 已提交
513 514 515
	__set_bit(FR_ISREPLY, &req->flags);
	if (!test_bit(FR_WAITING, &req->flags)) {
		__set_bit(FR_WAITING, &req->flags);
516 517
		atomic_inc(&fc->num_waiting);
	}
518 519
	__fuse_request_send(fc, req);
}
520
EXPORT_SYMBOL_GPL(fuse_request_send);
M
Miklos Szeredi 已提交
521

522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554
static void fuse_adjust_compat(struct fuse_conn *fc, struct fuse_args *args)
{
	if (fc->minor < 4 && args->in.h.opcode == FUSE_STATFS)
		args->out.args[0].size = FUSE_COMPAT_STATFS_SIZE;

	if (fc->minor < 9) {
		switch (args->in.h.opcode) {
		case FUSE_LOOKUP:
		case FUSE_CREATE:
		case FUSE_MKNOD:
		case FUSE_MKDIR:
		case FUSE_SYMLINK:
		case FUSE_LINK:
			args->out.args[0].size = FUSE_COMPAT_ENTRY_OUT_SIZE;
			break;
		case FUSE_GETATTR:
		case FUSE_SETATTR:
			args->out.args[0].size = FUSE_COMPAT_ATTR_OUT_SIZE;
			break;
		}
	}
	if (fc->minor < 12) {
		switch (args->in.h.opcode) {
		case FUSE_CREATE:
			args->in.args[0].size = sizeof(struct fuse_open_in);
			break;
		case FUSE_MKNOD:
			args->in.args[0].size = FUSE_COMPAT_MKNOD_IN_SIZE;
			break;
		}
	}
}

555 556 557 558 559 560 561 562 563
ssize_t fuse_simple_request(struct fuse_conn *fc, struct fuse_args *args)
{
	struct fuse_req *req;
	ssize_t ret;

	req = fuse_get_req(fc, 0);
	if (IS_ERR(req))
		return PTR_ERR(req);

564 565 566
	/* Needs to be done after fuse_get_req() so that fc->minor is valid */
	fuse_adjust_compat(fc, args);

567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586
	req->in.h.opcode = args->in.h.opcode;
	req->in.h.nodeid = args->in.h.nodeid;
	req->in.numargs = args->in.numargs;
	memcpy(req->in.args, args->in.args,
	       args->in.numargs * sizeof(struct fuse_in_arg));
	req->out.argvar = args->out.argvar;
	req->out.numargs = args->out.numargs;
	memcpy(req->out.args, args->out.args,
	       args->out.numargs * sizeof(struct fuse_arg));
	fuse_request_send(fc, req);
	ret = req->out.h.error;
	if (!ret && args->out.argvar) {
		BUG_ON(args->out.numargs != 1);
		ret = req->out.args[0].size;
	}
	fuse_put_request(fc, req);

	return ret;
}

587 588 589 590 591 592 593
/*
 * Called under fc->lock
 *
 * fc->connected must have been checked previously
 */
void fuse_request_send_background_locked(struct fuse_conn *fc,
					 struct fuse_req *req)
594
{
M
Miklos Szeredi 已提交
595 596 597
	BUG_ON(!test_bit(FR_BACKGROUND, &req->flags));
	if (!test_bit(FR_WAITING, &req->flags)) {
		__set_bit(FR_WAITING, &req->flags);
598 599
		atomic_inc(&fc->num_waiting);
	}
M
Miklos Szeredi 已提交
600
	__set_bit(FR_ISREPLY, &req->flags);
601
	fc->num_background++;
602
	if (fc->num_background == fc->max_background)
603
		fc->blocked = 1;
J
Jan Kara 已提交
604
	if (fc->num_background == fc->congestion_threshold && fc->sb) {
605 606
		set_bdi_congested(fc->sb->s_bdi, BLK_RW_SYNC);
		set_bdi_congested(fc->sb->s_bdi, BLK_RW_ASYNC);
607 608 609 610 611
	}
	list_add_tail(&req->list, &fc->bg_queue);
	flush_bg_queue(fc);
}

612
void fuse_request_send_background(struct fuse_conn *fc, struct fuse_req *req)
M
Miklos Szeredi 已提交
613
{
614
	BUG_ON(!req->end);
615
	spin_lock(&fc->lock);
M
Miklos Szeredi 已提交
616
	if (fc->connected) {
617
		fuse_request_send_background_locked(fc, req);
618
		spin_unlock(&fc->lock);
M
Miklos Szeredi 已提交
619
	} else {
620
		spin_unlock(&fc->lock);
M
Miklos Szeredi 已提交
621
		req->out.h.error = -ENOTCONN;
622 623
		req->end(fc, req);
		fuse_put_request(fc, req);
M
Miklos Szeredi 已提交
624 625
	}
}
626
EXPORT_SYMBOL_GPL(fuse_request_send_background);
M
Miklos Szeredi 已提交
627

M
Miklos Szeredi 已提交
628 629 630 631
static int fuse_request_send_notify_reply(struct fuse_conn *fc,
					  struct fuse_req *req, u64 unique)
{
	int err = -ENODEV;
M
Miklos Szeredi 已提交
632
	struct fuse_iqueue *fiq = &fc->iq;
M
Miklos Szeredi 已提交
633

M
Miklos Szeredi 已提交
634
	__clear_bit(FR_ISREPLY, &req->flags);
M
Miklos Szeredi 已提交
635
	req->in.h.unique = unique;
636
	spin_lock(&fiq->lock);
637
	if (fiq->connected) {
M
Miklos Szeredi 已提交
638
		queue_request(fiq, req);
M
Miklos Szeredi 已提交
639 640
		err = 0;
	}
641
	spin_unlock(&fiq->lock);
M
Miklos Szeredi 已提交
642 643 644 645

	return err;
}

646 647
void fuse_force_forget(struct file *file, u64 nodeid)
{
A
Al Viro 已提交
648
	struct inode *inode = file_inode(file);
649 650 651 652 653 654
	struct fuse_conn *fc = get_fuse_conn(inode);
	struct fuse_req *req;
	struct fuse_forget_in inarg;

	memset(&inarg, 0, sizeof(inarg));
	inarg.nlookup = 1;
M
Maxim Patlasov 已提交
655
	req = fuse_get_req_nofail_nopages(fc, file);
656 657 658 659 660
	req->in.h.opcode = FUSE_FORGET;
	req->in.h.nodeid = nodeid;
	req->in.numargs = 1;
	req->in.args[0].size = sizeof(inarg);
	req->in.args[0].value = &inarg;
M
Miklos Szeredi 已提交
661
	__clear_bit(FR_ISREPLY, &req->flags);
662 663 664
	__fuse_request_send(fc, req);
	/* ignore errors */
	fuse_put_request(fc, req);
665 666
}

M
Miklos Szeredi 已提交
667 668 669
/*
 * Lock the request.  Up to the next unlock_request() there mustn't be
 * anything that could cause a page-fault.  If the request was already
670
 * aborted bail out.
M
Miklos Szeredi 已提交
671
 */
672
static int lock_request(struct fuse_req *req)
M
Miklos Szeredi 已提交
673 674 675
{
	int err = 0;
	if (req) {
676
		spin_lock(&req->waitq.lock);
M
Miklos Szeredi 已提交
677
		if (test_bit(FR_ABORTED, &req->flags))
M
Miklos Szeredi 已提交
678 679
			err = -ENOENT;
		else
M
Miklos Szeredi 已提交
680
			set_bit(FR_LOCKED, &req->flags);
681
		spin_unlock(&req->waitq.lock);
M
Miklos Szeredi 已提交
682 683 684 685 686
	}
	return err;
}

/*
M
Miklos Szeredi 已提交
687 688
 * Unlock request.  If it was aborted while locked, caller is responsible
 * for unlocking and ending the request.
M
Miklos Szeredi 已提交
689
 */
690
static int unlock_request(struct fuse_req *req)
M
Miklos Szeredi 已提交
691
{
M
Miklos Szeredi 已提交
692
	int err = 0;
M
Miklos Szeredi 已提交
693
	if (req) {
694
		spin_lock(&req->waitq.lock);
M
Miklos Szeredi 已提交
695
		if (test_bit(FR_ABORTED, &req->flags))
M
Miklos Szeredi 已提交
696 697
			err = -ENOENT;
		else
M
Miklos Szeredi 已提交
698
			clear_bit(FR_LOCKED, &req->flags);
699
		spin_unlock(&req->waitq.lock);
M
Miklos Szeredi 已提交
700
	}
M
Miklos Szeredi 已提交
701
	return err;
M
Miklos Szeredi 已提交
702 703 704 705 706
}

struct fuse_copy_state {
	int write;
	struct fuse_req *req;
707
	struct iov_iter *iter;
708 709 710
	struct pipe_buffer *pipebufs;
	struct pipe_buffer *currbuf;
	struct pipe_inode_info *pipe;
M
Miklos Szeredi 已提交
711 712 713
	unsigned long nr_segs;
	struct page *pg;
	unsigned len;
714
	unsigned offset;
715
	unsigned move_pages:1;
M
Miklos Szeredi 已提交
716 717
};

718
static void fuse_copy_init(struct fuse_copy_state *cs, int write,
719
			   struct iov_iter *iter)
M
Miklos Szeredi 已提交
720 721 722
{
	memset(cs, 0, sizeof(*cs));
	cs->write = write;
723
	cs->iter = iter;
M
Miklos Szeredi 已提交
724 725 726
}

/* Unmap and put previous page of userspace buffer */
727
static void fuse_copy_finish(struct fuse_copy_state *cs)
M
Miklos Szeredi 已提交
728
{
729 730 731
	if (cs->currbuf) {
		struct pipe_buffer *buf = cs->currbuf;

732
		if (cs->write)
733
			buf->len = PAGE_SIZE - cs->len;
734
		cs->currbuf = NULL;
735
	} else if (cs->pg) {
M
Miklos Szeredi 已提交
736 737 738 739 740 741
		if (cs->write) {
			flush_dcache_page(cs->pg);
			set_page_dirty_lock(cs->pg);
		}
		put_page(cs->pg);
	}
742
	cs->pg = NULL;
M
Miklos Szeredi 已提交
743 744 745 746 747 748 749 750
}

/*
 * Get another pagefull of userspace buffer, and map it to kernel
 * address space, and lock request
 */
static int fuse_copy_fill(struct fuse_copy_state *cs)
{
751
	struct page *page;
M
Miklos Szeredi 已提交
752 753
	int err;

754
	err = unlock_request(cs->req);
M
Miklos Szeredi 已提交
755 756 757
	if (err)
		return err;

M
Miklos Szeredi 已提交
758
	fuse_copy_finish(cs);
759 760 761
	if (cs->pipebufs) {
		struct pipe_buffer *buf = cs->pipebufs;

762
		if (!cs->write) {
763
			err = pipe_buf_confirm(cs->pipe, buf);
764 765 766 767 768
			if (err)
				return err;

			BUG_ON(!cs->nr_segs);
			cs->currbuf = buf;
769 770
			cs->pg = buf->page;
			cs->offset = buf->offset;
771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786
			cs->len = buf->len;
			cs->pipebufs++;
			cs->nr_segs--;
		} else {
			if (cs->nr_segs == cs->pipe->buffers)
				return -EIO;

			page = alloc_page(GFP_HIGHUSER);
			if (!page)
				return -ENOMEM;

			buf->page = page;
			buf->offset = 0;
			buf->len = 0;

			cs->currbuf = buf;
787 788
			cs->pg = page;
			cs->offset = 0;
789 790 791 792
			cs->len = PAGE_SIZE;
			cs->pipebufs++;
			cs->nr_segs++;
		}
793
	} else {
794 795
		size_t off;
		err = iov_iter_get_pages(cs->iter, &page, PAGE_SIZE, 1, &off);
796 797
		if (err < 0)
			return err;
798 799 800
		BUG_ON(!err);
		cs->len = err;
		cs->offset = off;
801
		cs->pg = page;
802
		iov_iter_advance(cs->iter, err);
M
Miklos Szeredi 已提交
803 804
	}

805
	return lock_request(cs->req);
M
Miklos Szeredi 已提交
806 807 808
}

/* Do as much copy to/from userspace buffer as we can */
809
static int fuse_copy_do(struct fuse_copy_state *cs, void **val, unsigned *size)
M
Miklos Szeredi 已提交
810 811 812
{
	unsigned ncpy = min(*size, cs->len);
	if (val) {
813 814 815
		void *pgaddr = kmap_atomic(cs->pg);
		void *buf = pgaddr + cs->offset;

M
Miklos Szeredi 已提交
816
		if (cs->write)
817
			memcpy(buf, *val, ncpy);
M
Miklos Szeredi 已提交
818
		else
819 820 821
			memcpy(*val, buf, ncpy);

		kunmap_atomic(pgaddr);
M
Miklos Szeredi 已提交
822 823 824 825
		*val += ncpy;
	}
	*size -= ncpy;
	cs->len -= ncpy;
826
	cs->offset += ncpy;
M
Miklos Szeredi 已提交
827 828 829
	return ncpy;
}

830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855
static int fuse_check_page(struct page *page)
{
	if (page_mapcount(page) ||
	    page->mapping != NULL ||
	    page_count(page) != 1 ||
	    (page->flags & PAGE_FLAGS_CHECK_AT_PREP &
	     ~(1 << PG_locked |
	       1 << PG_referenced |
	       1 << PG_uptodate |
	       1 << PG_lru |
	       1 << PG_active |
	       1 << PG_reclaim))) {
		printk(KERN_WARNING "fuse: trying to steal weird page\n");
		printk(KERN_WARNING "  page=%p index=%li flags=%08lx, count=%i, mapcount=%i, mapping=%p\n", page, page->index, page->flags, page_count(page), page_mapcount(page), page->mapping);
		return 1;
	}
	return 0;
}

static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
{
	int err;
	struct page *oldpage = *pagep;
	struct page *newpage;
	struct pipe_buffer *buf = cs->pipebufs;

856
	err = unlock_request(cs->req);
M
Miklos Szeredi 已提交
857 858 859
	if (err)
		return err;

860 861
	fuse_copy_finish(cs);

862
	err = pipe_buf_confirm(cs->pipe, buf);
863 864 865 866 867 868 869 870 871 872 873 874
	if (err)
		return err;

	BUG_ON(!cs->nr_segs);
	cs->currbuf = buf;
	cs->len = buf->len;
	cs->pipebufs++;
	cs->nr_segs--;

	if (cs->len != PAGE_SIZE)
		goto out_fallback;

M
Miklos Szeredi 已提交
875
	if (pipe_buf_steal(cs->pipe, buf) != 0)
876 877 878 879
		goto out_fallback;

	newpage = buf->page;

M
Miklos Szeredi 已提交
880 881
	if (!PageUptodate(newpage))
		SetPageUptodate(newpage);
882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900

	ClearPageMappedToDisk(newpage);

	if (fuse_check_page(newpage) != 0)
		goto out_fallback_unlock;

	/*
	 * This is a new and locked page, it shouldn't be mapped or
	 * have any special flags on it
	 */
	if (WARN_ON(page_mapped(oldpage)))
		goto out_fallback_unlock;
	if (WARN_ON(page_has_private(oldpage)))
		goto out_fallback_unlock;
	if (WARN_ON(PageDirty(oldpage) || PageWriteback(oldpage)))
		goto out_fallback_unlock;
	if (WARN_ON(PageMlocked(oldpage)))
		goto out_fallback_unlock;

901
	err = replace_page_cache_page(oldpage, newpage, GFP_KERNEL);
902
	if (err) {
903 904
		unlock_page(newpage);
		return err;
905
	}
906

907
	get_page(newpage);
908 909 910 911 912

	if (!(buf->flags & PIPE_BUF_FLAG_LRU))
		lru_cache_add_file(newpage);

	err = 0;
913
	spin_lock(&cs->req->waitq.lock);
M
Miklos Szeredi 已提交
914
	if (test_bit(FR_ABORTED, &cs->req->flags))
915 916 917
		err = -ENOENT;
	else
		*pagep = newpage;
918
	spin_unlock(&cs->req->waitq.lock);
919 920 921

	if (err) {
		unlock_page(newpage);
922
		put_page(newpage);
923 924 925 926
		return err;
	}

	unlock_page(oldpage);
927
	put_page(oldpage);
928 929 930 931 932 933 934
	cs->len = 0;

	return 0;

out_fallback_unlock:
	unlock_page(newpage);
out_fallback:
935 936
	cs->pg = buf->page;
	cs->offset = buf->offset;
937

938
	err = lock_request(cs->req);
939 940 941 942 943 944
	if (err)
		return err;

	return 1;
}

945 946 947 948
static int fuse_ref_page(struct fuse_copy_state *cs, struct page *page,
			 unsigned offset, unsigned count)
{
	struct pipe_buffer *buf;
M
Miklos Szeredi 已提交
949
	int err;
950 951 952 953

	if (cs->nr_segs == cs->pipe->buffers)
		return -EIO;

954
	err = unlock_request(cs->req);
M
Miklos Szeredi 已提交
955 956 957
	if (err)
		return err;

958 959 960
	fuse_copy_finish(cs);

	buf = cs->pipebufs;
961
	get_page(page);
962 963 964 965 966 967 968 969 970 971 972
	buf->page = page;
	buf->offset = offset;
	buf->len = count;

	cs->pipebufs++;
	cs->nr_segs++;
	cs->len = 0;

	return 0;
}

M
Miklos Szeredi 已提交
973 974 975 976
/*
 * Copy a page in the request to/from the userspace buffer.  Must be
 * done atomically
 */
977
static int fuse_copy_page(struct fuse_copy_state *cs, struct page **pagep,
978
			  unsigned offset, unsigned count, int zeroing)
M
Miklos Szeredi 已提交
979
{
980 981 982
	int err;
	struct page *page = *pagep;

983 984 985
	if (page && zeroing && count < PAGE_SIZE)
		clear_highpage(page);

M
Miklos Szeredi 已提交
986
	while (count) {
987 988 989
		if (cs->write && cs->pipebufs && page) {
			return fuse_ref_page(cs, page, offset, count);
		} else if (!cs->len) {
990 991 992 993 994 995 996 997 998 999
			if (cs->move_pages && page &&
			    offset == 0 && count == PAGE_SIZE) {
				err = fuse_try_move_page(cs, pagep);
				if (err <= 0)
					return err;
			} else {
				err = fuse_copy_fill(cs);
				if (err)
					return err;
			}
M
Miklos Szeredi 已提交
1000
		}
M
Miklos Szeredi 已提交
1001
		if (page) {
1002
			void *mapaddr = kmap_atomic(page);
M
Miklos Szeredi 已提交
1003 1004
			void *buf = mapaddr + offset;
			offset += fuse_copy_do(cs, &buf, &count);
1005
			kunmap_atomic(mapaddr);
M
Miklos Szeredi 已提交
1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021
		} else
			offset += fuse_copy_do(cs, NULL, &count);
	}
	if (page && !cs->write)
		flush_dcache_page(page);
	return 0;
}

/* Copy pages in the request to/from userspace buffer */
static int fuse_copy_pages(struct fuse_copy_state *cs, unsigned nbytes,
			   int zeroing)
{
	unsigned i;
	struct fuse_req *req = cs->req;

	for (i = 0; i < req->num_pages && (nbytes || zeroing); i++) {
1022
		int err;
1023 1024
		unsigned offset = req->page_descs[i].offset;
		unsigned count = min(nbytes, req->page_descs[i].length);
1025 1026 1027

		err = fuse_copy_page(cs, &req->pages[i], offset, count,
				     zeroing);
M
Miklos Szeredi 已提交
1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039
		if (err)
			return err;

		nbytes -= count;
	}
	return 0;
}

/* Copy a single argument in the request to/from userspace buffer */
static int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size)
{
	while (size) {
M
Miklos Szeredi 已提交
1040 1041 1042 1043 1044
		if (!cs->len) {
			int err = fuse_copy_fill(cs);
			if (err)
				return err;
		}
M
Miklos Szeredi 已提交
1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067
		fuse_copy_do(cs, &val, &size);
	}
	return 0;
}

/* Copy request arguments to/from userspace buffer */
static int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs,
			  unsigned argpages, struct fuse_arg *args,
			  int zeroing)
{
	int err = 0;
	unsigned i;

	for (i = 0; !err && i < numargs; i++)  {
		struct fuse_arg *arg = &args[i];
		if (i == numargs - 1 && argpages)
			err = fuse_copy_pages(cs, arg->size, zeroing);
		else
			err = fuse_copy_one(cs, arg->value, arg->size);
	}
	return err;
}

M
Miklos Szeredi 已提交
1068
static int forget_pending(struct fuse_iqueue *fiq)
1069
{
M
Miklos Szeredi 已提交
1070
	return fiq->forget_list_head.next != NULL;
1071 1072
}

M
Miklos Szeredi 已提交
1073
static int request_pending(struct fuse_iqueue *fiq)
1074
{
M
Miklos Szeredi 已提交
1075 1076
	return !list_empty(&fiq->pending) || !list_empty(&fiq->interrupts) ||
		forget_pending(fiq);
1077 1078 1079 1080 1081 1082 1083 1084
}

/*
 * Transfer an interrupt request to userspace
 *
 * Unlike other requests this is assembled on demand, without a need
 * to allocate a separate fuse_req structure.
 *
1085
 * Called with fiq->lock held, releases it
1086
 */
1087 1088
static int fuse_read_interrupt(struct fuse_iqueue *fiq,
			       struct fuse_copy_state *cs,
1089
			       size_t nbytes, struct fuse_req *req)
1090
__releases(fiq->lock)
1091 1092 1093 1094 1095 1096 1097
{
	struct fuse_in_header ih;
	struct fuse_interrupt_in arg;
	unsigned reqsize = sizeof(ih) + sizeof(arg);
	int err;

	list_del_init(&req->intr_entry);
M
Miklos Szeredi 已提交
1098
	req->intr_unique = fuse_get_unique(fiq);
1099 1100 1101 1102 1103 1104 1105
	memset(&ih, 0, sizeof(ih));
	memset(&arg, 0, sizeof(arg));
	ih.len = reqsize;
	ih.opcode = FUSE_INTERRUPT;
	ih.unique = req->intr_unique;
	arg.unique = req->in.h.unique;

1106
	spin_unlock(&fiq->lock);
1107
	if (nbytes < reqsize)
1108 1109
		return -EINVAL;

1110
	err = fuse_copy_one(cs, &ih, sizeof(ih));
1111
	if (!err)
1112 1113
		err = fuse_copy_one(cs, &arg, sizeof(arg));
	fuse_copy_finish(cs);
1114 1115 1116 1117

	return err ? err : reqsize;
}

M
Miklos Szeredi 已提交
1118
static struct fuse_forget_link *dequeue_forget(struct fuse_iqueue *fiq,
1119 1120
					       unsigned max,
					       unsigned *countp)
1121
{
M
Miklos Szeredi 已提交
1122
	struct fuse_forget_link *head = fiq->forget_list_head.next;
1123 1124
	struct fuse_forget_link **newhead = &head;
	unsigned count;
1125

1126 1127 1128
	for (count = 0; *newhead != NULL && count < max; count++)
		newhead = &(*newhead)->next;

M
Miklos Szeredi 已提交
1129
	fiq->forget_list_head.next = *newhead;
1130
	*newhead = NULL;
M
Miklos Szeredi 已提交
1131 1132
	if (fiq->forget_list_head.next == NULL)
		fiq->forget_list_tail = &fiq->forget_list_head;
1133

1134 1135 1136 1137
	if (countp != NULL)
		*countp = count;

	return head;
1138 1139
}

1140
static int fuse_read_single_forget(struct fuse_iqueue *fiq,
1141 1142
				   struct fuse_copy_state *cs,
				   size_t nbytes)
1143
__releases(fiq->lock)
1144 1145
{
	int err;
M
Miklos Szeredi 已提交
1146
	struct fuse_forget_link *forget = dequeue_forget(fiq, 1, NULL);
1147
	struct fuse_forget_in arg = {
1148
		.nlookup = forget->forget_one.nlookup,
1149 1150 1151
	};
	struct fuse_in_header ih = {
		.opcode = FUSE_FORGET,
1152
		.nodeid = forget->forget_one.nodeid,
M
Miklos Szeredi 已提交
1153
		.unique = fuse_get_unique(fiq),
1154 1155 1156
		.len = sizeof(ih) + sizeof(arg),
	};

1157
	spin_unlock(&fiq->lock);
1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172
	kfree(forget);
	if (nbytes < ih.len)
		return -EINVAL;

	err = fuse_copy_one(cs, &ih, sizeof(ih));
	if (!err)
		err = fuse_copy_one(cs, &arg, sizeof(arg));
	fuse_copy_finish(cs);

	if (err)
		return err;

	return ih.len;
}

1173
static int fuse_read_batch_forget(struct fuse_iqueue *fiq,
1174
				   struct fuse_copy_state *cs, size_t nbytes)
1175
__releases(fiq->lock)
1176 1177 1178 1179 1180 1181 1182 1183
{
	int err;
	unsigned max_forgets;
	unsigned count;
	struct fuse_forget_link *head;
	struct fuse_batch_forget_in arg = { .count = 0 };
	struct fuse_in_header ih = {
		.opcode = FUSE_BATCH_FORGET,
M
Miklos Szeredi 已提交
1184
		.unique = fuse_get_unique(fiq),
1185 1186 1187 1188
		.len = sizeof(ih) + sizeof(arg),
	};

	if (nbytes < ih.len) {
1189
		spin_unlock(&fiq->lock);
1190 1191 1192 1193
		return -EINVAL;
	}

	max_forgets = (nbytes - ih.len) / sizeof(struct fuse_forget_one);
M
Miklos Szeredi 已提交
1194
	head = dequeue_forget(fiq, max_forgets, &count);
1195
	spin_unlock(&fiq->lock);
1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221

	arg.count = count;
	ih.len += count * sizeof(struct fuse_forget_one);
	err = fuse_copy_one(cs, &ih, sizeof(ih));
	if (!err)
		err = fuse_copy_one(cs, &arg, sizeof(arg));

	while (head) {
		struct fuse_forget_link *forget = head;

		if (!err) {
			err = fuse_copy_one(cs, &forget->forget_one,
					    sizeof(forget->forget_one));
		}
		head = forget->next;
		kfree(forget);
	}

	fuse_copy_finish(cs);

	if (err)
		return err;

	return ih.len;
}

1222 1223
static int fuse_read_forget(struct fuse_conn *fc, struct fuse_iqueue *fiq,
			    struct fuse_copy_state *cs,
1224
			    size_t nbytes)
1225
__releases(fiq->lock)
1226
{
M
Miklos Szeredi 已提交
1227
	if (fc->minor < 16 || fiq->forget_list_head.next->next == NULL)
1228
		return fuse_read_single_forget(fiq, cs, nbytes);
1229
	else
1230
		return fuse_read_batch_forget(fiq, cs, nbytes);
1231 1232
}

M
Miklos Szeredi 已提交
1233 1234 1235 1236
/*
 * Read a single request into the userspace filesystem's buffer.  This
 * function waits until a request is available, then removes it from
 * the pending list and copies request data to userspace buffer.  If
1237 1238
 * no reply is needed (FORGET) or request has been aborted or there
 * was an error during the copying then it's finished by calling
M
Miklos Szeredi 已提交
1239 1240 1241
 * request_end().  Otherwise add it to the processing list, and set
 * the 'sent' flag.
 */
1242
static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file,
1243
				struct fuse_copy_state *cs, size_t nbytes)
M
Miklos Szeredi 已提交
1244
{
1245
	ssize_t err;
1246
	struct fuse_conn *fc = fud->fc;
M
Miklos Szeredi 已提交
1247
	struct fuse_iqueue *fiq = &fc->iq;
1248
	struct fuse_pqueue *fpq = &fud->pq;
M
Miklos Szeredi 已提交
1249 1250 1251 1252
	struct fuse_req *req;
	struct fuse_in *in;
	unsigned reqsize;

1253
 restart:
1254 1255 1256 1257 1258
	for (;;) {
		spin_lock(&fiq->lock);
		if (!fiq->connected || request_pending(fiq))
			break;
		spin_unlock(&fiq->lock);
1259

1260 1261 1262
		if (file->f_flags & O_NONBLOCK)
			return -EAGAIN;
		err = wait_event_interruptible_exclusive(fiq->waitq,
M
Miklos Szeredi 已提交
1263
				!fiq->connected || request_pending(fiq));
1264 1265 1266
		if (err)
			return err;
	}
M
Miklos Szeredi 已提交
1267

1268 1269
	if (!fiq->connected) {
		err = (fc->aborted && fc->abort_err) ? -ECONNABORTED : -ENODEV;
M
Miklos Szeredi 已提交
1270
		goto err_unlock;
1271
	}
M
Miklos Szeredi 已提交
1272

M
Miklos Szeredi 已提交
1273 1274
	if (!list_empty(&fiq->interrupts)) {
		req = list_entry(fiq->interrupts.next, struct fuse_req,
1275
				 intr_entry);
1276
		return fuse_read_interrupt(fiq, cs, nbytes, req);
1277 1278
	}

M
Miklos Szeredi 已提交
1279 1280
	if (forget_pending(fiq)) {
		if (list_empty(&fiq->pending) || fiq->forget_batch-- > 0)
1281
			return fuse_read_forget(fc, fiq, cs, nbytes);
1282

M
Miklos Szeredi 已提交
1283 1284
		if (fiq->forget_batch <= -8)
			fiq->forget_batch = 16;
1285 1286
	}

M
Miklos Szeredi 已提交
1287
	req = list_entry(fiq->pending.next, struct fuse_req, list);
M
Miklos Szeredi 已提交
1288
	clear_bit(FR_PENDING, &req->flags);
M
Miklos Szeredi 已提交
1289
	list_del_init(&req->list);
1290
	spin_unlock(&fiq->lock);
M
Miklos Szeredi 已提交
1291

M
Miklos Szeredi 已提交
1292
	in = &req->in;
1293
	reqsize = in->h.len;
1294

1295
	/* If request is too large, reply with an error and restart the read */
1296
	if (nbytes < reqsize) {
1297 1298 1299 1300 1301 1302
		req->out.h.error = -EIO;
		/* SETXATTR is special, since it may contain too large data */
		if (in->h.opcode == FUSE_SETXATTR)
			req->out.h.error = -E2BIG;
		request_end(fc, req);
		goto restart;
M
Miklos Szeredi 已提交
1303
	}
M
Miklos Szeredi 已提交
1304
	spin_lock(&fpq->lock);
1305
	list_add(&req->list, &fpq->io);
M
Miklos Szeredi 已提交
1306
	spin_unlock(&fpq->lock);
1307 1308
	cs->req = req;
	err = fuse_copy_one(cs, &in->h, sizeof(in->h));
1309
	if (!err)
1310
		err = fuse_copy_args(cs, in->numargs, in->argpages,
1311
				     (struct fuse_arg *) in->args, 0);
1312
	fuse_copy_finish(cs);
M
Miklos Szeredi 已提交
1313
	spin_lock(&fpq->lock);
M
Miklos Szeredi 已提交
1314
	clear_bit(FR_LOCKED, &req->flags);
1315
	if (!fpq->connected) {
1316
		err = (fc->aborted && fc->abort_err) ? -ECONNABORTED : -ENODEV;
1317
		goto out_end;
1318
	}
M
Miklos Szeredi 已提交
1319
	if (err) {
1320
		req->out.h.error = -EIO;
1321
		goto out_end;
M
Miklos Szeredi 已提交
1322
	}
M
Miklos Szeredi 已提交
1323
	if (!test_bit(FR_ISREPLY, &req->flags)) {
1324 1325
		err = reqsize;
		goto out_end;
M
Miklos Szeredi 已提交
1326
	}
1327
	list_move_tail(&req->list, &fpq->processing);
1328
	__fuse_get_request(req);
1329
	set_bit(FR_SENT, &req->flags);
M
Miklos Szeredi 已提交
1330
	spin_unlock(&fpq->lock);
1331 1332 1333 1334
	/* matches barrier in request_wait_answer() */
	smp_mb__after_atomic();
	if (test_bit(FR_INTERRUPTED, &req->flags))
		queue_interrupt(fiq, req);
1335
	fuse_put_request(fc, req);
1336

M
Miklos Szeredi 已提交
1337 1338
	return reqsize;

1339
out_end:
1340 1341
	if (!test_bit(FR_PRIVATE, &req->flags))
		list_del_init(&req->list);
M
Miklos Szeredi 已提交
1342
	spin_unlock(&fpq->lock);
1343 1344 1345
	request_end(fc, req);
	return err;

M
Miklos Szeredi 已提交
1346
 err_unlock:
1347
	spin_unlock(&fiq->lock);
M
Miklos Szeredi 已提交
1348 1349 1350
	return err;
}

1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361
static int fuse_dev_open(struct inode *inode, struct file *file)
{
	/*
	 * The fuse device's file's private_data is used to hold
	 * the fuse_conn(ection) when it is mounted, and is used to
	 * keep track of whether the file has been mounted already.
	 */
	file->private_data = NULL;
	return 0;
}

1362
static ssize_t fuse_dev_read(struct kiocb *iocb, struct iov_iter *to)
1363 1364 1365
{
	struct fuse_copy_state cs;
	struct file *file = iocb->ki_filp;
1366 1367 1368
	struct fuse_dev *fud = fuse_get_dev(file);

	if (!fud)
1369 1370
		return -EPERM;

1371 1372 1373
	if (!iter_is_iovec(to))
		return -EINVAL;

1374
	fuse_copy_init(&cs, 1, to);
1375

1376
	return fuse_dev_do_read(fud, file, &cs, iov_iter_count(to));
1377 1378 1379 1380 1381 1382
}

static ssize_t fuse_dev_splice_read(struct file *in, loff_t *ppos,
				    struct pipe_inode_info *pipe,
				    size_t len, unsigned int flags)
{
1383
	int total, ret;
1384 1385 1386
	int page_nr = 0;
	struct pipe_buffer *bufs;
	struct fuse_copy_state cs;
1387 1388 1389
	struct fuse_dev *fud = fuse_get_dev(in);

	if (!fud)
1390 1391
		return -EPERM;

1392 1393
	bufs = kvmalloc_array(pipe->buffers, sizeof(struct pipe_buffer),
			      GFP_KERNEL);
1394 1395 1396
	if (!bufs)
		return -ENOMEM;

1397
	fuse_copy_init(&cs, 1, NULL);
1398 1399
	cs.pipebufs = bufs;
	cs.pipe = pipe;
1400
	ret = fuse_dev_do_read(fud, in, &cs, len);
1401 1402 1403 1404 1405
	if (ret < 0)
		goto out;

	if (pipe->nrbufs + cs.nr_segs > pipe->buffers) {
		ret = -EIO;
1406
		goto out;
1407 1408
	}

1409
	for (ret = total = 0; page_nr < cs.nr_segs; total += ret) {
M
Miklos Szeredi 已提交
1410 1411 1412 1413
		/*
		 * Need to be careful about this.  Having buf->ops in module
		 * code can Oops if the buffer persists after module unload.
		 */
1414
		bufs[page_nr].ops = &nosteal_pipe_buf_ops;
1415
		bufs[page_nr].flags = 0;
1416 1417 1418
		ret = add_to_pipe(pipe, &bufs[page_nr++]);
		if (unlikely(ret < 0))
			break;
1419
	}
1420 1421
	if (total)
		ret = total;
1422 1423
out:
	for (; page_nr < cs.nr_segs; page_nr++)
1424
		put_page(bufs[page_nr].page);
1425

1426
	kvfree(bufs);
1427 1428 1429
	return ret;
}

T
Tejun Heo 已提交
1430 1431 1432 1433
static int fuse_notify_poll(struct fuse_conn *fc, unsigned int size,
			    struct fuse_copy_state *cs)
{
	struct fuse_notify_poll_wakeup_out outarg;
M
Miklos Szeredi 已提交
1434
	int err = -EINVAL;
T
Tejun Heo 已提交
1435 1436

	if (size != sizeof(outarg))
M
Miklos Szeredi 已提交
1437
		goto err;
T
Tejun Heo 已提交
1438 1439 1440

	err = fuse_copy_one(cs, &outarg, sizeof(outarg));
	if (err)
M
Miklos Szeredi 已提交
1441
		goto err;
T
Tejun Heo 已提交
1442

M
Miklos Szeredi 已提交
1443
	fuse_copy_finish(cs);
T
Tejun Heo 已提交
1444
	return fuse_notify_poll_wakeup(fc, &outarg);
M
Miklos Szeredi 已提交
1445 1446 1447 1448

err:
	fuse_copy_finish(cs);
	return err;
T
Tejun Heo 已提交
1449 1450
}

J
John Muir 已提交
1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466
static int fuse_notify_inval_inode(struct fuse_conn *fc, unsigned int size,
				   struct fuse_copy_state *cs)
{
	struct fuse_notify_inval_inode_out outarg;
	int err = -EINVAL;

	if (size != sizeof(outarg))
		goto err;

	err = fuse_copy_one(cs, &outarg, sizeof(outarg));
	if (err)
		goto err;
	fuse_copy_finish(cs);

	down_read(&fc->killsb);
	err = -ENOENT;
1467 1468 1469 1470
	if (fc->sb) {
		err = fuse_reverse_inval_inode(fc->sb, outarg.ino,
					       outarg.off, outarg.len);
	}
J
John Muir 已提交
1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482
	up_read(&fc->killsb);
	return err;

err:
	fuse_copy_finish(cs);
	return err;
}

static int fuse_notify_inval_entry(struct fuse_conn *fc, unsigned int size,
				   struct fuse_copy_state *cs)
{
	struct fuse_notify_inval_entry_out outarg;
F
Fang Wenqi 已提交
1483 1484
	int err = -ENOMEM;
	char *buf;
J
John Muir 已提交
1485 1486
	struct qstr name;

F
Fang Wenqi 已提交
1487 1488 1489 1490 1491
	buf = kzalloc(FUSE_NAME_MAX + 1, GFP_KERNEL);
	if (!buf)
		goto err;

	err = -EINVAL;
J
John Muir 已提交
1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502
	if (size < sizeof(outarg))
		goto err;

	err = fuse_copy_one(cs, &outarg, sizeof(outarg));
	if (err)
		goto err;

	err = -ENAMETOOLONG;
	if (outarg.namelen > FUSE_NAME_MAX)
		goto err;

1503 1504 1505 1506
	err = -EINVAL;
	if (size != sizeof(outarg) + outarg.namelen + 1)
		goto err;

J
John Muir 已提交
1507 1508 1509 1510 1511 1512 1513 1514 1515 1516
	name.name = buf;
	name.len = outarg.namelen;
	err = fuse_copy_one(cs, buf, outarg.namelen + 1);
	if (err)
		goto err;
	fuse_copy_finish(cs);
	buf[outarg.namelen] = 0;

	down_read(&fc->killsb);
	err = -ENOENT;
1517
	if (fc->sb)
1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569
		err = fuse_reverse_inval_entry(fc->sb, outarg.parent, 0, &name);
	up_read(&fc->killsb);
	kfree(buf);
	return err;

err:
	kfree(buf);
	fuse_copy_finish(cs);
	return err;
}

static int fuse_notify_delete(struct fuse_conn *fc, unsigned int size,
			      struct fuse_copy_state *cs)
{
	struct fuse_notify_delete_out outarg;
	int err = -ENOMEM;
	char *buf;
	struct qstr name;

	buf = kzalloc(FUSE_NAME_MAX + 1, GFP_KERNEL);
	if (!buf)
		goto err;

	err = -EINVAL;
	if (size < sizeof(outarg))
		goto err;

	err = fuse_copy_one(cs, &outarg, sizeof(outarg));
	if (err)
		goto err;

	err = -ENAMETOOLONG;
	if (outarg.namelen > FUSE_NAME_MAX)
		goto err;

	err = -EINVAL;
	if (size != sizeof(outarg) + outarg.namelen + 1)
		goto err;

	name.name = buf;
	name.len = outarg.namelen;
	err = fuse_copy_one(cs, buf, outarg.namelen + 1);
	if (err)
		goto err;
	fuse_copy_finish(cs);
	buf[outarg.namelen] = 0;

	down_read(&fc->killsb);
	err = -ENOENT;
	if (fc->sb)
		err = fuse_reverse_inval_entry(fc->sb, outarg.parent,
					       outarg.child, &name);
J
John Muir 已提交
1570
	up_read(&fc->killsb);
F
Fang Wenqi 已提交
1571
	kfree(buf);
J
John Muir 已提交
1572 1573 1574
	return err;

err:
F
Fang Wenqi 已提交
1575
	kfree(buf);
J
John Muir 已提交
1576 1577 1578 1579
	fuse_copy_finish(cs);
	return err;
}

M
Miklos Szeredi 已提交
1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618
static int fuse_notify_store(struct fuse_conn *fc, unsigned int size,
			     struct fuse_copy_state *cs)
{
	struct fuse_notify_store_out outarg;
	struct inode *inode;
	struct address_space *mapping;
	u64 nodeid;
	int err;
	pgoff_t index;
	unsigned int offset;
	unsigned int num;
	loff_t file_size;
	loff_t end;

	err = -EINVAL;
	if (size < sizeof(outarg))
		goto out_finish;

	err = fuse_copy_one(cs, &outarg, sizeof(outarg));
	if (err)
		goto out_finish;

	err = -EINVAL;
	if (size - sizeof(outarg) != outarg.size)
		goto out_finish;

	nodeid = outarg.nodeid;

	down_read(&fc->killsb);

	err = -ENOENT;
	if (!fc->sb)
		goto out_up_killsb;

	inode = ilookup5(fc->sb, nodeid, fuse_inode_eq, &nodeid);
	if (!inode)
		goto out_up_killsb;

	mapping = inode->i_mapping;
1619 1620
	index = outarg.offset >> PAGE_SHIFT;
	offset = outarg.offset & ~PAGE_MASK;
M
Miklos Szeredi 已提交
1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638
	file_size = i_size_read(inode);
	end = outarg.offset + outarg.size;
	if (end > file_size) {
		file_size = end;
		fuse_write_update_size(inode, file_size);
	}

	num = outarg.size;
	while (num) {
		struct page *page;
		unsigned int this_num;

		err = -ENOMEM;
		page = find_or_create_page(mapping, index,
					   mapping_gfp_mask(mapping));
		if (!page)
			goto out_iput;

1639
		this_num = min_t(unsigned, num, PAGE_SIZE - offset);
M
Miklos Szeredi 已提交
1640
		err = fuse_copy_page(cs, &page, offset, this_num, 0);
1641
		if (!err && offset == 0 &&
1642
		    (this_num == PAGE_SIZE || file_size == end))
M
Miklos Szeredi 已提交
1643 1644
			SetPageUptodate(page);
		unlock_page(page);
1645
		put_page(page);
M
Miklos Szeredi 已提交
1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665

		if (err)
			goto out_iput;

		num -= this_num;
		offset = 0;
		index++;
	}

	err = 0;

out_iput:
	iput(inode);
out_up_killsb:
	up_read(&fc->killsb);
out_finish:
	fuse_copy_finish(cs);
	return err;
}

M
Miklos Szeredi 已提交
1666 1667
static void fuse_retrieve_end(struct fuse_conn *fc, struct fuse_req *req)
{
1668
	release_pages(req->pages, req->num_pages);
M
Miklos Szeredi 已提交
1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680
}

static int fuse_retrieve(struct fuse_conn *fc, struct inode *inode,
			 struct fuse_notify_retrieve_out *outarg)
{
	int err;
	struct address_space *mapping = inode->i_mapping;
	struct fuse_req *req;
	pgoff_t index;
	loff_t file_size;
	unsigned int num;
	unsigned int offset;
1681
	size_t total_len = 0;
M
Maxim Patlasov 已提交
1682
	int num_pages;
M
Miklos Szeredi 已提交
1683

1684
	offset = outarg->offset & ~PAGE_MASK;
M
Maxim Patlasov 已提交
1685 1686
	file_size = i_size_read(inode);

1687
	num = min(outarg->size, fc->max_write);
M
Maxim Patlasov 已提交
1688 1689 1690 1691 1692 1693 1694 1695 1696
	if (outarg->offset > file_size)
		num = 0;
	else if (outarg->offset + num > file_size)
		num = file_size - outarg->offset;

	num_pages = (num + offset + PAGE_SIZE - 1) >> PAGE_SHIFT;
	num_pages = min(num_pages, FUSE_MAX_PAGES_PER_REQ);

	req = fuse_get_req(fc, num_pages);
M
Miklos Szeredi 已提交
1697 1698 1699 1700 1701 1702 1703 1704 1705
	if (IS_ERR(req))
		return PTR_ERR(req);

	req->in.h.opcode = FUSE_NOTIFY_REPLY;
	req->in.h.nodeid = outarg->nodeid;
	req->in.numargs = 2;
	req->in.argpages = 1;
	req->end = fuse_retrieve_end;

1706
	index = outarg->offset >> PAGE_SHIFT;
M
Miklos Szeredi 已提交
1707

M
Maxim Patlasov 已提交
1708
	while (num && req->num_pages < num_pages) {
M
Miklos Szeredi 已提交
1709 1710 1711 1712 1713 1714 1715
		struct page *page;
		unsigned int this_num;

		page = find_get_page(mapping, index);
		if (!page)
			break;

1716
		this_num = min_t(unsigned, num, PAGE_SIZE - offset);
M
Miklos Szeredi 已提交
1717
		req->pages[req->num_pages] = page;
1718
		req->page_descs[req->num_pages].offset = offset;
1719
		req->page_descs[req->num_pages].length = this_num;
M
Miklos Szeredi 已提交
1720 1721
		req->num_pages++;

M
Miklos Szeredi 已提交
1722
		offset = 0;
M
Miklos Szeredi 已提交
1723 1724
		num -= this_num;
		total_len += this_num;
M
Miklos Szeredi 已提交
1725
		index++;
M
Miklos Szeredi 已提交
1726 1727 1728 1729 1730 1731 1732 1733
	}
	req->misc.retrieve_in.offset = outarg->offset;
	req->misc.retrieve_in.size = total_len;
	req->in.args[0].size = sizeof(req->misc.retrieve_in);
	req->in.args[0].value = &req->misc.retrieve_in;
	req->in.args[1].size = total_len;

	err = fuse_request_send_notify_reply(fc, req, outarg->notify_unique);
M
Miklos Szeredi 已提交
1734
	if (err) {
M
Miklos Szeredi 已提交
1735
		fuse_retrieve_end(fc, req);
M
Miklos Szeredi 已提交
1736 1737
		fuse_put_request(fc, req);
	}
M
Miklos Szeredi 已提交
1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778

	return err;
}

static int fuse_notify_retrieve(struct fuse_conn *fc, unsigned int size,
				struct fuse_copy_state *cs)
{
	struct fuse_notify_retrieve_out outarg;
	struct inode *inode;
	int err;

	err = -EINVAL;
	if (size != sizeof(outarg))
		goto copy_finish;

	err = fuse_copy_one(cs, &outarg, sizeof(outarg));
	if (err)
		goto copy_finish;

	fuse_copy_finish(cs);

	down_read(&fc->killsb);
	err = -ENOENT;
	if (fc->sb) {
		u64 nodeid = outarg.nodeid;

		inode = ilookup5(fc->sb, nodeid, fuse_inode_eq, &nodeid);
		if (inode) {
			err = fuse_retrieve(fc, inode, &outarg);
			iput(inode);
		}
	}
	up_read(&fc->killsb);

	return err;

copy_finish:
	fuse_copy_finish(cs);
	return err;
}

1779 1780 1781
static int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code,
		       unsigned int size, struct fuse_copy_state *cs)
{
M
Miklos Szeredi 已提交
1782 1783 1784
	/* Don't try to move pages (yet) */
	cs->move_pages = 0;

1785
	switch (code) {
T
Tejun Heo 已提交
1786 1787 1788
	case FUSE_NOTIFY_POLL:
		return fuse_notify_poll(fc, size, cs);

J
John Muir 已提交
1789 1790 1791 1792 1793 1794
	case FUSE_NOTIFY_INVAL_INODE:
		return fuse_notify_inval_inode(fc, size, cs);

	case FUSE_NOTIFY_INVAL_ENTRY:
		return fuse_notify_inval_entry(fc, size, cs);

M
Miklos Szeredi 已提交
1795 1796 1797
	case FUSE_NOTIFY_STORE:
		return fuse_notify_store(fc, size, cs);

M
Miklos Szeredi 已提交
1798 1799 1800
	case FUSE_NOTIFY_RETRIEVE:
		return fuse_notify_retrieve(fc, size, cs);

1801 1802 1803
	case FUSE_NOTIFY_DELETE:
		return fuse_notify_delete(fc, size, cs);

1804
	default:
M
Miklos Szeredi 已提交
1805
		fuse_copy_finish(cs);
1806 1807 1808 1809
		return -EINVAL;
	}
}

M
Miklos Szeredi 已提交
1810
/* Look up request on processing list by unique ID */
1811
static struct fuse_req *request_find(struct fuse_pqueue *fpq, u64 unique)
M
Miklos Szeredi 已提交
1812
{
1813
	struct fuse_req *req;
M
Miklos Szeredi 已提交
1814

1815
	list_for_each_entry(req, &fpq->processing, list) {
1816
		if (req->in.h.unique == unique || req->intr_unique == unique)
M
Miklos Szeredi 已提交
1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851
			return req;
	}
	return NULL;
}

static int copy_out_args(struct fuse_copy_state *cs, struct fuse_out *out,
			 unsigned nbytes)
{
	unsigned reqsize = sizeof(struct fuse_out_header);

	if (out->h.error)
		return nbytes != reqsize ? -EINVAL : 0;

	reqsize += len_args(out->numargs, out->args);

	if (reqsize < nbytes || (reqsize > nbytes && !out->argvar))
		return -EINVAL;
	else if (reqsize > nbytes) {
		struct fuse_arg *lastarg = &out->args[out->numargs-1];
		unsigned diffsize = reqsize - nbytes;
		if (diffsize > lastarg->size)
			return -EINVAL;
		lastarg->size -= diffsize;
	}
	return fuse_copy_args(cs, out->numargs, out->argpages, out->args,
			      out->page_zeroing);
}

/*
 * Write a single reply to a request.  First the header is copied from
 * the write buffer.  The request is then searched on the processing
 * list by the unique ID found in the header.  If found, then remove
 * it from the list and copy the rest of the buffer to the request.
 * The request is finished by calling request_end()
 */
1852
static ssize_t fuse_dev_do_write(struct fuse_dev *fud,
1853
				 struct fuse_copy_state *cs, size_t nbytes)
M
Miklos Szeredi 已提交
1854 1855
{
	int err;
1856 1857
	struct fuse_conn *fc = fud->fc;
	struct fuse_pqueue *fpq = &fud->pq;
M
Miklos Szeredi 已提交
1858 1859 1860 1861 1862 1863
	struct fuse_req *req;
	struct fuse_out_header oh;

	if (nbytes < sizeof(struct fuse_out_header))
		return -EINVAL;

1864
	err = fuse_copy_one(cs, &oh, sizeof(oh));
M
Miklos Szeredi 已提交
1865 1866
	if (err)
		goto err_finish;
1867 1868 1869 1870 1871 1872 1873 1874 1875 1876

	err = -EINVAL;
	if (oh.len != nbytes)
		goto err_finish;

	/*
	 * Zero oh.unique indicates unsolicited notification message
	 * and error contains notification code.
	 */
	if (!oh.unique) {
1877
		err = fuse_notify(fc, oh.error, nbytes - sizeof(oh), cs);
1878 1879 1880
		return err ? err : nbytes;
	}

M
Miklos Szeredi 已提交
1881
	err = -EINVAL;
1882
	if (oh.error <= -1000 || oh.error > 0)
M
Miklos Szeredi 已提交
1883 1884
		goto err_finish;

M
Miklos Szeredi 已提交
1885
	spin_lock(&fpq->lock);
1886
	err = -ENOENT;
1887
	if (!fpq->connected)
M
Miklos Szeredi 已提交
1888
		goto err_unlock_pq;
1889

1890
	req = request_find(fpq, oh.unique);
M
Miklos Szeredi 已提交
1891
	if (!req)
M
Miklos Szeredi 已提交
1892
		goto err_unlock_pq;
M
Miklos Szeredi 已提交
1893

1894 1895
	/* Is it an interrupt reply? */
	if (req->intr_unique == oh.unique) {
1896
		__fuse_get_request(req);
M
Miklos Szeredi 已提交
1897 1898
		spin_unlock(&fpq->lock);

1899
		err = -EINVAL;
1900 1901
		if (nbytes != sizeof(struct fuse_out_header)) {
			fuse_put_request(fc, req);
1902
			goto err_finish;
1903
		}
1904 1905 1906 1907

		if (oh.error == -ENOSYS)
			fc->no_interrupt = 1;
		else if (oh.error == -EAGAIN)
M
Miklos Szeredi 已提交
1908
			queue_interrupt(&fc->iq, req);
1909
		fuse_put_request(fc, req);
1910

1911
		fuse_copy_finish(cs);
1912 1913 1914
		return nbytes;
	}

M
Miklos Szeredi 已提交
1915
	clear_bit(FR_SENT, &req->flags);
1916
	list_move(&req->list, &fpq->io);
M
Miklos Szeredi 已提交
1917
	req->out.h = oh;
M
Miklos Szeredi 已提交
1918
	set_bit(FR_LOCKED, &req->flags);
M
Miklos Szeredi 已提交
1919
	spin_unlock(&fpq->lock);
1920
	cs->req = req;
1921 1922
	if (!req->out.page_replace)
		cs->move_pages = 0;
M
Miklos Szeredi 已提交
1923

1924 1925
	err = copy_out_args(cs, &req->out, nbytes);
	fuse_copy_finish(cs);
M
Miklos Szeredi 已提交
1926

M
Miklos Szeredi 已提交
1927
	spin_lock(&fpq->lock);
M
Miklos Szeredi 已提交
1928
	clear_bit(FR_LOCKED, &req->flags);
1929
	if (!fpq->connected)
M
Miklos Szeredi 已提交
1930 1931
		err = -ENOENT;
	else if (err)
M
Miklos Szeredi 已提交
1932
		req->out.h.error = -EIO;
1933 1934
	if (!test_bit(FR_PRIVATE, &req->flags))
		list_del_init(&req->list);
M
Miklos Szeredi 已提交
1935
	spin_unlock(&fpq->lock);
1936

M
Miklos Szeredi 已提交
1937 1938 1939 1940
	request_end(fc, req);

	return err ? err : nbytes;

M
Miklos Szeredi 已提交
1941 1942
 err_unlock_pq:
	spin_unlock(&fpq->lock);
M
Miklos Szeredi 已提交
1943
 err_finish:
1944
	fuse_copy_finish(cs);
M
Miklos Szeredi 已提交
1945 1946 1947
	return err;
}

1948
static ssize_t fuse_dev_write(struct kiocb *iocb, struct iov_iter *from)
1949 1950
{
	struct fuse_copy_state cs;
1951 1952 1953
	struct fuse_dev *fud = fuse_get_dev(iocb->ki_filp);

	if (!fud)
1954 1955
		return -EPERM;

1956 1957 1958
	if (!iter_is_iovec(from))
		return -EINVAL;

1959
	fuse_copy_init(&cs, 0, from);
1960

1961
	return fuse_dev_do_write(fud, &cs, iov_iter_count(from));
1962 1963 1964 1965 1966 1967 1968 1969 1970 1971
}

static ssize_t fuse_dev_splice_write(struct pipe_inode_info *pipe,
				     struct file *out, loff_t *ppos,
				     size_t len, unsigned int flags)
{
	unsigned nbuf;
	unsigned idx;
	struct pipe_buffer *bufs;
	struct fuse_copy_state cs;
1972
	struct fuse_dev *fud;
1973 1974 1975
	size_t rem;
	ssize_t ret;

1976 1977
	fud = fuse_get_dev(out);
	if (!fud)
1978 1979
		return -EPERM;

1980 1981
	pipe_lock(pipe);

1982
	bufs = kvmalloc_array(pipe->nrbufs, sizeof(struct pipe_buffer),
1983
			      GFP_KERNEL);
1984 1985
	if (!bufs) {
		pipe_unlock(pipe);
1986
		return -ENOMEM;
1987
	}
1988 1989 1990 1991 1992 1993 1994

	nbuf = 0;
	rem = 0;
	for (idx = 0; idx < pipe->nrbufs && rem < len; idx++)
		rem += pipe->bufs[(pipe->curbuf + idx) & (pipe->buffers - 1)].len;

	ret = -EINVAL;
1995 1996
	if (rem < len)
		goto out_free;
1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013

	rem = len;
	while (rem) {
		struct pipe_buffer *ibuf;
		struct pipe_buffer *obuf;

		BUG_ON(nbuf >= pipe->buffers);
		BUG_ON(!pipe->nrbufs);
		ibuf = &pipe->bufs[pipe->curbuf];
		obuf = &bufs[nbuf];

		if (rem >= ibuf->len) {
			*obuf = *ibuf;
			ibuf->ops = NULL;
			pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);
			pipe->nrbufs--;
		} else {
2014 2015 2016
			if (!pipe_buf_get(pipe, ibuf))
				goto out_free;

2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027
			*obuf = *ibuf;
			obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
			obuf->len = rem;
			ibuf->offset += obuf->len;
			ibuf->len -= obuf->len;
		}
		nbuf++;
		rem -= obuf->len;
	}
	pipe_unlock(pipe);

2028
	fuse_copy_init(&cs, 0, NULL);
2029
	cs.pipebufs = bufs;
2030
	cs.nr_segs = nbuf;
2031 2032
	cs.pipe = pipe;

2033 2034 2035
	if (flags & SPLICE_F_MOVE)
		cs.move_pages = 1;

2036
	ret = fuse_dev_do_write(fud, &cs, len);
2037

2038
	pipe_lock(pipe);
2039
out_free:
2040 2041
	for (idx = 0; idx < nbuf; idx++)
		pipe_buf_release(pipe, &bufs[idx]);
2042
	pipe_unlock(pipe);
2043

2044
	kvfree(bufs);
2045 2046 2047
	return ret;
}

A
Al Viro 已提交
2048
static __poll_t fuse_dev_poll(struct file *file, poll_table *wait)
M
Miklos Szeredi 已提交
2049
{
2050
	__poll_t mask = EPOLLOUT | EPOLLWRNORM;
M
Miklos Szeredi 已提交
2051
	struct fuse_iqueue *fiq;
2052 2053 2054
	struct fuse_dev *fud = fuse_get_dev(file);

	if (!fud)
2055
		return EPOLLERR;
M
Miklos Szeredi 已提交
2056

2057
	fiq = &fud->fc->iq;
M
Miklos Szeredi 已提交
2058
	poll_wait(file, &fiq->waitq, wait);
M
Miklos Szeredi 已提交
2059

2060
	spin_lock(&fiq->lock);
2061
	if (!fiq->connected)
2062
		mask = EPOLLERR;
M
Miklos Szeredi 已提交
2063
	else if (request_pending(fiq))
2064
		mask |= EPOLLIN | EPOLLRDNORM;
2065
	spin_unlock(&fiq->lock);
M
Miklos Szeredi 已提交
2066 2067 2068 2069

	return mask;
}

2070 2071 2072
/*
 * Abort all requests on the given list (pending or processing)
 *
2073
 * This function releases and reacquires fc->lock
2074
 */
M
Miklos Szeredi 已提交
2075 2076 2077 2078 2079 2080
static void end_requests(struct fuse_conn *fc, struct list_head *head)
{
	while (!list_empty(head)) {
		struct fuse_req *req;
		req = list_entry(head->next, struct fuse_req, list);
		req->out.h.error = -ECONNABORTED;
M
Miklos Szeredi 已提交
2081
		clear_bit(FR_SENT, &req->flags);
2082
		list_del_init(&req->list);
M
Miklos Szeredi 已提交
2083 2084 2085 2086
		request_end(fc, req);
	}
}

2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101
static void end_polls(struct fuse_conn *fc)
{
	struct rb_node *p;

	p = rb_first(&fc->polled_files);

	while (p) {
		struct fuse_file *ff;
		ff = rb_entry(p, struct fuse_file, polled_node);
		wake_up_interruptible_all(&ff->poll_wait);

		p = rb_next(p);
	}
}

2102 2103 2104
/*
 * Abort all requests.
 *
M
Miklos Szeredi 已提交
2105 2106 2107 2108 2109 2110 2111
 * Emergency exit in case of a malicious or accidental deadlock, or just a hung
 * filesystem.
 *
 * The same effect is usually achievable through killing the filesystem daemon
 * and all users of the filesystem.  The exception is the combination of an
 * asynchronous request and the tricky deadlock (see
 * Documentation/filesystems/fuse.txt).
2112
 *
M
Miklos Szeredi 已提交
2113 2114 2115 2116 2117 2118
 * Aborting requests under I/O goes as follows: 1: Separate out unlocked
 * requests, they should be finished off immediately.  Locked requests will be
 * finished after unlock; see unlock_request(). 2: Finish off the unlocked
 * requests.  It is possible that some request will finish before we can.  This
 * is OK, the request will in that case be removed from the list before we touch
 * it.
2119
 */
2120
void fuse_abort_conn(struct fuse_conn *fc, bool is_abort)
2121
{
M
Miklos Szeredi 已提交
2122 2123
	struct fuse_iqueue *fiq = &fc->iq;

2124
	spin_lock(&fc->lock);
2125
	if (fc->connected) {
2126
		struct fuse_dev *fud;
M
Miklos Szeredi 已提交
2127
		struct fuse_req *req, *next;
2128
		LIST_HEAD(to_end);
M
Miklos Szeredi 已提交
2129

2130
		fc->connected = 0;
2131
		fc->blocked = 0;
2132
		fc->aborted = is_abort;
2133
		fuse_set_initialized(fc);
2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144
		list_for_each_entry(fud, &fc->devices, entry) {
			struct fuse_pqueue *fpq = &fud->pq;

			spin_lock(&fpq->lock);
			fpq->connected = 0;
			list_for_each_entry_safe(req, next, &fpq->io, list) {
				req->out.h.error = -ECONNABORTED;
				spin_lock(&req->waitq.lock);
				set_bit(FR_ABORTED, &req->flags);
				if (!test_bit(FR_LOCKED, &req->flags)) {
					set_bit(FR_PRIVATE, &req->flags);
M
Miklos Szeredi 已提交
2145
					__fuse_get_request(req);
2146
					list_move(&req->list, &to_end);
2147 2148
				}
				spin_unlock(&req->waitq.lock);
2149
			}
2150
			list_splice_tail_init(&fpq->processing, &to_end);
2151
			spin_unlock(&fpq->lock);
M
Miklos Szeredi 已提交
2152
		}
M
Miklos Szeredi 已提交
2153 2154
		fc->max_background = UINT_MAX;
		flush_bg_queue(fc);
2155

2156
		spin_lock(&fiq->lock);
2157
		fiq->connected = 0;
2158
		list_for_each_entry(req, &fiq->pending, list)
2159
			clear_bit(FR_PENDING, &req->flags);
2160
		list_splice_tail_init(&fiq->pending, &to_end);
2161 2162
		while (forget_pending(fiq))
			kfree(dequeue_forget(fiq, 1, NULL));
2163 2164
		wake_up_all(&fiq->waitq);
		spin_unlock(&fiq->lock);
2165
		kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
2166 2167 2168
		end_polls(fc);
		wake_up_all(&fc->blocked_waitq);
		spin_unlock(&fc->lock);
2169

2170
		end_requests(fc, &to_end);
2171 2172
	} else {
		spin_unlock(&fc->lock);
2173 2174
	}
}
2175
EXPORT_SYMBOL_GPL(fuse_abort_conn);
2176

2177 2178
void fuse_wait_aborted(struct fuse_conn *fc)
{
2179 2180
	/* matches implicit memory barrier in fuse_drop_waiting() */
	smp_mb();
2181 2182 2183
	wait_event(fc->blocked_waitq, atomic_read(&fc->num_waiting) == 0);
}

2184
int fuse_dev_release(struct inode *inode, struct file *file)
M
Miklos Szeredi 已提交
2185
{
2186 2187 2188 2189
	struct fuse_dev *fud = fuse_get_dev(file);

	if (fud) {
		struct fuse_conn *fc = fud->fc;
2190
		struct fuse_pqueue *fpq = &fud->pq;
2191
		LIST_HEAD(to_end);
2192

2193
		spin_lock(&fpq->lock);
2194
		WARN_ON(!list_empty(&fpq->io));
2195 2196 2197 2198 2199
		list_splice_init(&fpq->processing, &to_end);
		spin_unlock(&fpq->lock);

		end_requests(fc, &to_end);

2200 2201 2202
		/* Are we the last open device? */
		if (atomic_dec_and_test(&fc->dev_count)) {
			WARN_ON(fc->iq.fasync != NULL);
2203
			fuse_abort_conn(fc, false);
2204
		}
2205
		fuse_dev_free(fud);
2206
	}
M
Miklos Szeredi 已提交
2207 2208
	return 0;
}
2209
EXPORT_SYMBOL_GPL(fuse_dev_release);
M
Miklos Szeredi 已提交
2210

2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245
/*
 * Flush all pending processing requests.
 *
 * The failover procedure will reuse the fuse_conn after usersapce crash and
 * recovery. But the reqs in the processing queue will never get the reply,
 * making the application stuck forever.
 *
 * So we need to flush these reqs by sysfs api. We only flush the reqs in
 * the processing queue, because these reqs have been sent to userspace.
 * Firstly we need to dequeue the req from the processing queue, and secondly
 * we need to call request_end to finish it.
 */
void fuse_flush_pq(struct fuse_conn *fc)
{
	struct fuse_dev *fud;
	LIST_HEAD(to_end);

	spin_lock(&fc->lock);
	if (!fc->connected) {
		spin_unlock(&fc->lock);
		return;
	}
	list_for_each_entry(fud, &fc->devices, entry) {
		struct fuse_pqueue *fpq = &fud->pq;

		spin_lock(&fpq->lock);
		WARN_ON(!list_empty(&fpq->io));
		list_splice_init(&fpq->processing, &to_end);
		spin_unlock(&fpq->lock);
	}
	spin_unlock(&fc->lock);

	end_requests(fc, &to_end);
}

2246 2247
static int fuse_dev_fasync(int fd, struct file *file, int on)
{
2248 2249 2250
	struct fuse_dev *fud = fuse_get_dev(file);

	if (!fud)
2251
		return -EPERM;
2252 2253

	/* No locking - fasync_helper does its own locking */
2254
	return fasync_helper(fd, file, on, &fud->fc->iq.fasync);
2255 2256
}

M
Miklos Szeredi 已提交
2257 2258
static int fuse_device_clone(struct fuse_conn *fc, struct file *new)
{
2259 2260
	struct fuse_dev *fud;

M
Miklos Szeredi 已提交
2261 2262 2263
	if (new->private_data)
		return -EINVAL;

2264 2265 2266 2267 2268
	fud = fuse_dev_alloc(fc);
	if (!fud)
		return -ENOMEM;

	new->private_data = fud;
2269
	atomic_inc(&fc->dev_count);
M
Miklos Szeredi 已提交
2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287

	return 0;
}

static long fuse_dev_ioctl(struct file *file, unsigned int cmd,
			   unsigned long arg)
{
	int err = -ENOTTY;

	if (cmd == FUSE_DEV_IOC_CLONE) {
		int oldfd;

		err = -EFAULT;
		if (!get_user(oldfd, (__u32 __user *) arg)) {
			struct file *old = fget(oldfd);

			err = -EINVAL;
			if (old) {
J
Jann Horn 已提交
2288 2289 2290 2291 2292 2293 2294 2295 2296
				struct fuse_dev *fud = NULL;

				/*
				 * Check against file->f_op because CUSE
				 * uses the same ioctl handler.
				 */
				if (old->f_op == file->f_op &&
				    old->f_cred->user_ns == file->f_cred->user_ns)
					fud = fuse_get_dev(old);
M
Miklos Szeredi 已提交
2297

2298
				if (fud) {
M
Miklos Szeredi 已提交
2299
					mutex_lock(&fuse_mutex);
2300
					err = fuse_device_clone(fud->fc, file);
M
Miklos Szeredi 已提交
2301 2302 2303 2304 2305 2306 2307 2308 2309
					mutex_unlock(&fuse_mutex);
				}
				fput(old);
			}
		}
	}
	return err;
}

2310
const struct file_operations fuse_dev_operations = {
M
Miklos Szeredi 已提交
2311
	.owner		= THIS_MODULE,
2312
	.open		= fuse_dev_open,
M
Miklos Szeredi 已提交
2313
	.llseek		= no_llseek,
2314
	.read_iter	= fuse_dev_read,
2315
	.splice_read	= fuse_dev_splice_read,
2316
	.write_iter	= fuse_dev_write,
2317
	.splice_write	= fuse_dev_splice_write,
M
Miklos Szeredi 已提交
2318 2319
	.poll		= fuse_dev_poll,
	.release	= fuse_dev_release,
2320
	.fasync		= fuse_dev_fasync,
M
Miklos Szeredi 已提交
2321 2322
	.unlocked_ioctl = fuse_dev_ioctl,
	.compat_ioctl   = fuse_dev_ioctl,
M
Miklos Szeredi 已提交
2323
};
2324
EXPORT_SYMBOL_GPL(fuse_dev_operations);
M
Miklos Szeredi 已提交
2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336

static struct miscdevice fuse_miscdevice = {
	.minor = FUSE_MINOR,
	.name  = "fuse",
	.fops = &fuse_dev_operations,
};

int __init fuse_dev_init(void)
{
	int err = -ENOMEM;
	fuse_req_cachep = kmem_cache_create("fuse_request",
					    sizeof(struct fuse_req),
2337
					    0, 0, NULL);
M
Miklos Szeredi 已提交
2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357
	if (!fuse_req_cachep)
		goto out;

	err = misc_register(&fuse_miscdevice);
	if (err)
		goto out_cache_clean;

	return 0;

 out_cache_clean:
	kmem_cache_destroy(fuse_req_cachep);
 out:
	return err;
}

void fuse_dev_cleanup(void)
{
	misc_deregister(&fuse_miscdevice);
	kmem_cache_destroy(fuse_req_cachep);
}