loop.c 50.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
/*
 *  linux/drivers/block/loop.c
 *
 *  Written by Theodore Ts'o, 3/29/93
 *
 * Copyright 1993 by Theodore Ts'o.  Redistribution of this file is
 * permitted under the GNU General Public License.
 *
 * DES encryption plus some minor changes by Werner Almesberger, 30-MAY-1993
 * more DES encryption plus IDEA encryption by Nicholas J. Leon, June 20, 1996
 *
 * Modularized and updated for 1.1.16 kernel - Mitch Dsouza 28th May 1994
 * Adapted for 1.3.59 kernel - Andries Brouwer, 1 Feb 1996
 *
 * Fixed do_loop_request() re-entrancy - Vincent.Renardias@waw.com Mar 20, 1997
 *
 * Added devfs support - Richard Gooch <rgooch@atnf.csiro.au> 16-Jan-1998
 *
 * Handle sparse backing files correctly - Kenn Humborg, Jun 28, 1998
 *
 * Loadable modules and other fixes by AK, 1998
 *
 * Make real block number available to downstream transfer functions, enables
 * CBC (and relatives) mode encryption requiring unique IVs per data block.
 * Reed H. Petty, rhp@draper.net
 *
 * Maximum number of loop devices now dynamic via max_loop module parameter.
 * Russell Kroll <rkroll@exploits.org> 19990701
 *
 * Maximum number of loop devices when compiled-in now selectable by passing
 * max_loop=<1-255> to the kernel on boot.
32
 * Erik I. Bolsø, <eriki@himolde.no>, Oct 31, 1999
L
Linus Torvalds 已提交
33 34 35 36 37 38 39 40 41 42
 *
 * Completely rewrite request handling to be make_request_fn style and
 * non blocking, pushing work to a helper thread. Lots of fixes from
 * Al Viro too.
 * Jens Axboe <axboe@suse.de>, Nov 2000
 *
 * Support up to 256 loop devices
 * Heinz Mauelshagen <mge@sistina.com>, Feb 2002
 *
 * Support for falling back on the write file operation when the address space
43
 * operations write_begin is not available on the backing filesystem.
L
Linus Torvalds 已提交
44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65
 * Anton Altaparmakov, 16 Feb 2005
 *
 * Still To Fix:
 * - Advisory locking is ignored here.
 * - Should use an own CAP_* category instead of CAP_SYS_ADMIN
 *
 */

#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/sched.h>
#include <linux/fs.h>
#include <linux/file.h>
#include <linux/stat.h>
#include <linux/errno.h>
#include <linux/major.h>
#include <linux/wait.h>
#include <linux/blkdev.h>
#include <linux/blkpg.h>
#include <linux/init.h>
#include <linux/swap.h>
#include <linux/slab.h>
66
#include <linux/compat.h>
L
Linus Torvalds 已提交
67
#include <linux/suspend.h>
68
#include <linux/freezer.h>
69
#include <linux/mutex.h>
L
Linus Torvalds 已提交
70 71 72
#include <linux/writeback.h>
#include <linux/completion.h>
#include <linux/highmem.h>
73
#include <linux/kthread.h>
74
#include <linux/splice.h>
75
#include <linux/sysfs.h>
76
#include <linux/miscdevice.h>
77
#include <linux/falloc.h>
A
Al Viro 已提交
78
#include <linux/uio.h>
A
Al Viro 已提交
79
#include "loop.h"
L
Linus Torvalds 已提交
80 81 82

#include <asm/uaccess.h>

83 84
static DEFINE_IDR(loop_index_idr);
static DEFINE_MUTEX(loop_index_mutex);
L
Linus Torvalds 已提交
85

86 87 88
static int max_part;
static int part_shift;

L
Linus Torvalds 已提交
89 90 91 92 93
static int transfer_xor(struct loop_device *lo, int cmd,
			struct page *raw_page, unsigned raw_off,
			struct page *loop_page, unsigned loop_off,
			int size, sector_t real_block)
{
94 95
	char *raw_buf = kmap_atomic(raw_page) + raw_off;
	char *loop_buf = kmap_atomic(loop_page) + loop_off;
L
Linus Torvalds 已提交
96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111
	char *in, *out, *key;
	int i, keysize;

	if (cmd == READ) {
		in = raw_buf;
		out = loop_buf;
	} else {
		in = loop_buf;
		out = raw_buf;
	}

	key = lo->lo_encrypt_key;
	keysize = lo->lo_encrypt_key_size;
	for (i = 0; i < size; i++)
		*out++ = *in++ ^ key[(i & 511) % keysize];

112 113
	kunmap_atomic(loop_buf);
	kunmap_atomic(raw_buf);
L
Linus Torvalds 已提交
114 115 116 117 118 119 120 121 122 123 124 125 126
	cond_resched();
	return 0;
}

static int xor_init(struct loop_device *lo, const struct loop_info64 *info)
{
	if (unlikely(info->lo_encrypt_key_size <= 0))
		return -EINVAL;
	return 0;
}

static struct loop_func_table none_funcs = {
	.number = LO_CRYPT_NONE,
127
}; 
L
Linus Torvalds 已提交
128 129 130 131 132

static struct loop_func_table xor_funcs = {
	.number = LO_CRYPT_XOR,
	.transfer = transfer_xor,
	.init = xor_init
133
}; 
L
Linus Torvalds 已提交
134 135 136 137 138 139 140

/* xfer_funcs[0] is special - its release function is never called */
static struct loop_func_table *xfer_funcs[MAX_LO_CRYPT] = {
	&none_funcs,
	&xor_funcs
};

141
static loff_t get_size(loff_t offset, loff_t sizelimit, struct file *file)
L
Linus Torvalds 已提交
142
{
143
	loff_t loopsize;
L
Linus Torvalds 已提交
144 145

	/* Compute loopsize in bytes */
146 147 148 149
	loopsize = i_size_read(file->f_mapping->host);
	if (offset > 0)
		loopsize -= offset;
	/* offset is beyond i_size, weird but possible */
150 151
	if (loopsize < 0)
		return 0;
L
Linus Torvalds 已提交
152

153 154
	if (sizelimit > 0 && sizelimit < loopsize)
		loopsize = sizelimit;
L
Linus Torvalds 已提交
155 156 157 158 159 160 161
	/*
	 * Unfortunately, if we want to do I/O on the device,
	 * the number of 512-byte sectors has to fit into a sector_t.
	 */
	return loopsize >> 9;
}

162 163 164 165 166
static loff_t get_loop_size(struct loop_device *lo, struct file *file)
{
	return get_size(lo->lo_offset, lo->lo_sizelimit, file);
}

167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222
static void __loop_update_dio(struct loop_device *lo, bool dio)
{
	struct file *file = lo->lo_backing_file;
	struct address_space *mapping = file->f_mapping;
	struct inode *inode = mapping->host;
	unsigned short sb_bsize = 0;
	unsigned dio_align = 0;
	bool use_dio;

	if (inode->i_sb->s_bdev) {
		sb_bsize = bdev_logical_block_size(inode->i_sb->s_bdev);
		dio_align = sb_bsize - 1;
	}

	/*
	 * We support direct I/O only if lo_offset is aligned with the
	 * logical I/O size of backing device, and the logical block
	 * size of loop is bigger than the backing device's and the loop
	 * needn't transform transfer.
	 *
	 * TODO: the above condition may be loosed in the future, and
	 * direct I/O may be switched runtime at that time because most
	 * of requests in sane appplications should be PAGE_SIZE algined
	 */
	if (dio) {
		if (queue_logical_block_size(lo->lo_queue) >= sb_bsize &&
				!(lo->lo_offset & dio_align) &&
				mapping->a_ops->direct_IO &&
				!lo->transfer)
			use_dio = true;
		else
			use_dio = false;
	} else {
		use_dio = false;
	}

	if (lo->use_dio == use_dio)
		return;

	/* flush dirty pages before changing direct IO */
	vfs_fsync(file, 0);

	/*
	 * The flag of LO_FLAGS_DIRECT_IO is handled similarly with
	 * LO_FLAGS_READ_ONLY, both are set from kernel, and losetup
	 * will get updated by ioctl(LOOP_GET_STATUS)
	 */
	blk_mq_freeze_queue(lo->lo_queue);
	lo->use_dio = use_dio;
	if (use_dio)
		lo->lo_flags |= LO_FLAGS_DIRECT_IO;
	else
		lo->lo_flags &= ~LO_FLAGS_DIRECT_IO;
	blk_mq_unfreeze_queue(lo->lo_queue);
}

L
Linus Torvalds 已提交
223
static int
224
figure_loop_size(struct loop_device *lo, loff_t offset, loff_t sizelimit)
L
Linus Torvalds 已提交
225
{
226
	loff_t size = get_size(offset, sizelimit, lo->lo_backing_file);
L
Linus Torvalds 已提交
227
	sector_t x = (sector_t)size;
228
	struct block_device *bdev = lo->lo_device;
L
Linus Torvalds 已提交
229 230 231

	if (unlikely((loff_t)x != size))
		return -EFBIG;
232 233 234 235
	if (lo->lo_offset != offset)
		lo->lo_offset = offset;
	if (lo->lo_sizelimit != sizelimit)
		lo->lo_sizelimit = sizelimit;
236
	set_capacity(lo->lo_disk, x);
237 238 239
	bd_set_size(bdev, (loff_t)get_capacity(bdev->bd_disk) << 9);
	/* let user-space know about the new size */
	kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE);
240
	return 0;
L
Linus Torvalds 已提交
241 242 243 244 245 246 247 248
}

static inline int
lo_do_transfer(struct loop_device *lo, int cmd,
	       struct page *rpage, unsigned roffs,
	       struct page *lpage, unsigned loffs,
	       int size, sector_t rblock)
{
249 250 251 252
	int ret;

	ret = lo->transfer(lo, cmd, rpage, roffs, lpage, loffs, size, rblock);
	if (likely(!ret))
L
Linus Torvalds 已提交
253 254
		return 0;

255 256 257 258
	printk_ratelimited(KERN_ERR
		"loop: Transfer error at byte offset %llu, length %i.\n",
		(unsigned long long)rblock << 9, size);
	return ret;
L
Linus Torvalds 已提交
259 260
}

261
static int lo_write_bvec(struct file *file, struct bio_vec *bvec, loff_t *ppos)
L
Linus Torvalds 已提交
262
{
263
	struct iov_iter i;
L
Linus Torvalds 已提交
264
	ssize_t bw;
A
Al Viro 已提交
265

266
	iov_iter_bvec(&i, ITER_BVEC, bvec, 1, bvec->bv_len);
L
Linus Torvalds 已提交
267

A
Al Viro 已提交
268
	file_start_write(file);
269
	bw = vfs_iter_write(file, &i, ppos);
A
Al Viro 已提交
270
	file_end_write(file);
271 272

	if (likely(bw ==  bvec->bv_len))
L
Linus Torvalds 已提交
273
		return 0;
274 275 276 277

	printk_ratelimited(KERN_ERR
		"loop: Write error at byte offset %llu, length %i.\n",
		(unsigned long long)*ppos, bvec->bv_len);
L
Linus Torvalds 已提交
278 279 280 281 282
	if (bw >= 0)
		bw = -EIO;
	return bw;
}

283 284
static int lo_write_simple(struct loop_device *lo, struct request *rq,
		loff_t pos)
L
Linus Torvalds 已提交
285
{
286 287 288 289 290 291 292 293 294 295 296 297
	struct bio_vec bvec;
	struct req_iterator iter;
	int ret = 0;

	rq_for_each_segment(bvec, rq, iter) {
		ret = lo_write_bvec(lo->lo_backing_file, &bvec, &pos);
		if (ret < 0)
			break;
		cond_resched();
	}

	return ret;
L
Linus Torvalds 已提交
298 299
}

300
/*
301 302 303
 * This is the slow, transforming version that needs to double buffer the
 * data as it cannot do the transformations in place without having direct
 * access to the destination pages of the backing file.
L
Linus Torvalds 已提交
304
 */
305 306
static int lo_write_transfer(struct loop_device *lo, struct request *rq,
		loff_t pos)
L
Linus Torvalds 已提交
307
{
308
	struct bio_vec bvec, b;
M
Ming Lei 已提交
309
	struct req_iterator iter;
310
	struct page *page;
311
	int ret = 0;
L
Linus Torvalds 已提交
312

313 314 315
	page = alloc_page(GFP_NOIO);
	if (unlikely(!page))
		return -ENOMEM;
316

M
Ming Lei 已提交
317
	rq_for_each_segment(bvec, rq, iter) {
318 319 320 321 322 323 324 325 326
		ret = lo_do_transfer(lo, WRITE, page, 0, bvec.bv_page,
			bvec.bv_offset, bvec.bv_len, pos >> 9);
		if (unlikely(ret))
			break;

		b.bv_page = page;
		b.bv_offset = 0;
		b.bv_len = bvec.bv_len;
		ret = lo_write_bvec(lo->lo_backing_file, &b, &pos);
L
Linus Torvalds 已提交
327 328 329
		if (ret < 0)
			break;
	}
330 331

	__free_page(page);
L
Linus Torvalds 已提交
332 333 334
	return ret;
}

335 336
static int lo_read_simple(struct loop_device *lo, struct request *rq,
		loff_t pos)
L
Linus Torvalds 已提交
337
{
338 339 340 341
	struct bio_vec bvec;
	struct req_iterator iter;
	struct iov_iter i;
	ssize_t len;
L
Linus Torvalds 已提交
342

343 344 345 346 347
	rq_for_each_segment(bvec, rq, iter) {
		iov_iter_bvec(&i, ITER_BVEC, &bvec, 1, bvec.bv_len);
		len = vfs_iter_read(lo->lo_backing_file, &i, &pos);
		if (len < 0)
			return len;
L
Linus Torvalds 已提交
348

349
		flush_dcache_page(bvec.bv_page);
350

351 352
		if (len != bvec.bv_len) {
			struct bio *bio;
L
Linus Torvalds 已提交
353

354 355 356 357 358 359 360 361
			__rq_for_each_bio(bio, rq)
				zero_fill_bio(bio);
			break;
		}
		cond_resched();
	}

	return 0;
362 363
}

364 365
static int lo_read_transfer(struct loop_device *lo, struct request *rq,
		loff_t pos)
L
Linus Torvalds 已提交
366
{
367 368 369 370 371 372
	struct bio_vec bvec, b;
	struct req_iterator iter;
	struct iov_iter i;
	struct page *page;
	ssize_t len;
	int ret = 0;
L
Linus Torvalds 已提交
373

374 375 376
	page = alloc_page(GFP_NOIO);
	if (unlikely(!page))
		return -ENOMEM;
377

378 379
	rq_for_each_segment(bvec, rq, iter) {
		loff_t offset = pos;
380

381 382 383
		b.bv_page = page;
		b.bv_offset = 0;
		b.bv_len = bvec.bv_len;
384

385 386 387 388 389 390
		iov_iter_bvec(&i, ITER_BVEC, &b, 1, b.bv_len);
		len = vfs_iter_read(lo->lo_backing_file, &i, &pos);
		if (len < 0) {
			ret = len;
			goto out_free_page;
		}
L
Linus Torvalds 已提交
391

392 393 394 395
		ret = lo_do_transfer(lo, READ, page, 0, bvec.bv_page,
			bvec.bv_offset, len, offset >> 9);
		if (ret)
			goto out_free_page;
L
Linus Torvalds 已提交
396

397
		flush_dcache_page(bvec.bv_page);
398

399
		if (len != bvec.bv_len) {
M
Ming Lei 已提交
400 401 402 403
			struct bio *bio;

			__rq_for_each_bio(bio, rq)
				zero_fill_bio(bio);
L
Linus Torvalds 已提交
404
			break;
405
		}
L
Linus Torvalds 已提交
406
	}
407 408 409 410 411

	ret = 0;
out_free_page:
	__free_page(page);
	return ret;
L
Linus Torvalds 已提交
412 413
}

414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447
static int lo_discard(struct loop_device *lo, struct request *rq, loff_t pos)
{
	/*
	 * We use punch hole to reclaim the free space used by the
	 * image a.k.a. discard. However we do not support discard if
	 * encryption is enabled, because it may give an attacker
	 * useful information.
	 */
	struct file *file = lo->lo_backing_file;
	int mode = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE;
	int ret;

	if ((!file->f_op->fallocate) || lo->lo_encrypt_key_size) {
		ret = -EOPNOTSUPP;
		goto out;
	}

	ret = file->f_op->fallocate(file, mode, pos, blk_rq_bytes(rq));
	if (unlikely(ret && ret != -EINVAL && ret != -EOPNOTSUPP))
		ret = -EIO;
 out:
	return ret;
}

static int lo_req_flush(struct loop_device *lo, struct request *rq)
{
	struct file *file = lo->lo_backing_file;
	int ret = vfs_fsync(file, 0);
	if (unlikely(ret && ret != -EINVAL))
		ret = -EIO;

	return ret;
}

M
Ming Lei 已提交
448 449
static inline void handle_partial_read(struct loop_cmd *cmd, long bytes)
{
450
	if (bytes < 0 || op_is_write(req_op(cmd->rq)))
M
Ming Lei 已提交
451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472
		return;

	if (unlikely(bytes < blk_rq_bytes(cmd->rq))) {
		struct bio *bio = cmd->rq->bio;

		bio_advance(bio, bytes);
		zero_fill_bio(bio);
	}
}

static void lo_rw_aio_complete(struct kiocb *iocb, long ret, long ret2)
{
	struct loop_cmd *cmd = container_of(iocb, struct loop_cmd, iocb);
	struct request *rq = cmd->rq;

	handle_partial_read(cmd, ret);

	if (ret > 0)
		ret = 0;
	else if (ret < 0)
		ret = -EIO;

473
	blk_mq_complete_request(rq, ret);
M
Ming Lei 已提交
474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490
}

static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd,
		     loff_t pos, bool rw)
{
	struct iov_iter iter;
	struct bio_vec *bvec;
	struct bio *bio = cmd->rq->bio;
	struct file *file = lo->lo_backing_file;
	int ret;

	/* nomerge for loop request queue */
	WARN_ON(cmd->rq->bio != cmd->rq->biotail);

	bvec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter);
	iov_iter_bvec(&iter, ITER_BVEC | rw, bvec,
		      bio_segments(bio), blk_rq_bytes(cmd->rq));
491 492 493 494 495 496
	/*
	 * This bio may be started from the middle of the 'bvec'
	 * because of bio splitting, so offset from the bvec must
	 * be passed to iov iterator
	 */
	iter.iov_offset = bio->bi_iter.bi_bvec_done;
M
Ming Lei 已提交
497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512

	cmd->iocb.ki_pos = pos;
	cmd->iocb.ki_filp = file;
	cmd->iocb.ki_complete = lo_rw_aio_complete;
	cmd->iocb.ki_flags = IOCB_DIRECT;

	if (rw == WRITE)
		ret = file->f_op->write_iter(&cmd->iocb, &iter);
	else
		ret = file->f_op->read_iter(&cmd->iocb, &iter);

	if (ret != -EIOCBQUEUED)
		cmd->iocb.ki_complete(&cmd->iocb, ret, 0);
	return 0;
}

513
static int do_req_filebacked(struct loop_device *lo, struct request *rq)
M
Ming Lei 已提交
514 515
{
	struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);
516
	loff_t pos = ((loff_t) blk_rq_pos(rq) << 9) + lo->lo_offset;
M
Ming Lei 已提交
517 518 519 520 521 522 523 524 525 526

	/*
	 * lo_write_simple and lo_read_simple should have been covered
	 * by io submit style function like lo_rw_aio(), one blocker
	 * is that lo_read_simple() need to call flush_dcache_page after
	 * the page is written from kernel, and it isn't easy to handle
	 * this in io submit style function which submits all segments
	 * of the req at one time. And direct read IO doesn't need to
	 * run flush_dcache_page().
	 */
527 528 529 530 531 532 533 534 535 536
	switch (req_op(rq)) {
	case REQ_OP_FLUSH:
		return lo_req_flush(lo, rq);
	case REQ_OP_DISCARD:
		return lo_discard(lo, rq, pos);
	case REQ_OP_WRITE:
		if (lo->transfer)
			return lo_write_transfer(lo, rq, pos);
		else if (cmd->use_aio)
			return lo_rw_aio(lo, cmd, pos, WRITE);
537
		else
538 539
			return lo_write_simple(lo, rq, pos);
	case REQ_OP_READ:
540
		if (lo->transfer)
541 542 543
			return lo_read_transfer(lo, rq, pos);
		else if (cmd->use_aio)
			return lo_rw_aio(lo, cmd, pos, READ);
544
		else
545 546 547 548 549
			return lo_read_simple(lo, rq, pos);
	default:
		WARN_ON_ONCE(1);
		return -EIO;
		break;
550
	}
L
Linus Torvalds 已提交
551 552 553 554 555 556 557
}

struct switch_request {
	struct file *file;
	struct completion wait;
};

558 559 560 561 562 563
static inline void loop_update_dio(struct loop_device *lo)
{
	__loop_update_dio(lo, io_is_direct(lo->lo_backing_file) |
			lo->use_dio);
}

L
Linus Torvalds 已提交
564
/*
565
 * Do the actual switch; called from the BIO completion routine
L
Linus Torvalds 已提交
566
 */
567
static void do_loop_switch(struct loop_device *lo, struct switch_request *p)
L
Linus Torvalds 已提交
568
{
569 570 571
	struct file *file = p->file;
	struct file *old_file = lo->lo_backing_file;
	struct address_space *mapping;
N
Nick Piggin 已提交
572

573 574 575
	/* if no new file, only flush of queued bios requested */
	if (!file)
		return;
L
Linus Torvalds 已提交
576

577 578 579 580 581 582 583
	mapping = file->f_mapping;
	mapping_set_gfp_mask(old_file->f_mapping, lo->old_gfp_mask);
	lo->lo_backing_file = file;
	lo->lo_blocksize = S_ISBLK(mapping->host->i_mode) ?
		mapping->host->i_bdev->bd_block_size : PAGE_SIZE;
	lo->old_gfp_mask = mapping_gfp_mask(mapping);
	mapping_set_gfp_mask(mapping, lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS));
584
	loop_update_dio(lo);
L
Linus Torvalds 已提交
585 586 587 588 589 590 591 592 593 594
}

/*
 * loop_switch performs the hard work of switching a backing store.
 * First it needs to flush existing IO, it does this by sending a magic
 * BIO down the pipe. The completion of this BIO does the actual switch.
 */
static int loop_switch(struct loop_device *lo, struct file *file)
{
	struct switch_request w;
595

L
Linus Torvalds 已提交
596
	w.file = file;
597 598 599 600 601 602 603 604 605 606

	/* freeze queue and wait for completion of scheduled requests */
	blk_mq_freeze_queue(lo->lo_queue);

	/* do the switch action */
	do_loop_switch(lo, &w);

	/* unfreeze */
	blk_mq_unfreeze_queue(lo->lo_queue);

L
Linus Torvalds 已提交
607 608 609
	return 0;
}

610 611 612 613 614 615 616 617
/*
 * Helper to flush the IOs in loop, but keeping loop thread running
 */
static int loop_flush(struct loop_device *lo)
{
	return loop_switch(lo, NULL);
}

618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639
static void loop_reread_partitions(struct loop_device *lo,
				   struct block_device *bdev)
{
	int rc;

	/*
	 * bd_mutex has been held already in release path, so don't
	 * acquire it if this function is called in such case.
	 *
	 * If the reread partition isn't from release path, lo_refcnt
	 * must be at least one and it can only become zero when the
	 * current holder is released.
	 */
	if (!atomic_read(&lo->lo_refcnt))
		rc = __blkdev_reread_part(bdev);
	else
		rc = blkdev_reread_part(bdev);
	if (rc)
		pr_warn("%s: partition scan of loop%d (%s) failed (rc=%d)\n",
			__func__, lo->lo_number, lo->lo_file_name, rc);
}

L
Linus Torvalds 已提交
640 641 642 643 644 645 646 647
/*
 * loop_change_fd switched the backing store of a loopback device to
 * a new file. This is useful for operating system installers to free up
 * the original file and in High Availability environments to switch to
 * an alternative location for the content in case of server meltdown.
 * This can only work if the loop device is used read-only, and if the
 * new backing store is the same size and type as the old backing store.
 */
A
Al Viro 已提交
648 649
static int loop_change_fd(struct loop_device *lo, struct block_device *bdev,
			  unsigned int arg)
L
Linus Torvalds 已提交
650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686
{
	struct file	*file, *old_file;
	struct inode	*inode;
	int		error;

	error = -ENXIO;
	if (lo->lo_state != Lo_bound)
		goto out;

	/* the loop device has to be read-only */
	error = -EINVAL;
	if (!(lo->lo_flags & LO_FLAGS_READ_ONLY))
		goto out;

	error = -EBADF;
	file = fget(arg);
	if (!file)
		goto out;

	inode = file->f_mapping->host;
	old_file = lo->lo_backing_file;

	error = -EINVAL;

	if (!S_ISREG(inode->i_mode) && !S_ISBLK(inode->i_mode))
		goto out_putf;

	/* size of the new backing store needs to be the same */
	if (get_loop_size(lo, file) != get_loop_size(lo, old_file))
		goto out_putf;

	/* and ... switch */
	error = loop_switch(lo, file);
	if (error)
		goto out_putf;

	fput(old_file);
687
	if (lo->lo_flags & LO_FLAGS_PARTSCAN)
688
		loop_reread_partitions(lo, bdev);
L
Linus Torvalds 已提交
689 690 691 692 693 694 695 696 697 698 699 700 701 702 703
	return 0;

 out_putf:
	fput(file);
 out:
	return error;
}

static inline int is_loop_device(struct file *file)
{
	struct inode *i = file->f_mapping->host;

	return i && S_ISBLK(i->i_mode) && MAJOR(i->i_rdev) == LOOP_MAJOR;
}

704 705 706 707 708
/* loop sysfs attributes */

static ssize_t loop_attr_show(struct device *dev, char *page,
			      ssize_t (*callback)(struct loop_device *, char *))
{
709 710
	struct gendisk *disk = dev_to_disk(dev);
	struct loop_device *lo = disk->private_data;
711

712
	return callback(lo, page);
713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729
}

#define LOOP_ATTR_RO(_name)						\
static ssize_t loop_attr_##_name##_show(struct loop_device *, char *);	\
static ssize_t loop_attr_do_show_##_name(struct device *d,		\
				struct device_attribute *attr, char *b)	\
{									\
	return loop_attr_show(d, b, loop_attr_##_name##_show);		\
}									\
static struct device_attribute loop_attr_##_name =			\
	__ATTR(_name, S_IRUGO, loop_attr_do_show_##_name, NULL);

static ssize_t loop_attr_backing_file_show(struct loop_device *lo, char *buf)
{
	ssize_t ret;
	char *p = NULL;

730
	spin_lock_irq(&lo->lo_lock);
731
	if (lo->lo_backing_file)
M
Miklos Szeredi 已提交
732
		p = file_path(lo->lo_backing_file, buf, PAGE_SIZE - 1);
733
	spin_unlock_irq(&lo->lo_lock);
734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763

	if (IS_ERR_OR_NULL(p))
		ret = PTR_ERR(p);
	else {
		ret = strlen(p);
		memmove(buf, p, ret);
		buf[ret++] = '\n';
		buf[ret] = 0;
	}

	return ret;
}

static ssize_t loop_attr_offset_show(struct loop_device *lo, char *buf)
{
	return sprintf(buf, "%llu\n", (unsigned long long)lo->lo_offset);
}

static ssize_t loop_attr_sizelimit_show(struct loop_device *lo, char *buf)
{
	return sprintf(buf, "%llu\n", (unsigned long long)lo->lo_sizelimit);
}

static ssize_t loop_attr_autoclear_show(struct loop_device *lo, char *buf)
{
	int autoclear = (lo->lo_flags & LO_FLAGS_AUTOCLEAR);

	return sprintf(buf, "%s\n", autoclear ? "1" : "0");
}

764 765 766 767 768 769 770
static ssize_t loop_attr_partscan_show(struct loop_device *lo, char *buf)
{
	int partscan = (lo->lo_flags & LO_FLAGS_PARTSCAN);

	return sprintf(buf, "%s\n", partscan ? "1" : "0");
}

771 772 773 774 775 776 777
static ssize_t loop_attr_dio_show(struct loop_device *lo, char *buf)
{
	int dio = (lo->lo_flags & LO_FLAGS_DIRECT_IO);

	return sprintf(buf, "%s\n", dio ? "1" : "0");
}

778 779 780 781
LOOP_ATTR_RO(backing_file);
LOOP_ATTR_RO(offset);
LOOP_ATTR_RO(sizelimit);
LOOP_ATTR_RO(autoclear);
782
LOOP_ATTR_RO(partscan);
783
LOOP_ATTR_RO(dio);
784 785 786 787 788 789

static struct attribute *loop_attrs[] = {
	&loop_attr_backing_file.attr,
	&loop_attr_offset.attr,
	&loop_attr_sizelimit.attr,
	&loop_attr_autoclear.attr,
790
	&loop_attr_partscan.attr,
791
	&loop_attr_dio.attr,
792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811
	NULL,
};

static struct attribute_group loop_attribute_group = {
	.name = "loop",
	.attrs= loop_attrs,
};

static int loop_sysfs_init(struct loop_device *lo)
{
	return sysfs_create_group(&disk_to_dev(lo->lo_disk)->kobj,
				  &loop_attribute_group);
}

static void loop_sysfs_exit(struct loop_device *lo)
{
	sysfs_remove_group(&disk_to_dev(lo->lo_disk)->kobj,
			   &loop_attribute_group);
}

812 813 814 815 816 817 818 819
static void loop_config_discard(struct loop_device *lo)
{
	struct file *file = lo->lo_backing_file;
	struct inode *inode = file->f_mapping->host;
	struct request_queue *q = lo->lo_queue;

	/*
	 * We use punch hole to reclaim the free space used by the
820
	 * image a.k.a. discard. However we do not support discard if
821 822 823 824 825 826 827
	 * encryption is enabled, because it may give an attacker
	 * useful information.
	 */
	if ((!file->f_op->fallocate) ||
	    lo->lo_encrypt_key_size) {
		q->limits.discard_granularity = 0;
		q->limits.discard_alignment = 0;
828
		blk_queue_max_discard_sectors(q, 0);
829 830 831 832 833 834
		q->limits.discard_zeroes_data = 0;
		queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, q);
		return;
	}

	q->limits.discard_granularity = inode->i_sb->s_blocksize;
835
	q->limits.discard_alignment = 0;
836
	blk_queue_max_discard_sectors(q, UINT_MAX >> 9);
837 838 839 840
	q->limits.discard_zeroes_data = 1;
	queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q);
}

M
Ming Lei 已提交
841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857
static void loop_unprepare_queue(struct loop_device *lo)
{
	flush_kthread_worker(&lo->worker);
	kthread_stop(lo->worker_task);
}

static int loop_prepare_queue(struct loop_device *lo)
{
	init_kthread_worker(&lo->worker);
	lo->worker_task = kthread_run(kthread_worker_fn,
			&lo->worker, "loop%d", lo->lo_number);
	if (IS_ERR(lo->worker_task))
		return -ENOMEM;
	set_user_nice(lo->worker_task, MIN_NICE);
	return 0;
}

A
Al Viro 已提交
858
static int loop_set_fd(struct loop_device *lo, fmode_t mode,
L
Linus Torvalds 已提交
859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885
		       struct block_device *bdev, unsigned int arg)
{
	struct file	*file, *f;
	struct inode	*inode;
	struct address_space *mapping;
	unsigned lo_blocksize;
	int		lo_flags = 0;
	int		error;
	loff_t		size;

	/* This is safe, since we have a reference from open(). */
	__module_get(THIS_MODULE);

	error = -EBADF;
	file = fget(arg);
	if (!file)
		goto out;

	error = -EBUSY;
	if (lo->lo_state != Lo_unbound)
		goto out_putf;

	/* Avoid recursion */
	f = file;
	while (is_loop_device(f)) {
		struct loop_device *l;

A
Al Viro 已提交
886
		if (f->f_mapping->host->i_bdev == bdev)
L
Linus Torvalds 已提交
887 888 889 890 891 892 893 894 895 896 897 898 899 900
			goto out_putf;

		l = f->f_mapping->host->i_bdev->bd_disk->private_data;
		if (l->lo_state == Lo_unbound) {
			error = -EINVAL;
			goto out_putf;
		}
		f = l->lo_backing_file;
	}

	mapping = file->f_mapping;
	inode = mapping->host;

	error = -EINVAL;
901 902
	if (!S_ISREG(inode->i_mode) && !S_ISBLK(inode->i_mode))
		goto out_putf;
L
Linus Torvalds 已提交
903

904
	if (!(file->f_mode & FMODE_WRITE) || !(mode & FMODE_WRITE) ||
A
Al Viro 已提交
905
	    !file->f_op->write_iter)
906
		lo_flags |= LO_FLAGS_READ_ONLY;
907

908 909
	lo_blocksize = S_ISBLK(inode->i_mode) ?
		inode->i_bdev->bd_block_size : PAGE_SIZE;
L
Linus Torvalds 已提交
910

911
	error = -EFBIG;
L
Linus Torvalds 已提交
912
	size = get_loop_size(lo, file);
913
	if ((loff_t)(sector_t)size != size)
L
Linus Torvalds 已提交
914
		goto out_putf;
M
Ming Lei 已提交
915 916
	error = loop_prepare_queue(lo);
	if (error)
917
		goto out_putf;
L
Linus Torvalds 已提交
918

919
	error = 0;
L
Linus Torvalds 已提交
920 921 922

	set_device_ro(bdev, (lo_flags & LO_FLAGS_READ_ONLY) != 0);

923
	lo->use_dio = false;
L
Linus Torvalds 已提交
924 925 926 927
	lo->lo_blocksize = lo_blocksize;
	lo->lo_device = bdev;
	lo->lo_flags = lo_flags;
	lo->lo_backing_file = file;
928
	lo->transfer = NULL;
L
Linus Torvalds 已提交
929 930 931 932 933
	lo->ioctl = NULL;
	lo->lo_sizelimit = 0;
	lo->old_gfp_mask = mapping_gfp_mask(mapping);
	mapping_set_gfp_mask(mapping, lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS));

934
	if (!(lo_flags & LO_FLAGS_READ_ONLY) && file->f_op->fsync)
935
		blk_queue_write_cache(lo->lo_queue, true, false);
936

937
	loop_update_dio(lo);
938
	set_capacity(lo->lo_disk, size);
L
Linus Torvalds 已提交
939
	bd_set_size(bdev, size << 9);
940
	loop_sysfs_init(lo);
941 942
	/* let user-space know about the new size */
	kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE);
L
Linus Torvalds 已提交
943 944 945

	set_blocksize(bdev, lo_blocksize);

946
	lo->lo_state = Lo_bound;
947 948 949
	if (part_shift)
		lo->lo_flags |= LO_FLAGS_PARTSCAN;
	if (lo->lo_flags & LO_FLAGS_PARTSCAN)
950
		loop_reread_partitions(lo, bdev);
951 952 953 954 955

	/* Grab the block_device to prevent its destruction after we
	 * put /dev/loopXX inode. Later in loop_clr_fd() we bdput(bdev).
	 */
	bdgrab(bdev);
L
Linus Torvalds 已提交
956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002
	return 0;

 out_putf:
	fput(file);
 out:
	/* This is safe: open() is still holding a reference. */
	module_put(THIS_MODULE);
	return error;
}

static int
loop_release_xfer(struct loop_device *lo)
{
	int err = 0;
	struct loop_func_table *xfer = lo->lo_encryption;

	if (xfer) {
		if (xfer->release)
			err = xfer->release(lo);
		lo->transfer = NULL;
		lo->lo_encryption = NULL;
		module_put(xfer->owner);
	}
	return err;
}

static int
loop_init_xfer(struct loop_device *lo, struct loop_func_table *xfer,
	       const struct loop_info64 *i)
{
	int err = 0;

	if (xfer) {
		struct module *owner = xfer->owner;

		if (!try_module_get(owner))
			return -EINVAL;
		if (xfer->init)
			err = xfer->init(lo, i);
		if (err)
			module_put(owner);
		else
			lo->lo_encryption = xfer;
	}
	return err;
}

1003
static int loop_clr_fd(struct loop_device *lo)
L
Linus Torvalds 已提交
1004 1005
{
	struct file *filp = lo->lo_backing_file;
1006
	gfp_t gfp = lo->old_gfp_mask;
1007
	struct block_device *bdev = lo->lo_device;
L
Linus Torvalds 已提交
1008 1009 1010 1011

	if (lo->lo_state != Lo_bound)
		return -ENXIO;

1012 1013 1014 1015 1016 1017 1018 1019 1020 1021
	/*
	 * If we've explicitly asked to tear down the loop device,
	 * and it has an elevated reference count, set it for auto-teardown when
	 * the last reference goes away. This stops $!~#$@ udev from
	 * preventing teardown because it decided that it needs to run blkid on
	 * the loopback device whenever they appear. xfstests is notorious for
	 * failing tests because blkid via udev races with a losetup
	 * <dev>/do something like mkfs/losetup -d <dev> causing the losetup -d
	 * command to fail with EBUSY.
	 */
1022
	if (atomic_read(&lo->lo_refcnt) > 1) {
1023 1024 1025 1026
		lo->lo_flags |= LO_FLAGS_AUTOCLEAR;
		mutex_unlock(&lo->lo_ctl_mutex);
		return 0;
	}
L
Linus Torvalds 已提交
1027 1028 1029 1030

	if (filp == NULL)
		return -EINVAL;

1031 1032 1033
	/* freeze request queue during the transition */
	blk_mq_freeze_queue(lo->lo_queue);

L
Linus Torvalds 已提交
1034 1035 1036
	spin_lock_irq(&lo->lo_lock);
	lo->lo_state = Lo_rundown;
	lo->lo_backing_file = NULL;
1037
	spin_unlock_irq(&lo->lo_lock);
L
Linus Torvalds 已提交
1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049

	loop_release_xfer(lo);
	lo->transfer = NULL;
	lo->ioctl = NULL;
	lo->lo_device = NULL;
	lo->lo_encryption = NULL;
	lo->lo_offset = 0;
	lo->lo_sizelimit = 0;
	lo->lo_encrypt_key_size = 0;
	memset(lo->lo_encrypt_key, 0, LO_KEY_SIZE);
	memset(lo->lo_crypt_name, 0, LO_NAME_SIZE);
	memset(lo->lo_file_name, 0, LO_NAME_SIZE);
1050 1051
	if (bdev) {
		bdput(bdev);
A
Al Viro 已提交
1052
		invalidate_bdev(bdev);
1053
	}
1054
	set_capacity(lo->lo_disk, 0);
1055
	loop_sysfs_exit(lo);
1056
	if (bdev) {
A
Al Viro 已提交
1057
		bd_set_size(bdev, 0);
1058 1059 1060
		/* let user-space know about this change */
		kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE);
	}
L
Linus Torvalds 已提交
1061 1062 1063 1064
	mapping_set_gfp_mask(filp->f_mapping, gfp);
	lo->lo_state = Lo_unbound;
	/* This is safe: open() is still holding a reference. */
	module_put(THIS_MODULE);
1065 1066
	blk_mq_unfreeze_queue(lo->lo_queue);

1067
	if (lo->lo_flags & LO_FLAGS_PARTSCAN && bdev)
1068
		loop_reread_partitions(lo, bdev);
1069 1070 1071
	lo->lo_flags = 0;
	if (!part_shift)
		lo->lo_disk->flags |= GENHD_FL_NO_PART_SCAN;
M
Ming Lei 已提交
1072
	loop_unprepare_queue(lo);
1073 1074 1075 1076 1077 1078 1079 1080
	mutex_unlock(&lo->lo_ctl_mutex);
	/*
	 * Need not hold lo_ctl_mutex to fput backing file.
	 * Calling fput holding lo_ctl_mutex triggers a circular
	 * lock dependency possibility warning as fput can take
	 * bd_mutex which is usually taken before lo_ctl_mutex.
	 */
	fput(filp);
L
Linus Torvalds 已提交
1081 1082 1083 1084 1085 1086 1087 1088
	return 0;
}

static int
loop_set_status(struct loop_device *lo, const struct loop_info64 *info)
{
	int err;
	struct loop_func_table *xfer;
1089
	kuid_t uid = current_uid();
L
Linus Torvalds 已提交
1090

1091
	if (lo->lo_encrypt_key_size &&
1092
	    !uid_eq(lo->lo_key_owner, uid) &&
L
Linus Torvalds 已提交
1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119
	    !capable(CAP_SYS_ADMIN))
		return -EPERM;
	if (lo->lo_state != Lo_bound)
		return -ENXIO;
	if ((unsigned int) info->lo_encrypt_key_size > LO_KEY_SIZE)
		return -EINVAL;

	err = loop_release_xfer(lo);
	if (err)
		return err;

	if (info->lo_encrypt_type) {
		unsigned int type = info->lo_encrypt_type;

		if (type >= MAX_LO_CRYPT)
			return -EINVAL;
		xfer = xfer_funcs[type];
		if (xfer == NULL)
			return -EINVAL;
	} else
		xfer = NULL;

	err = loop_init_xfer(lo, xfer, info);
	if (err)
		return err;

	if (lo->lo_offset != info->lo_offset ||
1120
	    lo->lo_sizelimit != info->lo_sizelimit)
1121
		if (figure_loop_size(lo, info->lo_offset, info->lo_sizelimit))
L
Linus Torvalds 已提交
1122
			return -EFBIG;
1123

1124
	loop_config_discard(lo);
L
Linus Torvalds 已提交
1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135

	memcpy(lo->lo_file_name, info->lo_file_name, LO_NAME_SIZE);
	memcpy(lo->lo_crypt_name, info->lo_crypt_name, LO_NAME_SIZE);
	lo->lo_file_name[LO_NAME_SIZE-1] = 0;
	lo->lo_crypt_name[LO_NAME_SIZE-1] = 0;

	if (!xfer)
		xfer = &none_funcs;
	lo->transfer = xfer->transfer;
	lo->ioctl = xfer->ioctl;

1136 1137 1138 1139
	if ((lo->lo_flags & LO_FLAGS_AUTOCLEAR) !=
	     (info->lo_flags & LO_FLAGS_AUTOCLEAR))
		lo->lo_flags ^= LO_FLAGS_AUTOCLEAR;

1140 1141 1142 1143
	if ((info->lo_flags & LO_FLAGS_PARTSCAN) &&
	     !(lo->lo_flags & LO_FLAGS_PARTSCAN)) {
		lo->lo_flags |= LO_FLAGS_PARTSCAN;
		lo->lo_disk->flags &= ~GENHD_FL_NO_PART_SCAN;
1144
		loop_reread_partitions(lo, lo->lo_device);
1145 1146
	}

L
Linus Torvalds 已提交
1147 1148 1149 1150 1151 1152
	lo->lo_encrypt_key_size = info->lo_encrypt_key_size;
	lo->lo_init[0] = info->lo_init[0];
	lo->lo_init[1] = info->lo_init[1];
	if (info->lo_encrypt_key_size) {
		memcpy(lo->lo_encrypt_key, info->lo_encrypt_key,
		       info->lo_encrypt_key_size);
1153
		lo->lo_key_owner = uid;
1154
	}
L
Linus Torvalds 已提交
1155

1156 1157 1158
	/* update dio if lo_offset or transfer is changed */
	__loop_update_dio(lo, lo->use_dio);

L
Linus Torvalds 已提交
1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170
	return 0;
}

static int
loop_get_status(struct loop_device *lo, struct loop_info64 *info)
{
	struct file *file = lo->lo_backing_file;
	struct kstat stat;
	int error;

	if (lo->lo_state != Lo_bound)
		return -ENXIO;
A
Al Viro 已提交
1171
	error = vfs_getattr(&file->f_path, &stat);
L
Linus Torvalds 已提交
1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300
	if (error)
		return error;
	memset(info, 0, sizeof(*info));
	info->lo_number = lo->lo_number;
	info->lo_device = huge_encode_dev(stat.dev);
	info->lo_inode = stat.ino;
	info->lo_rdevice = huge_encode_dev(lo->lo_device ? stat.rdev : stat.dev);
	info->lo_offset = lo->lo_offset;
	info->lo_sizelimit = lo->lo_sizelimit;
	info->lo_flags = lo->lo_flags;
	memcpy(info->lo_file_name, lo->lo_file_name, LO_NAME_SIZE);
	memcpy(info->lo_crypt_name, lo->lo_crypt_name, LO_NAME_SIZE);
	info->lo_encrypt_type =
		lo->lo_encryption ? lo->lo_encryption->number : 0;
	if (lo->lo_encrypt_key_size && capable(CAP_SYS_ADMIN)) {
		info->lo_encrypt_key_size = lo->lo_encrypt_key_size;
		memcpy(info->lo_encrypt_key, lo->lo_encrypt_key,
		       lo->lo_encrypt_key_size);
	}
	return 0;
}

static void
loop_info64_from_old(const struct loop_info *info, struct loop_info64 *info64)
{
	memset(info64, 0, sizeof(*info64));
	info64->lo_number = info->lo_number;
	info64->lo_device = info->lo_device;
	info64->lo_inode = info->lo_inode;
	info64->lo_rdevice = info->lo_rdevice;
	info64->lo_offset = info->lo_offset;
	info64->lo_sizelimit = 0;
	info64->lo_encrypt_type = info->lo_encrypt_type;
	info64->lo_encrypt_key_size = info->lo_encrypt_key_size;
	info64->lo_flags = info->lo_flags;
	info64->lo_init[0] = info->lo_init[0];
	info64->lo_init[1] = info->lo_init[1];
	if (info->lo_encrypt_type == LO_CRYPT_CRYPTOAPI)
		memcpy(info64->lo_crypt_name, info->lo_name, LO_NAME_SIZE);
	else
		memcpy(info64->lo_file_name, info->lo_name, LO_NAME_SIZE);
	memcpy(info64->lo_encrypt_key, info->lo_encrypt_key, LO_KEY_SIZE);
}

static int
loop_info64_to_old(const struct loop_info64 *info64, struct loop_info *info)
{
	memset(info, 0, sizeof(*info));
	info->lo_number = info64->lo_number;
	info->lo_device = info64->lo_device;
	info->lo_inode = info64->lo_inode;
	info->lo_rdevice = info64->lo_rdevice;
	info->lo_offset = info64->lo_offset;
	info->lo_encrypt_type = info64->lo_encrypt_type;
	info->lo_encrypt_key_size = info64->lo_encrypt_key_size;
	info->lo_flags = info64->lo_flags;
	info->lo_init[0] = info64->lo_init[0];
	info->lo_init[1] = info64->lo_init[1];
	if (info->lo_encrypt_type == LO_CRYPT_CRYPTOAPI)
		memcpy(info->lo_name, info64->lo_crypt_name, LO_NAME_SIZE);
	else
		memcpy(info->lo_name, info64->lo_file_name, LO_NAME_SIZE);
	memcpy(info->lo_encrypt_key, info64->lo_encrypt_key, LO_KEY_SIZE);

	/* error in case values were truncated */
	if (info->lo_device != info64->lo_device ||
	    info->lo_rdevice != info64->lo_rdevice ||
	    info->lo_inode != info64->lo_inode ||
	    info->lo_offset != info64->lo_offset)
		return -EOVERFLOW;

	return 0;
}

static int
loop_set_status_old(struct loop_device *lo, const struct loop_info __user *arg)
{
	struct loop_info info;
	struct loop_info64 info64;

	if (copy_from_user(&info, arg, sizeof (struct loop_info)))
		return -EFAULT;
	loop_info64_from_old(&info, &info64);
	return loop_set_status(lo, &info64);
}

static int
loop_set_status64(struct loop_device *lo, const struct loop_info64 __user *arg)
{
	struct loop_info64 info64;

	if (copy_from_user(&info64, arg, sizeof (struct loop_info64)))
		return -EFAULT;
	return loop_set_status(lo, &info64);
}

static int
loop_get_status_old(struct loop_device *lo, struct loop_info __user *arg) {
	struct loop_info info;
	struct loop_info64 info64;
	int err = 0;

	if (!arg)
		err = -EINVAL;
	if (!err)
		err = loop_get_status(lo, &info64);
	if (!err)
		err = loop_info64_to_old(&info64, &info);
	if (!err && copy_to_user(arg, &info, sizeof(info)))
		err = -EFAULT;

	return err;
}

static int
loop_get_status64(struct loop_device *lo, struct loop_info64 __user *arg) {
	struct loop_info64 info64;
	int err = 0;

	if (!arg)
		err = -EINVAL;
	if (!err)
		err = loop_get_status(lo, &info64);
	if (!err && copy_to_user(arg, &info64, sizeof(info64)))
		err = -EFAULT;

	return err;
}

1301 1302 1303
static int loop_set_capacity(struct loop_device *lo, struct block_device *bdev)
{
	if (unlikely(lo->lo_state != Lo_bound))
1304
		return -ENXIO;
1305

1306
	return figure_loop_size(lo, lo->lo_offset, lo->lo_sizelimit);
1307 1308
}

1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322
static int loop_set_dio(struct loop_device *lo, unsigned long arg)
{
	int error = -ENXIO;
	if (lo->lo_state != Lo_bound)
		goto out;

	__loop_update_dio(lo, !!arg);
	if (lo->use_dio == !!arg)
		return 0;
	error = -EINVAL;
 out:
	return error;
}

A
Al Viro 已提交
1323
static int lo_ioctl(struct block_device *bdev, fmode_t mode,
L
Linus Torvalds 已提交
1324 1325
	unsigned int cmd, unsigned long arg)
{
A
Al Viro 已提交
1326
	struct loop_device *lo = bdev->bd_disk->private_data;
L
Linus Torvalds 已提交
1327 1328
	int err;

1329
	mutex_lock_nested(&lo->lo_ctl_mutex, 1);
L
Linus Torvalds 已提交
1330 1331
	switch (cmd) {
	case LOOP_SET_FD:
A
Al Viro 已提交
1332
		err = loop_set_fd(lo, mode, bdev, arg);
L
Linus Torvalds 已提交
1333 1334
		break;
	case LOOP_CHANGE_FD:
A
Al Viro 已提交
1335
		err = loop_change_fd(lo, bdev, arg);
L
Linus Torvalds 已提交
1336 1337
		break;
	case LOOP_CLR_FD:
1338
		/* loop_clr_fd would have unlocked lo_ctl_mutex on success */
1339
		err = loop_clr_fd(lo);
1340 1341
		if (!err)
			goto out_unlocked;
L
Linus Torvalds 已提交
1342 1343
		break;
	case LOOP_SET_STATUS:
1344 1345 1346 1347
		err = -EPERM;
		if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN))
			err = loop_set_status_old(lo,
					(struct loop_info __user *)arg);
L
Linus Torvalds 已提交
1348 1349 1350 1351 1352
		break;
	case LOOP_GET_STATUS:
		err = loop_get_status_old(lo, (struct loop_info __user *) arg);
		break;
	case LOOP_SET_STATUS64:
1353 1354 1355 1356
		err = -EPERM;
		if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN))
			err = loop_set_status64(lo,
					(struct loop_info64 __user *) arg);
L
Linus Torvalds 已提交
1357 1358 1359 1360
		break;
	case LOOP_GET_STATUS64:
		err = loop_get_status64(lo, (struct loop_info64 __user *) arg);
		break;
1361 1362 1363 1364 1365
	case LOOP_SET_CAPACITY:
		err = -EPERM;
		if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN))
			err = loop_set_capacity(lo, bdev);
		break;
1366 1367 1368 1369 1370
	case LOOP_SET_DIRECT_IO:
		err = -EPERM;
		if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN))
			err = loop_set_dio(lo, arg);
		break;
L
Linus Torvalds 已提交
1371 1372 1373
	default:
		err = lo->ioctl ? lo->ioctl(lo, cmd, arg) : -EINVAL;
	}
1374
	mutex_unlock(&lo->lo_ctl_mutex);
1375 1376

out_unlocked:
L
Linus Torvalds 已提交
1377 1378 1379
	return err;
}

1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400
#ifdef CONFIG_COMPAT
struct compat_loop_info {
	compat_int_t	lo_number;      /* ioctl r/o */
	compat_dev_t	lo_device;      /* ioctl r/o */
	compat_ulong_t	lo_inode;       /* ioctl r/o */
	compat_dev_t	lo_rdevice;     /* ioctl r/o */
	compat_int_t	lo_offset;
	compat_int_t	lo_encrypt_type;
	compat_int_t	lo_encrypt_key_size;    /* ioctl w/o */
	compat_int_t	lo_flags;       /* ioctl r/o */
	char		lo_name[LO_NAME_SIZE];
	unsigned char	lo_encrypt_key[LO_KEY_SIZE]; /* ioctl w/o */
	compat_ulong_t	lo_init[2];
	char		reserved[4];
};

/*
 * Transfer 32-bit compatibility structure in userspace to 64-bit loop info
 * - noinlined to reduce stack space usage in main part of driver
 */
static noinline int
A
Al Viro 已提交
1401
loop_info64_from_compat(const struct compat_loop_info __user *arg,
1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498
			struct loop_info64 *info64)
{
	struct compat_loop_info info;

	if (copy_from_user(&info, arg, sizeof(info)))
		return -EFAULT;

	memset(info64, 0, sizeof(*info64));
	info64->lo_number = info.lo_number;
	info64->lo_device = info.lo_device;
	info64->lo_inode = info.lo_inode;
	info64->lo_rdevice = info.lo_rdevice;
	info64->lo_offset = info.lo_offset;
	info64->lo_sizelimit = 0;
	info64->lo_encrypt_type = info.lo_encrypt_type;
	info64->lo_encrypt_key_size = info.lo_encrypt_key_size;
	info64->lo_flags = info.lo_flags;
	info64->lo_init[0] = info.lo_init[0];
	info64->lo_init[1] = info.lo_init[1];
	if (info.lo_encrypt_type == LO_CRYPT_CRYPTOAPI)
		memcpy(info64->lo_crypt_name, info.lo_name, LO_NAME_SIZE);
	else
		memcpy(info64->lo_file_name, info.lo_name, LO_NAME_SIZE);
	memcpy(info64->lo_encrypt_key, info.lo_encrypt_key, LO_KEY_SIZE);
	return 0;
}

/*
 * Transfer 64-bit loop info to 32-bit compatibility structure in userspace
 * - noinlined to reduce stack space usage in main part of driver
 */
static noinline int
loop_info64_to_compat(const struct loop_info64 *info64,
		      struct compat_loop_info __user *arg)
{
	struct compat_loop_info info;

	memset(&info, 0, sizeof(info));
	info.lo_number = info64->lo_number;
	info.lo_device = info64->lo_device;
	info.lo_inode = info64->lo_inode;
	info.lo_rdevice = info64->lo_rdevice;
	info.lo_offset = info64->lo_offset;
	info.lo_encrypt_type = info64->lo_encrypt_type;
	info.lo_encrypt_key_size = info64->lo_encrypt_key_size;
	info.lo_flags = info64->lo_flags;
	info.lo_init[0] = info64->lo_init[0];
	info.lo_init[1] = info64->lo_init[1];
	if (info.lo_encrypt_type == LO_CRYPT_CRYPTOAPI)
		memcpy(info.lo_name, info64->lo_crypt_name, LO_NAME_SIZE);
	else
		memcpy(info.lo_name, info64->lo_file_name, LO_NAME_SIZE);
	memcpy(info.lo_encrypt_key, info64->lo_encrypt_key, LO_KEY_SIZE);

	/* error in case values were truncated */
	if (info.lo_device != info64->lo_device ||
	    info.lo_rdevice != info64->lo_rdevice ||
	    info.lo_inode != info64->lo_inode ||
	    info.lo_offset != info64->lo_offset ||
	    info.lo_init[0] != info64->lo_init[0] ||
	    info.lo_init[1] != info64->lo_init[1])
		return -EOVERFLOW;

	if (copy_to_user(arg, &info, sizeof(info)))
		return -EFAULT;
	return 0;
}

static int
loop_set_status_compat(struct loop_device *lo,
		       const struct compat_loop_info __user *arg)
{
	struct loop_info64 info64;
	int ret;

	ret = loop_info64_from_compat(arg, &info64);
	if (ret < 0)
		return ret;
	return loop_set_status(lo, &info64);
}

static int
loop_get_status_compat(struct loop_device *lo,
		       struct compat_loop_info __user *arg)
{
	struct loop_info64 info64;
	int err = 0;

	if (!arg)
		err = -EINVAL;
	if (!err)
		err = loop_get_status(lo, &info64);
	if (!err)
		err = loop_info64_to_compat(&info64, arg);
	return err;
}

A
Al Viro 已提交
1499 1500
static int lo_compat_ioctl(struct block_device *bdev, fmode_t mode,
			   unsigned int cmd, unsigned long arg)
1501
{
A
Al Viro 已提交
1502
	struct loop_device *lo = bdev->bd_disk->private_data;
1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517
	int err;

	switch(cmd) {
	case LOOP_SET_STATUS:
		mutex_lock(&lo->lo_ctl_mutex);
		err = loop_set_status_compat(
			lo, (const struct compat_loop_info __user *) arg);
		mutex_unlock(&lo->lo_ctl_mutex);
		break;
	case LOOP_GET_STATUS:
		mutex_lock(&lo->lo_ctl_mutex);
		err = loop_get_status_compat(
			lo, (struct compat_loop_info __user *) arg);
		mutex_unlock(&lo->lo_ctl_mutex);
		break;
1518
	case LOOP_SET_CAPACITY:
1519 1520 1521 1522 1523 1524
	case LOOP_CLR_FD:
	case LOOP_GET_STATUS64:
	case LOOP_SET_STATUS64:
		arg = (unsigned long) compat_ptr(arg);
	case LOOP_SET_FD:
	case LOOP_CHANGE_FD:
A
Al Viro 已提交
1525
		err = lo_ioctl(bdev, mode, cmd, arg);
1526 1527 1528 1529 1530 1531 1532 1533 1534
		break;
	default:
		err = -ENOIOCTLCMD;
		break;
	}
	return err;
}
#endif

A
Al Viro 已提交
1535
static int lo_open(struct block_device *bdev, fmode_t mode)
L
Linus Torvalds 已提交
1536
{
1537 1538 1539 1540 1541 1542 1543 1544 1545
	struct loop_device *lo;
	int err = 0;

	mutex_lock(&loop_index_mutex);
	lo = bdev->bd_disk->private_data;
	if (!lo) {
		err = -ENXIO;
		goto out;
	}
L
Linus Torvalds 已提交
1546

1547
	atomic_inc(&lo->lo_refcnt);
1548 1549 1550
out:
	mutex_unlock(&loop_index_mutex);
	return err;
L
Linus Torvalds 已提交
1551 1552
}

1553
static void lo_release(struct gendisk *disk, fmode_t mode)
L
Linus Torvalds 已提交
1554
{
A
Al Viro 已提交
1555
	struct loop_device *lo = disk->private_data;
1556
	int err;
L
Linus Torvalds 已提交
1557

1558 1559
	if (atomic_dec_return(&lo->lo_refcnt))
		return;
1560

1561
	mutex_lock(&lo->lo_ctl_mutex);
1562 1563 1564 1565 1566
	if (lo->lo_flags & LO_FLAGS_AUTOCLEAR) {
		/*
		 * In autoclear mode, stop the loop thread
		 * and remove configuration after last close.
		 */
1567
		err = loop_clr_fd(lo);
1568
		if (!err)
1569
			return;
1570 1571 1572 1573 1574 1575 1576
	} else {
		/*
		 * Otherwise keep thread (if running) and config,
		 * but flush possible ongoing bios in thread.
		 */
		loop_flush(lo);
	}
1577

1578
	mutex_unlock(&lo->lo_ctl_mutex);
L
Linus Torvalds 已提交
1579 1580
}

1581
static const struct block_device_operations lo_fops = {
L
Linus Torvalds 已提交
1582
	.owner =	THIS_MODULE,
A
Al Viro 已提交
1583 1584 1585
	.open =		lo_open,
	.release =	lo_release,
	.ioctl =	lo_ioctl,
1586
#ifdef CONFIG_COMPAT
A
Al Viro 已提交
1587
	.compat_ioctl =	lo_compat_ioctl,
1588
#endif
L
Linus Torvalds 已提交
1589 1590 1591 1592 1593
};

/*
 * And now the modules code and kernel interface.
 */
1594
static int max_loop;
N
Namhyung Kim 已提交
1595
module_param(max_loop, int, S_IRUGO);
K
Ken Chen 已提交
1596
MODULE_PARM_DESC(max_loop, "Maximum number of loop devices");
N
Namhyung Kim 已提交
1597
module_param(max_part, int, S_IRUGO);
1598
MODULE_PARM_DESC(max_part, "Maximum number of partitions per loop device");
L
Linus Torvalds 已提交
1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611
MODULE_LICENSE("GPL");
MODULE_ALIAS_BLOCKDEV_MAJOR(LOOP_MAJOR);

int loop_register_transfer(struct loop_func_table *funcs)
{
	unsigned int n = funcs->number;

	if (n >= MAX_LO_CRYPT || xfer_funcs[n])
		return -EINVAL;
	xfer_funcs[n] = funcs;
	return 0;
}

1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623
static int unregister_transfer_cb(int id, void *ptr, void *data)
{
	struct loop_device *lo = ptr;
	struct loop_func_table *xfer = data;

	mutex_lock(&lo->lo_ctl_mutex);
	if (lo->lo_encryption == xfer)
		loop_release_xfer(lo);
	mutex_unlock(&lo->lo_ctl_mutex);
	return 0;
}

L
Linus Torvalds 已提交
1624 1625 1626 1627 1628 1629 1630 1631 1632
int loop_unregister_transfer(int number)
{
	unsigned int n = number;
	struct loop_func_table *xfer;

	if (n == 0 || n >= MAX_LO_CRYPT || (xfer = xfer_funcs[n]) == NULL)
		return -EINVAL;

	xfer_funcs[n] = NULL;
1633
	idr_for_each(&loop_index_idr, &unregister_transfer_cb, xfer);
L
Linus Torvalds 已提交
1634 1635 1636 1637 1638 1639
	return 0;
}

EXPORT_SYMBOL(loop_register_transfer);
EXPORT_SYMBOL(loop_unregister_transfer);

1640 1641 1642 1643
static int loop_queue_rq(struct blk_mq_hw_ctx *hctx,
		const struct blk_mq_queue_data *bd)
{
	struct loop_cmd *cmd = blk_mq_rq_to_pdu(bd->rq);
1644
	struct loop_device *lo = cmd->rq->q->queuedata;
1645 1646 1647

	blk_mq_start_request(bd->rq);

1648 1649 1650
	if (lo->lo_state != Lo_bound)
		return -EIO;

1651 1652 1653
	switch (req_op(cmd->rq)) {
	case REQ_OP_FLUSH:
	case REQ_OP_DISCARD:
M
Ming Lei 已提交
1654
		cmd->use_aio = false;
1655 1656 1657 1658 1659
		break;
	default:
		cmd->use_aio = lo->use_dio;
		break;
	}
M
Ming Lei 已提交
1660

M
Ming Lei 已提交
1661
	queue_kthread_work(&lo->worker, &cmd->work);
1662 1663 1664 1665 1666 1667

	return BLK_MQ_RQ_QUEUE_OK;
}

static void loop_handle_cmd(struct loop_cmd *cmd)
{
1668
	const bool write = op_is_write(req_op(cmd->rq));
1669
	struct loop_device *lo = cmd->rq->q->queuedata;
1670
	int ret = 0;
1671

1672 1673
	if (write && (lo->lo_flags & LO_FLAGS_READ_ONLY)) {
		ret = -EIO;
1674
		goto failed;
1675
	}
1676

M
Ming Lei 已提交
1677
	ret = do_req_filebacked(lo, cmd->rq);
1678
 failed:
M
Ming Lei 已提交
1679 1680
	/* complete non-aio request */
	if (!cmd->use_aio || ret)
1681
		blk_mq_complete_request(cmd->rq, ret ? -EIO : 0);
1682 1683
}

M
Ming Lei 已提交
1684
static void loop_queue_work(struct kthread_work *work)
1685 1686
{
	struct loop_cmd *cmd =
M
Ming Lei 已提交
1687
		container_of(work, struct loop_cmd, work);
1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698

	loop_handle_cmd(cmd);
}

static int loop_init_request(void *data, struct request *rq,
		unsigned int hctx_idx, unsigned int request_idx,
		unsigned int numa_node)
{
	struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);

	cmd->rq = rq;
M
Ming Lei 已提交
1699
	init_kthread_work(&cmd->work, loop_queue_work);
1700 1701 1702 1703 1704 1705 1706 1707 1708 1709

	return 0;
}

static struct blk_mq_ops loop_mq_ops = {
	.queue_rq       = loop_queue_rq,
	.map_queue      = blk_mq_map_queue,
	.init_request	= loop_init_request,
};

1710
static int loop_add(struct loop_device **l, int i)
1711 1712 1713
{
	struct loop_device *lo;
	struct gendisk *disk;
1714
	int err;
1715

1716
	err = -ENOMEM;
1717
	lo = kzalloc(sizeof(*lo), GFP_KERNEL);
1718
	if (!lo)
1719
		goto out;
1720

1721 1722
	lo->lo_state = Lo_unbound;

T
Tejun Heo 已提交
1723
	/* allocate id, if @id >= 0, we're requesting that specific id */
1724
	if (i >= 0) {
T
Tejun Heo 已提交
1725 1726
		err = idr_alloc(&loop_index_idr, lo, i, i + 1, GFP_KERNEL);
		if (err == -ENOSPC)
1727 1728
			err = -EEXIST;
	} else {
T
Tejun Heo 已提交
1729
		err = idr_alloc(&loop_index_idr, lo, 0, 0, GFP_KERNEL);
1730 1731 1732
	}
	if (err < 0)
		goto out_free_dev;
T
Tejun Heo 已提交
1733
	i = err;
1734

1735
	err = -ENOMEM;
1736 1737 1738 1739 1740 1741 1742 1743 1744 1745
	lo->tag_set.ops = &loop_mq_ops;
	lo->tag_set.nr_hw_queues = 1;
	lo->tag_set.queue_depth = 128;
	lo->tag_set.numa_node = NUMA_NO_NODE;
	lo->tag_set.cmd_size = sizeof(struct loop_cmd);
	lo->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE;
	lo->tag_set.driver_data = lo;

	err = blk_mq_alloc_tag_set(&lo->tag_set);
	if (err)
1746
		goto out_free_idr;
1747

1748 1749 1750 1751 1752
	lo->lo_queue = blk_mq_init_queue(&lo->tag_set);
	if (IS_ERR_OR_NULL(lo->lo_queue)) {
		err = PTR_ERR(lo->lo_queue);
		goto out_cleanup_tags;
	}
1753 1754
	lo->lo_queue->queuedata = lo;

1755 1756 1757 1758 1759 1760
	/*
	 * It doesn't make sense to enable merge because the I/O
	 * submitted to backing file is handled page by page.
	 */
	queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, lo->lo_queue);

1761
	err = -ENOMEM;
1762
	disk = lo->lo_disk = alloc_disk(1 << part_shift);
1763 1764 1765
	if (!disk)
		goto out_free_queue;

1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786
	/*
	 * Disable partition scanning by default. The in-kernel partition
	 * scanning can be requested individually per-device during its
	 * setup. Userspace can always add and remove partitions from all
	 * devices. The needed partition minors are allocated from the
	 * extended minor space, the main loop device numbers will continue
	 * to match the loop minors, regardless of the number of partitions
	 * used.
	 *
	 * If max_part is given, partition scanning is globally enabled for
	 * all loop devices. The minors for the main loop devices will be
	 * multiples of max_part.
	 *
	 * Note: Global-for-all-devices, set-only-at-init, read-only module
	 * parameteters like 'max_loop' and 'max_part' make things needlessly
	 * complicated, are too static, inflexible and may surprise
	 * userspace tools. Parameters like this in general should be avoided.
	 */
	if (!part_shift)
		disk->flags |= GENHD_FL_NO_PART_SCAN;
	disk->flags |= GENHD_FL_EXT_DEVT;
1787
	mutex_init(&lo->lo_ctl_mutex);
1788
	atomic_set(&lo->lo_refcnt, 0);
1789 1790 1791
	lo->lo_number		= i;
	spin_lock_init(&lo->lo_lock);
	disk->major		= LOOP_MAJOR;
1792
	disk->first_minor	= i << part_shift;
1793 1794 1795 1796
	disk->fops		= &lo_fops;
	disk->private_data	= lo;
	disk->queue		= lo->lo_queue;
	sprintf(disk->disk_name, "loop%d", i);
1797 1798 1799
	add_disk(disk);
	*l = lo;
	return lo->lo_number;
1800 1801 1802

out_free_queue:
	blk_cleanup_queue(lo->lo_queue);
1803 1804
out_cleanup_tags:
	blk_mq_free_tag_set(&lo->tag_set);
1805 1806
out_free_idr:
	idr_remove(&loop_index_idr, i);
1807 1808 1809
out_free_dev:
	kfree(lo);
out:
1810
	return err;
1811 1812
}

1813
static void loop_remove(struct loop_device *lo)
L
Linus Torvalds 已提交
1814
{
1815
	blk_cleanup_queue(lo->lo_queue);
1816
	del_gendisk(lo->lo_disk);
1817
	blk_mq_free_tag_set(&lo->tag_set);
1818 1819 1820
	put_disk(lo->lo_disk);
	kfree(lo);
}
L
Linus Torvalds 已提交
1821

1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833
static int find_free_cb(int id, void *ptr, void *data)
{
	struct loop_device *lo = ptr;
	struct loop_device **l = data;

	if (lo->lo_state == Lo_unbound) {
		*l = lo;
		return 1;
	}
	return 0;
}

1834
static int loop_lookup(struct loop_device **l, int i)
K
Ken Chen 已提交
1835 1836
{
	struct loop_device *lo;
1837
	int ret = -ENODEV;
K
Ken Chen 已提交
1838

1839 1840 1841 1842 1843 1844 1845 1846 1847
	if (i < 0) {
		int err;

		err = idr_for_each(&loop_index_idr, &find_free_cb, &lo);
		if (err == 1) {
			*l = lo;
			ret = lo->lo_number;
		}
		goto out;
K
Ken Chen 已提交
1848 1849
	}

1850
	/* lookup and return a specific i */
1851
	lo = idr_find(&loop_index_idr, i);
K
Ken Chen 已提交
1852
	if (lo) {
1853 1854
		*l = lo;
		ret = lo->lo_number;
K
Ken Chen 已提交
1855
	}
1856
out:
1857
	return ret;
K
Ken Chen 已提交
1858 1859
}

1860 1861
static struct kobject *loop_probe(dev_t dev, int *part, void *data)
{
A
Al Viro 已提交
1862
	struct loop_device *lo;
1863
	struct kobject *kobj;
1864
	int err;
1865

1866 1867 1868 1869 1870
	mutex_lock(&loop_index_mutex);
	err = loop_lookup(&lo, MINOR(dev) >> part_shift);
	if (err < 0)
		err = loop_add(&lo, MINOR(dev) >> part_shift);
	if (err < 0)
1871
		kobj = NULL;
1872 1873 1874
	else
		kobj = get_disk(lo->lo_disk);
	mutex_unlock(&loop_index_mutex);
1875 1876

	*part = 0;
1877
	return kobj;
1878 1879
}

1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905
static long loop_control_ioctl(struct file *file, unsigned int cmd,
			       unsigned long parm)
{
	struct loop_device *lo;
	int ret = -ENOSYS;

	mutex_lock(&loop_index_mutex);
	switch (cmd) {
	case LOOP_CTL_ADD:
		ret = loop_lookup(&lo, parm);
		if (ret >= 0) {
			ret = -EEXIST;
			break;
		}
		ret = loop_add(&lo, parm);
		break;
	case LOOP_CTL_REMOVE:
		ret = loop_lookup(&lo, parm);
		if (ret < 0)
			break;
		mutex_lock(&lo->lo_ctl_mutex);
		if (lo->lo_state != Lo_unbound) {
			ret = -EBUSY;
			mutex_unlock(&lo->lo_ctl_mutex);
			break;
		}
1906
		if (atomic_read(&lo->lo_refcnt) > 0) {
1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943
			ret = -EBUSY;
			mutex_unlock(&lo->lo_ctl_mutex);
			break;
		}
		lo->lo_disk->private_data = NULL;
		mutex_unlock(&lo->lo_ctl_mutex);
		idr_remove(&loop_index_idr, lo->lo_number);
		loop_remove(lo);
		break;
	case LOOP_CTL_GET_FREE:
		ret = loop_lookup(&lo, -1);
		if (ret >= 0)
			break;
		ret = loop_add(&lo, -1);
	}
	mutex_unlock(&loop_index_mutex);

	return ret;
}

static const struct file_operations loop_ctl_fops = {
	.open		= nonseekable_open,
	.unlocked_ioctl	= loop_control_ioctl,
	.compat_ioctl	= loop_control_ioctl,
	.owner		= THIS_MODULE,
	.llseek		= noop_llseek,
};

static struct miscdevice loop_misc = {
	.minor		= LOOP_CTRL_MINOR,
	.name		= "loop-control",
	.fops		= &loop_ctl_fops,
};

MODULE_ALIAS_MISCDEV(LOOP_CTRL_MINOR);
MODULE_ALIAS("devname:loop-control");

1944 1945
static int __init loop_init(void)
{
K
Ken Chen 已提交
1946 1947
	int i, nr;
	unsigned long range;
1948
	struct loop_device *lo;
1949
	int err;
K
Ken Chen 已提交
1950

1951 1952 1953
	err = misc_register(&loop_misc);
	if (err < 0)
		return err;
1954 1955

	part_shift = 0;
N
Namhyung Kim 已提交
1956
	if (max_part > 0) {
1957 1958
		part_shift = fls(max_part);

N
Namhyung Kim 已提交
1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969
		/*
		 * Adjust max_part according to part_shift as it is exported
		 * to user space so that user can decide correct minor number
		 * if [s]he want to create more devices.
		 *
		 * Note that -1 is required because partition 0 is reserved
		 * for the whole disk.
		 */
		max_part = (1UL << part_shift) - 1;
	}

1970 1971 1972 1973
	if ((1UL << part_shift) > DISK_MAX_PARTS) {
		err = -EINVAL;
		goto misc_out;
	}
1974

1975 1976 1977 1978
	if (max_loop > 1UL << (MINORBITS - part_shift)) {
		err = -EINVAL;
		goto misc_out;
	}
L
Linus Torvalds 已提交
1979

1980 1981 1982 1983 1984 1985 1986 1987
	/*
	 * If max_loop is specified, create that many devices upfront.
	 * This also becomes a hard limit. If max_loop is not specified,
	 * create CONFIG_BLK_DEV_LOOP_MIN_COUNT loop devices at module
	 * init time. Loop devices can be requested on-demand with the
	 * /dev/loop-control interface, or be instantiated by accessing
	 * a 'dead' device node.
	 */
1988
	if (max_loop) {
K
Ken Chen 已提交
1989
		nr = max_loop;
1990
		range = max_loop << part_shift;
K
Ken Chen 已提交
1991
	} else {
1992
		nr = CONFIG_BLK_DEV_LOOP_MIN_COUNT;
1993
		range = 1UL << MINORBITS;
K
Ken Chen 已提交
1994 1995
	}

1996 1997 1998 1999
	if (register_blkdev(LOOP_MAJOR, "loop")) {
		err = -EIO;
		goto misc_out;
	}
L
Linus Torvalds 已提交
2000

K
Ken Chen 已提交
2001 2002 2003
	blk_register_region(MKDEV(LOOP_MAJOR, 0), range,
				  THIS_MODULE, loop_probe, NULL, NULL);

2004
	/* pre-create number of devices given by config or max_loop */
2005 2006 2007 2008 2009
	mutex_lock(&loop_index_mutex);
	for (i = 0; i < nr; i++)
		loop_add(&lo, i);
	mutex_unlock(&loop_index_mutex);

2010
	printk(KERN_INFO "loop: module loaded\n");
L
Linus Torvalds 已提交
2011
	return 0;
2012 2013 2014 2015

misc_out:
	misc_deregister(&loop_misc);
	return err;
2016
}
K
Ken Chen 已提交
2017

2018 2019 2020
static int loop_exit_cb(int id, void *ptr, void *data)
{
	struct loop_device *lo = ptr;
K
Ken Chen 已提交
2021

2022 2023
	loop_remove(lo);
	return 0;
L
Linus Torvalds 已提交
2024 2025
}

2026
static void __exit loop_exit(void)
L
Linus Torvalds 已提交
2027
{
K
Ken Chen 已提交
2028
	unsigned long range;
L
Linus Torvalds 已提交
2029

2030
	range = max_loop ? max_loop << part_shift : 1UL << MINORBITS;
K
Ken Chen 已提交
2031

2032 2033
	idr_for_each(&loop_index_idr, &loop_exit_cb, NULL);
	idr_destroy(&loop_index_idr);
2034

K
Ken Chen 已提交
2035
	blk_unregister_region(MKDEV(LOOP_MAJOR, 0), range);
2036
	unregister_blkdev(LOOP_MAJOR, "loop");
2037 2038

	misc_deregister(&loop_misc);
L
Linus Torvalds 已提交
2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052
}

module_init(loop_init);
module_exit(loop_exit);

#ifndef MODULE
static int __init max_loop_setup(char *str)
{
	max_loop = simple_strtol(str, NULL, 0);
	return 1;
}

__setup("max_loop=", max_loop_setup);
#endif