dm.c 34.3 KB
Newer Older
L
Linus Torvalds 已提交
1 2
/*
 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
3
 * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved.
L
Linus Torvalds 已提交
4 5 6 7 8 9
 *
 * This file is released under the GPL.
 */

#include "dm.h"
#include "dm-bio-list.h"
M
Mike Anderson 已提交
10
#include "dm-uevent.h"
L
Linus Torvalds 已提交
11 12 13

#include <linux/init.h>
#include <linux/module.h>
A
Arjan van de Ven 已提交
14
#include <linux/mutex.h>
L
Linus Torvalds 已提交
15 16 17 18 19 20 21
#include <linux/moduleparam.h>
#include <linux/blkpg.h>
#include <linux/bio.h>
#include <linux/buffer_head.h>
#include <linux/mempool.h>
#include <linux/slab.h>
#include <linux/idr.h>
D
Darrick J. Wong 已提交
22
#include <linux/hdreg.h>
23
#include <linux/blktrace_api.h>
24
#include <linux/smp_lock.h>
L
Linus Torvalds 已提交
25

26 27
#define DM_MSG_PREFIX "core"

L
Linus Torvalds 已提交
28 29 30 31 32
static const char *_name = DM_NAME;

static unsigned int major = 0;
static unsigned int _major = 0;

33
static DEFINE_SPINLOCK(_minor_lock);
L
Linus Torvalds 已提交
34 35 36 37 38 39 40
/*
 * One of these is allocated per bio.
 */
struct dm_io {
	struct mapped_device *md;
	int error;
	atomic_t io_count;
R
Richard Kennedy 已提交
41
	struct bio *bio;
42
	unsigned long start_time;
L
Linus Torvalds 已提交
43 44 45 46 47 48
};

/*
 * One of these is allocated per target within a bio.  Hopefully
 * this will be simplified out one day.
 */
A
Alasdair G Kergon 已提交
49
struct dm_target_io {
L
Linus Torvalds 已提交
50 51 52 53 54 55 56
	struct dm_io *io;
	struct dm_target *ti;
	union map_info info;
};

union map_info *dm_get_mapinfo(struct bio *bio)
{
A
Alasdair G Kergon 已提交
57
	if (bio && bio->bi_private)
A
Alasdair G Kergon 已提交
58
		return &((struct dm_target_io *)bio->bi_private)->info;
A
Alasdair G Kergon 已提交
59
	return NULL;
L
Linus Torvalds 已提交
60 61
}

62 63
#define MINOR_ALLOCED ((void *)-1)

L
Linus Torvalds 已提交
64 65 66 67 68
/*
 * Bits for the md->flags field.
 */
#define DMF_BLOCK_IO 0
#define DMF_SUSPENDED 1
69
#define DMF_FROZEN 2
J
Jeff Mahoney 已提交
70
#define DMF_FREEING 3
71
#define DMF_DELETING 4
72
#define DMF_NOFLUSH_SUSPENDING 5
L
Linus Torvalds 已提交
73

74 75 76 77 78 79 80 81 82 83 84 85 86
/*
 * Work processed by per-device workqueue.
 */
struct dm_wq_req {
	enum {
		DM_WQ_FLUSH_ALL,
		DM_WQ_FLUSH_DEFERRED,
	} type;
	struct work_struct work;
	struct mapped_device *md;
	void *context;
};

L
Linus Torvalds 已提交
87
struct mapped_device {
88
	struct rw_semaphore io_lock;
89
	struct mutex suspend_lock;
90
	spinlock_t pushback_lock;
L
Linus Torvalds 已提交
91 92
	rwlock_t map_lock;
	atomic_t holders;
93
	atomic_t open_count;
L
Linus Torvalds 已提交
94 95 96

	unsigned long flags;

97
	struct request_queue *queue;
L
Linus Torvalds 已提交
98
	struct gendisk *disk;
M
Mike Anderson 已提交
99
	char name[16];
L
Linus Torvalds 已提交
100 101 102 103 104 105 106 107

	void *interface_ptr;

	/*
	 * A list of ios that arrived while we were suspended.
	 */
	atomic_t pending;
	wait_queue_head_t wait;
K
Kiyoshi Ueda 已提交
108
	struct bio_list deferred;
109
	struct bio_list pushback;
L
Linus Torvalds 已提交
110

111 112 113 114 115
	/*
	 * Processing queue (flush/barriers)
	 */
	struct workqueue_struct *wq;

L
Linus Torvalds 已提交
116 117 118 119 120 121 122 123 124 125 126
	/*
	 * The current mapping.
	 */
	struct dm_table *map;

	/*
	 * io objects are allocated from here.
	 */
	mempool_t *io_pool;
	mempool_t *tio_pool;

S
Stefan Bader 已提交
127 128
	struct bio_set *bs;

L
Linus Torvalds 已提交
129 130 131 132 133
	/*
	 * Event handling.
	 */
	atomic_t event_nr;
	wait_queue_head_t eventq;
M
Mike Anderson 已提交
134 135 136
	atomic_t uevent_seq;
	struct list_head uevent_list;
	spinlock_t uevent_lock; /* Protect access to uevent_list */
L
Linus Torvalds 已提交
137 138 139 140 141

	/*
	 * freeze/thaw support require holding onto a super block
	 */
	struct super_block *frozen_sb;
142
	struct block_device *suspended_bdev;
D
Darrick J. Wong 已提交
143 144 145

	/* forced geometry settings */
	struct hd_geometry geometry;
L
Linus Torvalds 已提交
146 147 148
};

#define MIN_IOS 256
149 150
static struct kmem_cache *_io_cache;
static struct kmem_cache *_tio_cache;
L
Linus Torvalds 已提交
151 152 153 154 155 156

static int __init local_init(void)
{
	int r;

	/* allocate a slab for the dm_ios */
A
Alasdair G Kergon 已提交
157
	_io_cache = KMEM_CACHE(dm_io, 0);
L
Linus Torvalds 已提交
158 159 160 161
	if (!_io_cache)
		return -ENOMEM;

	/* allocate a slab for the target ios */
A
Alasdair G Kergon 已提交
162
	_tio_cache = KMEM_CACHE(dm_target_io, 0);
L
Linus Torvalds 已提交
163 164 165 166 167
	if (!_tio_cache) {
		kmem_cache_destroy(_io_cache);
		return -ENOMEM;
	}

M
Mike Anderson 已提交
168 169 170 171 172 173 174
	r = dm_uevent_init();
	if (r) {
		kmem_cache_destroy(_tio_cache);
		kmem_cache_destroy(_io_cache);
		return r;
	}

L
Linus Torvalds 已提交
175 176 177 178 179
	_major = major;
	r = register_blkdev(_major, _name);
	if (r < 0) {
		kmem_cache_destroy(_tio_cache);
		kmem_cache_destroy(_io_cache);
M
Mike Anderson 已提交
180
		dm_uevent_exit();
L
Linus Torvalds 已提交
181 182 183 184 185 186 187 188 189 190 191 192 193
		return r;
	}

	if (!_major)
		_major = r;

	return 0;
}

static void local_exit(void)
{
	kmem_cache_destroy(_tio_cache);
	kmem_cache_destroy(_io_cache);
194
	unregister_blkdev(_major, _name);
M
Mike Anderson 已提交
195
	dm_uevent_exit();
L
Linus Torvalds 已提交
196 197 198 199 200 201

	_major = 0;

	DMINFO("cleaned up");
}

202
static int (*_inits[])(void) __initdata = {
L
Linus Torvalds 已提交
203 204 205 206
	local_init,
	dm_target_init,
	dm_linear_init,
	dm_stripe_init,
207
	dm_kcopyd_init,
L
Linus Torvalds 已提交
208 209 210
	dm_interface_init,
};

211
static void (*_exits[])(void) = {
L
Linus Torvalds 已提交
212 213 214 215
	local_exit,
	dm_target_exit,
	dm_linear_exit,
	dm_stripe_exit,
216
	dm_kcopyd_exit,
L
Linus Torvalds 已提交
217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255
	dm_interface_exit,
};

static int __init dm_init(void)
{
	const int count = ARRAY_SIZE(_inits);

	int r, i;

	for (i = 0; i < count; i++) {
		r = _inits[i]();
		if (r)
			goto bad;
	}

	return 0;

      bad:
	while (i--)
		_exits[i]();

	return r;
}

static void __exit dm_exit(void)
{
	int i = ARRAY_SIZE(_exits);

	while (i--)
		_exits[i]();
}

/*
 * Block device functions
 */
static int dm_blk_open(struct inode *inode, struct file *file)
{
	struct mapped_device *md;

J
Jeff Mahoney 已提交
256 257
	spin_lock(&_minor_lock);

L
Linus Torvalds 已提交
258
	md = inode->i_bdev->bd_disk->private_data;
J
Jeff Mahoney 已提交
259 260 261
	if (!md)
		goto out;

262 263
	if (test_bit(DMF_FREEING, &md->flags) ||
	    test_bit(DMF_DELETING, &md->flags)) {
J
Jeff Mahoney 已提交
264 265 266 267
		md = NULL;
		goto out;
	}

L
Linus Torvalds 已提交
268
	dm_get(md);
269
	atomic_inc(&md->open_count);
J
Jeff Mahoney 已提交
270 271 272 273 274

out:
	spin_unlock(&_minor_lock);

	return md ? 0 : -ENXIO;
L
Linus Torvalds 已提交
275 276 277 278 279 280 281
}

static int dm_blk_close(struct inode *inode, struct file *file)
{
	struct mapped_device *md;

	md = inode->i_bdev->bd_disk->private_data;
282
	atomic_dec(&md->open_count);
L
Linus Torvalds 已提交
283 284 285 286
	dm_put(md);
	return 0;
}

287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310
int dm_open_count(struct mapped_device *md)
{
	return atomic_read(&md->open_count);
}

/*
 * Guarantees nothing is using the device before it's deleted.
 */
int dm_lock_for_deletion(struct mapped_device *md)
{
	int r = 0;

	spin_lock(&_minor_lock);

	if (dm_open_count(md))
		r = -EBUSY;
	else
		set_bit(DMF_DELETING, &md->flags);

	spin_unlock(&_minor_lock);

	return r;
}

D
Darrick J. Wong 已提交
311 312 313 314 315 316 317
static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
{
	struct mapped_device *md = bdev->bd_disk->private_data;

	return dm_get_geometry(md, geo);
}

318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356
static int dm_blk_ioctl(struct inode *inode, struct file *file,
			unsigned int cmd, unsigned long arg)
{
	struct mapped_device *md;
	struct dm_table *map;
	struct dm_target *tgt;
	int r = -ENOTTY;

	/* We don't really need this lock, but we do need 'inode'. */
	unlock_kernel();

	md = inode->i_bdev->bd_disk->private_data;

	map = dm_get_table(md);

	if (!map || !dm_table_get_size(map))
		goto out;

	/* We only support devices that have a single target */
	if (dm_table_get_num_targets(map) != 1)
		goto out;

	tgt = dm_table_get_target(map, 0);

	if (dm_suspended(md)) {
		r = -EAGAIN;
		goto out;
	}

	if (tgt->type->ioctl)
		r = tgt->type->ioctl(tgt, inode, file, cmd, arg);

out:
	dm_table_put(map);

	lock_kernel();
	return r;
}

A
Alasdair G Kergon 已提交
357
static struct dm_io *alloc_io(struct mapped_device *md)
L
Linus Torvalds 已提交
358 359 360 361
{
	return mempool_alloc(md->io_pool, GFP_NOIO);
}

A
Alasdair G Kergon 已提交
362
static void free_io(struct mapped_device *md, struct dm_io *io)
L
Linus Torvalds 已提交
363 364 365 366
{
	mempool_free(io, md->io_pool);
}

A
Alasdair G Kergon 已提交
367
static struct dm_target_io *alloc_tio(struct mapped_device *md)
L
Linus Torvalds 已提交
368 369 370 371
{
	return mempool_alloc(md->tio_pool, GFP_NOIO);
}

A
Alasdair G Kergon 已提交
372
static void free_tio(struct mapped_device *md, struct dm_target_io *tio)
L
Linus Torvalds 已提交
373 374 375 376
{
	mempool_free(tio, md->tio_pool);
}

377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406
static void start_io_acct(struct dm_io *io)
{
	struct mapped_device *md = io->md;

	io->start_time = jiffies;

	preempt_disable();
	disk_round_stats(dm_disk(md));
	preempt_enable();
	dm_disk(md)->in_flight = atomic_inc_return(&md->pending);
}

static int end_io_acct(struct dm_io *io)
{
	struct mapped_device *md = io->md;
	struct bio *bio = io->bio;
	unsigned long duration = jiffies - io->start_time;
	int pending;
	int rw = bio_data_dir(bio);

	preempt_disable();
	disk_round_stats(dm_disk(md));
	preempt_enable();
	dm_disk(md)->in_flight = pending = atomic_dec_return(&md->pending);

	disk_stat_add(dm_disk(md), ticks[rw], duration);

	return !pending;
}

L
Linus Torvalds 已提交
407 408 409 410 411
/*
 * Add the bio to the list of deferred io.
 */
static int queue_io(struct mapped_device *md, struct bio *bio)
{
412
	down_write(&md->io_lock);
L
Linus Torvalds 已提交
413 414

	if (!test_bit(DMF_BLOCK_IO, &md->flags)) {
415
		up_write(&md->io_lock);
L
Linus Torvalds 已提交
416 417 418 419 420
		return 1;
	}

	bio_list_add(&md->deferred, bio);

421
	up_write(&md->io_lock);
L
Linus Torvalds 已提交
422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442
	return 0;		/* deferred successfully */
}

/*
 * Everyone (including functions in this file), should use this
 * function to access the md->map field, and make sure they call
 * dm_table_put() when finished.
 */
struct dm_table *dm_get_table(struct mapped_device *md)
{
	struct dm_table *t;

	read_lock(&md->map_lock);
	t = md->map;
	if (t)
		dm_table_get(t);
	read_unlock(&md->map_lock);

	return t;
}

D
Darrick J. Wong 已提交
443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469
/*
 * Get the geometry associated with a dm device
 */
int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
{
	*geo = md->geometry;

	return 0;
}

/*
 * Set the geometry of a device.
 */
int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
{
	sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;

	if (geo->start > sz) {
		DMWARN("Start sector is beyond the geometry limits.");
		return -EINVAL;
	}

	md->geometry = *geo;

	return 0;
}

L
Linus Torvalds 已提交
470 471 472 473 474 475 476 477 478
/*-----------------------------------------------------------------
 * CRUD START:
 *   A more elegant soln is in the works that uses the queue
 *   merge fn, unfortunately there are a couple of changes to
 *   the block layer that I want to make for this.  So in the
 *   interests of getting something for people to use I give
 *   you this clearly demarcated crap.
 *---------------------------------------------------------------*/

479 480 481 482 483
static int __noflush_suspending(struct mapped_device *md)
{
	return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
}

L
Linus Torvalds 已提交
484 485 486 487
/*
 * Decrements the number of outstanding ios that a bio has been
 * cloned into, completing the original io if necc.
 */
488
static void dec_pending(struct dm_io *io, int error)
L
Linus Torvalds 已提交
489
{
490 491 492 493
	unsigned long flags;

	/* Push-back supersedes any I/O errors */
	if (error && !(io->error > 0 && __noflush_suspending(io->md)))
L
Linus Torvalds 已提交
494 495 496
		io->error = error;

	if (atomic_dec_and_test(&io->io_count)) {
497 498 499 500 501 502 503 504 505 506 507 508 509 510 511
		if (io->error == DM_ENDIO_REQUEUE) {
			/*
			 * Target requested pushing back the I/O.
			 * This must be handled before the sleeper on
			 * suspend queue merges the pushback list.
			 */
			spin_lock_irqsave(&io->md->pushback_lock, flags);
			if (__noflush_suspending(io->md))
				bio_list_add(&io->md->pushback, io->bio);
			else
				/* noflush suspend was interrupted. */
				io->error = -EIO;
			spin_unlock_irqrestore(&io->md->pushback_lock, flags);
		}

512
		if (end_io_acct(io))
L
Linus Torvalds 已提交
513 514 515
			/* nudge anyone waiting on suspend queue */
			wake_up(&io->md->wait);

516 517 518 519
		if (io->error != DM_ENDIO_REQUEUE) {
			blk_add_trace_bio(io->md->queue, io->bio,
					  BLK_TA_COMPLETE);

520
			bio_endio(io->bio, io->error);
521
		}
522

L
Linus Torvalds 已提交
523 524 525 526
		free_io(io->md, io);
	}
}

527
static void clone_endio(struct bio *bio, int error)
L
Linus Torvalds 已提交
528 529
{
	int r = 0;
A
Alasdair G Kergon 已提交
530
	struct dm_target_io *tio = bio->bi_private;
S
Stefan Bader 已提交
531
	struct mapped_device *md = tio->io->md;
L
Linus Torvalds 已提交
532 533 534 535 536 537 538
	dm_endio_fn endio = tio->ti->type->end_io;

	if (!bio_flagged(bio, BIO_UPTODATE) && !error)
		error = -EIO;

	if (endio) {
		r = endio(tio->ti, bio, error, &tio->info);
539 540 541 542 543
		if (r < 0 || r == DM_ENDIO_REQUEUE)
			/*
			 * error and requeue request are handled
			 * in dec_pending().
			 */
L
Linus Torvalds 已提交
544
			error = r;
545 546
		else if (r == DM_ENDIO_INCOMPLETE)
			/* The target will handle the io */
547
			return;
548 549 550 551
		else if (r) {
			DMWARN("unimplemented target endio return value: %d", r);
			BUG();
		}
L
Linus Torvalds 已提交
552 553
	}

S
Stefan Bader 已提交
554 555 556 557 558 559 560
	dec_pending(tio->io, error);

	/*
	 * Store md for cleanup instead of tio which is about to get freed.
	 */
	bio->bi_private = md->bs;

L
Linus Torvalds 已提交
561
	bio_put(bio);
S
Stefan Bader 已提交
562
	free_tio(md, tio);
L
Linus Torvalds 已提交
563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585
}

static sector_t max_io_len(struct mapped_device *md,
			   sector_t sector, struct dm_target *ti)
{
	sector_t offset = sector - ti->begin;
	sector_t len = ti->len - offset;

	/*
	 * Does the target need to split even further ?
	 */
	if (ti->split_io) {
		sector_t boundary;
		boundary = ((offset + ti->split_io) & ~(ti->split_io - 1))
			   - offset;
		if (len > boundary)
			len = boundary;
	}

	return len;
}

static void __map_bio(struct dm_target *ti, struct bio *clone,
A
Alasdair G Kergon 已提交
586
		      struct dm_target_io *tio)
L
Linus Torvalds 已提交
587 588
{
	int r;
589
	sector_t sector;
S
Stefan Bader 已提交
590
	struct mapped_device *md;
L
Linus Torvalds 已提交
591 592 593 594 595 596 597 598 599 600 601 602 603 604 605

	/*
	 * Sanity checks.
	 */
	BUG_ON(!clone->bi_size);

	clone->bi_end_io = clone_endio;
	clone->bi_private = tio;

	/*
	 * Map the clone.  If r == 0 we don't need to do
	 * anything, the target has assumed ownership of
	 * this io.
	 */
	atomic_inc(&tio->io->io_count);
606
	sector = clone->bi_sector;
L
Linus Torvalds 已提交
607
	r = ti->type->map(ti, clone, &tio->info);
608
	if (r == DM_MAPIO_REMAPPED) {
L
Linus Torvalds 已提交
609
		/* the bio has been remapped so dispatch it */
610

A
Alasdair G Kergon 已提交
611
		blk_add_trace_remap(bdev_get_queue(clone->bi_bdev), clone,
612 613
				    tio->io->bio->bi_bdev->bd_dev,
				    clone->bi_sector, sector);
614

L
Linus Torvalds 已提交
615
		generic_make_request(clone);
616 617
	} else if (r < 0 || r == DM_MAPIO_REQUEUE) {
		/* error the io and bail out, or requeue it if needed */
S
Stefan Bader 已提交
618 619 620 621 622 623
		md = tio->io->md;
		dec_pending(tio->io, r);
		/*
		 * Store bio_set for cleanup.
		 */
		clone->bi_private = md->bs;
L
Linus Torvalds 已提交
624
		bio_put(clone);
S
Stefan Bader 已提交
625
		free_tio(md, tio);
626 627 628
	} else if (r) {
		DMWARN("unimplemented target map return value: %d", r);
		BUG();
L
Linus Torvalds 已提交
629 630 631 632 633 634 635 636 637 638 639 640 641
	}
}

struct clone_info {
	struct mapped_device *md;
	struct dm_table *map;
	struct bio *bio;
	struct dm_io *io;
	sector_t sector;
	sector_t sector_count;
	unsigned short idx;
};

P
Peter Osterlund 已提交
642 643
static void dm_bio_destructor(struct bio *bio)
{
S
Stefan Bader 已提交
644 645 646
	struct bio_set *bs = bio->bi_private;

	bio_free(bio, bs);
P
Peter Osterlund 已提交
647 648
}

L
Linus Torvalds 已提交
649 650 651 652 653
/*
 * Creates a little bio that is just does part of a bvec.
 */
static struct bio *split_bvec(struct bio *bio, sector_t sector,
			      unsigned short idx, unsigned int offset,
S
Stefan Bader 已提交
654
			      unsigned int len, struct bio_set *bs)
L
Linus Torvalds 已提交
655 656 657 658
{
	struct bio *clone;
	struct bio_vec *bv = bio->bi_io_vec + idx;

S
Stefan Bader 已提交
659
	clone = bio_alloc_bioset(GFP_NOIO, 1, bs);
P
Peter Osterlund 已提交
660
	clone->bi_destructor = dm_bio_destructor;
L
Linus Torvalds 已提交
661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678
	*clone->bi_io_vec = *bv;

	clone->bi_sector = sector;
	clone->bi_bdev = bio->bi_bdev;
	clone->bi_rw = bio->bi_rw;
	clone->bi_vcnt = 1;
	clone->bi_size = to_bytes(len);
	clone->bi_io_vec->bv_offset = offset;
	clone->bi_io_vec->bv_len = clone->bi_size;

	return clone;
}

/*
 * Creates a bio that consists of range of complete bvecs.
 */
static struct bio *clone_bio(struct bio *bio, sector_t sector,
			     unsigned short idx, unsigned short bv_count,
S
Stefan Bader 已提交
679
			     unsigned int len, struct bio_set *bs)
L
Linus Torvalds 已提交
680 681 682
{
	struct bio *clone;

S
Stefan Bader 已提交
683 684 685
	clone = bio_alloc_bioset(GFP_NOIO, bio->bi_max_vecs, bs);
	__bio_clone(clone, bio);
	clone->bi_destructor = dm_bio_destructor;
L
Linus Torvalds 已提交
686 687 688 689 690 691 692 693 694
	clone->bi_sector = sector;
	clone->bi_idx = idx;
	clone->bi_vcnt = idx + bv_count;
	clone->bi_size = to_bytes(len);
	clone->bi_flags &= ~(1 << BIO_SEG_VALID);

	return clone;
}

695
static int __clone_and_map(struct clone_info *ci)
L
Linus Torvalds 已提交
696 697
{
	struct bio *clone, *bio = ci->bio;
698 699
	struct dm_target *ti;
	sector_t len = 0, max;
A
Alasdair G Kergon 已提交
700
	struct dm_target_io *tio;
L
Linus Torvalds 已提交
701

702 703 704 705 706 707
	ti = dm_table_find_target(ci->map, ci->sector);
	if (!dm_target_is_valid(ti))
		return -EIO;

	max = max_io_len(ci->md, ci->sector, ti);

L
Linus Torvalds 已提交
708 709 710 711 712 713 714 715 716 717 718 719 720 721
	/*
	 * Allocate a target io object.
	 */
	tio = alloc_tio(ci->md);
	tio->io = ci->io;
	tio->ti = ti;
	memset(&tio->info, 0, sizeof(tio->info));

	if (ci->sector_count <= max) {
		/*
		 * Optimise for the simple case where we can do all of
		 * the remaining io with a single clone.
		 */
		clone = clone_bio(bio, ci->sector, ci->idx,
S
Stefan Bader 已提交
722 723
				  bio->bi_vcnt - ci->idx, ci->sector_count,
				  ci->md->bs);
L
Linus Torvalds 已提交
724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745
		__map_bio(ti, clone, tio);
		ci->sector_count = 0;

	} else if (to_sector(bio->bi_io_vec[ci->idx].bv_len) <= max) {
		/*
		 * There are some bvecs that don't span targets.
		 * Do as many of these as possible.
		 */
		int i;
		sector_t remaining = max;
		sector_t bv_len;

		for (i = ci->idx; remaining && (i < bio->bi_vcnt); i++) {
			bv_len = to_sector(bio->bi_io_vec[i].bv_len);

			if (bv_len > remaining)
				break;

			remaining -= bv_len;
			len += bv_len;
		}

S
Stefan Bader 已提交
746 747
		clone = clone_bio(bio, ci->sector, ci->idx, i - ci->idx, len,
				  ci->md->bs);
L
Linus Torvalds 已提交
748 749 750 751 752 753 754 755
		__map_bio(ti, clone, tio);

		ci->sector += len;
		ci->sector_count -= len;
		ci->idx = i;

	} else {
		/*
756
		 * Handle a bvec that must be split between two or more targets.
L
Linus Torvalds 已提交
757 758
		 */
		struct bio_vec *bv = bio->bi_io_vec + ci->idx;
759 760
		sector_t remaining = to_sector(bv->bv_len);
		unsigned int offset = 0;
L
Linus Torvalds 已提交
761

762 763 764
		do {
			if (offset) {
				ti = dm_table_find_target(ci->map, ci->sector);
765 766 767
				if (!dm_target_is_valid(ti))
					return -EIO;

768
				max = max_io_len(ci->md, ci->sector, ti);
L
Linus Torvalds 已提交
769

770 771 772 773 774 775 776 777 778
				tio = alloc_tio(ci->md);
				tio->io = ci->io;
				tio->ti = ti;
				memset(&tio->info, 0, sizeof(tio->info));
			}

			len = min(remaining, max);

			clone = split_bvec(bio, ci->sector, ci->idx,
S
Stefan Bader 已提交
779 780
					   bv->bv_offset + offset, len,
					   ci->md->bs);
781 782 783 784 785 786 787

			__map_bio(ti, clone, tio);

			ci->sector += len;
			ci->sector_count -= len;
			offset += to_bytes(len);
		} while (remaining -= len);
L
Linus Torvalds 已提交
788 789 790

		ci->idx++;
	}
791 792

	return 0;
L
Linus Torvalds 已提交
793 794 795 796 797
}

/*
 * Split the bio into several clones.
 */
M
Milan Broz 已提交
798
static int __split_bio(struct mapped_device *md, struct bio *bio)
L
Linus Torvalds 已提交
799 800
{
	struct clone_info ci;
801
	int error = 0;
L
Linus Torvalds 已提交
802 803

	ci.map = dm_get_table(md);
M
Milan Broz 已提交
804 805
	if (unlikely(!ci.map))
		return -EIO;
L
Linus Torvalds 已提交
806 807 808 809 810 811 812 813 814 815 816 817

	ci.md = md;
	ci.bio = bio;
	ci.io = alloc_io(md);
	ci.io->error = 0;
	atomic_set(&ci.io->io_count, 1);
	ci.io->bio = bio;
	ci.io->md = md;
	ci.sector = bio->bi_sector;
	ci.sector_count = bio_sectors(bio);
	ci.idx = bio->bi_idx;

818
	start_io_acct(ci.io);
819 820
	while (ci.sector_count && !error)
		error = __clone_and_map(&ci);
L
Linus Torvalds 已提交
821 822

	/* drop the extra reference count */
823
	dec_pending(ci.io, error);
L
Linus Torvalds 已提交
824
	dm_table_put(ci.map);
M
Milan Broz 已提交
825 826

	return 0;
L
Linus Torvalds 已提交
827 828 829 830 831
}
/*-----------------------------------------------------------------
 * CRUD END
 *---------------------------------------------------------------*/

M
Milan Broz 已提交
832 833 834 835 836 837 838 839
static int dm_merge_bvec(struct request_queue *q,
			 struct bvec_merge_data *bvm,
			 struct bio_vec *biovec)
{
	struct mapped_device *md = q->queuedata;
	struct dm_table *map = dm_get_table(md);
	struct dm_target *ti;
	sector_t max_sectors;
840
	int max_size = 0;
M
Milan Broz 已提交
841 842

	if (unlikely(!map))
843
		goto out;
M
Milan Broz 已提交
844 845

	ti = dm_table_find_target(map, bvm->bi_sector);
846 847
	if (!dm_target_is_valid(ti))
		goto out_table;
M
Milan Broz 已提交
848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865

	/*
	 * Find maximum amount of I/O that won't need splitting
	 */
	max_sectors = min(max_io_len(md, bvm->bi_sector, ti),
			  (sector_t) BIO_MAX_SECTORS);
	max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size;
	if (max_size < 0)
		max_size = 0;

	/*
	 * merge_bvec_fn() returns number of bytes
	 * it can accept at this offset
	 * max is precomputed maximal io size
	 */
	if (max_size && ti->type->merge)
		max_size = ti->type->merge(ti, bvm, biovec, max_size);

866
out_table:
867 868 869
	dm_table_put(map);

out:
M
Milan Broz 已提交
870 871 872 873 874 875 876 877 878
	/*
	 * Always allow an entire first page
	 */
	if (max_size <= biovec->bv_len && !(bvm->bi_size >> SECTOR_SHIFT))
		max_size = biovec->bv_len;

	return max_size;
}

L
Linus Torvalds 已提交
879 880 881 882
/*
 * The request function that just remaps the bio built up by
 * dm_merge_bvec.
 */
883
static int dm_request(struct request_queue *q, struct bio *bio)
L
Linus Torvalds 已提交
884
{
M
Milan Broz 已提交
885
	int r = -EIO;
886
	int rw = bio_data_dir(bio);
L
Linus Torvalds 已提交
887 888
	struct mapped_device *md = q->queuedata;

S
Stefan Bader 已提交
889 890 891 892 893
	/*
	 * There is no use in forwarding any barrier request since we can't
	 * guarantee it is (or can be) handled by the targets correctly.
	 */
	if (unlikely(bio_barrier(bio))) {
894
		bio_endio(bio, -EOPNOTSUPP);
S
Stefan Bader 已提交
895 896 897
		return 0;
	}

898
	down_read(&md->io_lock);
L
Linus Torvalds 已提交
899

900 901 902
	disk_stat_inc(dm_disk(md), ios[rw]);
	disk_stat_add(dm_disk(md), sectors[rw], bio_sectors(bio));

L
Linus Torvalds 已提交
903 904 905 906 907
	/*
	 * If we're suspended we have to queue
	 * this io for later.
	 */
	while (test_bit(DMF_BLOCK_IO, &md->flags)) {
908
		up_read(&md->io_lock);
L
Linus Torvalds 已提交
909

M
Milan Broz 已提交
910 911
		if (bio_rw(bio) != READA)
			r = queue_io(md, bio);
L
Linus Torvalds 已提交
912

M
Milan Broz 已提交
913 914
		if (r <= 0)
			goto out_req;
L
Linus Torvalds 已提交
915 916 917 918 919

		/*
		 * We're in a while loop, because someone could suspend
		 * before we get to the following read lock.
		 */
920
		down_read(&md->io_lock);
L
Linus Torvalds 已提交
921 922
	}

M
Milan Broz 已提交
923
	r = __split_bio(md, bio);
924
	up_read(&md->io_lock);
M
Milan Broz 已提交
925 926 927 928 929

out_req:
	if (r < 0)
		bio_io_error(bio);

L
Linus Torvalds 已提交
930 931 932
	return 0;
}

933
static void dm_unplug_all(struct request_queue *q)
L
Linus Torvalds 已提交
934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963
{
	struct mapped_device *md = q->queuedata;
	struct dm_table *map = dm_get_table(md);

	if (map) {
		dm_table_unplug_all(map);
		dm_table_put(map);
	}
}

static int dm_any_congested(void *congested_data, int bdi_bits)
{
	int r;
	struct mapped_device *md = (struct mapped_device *) congested_data;
	struct dm_table *map = dm_get_table(md);

	if (!map || test_bit(DMF_BLOCK_IO, &md->flags))
		r = bdi_bits;
	else
		r = dm_table_any_congested(map, bdi_bits);

	dm_table_put(map);
	return r;
}

/*-----------------------------------------------------------------
 * An IDR is used to keep track of allocated minor numbers.
 *---------------------------------------------------------------*/
static DEFINE_IDR(_minor_idr);

964
static void free_minor(int minor)
L
Linus Torvalds 已提交
965
{
966
	spin_lock(&_minor_lock);
L
Linus Torvalds 已提交
967
	idr_remove(&_minor_idr, minor);
968
	spin_unlock(&_minor_lock);
L
Linus Torvalds 已提交
969 970 971 972 973
}

/*
 * See if the device with a specific minor # is free.
 */
974
static int specific_minor(int minor)
L
Linus Torvalds 已提交
975 976 977 978 979 980
{
	int r, m;

	if (minor >= (1 << MINORBITS))
		return -EINVAL;

J
Jeff Mahoney 已提交
981 982 983 984
	r = idr_pre_get(&_minor_idr, GFP_KERNEL);
	if (!r)
		return -ENOMEM;

985
	spin_lock(&_minor_lock);
L
Linus Torvalds 已提交
986 987 988 989 990 991

	if (idr_find(&_minor_idr, minor)) {
		r = -EBUSY;
		goto out;
	}

992
	r = idr_get_new_above(&_minor_idr, MINOR_ALLOCED, minor, &m);
J
Jeff Mahoney 已提交
993
	if (r)
L
Linus Torvalds 已提交
994 995 996 997 998 999 1000 1001 1002
		goto out;

	if (m != minor) {
		idr_remove(&_minor_idr, m);
		r = -EBUSY;
		goto out;
	}

out:
1003
	spin_unlock(&_minor_lock);
L
Linus Torvalds 已提交
1004 1005 1006
	return r;
}

1007
static int next_free_minor(int *minor)
L
Linus Torvalds 已提交
1008
{
1009
	int r, m;
L
Linus Torvalds 已提交
1010 1011

	r = idr_pre_get(&_minor_idr, GFP_KERNEL);
J
Jeff Mahoney 已提交
1012 1013 1014
	if (!r)
		return -ENOMEM;

1015
	spin_lock(&_minor_lock);
L
Linus Torvalds 已提交
1016

1017
	r = idr_get_new(&_minor_idr, MINOR_ALLOCED, &m);
1018
	if (r)
L
Linus Torvalds 已提交
1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029
		goto out;

	if (m >= (1 << MINORBITS)) {
		idr_remove(&_minor_idr, m);
		r = -ENOSPC;
		goto out;
	}

	*minor = m;

out:
1030
	spin_unlock(&_minor_lock);
L
Linus Torvalds 已提交
1031 1032 1033 1034 1035 1036 1037 1038
	return r;
}

static struct block_device_operations dm_blk_dops;

/*
 * Allocate and initialise a blank device with a given minor.
 */
1039
static struct mapped_device *alloc_dev(int minor)
L
Linus Torvalds 已提交
1040 1041
{
	int r;
1042
	struct mapped_device *md = kzalloc(sizeof(*md), GFP_KERNEL);
1043
	void *old_md;
L
Linus Torvalds 已提交
1044 1045 1046 1047 1048 1049

	if (!md) {
		DMWARN("unable to allocate device, out of memory.");
		return NULL;
	}

1050
	if (!try_module_get(THIS_MODULE))
M
Milan Broz 已提交
1051
		goto bad_module_get;
1052

L
Linus Torvalds 已提交
1053
	/* get a minor number for the dev */
1054
	if (minor == DM_ANY_MINOR)
1055
		r = next_free_minor(&minor);
1056
	else
1057
		r = specific_minor(minor);
L
Linus Torvalds 已提交
1058
	if (r < 0)
M
Milan Broz 已提交
1059
		goto bad_minor;
L
Linus Torvalds 已提交
1060

1061
	init_rwsem(&md->io_lock);
1062
	mutex_init(&md->suspend_lock);
1063
	spin_lock_init(&md->pushback_lock);
L
Linus Torvalds 已提交
1064 1065
	rwlock_init(&md->map_lock);
	atomic_set(&md->holders, 1);
1066
	atomic_set(&md->open_count, 0);
L
Linus Torvalds 已提交
1067
	atomic_set(&md->event_nr, 0);
M
Mike Anderson 已提交
1068 1069 1070
	atomic_set(&md->uevent_seq, 0);
	INIT_LIST_HEAD(&md->uevent_list);
	spin_lock_init(&md->uevent_lock);
L
Linus Torvalds 已提交
1071 1072 1073

	md->queue = blk_alloc_queue(GFP_KERNEL);
	if (!md->queue)
M
Milan Broz 已提交
1074
		goto bad_queue;
L
Linus Torvalds 已提交
1075 1076 1077 1078 1079

	md->queue->queuedata = md;
	md->queue->backing_dev_info.congested_fn = dm_any_congested;
	md->queue->backing_dev_info.congested_data = md;
	blk_queue_make_request(md->queue, dm_request);
1080
	blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);
L
Linus Torvalds 已提交
1081
	md->queue->unplug_fn = dm_unplug_all;
M
Milan Broz 已提交
1082
	blk_queue_merge_bvec(md->queue, dm_merge_bvec);
L
Linus Torvalds 已提交
1083

1084
	md->io_pool = mempool_create_slab_pool(MIN_IOS, _io_cache);
K
Kiyoshi Ueda 已提交
1085
	if (!md->io_pool)
M
Milan Broz 已提交
1086
		goto bad_io_pool;
L
Linus Torvalds 已提交
1087

1088
	md->tio_pool = mempool_create_slab_pool(MIN_IOS, _tio_cache);
L
Linus Torvalds 已提交
1089
	if (!md->tio_pool)
M
Milan Broz 已提交
1090
		goto bad_tio_pool;
L
Linus Torvalds 已提交
1091

1092
	md->bs = bioset_create(16, 16);
S
Stefan Bader 已提交
1093 1094 1095
	if (!md->bs)
		goto bad_no_bioset;

L
Linus Torvalds 已提交
1096 1097
	md->disk = alloc_disk(1);
	if (!md->disk)
M
Milan Broz 已提交
1098
		goto bad_disk;
L
Linus Torvalds 已提交
1099

1100 1101 1102 1103
	atomic_set(&md->pending, 0);
	init_waitqueue_head(&md->wait);
	init_waitqueue_head(&md->eventq);

L
Linus Torvalds 已提交
1104 1105 1106 1107 1108 1109 1110
	md->disk->major = _major;
	md->disk->first_minor = minor;
	md->disk->fops = &dm_blk_dops;
	md->disk->queue = md->queue;
	md->disk->private_data = md;
	sprintf(md->disk->disk_name, "dm-%d", minor);
	add_disk(md->disk);
M
Mike Anderson 已提交
1111
	format_dev_t(md->name, MKDEV(_major, minor));
L
Linus Torvalds 已提交
1112

1113 1114 1115 1116
	md->wq = create_singlethread_workqueue("kdmflush");
	if (!md->wq)
		goto bad_thread;

1117
	/* Populate the mapping, nobody knows we exist yet */
1118
	spin_lock(&_minor_lock);
1119
	old_md = idr_replace(&_minor_idr, md, minor);
1120
	spin_unlock(&_minor_lock);
1121 1122 1123

	BUG_ON(old_md != MINOR_ALLOCED);

L
Linus Torvalds 已提交
1124 1125
	return md;

1126 1127
bad_thread:
	put_disk(md->disk);
M
Milan Broz 已提交
1128
bad_disk:
S
Stefan Bader 已提交
1129
	bioset_free(md->bs);
M
Milan Broz 已提交
1130
bad_no_bioset:
L
Linus Torvalds 已提交
1131
	mempool_destroy(md->tio_pool);
M
Milan Broz 已提交
1132
bad_tio_pool:
L
Linus Torvalds 已提交
1133
	mempool_destroy(md->io_pool);
M
Milan Broz 已提交
1134
bad_io_pool:
1135
	blk_cleanup_queue(md->queue);
M
Milan Broz 已提交
1136
bad_queue:
L
Linus Torvalds 已提交
1137
	free_minor(minor);
M
Milan Broz 已提交
1138
bad_minor:
1139
	module_put(THIS_MODULE);
M
Milan Broz 已提交
1140
bad_module_get:
L
Linus Torvalds 已提交
1141 1142 1143 1144
	kfree(md);
	return NULL;
}

J
Jun'ichi Nomura 已提交
1145 1146
static void unlock_fs(struct mapped_device *md);

L
Linus Torvalds 已提交
1147 1148
static void free_dev(struct mapped_device *md)
{
1149
	int minor = md->disk->first_minor;
1150

1151
	if (md->suspended_bdev) {
J
Jun'ichi Nomura 已提交
1152
		unlock_fs(md);
1153 1154
		bdput(md->suspended_bdev);
	}
1155
	destroy_workqueue(md->wq);
L
Linus Torvalds 已提交
1156 1157
	mempool_destroy(md->tio_pool);
	mempool_destroy(md->io_pool);
S
Stefan Bader 已提交
1158
	bioset_free(md->bs);
L
Linus Torvalds 已提交
1159
	del_gendisk(md->disk);
1160
	free_minor(minor);
J
Jeff Mahoney 已提交
1161 1162 1163 1164 1165

	spin_lock(&_minor_lock);
	md->disk->private_data = NULL;
	spin_unlock(&_minor_lock);

L
Linus Torvalds 已提交
1166
	put_disk(md->disk);
1167
	blk_cleanup_queue(md->queue);
1168
	module_put(THIS_MODULE);
L
Linus Torvalds 已提交
1169 1170 1171 1172 1173 1174 1175 1176
	kfree(md);
}

/*
 * Bind a table to the device.
 */
static void event_callback(void *context)
{
M
Mike Anderson 已提交
1177 1178
	unsigned long flags;
	LIST_HEAD(uevents);
L
Linus Torvalds 已提交
1179 1180
	struct mapped_device *md = (struct mapped_device *) context;

M
Mike Anderson 已提交
1181 1182 1183 1184
	spin_lock_irqsave(&md->uevent_lock, flags);
	list_splice_init(&md->uevent_list, &uevents);
	spin_unlock_irqrestore(&md->uevent_lock, flags);

1185
	dm_send_uevents(&uevents, &md->disk->dev.kobj);
M
Mike Anderson 已提交
1186

L
Linus Torvalds 已提交
1187 1188 1189 1190
	atomic_inc(&md->event_nr);
	wake_up(&md->eventq);
}

1191
static void __set_size(struct mapped_device *md, sector_t size)
L
Linus Torvalds 已提交
1192
{
1193
	set_capacity(md->disk, size);
L
Linus Torvalds 已提交
1194

1195
	mutex_lock(&md->suspended_bdev->bd_inode->i_mutex);
1196
	i_size_write(md->suspended_bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
1197
	mutex_unlock(&md->suspended_bdev->bd_inode->i_mutex);
L
Linus Torvalds 已提交
1198 1199 1200 1201
}

static int __bind(struct mapped_device *md, struct dm_table *t)
{
1202
	struct request_queue *q = md->queue;
L
Linus Torvalds 已提交
1203 1204 1205
	sector_t size;

	size = dm_table_get_size(t);
D
Darrick J. Wong 已提交
1206 1207 1208 1209 1210 1211 1212

	/*
	 * Wipe any geometry if the size of the table changed.
	 */
	if (size != get_capacity(md->disk))
		memset(&md->geometry, 0, sizeof(md->geometry));

1213 1214
	if (md->suspended_bdev)
		__set_size(md, size);
L
Linus Torvalds 已提交
1215 1216 1217
	if (size == 0)
		return 0;

1218 1219 1220
	dm_table_get(t);
	dm_table_event_callback(t, event_callback, md);

L
Linus Torvalds 已提交
1221 1222
	write_lock(&md->map_lock);
	md->map = t;
1223
	dm_table_set_restrictions(t, q);
L
Linus Torvalds 已提交
1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245
	write_unlock(&md->map_lock);

	return 0;
}

static void __unbind(struct mapped_device *md)
{
	struct dm_table *map = md->map;

	if (!map)
		return;

	dm_table_event_callback(map, NULL, NULL);
	write_lock(&md->map_lock);
	md->map = NULL;
	write_unlock(&md->map_lock);
	dm_table_put(map);
}

/*
 * Constructor for a new device.
 */
1246
int dm_create(int minor, struct mapped_device **result)
L
Linus Torvalds 已提交
1247 1248 1249
{
	struct mapped_device *md;

1250
	md = alloc_dev(minor);
L
Linus Torvalds 已提交
1251 1252 1253 1254 1255 1256 1257
	if (!md)
		return -ENXIO;

	*result = md;
	return 0;
}

1258
static struct mapped_device *dm_find_md(dev_t dev)
L
Linus Torvalds 已提交
1259 1260 1261 1262 1263 1264 1265
{
	struct mapped_device *md;
	unsigned minor = MINOR(dev);

	if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
		return NULL;

1266
	spin_lock(&_minor_lock);
L
Linus Torvalds 已提交
1267 1268

	md = idr_find(&_minor_idr, minor);
J
Jeff Mahoney 已提交
1269 1270
	if (md && (md == MINOR_ALLOCED ||
		   (dm_disk(md)->first_minor != minor) ||
A
Alasdair G Kergon 已提交
1271
		   test_bit(DMF_FREEING, &md->flags))) {
1272
		md = NULL;
J
Jeff Mahoney 已提交
1273 1274
		goto out;
	}
L
Linus Torvalds 已提交
1275

J
Jeff Mahoney 已提交
1276
out:
1277
	spin_unlock(&_minor_lock);
L
Linus Torvalds 已提交
1278

1279 1280 1281
	return md;
}

1282 1283 1284 1285 1286 1287 1288 1289 1290 1291
struct mapped_device *dm_get_md(dev_t dev)
{
	struct mapped_device *md = dm_find_md(dev);

	if (md)
		dm_get(md);

	return md;
}

A
Alasdair G Kergon 已提交
1292
void *dm_get_mdptr(struct mapped_device *md)
1293
{
A
Alasdair G Kergon 已提交
1294
	return md->interface_ptr;
L
Linus Torvalds 已提交
1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306
}

void dm_set_mdptr(struct mapped_device *md, void *ptr)
{
	md->interface_ptr = ptr;
}

void dm_get(struct mapped_device *md)
{
	atomic_inc(&md->holders);
}

1307 1308 1309 1310 1311 1312
const char *dm_device_name(struct mapped_device *md)
{
	return md->name;
}
EXPORT_SYMBOL_GPL(dm_device_name);

L
Linus Torvalds 已提交
1313 1314
void dm_put(struct mapped_device *md)
{
M
Mike Anderson 已提交
1315
	struct dm_table *map;
L
Linus Torvalds 已提交
1316

J
Jeff Mahoney 已提交
1317 1318
	BUG_ON(test_bit(DMF_FREEING, &md->flags));

1319
	if (atomic_dec_and_lock(&md->holders, &_minor_lock)) {
M
Mike Anderson 已提交
1320
		map = dm_get_table(md);
1321
		idr_replace(&_minor_idr, MINOR_ALLOCED, dm_disk(md)->first_minor);
J
Jeff Mahoney 已提交
1322
		set_bit(DMF_FREEING, &md->flags);
1323
		spin_unlock(&_minor_lock);
1324
		if (!dm_suspended(md)) {
L
Linus Torvalds 已提交
1325 1326 1327 1328
			dm_table_presuspend_targets(map);
			dm_table_postsuspend_targets(map);
		}
		__unbind(md);
M
Mike Anderson 已提交
1329
		dm_table_put(map);
L
Linus Torvalds 已提交
1330 1331 1332
		free_dev(md);
	}
}
E
Edward Goggin 已提交
1333
EXPORT_SYMBOL_GPL(dm_put);
L
Linus Torvalds 已提交
1334

1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357
static int dm_wait_for_completion(struct mapped_device *md)
{
	int r = 0;

	while (1) {
		set_current_state(TASK_INTERRUPTIBLE);

		smp_mb();
		if (!atomic_read(&md->pending))
			break;

		if (signal_pending(current)) {
			r = -EINTR;
			break;
		}

		io_schedule();
	}
	set_current_state(TASK_RUNNING);

	return r;
}

L
Linus Torvalds 已提交
1358 1359 1360
/*
 * Process the deferred bios
 */
1361
static void __flush_deferred_io(struct mapped_device *md)
L
Linus Torvalds 已提交
1362
{
1363
	struct bio *c;
L
Linus Torvalds 已提交
1364

1365
	while ((c = bio_list_pop(&md->deferred))) {
M
Milan Broz 已提交
1366 1367
		if (__split_bio(md, c))
			bio_io_error(c);
L
Linus Torvalds 已提交
1368
	}
M
Milan Broz 已提交
1369 1370

	clear_bit(DMF_BLOCK_IO, &md->flags);
L
Linus Torvalds 已提交
1371 1372
}

1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383
static void __merge_pushback_list(struct mapped_device *md)
{
	unsigned long flags;

	spin_lock_irqsave(&md->pushback_lock, flags);
	clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
	bio_list_merge_head(&md->deferred, &md->pushback);
	bio_list_init(&md->pushback);
	spin_unlock_irqrestore(&md->pushback_lock, flags);
}

1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421
static void dm_wq_work(struct work_struct *work)
{
	struct dm_wq_req *req = container_of(work, struct dm_wq_req, work);
	struct mapped_device *md = req->md;

	down_write(&md->io_lock);
	switch (req->type) {
	case DM_WQ_FLUSH_ALL:
		__merge_pushback_list(md);
		/* pass through */
	case DM_WQ_FLUSH_DEFERRED:
		__flush_deferred_io(md);
		break;
	default:
		DMERR("dm_wq_work: unrecognised work type %d", req->type);
		BUG();
	}
	up_write(&md->io_lock);
}

static void dm_wq_queue(struct mapped_device *md, int type, void *context,
			struct dm_wq_req *req)
{
	req->type = type;
	req->md = md;
	req->context = context;
	INIT_WORK(&req->work, dm_wq_work);
	queue_work(md->wq, &req->work);
}

static void dm_queue_flush(struct mapped_device *md, int type, void *context)
{
	struct dm_wq_req req;

	dm_wq_queue(md, type, context, &req);
	flush_workqueue(md->wq);
}

L
Linus Torvalds 已提交
1422 1423 1424 1425 1426
/*
 * Swap in a new table (destroying old one).
 */
int dm_swap_table(struct mapped_device *md, struct dm_table *table)
{
1427
	int r = -EINVAL;
L
Linus Torvalds 已提交
1428

1429
	mutex_lock(&md->suspend_lock);
L
Linus Torvalds 已提交
1430 1431

	/* device must be suspended */
1432
	if (!dm_suspended(md))
1433
		goto out;
L
Linus Torvalds 已提交
1434

1435 1436 1437 1438 1439
	/* without bdev, the device size cannot be changed */
	if (!md->suspended_bdev)
		if (get_capacity(md->disk) != dm_table_get_size(table))
			goto out;

L
Linus Torvalds 已提交
1440 1441 1442
	__unbind(md);
	r = __bind(md, table);

1443
out:
1444
	mutex_unlock(&md->suspend_lock);
1445
	return r;
L
Linus Torvalds 已提交
1446 1447 1448 1449 1450 1451
}

/*
 * Functions to lock and unlock any filesystem running on the
 * device.
 */
1452
static int lock_fs(struct mapped_device *md)
L
Linus Torvalds 已提交
1453
{
1454
	int r;
L
Linus Torvalds 已提交
1455 1456

	WARN_ON(md->frozen_sb);
1457

1458
	md->frozen_sb = freeze_bdev(md->suspended_bdev);
1459
	if (IS_ERR(md->frozen_sb)) {
1460
		r = PTR_ERR(md->frozen_sb);
1461 1462
		md->frozen_sb = NULL;
		return r;
1463 1464
	}

1465 1466
	set_bit(DMF_FROZEN, &md->flags);

L
Linus Torvalds 已提交
1467
	/* don't bdput right now, we don't want the bdev
1468
	 * to go away while it is locked.
L
Linus Torvalds 已提交
1469 1470 1471 1472
	 */
	return 0;
}

1473
static void unlock_fs(struct mapped_device *md)
L
Linus Torvalds 已提交
1474
{
1475 1476 1477
	if (!test_bit(DMF_FROZEN, &md->flags))
		return;

1478
	thaw_bdev(md->suspended_bdev, md->frozen_sb);
L
Linus Torvalds 已提交
1479
	md->frozen_sb = NULL;
1480
	clear_bit(DMF_FROZEN, &md->flags);
L
Linus Torvalds 已提交
1481 1482 1483 1484 1485 1486 1487 1488 1489
}

/*
 * We need to be able to change a mapping table under a mounted
 * filesystem.  For example we might want to move some data in
 * the background.  Before the table can be swapped with
 * dm_bind_table, dm_suspend must be called to flush any in
 * flight bios and ensure that any further io gets deferred.
 */
1490
int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
L
Linus Torvalds 已提交
1491
{
1492
	struct dm_table *map = NULL;
L
Linus Torvalds 已提交
1493
	DECLARE_WAITQUEUE(wait, current);
1494
	int r = 0;
1495
	int do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG ? 1 : 0;
1496
	int noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG ? 1 : 0;
L
Linus Torvalds 已提交
1497

1498
	mutex_lock(&md->suspend_lock);
1499

M
Milan Broz 已提交
1500 1501
	if (dm_suspended(md)) {
		r = -EINVAL;
1502
		goto out_unlock;
M
Milan Broz 已提交
1503
	}
L
Linus Torvalds 已提交
1504 1505 1506

	map = dm_get_table(md);

1507 1508 1509 1510 1511 1512 1513
	/*
	 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
	 * This flag is cleared before dm_suspend returns.
	 */
	if (noflush)
		set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);

1514 1515 1516
	/* This does not get reverted if there's an error later. */
	dm_table_presuspend_targets(map);

1517 1518 1519 1520 1521 1522 1523 1524
	/* bdget() can stall if the pending I/Os are not flushed */
	if (!noflush) {
		md->suspended_bdev = bdget_disk(md->disk, 0);
		if (!md->suspended_bdev) {
			DMWARN("bdget failed in dm_suspend");
			r = -ENOMEM;
			goto flush_and_out;
		}
1525

1526 1527 1528 1529 1530 1531 1532 1533 1534
		/*
		 * Flush I/O to the device. noflush supersedes do_lockfs,
		 * because lock_fs() needs to flush I/Os.
		 */
		if (do_lockfs) {
			r = lock_fs(md);
			if (r)
				goto out;
		}
1535
	}
L
Linus Torvalds 已提交
1536 1537

	/*
1538
	 * First we set the BLOCK_IO flag so no more ios will be mapped.
L
Linus Torvalds 已提交
1539
	 */
1540 1541
	down_write(&md->io_lock);
	set_bit(DMF_BLOCK_IO, &md->flags);
L
Linus Torvalds 已提交
1542 1543

	add_wait_queue(&md->wait, &wait);
1544
	up_write(&md->io_lock);
L
Linus Torvalds 已提交
1545 1546

	/* unplug */
1547
	if (map)
L
Linus Torvalds 已提交
1548 1549 1550
		dm_table_unplug_all(map);

	/*
1551
	 * Wait for the already-mapped ios to complete.
L
Linus Torvalds 已提交
1552
	 */
1553
	r = dm_wait_for_completion(md);
L
Linus Torvalds 已提交
1554

1555
	down_write(&md->io_lock);
L
Linus Torvalds 已提交
1556 1557
	remove_wait_queue(&md->wait, &wait);

1558 1559
	if (noflush)
		__merge_pushback_list(md);
1560
	up_write(&md->io_lock);
1561

L
Linus Torvalds 已提交
1562
	/* were we interrupted ? */
1563
	if (r < 0) {
1564
		dm_queue_flush(md, DM_WQ_FLUSH_DEFERRED, NULL);
M
Milan Broz 已提交
1565

1566
		unlock_fs(md);
1567
		goto out; /* pushback list is already flushed, so skip flush */
1568
	}
L
Linus Torvalds 已提交
1569

1570
	dm_table_postsuspend_targets(map);
L
Linus Torvalds 已提交
1571

1572
	set_bit(DMF_SUSPENDED, &md->flags);
1573

1574
flush_and_out:
1575
	if (r && noflush)
1576 1577 1578 1579
		/*
		 * Because there may be already I/Os in the pushback list,
		 * flush them before return.
		 */
1580
		dm_queue_flush(md, DM_WQ_FLUSH_ALL, NULL);
1581

1582
out:
1583 1584 1585 1586 1587
	if (r && md->suspended_bdev) {
		bdput(md->suspended_bdev);
		md->suspended_bdev = NULL;
	}

1588
	dm_table_put(map);
1589 1590

out_unlock:
1591
	mutex_unlock(&md->suspend_lock);
1592
	return r;
L
Linus Torvalds 已提交
1593 1594 1595 1596
}

int dm_resume(struct mapped_device *md)
{
1597 1598
	int r = -EINVAL;
	struct dm_table *map = NULL;
L
Linus Torvalds 已提交
1599

1600
	mutex_lock(&md->suspend_lock);
1601
	if (!dm_suspended(md))
1602 1603 1604
		goto out;

	map = dm_get_table(md);
1605
	if (!map || !dm_table_get_size(map))
1606
		goto out;
L
Linus Torvalds 已提交
1607

1608 1609 1610
	r = dm_table_resume_targets(map);
	if (r)
		goto out;
1611

1612
	dm_queue_flush(md, DM_WQ_FLUSH_DEFERRED, NULL);
1613 1614 1615

	unlock_fs(md);

1616 1617 1618 1619
	if (md->suspended_bdev) {
		bdput(md->suspended_bdev);
		md->suspended_bdev = NULL;
	}
1620

1621 1622
	clear_bit(DMF_SUSPENDED, &md->flags);

L
Linus Torvalds 已提交
1623 1624
	dm_table_unplug_all(map);

1625
	dm_kobject_uevent(md);
1626

1627
	r = 0;
1628

1629 1630
out:
	dm_table_put(map);
1631
	mutex_unlock(&md->suspend_lock);
1632

1633
	return r;
L
Linus Torvalds 已提交
1634 1635 1636 1637 1638
}

/*-----------------------------------------------------------------
 * Event notification.
 *---------------------------------------------------------------*/
1639 1640
void dm_kobject_uevent(struct mapped_device *md)
{
1641
	kobject_uevent(&md->disk->dev.kobj, KOBJ_CHANGE);
1642 1643
}

M
Mike Anderson 已提交
1644 1645 1646 1647 1648
uint32_t dm_next_uevent_seq(struct mapped_device *md)
{
	return atomic_add_return(1, &md->uevent_seq);
}

L
Linus Torvalds 已提交
1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659
uint32_t dm_get_event_nr(struct mapped_device *md)
{
	return atomic_read(&md->event_nr);
}

int dm_wait_event(struct mapped_device *md, int event_nr)
{
	return wait_event_interruptible(md->eventq,
			(event_nr != atomic_read(&md->event_nr)));
}

M
Mike Anderson 已提交
1660 1661 1662 1663 1664 1665 1666 1667 1668
void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
{
	unsigned long flags;

	spin_lock_irqsave(&md->uevent_lock, flags);
	list_add(elist, &md->uevent_list);
	spin_unlock_irqrestore(&md->uevent_lock, flags);
}

L
Linus Torvalds 已提交
1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682
/*
 * The gendisk is only valid as long as you have a reference
 * count on 'md'.
 */
struct gendisk *dm_disk(struct mapped_device *md)
{
	return md->disk;
}

int dm_suspended(struct mapped_device *md)
{
	return test_bit(DMF_SUSPENDED, &md->flags);
}

1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693
int dm_noflush_suspending(struct dm_target *ti)
{
	struct mapped_device *md = dm_table_get_md(ti->table);
	int r = __noflush_suspending(md);

	dm_put(md);

	return r;
}
EXPORT_SYMBOL_GPL(dm_noflush_suspending);

L
Linus Torvalds 已提交
1694 1695 1696
static struct block_device_operations dm_blk_dops = {
	.open = dm_blk_open,
	.release = dm_blk_close,
1697
	.ioctl = dm_blk_ioctl,
D
Darrick J. Wong 已提交
1698
	.getgeo = dm_blk_getgeo,
L
Linus Torvalds 已提交
1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714
	.owner = THIS_MODULE
};

EXPORT_SYMBOL(dm_get_mapinfo);

/*
 * module hooks
 */
module_init(dm_init);
module_exit(dm_exit);

module_param(major, uint, 0);
MODULE_PARM_DESC(major, "The major number of the device mapper");
MODULE_DESCRIPTION(DM_NAME " driver");
MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
MODULE_LICENSE("GPL");