dm.c 31.4 KB
Newer Older
L
Linus Torvalds 已提交
1 2
/*
 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
3
 * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved.
L
Linus Torvalds 已提交
4 5 6 7 8 9 10 11 12
 *
 * This file is released under the GPL.
 */

#include "dm.h"
#include "dm-bio-list.h"

#include <linux/init.h>
#include <linux/module.h>
A
Arjan van de Ven 已提交
13
#include <linux/mutex.h>
L
Linus Torvalds 已提交
14 15 16 17 18 19 20
#include <linux/moduleparam.h>
#include <linux/blkpg.h>
#include <linux/bio.h>
#include <linux/buffer_head.h>
#include <linux/mempool.h>
#include <linux/slab.h>
#include <linux/idr.h>
D
Darrick J. Wong 已提交
21
#include <linux/hdreg.h>
22
#include <linux/blktrace_api.h>
23
#include <linux/smp_lock.h>
L
Linus Torvalds 已提交
24

25 26
#define DM_MSG_PREFIX "core"

L
Linus Torvalds 已提交
27 28 29 30 31
static const char *_name = DM_NAME;

static unsigned int major = 0;
static unsigned int _major = 0;

32
static DEFINE_SPINLOCK(_minor_lock);
L
Linus Torvalds 已提交
33 34 35 36 37 38 39 40
/*
 * One of these is allocated per bio.
 */
struct dm_io {
	struct mapped_device *md;
	int error;
	struct bio *bio;
	atomic_t io_count;
41
	unsigned long start_time;
L
Linus Torvalds 已提交
42 43 44 45 46 47 48 49 50 51 52 53 54 55
};

/*
 * One of these is allocated per target within a bio.  Hopefully
 * this will be simplified out one day.
 */
struct target_io {
	struct dm_io *io;
	struct dm_target *ti;
	union map_info info;
};

union map_info *dm_get_mapinfo(struct bio *bio)
{
A
Alasdair G Kergon 已提交
56 57 58
	if (bio && bio->bi_private)
		return &((struct target_io *)bio->bi_private)->info;
	return NULL;
L
Linus Torvalds 已提交
59 60
}

61 62
#define MINOR_ALLOCED ((void *)-1)

L
Linus Torvalds 已提交
63 64 65 66 67
/*
 * Bits for the md->flags field.
 */
#define DMF_BLOCK_IO 0
#define DMF_SUSPENDED 1
68
#define DMF_FROZEN 2
J
Jeff Mahoney 已提交
69
#define DMF_FREEING 3
70
#define DMF_DELETING 4
71
#define DMF_NOFLUSH_SUSPENDING 5
L
Linus Torvalds 已提交
72 73

struct mapped_device {
74 75
	struct rw_semaphore io_lock;
	struct semaphore suspend_lock;
76
	spinlock_t pushback_lock;
L
Linus Torvalds 已提交
77 78
	rwlock_t map_lock;
	atomic_t holders;
79
	atomic_t open_count;
L
Linus Torvalds 已提交
80 81 82 83 84

	unsigned long flags;

	request_queue_t *queue;
	struct gendisk *disk;
M
Mike Anderson 已提交
85
	char name[16];
L
Linus Torvalds 已提交
86 87 88 89 90 91 92 93

	void *interface_ptr;

	/*
	 * A list of ios that arrived while we were suspended.
	 */
	atomic_t pending;
	wait_queue_head_t wait;
K
Kiyoshi Ueda 已提交
94
	struct bio_list deferred;
95
	struct bio_list pushback;
L
Linus Torvalds 已提交
96 97 98 99 100 101 102 103 104 105 106 107

	/*
	 * The current mapping.
	 */
	struct dm_table *map;

	/*
	 * io objects are allocated from here.
	 */
	mempool_t *io_pool;
	mempool_t *tio_pool;

S
Stefan Bader 已提交
108 109
	struct bio_set *bs;

L
Linus Torvalds 已提交
110 111 112 113 114 115 116 117 118 119
	/*
	 * Event handling.
	 */
	atomic_t event_nr;
	wait_queue_head_t eventq;

	/*
	 * freeze/thaw support require holding onto a super block
	 */
	struct super_block *frozen_sb;
120
	struct block_device *suspended_bdev;
D
Darrick J. Wong 已提交
121 122 123

	/* forced geometry settings */
	struct hd_geometry geometry;
L
Linus Torvalds 已提交
124 125 126
};

#define MIN_IOS 256
127 128
static struct kmem_cache *_io_cache;
static struct kmem_cache *_tio_cache;
L
Linus Torvalds 已提交
129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167

static int __init local_init(void)
{
	int r;

	/* allocate a slab for the dm_ios */
	_io_cache = kmem_cache_create("dm_io",
				      sizeof(struct dm_io), 0, 0, NULL, NULL);
	if (!_io_cache)
		return -ENOMEM;

	/* allocate a slab for the target ios */
	_tio_cache = kmem_cache_create("dm_tio", sizeof(struct target_io),
				       0, 0, NULL, NULL);
	if (!_tio_cache) {
		kmem_cache_destroy(_io_cache);
		return -ENOMEM;
	}

	_major = major;
	r = register_blkdev(_major, _name);
	if (r < 0) {
		kmem_cache_destroy(_tio_cache);
		kmem_cache_destroy(_io_cache);
		return r;
	}

	if (!_major)
		_major = r;

	return 0;
}

static void local_exit(void)
{
	kmem_cache_destroy(_tio_cache);
	kmem_cache_destroy(_io_cache);

	if (unregister_blkdev(_major, _name) < 0)
168
		DMERR("unregister_blkdev failed");
L
Linus Torvalds 已提交
169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226

	_major = 0;

	DMINFO("cleaned up");
}

int (*_inits[])(void) __initdata = {
	local_init,
	dm_target_init,
	dm_linear_init,
	dm_stripe_init,
	dm_interface_init,
};

void (*_exits[])(void) = {
	local_exit,
	dm_target_exit,
	dm_linear_exit,
	dm_stripe_exit,
	dm_interface_exit,
};

static int __init dm_init(void)
{
	const int count = ARRAY_SIZE(_inits);

	int r, i;

	for (i = 0; i < count; i++) {
		r = _inits[i]();
		if (r)
			goto bad;
	}

	return 0;

      bad:
	while (i--)
		_exits[i]();

	return r;
}

static void __exit dm_exit(void)
{
	int i = ARRAY_SIZE(_exits);

	while (i--)
		_exits[i]();
}

/*
 * Block device functions
 */
static int dm_blk_open(struct inode *inode, struct file *file)
{
	struct mapped_device *md;

J
Jeff Mahoney 已提交
227 228
	spin_lock(&_minor_lock);

L
Linus Torvalds 已提交
229
	md = inode->i_bdev->bd_disk->private_data;
J
Jeff Mahoney 已提交
230 231 232
	if (!md)
		goto out;

233 234
	if (test_bit(DMF_FREEING, &md->flags) ||
	    test_bit(DMF_DELETING, &md->flags)) {
J
Jeff Mahoney 已提交
235 236 237 238
		md = NULL;
		goto out;
	}

L
Linus Torvalds 已提交
239
	dm_get(md);
240
	atomic_inc(&md->open_count);
J
Jeff Mahoney 已提交
241 242 243 244 245

out:
	spin_unlock(&_minor_lock);

	return md ? 0 : -ENXIO;
L
Linus Torvalds 已提交
246 247 248 249 250 251 252
}

static int dm_blk_close(struct inode *inode, struct file *file)
{
	struct mapped_device *md;

	md = inode->i_bdev->bd_disk->private_data;
253
	atomic_dec(&md->open_count);
L
Linus Torvalds 已提交
254 255 256 257
	dm_put(md);
	return 0;
}

258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281
int dm_open_count(struct mapped_device *md)
{
	return atomic_read(&md->open_count);
}

/*
 * Guarantees nothing is using the device before it's deleted.
 */
int dm_lock_for_deletion(struct mapped_device *md)
{
	int r = 0;

	spin_lock(&_minor_lock);

	if (dm_open_count(md))
		r = -EBUSY;
	else
		set_bit(DMF_DELETING, &md->flags);

	spin_unlock(&_minor_lock);

	return r;
}

D
Darrick J. Wong 已提交
282 283 284 285 286 287 288
static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
{
	struct mapped_device *md = bdev->bd_disk->private_data;

	return dm_get_geometry(md, geo);
}

289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327
static int dm_blk_ioctl(struct inode *inode, struct file *file,
			unsigned int cmd, unsigned long arg)
{
	struct mapped_device *md;
	struct dm_table *map;
	struct dm_target *tgt;
	int r = -ENOTTY;

	/* We don't really need this lock, but we do need 'inode'. */
	unlock_kernel();

	md = inode->i_bdev->bd_disk->private_data;

	map = dm_get_table(md);

	if (!map || !dm_table_get_size(map))
		goto out;

	/* We only support devices that have a single target */
	if (dm_table_get_num_targets(map) != 1)
		goto out;

	tgt = dm_table_get_target(map, 0);

	if (dm_suspended(md)) {
		r = -EAGAIN;
		goto out;
	}

	if (tgt->type->ioctl)
		r = tgt->type->ioctl(tgt, inode, file, cmd, arg);

out:
	dm_table_put(map);

	lock_kernel();
	return r;
}

L
Linus Torvalds 已提交
328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347
static inline struct dm_io *alloc_io(struct mapped_device *md)
{
	return mempool_alloc(md->io_pool, GFP_NOIO);
}

static inline void free_io(struct mapped_device *md, struct dm_io *io)
{
	mempool_free(io, md->io_pool);
}

static inline struct target_io *alloc_tio(struct mapped_device *md)
{
	return mempool_alloc(md->tio_pool, GFP_NOIO);
}

static inline void free_tio(struct mapped_device *md, struct target_io *tio)
{
	mempool_free(tio, md->tio_pool);
}

348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377
static void start_io_acct(struct dm_io *io)
{
	struct mapped_device *md = io->md;

	io->start_time = jiffies;

	preempt_disable();
	disk_round_stats(dm_disk(md));
	preempt_enable();
	dm_disk(md)->in_flight = atomic_inc_return(&md->pending);
}

static int end_io_acct(struct dm_io *io)
{
	struct mapped_device *md = io->md;
	struct bio *bio = io->bio;
	unsigned long duration = jiffies - io->start_time;
	int pending;
	int rw = bio_data_dir(bio);

	preempt_disable();
	disk_round_stats(dm_disk(md));
	preempt_enable();
	dm_disk(md)->in_flight = pending = atomic_dec_return(&md->pending);

	disk_stat_add(dm_disk(md), ticks[rw], duration);

	return !pending;
}

L
Linus Torvalds 已提交
378 379 380 381 382
/*
 * Add the bio to the list of deferred io.
 */
static int queue_io(struct mapped_device *md, struct bio *bio)
{
383
	down_write(&md->io_lock);
L
Linus Torvalds 已提交
384 385

	if (!test_bit(DMF_BLOCK_IO, &md->flags)) {
386
		up_write(&md->io_lock);
L
Linus Torvalds 已提交
387 388 389 390 391
		return 1;
	}

	bio_list_add(&md->deferred, bio);

392
	up_write(&md->io_lock);
L
Linus Torvalds 已提交
393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413
	return 0;		/* deferred successfully */
}

/*
 * Everyone (including functions in this file), should use this
 * function to access the md->map field, and make sure they call
 * dm_table_put() when finished.
 */
struct dm_table *dm_get_table(struct mapped_device *md)
{
	struct dm_table *t;

	read_lock(&md->map_lock);
	t = md->map;
	if (t)
		dm_table_get(t);
	read_unlock(&md->map_lock);

	return t;
}

D
Darrick J. Wong 已提交
414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440
/*
 * Get the geometry associated with a dm device
 */
int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
{
	*geo = md->geometry;

	return 0;
}

/*
 * Set the geometry of a device.
 */
int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
{
	sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;

	if (geo->start > sz) {
		DMWARN("Start sector is beyond the geometry limits.");
		return -EINVAL;
	}

	md->geometry = *geo;

	return 0;
}

L
Linus Torvalds 已提交
441 442 443 444 445 446 447 448 449
/*-----------------------------------------------------------------
 * CRUD START:
 *   A more elegant soln is in the works that uses the queue
 *   merge fn, unfortunately there are a couple of changes to
 *   the block layer that I want to make for this.  So in the
 *   interests of getting something for people to use I give
 *   you this clearly demarcated crap.
 *---------------------------------------------------------------*/

450 451 452 453 454
static int __noflush_suspending(struct mapped_device *md)
{
	return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
}

L
Linus Torvalds 已提交
455 456 457 458
/*
 * Decrements the number of outstanding ios that a bio has been
 * cloned into, completing the original io if necc.
 */
459
static void dec_pending(struct dm_io *io, int error)
L
Linus Torvalds 已提交
460
{
461 462 463 464
	unsigned long flags;

	/* Push-back supersedes any I/O errors */
	if (error && !(io->error > 0 && __noflush_suspending(io->md)))
L
Linus Torvalds 已提交
465 466 467
		io->error = error;

	if (atomic_dec_and_test(&io->io_count)) {
468 469 470 471 472 473 474 475 476 477 478 479 480 481 482
		if (io->error == DM_ENDIO_REQUEUE) {
			/*
			 * Target requested pushing back the I/O.
			 * This must be handled before the sleeper on
			 * suspend queue merges the pushback list.
			 */
			spin_lock_irqsave(&io->md->pushback_lock, flags);
			if (__noflush_suspending(io->md))
				bio_list_add(&io->md->pushback, io->bio);
			else
				/* noflush suspend was interrupted. */
				io->error = -EIO;
			spin_unlock_irqrestore(&io->md->pushback_lock, flags);
		}

483
		if (end_io_acct(io))
L
Linus Torvalds 已提交
484 485 486
			/* nudge anyone waiting on suspend queue */
			wake_up(&io->md->wait);

487 488 489 490 491 492
		if (io->error != DM_ENDIO_REQUEUE) {
			blk_add_trace_bio(io->md->queue, io->bio,
					  BLK_TA_COMPLETE);

			bio_endio(io->bio, io->bio->bi_size, io->error);
		}
493

L
Linus Torvalds 已提交
494 495 496 497 498 499 500 501
		free_io(io->md, io);
	}
}

static int clone_endio(struct bio *bio, unsigned int done, int error)
{
	int r = 0;
	struct target_io *tio = bio->bi_private;
S
Stefan Bader 已提交
502
	struct mapped_device *md = tio->io->md;
L
Linus Torvalds 已提交
503 504 505 506 507 508 509 510 511 512
	dm_endio_fn endio = tio->ti->type->end_io;

	if (bio->bi_size)
		return 1;

	if (!bio_flagged(bio, BIO_UPTODATE) && !error)
		error = -EIO;

	if (endio) {
		r = endio(tio->ti, bio, error, &tio->info);
513 514 515 516 517
		if (r < 0 || r == DM_ENDIO_REQUEUE)
			/*
			 * error and requeue request are handled
			 * in dec_pending().
			 */
L
Linus Torvalds 已提交
518
			error = r;
519 520
		else if (r == DM_ENDIO_INCOMPLETE)
			/* The target will handle the io */
L
Linus Torvalds 已提交
521
			return 1;
522 523 524 525
		else if (r) {
			DMWARN("unimplemented target endio return value: %d", r);
			BUG();
		}
L
Linus Torvalds 已提交
526 527
	}

S
Stefan Bader 已提交
528 529 530 531 532 533 534
	dec_pending(tio->io, error);

	/*
	 * Store md for cleanup instead of tio which is about to get freed.
	 */
	bio->bi_private = md->bs;

L
Linus Torvalds 已提交
535
	bio_put(bio);
S
Stefan Bader 已提交
536
	free_tio(md, tio);
L
Linus Torvalds 已提交
537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563
	return r;
}

static sector_t max_io_len(struct mapped_device *md,
			   sector_t sector, struct dm_target *ti)
{
	sector_t offset = sector - ti->begin;
	sector_t len = ti->len - offset;

	/*
	 * Does the target need to split even further ?
	 */
	if (ti->split_io) {
		sector_t boundary;
		boundary = ((offset + ti->split_io) & ~(ti->split_io - 1))
			   - offset;
		if (len > boundary)
			len = boundary;
	}

	return len;
}

static void __map_bio(struct dm_target *ti, struct bio *clone,
		      struct target_io *tio)
{
	int r;
564
	sector_t sector;
S
Stefan Bader 已提交
565
	struct mapped_device *md;
L
Linus Torvalds 已提交
566 567 568 569 570 571 572 573 574 575 576 577 578 579 580

	/*
	 * Sanity checks.
	 */
	BUG_ON(!clone->bi_size);

	clone->bi_end_io = clone_endio;
	clone->bi_private = tio;

	/*
	 * Map the clone.  If r == 0 we don't need to do
	 * anything, the target has assumed ownership of
	 * this io.
	 */
	atomic_inc(&tio->io->io_count);
581
	sector = clone->bi_sector;
L
Linus Torvalds 已提交
582
	r = ti->type->map(ti, clone, &tio->info);
583
	if (r == DM_MAPIO_REMAPPED) {
L
Linus Torvalds 已提交
584
		/* the bio has been remapped so dispatch it */
585

A
Alasdair G Kergon 已提交
586 587
		blk_add_trace_remap(bdev_get_queue(clone->bi_bdev), clone,
				    tio->io->bio->bi_bdev->bd_dev, sector,
588 589
				    clone->bi_sector);

L
Linus Torvalds 已提交
590
		generic_make_request(clone);
591 592
	} else if (r < 0 || r == DM_MAPIO_REQUEUE) {
		/* error the io and bail out, or requeue it if needed */
S
Stefan Bader 已提交
593 594 595 596 597 598
		md = tio->io->md;
		dec_pending(tio->io, r);
		/*
		 * Store bio_set for cleanup.
		 */
		clone->bi_private = md->bs;
L
Linus Torvalds 已提交
599
		bio_put(clone);
S
Stefan Bader 已提交
600
		free_tio(md, tio);
601 602 603
	} else if (r) {
		DMWARN("unimplemented target map return value: %d", r);
		BUG();
L
Linus Torvalds 已提交
604 605 606 607 608 609 610 611 612 613 614 615 616
	}
}

struct clone_info {
	struct mapped_device *md;
	struct dm_table *map;
	struct bio *bio;
	struct dm_io *io;
	sector_t sector;
	sector_t sector_count;
	unsigned short idx;
};

P
Peter Osterlund 已提交
617 618
static void dm_bio_destructor(struct bio *bio)
{
S
Stefan Bader 已提交
619 620 621
	struct bio_set *bs = bio->bi_private;

	bio_free(bio, bs);
P
Peter Osterlund 已提交
622 623
}

L
Linus Torvalds 已提交
624 625 626 627 628
/*
 * Creates a little bio that is just does part of a bvec.
 */
static struct bio *split_bvec(struct bio *bio, sector_t sector,
			      unsigned short idx, unsigned int offset,
S
Stefan Bader 已提交
629
			      unsigned int len, struct bio_set *bs)
L
Linus Torvalds 已提交
630 631 632 633
{
	struct bio *clone;
	struct bio_vec *bv = bio->bi_io_vec + idx;

S
Stefan Bader 已提交
634
	clone = bio_alloc_bioset(GFP_NOIO, 1, bs);
P
Peter Osterlund 已提交
635
	clone->bi_destructor = dm_bio_destructor;
L
Linus Torvalds 已提交
636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653
	*clone->bi_io_vec = *bv;

	clone->bi_sector = sector;
	clone->bi_bdev = bio->bi_bdev;
	clone->bi_rw = bio->bi_rw;
	clone->bi_vcnt = 1;
	clone->bi_size = to_bytes(len);
	clone->bi_io_vec->bv_offset = offset;
	clone->bi_io_vec->bv_len = clone->bi_size;

	return clone;
}

/*
 * Creates a bio that consists of range of complete bvecs.
 */
static struct bio *clone_bio(struct bio *bio, sector_t sector,
			     unsigned short idx, unsigned short bv_count,
S
Stefan Bader 已提交
654
			     unsigned int len, struct bio_set *bs)
L
Linus Torvalds 已提交
655 656 657
{
	struct bio *clone;

S
Stefan Bader 已提交
658 659 660
	clone = bio_alloc_bioset(GFP_NOIO, bio->bi_max_vecs, bs);
	__bio_clone(clone, bio);
	clone->bi_destructor = dm_bio_destructor;
L
Linus Torvalds 已提交
661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690
	clone->bi_sector = sector;
	clone->bi_idx = idx;
	clone->bi_vcnt = idx + bv_count;
	clone->bi_size = to_bytes(len);
	clone->bi_flags &= ~(1 << BIO_SEG_VALID);

	return clone;
}

static void __clone_and_map(struct clone_info *ci)
{
	struct bio *clone, *bio = ci->bio;
	struct dm_target *ti = dm_table_find_target(ci->map, ci->sector);
	sector_t len = 0, max = max_io_len(ci->md, ci->sector, ti);
	struct target_io *tio;

	/*
	 * Allocate a target io object.
	 */
	tio = alloc_tio(ci->md);
	tio->io = ci->io;
	tio->ti = ti;
	memset(&tio->info, 0, sizeof(tio->info));

	if (ci->sector_count <= max) {
		/*
		 * Optimise for the simple case where we can do all of
		 * the remaining io with a single clone.
		 */
		clone = clone_bio(bio, ci->sector, ci->idx,
S
Stefan Bader 已提交
691 692
				  bio->bi_vcnt - ci->idx, ci->sector_count,
				  ci->md->bs);
L
Linus Torvalds 已提交
693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714
		__map_bio(ti, clone, tio);
		ci->sector_count = 0;

	} else if (to_sector(bio->bi_io_vec[ci->idx].bv_len) <= max) {
		/*
		 * There are some bvecs that don't span targets.
		 * Do as many of these as possible.
		 */
		int i;
		sector_t remaining = max;
		sector_t bv_len;

		for (i = ci->idx; remaining && (i < bio->bi_vcnt); i++) {
			bv_len = to_sector(bio->bi_io_vec[i].bv_len);

			if (bv_len > remaining)
				break;

			remaining -= bv_len;
			len += bv_len;
		}

S
Stefan Bader 已提交
715 716
		clone = clone_bio(bio, ci->sector, ci->idx, i - ci->idx, len,
				  ci->md->bs);
L
Linus Torvalds 已提交
717 718 719 720 721 722 723 724
		__map_bio(ti, clone, tio);

		ci->sector += len;
		ci->sector_count -= len;
		ci->idx = i;

	} else {
		/*
725
		 * Handle a bvec that must be split between two or more targets.
L
Linus Torvalds 已提交
726 727
		 */
		struct bio_vec *bv = bio->bi_io_vec + ci->idx;
728 729
		sector_t remaining = to_sector(bv->bv_len);
		unsigned int offset = 0;
L
Linus Torvalds 已提交
730

731 732 733 734
		do {
			if (offset) {
				ti = dm_table_find_target(ci->map, ci->sector);
				max = max_io_len(ci->md, ci->sector, ti);
L
Linus Torvalds 已提交
735

736 737 738 739 740 741 742 743 744
				tio = alloc_tio(ci->md);
				tio->io = ci->io;
				tio->ti = ti;
				memset(&tio->info, 0, sizeof(tio->info));
			}

			len = min(remaining, max);

			clone = split_bvec(bio, ci->sector, ci->idx,
S
Stefan Bader 已提交
745 746
					   bv->bv_offset + offset, len,
					   ci->md->bs);
747 748 749 750 751 752 753

			__map_bio(ti, clone, tio);

			ci->sector += len;
			ci->sector_count -= len;
			offset += to_bytes(len);
		} while (remaining -= len);
L
Linus Torvalds 已提交
754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782

		ci->idx++;
	}
}

/*
 * Split the bio into several clones.
 */
static void __split_bio(struct mapped_device *md, struct bio *bio)
{
	struct clone_info ci;

	ci.map = dm_get_table(md);
	if (!ci.map) {
		bio_io_error(bio, bio->bi_size);
		return;
	}

	ci.md = md;
	ci.bio = bio;
	ci.io = alloc_io(md);
	ci.io->error = 0;
	atomic_set(&ci.io->io_count, 1);
	ci.io->bio = bio;
	ci.io->md = md;
	ci.sector = bio->bi_sector;
	ci.sector_count = bio_sectors(bio);
	ci.idx = bio->bi_idx;

783
	start_io_acct(ci.io);
L
Linus Torvalds 已提交
784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801
	while (ci.sector_count)
		__clone_and_map(&ci);

	/* drop the extra reference count */
	dec_pending(ci.io, 0);
	dm_table_put(ci.map);
}
/*-----------------------------------------------------------------
 * CRUD END
 *---------------------------------------------------------------*/

/*
 * The request function that just remaps the bio built up by
 * dm_merge_bvec.
 */
static int dm_request(request_queue_t *q, struct bio *bio)
{
	int r;
802
	int rw = bio_data_dir(bio);
L
Linus Torvalds 已提交
803 804
	struct mapped_device *md = q->queuedata;

805
	down_read(&md->io_lock);
L
Linus Torvalds 已提交
806

807 808 809
	disk_stat_inc(dm_disk(md), ios[rw]);
	disk_stat_add(dm_disk(md), sectors[rw], bio_sectors(bio));

L
Linus Torvalds 已提交
810 811 812 813 814
	/*
	 * If we're suspended we have to queue
	 * this io for later.
	 */
	while (test_bit(DMF_BLOCK_IO, &md->flags)) {
815
		up_read(&md->io_lock);
L
Linus Torvalds 已提交
816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833

		if (bio_rw(bio) == READA) {
			bio_io_error(bio, bio->bi_size);
			return 0;
		}

		r = queue_io(md, bio);
		if (r < 0) {
			bio_io_error(bio, bio->bi_size);
			return 0;

		} else if (r == 0)
			return 0;	/* deferred successfully */

		/*
		 * We're in a while loop, because someone could suspend
		 * before we get to the following read lock.
		 */
834
		down_read(&md->io_lock);
L
Linus Torvalds 已提交
835 836 837
	}

	__split_bio(md, bio);
838
	up_read(&md->io_lock);
L
Linus Torvalds 已提交
839 840 841 842 843 844 845 846 847 848 849
	return 0;
}

static int dm_flush_all(request_queue_t *q, struct gendisk *disk,
			sector_t *error_sector)
{
	struct mapped_device *md = q->queuedata;
	struct dm_table *map = dm_get_table(md);
	int ret = -ENXIO;

	if (map) {
850
		ret = dm_table_flush_all(map);
L
Linus Torvalds 已提交
851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887
		dm_table_put(map);
	}

	return ret;
}

static void dm_unplug_all(request_queue_t *q)
{
	struct mapped_device *md = q->queuedata;
	struct dm_table *map = dm_get_table(md);

	if (map) {
		dm_table_unplug_all(map);
		dm_table_put(map);
	}
}

static int dm_any_congested(void *congested_data, int bdi_bits)
{
	int r;
	struct mapped_device *md = (struct mapped_device *) congested_data;
	struct dm_table *map = dm_get_table(md);

	if (!map || test_bit(DMF_BLOCK_IO, &md->flags))
		r = bdi_bits;
	else
		r = dm_table_any_congested(map, bdi_bits);

	dm_table_put(map);
	return r;
}

/*-----------------------------------------------------------------
 * An IDR is used to keep track of allocated minor numbers.
 *---------------------------------------------------------------*/
static DEFINE_IDR(_minor_idr);

888
static void free_minor(int minor)
L
Linus Torvalds 已提交
889
{
890
	spin_lock(&_minor_lock);
L
Linus Torvalds 已提交
891
	idr_remove(&_minor_idr, minor);
892
	spin_unlock(&_minor_lock);
L
Linus Torvalds 已提交
893 894 895 896 897
}

/*
 * See if the device with a specific minor # is free.
 */
898
static int specific_minor(struct mapped_device *md, int minor)
L
Linus Torvalds 已提交
899 900 901 902 903 904
{
	int r, m;

	if (minor >= (1 << MINORBITS))
		return -EINVAL;

J
Jeff Mahoney 已提交
905 906 907 908
	r = idr_pre_get(&_minor_idr, GFP_KERNEL);
	if (!r)
		return -ENOMEM;

909
	spin_lock(&_minor_lock);
L
Linus Torvalds 已提交
910 911 912 913 914 915

	if (idr_find(&_minor_idr, minor)) {
		r = -EBUSY;
		goto out;
	}

916
	r = idr_get_new_above(&_minor_idr, MINOR_ALLOCED, minor, &m);
J
Jeff Mahoney 已提交
917
	if (r)
L
Linus Torvalds 已提交
918 919 920 921 922 923 924 925 926
		goto out;

	if (m != minor) {
		idr_remove(&_minor_idr, m);
		r = -EBUSY;
		goto out;
	}

out:
927
	spin_unlock(&_minor_lock);
L
Linus Torvalds 已提交
928 929 930
	return r;
}

931
static int next_free_minor(struct mapped_device *md, int *minor)
L
Linus Torvalds 已提交
932
{
933
	int r, m;
L
Linus Torvalds 已提交
934 935

	r = idr_pre_get(&_minor_idr, GFP_KERNEL);
J
Jeff Mahoney 已提交
936 937 938
	if (!r)
		return -ENOMEM;

939
	spin_lock(&_minor_lock);
L
Linus Torvalds 已提交
940

941
	r = idr_get_new(&_minor_idr, MINOR_ALLOCED, &m);
L
Linus Torvalds 已提交
942 943 944 945 946 947 948 949 950 951 952 953 954
	if (r) {
		goto out;
	}

	if (m >= (1 << MINORBITS)) {
		idr_remove(&_minor_idr, m);
		r = -ENOSPC;
		goto out;
	}

	*minor = m;

out:
955
	spin_unlock(&_minor_lock);
L
Linus Torvalds 已提交
956 957 958 959 960 961 962 963
	return r;
}

static struct block_device_operations dm_blk_dops;

/*
 * Allocate and initialise a blank device with a given minor.
 */
964
static struct mapped_device *alloc_dev(int minor)
L
Linus Torvalds 已提交
965 966 967
{
	int r;
	struct mapped_device *md = kmalloc(sizeof(*md), GFP_KERNEL);
968
	void *old_md;
L
Linus Torvalds 已提交
969 970 971 972 973 974

	if (!md) {
		DMWARN("unable to allocate device, out of memory.");
		return NULL;
	}

975 976 977
	if (!try_module_get(THIS_MODULE))
		goto bad0;

L
Linus Torvalds 已提交
978
	/* get a minor number for the dev */
979 980 981 982
	if (minor == DM_ANY_MINOR)
		r = next_free_minor(md, &minor);
	else
		r = specific_minor(md, minor);
L
Linus Torvalds 已提交
983 984 985 986
	if (r < 0)
		goto bad1;

	memset(md, 0, sizeof(*md));
987 988
	init_rwsem(&md->io_lock);
	init_MUTEX(&md->suspend_lock);
989
	spin_lock_init(&md->pushback_lock);
L
Linus Torvalds 已提交
990 991
	rwlock_init(&md->map_lock);
	atomic_set(&md->holders, 1);
992
	atomic_set(&md->open_count, 0);
L
Linus Torvalds 已提交
993 994 995 996
	atomic_set(&md->event_nr, 0);

	md->queue = blk_alloc_queue(GFP_KERNEL);
	if (!md->queue)
997
		goto bad1_free_minor;
L
Linus Torvalds 已提交
998 999 1000 1001 1002

	md->queue->queuedata = md;
	md->queue->backing_dev_info.congested_fn = dm_any_congested;
	md->queue->backing_dev_info.congested_data = md;
	blk_queue_make_request(md->queue, dm_request);
1003
	blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);
L
Linus Torvalds 已提交
1004 1005 1006
	md->queue->unplug_fn = dm_unplug_all;
	md->queue->issue_flush_fn = dm_flush_all;

1007
	md->io_pool = mempool_create_slab_pool(MIN_IOS, _io_cache);
K
Kiyoshi Ueda 已提交
1008 1009
	if (!md->io_pool)
		goto bad2;
L
Linus Torvalds 已提交
1010

1011
	md->tio_pool = mempool_create_slab_pool(MIN_IOS, _tio_cache);
L
Linus Torvalds 已提交
1012 1013 1014
	if (!md->tio_pool)
		goto bad3;

1015
	md->bs = bioset_create(16, 16);
S
Stefan Bader 已提交
1016 1017 1018
	if (!md->bs)
		goto bad_no_bioset;

L
Linus Torvalds 已提交
1019 1020 1021 1022
	md->disk = alloc_disk(1);
	if (!md->disk)
		goto bad4;

1023 1024 1025 1026
	atomic_set(&md->pending, 0);
	init_waitqueue_head(&md->wait);
	init_waitqueue_head(&md->eventq);

L
Linus Torvalds 已提交
1027 1028 1029 1030 1031 1032 1033
	md->disk->major = _major;
	md->disk->first_minor = minor;
	md->disk->fops = &dm_blk_dops;
	md->disk->queue = md->queue;
	md->disk->private_data = md;
	sprintf(md->disk->disk_name, "dm-%d", minor);
	add_disk(md->disk);
M
Mike Anderson 已提交
1034
	format_dev_t(md->name, MKDEV(_major, minor));
L
Linus Torvalds 已提交
1035

1036
	/* Populate the mapping, nobody knows we exist yet */
1037
	spin_lock(&_minor_lock);
1038
	old_md = idr_replace(&_minor_idr, md, minor);
1039
	spin_unlock(&_minor_lock);
1040 1041 1042

	BUG_ON(old_md != MINOR_ALLOCED);

L
Linus Torvalds 已提交
1043 1044 1045
	return md;

 bad4:
S
Stefan Bader 已提交
1046 1047
	bioset_free(md->bs);
 bad_no_bioset:
L
Linus Torvalds 已提交
1048 1049 1050 1051
	mempool_destroy(md->tio_pool);
 bad3:
	mempool_destroy(md->io_pool);
 bad2:
1052
	blk_cleanup_queue(md->queue);
1053
 bad1_free_minor:
L
Linus Torvalds 已提交
1054 1055
	free_minor(minor);
 bad1:
1056 1057
	module_put(THIS_MODULE);
 bad0:
L
Linus Torvalds 已提交
1058 1059 1060 1061 1062 1063
	kfree(md);
	return NULL;
}

static void free_dev(struct mapped_device *md)
{
1064
	int minor = md->disk->first_minor;
1065

1066 1067 1068 1069
	if (md->suspended_bdev) {
		thaw_bdev(md->suspended_bdev, NULL);
		bdput(md->suspended_bdev);
	}
L
Linus Torvalds 已提交
1070 1071
	mempool_destroy(md->tio_pool);
	mempool_destroy(md->io_pool);
S
Stefan Bader 已提交
1072
	bioset_free(md->bs);
L
Linus Torvalds 已提交
1073
	del_gendisk(md->disk);
1074
	free_minor(minor);
J
Jeff Mahoney 已提交
1075 1076 1077 1078 1079

	spin_lock(&_minor_lock);
	md->disk->private_data = NULL;
	spin_unlock(&_minor_lock);

L
Linus Torvalds 已提交
1080
	put_disk(md->disk);
1081
	blk_cleanup_queue(md->queue);
1082
	module_put(THIS_MODULE);
L
Linus Torvalds 已提交
1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096
	kfree(md);
}

/*
 * Bind a table to the device.
 */
static void event_callback(void *context)
{
	struct mapped_device *md = (struct mapped_device *) context;

	atomic_inc(&md->event_nr);
	wake_up(&md->eventq);
}

1097
static void __set_size(struct mapped_device *md, sector_t size)
L
Linus Torvalds 已提交
1098
{
1099
	set_capacity(md->disk, size);
L
Linus Torvalds 已提交
1100

1101
	mutex_lock(&md->suspended_bdev->bd_inode->i_mutex);
1102
	i_size_write(md->suspended_bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
1103
	mutex_unlock(&md->suspended_bdev->bd_inode->i_mutex);
L
Linus Torvalds 已提交
1104 1105 1106 1107 1108 1109 1110 1111
}

static int __bind(struct mapped_device *md, struct dm_table *t)
{
	request_queue_t *q = md->queue;
	sector_t size;

	size = dm_table_get_size(t);
D
Darrick J. Wong 已提交
1112 1113 1114 1115 1116 1117 1118

	/*
	 * Wipe any geometry if the size of the table changed.
	 */
	if (size != get_capacity(md->disk))
		memset(&md->geometry, 0, sizeof(md->geometry));

1119 1120
	if (md->suspended_bdev)
		__set_size(md, size);
L
Linus Torvalds 已提交
1121 1122 1123
	if (size == 0)
		return 0;

1124 1125 1126
	dm_table_get(t);
	dm_table_event_callback(t, event_callback, md);

L
Linus Torvalds 已提交
1127 1128
	write_lock(&md->map_lock);
	md->map = t;
1129
	dm_table_set_restrictions(t, q);
L
Linus Torvalds 已提交
1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151
	write_unlock(&md->map_lock);

	return 0;
}

static void __unbind(struct mapped_device *md)
{
	struct dm_table *map = md->map;

	if (!map)
		return;

	dm_table_event_callback(map, NULL, NULL);
	write_lock(&md->map_lock);
	md->map = NULL;
	write_unlock(&md->map_lock);
	dm_table_put(map);
}

/*
 * Constructor for a new device.
 */
1152
int dm_create(int minor, struct mapped_device **result)
L
Linus Torvalds 已提交
1153 1154 1155
{
	struct mapped_device *md;

1156
	md = alloc_dev(minor);
L
Linus Torvalds 已提交
1157 1158 1159 1160 1161 1162 1163
	if (!md)
		return -ENXIO;

	*result = md;
	return 0;
}

1164
static struct mapped_device *dm_find_md(dev_t dev)
L
Linus Torvalds 已提交
1165 1166 1167 1168 1169 1170 1171
{
	struct mapped_device *md;
	unsigned minor = MINOR(dev);

	if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
		return NULL;

1172
	spin_lock(&_minor_lock);
L
Linus Torvalds 已提交
1173 1174

	md = idr_find(&_minor_idr, minor);
J
Jeff Mahoney 已提交
1175 1176
	if (md && (md == MINOR_ALLOCED ||
		   (dm_disk(md)->first_minor != minor) ||
A
Alasdair G Kergon 已提交
1177
		   test_bit(DMF_FREEING, &md->flags))) {
1178
		md = NULL;
J
Jeff Mahoney 已提交
1179 1180
		goto out;
	}
L
Linus Torvalds 已提交
1181

J
Jeff Mahoney 已提交
1182
out:
1183
	spin_unlock(&_minor_lock);
L
Linus Torvalds 已提交
1184

1185 1186 1187
	return md;
}

1188 1189 1190 1191 1192 1193 1194 1195 1196 1197
struct mapped_device *dm_get_md(dev_t dev)
{
	struct mapped_device *md = dm_find_md(dev);

	if (md)
		dm_get(md);

	return md;
}

A
Alasdair G Kergon 已提交
1198
void *dm_get_mdptr(struct mapped_device *md)
1199
{
A
Alasdair G Kergon 已提交
1200
	return md->interface_ptr;
L
Linus Torvalds 已提交
1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212
}

void dm_set_mdptr(struct mapped_device *md, void *ptr)
{
	md->interface_ptr = ptr;
}

void dm_get(struct mapped_device *md)
{
	atomic_inc(&md->holders);
}

1213 1214 1215 1216 1217 1218
const char *dm_device_name(struct mapped_device *md)
{
	return md->name;
}
EXPORT_SYMBOL_GPL(dm_device_name);

L
Linus Torvalds 已提交
1219 1220
void dm_put(struct mapped_device *md)
{
M
Mike Anderson 已提交
1221
	struct dm_table *map;
L
Linus Torvalds 已提交
1222

J
Jeff Mahoney 已提交
1223 1224
	BUG_ON(test_bit(DMF_FREEING, &md->flags));

1225
	if (atomic_dec_and_lock(&md->holders, &_minor_lock)) {
M
Mike Anderson 已提交
1226
		map = dm_get_table(md);
1227
		idr_replace(&_minor_idr, MINOR_ALLOCED, dm_disk(md)->first_minor);
J
Jeff Mahoney 已提交
1228
		set_bit(DMF_FREEING, &md->flags);
1229
		spin_unlock(&_minor_lock);
1230
		if (!dm_suspended(md)) {
L
Linus Torvalds 已提交
1231 1232 1233 1234
			dm_table_presuspend_targets(map);
			dm_table_postsuspend_targets(map);
		}
		__unbind(md);
M
Mike Anderson 已提交
1235
		dm_table_put(map);
L
Linus Torvalds 已提交
1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259
		free_dev(md);
	}
}

/*
 * Process the deferred bios
 */
static void __flush_deferred_io(struct mapped_device *md, struct bio *c)
{
	struct bio *n;

	while (c) {
		n = c->bi_next;
		c->bi_next = NULL;
		__split_bio(md, c);
		c = n;
	}
}

/*
 * Swap in a new table (destroying old one).
 */
int dm_swap_table(struct mapped_device *md, struct dm_table *table)
{
1260
	int r = -EINVAL;
L
Linus Torvalds 已提交
1261

1262
	down(&md->suspend_lock);
L
Linus Torvalds 已提交
1263 1264

	/* device must be suspended */
1265
	if (!dm_suspended(md))
1266
		goto out;
L
Linus Torvalds 已提交
1267

1268 1269 1270 1271 1272
	/* without bdev, the device size cannot be changed */
	if (!md->suspended_bdev)
		if (get_capacity(md->disk) != dm_table_get_size(table))
			goto out;

L
Linus Torvalds 已提交
1273 1274 1275
	__unbind(md);
	r = __bind(md, table);

1276
out:
1277
	up(&md->suspend_lock);
1278
	return r;
L
Linus Torvalds 已提交
1279 1280 1281 1282 1283 1284
}

/*
 * Functions to lock and unlock any filesystem running on the
 * device.
 */
1285
static int lock_fs(struct mapped_device *md)
L
Linus Torvalds 已提交
1286
{
1287
	int r;
L
Linus Torvalds 已提交
1288 1289

	WARN_ON(md->frozen_sb);
1290

1291
	md->frozen_sb = freeze_bdev(md->suspended_bdev);
1292
	if (IS_ERR(md->frozen_sb)) {
1293
		r = PTR_ERR(md->frozen_sb);
1294 1295
		md->frozen_sb = NULL;
		return r;
1296 1297
	}

1298 1299
	set_bit(DMF_FROZEN, &md->flags);

L
Linus Torvalds 已提交
1300
	/* don't bdput right now, we don't want the bdev
1301
	 * to go away while it is locked.
L
Linus Torvalds 已提交
1302 1303 1304 1305
	 */
	return 0;
}

1306
static void unlock_fs(struct mapped_device *md)
L
Linus Torvalds 已提交
1307
{
1308 1309 1310
	if (!test_bit(DMF_FROZEN, &md->flags))
		return;

1311
	thaw_bdev(md->suspended_bdev, md->frozen_sb);
L
Linus Torvalds 已提交
1312
	md->frozen_sb = NULL;
1313
	clear_bit(DMF_FROZEN, &md->flags);
L
Linus Torvalds 已提交
1314 1315 1316 1317 1318 1319 1320 1321 1322
}

/*
 * We need to be able to change a mapping table under a mounted
 * filesystem.  For example we might want to move some data in
 * the background.  Before the table can be swapped with
 * dm_bind_table, dm_suspend must be called to flush any in
 * flight bios and ensure that any further io gets deferred.
 */
1323
int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
L
Linus Torvalds 已提交
1324
{
1325
	struct dm_table *map = NULL;
1326
	unsigned long flags;
L
Linus Torvalds 已提交
1327
	DECLARE_WAITQUEUE(wait, current);
J
Jun'ichi Nomura 已提交
1328
	struct bio *def;
1329
	int r = -EINVAL;
1330
	int do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG ? 1 : 0;
1331
	int noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG ? 1 : 0;
L
Linus Torvalds 已提交
1332

1333 1334 1335
	down(&md->suspend_lock);

	if (dm_suspended(md))
1336
		goto out_unlock;
L
Linus Torvalds 已提交
1337 1338 1339

	map = dm_get_table(md);

1340 1341 1342 1343 1344 1345 1346
	/*
	 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
	 * This flag is cleared before dm_suspend returns.
	 */
	if (noflush)
		set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);

1347 1348 1349
	/* This does not get reverted if there's an error later. */
	dm_table_presuspend_targets(map);

1350 1351 1352 1353 1354 1355 1356 1357
	/* bdget() can stall if the pending I/Os are not flushed */
	if (!noflush) {
		md->suspended_bdev = bdget_disk(md->disk, 0);
		if (!md->suspended_bdev) {
			DMWARN("bdget failed in dm_suspend");
			r = -ENOMEM;
			goto flush_and_out;
		}
1358 1359
	}

1360 1361 1362 1363 1364
	/*
	 * Flush I/O to the device.
	 * noflush supersedes do_lockfs, because lock_fs() needs to flush I/Os.
	 */
	if (do_lockfs && !noflush) {
1365 1366 1367 1368
		r = lock_fs(md);
		if (r)
			goto out;
	}
L
Linus Torvalds 已提交
1369 1370

	/*
1371
	 * First we set the BLOCK_IO flag so no more ios will be mapped.
L
Linus Torvalds 已提交
1372
	 */
1373 1374
	down_write(&md->io_lock);
	set_bit(DMF_BLOCK_IO, &md->flags);
L
Linus Torvalds 已提交
1375 1376

	add_wait_queue(&md->wait, &wait);
1377
	up_write(&md->io_lock);
L
Linus Torvalds 已提交
1378 1379

	/* unplug */
1380
	if (map)
L
Linus Torvalds 已提交
1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396
		dm_table_unplug_all(map);

	/*
	 * Then we wait for the already mapped ios to
	 * complete.
	 */
	while (1) {
		set_current_state(TASK_INTERRUPTIBLE);

		if (!atomic_read(&md->pending) || signal_pending(current))
			break;

		io_schedule();
	}
	set_current_state(TASK_RUNNING);

1397
	down_write(&md->io_lock);
L
Linus Torvalds 已提交
1398 1399
	remove_wait_queue(&md->wait, &wait);

1400 1401 1402 1403 1404 1405 1406 1407
	if (noflush) {
		spin_lock_irqsave(&md->pushback_lock, flags);
		clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
		bio_list_merge_head(&md->deferred, &md->pushback);
		bio_list_init(&md->pushback);
		spin_unlock_irqrestore(&md->pushback_lock, flags);
	}

L
Linus Torvalds 已提交
1408
	/* were we interrupted ? */
1409
	r = -EINTR;
1410
	if (atomic_read(&md->pending)) {
J
Jun'ichi Nomura 已提交
1411 1412 1413
		clear_bit(DMF_BLOCK_IO, &md->flags);
		def = bio_list_get(&md->deferred);
		__flush_deferred_io(md, def);
1414 1415
		up_write(&md->io_lock);
		unlock_fs(md);
1416
		goto out; /* pushback list is already flushed, so skip flush */
1417 1418
	}
	up_write(&md->io_lock);
L
Linus Torvalds 已提交
1419

1420
	dm_table_postsuspend_targets(map);
L
Linus Torvalds 已提交
1421

1422
	set_bit(DMF_SUSPENDED, &md->flags);
1423

1424
	r = 0;
1425

1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444
flush_and_out:
	if (r && noflush) {
		/*
		 * Because there may be already I/Os in the pushback list,
		 * flush them before return.
		 */
		down_write(&md->io_lock);

		spin_lock_irqsave(&md->pushback_lock, flags);
		clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
		bio_list_merge_head(&md->deferred, &md->pushback);
		bio_list_init(&md->pushback);
		spin_unlock_irqrestore(&md->pushback_lock, flags);

		def = bio_list_get(&md->deferred);
		__flush_deferred_io(md, def);
		up_write(&md->io_lock);
	}

1445
out:
1446 1447 1448 1449 1450
	if (r && md->suspended_bdev) {
		bdput(md->suspended_bdev);
		md->suspended_bdev = NULL;
	}

1451
	dm_table_put(map);
1452 1453

out_unlock:
1454
	up(&md->suspend_lock);
1455
	return r;
L
Linus Torvalds 已提交
1456 1457 1458 1459
}

int dm_resume(struct mapped_device *md)
{
1460
	int r = -EINVAL;
L
Linus Torvalds 已提交
1461
	struct bio *def;
1462
	struct dm_table *map = NULL;
L
Linus Torvalds 已提交
1463

1464 1465
	down(&md->suspend_lock);
	if (!dm_suspended(md))
1466 1467 1468
		goto out;

	map = dm_get_table(md);
1469
	if (!map || !dm_table_get_size(map))
1470
		goto out;
L
Linus Torvalds 已提交
1471

1472 1473 1474
	r = dm_table_resume_targets(map);
	if (r)
		goto out;
1475 1476

	down_write(&md->io_lock);
L
Linus Torvalds 已提交
1477 1478 1479 1480
	clear_bit(DMF_BLOCK_IO, &md->flags);

	def = bio_list_get(&md->deferred);
	__flush_deferred_io(md, def);
1481 1482 1483 1484
	up_write(&md->io_lock);

	unlock_fs(md);

1485 1486 1487 1488
	if (md->suspended_bdev) {
		bdput(md->suspended_bdev);
		md->suspended_bdev = NULL;
	}
1489

1490 1491
	clear_bit(DMF_SUSPENDED, &md->flags);

L
Linus Torvalds 已提交
1492 1493
	dm_table_unplug_all(map);

1494 1495
	kobject_uevent(&md->disk->kobj, KOBJ_CHANGE);

1496
	r = 0;
1497

1498 1499
out:
	dm_table_put(map);
1500 1501
	up(&md->suspend_lock);

1502
	return r;
L
Linus Torvalds 已提交
1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532
}

/*-----------------------------------------------------------------
 * Event notification.
 *---------------------------------------------------------------*/
uint32_t dm_get_event_nr(struct mapped_device *md)
{
	return atomic_read(&md->event_nr);
}

int dm_wait_event(struct mapped_device *md, int event_nr)
{
	return wait_event_interruptible(md->eventq,
			(event_nr != atomic_read(&md->event_nr)));
}

/*
 * The gendisk is only valid as long as you have a reference
 * count on 'md'.
 */
struct gendisk *dm_disk(struct mapped_device *md)
{
	return md->disk;
}

int dm_suspended(struct mapped_device *md)
{
	return test_bit(DMF_SUSPENDED, &md->flags);
}

1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543
int dm_noflush_suspending(struct dm_target *ti)
{
	struct mapped_device *md = dm_table_get_md(ti->table);
	int r = __noflush_suspending(md);

	dm_put(md);

	return r;
}
EXPORT_SYMBOL_GPL(dm_noflush_suspending);

L
Linus Torvalds 已提交
1544 1545 1546
static struct block_device_operations dm_blk_dops = {
	.open = dm_blk_open,
	.release = dm_blk_close,
1547
	.ioctl = dm_blk_ioctl,
D
Darrick J. Wong 已提交
1548
	.getgeo = dm_blk_getgeo,
L
Linus Torvalds 已提交
1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564
	.owner = THIS_MODULE
};

EXPORT_SYMBOL(dm_get_mapinfo);

/*
 * module hooks
 */
module_init(dm_init);
module_exit(dm_exit);

module_param(major, uint, 0);
MODULE_PARM_DESC(major, "The major number of the device mapper");
MODULE_DESCRIPTION(DM_NAME " driver");
MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
MODULE_LICENSE("GPL");