dm.c 34.5 KB
Newer Older
L
Linus Torvalds 已提交
1 2
/*
 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
3
 * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved.
L
Linus Torvalds 已提交
4 5 6 7 8 9
 *
 * This file is released under the GPL.
 */

#include "dm.h"
#include "dm-bio-list.h"
M
Mike Anderson 已提交
10
#include "dm-uevent.h"
L
Linus Torvalds 已提交
11 12 13

#include <linux/init.h>
#include <linux/module.h>
A
Arjan van de Ven 已提交
14
#include <linux/mutex.h>
L
Linus Torvalds 已提交
15 16 17 18 19 20 21
#include <linux/moduleparam.h>
#include <linux/blkpg.h>
#include <linux/bio.h>
#include <linux/buffer_head.h>
#include <linux/mempool.h>
#include <linux/slab.h>
#include <linux/idr.h>
D
Darrick J. Wong 已提交
22
#include <linux/hdreg.h>
23
#include <linux/blktrace_api.h>
24
#include <linux/smp_lock.h>
L
Linus Torvalds 已提交
25

26 27
#define DM_MSG_PREFIX "core"

L
Linus Torvalds 已提交
28 29 30 31 32
static const char *_name = DM_NAME;

static unsigned int major = 0;
static unsigned int _major = 0;

33
static DEFINE_SPINLOCK(_minor_lock);
L
Linus Torvalds 已提交
34 35 36 37 38 39 40
/*
 * One of these is allocated per bio.
 */
struct dm_io {
	struct mapped_device *md;
	int error;
	atomic_t io_count;
R
Richard Kennedy 已提交
41
	struct bio *bio;
42
	unsigned long start_time;
L
Linus Torvalds 已提交
43 44 45 46 47 48
};

/*
 * One of these is allocated per target within a bio.  Hopefully
 * this will be simplified out one day.
 */
A
Alasdair G Kergon 已提交
49
struct dm_target_io {
L
Linus Torvalds 已提交
50 51 52 53 54 55 56
	struct dm_io *io;
	struct dm_target *ti;
	union map_info info;
};

union map_info *dm_get_mapinfo(struct bio *bio)
{
A
Alasdair G Kergon 已提交
57
	if (bio && bio->bi_private)
A
Alasdair G Kergon 已提交
58
		return &((struct dm_target_io *)bio->bi_private)->info;
A
Alasdair G Kergon 已提交
59
	return NULL;
L
Linus Torvalds 已提交
60 61
}

62 63
#define MINOR_ALLOCED ((void *)-1)

L
Linus Torvalds 已提交
64 65 66 67 68
/*
 * Bits for the md->flags field.
 */
#define DMF_BLOCK_IO 0
#define DMF_SUSPENDED 1
69
#define DMF_FROZEN 2
J
Jeff Mahoney 已提交
70
#define DMF_FREEING 3
71
#define DMF_DELETING 4
72
#define DMF_NOFLUSH_SUSPENDING 5
L
Linus Torvalds 已提交
73

74 75 76 77 78 79 80 81 82 83 84 85 86
/*
 * Work processed by per-device workqueue.
 */
struct dm_wq_req {
	enum {
		DM_WQ_FLUSH_ALL,
		DM_WQ_FLUSH_DEFERRED,
	} type;
	struct work_struct work;
	struct mapped_device *md;
	void *context;
};

L
Linus Torvalds 已提交
87
struct mapped_device {
88
	struct rw_semaphore io_lock;
89
	struct mutex suspend_lock;
90
	spinlock_t pushback_lock;
L
Linus Torvalds 已提交
91 92
	rwlock_t map_lock;
	atomic_t holders;
93
	atomic_t open_count;
L
Linus Torvalds 已提交
94 95 96

	unsigned long flags;

97
	struct request_queue *queue;
L
Linus Torvalds 已提交
98
	struct gendisk *disk;
M
Mike Anderson 已提交
99
	char name[16];
L
Linus Torvalds 已提交
100 101 102 103 104 105 106 107

	void *interface_ptr;

	/*
	 * A list of ios that arrived while we were suspended.
	 */
	atomic_t pending;
	wait_queue_head_t wait;
K
Kiyoshi Ueda 已提交
108
	struct bio_list deferred;
109
	struct bio_list pushback;
L
Linus Torvalds 已提交
110

111 112 113 114 115
	/*
	 * Processing queue (flush/barriers)
	 */
	struct workqueue_struct *wq;

L
Linus Torvalds 已提交
116 117 118 119 120 121 122 123 124 125 126
	/*
	 * The current mapping.
	 */
	struct dm_table *map;

	/*
	 * io objects are allocated from here.
	 */
	mempool_t *io_pool;
	mempool_t *tio_pool;

S
Stefan Bader 已提交
127 128
	struct bio_set *bs;

L
Linus Torvalds 已提交
129 130 131 132 133
	/*
	 * Event handling.
	 */
	atomic_t event_nr;
	wait_queue_head_t eventq;
M
Mike Anderson 已提交
134 135 136
	atomic_t uevent_seq;
	struct list_head uevent_list;
	spinlock_t uevent_lock; /* Protect access to uevent_list */
L
Linus Torvalds 已提交
137 138 139 140 141

	/*
	 * freeze/thaw support require holding onto a super block
	 */
	struct super_block *frozen_sb;
142
	struct block_device *suspended_bdev;
D
Darrick J. Wong 已提交
143 144 145

	/* forced geometry settings */
	struct hd_geometry geometry;
L
Linus Torvalds 已提交
146 147 148
};

#define MIN_IOS 256
149 150
static struct kmem_cache *_io_cache;
static struct kmem_cache *_tio_cache;
L
Linus Torvalds 已提交
151 152 153 154 155 156

static int __init local_init(void)
{
	int r;

	/* allocate a slab for the dm_ios */
A
Alasdair G Kergon 已提交
157
	_io_cache = KMEM_CACHE(dm_io, 0);
L
Linus Torvalds 已提交
158 159 160 161
	if (!_io_cache)
		return -ENOMEM;

	/* allocate a slab for the target ios */
A
Alasdair G Kergon 已提交
162
	_tio_cache = KMEM_CACHE(dm_target_io, 0);
L
Linus Torvalds 已提交
163 164 165 166 167
	if (!_tio_cache) {
		kmem_cache_destroy(_io_cache);
		return -ENOMEM;
	}

M
Mike Anderson 已提交
168 169 170 171 172 173 174
	r = dm_uevent_init();
	if (r) {
		kmem_cache_destroy(_tio_cache);
		kmem_cache_destroy(_io_cache);
		return r;
	}

L
Linus Torvalds 已提交
175 176 177 178 179
	_major = major;
	r = register_blkdev(_major, _name);
	if (r < 0) {
		kmem_cache_destroy(_tio_cache);
		kmem_cache_destroy(_io_cache);
M
Mike Anderson 已提交
180
		dm_uevent_exit();
L
Linus Torvalds 已提交
181 182 183 184 185 186 187 188 189 190 191 192 193
		return r;
	}

	if (!_major)
		_major = r;

	return 0;
}

static void local_exit(void)
{
	kmem_cache_destroy(_tio_cache);
	kmem_cache_destroy(_io_cache);
194
	unregister_blkdev(_major, _name);
M
Mike Anderson 已提交
195
	dm_uevent_exit();
L
Linus Torvalds 已提交
196 197 198 199 200 201

	_major = 0;

	DMINFO("cleaned up");
}

202
static int (*_inits[])(void) __initdata = {
L
Linus Torvalds 已提交
203 204 205 206
	local_init,
	dm_target_init,
	dm_linear_init,
	dm_stripe_init,
207
	dm_kcopyd_init,
L
Linus Torvalds 已提交
208 209 210
	dm_interface_init,
};

211
static void (*_exits[])(void) = {
L
Linus Torvalds 已提交
212 213 214 215
	local_exit,
	dm_target_exit,
	dm_linear_exit,
	dm_stripe_exit,
216
	dm_kcopyd_exit,
L
Linus Torvalds 已提交
217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255
	dm_interface_exit,
};

static int __init dm_init(void)
{
	const int count = ARRAY_SIZE(_inits);

	int r, i;

	for (i = 0; i < count; i++) {
		r = _inits[i]();
		if (r)
			goto bad;
	}

	return 0;

      bad:
	while (i--)
		_exits[i]();

	return r;
}

static void __exit dm_exit(void)
{
	int i = ARRAY_SIZE(_exits);

	while (i--)
		_exits[i]();
}

/*
 * Block device functions
 */
static int dm_blk_open(struct inode *inode, struct file *file)
{
	struct mapped_device *md;

J
Jeff Mahoney 已提交
256 257
	spin_lock(&_minor_lock);

L
Linus Torvalds 已提交
258
	md = inode->i_bdev->bd_disk->private_data;
J
Jeff Mahoney 已提交
259 260 261
	if (!md)
		goto out;

262 263
	if (test_bit(DMF_FREEING, &md->flags) ||
	    test_bit(DMF_DELETING, &md->flags)) {
J
Jeff Mahoney 已提交
264 265 266 267
		md = NULL;
		goto out;
	}

L
Linus Torvalds 已提交
268
	dm_get(md);
269
	atomic_inc(&md->open_count);
J
Jeff Mahoney 已提交
270 271 272 273 274

out:
	spin_unlock(&_minor_lock);

	return md ? 0 : -ENXIO;
L
Linus Torvalds 已提交
275 276 277 278 279 280 281
}

static int dm_blk_close(struct inode *inode, struct file *file)
{
	struct mapped_device *md;

	md = inode->i_bdev->bd_disk->private_data;
282
	atomic_dec(&md->open_count);
L
Linus Torvalds 已提交
283 284 285 286
	dm_put(md);
	return 0;
}

287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310
int dm_open_count(struct mapped_device *md)
{
	return atomic_read(&md->open_count);
}

/*
 * Guarantees nothing is using the device before it's deleted.
 */
int dm_lock_for_deletion(struct mapped_device *md)
{
	int r = 0;

	spin_lock(&_minor_lock);

	if (dm_open_count(md))
		r = -EBUSY;
	else
		set_bit(DMF_DELETING, &md->flags);

	spin_unlock(&_minor_lock);

	return r;
}

D
Darrick J. Wong 已提交
311 312 313 314 315 316 317
static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
{
	struct mapped_device *md = bdev->bd_disk->private_data;

	return dm_get_geometry(md, geo);
}

318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356
static int dm_blk_ioctl(struct inode *inode, struct file *file,
			unsigned int cmd, unsigned long arg)
{
	struct mapped_device *md;
	struct dm_table *map;
	struct dm_target *tgt;
	int r = -ENOTTY;

	/* We don't really need this lock, but we do need 'inode'. */
	unlock_kernel();

	md = inode->i_bdev->bd_disk->private_data;

	map = dm_get_table(md);

	if (!map || !dm_table_get_size(map))
		goto out;

	/* We only support devices that have a single target */
	if (dm_table_get_num_targets(map) != 1)
		goto out;

	tgt = dm_table_get_target(map, 0);

	if (dm_suspended(md)) {
		r = -EAGAIN;
		goto out;
	}

	if (tgt->type->ioctl)
		r = tgt->type->ioctl(tgt, inode, file, cmd, arg);

out:
	dm_table_put(map);

	lock_kernel();
	return r;
}

A
Alasdair G Kergon 已提交
357
static struct dm_io *alloc_io(struct mapped_device *md)
L
Linus Torvalds 已提交
358 359 360 361
{
	return mempool_alloc(md->io_pool, GFP_NOIO);
}

A
Alasdair G Kergon 已提交
362
static void free_io(struct mapped_device *md, struct dm_io *io)
L
Linus Torvalds 已提交
363 364 365 366
{
	mempool_free(io, md->io_pool);
}

A
Alasdair G Kergon 已提交
367
static struct dm_target_io *alloc_tio(struct mapped_device *md)
L
Linus Torvalds 已提交
368 369 370 371
{
	return mempool_alloc(md->tio_pool, GFP_NOIO);
}

A
Alasdair G Kergon 已提交
372
static void free_tio(struct mapped_device *md, struct dm_target_io *tio)
L
Linus Torvalds 已提交
373 374 375 376
{
	mempool_free(tio, md->tio_pool);
}

377 378 379
static void start_io_acct(struct dm_io *io)
{
	struct mapped_device *md = io->md;
T
Tejun Heo 已提交
380
	int cpu;
381 382 383

	io->start_time = jiffies;

T
Tejun Heo 已提交
384 385 386 387
	cpu = part_stat_lock();
	part_round_stats(cpu, &dm_disk(md)->part0);
	part_stat_unlock();
	dm_disk(md)->part0.in_flight = atomic_inc_return(&md->pending);
388 389 390 391 392 393 394
}

static int end_io_acct(struct dm_io *io)
{
	struct mapped_device *md = io->md;
	struct bio *bio = io->bio;
	unsigned long duration = jiffies - io->start_time;
T
Tejun Heo 已提交
395
	int pending, cpu;
396 397
	int rw = bio_data_dir(bio);

T
Tejun Heo 已提交
398 399 400 401
	cpu = part_stat_lock();
	part_round_stats(cpu, &dm_disk(md)->part0);
	part_stat_add(cpu, &dm_disk(md)->part0, ticks[rw], duration);
	part_stat_unlock();
402

T
Tejun Heo 已提交
403 404
	dm_disk(md)->part0.in_flight = pending =
		atomic_dec_return(&md->pending);
405 406 407 408

	return !pending;
}

L
Linus Torvalds 已提交
409 410 411 412 413
/*
 * Add the bio to the list of deferred io.
 */
static int queue_io(struct mapped_device *md, struct bio *bio)
{
414
	down_write(&md->io_lock);
L
Linus Torvalds 已提交
415 416

	if (!test_bit(DMF_BLOCK_IO, &md->flags)) {
417
		up_write(&md->io_lock);
L
Linus Torvalds 已提交
418 419 420 421 422
		return 1;
	}

	bio_list_add(&md->deferred, bio);

423
	up_write(&md->io_lock);
L
Linus Torvalds 已提交
424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444
	return 0;		/* deferred successfully */
}

/*
 * Everyone (including functions in this file), should use this
 * function to access the md->map field, and make sure they call
 * dm_table_put() when finished.
 */
struct dm_table *dm_get_table(struct mapped_device *md)
{
	struct dm_table *t;

	read_lock(&md->map_lock);
	t = md->map;
	if (t)
		dm_table_get(t);
	read_unlock(&md->map_lock);

	return t;
}

D
Darrick J. Wong 已提交
445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471
/*
 * Get the geometry associated with a dm device
 */
int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
{
	*geo = md->geometry;

	return 0;
}

/*
 * Set the geometry of a device.
 */
int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
{
	sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;

	if (geo->start > sz) {
		DMWARN("Start sector is beyond the geometry limits.");
		return -EINVAL;
	}

	md->geometry = *geo;

	return 0;
}

L
Linus Torvalds 已提交
472 473 474 475 476 477 478 479 480
/*-----------------------------------------------------------------
 * CRUD START:
 *   A more elegant soln is in the works that uses the queue
 *   merge fn, unfortunately there are a couple of changes to
 *   the block layer that I want to make for this.  So in the
 *   interests of getting something for people to use I give
 *   you this clearly demarcated crap.
 *---------------------------------------------------------------*/

481 482 483 484 485
static int __noflush_suspending(struct mapped_device *md)
{
	return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
}

L
Linus Torvalds 已提交
486 487 488 489
/*
 * Decrements the number of outstanding ios that a bio has been
 * cloned into, completing the original io if necc.
 */
490
static void dec_pending(struct dm_io *io, int error)
L
Linus Torvalds 已提交
491
{
492 493 494 495
	unsigned long flags;

	/* Push-back supersedes any I/O errors */
	if (error && !(io->error > 0 && __noflush_suspending(io->md)))
L
Linus Torvalds 已提交
496 497 498
		io->error = error;

	if (atomic_dec_and_test(&io->io_count)) {
499 500 501 502 503 504 505 506 507 508 509 510 511 512 513
		if (io->error == DM_ENDIO_REQUEUE) {
			/*
			 * Target requested pushing back the I/O.
			 * This must be handled before the sleeper on
			 * suspend queue merges the pushback list.
			 */
			spin_lock_irqsave(&io->md->pushback_lock, flags);
			if (__noflush_suspending(io->md))
				bio_list_add(&io->md->pushback, io->bio);
			else
				/* noflush suspend was interrupted. */
				io->error = -EIO;
			spin_unlock_irqrestore(&io->md->pushback_lock, flags);
		}

514
		if (end_io_acct(io))
L
Linus Torvalds 已提交
515 516 517
			/* nudge anyone waiting on suspend queue */
			wake_up(&io->md->wait);

518 519 520 521
		if (io->error != DM_ENDIO_REQUEUE) {
			blk_add_trace_bio(io->md->queue, io->bio,
					  BLK_TA_COMPLETE);

522
			bio_endio(io->bio, io->error);
523
		}
524

L
Linus Torvalds 已提交
525 526 527 528
		free_io(io->md, io);
	}
}

529
static void clone_endio(struct bio *bio, int error)
L
Linus Torvalds 已提交
530 531
{
	int r = 0;
A
Alasdair G Kergon 已提交
532
	struct dm_target_io *tio = bio->bi_private;
S
Stefan Bader 已提交
533
	struct mapped_device *md = tio->io->md;
L
Linus Torvalds 已提交
534 535 536 537 538 539 540
	dm_endio_fn endio = tio->ti->type->end_io;

	if (!bio_flagged(bio, BIO_UPTODATE) && !error)
		error = -EIO;

	if (endio) {
		r = endio(tio->ti, bio, error, &tio->info);
541 542 543 544 545
		if (r < 0 || r == DM_ENDIO_REQUEUE)
			/*
			 * error and requeue request are handled
			 * in dec_pending().
			 */
L
Linus Torvalds 已提交
546
			error = r;
547 548
		else if (r == DM_ENDIO_INCOMPLETE)
			/* The target will handle the io */
549
			return;
550 551 552 553
		else if (r) {
			DMWARN("unimplemented target endio return value: %d", r);
			BUG();
		}
L
Linus Torvalds 已提交
554 555
	}

S
Stefan Bader 已提交
556 557 558 559 560 561 562
	dec_pending(tio->io, error);

	/*
	 * Store md for cleanup instead of tio which is about to get freed.
	 */
	bio->bi_private = md->bs;

L
Linus Torvalds 已提交
563
	bio_put(bio);
S
Stefan Bader 已提交
564
	free_tio(md, tio);
L
Linus Torvalds 已提交
565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587
}

static sector_t max_io_len(struct mapped_device *md,
			   sector_t sector, struct dm_target *ti)
{
	sector_t offset = sector - ti->begin;
	sector_t len = ti->len - offset;

	/*
	 * Does the target need to split even further ?
	 */
	if (ti->split_io) {
		sector_t boundary;
		boundary = ((offset + ti->split_io) & ~(ti->split_io - 1))
			   - offset;
		if (len > boundary)
			len = boundary;
	}

	return len;
}

static void __map_bio(struct dm_target *ti, struct bio *clone,
A
Alasdair G Kergon 已提交
588
		      struct dm_target_io *tio)
L
Linus Torvalds 已提交
589 590
{
	int r;
591
	sector_t sector;
S
Stefan Bader 已提交
592
	struct mapped_device *md;
L
Linus Torvalds 已提交
593 594 595 596 597 598 599 600 601 602 603 604 605 606 607

	/*
	 * Sanity checks.
	 */
	BUG_ON(!clone->bi_size);

	clone->bi_end_io = clone_endio;
	clone->bi_private = tio;

	/*
	 * Map the clone.  If r == 0 we don't need to do
	 * anything, the target has assumed ownership of
	 * this io.
	 */
	atomic_inc(&tio->io->io_count);
608
	sector = clone->bi_sector;
L
Linus Torvalds 已提交
609
	r = ti->type->map(ti, clone, &tio->info);
610
	if (r == DM_MAPIO_REMAPPED) {
L
Linus Torvalds 已提交
611
		/* the bio has been remapped so dispatch it */
612

A
Alasdair G Kergon 已提交
613
		blk_add_trace_remap(bdev_get_queue(clone->bi_bdev), clone,
614 615
				    tio->io->bio->bi_bdev->bd_dev,
				    clone->bi_sector, sector);
616

L
Linus Torvalds 已提交
617
		generic_make_request(clone);
618 619
	} else if (r < 0 || r == DM_MAPIO_REQUEUE) {
		/* error the io and bail out, or requeue it if needed */
S
Stefan Bader 已提交
620 621 622 623 624 625
		md = tio->io->md;
		dec_pending(tio->io, r);
		/*
		 * Store bio_set for cleanup.
		 */
		clone->bi_private = md->bs;
L
Linus Torvalds 已提交
626
		bio_put(clone);
S
Stefan Bader 已提交
627
		free_tio(md, tio);
628 629 630
	} else if (r) {
		DMWARN("unimplemented target map return value: %d", r);
		BUG();
L
Linus Torvalds 已提交
631 632 633 634 635 636 637 638 639 640 641 642 643
	}
}

struct clone_info {
	struct mapped_device *md;
	struct dm_table *map;
	struct bio *bio;
	struct dm_io *io;
	sector_t sector;
	sector_t sector_count;
	unsigned short idx;
};

P
Peter Osterlund 已提交
644 645
static void dm_bio_destructor(struct bio *bio)
{
S
Stefan Bader 已提交
646 647 648
	struct bio_set *bs = bio->bi_private;

	bio_free(bio, bs);
P
Peter Osterlund 已提交
649 650
}

L
Linus Torvalds 已提交
651 652 653 654 655
/*
 * Creates a little bio that is just does part of a bvec.
 */
static struct bio *split_bvec(struct bio *bio, sector_t sector,
			      unsigned short idx, unsigned int offset,
S
Stefan Bader 已提交
656
			      unsigned int len, struct bio_set *bs)
L
Linus Torvalds 已提交
657 658 659 660
{
	struct bio *clone;
	struct bio_vec *bv = bio->bi_io_vec + idx;

S
Stefan Bader 已提交
661
	clone = bio_alloc_bioset(GFP_NOIO, 1, bs);
P
Peter Osterlund 已提交
662
	clone->bi_destructor = dm_bio_destructor;
L
Linus Torvalds 已提交
663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680
	*clone->bi_io_vec = *bv;

	clone->bi_sector = sector;
	clone->bi_bdev = bio->bi_bdev;
	clone->bi_rw = bio->bi_rw;
	clone->bi_vcnt = 1;
	clone->bi_size = to_bytes(len);
	clone->bi_io_vec->bv_offset = offset;
	clone->bi_io_vec->bv_len = clone->bi_size;

	return clone;
}

/*
 * Creates a bio that consists of range of complete bvecs.
 */
static struct bio *clone_bio(struct bio *bio, sector_t sector,
			     unsigned short idx, unsigned short bv_count,
S
Stefan Bader 已提交
681
			     unsigned int len, struct bio_set *bs)
L
Linus Torvalds 已提交
682 683 684
{
	struct bio *clone;

S
Stefan Bader 已提交
685 686 687
	clone = bio_alloc_bioset(GFP_NOIO, bio->bi_max_vecs, bs);
	__bio_clone(clone, bio);
	clone->bi_destructor = dm_bio_destructor;
L
Linus Torvalds 已提交
688 689 690 691 692 693 694 695 696
	clone->bi_sector = sector;
	clone->bi_idx = idx;
	clone->bi_vcnt = idx + bv_count;
	clone->bi_size = to_bytes(len);
	clone->bi_flags &= ~(1 << BIO_SEG_VALID);

	return clone;
}

697
static int __clone_and_map(struct clone_info *ci)
L
Linus Torvalds 已提交
698 699
{
	struct bio *clone, *bio = ci->bio;
700 701
	struct dm_target *ti;
	sector_t len = 0, max;
A
Alasdair G Kergon 已提交
702
	struct dm_target_io *tio;
L
Linus Torvalds 已提交
703

704 705 706 707 708 709
	ti = dm_table_find_target(ci->map, ci->sector);
	if (!dm_target_is_valid(ti))
		return -EIO;

	max = max_io_len(ci->md, ci->sector, ti);

L
Linus Torvalds 已提交
710 711 712 713 714 715 716 717 718 719 720 721 722 723
	/*
	 * Allocate a target io object.
	 */
	tio = alloc_tio(ci->md);
	tio->io = ci->io;
	tio->ti = ti;
	memset(&tio->info, 0, sizeof(tio->info));

	if (ci->sector_count <= max) {
		/*
		 * Optimise for the simple case where we can do all of
		 * the remaining io with a single clone.
		 */
		clone = clone_bio(bio, ci->sector, ci->idx,
S
Stefan Bader 已提交
724 725
				  bio->bi_vcnt - ci->idx, ci->sector_count,
				  ci->md->bs);
L
Linus Torvalds 已提交
726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747
		__map_bio(ti, clone, tio);
		ci->sector_count = 0;

	} else if (to_sector(bio->bi_io_vec[ci->idx].bv_len) <= max) {
		/*
		 * There are some bvecs that don't span targets.
		 * Do as many of these as possible.
		 */
		int i;
		sector_t remaining = max;
		sector_t bv_len;

		for (i = ci->idx; remaining && (i < bio->bi_vcnt); i++) {
			bv_len = to_sector(bio->bi_io_vec[i].bv_len);

			if (bv_len > remaining)
				break;

			remaining -= bv_len;
			len += bv_len;
		}

S
Stefan Bader 已提交
748 749
		clone = clone_bio(bio, ci->sector, ci->idx, i - ci->idx, len,
				  ci->md->bs);
L
Linus Torvalds 已提交
750 751 752 753 754 755 756 757
		__map_bio(ti, clone, tio);

		ci->sector += len;
		ci->sector_count -= len;
		ci->idx = i;

	} else {
		/*
758
		 * Handle a bvec that must be split between two or more targets.
L
Linus Torvalds 已提交
759 760
		 */
		struct bio_vec *bv = bio->bi_io_vec + ci->idx;
761 762
		sector_t remaining = to_sector(bv->bv_len);
		unsigned int offset = 0;
L
Linus Torvalds 已提交
763

764 765 766
		do {
			if (offset) {
				ti = dm_table_find_target(ci->map, ci->sector);
767 768 769
				if (!dm_target_is_valid(ti))
					return -EIO;

770
				max = max_io_len(ci->md, ci->sector, ti);
L
Linus Torvalds 已提交
771

772 773 774 775 776 777 778 779 780
				tio = alloc_tio(ci->md);
				tio->io = ci->io;
				tio->ti = ti;
				memset(&tio->info, 0, sizeof(tio->info));
			}

			len = min(remaining, max);

			clone = split_bvec(bio, ci->sector, ci->idx,
S
Stefan Bader 已提交
781 782
					   bv->bv_offset + offset, len,
					   ci->md->bs);
783 784 785 786 787 788 789

			__map_bio(ti, clone, tio);

			ci->sector += len;
			ci->sector_count -= len;
			offset += to_bytes(len);
		} while (remaining -= len);
L
Linus Torvalds 已提交
790 791 792

		ci->idx++;
	}
793 794

	return 0;
L
Linus Torvalds 已提交
795 796 797 798 799
}

/*
 * Split the bio into several clones.
 */
M
Milan Broz 已提交
800
static int __split_bio(struct mapped_device *md, struct bio *bio)
L
Linus Torvalds 已提交
801 802
{
	struct clone_info ci;
803
	int error = 0;
L
Linus Torvalds 已提交
804 805

	ci.map = dm_get_table(md);
M
Milan Broz 已提交
806 807
	if (unlikely(!ci.map))
		return -EIO;
L
Linus Torvalds 已提交
808 809 810 811 812 813 814 815 816 817 818 819

	ci.md = md;
	ci.bio = bio;
	ci.io = alloc_io(md);
	ci.io->error = 0;
	atomic_set(&ci.io->io_count, 1);
	ci.io->bio = bio;
	ci.io->md = md;
	ci.sector = bio->bi_sector;
	ci.sector_count = bio_sectors(bio);
	ci.idx = bio->bi_idx;

820
	start_io_acct(ci.io);
821 822
	while (ci.sector_count && !error)
		error = __clone_and_map(&ci);
L
Linus Torvalds 已提交
823 824

	/* drop the extra reference count */
825
	dec_pending(ci.io, error);
L
Linus Torvalds 已提交
826
	dm_table_put(ci.map);
M
Milan Broz 已提交
827 828

	return 0;
L
Linus Torvalds 已提交
829 830 831 832 833
}
/*-----------------------------------------------------------------
 * CRUD END
 *---------------------------------------------------------------*/

M
Milan Broz 已提交
834 835 836 837 838 839 840 841
static int dm_merge_bvec(struct request_queue *q,
			 struct bvec_merge_data *bvm,
			 struct bio_vec *biovec)
{
	struct mapped_device *md = q->queuedata;
	struct dm_table *map = dm_get_table(md);
	struct dm_target *ti;
	sector_t max_sectors;
842
	int max_size = 0;
M
Milan Broz 已提交
843 844

	if (unlikely(!map))
845
		goto out;
M
Milan Broz 已提交
846 847

	ti = dm_table_find_target(map, bvm->bi_sector);
848 849
	if (!dm_target_is_valid(ti))
		goto out_table;
M
Milan Broz 已提交
850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867

	/*
	 * Find maximum amount of I/O that won't need splitting
	 */
	max_sectors = min(max_io_len(md, bvm->bi_sector, ti),
			  (sector_t) BIO_MAX_SECTORS);
	max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size;
	if (max_size < 0)
		max_size = 0;

	/*
	 * merge_bvec_fn() returns number of bytes
	 * it can accept at this offset
	 * max is precomputed maximal io size
	 */
	if (max_size && ti->type->merge)
		max_size = ti->type->merge(ti, bvm, biovec, max_size);

868
out_table:
869 870 871
	dm_table_put(map);

out:
M
Milan Broz 已提交
872 873 874 875 876 877 878 879 880
	/*
	 * Always allow an entire first page
	 */
	if (max_size <= biovec->bv_len && !(bvm->bi_size >> SECTOR_SHIFT))
		max_size = biovec->bv_len;

	return max_size;
}

L
Linus Torvalds 已提交
881 882 883 884
/*
 * The request function that just remaps the bio built up by
 * dm_merge_bvec.
 */
885
static int dm_request(struct request_queue *q, struct bio *bio)
L
Linus Torvalds 已提交
886
{
M
Milan Broz 已提交
887
	int r = -EIO;
888
	int rw = bio_data_dir(bio);
L
Linus Torvalds 已提交
889
	struct mapped_device *md = q->queuedata;
T
Tejun Heo 已提交
890
	int cpu;
L
Linus Torvalds 已提交
891

S
Stefan Bader 已提交
892 893 894 895 896
	/*
	 * There is no use in forwarding any barrier request since we can't
	 * guarantee it is (or can be) handled by the targets correctly.
	 */
	if (unlikely(bio_barrier(bio))) {
897
		bio_endio(bio, -EOPNOTSUPP);
S
Stefan Bader 已提交
898 899 900
		return 0;
	}

901
	down_read(&md->io_lock);
L
Linus Torvalds 已提交
902

T
Tejun Heo 已提交
903 904 905 906
	cpu = part_stat_lock();
	part_stat_inc(cpu, &dm_disk(md)->part0, ios[rw]);
	part_stat_add(cpu, &dm_disk(md)->part0, sectors[rw], bio_sectors(bio));
	part_stat_unlock();
907

L
Linus Torvalds 已提交
908 909 910 911 912
	/*
	 * If we're suspended we have to queue
	 * this io for later.
	 */
	while (test_bit(DMF_BLOCK_IO, &md->flags)) {
913
		up_read(&md->io_lock);
L
Linus Torvalds 已提交
914

M
Milan Broz 已提交
915 916
		if (bio_rw(bio) != READA)
			r = queue_io(md, bio);
L
Linus Torvalds 已提交
917

M
Milan Broz 已提交
918 919
		if (r <= 0)
			goto out_req;
L
Linus Torvalds 已提交
920 921 922 923 924

		/*
		 * We're in a while loop, because someone could suspend
		 * before we get to the following read lock.
		 */
925
		down_read(&md->io_lock);
L
Linus Torvalds 已提交
926 927
	}

M
Milan Broz 已提交
928
	r = __split_bio(md, bio);
929
	up_read(&md->io_lock);
M
Milan Broz 已提交
930 931 932 933 934

out_req:
	if (r < 0)
		bio_io_error(bio);

L
Linus Torvalds 已提交
935 936 937
	return 0;
}

938
static void dm_unplug_all(struct request_queue *q)
L
Linus Torvalds 已提交
939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968
{
	struct mapped_device *md = q->queuedata;
	struct dm_table *map = dm_get_table(md);

	if (map) {
		dm_table_unplug_all(map);
		dm_table_put(map);
	}
}

static int dm_any_congested(void *congested_data, int bdi_bits)
{
	int r;
	struct mapped_device *md = (struct mapped_device *) congested_data;
	struct dm_table *map = dm_get_table(md);

	if (!map || test_bit(DMF_BLOCK_IO, &md->flags))
		r = bdi_bits;
	else
		r = dm_table_any_congested(map, bdi_bits);

	dm_table_put(map);
	return r;
}

/*-----------------------------------------------------------------
 * An IDR is used to keep track of allocated minor numbers.
 *---------------------------------------------------------------*/
static DEFINE_IDR(_minor_idr);

969
static void free_minor(int minor)
L
Linus Torvalds 已提交
970
{
971
	spin_lock(&_minor_lock);
L
Linus Torvalds 已提交
972
	idr_remove(&_minor_idr, minor);
973
	spin_unlock(&_minor_lock);
L
Linus Torvalds 已提交
974 975 976 977 978
}

/*
 * See if the device with a specific minor # is free.
 */
979
static int specific_minor(int minor)
L
Linus Torvalds 已提交
980 981 982 983 984 985
{
	int r, m;

	if (minor >= (1 << MINORBITS))
		return -EINVAL;

J
Jeff Mahoney 已提交
986 987 988 989
	r = idr_pre_get(&_minor_idr, GFP_KERNEL);
	if (!r)
		return -ENOMEM;

990
	spin_lock(&_minor_lock);
L
Linus Torvalds 已提交
991 992 993 994 995 996

	if (idr_find(&_minor_idr, minor)) {
		r = -EBUSY;
		goto out;
	}

997
	r = idr_get_new_above(&_minor_idr, MINOR_ALLOCED, minor, &m);
J
Jeff Mahoney 已提交
998
	if (r)
L
Linus Torvalds 已提交
999 1000 1001 1002 1003 1004 1005 1006 1007
		goto out;

	if (m != minor) {
		idr_remove(&_minor_idr, m);
		r = -EBUSY;
		goto out;
	}

out:
1008
	spin_unlock(&_minor_lock);
L
Linus Torvalds 已提交
1009 1010 1011
	return r;
}

1012
static int next_free_minor(int *minor)
L
Linus Torvalds 已提交
1013
{
1014
	int r, m;
L
Linus Torvalds 已提交
1015 1016

	r = idr_pre_get(&_minor_idr, GFP_KERNEL);
J
Jeff Mahoney 已提交
1017 1018 1019
	if (!r)
		return -ENOMEM;

1020
	spin_lock(&_minor_lock);
L
Linus Torvalds 已提交
1021

1022
	r = idr_get_new(&_minor_idr, MINOR_ALLOCED, &m);
1023
	if (r)
L
Linus Torvalds 已提交
1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034
		goto out;

	if (m >= (1 << MINORBITS)) {
		idr_remove(&_minor_idr, m);
		r = -ENOSPC;
		goto out;
	}

	*minor = m;

out:
1035
	spin_unlock(&_minor_lock);
L
Linus Torvalds 已提交
1036 1037 1038 1039 1040 1041 1042 1043
	return r;
}

static struct block_device_operations dm_blk_dops;

/*
 * Allocate and initialise a blank device with a given minor.
 */
1044
static struct mapped_device *alloc_dev(int minor)
L
Linus Torvalds 已提交
1045 1046
{
	int r;
1047
	struct mapped_device *md = kzalloc(sizeof(*md), GFP_KERNEL);
1048
	void *old_md;
L
Linus Torvalds 已提交
1049 1050 1051 1052 1053 1054

	if (!md) {
		DMWARN("unable to allocate device, out of memory.");
		return NULL;
	}

1055
	if (!try_module_get(THIS_MODULE))
M
Milan Broz 已提交
1056
		goto bad_module_get;
1057

L
Linus Torvalds 已提交
1058
	/* get a minor number for the dev */
1059
	if (minor == DM_ANY_MINOR)
1060
		r = next_free_minor(&minor);
1061
	else
1062
		r = specific_minor(minor);
L
Linus Torvalds 已提交
1063
	if (r < 0)
M
Milan Broz 已提交
1064
		goto bad_minor;
L
Linus Torvalds 已提交
1065

1066
	init_rwsem(&md->io_lock);
1067
	mutex_init(&md->suspend_lock);
1068
	spin_lock_init(&md->pushback_lock);
L
Linus Torvalds 已提交
1069 1070
	rwlock_init(&md->map_lock);
	atomic_set(&md->holders, 1);
1071
	atomic_set(&md->open_count, 0);
L
Linus Torvalds 已提交
1072
	atomic_set(&md->event_nr, 0);
M
Mike Anderson 已提交
1073 1074 1075
	atomic_set(&md->uevent_seq, 0);
	INIT_LIST_HEAD(&md->uevent_list);
	spin_lock_init(&md->uevent_lock);
L
Linus Torvalds 已提交
1076 1077 1078

	md->queue = blk_alloc_queue(GFP_KERNEL);
	if (!md->queue)
M
Milan Broz 已提交
1079
		goto bad_queue;
L
Linus Torvalds 已提交
1080 1081 1082 1083 1084

	md->queue->queuedata = md;
	md->queue->backing_dev_info.congested_fn = dm_any_congested;
	md->queue->backing_dev_info.congested_data = md;
	blk_queue_make_request(md->queue, dm_request);
1085
	blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);
L
Linus Torvalds 已提交
1086
	md->queue->unplug_fn = dm_unplug_all;
M
Milan Broz 已提交
1087
	blk_queue_merge_bvec(md->queue, dm_merge_bvec);
L
Linus Torvalds 已提交
1088

1089
	md->io_pool = mempool_create_slab_pool(MIN_IOS, _io_cache);
K
Kiyoshi Ueda 已提交
1090
	if (!md->io_pool)
M
Milan Broz 已提交
1091
		goto bad_io_pool;
L
Linus Torvalds 已提交
1092

1093
	md->tio_pool = mempool_create_slab_pool(MIN_IOS, _tio_cache);
L
Linus Torvalds 已提交
1094
	if (!md->tio_pool)
M
Milan Broz 已提交
1095
		goto bad_tio_pool;
L
Linus Torvalds 已提交
1096

1097
	md->bs = bioset_create(16, 16);
S
Stefan Bader 已提交
1098 1099 1100
	if (!md->bs)
		goto bad_no_bioset;

L
Linus Torvalds 已提交
1101 1102
	md->disk = alloc_disk(1);
	if (!md->disk)
M
Milan Broz 已提交
1103
		goto bad_disk;
L
Linus Torvalds 已提交
1104

1105 1106 1107 1108
	atomic_set(&md->pending, 0);
	init_waitqueue_head(&md->wait);
	init_waitqueue_head(&md->eventq);

L
Linus Torvalds 已提交
1109 1110 1111 1112 1113 1114 1115
	md->disk->major = _major;
	md->disk->first_minor = minor;
	md->disk->fops = &dm_blk_dops;
	md->disk->queue = md->queue;
	md->disk->private_data = md;
	sprintf(md->disk->disk_name, "dm-%d", minor);
	add_disk(md->disk);
M
Mike Anderson 已提交
1116
	format_dev_t(md->name, MKDEV(_major, minor));
L
Linus Torvalds 已提交
1117

1118 1119 1120 1121
	md->wq = create_singlethread_workqueue("kdmflush");
	if (!md->wq)
		goto bad_thread;

1122
	/* Populate the mapping, nobody knows we exist yet */
1123
	spin_lock(&_minor_lock);
1124
	old_md = idr_replace(&_minor_idr, md, minor);
1125
	spin_unlock(&_minor_lock);
1126 1127 1128

	BUG_ON(old_md != MINOR_ALLOCED);

L
Linus Torvalds 已提交
1129 1130
	return md;

1131 1132
bad_thread:
	put_disk(md->disk);
M
Milan Broz 已提交
1133
bad_disk:
S
Stefan Bader 已提交
1134
	bioset_free(md->bs);
M
Milan Broz 已提交
1135
bad_no_bioset:
L
Linus Torvalds 已提交
1136
	mempool_destroy(md->tio_pool);
M
Milan Broz 已提交
1137
bad_tio_pool:
L
Linus Torvalds 已提交
1138
	mempool_destroy(md->io_pool);
M
Milan Broz 已提交
1139
bad_io_pool:
1140
	blk_cleanup_queue(md->queue);
M
Milan Broz 已提交
1141
bad_queue:
L
Linus Torvalds 已提交
1142
	free_minor(minor);
M
Milan Broz 已提交
1143
bad_minor:
1144
	module_put(THIS_MODULE);
M
Milan Broz 已提交
1145
bad_module_get:
L
Linus Torvalds 已提交
1146 1147 1148 1149
	kfree(md);
	return NULL;
}

J
Jun'ichi Nomura 已提交
1150 1151
static void unlock_fs(struct mapped_device *md);

L
Linus Torvalds 已提交
1152 1153
static void free_dev(struct mapped_device *md)
{
1154
	int minor = MINOR(disk_devt(md->disk));
1155

1156
	if (md->suspended_bdev) {
J
Jun'ichi Nomura 已提交
1157
		unlock_fs(md);
1158 1159
		bdput(md->suspended_bdev);
	}
1160
	destroy_workqueue(md->wq);
L
Linus Torvalds 已提交
1161 1162
	mempool_destroy(md->tio_pool);
	mempool_destroy(md->io_pool);
S
Stefan Bader 已提交
1163
	bioset_free(md->bs);
L
Linus Torvalds 已提交
1164
	del_gendisk(md->disk);
1165
	free_minor(minor);
J
Jeff Mahoney 已提交
1166 1167 1168 1169 1170

	spin_lock(&_minor_lock);
	md->disk->private_data = NULL;
	spin_unlock(&_minor_lock);

L
Linus Torvalds 已提交
1171
	put_disk(md->disk);
1172
	blk_cleanup_queue(md->queue);
1173
	module_put(THIS_MODULE);
L
Linus Torvalds 已提交
1174 1175 1176 1177 1178 1179 1180 1181
	kfree(md);
}

/*
 * Bind a table to the device.
 */
static void event_callback(void *context)
{
M
Mike Anderson 已提交
1182 1183
	unsigned long flags;
	LIST_HEAD(uevents);
L
Linus Torvalds 已提交
1184 1185
	struct mapped_device *md = (struct mapped_device *) context;

M
Mike Anderson 已提交
1186 1187 1188 1189
	spin_lock_irqsave(&md->uevent_lock, flags);
	list_splice_init(&md->uevent_list, &uevents);
	spin_unlock_irqrestore(&md->uevent_lock, flags);

1190
	dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
M
Mike Anderson 已提交
1191

L
Linus Torvalds 已提交
1192 1193 1194 1195
	atomic_inc(&md->event_nr);
	wake_up(&md->eventq);
}

1196
static void __set_size(struct mapped_device *md, sector_t size)
L
Linus Torvalds 已提交
1197
{
1198
	set_capacity(md->disk, size);
L
Linus Torvalds 已提交
1199

1200
	mutex_lock(&md->suspended_bdev->bd_inode->i_mutex);
1201
	i_size_write(md->suspended_bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
1202
	mutex_unlock(&md->suspended_bdev->bd_inode->i_mutex);
L
Linus Torvalds 已提交
1203 1204 1205 1206
}

static int __bind(struct mapped_device *md, struct dm_table *t)
{
1207
	struct request_queue *q = md->queue;
L
Linus Torvalds 已提交
1208 1209 1210
	sector_t size;

	size = dm_table_get_size(t);
D
Darrick J. Wong 已提交
1211 1212 1213 1214 1215 1216 1217

	/*
	 * Wipe any geometry if the size of the table changed.
	 */
	if (size != get_capacity(md->disk))
		memset(&md->geometry, 0, sizeof(md->geometry));

1218 1219
	if (md->suspended_bdev)
		__set_size(md, size);
L
Linus Torvalds 已提交
1220 1221 1222
	if (size == 0)
		return 0;

1223 1224 1225
	dm_table_get(t);
	dm_table_event_callback(t, event_callback, md);

L
Linus Torvalds 已提交
1226 1227
	write_lock(&md->map_lock);
	md->map = t;
1228
	dm_table_set_restrictions(t, q);
L
Linus Torvalds 已提交
1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250
	write_unlock(&md->map_lock);

	return 0;
}

static void __unbind(struct mapped_device *md)
{
	struct dm_table *map = md->map;

	if (!map)
		return;

	dm_table_event_callback(map, NULL, NULL);
	write_lock(&md->map_lock);
	md->map = NULL;
	write_unlock(&md->map_lock);
	dm_table_put(map);
}

/*
 * Constructor for a new device.
 */
1251
int dm_create(int minor, struct mapped_device **result)
L
Linus Torvalds 已提交
1252 1253 1254
{
	struct mapped_device *md;

1255
	md = alloc_dev(minor);
L
Linus Torvalds 已提交
1256 1257 1258 1259 1260 1261 1262
	if (!md)
		return -ENXIO;

	*result = md;
	return 0;
}

1263
static struct mapped_device *dm_find_md(dev_t dev)
L
Linus Torvalds 已提交
1264 1265 1266 1267 1268 1269 1270
{
	struct mapped_device *md;
	unsigned minor = MINOR(dev);

	if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
		return NULL;

1271
	spin_lock(&_minor_lock);
L
Linus Torvalds 已提交
1272 1273

	md = idr_find(&_minor_idr, minor);
J
Jeff Mahoney 已提交
1274
	if (md && (md == MINOR_ALLOCED ||
1275
		   (MINOR(disk_devt(dm_disk(md))) != minor) ||
A
Alasdair G Kergon 已提交
1276
		   test_bit(DMF_FREEING, &md->flags))) {
1277
		md = NULL;
J
Jeff Mahoney 已提交
1278 1279
		goto out;
	}
L
Linus Torvalds 已提交
1280

J
Jeff Mahoney 已提交
1281
out:
1282
	spin_unlock(&_minor_lock);
L
Linus Torvalds 已提交
1283

1284 1285 1286
	return md;
}

1287 1288 1289 1290 1291 1292 1293 1294 1295 1296
struct mapped_device *dm_get_md(dev_t dev)
{
	struct mapped_device *md = dm_find_md(dev);

	if (md)
		dm_get(md);

	return md;
}

A
Alasdair G Kergon 已提交
1297
void *dm_get_mdptr(struct mapped_device *md)
1298
{
A
Alasdair G Kergon 已提交
1299
	return md->interface_ptr;
L
Linus Torvalds 已提交
1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311
}

void dm_set_mdptr(struct mapped_device *md, void *ptr)
{
	md->interface_ptr = ptr;
}

void dm_get(struct mapped_device *md)
{
	atomic_inc(&md->holders);
}

1312 1313 1314 1315 1316 1317
const char *dm_device_name(struct mapped_device *md)
{
	return md->name;
}
EXPORT_SYMBOL_GPL(dm_device_name);

L
Linus Torvalds 已提交
1318 1319
void dm_put(struct mapped_device *md)
{
M
Mike Anderson 已提交
1320
	struct dm_table *map;
L
Linus Torvalds 已提交
1321

J
Jeff Mahoney 已提交
1322 1323
	BUG_ON(test_bit(DMF_FREEING, &md->flags));

1324
	if (atomic_dec_and_lock(&md->holders, &_minor_lock)) {
M
Mike Anderson 已提交
1325
		map = dm_get_table(md);
1326 1327
		idr_replace(&_minor_idr, MINOR_ALLOCED,
			    MINOR(disk_devt(dm_disk(md))));
J
Jeff Mahoney 已提交
1328
		set_bit(DMF_FREEING, &md->flags);
1329
		spin_unlock(&_minor_lock);
1330
		if (!dm_suspended(md)) {
L
Linus Torvalds 已提交
1331 1332 1333 1334
			dm_table_presuspend_targets(map);
			dm_table_postsuspend_targets(map);
		}
		__unbind(md);
M
Mike Anderson 已提交
1335
		dm_table_put(map);
L
Linus Torvalds 已提交
1336 1337 1338
		free_dev(md);
	}
}
E
Edward Goggin 已提交
1339
EXPORT_SYMBOL_GPL(dm_put);
L
Linus Torvalds 已提交
1340

1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363
static int dm_wait_for_completion(struct mapped_device *md)
{
	int r = 0;

	while (1) {
		set_current_state(TASK_INTERRUPTIBLE);

		smp_mb();
		if (!atomic_read(&md->pending))
			break;

		if (signal_pending(current)) {
			r = -EINTR;
			break;
		}

		io_schedule();
	}
	set_current_state(TASK_RUNNING);

	return r;
}

L
Linus Torvalds 已提交
1364 1365 1366
/*
 * Process the deferred bios
 */
1367
static void __flush_deferred_io(struct mapped_device *md)
L
Linus Torvalds 已提交
1368
{
1369
	struct bio *c;
L
Linus Torvalds 已提交
1370

1371
	while ((c = bio_list_pop(&md->deferred))) {
M
Milan Broz 已提交
1372 1373
		if (__split_bio(md, c))
			bio_io_error(c);
L
Linus Torvalds 已提交
1374
	}
M
Milan Broz 已提交
1375 1376

	clear_bit(DMF_BLOCK_IO, &md->flags);
L
Linus Torvalds 已提交
1377 1378
}

1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389
static void __merge_pushback_list(struct mapped_device *md)
{
	unsigned long flags;

	spin_lock_irqsave(&md->pushback_lock, flags);
	clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
	bio_list_merge_head(&md->deferred, &md->pushback);
	bio_list_init(&md->pushback);
	spin_unlock_irqrestore(&md->pushback_lock, flags);
}

1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427
static void dm_wq_work(struct work_struct *work)
{
	struct dm_wq_req *req = container_of(work, struct dm_wq_req, work);
	struct mapped_device *md = req->md;

	down_write(&md->io_lock);
	switch (req->type) {
	case DM_WQ_FLUSH_ALL:
		__merge_pushback_list(md);
		/* pass through */
	case DM_WQ_FLUSH_DEFERRED:
		__flush_deferred_io(md);
		break;
	default:
		DMERR("dm_wq_work: unrecognised work type %d", req->type);
		BUG();
	}
	up_write(&md->io_lock);
}

static void dm_wq_queue(struct mapped_device *md, int type, void *context,
			struct dm_wq_req *req)
{
	req->type = type;
	req->md = md;
	req->context = context;
	INIT_WORK(&req->work, dm_wq_work);
	queue_work(md->wq, &req->work);
}

static void dm_queue_flush(struct mapped_device *md, int type, void *context)
{
	struct dm_wq_req req;

	dm_wq_queue(md, type, context, &req);
	flush_workqueue(md->wq);
}

L
Linus Torvalds 已提交
1428 1429 1430 1431 1432
/*
 * Swap in a new table (destroying old one).
 */
int dm_swap_table(struct mapped_device *md, struct dm_table *table)
{
1433
	int r = -EINVAL;
L
Linus Torvalds 已提交
1434

1435
	mutex_lock(&md->suspend_lock);
L
Linus Torvalds 已提交
1436 1437

	/* device must be suspended */
1438
	if (!dm_suspended(md))
1439
		goto out;
L
Linus Torvalds 已提交
1440

1441 1442 1443 1444 1445
	/* without bdev, the device size cannot be changed */
	if (!md->suspended_bdev)
		if (get_capacity(md->disk) != dm_table_get_size(table))
			goto out;

L
Linus Torvalds 已提交
1446 1447 1448
	__unbind(md);
	r = __bind(md, table);

1449
out:
1450
	mutex_unlock(&md->suspend_lock);
1451
	return r;
L
Linus Torvalds 已提交
1452 1453 1454 1455 1456 1457
}

/*
 * Functions to lock and unlock any filesystem running on the
 * device.
 */
1458
static int lock_fs(struct mapped_device *md)
L
Linus Torvalds 已提交
1459
{
1460
	int r;
L
Linus Torvalds 已提交
1461 1462

	WARN_ON(md->frozen_sb);
1463

1464
	md->frozen_sb = freeze_bdev(md->suspended_bdev);
1465
	if (IS_ERR(md->frozen_sb)) {
1466
		r = PTR_ERR(md->frozen_sb);
1467 1468
		md->frozen_sb = NULL;
		return r;
1469 1470
	}

1471 1472
	set_bit(DMF_FROZEN, &md->flags);

L
Linus Torvalds 已提交
1473
	/* don't bdput right now, we don't want the bdev
1474
	 * to go away while it is locked.
L
Linus Torvalds 已提交
1475 1476 1477 1478
	 */
	return 0;
}

1479
static void unlock_fs(struct mapped_device *md)
L
Linus Torvalds 已提交
1480
{
1481 1482 1483
	if (!test_bit(DMF_FROZEN, &md->flags))
		return;

1484
	thaw_bdev(md->suspended_bdev, md->frozen_sb);
L
Linus Torvalds 已提交
1485
	md->frozen_sb = NULL;
1486
	clear_bit(DMF_FROZEN, &md->flags);
L
Linus Torvalds 已提交
1487 1488 1489 1490 1491 1492 1493 1494 1495
}

/*
 * We need to be able to change a mapping table under a mounted
 * filesystem.  For example we might want to move some data in
 * the background.  Before the table can be swapped with
 * dm_bind_table, dm_suspend must be called to flush any in
 * flight bios and ensure that any further io gets deferred.
 */
1496
int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
L
Linus Torvalds 已提交
1497
{
1498
	struct dm_table *map = NULL;
L
Linus Torvalds 已提交
1499
	DECLARE_WAITQUEUE(wait, current);
1500
	int r = 0;
1501
	int do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG ? 1 : 0;
1502
	int noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG ? 1 : 0;
L
Linus Torvalds 已提交
1503

1504
	mutex_lock(&md->suspend_lock);
1505

M
Milan Broz 已提交
1506 1507
	if (dm_suspended(md)) {
		r = -EINVAL;
1508
		goto out_unlock;
M
Milan Broz 已提交
1509
	}
L
Linus Torvalds 已提交
1510 1511 1512

	map = dm_get_table(md);

1513 1514 1515 1516 1517 1518 1519
	/*
	 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
	 * This flag is cleared before dm_suspend returns.
	 */
	if (noflush)
		set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);

1520 1521 1522
	/* This does not get reverted if there's an error later. */
	dm_table_presuspend_targets(map);

1523 1524 1525 1526 1527 1528 1529 1530
	/* bdget() can stall if the pending I/Os are not flushed */
	if (!noflush) {
		md->suspended_bdev = bdget_disk(md->disk, 0);
		if (!md->suspended_bdev) {
			DMWARN("bdget failed in dm_suspend");
			r = -ENOMEM;
			goto flush_and_out;
		}
1531

1532 1533 1534 1535 1536 1537 1538 1539 1540
		/*
		 * Flush I/O to the device. noflush supersedes do_lockfs,
		 * because lock_fs() needs to flush I/Os.
		 */
		if (do_lockfs) {
			r = lock_fs(md);
			if (r)
				goto out;
		}
1541
	}
L
Linus Torvalds 已提交
1542 1543

	/*
1544
	 * First we set the BLOCK_IO flag so no more ios will be mapped.
L
Linus Torvalds 已提交
1545
	 */
1546 1547
	down_write(&md->io_lock);
	set_bit(DMF_BLOCK_IO, &md->flags);
L
Linus Torvalds 已提交
1548 1549

	add_wait_queue(&md->wait, &wait);
1550
	up_write(&md->io_lock);
L
Linus Torvalds 已提交
1551 1552

	/* unplug */
1553
	if (map)
L
Linus Torvalds 已提交
1554 1555 1556
		dm_table_unplug_all(map);

	/*
1557
	 * Wait for the already-mapped ios to complete.
L
Linus Torvalds 已提交
1558
	 */
1559
	r = dm_wait_for_completion(md);
L
Linus Torvalds 已提交
1560

1561
	down_write(&md->io_lock);
L
Linus Torvalds 已提交
1562 1563
	remove_wait_queue(&md->wait, &wait);

1564 1565
	if (noflush)
		__merge_pushback_list(md);
1566
	up_write(&md->io_lock);
1567

L
Linus Torvalds 已提交
1568
	/* were we interrupted ? */
1569
	if (r < 0) {
1570
		dm_queue_flush(md, DM_WQ_FLUSH_DEFERRED, NULL);
M
Milan Broz 已提交
1571

1572
		unlock_fs(md);
1573
		goto out; /* pushback list is already flushed, so skip flush */
1574
	}
L
Linus Torvalds 已提交
1575

1576
	dm_table_postsuspend_targets(map);
L
Linus Torvalds 已提交
1577

1578
	set_bit(DMF_SUSPENDED, &md->flags);
1579

1580
flush_and_out:
1581
	if (r && noflush)
1582 1583 1584 1585
		/*
		 * Because there may be already I/Os in the pushback list,
		 * flush them before return.
		 */
1586
		dm_queue_flush(md, DM_WQ_FLUSH_ALL, NULL);
1587

1588
out:
1589 1590 1591 1592 1593
	if (r && md->suspended_bdev) {
		bdput(md->suspended_bdev);
		md->suspended_bdev = NULL;
	}

1594
	dm_table_put(map);
1595 1596

out_unlock:
1597
	mutex_unlock(&md->suspend_lock);
1598
	return r;
L
Linus Torvalds 已提交
1599 1600 1601 1602
}

int dm_resume(struct mapped_device *md)
{
1603 1604
	int r = -EINVAL;
	struct dm_table *map = NULL;
L
Linus Torvalds 已提交
1605

1606
	mutex_lock(&md->suspend_lock);
1607
	if (!dm_suspended(md))
1608 1609 1610
		goto out;

	map = dm_get_table(md);
1611
	if (!map || !dm_table_get_size(map))
1612
		goto out;
L
Linus Torvalds 已提交
1613

1614 1615 1616
	r = dm_table_resume_targets(map);
	if (r)
		goto out;
1617

1618
	dm_queue_flush(md, DM_WQ_FLUSH_DEFERRED, NULL);
1619 1620 1621

	unlock_fs(md);

1622 1623 1624 1625
	if (md->suspended_bdev) {
		bdput(md->suspended_bdev);
		md->suspended_bdev = NULL;
	}
1626

1627 1628
	clear_bit(DMF_SUSPENDED, &md->flags);

L
Linus Torvalds 已提交
1629 1630
	dm_table_unplug_all(map);

1631
	dm_kobject_uevent(md);
1632

1633
	r = 0;
1634

1635 1636
out:
	dm_table_put(map);
1637
	mutex_unlock(&md->suspend_lock);
1638

1639
	return r;
L
Linus Torvalds 已提交
1640 1641 1642 1643 1644
}

/*-----------------------------------------------------------------
 * Event notification.
 *---------------------------------------------------------------*/
1645 1646
void dm_kobject_uevent(struct mapped_device *md)
{
1647
	kobject_uevent(&disk_to_dev(md->disk)->kobj, KOBJ_CHANGE);
1648 1649
}

M
Mike Anderson 已提交
1650 1651 1652 1653 1654
uint32_t dm_next_uevent_seq(struct mapped_device *md)
{
	return atomic_add_return(1, &md->uevent_seq);
}

L
Linus Torvalds 已提交
1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665
uint32_t dm_get_event_nr(struct mapped_device *md)
{
	return atomic_read(&md->event_nr);
}

int dm_wait_event(struct mapped_device *md, int event_nr)
{
	return wait_event_interruptible(md->eventq,
			(event_nr != atomic_read(&md->event_nr)));
}

M
Mike Anderson 已提交
1666 1667 1668 1669 1670 1671 1672 1673 1674
void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
{
	unsigned long flags;

	spin_lock_irqsave(&md->uevent_lock, flags);
	list_add(elist, &md->uevent_list);
	spin_unlock_irqrestore(&md->uevent_lock, flags);
}

L
Linus Torvalds 已提交
1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688
/*
 * The gendisk is only valid as long as you have a reference
 * count on 'md'.
 */
struct gendisk *dm_disk(struct mapped_device *md)
{
	return md->disk;
}

int dm_suspended(struct mapped_device *md)
{
	return test_bit(DMF_SUSPENDED, &md->flags);
}

1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699
int dm_noflush_suspending(struct dm_target *ti)
{
	struct mapped_device *md = dm_table_get_md(ti->table);
	int r = __noflush_suspending(md);

	dm_put(md);

	return r;
}
EXPORT_SYMBOL_GPL(dm_noflush_suspending);

L
Linus Torvalds 已提交
1700 1701 1702
static struct block_device_operations dm_blk_dops = {
	.open = dm_blk_open,
	.release = dm_blk_close,
1703
	.ioctl = dm_blk_ioctl,
D
Darrick J. Wong 已提交
1704
	.getgeo = dm_blk_getgeo,
L
Linus Torvalds 已提交
1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720
	.owner = THIS_MODULE
};

EXPORT_SYMBOL(dm_get_mapinfo);

/*
 * module hooks
 */
module_init(dm_init);
module_exit(dm_exit);

module_param(major, uint, 0);
MODULE_PARM_DESC(major, "The major number of the device mapper");
MODULE_DESCRIPTION(DM_NAME " driver");
MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
MODULE_LICENSE("GPL");