dm.c 35.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2
/*
 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
M
Milan Broz 已提交
3
 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
L
Linus Torvalds 已提交
4 5 6 7 8 9
 *
 * This file is released under the GPL.
 */

#include "dm.h"
#include "dm-bio-list.h"
M
Mike Anderson 已提交
10
#include "dm-uevent.h"
L
Linus Torvalds 已提交
11 12 13

#include <linux/init.h>
#include <linux/module.h>
A
Arjan van de Ven 已提交
14
#include <linux/mutex.h>
L
Linus Torvalds 已提交
15 16 17 18 19 20 21
#include <linux/moduleparam.h>
#include <linux/blkpg.h>
#include <linux/bio.h>
#include <linux/buffer_head.h>
#include <linux/mempool.h>
#include <linux/slab.h>
#include <linux/idr.h>
D
Darrick J. Wong 已提交
22
#include <linux/hdreg.h>
23
#include <linux/blktrace_api.h>
24
#include <trace/block.h>
L
Linus Torvalds 已提交
25

26 27
#define DM_MSG_PREFIX "core"

L
Linus Torvalds 已提交
28 29 30 31 32
static const char *_name = DM_NAME;

static unsigned int major = 0;
static unsigned int _major = 0;

33
static DEFINE_SPINLOCK(_minor_lock);
L
Linus Torvalds 已提交
34
/*
K
Kiyoshi Ueda 已提交
35
 * For bio-based dm.
L
Linus Torvalds 已提交
36 37 38 39 40 41
 * One of these is allocated per bio.
 */
struct dm_io {
	struct mapped_device *md;
	int error;
	atomic_t io_count;
R
Richard Kennedy 已提交
42
	struct bio *bio;
43
	unsigned long start_time;
L
Linus Torvalds 已提交
44 45 46
};

/*
K
Kiyoshi Ueda 已提交
47
 * For bio-based dm.
L
Linus Torvalds 已提交
48 49 50
 * One of these is allocated per target within a bio.  Hopefully
 * this will be simplified out one day.
 */
A
Alasdair G Kergon 已提交
51
struct dm_target_io {
L
Linus Torvalds 已提交
52 53 54 55 56
	struct dm_io *io;
	struct dm_target *ti;
	union map_info info;
};

57 58
DEFINE_TRACE(block_bio_complete);

K
Kiyoshi Ueda 已提交
59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79
/*
 * For request-based dm.
 * One of these is allocated per request.
 */
struct dm_rq_target_io {
	struct mapped_device *md;
	struct dm_target *ti;
	struct request *orig, clone;
	int error;
	union map_info info;
};

/*
 * For request-based dm.
 * One of these is allocated per bio.
 */
struct dm_rq_clone_bio_info {
	struct bio *orig;
	struct request *rq;
};

L
Linus Torvalds 已提交
80 81
union map_info *dm_get_mapinfo(struct bio *bio)
{
A
Alasdair G Kergon 已提交
82
	if (bio && bio->bi_private)
A
Alasdair G Kergon 已提交
83
		return &((struct dm_target_io *)bio->bi_private)->info;
A
Alasdair G Kergon 已提交
84
	return NULL;
L
Linus Torvalds 已提交
85 86
}

87 88
#define MINOR_ALLOCED ((void *)-1)

L
Linus Torvalds 已提交
89 90 91 92 93
/*
 * Bits for the md->flags field.
 */
#define DMF_BLOCK_IO 0
#define DMF_SUSPENDED 1
94
#define DMF_FROZEN 2
J
Jeff Mahoney 已提交
95
#define DMF_FREEING 3
96
#define DMF_DELETING 4
97
#define DMF_NOFLUSH_SUSPENDING 5
L
Linus Torvalds 已提交
98

99 100 101
/*
 * Work processed by per-device workqueue.
 */
L
Linus Torvalds 已提交
102
struct mapped_device {
103
	struct rw_semaphore io_lock;
104
	struct mutex suspend_lock;
105
	spinlock_t pushback_lock;
L
Linus Torvalds 已提交
106 107
	rwlock_t map_lock;
	atomic_t holders;
108
	atomic_t open_count;
L
Linus Torvalds 已提交
109 110 111

	unsigned long flags;

112
	struct request_queue *queue;
L
Linus Torvalds 已提交
113
	struct gendisk *disk;
M
Mike Anderson 已提交
114
	char name[16];
L
Linus Torvalds 已提交
115 116 117 118 119 120 121 122

	void *interface_ptr;

	/*
	 * A list of ios that arrived while we were suspended.
	 */
	atomic_t pending;
	wait_queue_head_t wait;
123
	struct work_struct work;
K
Kiyoshi Ueda 已提交
124
	struct bio_list deferred;
125
	struct bio_list pushback;
L
Linus Torvalds 已提交
126

127 128 129 130 131
	/*
	 * Processing queue (flush/barriers)
	 */
	struct workqueue_struct *wq;

L
Linus Torvalds 已提交
132 133 134 135 136 137 138 139 140 141 142
	/*
	 * The current mapping.
	 */
	struct dm_table *map;

	/*
	 * io objects are allocated from here.
	 */
	mempool_t *io_pool;
	mempool_t *tio_pool;

S
Stefan Bader 已提交
143 144
	struct bio_set *bs;

L
Linus Torvalds 已提交
145 146 147 148 149
	/*
	 * Event handling.
	 */
	atomic_t event_nr;
	wait_queue_head_t eventq;
M
Mike Anderson 已提交
150 151 152
	atomic_t uevent_seq;
	struct list_head uevent_list;
	spinlock_t uevent_lock; /* Protect access to uevent_list */
L
Linus Torvalds 已提交
153 154 155 156 157

	/*
	 * freeze/thaw support require holding onto a super block
	 */
	struct super_block *frozen_sb;
158
	struct block_device *suspended_bdev;
D
Darrick J. Wong 已提交
159 160 161

	/* forced geometry settings */
	struct hd_geometry geometry;
M
Milan Broz 已提交
162 163 164

	/* sysfs handle */
	struct kobject kobj;
L
Linus Torvalds 已提交
165 166 167
};

#define MIN_IOS 256
168 169
static struct kmem_cache *_io_cache;
static struct kmem_cache *_tio_cache;
K
Kiyoshi Ueda 已提交
170 171
static struct kmem_cache *_rq_tio_cache;
static struct kmem_cache *_rq_bio_info_cache;
L
Linus Torvalds 已提交
172 173 174

static int __init local_init(void)
{
K
Kiyoshi Ueda 已提交
175
	int r = -ENOMEM;
L
Linus Torvalds 已提交
176 177

	/* allocate a slab for the dm_ios */
A
Alasdair G Kergon 已提交
178
	_io_cache = KMEM_CACHE(dm_io, 0);
L
Linus Torvalds 已提交
179
	if (!_io_cache)
K
Kiyoshi Ueda 已提交
180
		return r;
L
Linus Torvalds 已提交
181 182

	/* allocate a slab for the target ios */
A
Alasdair G Kergon 已提交
183
	_tio_cache = KMEM_CACHE(dm_target_io, 0);
K
Kiyoshi Ueda 已提交
184 185
	if (!_tio_cache)
		goto out_free_io_cache;
L
Linus Torvalds 已提交
186

K
Kiyoshi Ueda 已提交
187 188 189 190 191 192 193 194
	_rq_tio_cache = KMEM_CACHE(dm_rq_target_io, 0);
	if (!_rq_tio_cache)
		goto out_free_tio_cache;

	_rq_bio_info_cache = KMEM_CACHE(dm_rq_clone_bio_info, 0);
	if (!_rq_bio_info_cache)
		goto out_free_rq_tio_cache;

M
Mike Anderson 已提交
195
	r = dm_uevent_init();
K
Kiyoshi Ueda 已提交
196
	if (r)
K
Kiyoshi Ueda 已提交
197
		goto out_free_rq_bio_info_cache;
M
Mike Anderson 已提交
198

L
Linus Torvalds 已提交
199 200
	_major = major;
	r = register_blkdev(_major, _name);
K
Kiyoshi Ueda 已提交
201 202
	if (r < 0)
		goto out_uevent_exit;
L
Linus Torvalds 已提交
203 204 205 206 207

	if (!_major)
		_major = r;

	return 0;
K
Kiyoshi Ueda 已提交
208 209 210

out_uevent_exit:
	dm_uevent_exit();
K
Kiyoshi Ueda 已提交
211 212 213 214
out_free_rq_bio_info_cache:
	kmem_cache_destroy(_rq_bio_info_cache);
out_free_rq_tio_cache:
	kmem_cache_destroy(_rq_tio_cache);
K
Kiyoshi Ueda 已提交
215 216 217 218 219 220
out_free_tio_cache:
	kmem_cache_destroy(_tio_cache);
out_free_io_cache:
	kmem_cache_destroy(_io_cache);

	return r;
L
Linus Torvalds 已提交
221 222 223 224
}

static void local_exit(void)
{
K
Kiyoshi Ueda 已提交
225 226
	kmem_cache_destroy(_rq_bio_info_cache);
	kmem_cache_destroy(_rq_tio_cache);
L
Linus Torvalds 已提交
227 228
	kmem_cache_destroy(_tio_cache);
	kmem_cache_destroy(_io_cache);
229
	unregister_blkdev(_major, _name);
M
Mike Anderson 已提交
230
	dm_uevent_exit();
L
Linus Torvalds 已提交
231 232 233 234 235 236

	_major = 0;

	DMINFO("cleaned up");
}

237
static int (*_inits[])(void) __initdata = {
L
Linus Torvalds 已提交
238 239 240 241
	local_init,
	dm_target_init,
	dm_linear_init,
	dm_stripe_init,
242
	dm_kcopyd_init,
L
Linus Torvalds 已提交
243 244 245
	dm_interface_init,
};

246
static void (*_exits[])(void) = {
L
Linus Torvalds 已提交
247 248 249 250
	local_exit,
	dm_target_exit,
	dm_linear_exit,
	dm_stripe_exit,
251
	dm_kcopyd_exit,
L
Linus Torvalds 已提交
252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286
	dm_interface_exit,
};

static int __init dm_init(void)
{
	const int count = ARRAY_SIZE(_inits);

	int r, i;

	for (i = 0; i < count; i++) {
		r = _inits[i]();
		if (r)
			goto bad;
	}

	return 0;

      bad:
	while (i--)
		_exits[i]();

	return r;
}

static void __exit dm_exit(void)
{
	int i = ARRAY_SIZE(_exits);

	while (i--)
		_exits[i]();
}

/*
 * Block device functions
 */
A
Al Viro 已提交
287
static int dm_blk_open(struct block_device *bdev, fmode_t mode)
L
Linus Torvalds 已提交
288 289 290
{
	struct mapped_device *md;

J
Jeff Mahoney 已提交
291 292
	spin_lock(&_minor_lock);

A
Al Viro 已提交
293
	md = bdev->bd_disk->private_data;
J
Jeff Mahoney 已提交
294 295 296
	if (!md)
		goto out;

297 298
	if (test_bit(DMF_FREEING, &md->flags) ||
	    test_bit(DMF_DELETING, &md->flags)) {
J
Jeff Mahoney 已提交
299 300 301 302
		md = NULL;
		goto out;
	}

L
Linus Torvalds 已提交
303
	dm_get(md);
304
	atomic_inc(&md->open_count);
J
Jeff Mahoney 已提交
305 306 307 308 309

out:
	spin_unlock(&_minor_lock);

	return md ? 0 : -ENXIO;
L
Linus Torvalds 已提交
310 311
}

A
Al Viro 已提交
312
static int dm_blk_close(struct gendisk *disk, fmode_t mode)
L
Linus Torvalds 已提交
313
{
A
Al Viro 已提交
314
	struct mapped_device *md = disk->private_data;
315
	atomic_dec(&md->open_count);
L
Linus Torvalds 已提交
316 317 318 319
	dm_put(md);
	return 0;
}

320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343
int dm_open_count(struct mapped_device *md)
{
	return atomic_read(&md->open_count);
}

/*
 * Guarantees nothing is using the device before it's deleted.
 */
int dm_lock_for_deletion(struct mapped_device *md)
{
	int r = 0;

	spin_lock(&_minor_lock);

	if (dm_open_count(md))
		r = -EBUSY;
	else
		set_bit(DMF_DELETING, &md->flags);

	spin_unlock(&_minor_lock);

	return r;
}

D
Darrick J. Wong 已提交
344 345 346 347 348 349 350
static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
{
	struct mapped_device *md = bdev->bd_disk->private_data;

	return dm_get_geometry(md, geo);
}

A
Al Viro 已提交
351
static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
352 353
			unsigned int cmd, unsigned long arg)
{
A
Al Viro 已提交
354 355
	struct mapped_device *md = bdev->bd_disk->private_data;
	struct dm_table *map = dm_get_table(md);
356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373
	struct dm_target *tgt;
	int r = -ENOTTY;

	if (!map || !dm_table_get_size(map))
		goto out;

	/* We only support devices that have a single target */
	if (dm_table_get_num_targets(map) != 1)
		goto out;

	tgt = dm_table_get_target(map, 0);

	if (dm_suspended(md)) {
		r = -EAGAIN;
		goto out;
	}

	if (tgt->type->ioctl)
374
		r = tgt->type->ioctl(tgt, cmd, arg);
375 376 377 378 379 380 381

out:
	dm_table_put(map);

	return r;
}

A
Alasdair G Kergon 已提交
382
static struct dm_io *alloc_io(struct mapped_device *md)
L
Linus Torvalds 已提交
383 384 385 386
{
	return mempool_alloc(md->io_pool, GFP_NOIO);
}

A
Alasdair G Kergon 已提交
387
static void free_io(struct mapped_device *md, struct dm_io *io)
L
Linus Torvalds 已提交
388 389 390 391
{
	mempool_free(io, md->io_pool);
}

A
Alasdair G Kergon 已提交
392
static struct dm_target_io *alloc_tio(struct mapped_device *md)
L
Linus Torvalds 已提交
393 394 395 396
{
	return mempool_alloc(md->tio_pool, GFP_NOIO);
}

A
Alasdair G Kergon 已提交
397
static void free_tio(struct mapped_device *md, struct dm_target_io *tio)
L
Linus Torvalds 已提交
398 399 400 401
{
	mempool_free(tio, md->tio_pool);
}

402 403 404
static void start_io_acct(struct dm_io *io)
{
	struct mapped_device *md = io->md;
T
Tejun Heo 已提交
405
	int cpu;
406 407 408

	io->start_time = jiffies;

T
Tejun Heo 已提交
409 410 411 412
	cpu = part_stat_lock();
	part_round_stats(cpu, &dm_disk(md)->part0);
	part_stat_unlock();
	dm_disk(md)->part0.in_flight = atomic_inc_return(&md->pending);
413 414
}

415
static void end_io_acct(struct dm_io *io)
416 417 418 419
{
	struct mapped_device *md = io->md;
	struct bio *bio = io->bio;
	unsigned long duration = jiffies - io->start_time;
T
Tejun Heo 已提交
420
	int pending, cpu;
421 422
	int rw = bio_data_dir(bio);

T
Tejun Heo 已提交
423 424 425 426
	cpu = part_stat_lock();
	part_round_stats(cpu, &dm_disk(md)->part0);
	part_stat_add(cpu, &dm_disk(md)->part0, ticks[rw], duration);
	part_stat_unlock();
427

T
Tejun Heo 已提交
428 429
	dm_disk(md)->part0.in_flight = pending =
		atomic_dec_return(&md->pending);
430

431 432 433
	/* nudge anyone waiting on suspend queue */
	if (!pending)
		wake_up(&md->wait);
434 435
}

L
Linus Torvalds 已提交
436 437 438 439 440
/*
 * Add the bio to the list of deferred io.
 */
static int queue_io(struct mapped_device *md, struct bio *bio)
{
441
	down_write(&md->io_lock);
L
Linus Torvalds 已提交
442 443

	if (!test_bit(DMF_BLOCK_IO, &md->flags)) {
444
		up_write(&md->io_lock);
L
Linus Torvalds 已提交
445 446 447 448 449
		return 1;
	}

	bio_list_add(&md->deferred, bio);

450
	up_write(&md->io_lock);
L
Linus Torvalds 已提交
451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471
	return 0;		/* deferred successfully */
}

/*
 * Everyone (including functions in this file), should use this
 * function to access the md->map field, and make sure they call
 * dm_table_put() when finished.
 */
struct dm_table *dm_get_table(struct mapped_device *md)
{
	struct dm_table *t;

	read_lock(&md->map_lock);
	t = md->map;
	if (t)
		dm_table_get(t);
	read_unlock(&md->map_lock);

	return t;
}

D
Darrick J. Wong 已提交
472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498
/*
 * Get the geometry associated with a dm device
 */
int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
{
	*geo = md->geometry;

	return 0;
}

/*
 * Set the geometry of a device.
 */
int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
{
	sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;

	if (geo->start > sz) {
		DMWARN("Start sector is beyond the geometry limits.");
		return -EINVAL;
	}

	md->geometry = *geo;

	return 0;
}

L
Linus Torvalds 已提交
499 500 501 502 503 504 505 506 507
/*-----------------------------------------------------------------
 * CRUD START:
 *   A more elegant soln is in the works that uses the queue
 *   merge fn, unfortunately there are a couple of changes to
 *   the block layer that I want to make for this.  So in the
 *   interests of getting something for people to use I give
 *   you this clearly demarcated crap.
 *---------------------------------------------------------------*/

508 509 510 511 512
static int __noflush_suspending(struct mapped_device *md)
{
	return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
}

L
Linus Torvalds 已提交
513 514 515 516
/*
 * Decrements the number of outstanding ios that a bio has been
 * cloned into, completing the original io if necc.
 */
517
static void dec_pending(struct dm_io *io, int error)
L
Linus Torvalds 已提交
518
{
519
	unsigned long flags;
520 521 522
	int io_error;
	struct bio *bio;
	struct mapped_device *md = io->md;
523 524

	/* Push-back supersedes any I/O errors */
525
	if (error && !(io->error > 0 && __noflush_suspending(md)))
L
Linus Torvalds 已提交
526 527 528
		io->error = error;

	if (atomic_dec_and_test(&io->io_count)) {
529 530 531 532 533 534
		if (io->error == DM_ENDIO_REQUEUE) {
			/*
			 * Target requested pushing back the I/O.
			 * This must be handled before the sleeper on
			 * suspend queue merges the pushback list.
			 */
535 536 537
			spin_lock_irqsave(&md->pushback_lock, flags);
			if (__noflush_suspending(md))
				bio_list_add(&md->pushback, io->bio);
538 539 540
			else
				/* noflush suspend was interrupted. */
				io->error = -EIO;
541
			spin_unlock_irqrestore(&md->pushback_lock, flags);
542 543
		}

544
		end_io_acct(io);
L
Linus Torvalds 已提交
545

546 547
		io_error = io->error;
		bio = io->bio;
548

549 550 551 552
		free_io(md, io);

		if (io_error != DM_ENDIO_REQUEUE) {
			trace_block_bio_complete(md->queue, bio);
553

554 555
			bio_endio(bio, io_error);
		}
L
Linus Torvalds 已提交
556 557 558
	}
}

559
static void clone_endio(struct bio *bio, int error)
L
Linus Torvalds 已提交
560 561
{
	int r = 0;
A
Alasdair G Kergon 已提交
562
	struct dm_target_io *tio = bio->bi_private;
563
	struct dm_io *io = tio->io;
S
Stefan Bader 已提交
564
	struct mapped_device *md = tio->io->md;
L
Linus Torvalds 已提交
565 566 567 568 569 570 571
	dm_endio_fn endio = tio->ti->type->end_io;

	if (!bio_flagged(bio, BIO_UPTODATE) && !error)
		error = -EIO;

	if (endio) {
		r = endio(tio->ti, bio, error, &tio->info);
572 573 574 575 576
		if (r < 0 || r == DM_ENDIO_REQUEUE)
			/*
			 * error and requeue request are handled
			 * in dec_pending().
			 */
L
Linus Torvalds 已提交
577
			error = r;
578 579
		else if (r == DM_ENDIO_INCOMPLETE)
			/* The target will handle the io */
580
			return;
581 582 583 584
		else if (r) {
			DMWARN("unimplemented target endio return value: %d", r);
			BUG();
		}
L
Linus Torvalds 已提交
585 586
	}

S
Stefan Bader 已提交
587 588 589 590 591 592
	/*
	 * Store md for cleanup instead of tio which is about to get freed.
	 */
	bio->bi_private = md->bs;

	free_tio(md, tio);
593 594
	bio_put(bio);
	dec_pending(io, error);
L
Linus Torvalds 已提交
595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617
}

static sector_t max_io_len(struct mapped_device *md,
			   sector_t sector, struct dm_target *ti)
{
	sector_t offset = sector - ti->begin;
	sector_t len = ti->len - offset;

	/*
	 * Does the target need to split even further ?
	 */
	if (ti->split_io) {
		sector_t boundary;
		boundary = ((offset + ti->split_io) & ~(ti->split_io - 1))
			   - offset;
		if (len > boundary)
			len = boundary;
	}

	return len;
}

static void __map_bio(struct dm_target *ti, struct bio *clone,
A
Alasdair G Kergon 已提交
618
		      struct dm_target_io *tio)
L
Linus Torvalds 已提交
619 620
{
	int r;
621
	sector_t sector;
S
Stefan Bader 已提交
622
	struct mapped_device *md;
L
Linus Torvalds 已提交
623 624 625 626 627 628 629 630 631 632 633 634 635 636 637

	/*
	 * Sanity checks.
	 */
	BUG_ON(!clone->bi_size);

	clone->bi_end_io = clone_endio;
	clone->bi_private = tio;

	/*
	 * Map the clone.  If r == 0 we don't need to do
	 * anything, the target has assumed ownership of
	 * this io.
	 */
	atomic_inc(&tio->io->io_count);
638
	sector = clone->bi_sector;
L
Linus Torvalds 已提交
639
	r = ti->type->map(ti, clone, &tio->info);
640
	if (r == DM_MAPIO_REMAPPED) {
L
Linus Torvalds 已提交
641
		/* the bio has been remapped so dispatch it */
642

643
		trace_block_remap(bdev_get_queue(clone->bi_bdev), clone,
644 645
				    tio->io->bio->bi_bdev->bd_dev,
				    clone->bi_sector, sector);
646

L
Linus Torvalds 已提交
647
		generic_make_request(clone);
648 649
	} else if (r < 0 || r == DM_MAPIO_REQUEUE) {
		/* error the io and bail out, or requeue it if needed */
S
Stefan Bader 已提交
650 651 652 653 654 655
		md = tio->io->md;
		dec_pending(tio->io, r);
		/*
		 * Store bio_set for cleanup.
		 */
		clone->bi_private = md->bs;
L
Linus Torvalds 已提交
656
		bio_put(clone);
S
Stefan Bader 已提交
657
		free_tio(md, tio);
658 659 660
	} else if (r) {
		DMWARN("unimplemented target map return value: %d", r);
		BUG();
L
Linus Torvalds 已提交
661 662 663 664 665 666 667 668 669 670 671 672 673
	}
}

struct clone_info {
	struct mapped_device *md;
	struct dm_table *map;
	struct bio *bio;
	struct dm_io *io;
	sector_t sector;
	sector_t sector_count;
	unsigned short idx;
};

P
Peter Osterlund 已提交
674 675
static void dm_bio_destructor(struct bio *bio)
{
S
Stefan Bader 已提交
676 677 678
	struct bio_set *bs = bio->bi_private;

	bio_free(bio, bs);
P
Peter Osterlund 已提交
679 680
}

L
Linus Torvalds 已提交
681 682 683 684 685
/*
 * Creates a little bio that is just does part of a bvec.
 */
static struct bio *split_bvec(struct bio *bio, sector_t sector,
			      unsigned short idx, unsigned int offset,
S
Stefan Bader 已提交
686
			      unsigned int len, struct bio_set *bs)
L
Linus Torvalds 已提交
687 688 689 690
{
	struct bio *clone;
	struct bio_vec *bv = bio->bi_io_vec + idx;

S
Stefan Bader 已提交
691
	clone = bio_alloc_bioset(GFP_NOIO, 1, bs);
P
Peter Osterlund 已提交
692
	clone->bi_destructor = dm_bio_destructor;
L
Linus Torvalds 已提交
693 694 695 696 697 698 699 700 701
	*clone->bi_io_vec = *bv;

	clone->bi_sector = sector;
	clone->bi_bdev = bio->bi_bdev;
	clone->bi_rw = bio->bi_rw;
	clone->bi_vcnt = 1;
	clone->bi_size = to_bytes(len);
	clone->bi_io_vec->bv_offset = offset;
	clone->bi_io_vec->bv_len = clone->bi_size;
M
Martin K. Petersen 已提交
702
	clone->bi_flags |= 1 << BIO_CLONED;
L
Linus Torvalds 已提交
703 704 705 706 707 708 709 710 711

	return clone;
}

/*
 * Creates a bio that consists of range of complete bvecs.
 */
static struct bio *clone_bio(struct bio *bio, sector_t sector,
			     unsigned short idx, unsigned short bv_count,
S
Stefan Bader 已提交
712
			     unsigned int len, struct bio_set *bs)
L
Linus Torvalds 已提交
713 714 715
{
	struct bio *clone;

S
Stefan Bader 已提交
716 717 718
	clone = bio_alloc_bioset(GFP_NOIO, bio->bi_max_vecs, bs);
	__bio_clone(clone, bio);
	clone->bi_destructor = dm_bio_destructor;
L
Linus Torvalds 已提交
719 720 721 722 723 724 725 726 727
	clone->bi_sector = sector;
	clone->bi_idx = idx;
	clone->bi_vcnt = idx + bv_count;
	clone->bi_size = to_bytes(len);
	clone->bi_flags &= ~(1 << BIO_SEG_VALID);

	return clone;
}

728
static int __clone_and_map(struct clone_info *ci)
L
Linus Torvalds 已提交
729 730
{
	struct bio *clone, *bio = ci->bio;
731 732
	struct dm_target *ti;
	sector_t len = 0, max;
A
Alasdair G Kergon 已提交
733
	struct dm_target_io *tio;
L
Linus Torvalds 已提交
734

735 736 737 738 739 740
	ti = dm_table_find_target(ci->map, ci->sector);
	if (!dm_target_is_valid(ti))
		return -EIO;

	max = max_io_len(ci->md, ci->sector, ti);

L
Linus Torvalds 已提交
741 742 743 744 745 746 747 748 749 750 751 752 753 754
	/*
	 * Allocate a target io object.
	 */
	tio = alloc_tio(ci->md);
	tio->io = ci->io;
	tio->ti = ti;
	memset(&tio->info, 0, sizeof(tio->info));

	if (ci->sector_count <= max) {
		/*
		 * Optimise for the simple case where we can do all of
		 * the remaining io with a single clone.
		 */
		clone = clone_bio(bio, ci->sector, ci->idx,
S
Stefan Bader 已提交
755 756
				  bio->bi_vcnt - ci->idx, ci->sector_count,
				  ci->md->bs);
L
Linus Torvalds 已提交
757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778
		__map_bio(ti, clone, tio);
		ci->sector_count = 0;

	} else if (to_sector(bio->bi_io_vec[ci->idx].bv_len) <= max) {
		/*
		 * There are some bvecs that don't span targets.
		 * Do as many of these as possible.
		 */
		int i;
		sector_t remaining = max;
		sector_t bv_len;

		for (i = ci->idx; remaining && (i < bio->bi_vcnt); i++) {
			bv_len = to_sector(bio->bi_io_vec[i].bv_len);

			if (bv_len > remaining)
				break;

			remaining -= bv_len;
			len += bv_len;
		}

S
Stefan Bader 已提交
779 780
		clone = clone_bio(bio, ci->sector, ci->idx, i - ci->idx, len,
				  ci->md->bs);
L
Linus Torvalds 已提交
781 782 783 784 785 786 787 788
		__map_bio(ti, clone, tio);

		ci->sector += len;
		ci->sector_count -= len;
		ci->idx = i;

	} else {
		/*
789
		 * Handle a bvec that must be split between two or more targets.
L
Linus Torvalds 已提交
790 791
		 */
		struct bio_vec *bv = bio->bi_io_vec + ci->idx;
792 793
		sector_t remaining = to_sector(bv->bv_len);
		unsigned int offset = 0;
L
Linus Torvalds 已提交
794

795 796 797
		do {
			if (offset) {
				ti = dm_table_find_target(ci->map, ci->sector);
798 799 800
				if (!dm_target_is_valid(ti))
					return -EIO;

801
				max = max_io_len(ci->md, ci->sector, ti);
L
Linus Torvalds 已提交
802

803 804 805 806 807 808 809 810 811
				tio = alloc_tio(ci->md);
				tio->io = ci->io;
				tio->ti = ti;
				memset(&tio->info, 0, sizeof(tio->info));
			}

			len = min(remaining, max);

			clone = split_bvec(bio, ci->sector, ci->idx,
S
Stefan Bader 已提交
812 813
					   bv->bv_offset + offset, len,
					   ci->md->bs);
814 815 816 817 818 819 820

			__map_bio(ti, clone, tio);

			ci->sector += len;
			ci->sector_count -= len;
			offset += to_bytes(len);
		} while (remaining -= len);
L
Linus Torvalds 已提交
821 822 823

		ci->idx++;
	}
824 825

	return 0;
L
Linus Torvalds 已提交
826 827 828
}

/*
M
Mikulas Patocka 已提交
829
 * Split the bio into several clones and submit it to targets.
L
Linus Torvalds 已提交
830
 */
831
static void __split_and_process_bio(struct mapped_device *md, struct bio *bio)
L
Linus Torvalds 已提交
832 833
{
	struct clone_info ci;
834
	int error = 0;
L
Linus Torvalds 已提交
835 836

	ci.map = dm_get_table(md);
837 838 839 840
	if (unlikely(!ci.map)) {
		bio_io_error(bio);
		return;
	}
841 842 843
	if (unlikely(bio_barrier(bio) && !dm_table_barrier_ok(ci.map))) {
		dm_table_put(ci.map);
		bio_endio(bio, -EOPNOTSUPP);
844
		return;
845
	}
L
Linus Torvalds 已提交
846 847 848 849 850 851 852 853 854 855 856
	ci.md = md;
	ci.bio = bio;
	ci.io = alloc_io(md);
	ci.io->error = 0;
	atomic_set(&ci.io->io_count, 1);
	ci.io->bio = bio;
	ci.io->md = md;
	ci.sector = bio->bi_sector;
	ci.sector_count = bio_sectors(bio);
	ci.idx = bio->bi_idx;

857
	start_io_acct(ci.io);
858 859
	while (ci.sector_count && !error)
		error = __clone_and_map(&ci);
L
Linus Torvalds 已提交
860 861

	/* drop the extra reference count */
862
	dec_pending(ci.io, error);
L
Linus Torvalds 已提交
863 864 865 866 867 868
	dm_table_put(ci.map);
}
/*-----------------------------------------------------------------
 * CRUD END
 *---------------------------------------------------------------*/

M
Milan Broz 已提交
869 870 871 872 873 874 875 876
static int dm_merge_bvec(struct request_queue *q,
			 struct bvec_merge_data *bvm,
			 struct bio_vec *biovec)
{
	struct mapped_device *md = q->queuedata;
	struct dm_table *map = dm_get_table(md);
	struct dm_target *ti;
	sector_t max_sectors;
877
	int max_size = 0;
M
Milan Broz 已提交
878 879

	if (unlikely(!map))
880
		goto out;
M
Milan Broz 已提交
881 882

	ti = dm_table_find_target(map, bvm->bi_sector);
883 884
	if (!dm_target_is_valid(ti))
		goto out_table;
M
Milan Broz 已提交
885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902

	/*
	 * Find maximum amount of I/O that won't need splitting
	 */
	max_sectors = min(max_io_len(md, bvm->bi_sector, ti),
			  (sector_t) BIO_MAX_SECTORS);
	max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size;
	if (max_size < 0)
		max_size = 0;

	/*
	 * merge_bvec_fn() returns number of bytes
	 * it can accept at this offset
	 * max is precomputed maximal io size
	 */
	if (max_size && ti->type->merge)
		max_size = ti->type->merge(ti, bvm, biovec, max_size);

903
out_table:
904 905 906
	dm_table_put(map);

out:
M
Milan Broz 已提交
907 908 909 910 911 912 913 914 915
	/*
	 * Always allow an entire first page
	 */
	if (max_size <= biovec->bv_len && !(bvm->bi_size >> SECTOR_SHIFT))
		max_size = biovec->bv_len;

	return max_size;
}

L
Linus Torvalds 已提交
916 917 918 919
/*
 * The request function that just remaps the bio built up by
 * dm_merge_bvec.
 */
920
static int dm_request(struct request_queue *q, struct bio *bio)
L
Linus Torvalds 已提交
921
{
M
Milan Broz 已提交
922
	int r = -EIO;
923
	int rw = bio_data_dir(bio);
L
Linus Torvalds 已提交
924
	struct mapped_device *md = q->queuedata;
T
Tejun Heo 已提交
925
	int cpu;
L
Linus Torvalds 已提交
926

927
	down_read(&md->io_lock);
L
Linus Torvalds 已提交
928

T
Tejun Heo 已提交
929 930 931 932
	cpu = part_stat_lock();
	part_stat_inc(cpu, &dm_disk(md)->part0, ios[rw]);
	part_stat_add(cpu, &dm_disk(md)->part0, sectors[rw], bio_sectors(bio));
	part_stat_unlock();
933

L
Linus Torvalds 已提交
934 935 936 937 938
	/*
	 * If we're suspended we have to queue
	 * this io for later.
	 */
	while (test_bit(DMF_BLOCK_IO, &md->flags)) {
939
		up_read(&md->io_lock);
L
Linus Torvalds 已提交
940

M
Milan Broz 已提交
941 942
		if (bio_rw(bio) != READA)
			r = queue_io(md, bio);
L
Linus Torvalds 已提交
943

M
Milan Broz 已提交
944 945
		if (r <= 0)
			goto out_req;
L
Linus Torvalds 已提交
946 947 948 949 950

		/*
		 * We're in a while loop, because someone could suspend
		 * before we get to the following read lock.
		 */
951
		down_read(&md->io_lock);
L
Linus Torvalds 已提交
952 953
	}

954
	__split_and_process_bio(md, bio);
955
	up_read(&md->io_lock);
956
	return 0;
M
Milan Broz 已提交
957 958 959 960 961

out_req:
	if (r < 0)
		bio_io_error(bio);

L
Linus Torvalds 已提交
962 963 964
	return 0;
}

965
static void dm_unplug_all(struct request_queue *q)
L
Linus Torvalds 已提交
966 967 968 969 970 971 972 973 974 975 976 977
{
	struct mapped_device *md = q->queuedata;
	struct dm_table *map = dm_get_table(md);

	if (map) {
		dm_table_unplug_all(map);
		dm_table_put(map);
	}
}

static int dm_any_congested(void *congested_data, int bdi_bits)
{
978 979 980
	int r = bdi_bits;
	struct mapped_device *md = congested_data;
	struct dm_table *map;
L
Linus Torvalds 已提交
981

982 983 984 985 986 987 988 989
	if (!test_bit(DMF_BLOCK_IO, &md->flags)) {
		map = dm_get_table(md);
		if (map) {
			r = dm_table_any_congested(map, bdi_bits);
			dm_table_put(map);
		}
	}

L
Linus Torvalds 已提交
990 991 992 993 994 995 996 997
	return r;
}

/*-----------------------------------------------------------------
 * An IDR is used to keep track of allocated minor numbers.
 *---------------------------------------------------------------*/
static DEFINE_IDR(_minor_idr);

998
static void free_minor(int minor)
L
Linus Torvalds 已提交
999
{
1000
	spin_lock(&_minor_lock);
L
Linus Torvalds 已提交
1001
	idr_remove(&_minor_idr, minor);
1002
	spin_unlock(&_minor_lock);
L
Linus Torvalds 已提交
1003 1004 1005 1006 1007
}

/*
 * See if the device with a specific minor # is free.
 */
1008
static int specific_minor(int minor)
L
Linus Torvalds 已提交
1009 1010 1011 1012 1013 1014
{
	int r, m;

	if (minor >= (1 << MINORBITS))
		return -EINVAL;

J
Jeff Mahoney 已提交
1015 1016 1017 1018
	r = idr_pre_get(&_minor_idr, GFP_KERNEL);
	if (!r)
		return -ENOMEM;

1019
	spin_lock(&_minor_lock);
L
Linus Torvalds 已提交
1020 1021 1022 1023 1024 1025

	if (idr_find(&_minor_idr, minor)) {
		r = -EBUSY;
		goto out;
	}

1026
	r = idr_get_new_above(&_minor_idr, MINOR_ALLOCED, minor, &m);
J
Jeff Mahoney 已提交
1027
	if (r)
L
Linus Torvalds 已提交
1028 1029 1030 1031 1032 1033 1034 1035 1036
		goto out;

	if (m != minor) {
		idr_remove(&_minor_idr, m);
		r = -EBUSY;
		goto out;
	}

out:
1037
	spin_unlock(&_minor_lock);
L
Linus Torvalds 已提交
1038 1039 1040
	return r;
}

1041
static int next_free_minor(int *minor)
L
Linus Torvalds 已提交
1042
{
1043
	int r, m;
L
Linus Torvalds 已提交
1044 1045

	r = idr_pre_get(&_minor_idr, GFP_KERNEL);
J
Jeff Mahoney 已提交
1046 1047 1048
	if (!r)
		return -ENOMEM;

1049
	spin_lock(&_minor_lock);
L
Linus Torvalds 已提交
1050

1051
	r = idr_get_new(&_minor_idr, MINOR_ALLOCED, &m);
1052
	if (r)
L
Linus Torvalds 已提交
1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063
		goto out;

	if (m >= (1 << MINORBITS)) {
		idr_remove(&_minor_idr, m);
		r = -ENOSPC;
		goto out;
	}

	*minor = m;

out:
1064
	spin_unlock(&_minor_lock);
L
Linus Torvalds 已提交
1065 1066 1067 1068 1069
	return r;
}

static struct block_device_operations dm_blk_dops;

1070 1071
static void dm_wq_work(struct work_struct *work);

L
Linus Torvalds 已提交
1072 1073 1074
/*
 * Allocate and initialise a blank device with a given minor.
 */
1075
static struct mapped_device *alloc_dev(int minor)
L
Linus Torvalds 已提交
1076 1077
{
	int r;
1078
	struct mapped_device *md = kzalloc(sizeof(*md), GFP_KERNEL);
1079
	void *old_md;
L
Linus Torvalds 已提交
1080 1081 1082 1083 1084 1085

	if (!md) {
		DMWARN("unable to allocate device, out of memory.");
		return NULL;
	}

1086
	if (!try_module_get(THIS_MODULE))
M
Milan Broz 已提交
1087
		goto bad_module_get;
1088

L
Linus Torvalds 已提交
1089
	/* get a minor number for the dev */
1090
	if (minor == DM_ANY_MINOR)
1091
		r = next_free_minor(&minor);
1092
	else
1093
		r = specific_minor(minor);
L
Linus Torvalds 已提交
1094
	if (r < 0)
M
Milan Broz 已提交
1095
		goto bad_minor;
L
Linus Torvalds 已提交
1096

1097
	init_rwsem(&md->io_lock);
1098
	mutex_init(&md->suspend_lock);
1099
	spin_lock_init(&md->pushback_lock);
L
Linus Torvalds 已提交
1100 1101
	rwlock_init(&md->map_lock);
	atomic_set(&md->holders, 1);
1102
	atomic_set(&md->open_count, 0);
L
Linus Torvalds 已提交
1103
	atomic_set(&md->event_nr, 0);
M
Mike Anderson 已提交
1104 1105 1106
	atomic_set(&md->uevent_seq, 0);
	INIT_LIST_HEAD(&md->uevent_list);
	spin_lock_init(&md->uevent_lock);
L
Linus Torvalds 已提交
1107 1108 1109

	md->queue = blk_alloc_queue(GFP_KERNEL);
	if (!md->queue)
M
Milan Broz 已提交
1110
		goto bad_queue;
L
Linus Torvalds 已提交
1111 1112 1113 1114 1115

	md->queue->queuedata = md;
	md->queue->backing_dev_info.congested_fn = dm_any_congested;
	md->queue->backing_dev_info.congested_data = md;
	blk_queue_make_request(md->queue, dm_request);
1116
	blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);
L
Linus Torvalds 已提交
1117
	md->queue->unplug_fn = dm_unplug_all;
M
Milan Broz 已提交
1118
	blk_queue_merge_bvec(md->queue, dm_merge_bvec);
L
Linus Torvalds 已提交
1119

1120
	md->io_pool = mempool_create_slab_pool(MIN_IOS, _io_cache);
K
Kiyoshi Ueda 已提交
1121
	if (!md->io_pool)
M
Milan Broz 已提交
1122
		goto bad_io_pool;
L
Linus Torvalds 已提交
1123

1124
	md->tio_pool = mempool_create_slab_pool(MIN_IOS, _tio_cache);
L
Linus Torvalds 已提交
1125
	if (!md->tio_pool)
M
Milan Broz 已提交
1126
		goto bad_tio_pool;
L
Linus Torvalds 已提交
1127

1128
	md->bs = bioset_create(16, 0);
S
Stefan Bader 已提交
1129 1130 1131
	if (!md->bs)
		goto bad_no_bioset;

L
Linus Torvalds 已提交
1132 1133
	md->disk = alloc_disk(1);
	if (!md->disk)
M
Milan Broz 已提交
1134
		goto bad_disk;
L
Linus Torvalds 已提交
1135

1136 1137
	atomic_set(&md->pending, 0);
	init_waitqueue_head(&md->wait);
1138
	INIT_WORK(&md->work, dm_wq_work);
1139 1140
	init_waitqueue_head(&md->eventq);

L
Linus Torvalds 已提交
1141 1142 1143 1144 1145 1146 1147
	md->disk->major = _major;
	md->disk->first_minor = minor;
	md->disk->fops = &dm_blk_dops;
	md->disk->queue = md->queue;
	md->disk->private_data = md;
	sprintf(md->disk->disk_name, "dm-%d", minor);
	add_disk(md->disk);
M
Mike Anderson 已提交
1148
	format_dev_t(md->name, MKDEV(_major, minor));
L
Linus Torvalds 已提交
1149

1150 1151 1152 1153
	md->wq = create_singlethread_workqueue("kdmflush");
	if (!md->wq)
		goto bad_thread;

1154
	/* Populate the mapping, nobody knows we exist yet */
1155
	spin_lock(&_minor_lock);
1156
	old_md = idr_replace(&_minor_idr, md, minor);
1157
	spin_unlock(&_minor_lock);
1158 1159 1160

	BUG_ON(old_md != MINOR_ALLOCED);

L
Linus Torvalds 已提交
1161 1162
	return md;

1163 1164
bad_thread:
	put_disk(md->disk);
M
Milan Broz 已提交
1165
bad_disk:
S
Stefan Bader 已提交
1166
	bioset_free(md->bs);
M
Milan Broz 已提交
1167
bad_no_bioset:
L
Linus Torvalds 已提交
1168
	mempool_destroy(md->tio_pool);
M
Milan Broz 已提交
1169
bad_tio_pool:
L
Linus Torvalds 已提交
1170
	mempool_destroy(md->io_pool);
M
Milan Broz 已提交
1171
bad_io_pool:
1172
	blk_cleanup_queue(md->queue);
M
Milan Broz 已提交
1173
bad_queue:
L
Linus Torvalds 已提交
1174
	free_minor(minor);
M
Milan Broz 已提交
1175
bad_minor:
1176
	module_put(THIS_MODULE);
M
Milan Broz 已提交
1177
bad_module_get:
L
Linus Torvalds 已提交
1178 1179 1180 1181
	kfree(md);
	return NULL;
}

J
Jun'ichi Nomura 已提交
1182 1183
static void unlock_fs(struct mapped_device *md);

L
Linus Torvalds 已提交
1184 1185
static void free_dev(struct mapped_device *md)
{
1186
	int minor = MINOR(disk_devt(md->disk));
1187

1188
	if (md->suspended_bdev) {
J
Jun'ichi Nomura 已提交
1189
		unlock_fs(md);
1190 1191
		bdput(md->suspended_bdev);
	}
1192
	destroy_workqueue(md->wq);
L
Linus Torvalds 已提交
1193 1194
	mempool_destroy(md->tio_pool);
	mempool_destroy(md->io_pool);
S
Stefan Bader 已提交
1195
	bioset_free(md->bs);
L
Linus Torvalds 已提交
1196
	del_gendisk(md->disk);
1197
	free_minor(minor);
J
Jeff Mahoney 已提交
1198 1199 1200 1201 1202

	spin_lock(&_minor_lock);
	md->disk->private_data = NULL;
	spin_unlock(&_minor_lock);

L
Linus Torvalds 已提交
1203
	put_disk(md->disk);
1204
	blk_cleanup_queue(md->queue);
1205
	module_put(THIS_MODULE);
L
Linus Torvalds 已提交
1206 1207 1208 1209 1210 1211 1212 1213
	kfree(md);
}

/*
 * Bind a table to the device.
 */
static void event_callback(void *context)
{
M
Mike Anderson 已提交
1214 1215
	unsigned long flags;
	LIST_HEAD(uevents);
L
Linus Torvalds 已提交
1216 1217
	struct mapped_device *md = (struct mapped_device *) context;

M
Mike Anderson 已提交
1218 1219 1220 1221
	spin_lock_irqsave(&md->uevent_lock, flags);
	list_splice_init(&md->uevent_list, &uevents);
	spin_unlock_irqrestore(&md->uevent_lock, flags);

1222
	dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
M
Mike Anderson 已提交
1223

L
Linus Torvalds 已提交
1224 1225 1226 1227
	atomic_inc(&md->event_nr);
	wake_up(&md->eventq);
}

1228
static void __set_size(struct mapped_device *md, sector_t size)
L
Linus Torvalds 已提交
1229
{
1230
	set_capacity(md->disk, size);
L
Linus Torvalds 已提交
1231

1232
	mutex_lock(&md->suspended_bdev->bd_inode->i_mutex);
1233
	i_size_write(md->suspended_bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
1234
	mutex_unlock(&md->suspended_bdev->bd_inode->i_mutex);
L
Linus Torvalds 已提交
1235 1236 1237 1238
}

static int __bind(struct mapped_device *md, struct dm_table *t)
{
1239
	struct request_queue *q = md->queue;
L
Linus Torvalds 已提交
1240 1241 1242
	sector_t size;

	size = dm_table_get_size(t);
D
Darrick J. Wong 已提交
1243 1244 1245 1246 1247 1248 1249

	/*
	 * Wipe any geometry if the size of the table changed.
	 */
	if (size != get_capacity(md->disk))
		memset(&md->geometry, 0, sizeof(md->geometry));

1250 1251
	if (md->suspended_bdev)
		__set_size(md, size);
1252 1253 1254

	if (!size) {
		dm_table_destroy(t);
L
Linus Torvalds 已提交
1255
		return 0;
1256
	}
L
Linus Torvalds 已提交
1257

1258 1259
	dm_table_event_callback(t, event_callback, md);

L
Linus Torvalds 已提交
1260 1261
	write_lock(&md->map_lock);
	md->map = t;
1262
	dm_table_set_restrictions(t, q);
L
Linus Torvalds 已提交
1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278
	write_unlock(&md->map_lock);

	return 0;
}

static void __unbind(struct mapped_device *md)
{
	struct dm_table *map = md->map;

	if (!map)
		return;

	dm_table_event_callback(map, NULL, NULL);
	write_lock(&md->map_lock);
	md->map = NULL;
	write_unlock(&md->map_lock);
1279
	dm_table_destroy(map);
L
Linus Torvalds 已提交
1280 1281 1282 1283 1284
}

/*
 * Constructor for a new device.
 */
1285
int dm_create(int minor, struct mapped_device **result)
L
Linus Torvalds 已提交
1286 1287 1288
{
	struct mapped_device *md;

1289
	md = alloc_dev(minor);
L
Linus Torvalds 已提交
1290 1291 1292
	if (!md)
		return -ENXIO;

M
Milan Broz 已提交
1293 1294
	dm_sysfs_init(md);

L
Linus Torvalds 已提交
1295 1296 1297 1298
	*result = md;
	return 0;
}

1299
static struct mapped_device *dm_find_md(dev_t dev)
L
Linus Torvalds 已提交
1300 1301 1302 1303 1304 1305 1306
{
	struct mapped_device *md;
	unsigned minor = MINOR(dev);

	if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
		return NULL;

1307
	spin_lock(&_minor_lock);
L
Linus Torvalds 已提交
1308 1309

	md = idr_find(&_minor_idr, minor);
J
Jeff Mahoney 已提交
1310
	if (md && (md == MINOR_ALLOCED ||
1311
		   (MINOR(disk_devt(dm_disk(md))) != minor) ||
A
Alasdair G Kergon 已提交
1312
		   test_bit(DMF_FREEING, &md->flags))) {
1313
		md = NULL;
J
Jeff Mahoney 已提交
1314 1315
		goto out;
	}
L
Linus Torvalds 已提交
1316

J
Jeff Mahoney 已提交
1317
out:
1318
	spin_unlock(&_minor_lock);
L
Linus Torvalds 已提交
1319

1320 1321 1322
	return md;
}

1323 1324 1325 1326 1327 1328 1329 1330 1331 1332
struct mapped_device *dm_get_md(dev_t dev)
{
	struct mapped_device *md = dm_find_md(dev);

	if (md)
		dm_get(md);

	return md;
}

A
Alasdair G Kergon 已提交
1333
void *dm_get_mdptr(struct mapped_device *md)
1334
{
A
Alasdair G Kergon 已提交
1335
	return md->interface_ptr;
L
Linus Torvalds 已提交
1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347
}

void dm_set_mdptr(struct mapped_device *md, void *ptr)
{
	md->interface_ptr = ptr;
}

void dm_get(struct mapped_device *md)
{
	atomic_inc(&md->holders);
}

1348 1349 1350 1351 1352 1353
const char *dm_device_name(struct mapped_device *md)
{
	return md->name;
}
EXPORT_SYMBOL_GPL(dm_device_name);

L
Linus Torvalds 已提交
1354 1355
void dm_put(struct mapped_device *md)
{
M
Mike Anderson 已提交
1356
	struct dm_table *map;
L
Linus Torvalds 已提交
1357

J
Jeff Mahoney 已提交
1358 1359
	BUG_ON(test_bit(DMF_FREEING, &md->flags));

1360
	if (atomic_dec_and_lock(&md->holders, &_minor_lock)) {
M
Mike Anderson 已提交
1361
		map = dm_get_table(md);
1362 1363
		idr_replace(&_minor_idr, MINOR_ALLOCED,
			    MINOR(disk_devt(dm_disk(md))));
J
Jeff Mahoney 已提交
1364
		set_bit(DMF_FREEING, &md->flags);
1365
		spin_unlock(&_minor_lock);
1366
		if (!dm_suspended(md)) {
L
Linus Torvalds 已提交
1367 1368 1369
			dm_table_presuspend_targets(map);
			dm_table_postsuspend_targets(map);
		}
M
Milan Broz 已提交
1370
		dm_sysfs_exit(md);
M
Mike Anderson 已提交
1371
		dm_table_put(map);
1372
		__unbind(md);
L
Linus Torvalds 已提交
1373 1374 1375
		free_dev(md);
	}
}
E
Edward Goggin 已提交
1376
EXPORT_SYMBOL_GPL(dm_put);
L
Linus Torvalds 已提交
1377

1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400
static int dm_wait_for_completion(struct mapped_device *md)
{
	int r = 0;

	while (1) {
		set_current_state(TASK_INTERRUPTIBLE);

		smp_mb();
		if (!atomic_read(&md->pending))
			break;

		if (signal_pending(current)) {
			r = -EINTR;
			break;
		}

		io_schedule();
	}
	set_current_state(TASK_RUNNING);

	return r;
}

L
Linus Torvalds 已提交
1401 1402 1403
/*
 * Process the deferred bios
 */
1404
static void __flush_deferred_io(struct mapped_device *md)
L
Linus Torvalds 已提交
1405
{
1406
	struct bio *c;
L
Linus Torvalds 已提交
1407

1408 1409
	while ((c = bio_list_pop(&md->deferred)))
		__split_and_process_bio(md, c);
M
Milan Broz 已提交
1410 1411

	clear_bit(DMF_BLOCK_IO, &md->flags);
L
Linus Torvalds 已提交
1412 1413
}

1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424
static void __merge_pushback_list(struct mapped_device *md)
{
	unsigned long flags;

	spin_lock_irqsave(&md->pushback_lock, flags);
	clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
	bio_list_merge_head(&md->deferred, &md->pushback);
	bio_list_init(&md->pushback);
	spin_unlock_irqrestore(&md->pushback_lock, flags);
}

1425 1426
static void dm_wq_work(struct work_struct *work)
{
1427 1428
	struct mapped_device *md = container_of(work, struct mapped_device,
						work);
1429 1430

	down_write(&md->io_lock);
1431
	__flush_deferred_io(md);
1432 1433 1434
	up_write(&md->io_lock);
}

1435
static void dm_queue_flush(struct mapped_device *md)
1436
{
1437
	queue_work(md->wq, &md->work);
1438 1439 1440
	flush_workqueue(md->wq);
}

L
Linus Torvalds 已提交
1441 1442 1443 1444 1445
/*
 * Swap in a new table (destroying old one).
 */
int dm_swap_table(struct mapped_device *md, struct dm_table *table)
{
1446
	int r = -EINVAL;
L
Linus Torvalds 已提交
1447

1448
	mutex_lock(&md->suspend_lock);
L
Linus Torvalds 已提交
1449 1450

	/* device must be suspended */
1451
	if (!dm_suspended(md))
1452
		goto out;
L
Linus Torvalds 已提交
1453

1454 1455 1456 1457 1458
	/* without bdev, the device size cannot be changed */
	if (!md->suspended_bdev)
		if (get_capacity(md->disk) != dm_table_get_size(table))
			goto out;

L
Linus Torvalds 已提交
1459 1460 1461
	__unbind(md);
	r = __bind(md, table);

1462
out:
1463
	mutex_unlock(&md->suspend_lock);
1464
	return r;
L
Linus Torvalds 已提交
1465 1466 1467 1468 1469 1470
}

/*
 * Functions to lock and unlock any filesystem running on the
 * device.
 */
1471
static int lock_fs(struct mapped_device *md)
L
Linus Torvalds 已提交
1472
{
1473
	int r;
L
Linus Torvalds 已提交
1474 1475

	WARN_ON(md->frozen_sb);
1476

1477
	md->frozen_sb = freeze_bdev(md->suspended_bdev);
1478
	if (IS_ERR(md->frozen_sb)) {
1479
		r = PTR_ERR(md->frozen_sb);
1480 1481
		md->frozen_sb = NULL;
		return r;
1482 1483
	}

1484 1485
	set_bit(DMF_FROZEN, &md->flags);

L
Linus Torvalds 已提交
1486
	/* don't bdput right now, we don't want the bdev
1487
	 * to go away while it is locked.
L
Linus Torvalds 已提交
1488 1489 1490 1491
	 */
	return 0;
}

1492
static void unlock_fs(struct mapped_device *md)
L
Linus Torvalds 已提交
1493
{
1494 1495 1496
	if (!test_bit(DMF_FROZEN, &md->flags))
		return;

1497
	thaw_bdev(md->suspended_bdev, md->frozen_sb);
L
Linus Torvalds 已提交
1498
	md->frozen_sb = NULL;
1499
	clear_bit(DMF_FROZEN, &md->flags);
L
Linus Torvalds 已提交
1500 1501 1502 1503 1504 1505 1506 1507 1508
}

/*
 * We need to be able to change a mapping table under a mounted
 * filesystem.  For example we might want to move some data in
 * the background.  Before the table can be swapped with
 * dm_bind_table, dm_suspend must be called to flush any in
 * flight bios and ensure that any further io gets deferred.
 */
1509
int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
L
Linus Torvalds 已提交
1510
{
1511
	struct dm_table *map = NULL;
L
Linus Torvalds 已提交
1512
	DECLARE_WAITQUEUE(wait, current);
1513
	int r = 0;
1514
	int do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG ? 1 : 0;
1515
	int noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG ? 1 : 0;
L
Linus Torvalds 已提交
1516

1517
	mutex_lock(&md->suspend_lock);
1518

M
Milan Broz 已提交
1519 1520
	if (dm_suspended(md)) {
		r = -EINVAL;
1521
		goto out_unlock;
M
Milan Broz 已提交
1522
	}
L
Linus Torvalds 已提交
1523 1524 1525

	map = dm_get_table(md);

1526 1527 1528 1529 1530 1531 1532
	/*
	 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
	 * This flag is cleared before dm_suspend returns.
	 */
	if (noflush)
		set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);

1533 1534 1535
	/* This does not get reverted if there's an error later. */
	dm_table_presuspend_targets(map);

1536 1537 1538 1539 1540 1541
	/* bdget() can stall if the pending I/Os are not flushed */
	if (!noflush) {
		md->suspended_bdev = bdget_disk(md->disk, 0);
		if (!md->suspended_bdev) {
			DMWARN("bdget failed in dm_suspend");
			r = -ENOMEM;
K
Kiyoshi Ueda 已提交
1542
			goto out;
1543
		}
1544

1545 1546 1547 1548 1549 1550 1551 1552 1553
		/*
		 * Flush I/O to the device. noflush supersedes do_lockfs,
		 * because lock_fs() needs to flush I/Os.
		 */
		if (do_lockfs) {
			r = lock_fs(md);
			if (r)
				goto out;
		}
1554
	}
L
Linus Torvalds 已提交
1555 1556

	/*
1557
	 * First we set the BLOCK_IO flag so no more ios will be mapped.
L
Linus Torvalds 已提交
1558
	 */
1559 1560
	down_write(&md->io_lock);
	set_bit(DMF_BLOCK_IO, &md->flags);
L
Linus Torvalds 已提交
1561 1562

	add_wait_queue(&md->wait, &wait);
1563
	up_write(&md->io_lock);
L
Linus Torvalds 已提交
1564 1565

	/* unplug */
1566
	if (map)
L
Linus Torvalds 已提交
1567 1568 1569
		dm_table_unplug_all(map);

	/*
1570
	 * Wait for the already-mapped ios to complete.
L
Linus Torvalds 已提交
1571
	 */
1572
	r = dm_wait_for_completion(md);
L
Linus Torvalds 已提交
1573

1574
	down_write(&md->io_lock);
L
Linus Torvalds 已提交
1575 1576
	remove_wait_queue(&md->wait, &wait);

1577 1578
	if (noflush)
		__merge_pushback_list(md);
1579
	up_write(&md->io_lock);
1580

L
Linus Torvalds 已提交
1581
	/* were we interrupted ? */
1582
	if (r < 0) {
1583
		dm_queue_flush(md);
M
Milan Broz 已提交
1584

1585
		unlock_fs(md);
1586
		goto out; /* pushback list is already flushed, so skip flush */
1587
	}
L
Linus Torvalds 已提交
1588

1589
	dm_table_postsuspend_targets(map);
L
Linus Torvalds 已提交
1590

1591
	set_bit(DMF_SUSPENDED, &md->flags);
1592

1593
out:
1594 1595 1596 1597 1598
	if (r && md->suspended_bdev) {
		bdput(md->suspended_bdev);
		md->suspended_bdev = NULL;
	}

1599
	dm_table_put(map);
1600 1601

out_unlock:
1602
	mutex_unlock(&md->suspend_lock);
1603
	return r;
L
Linus Torvalds 已提交
1604 1605 1606 1607
}

int dm_resume(struct mapped_device *md)
{
1608 1609
	int r = -EINVAL;
	struct dm_table *map = NULL;
L
Linus Torvalds 已提交
1610

1611
	mutex_lock(&md->suspend_lock);
1612
	if (!dm_suspended(md))
1613 1614 1615
		goto out;

	map = dm_get_table(md);
1616
	if (!map || !dm_table_get_size(map))
1617
		goto out;
L
Linus Torvalds 已提交
1618

1619 1620 1621
	r = dm_table_resume_targets(map);
	if (r)
		goto out;
1622

1623
	dm_queue_flush(md);
1624 1625 1626

	unlock_fs(md);

1627 1628 1629 1630
	if (md->suspended_bdev) {
		bdput(md->suspended_bdev);
		md->suspended_bdev = NULL;
	}
1631

1632 1633
	clear_bit(DMF_SUSPENDED, &md->flags);

L
Linus Torvalds 已提交
1634 1635
	dm_table_unplug_all(map);

1636
	dm_kobject_uevent(md);
1637

1638
	r = 0;
1639

1640 1641
out:
	dm_table_put(map);
1642
	mutex_unlock(&md->suspend_lock);
1643

1644
	return r;
L
Linus Torvalds 已提交
1645 1646 1647 1648 1649
}

/*-----------------------------------------------------------------
 * Event notification.
 *---------------------------------------------------------------*/
1650 1651
void dm_kobject_uevent(struct mapped_device *md)
{
1652
	kobject_uevent(&disk_to_dev(md->disk)->kobj, KOBJ_CHANGE);
1653 1654
}

M
Mike Anderson 已提交
1655 1656 1657 1658 1659
uint32_t dm_next_uevent_seq(struct mapped_device *md)
{
	return atomic_add_return(1, &md->uevent_seq);
}

L
Linus Torvalds 已提交
1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670
uint32_t dm_get_event_nr(struct mapped_device *md)
{
	return atomic_read(&md->event_nr);
}

int dm_wait_event(struct mapped_device *md, int event_nr)
{
	return wait_event_interruptible(md->eventq,
			(event_nr != atomic_read(&md->event_nr)));
}

M
Mike Anderson 已提交
1671 1672 1673 1674 1675 1676 1677 1678 1679
void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
{
	unsigned long flags;

	spin_lock_irqsave(&md->uevent_lock, flags);
	list_add(elist, &md->uevent_list);
	spin_unlock_irqrestore(&md->uevent_lock, flags);
}

L
Linus Torvalds 已提交
1680 1681 1682 1683 1684 1685 1686 1687 1688
/*
 * The gendisk is only valid as long as you have a reference
 * count on 'md'.
 */
struct gendisk *dm_disk(struct mapped_device *md)
{
	return md->disk;
}

M
Milan Broz 已提交
1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709
struct kobject *dm_kobject(struct mapped_device *md)
{
	return &md->kobj;
}

/*
 * struct mapped_device should not be exported outside of dm.c
 * so use this check to verify that kobj is part of md structure
 */
struct mapped_device *dm_get_from_kobject(struct kobject *kobj)
{
	struct mapped_device *md;

	md = container_of(kobj, struct mapped_device, kobj);
	if (&md->kobj != kobj)
		return NULL;

	dm_get(md);
	return md;
}

L
Linus Torvalds 已提交
1710 1711 1712 1713 1714
int dm_suspended(struct mapped_device *md)
{
	return test_bit(DMF_SUSPENDED, &md->flags);
}

1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725
int dm_noflush_suspending(struct dm_target *ti)
{
	struct mapped_device *md = dm_table_get_md(ti->table);
	int r = __noflush_suspending(md);

	dm_put(md);

	return r;
}
EXPORT_SYMBOL_GPL(dm_noflush_suspending);

L
Linus Torvalds 已提交
1726 1727 1728
static struct block_device_operations dm_blk_dops = {
	.open = dm_blk_open,
	.release = dm_blk_close,
1729
	.ioctl = dm_blk_ioctl,
D
Darrick J. Wong 已提交
1730
	.getgeo = dm_blk_getgeo,
L
Linus Torvalds 已提交
1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746
	.owner = THIS_MODULE
};

EXPORT_SYMBOL(dm_get_mapinfo);

/*
 * module hooks
 */
module_init(dm_init);
module_exit(dm_exit);

module_param(major, uint, 0);
MODULE_PARM_DESC(major, "The major number of the device mapper");
MODULE_DESCRIPTION(DM_NAME " driver");
MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
MODULE_LICENSE("GPL");