dm.c 35.4 KB
Newer Older
L
Linus Torvalds 已提交
1 2
/*
 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
M
Milan Broz 已提交
3
 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
L
Linus Torvalds 已提交
4 5 6 7 8 9
 *
 * This file is released under the GPL.
 */

#include "dm.h"
#include "dm-bio-list.h"
M
Mike Anderson 已提交
10
#include "dm-uevent.h"
L
Linus Torvalds 已提交
11 12 13

#include <linux/init.h>
#include <linux/module.h>
A
Arjan van de Ven 已提交
14
#include <linux/mutex.h>
L
Linus Torvalds 已提交
15 16 17 18 19 20 21
#include <linux/moduleparam.h>
#include <linux/blkpg.h>
#include <linux/bio.h>
#include <linux/buffer_head.h>
#include <linux/mempool.h>
#include <linux/slab.h>
#include <linux/idr.h>
D
Darrick J. Wong 已提交
22
#include <linux/hdreg.h>
23
#include <linux/blktrace_api.h>
24
#include <trace/block.h>
L
Linus Torvalds 已提交
25

26 27
#define DM_MSG_PREFIX "core"

L
Linus Torvalds 已提交
28 29 30 31 32
static const char *_name = DM_NAME;

static unsigned int major = 0;
static unsigned int _major = 0;

33
static DEFINE_SPINLOCK(_minor_lock);
L
Linus Torvalds 已提交
34
/*
K
Kiyoshi Ueda 已提交
35
 * For bio-based dm.
L
Linus Torvalds 已提交
36 37 38 39 40 41
 * One of these is allocated per bio.
 */
struct dm_io {
	struct mapped_device *md;
	int error;
	atomic_t io_count;
R
Richard Kennedy 已提交
42
	struct bio *bio;
43
	unsigned long start_time;
L
Linus Torvalds 已提交
44 45 46
};

/*
K
Kiyoshi Ueda 已提交
47
 * For bio-based dm.
L
Linus Torvalds 已提交
48 49 50
 * One of these is allocated per target within a bio.  Hopefully
 * this will be simplified out one day.
 */
A
Alasdair G Kergon 已提交
51
struct dm_target_io {
L
Linus Torvalds 已提交
52 53 54 55 56
	struct dm_io *io;
	struct dm_target *ti;
	union map_info info;
};

57 58
DEFINE_TRACE(block_bio_complete);

K
Kiyoshi Ueda 已提交
59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79
/*
 * For request-based dm.
 * One of these is allocated per request.
 */
struct dm_rq_target_io {
	struct mapped_device *md;
	struct dm_target *ti;
	struct request *orig, clone;
	int error;
	union map_info info;
};

/*
 * For request-based dm.
 * One of these is allocated per bio.
 */
struct dm_rq_clone_bio_info {
	struct bio *orig;
	struct request *rq;
};

L
Linus Torvalds 已提交
80 81
union map_info *dm_get_mapinfo(struct bio *bio)
{
A
Alasdair G Kergon 已提交
82
	if (bio && bio->bi_private)
A
Alasdair G Kergon 已提交
83
		return &((struct dm_target_io *)bio->bi_private)->info;
A
Alasdair G Kergon 已提交
84
	return NULL;
L
Linus Torvalds 已提交
85 86
}

87 88
#define MINOR_ALLOCED ((void *)-1)

L
Linus Torvalds 已提交
89 90 91 92 93
/*
 * Bits for the md->flags field.
 */
#define DMF_BLOCK_IO 0
#define DMF_SUSPENDED 1
94
#define DMF_FROZEN 2
J
Jeff Mahoney 已提交
95
#define DMF_FREEING 3
96
#define DMF_DELETING 4
97
#define DMF_NOFLUSH_SUSPENDING 5
L
Linus Torvalds 已提交
98

99 100 101 102 103 104 105 106 107 108 109 110
/*
 * Work processed by per-device workqueue.
 */
struct dm_wq_req {
	enum {
		DM_WQ_FLUSH_DEFERRED,
	} type;
	struct work_struct work;
	struct mapped_device *md;
	void *context;
};

L
Linus Torvalds 已提交
111
struct mapped_device {
112
	struct rw_semaphore io_lock;
113
	struct mutex suspend_lock;
114
	spinlock_t pushback_lock;
L
Linus Torvalds 已提交
115 116
	rwlock_t map_lock;
	atomic_t holders;
117
	atomic_t open_count;
L
Linus Torvalds 已提交
118 119 120

	unsigned long flags;

121
	struct request_queue *queue;
L
Linus Torvalds 已提交
122
	struct gendisk *disk;
M
Mike Anderson 已提交
123
	char name[16];
L
Linus Torvalds 已提交
124 125 126 127 128 129 130 131

	void *interface_ptr;

	/*
	 * A list of ios that arrived while we were suspended.
	 */
	atomic_t pending;
	wait_queue_head_t wait;
K
Kiyoshi Ueda 已提交
132
	struct bio_list deferred;
133
	struct bio_list pushback;
L
Linus Torvalds 已提交
134

135 136 137 138 139
	/*
	 * Processing queue (flush/barriers)
	 */
	struct workqueue_struct *wq;

L
Linus Torvalds 已提交
140 141 142 143 144 145 146 147 148 149 150
	/*
	 * The current mapping.
	 */
	struct dm_table *map;

	/*
	 * io objects are allocated from here.
	 */
	mempool_t *io_pool;
	mempool_t *tio_pool;

S
Stefan Bader 已提交
151 152
	struct bio_set *bs;

L
Linus Torvalds 已提交
153 154 155 156 157
	/*
	 * Event handling.
	 */
	atomic_t event_nr;
	wait_queue_head_t eventq;
M
Mike Anderson 已提交
158 159 160
	atomic_t uevent_seq;
	struct list_head uevent_list;
	spinlock_t uevent_lock; /* Protect access to uevent_list */
L
Linus Torvalds 已提交
161 162 163 164 165

	/*
	 * freeze/thaw support require holding onto a super block
	 */
	struct super_block *frozen_sb;
166
	struct block_device *suspended_bdev;
D
Darrick J. Wong 已提交
167 168 169

	/* forced geometry settings */
	struct hd_geometry geometry;
M
Milan Broz 已提交
170 171 172

	/* sysfs handle */
	struct kobject kobj;
L
Linus Torvalds 已提交
173 174 175
};

#define MIN_IOS 256
176 177
static struct kmem_cache *_io_cache;
static struct kmem_cache *_tio_cache;
K
Kiyoshi Ueda 已提交
178 179
static struct kmem_cache *_rq_tio_cache;
static struct kmem_cache *_rq_bio_info_cache;
L
Linus Torvalds 已提交
180 181 182

static int __init local_init(void)
{
K
Kiyoshi Ueda 已提交
183
	int r = -ENOMEM;
L
Linus Torvalds 已提交
184 185

	/* allocate a slab for the dm_ios */
A
Alasdair G Kergon 已提交
186
	_io_cache = KMEM_CACHE(dm_io, 0);
L
Linus Torvalds 已提交
187
	if (!_io_cache)
K
Kiyoshi Ueda 已提交
188
		return r;
L
Linus Torvalds 已提交
189 190

	/* allocate a slab for the target ios */
A
Alasdair G Kergon 已提交
191
	_tio_cache = KMEM_CACHE(dm_target_io, 0);
K
Kiyoshi Ueda 已提交
192 193
	if (!_tio_cache)
		goto out_free_io_cache;
L
Linus Torvalds 已提交
194

K
Kiyoshi Ueda 已提交
195 196 197 198 199 200 201 202
	_rq_tio_cache = KMEM_CACHE(dm_rq_target_io, 0);
	if (!_rq_tio_cache)
		goto out_free_tio_cache;

	_rq_bio_info_cache = KMEM_CACHE(dm_rq_clone_bio_info, 0);
	if (!_rq_bio_info_cache)
		goto out_free_rq_tio_cache;

M
Mike Anderson 已提交
203
	r = dm_uevent_init();
K
Kiyoshi Ueda 已提交
204
	if (r)
K
Kiyoshi Ueda 已提交
205
		goto out_free_rq_bio_info_cache;
M
Mike Anderson 已提交
206

L
Linus Torvalds 已提交
207 208
	_major = major;
	r = register_blkdev(_major, _name);
K
Kiyoshi Ueda 已提交
209 210
	if (r < 0)
		goto out_uevent_exit;
L
Linus Torvalds 已提交
211 212 213 214 215

	if (!_major)
		_major = r;

	return 0;
K
Kiyoshi Ueda 已提交
216 217 218

out_uevent_exit:
	dm_uevent_exit();
K
Kiyoshi Ueda 已提交
219 220 221 222
out_free_rq_bio_info_cache:
	kmem_cache_destroy(_rq_bio_info_cache);
out_free_rq_tio_cache:
	kmem_cache_destroy(_rq_tio_cache);
K
Kiyoshi Ueda 已提交
223 224 225 226 227 228
out_free_tio_cache:
	kmem_cache_destroy(_tio_cache);
out_free_io_cache:
	kmem_cache_destroy(_io_cache);

	return r;
L
Linus Torvalds 已提交
229 230 231 232
}

static void local_exit(void)
{
K
Kiyoshi Ueda 已提交
233 234
	kmem_cache_destroy(_rq_bio_info_cache);
	kmem_cache_destroy(_rq_tio_cache);
L
Linus Torvalds 已提交
235 236
	kmem_cache_destroy(_tio_cache);
	kmem_cache_destroy(_io_cache);
237
	unregister_blkdev(_major, _name);
M
Mike Anderson 已提交
238
	dm_uevent_exit();
L
Linus Torvalds 已提交
239 240 241 242 243 244

	_major = 0;

	DMINFO("cleaned up");
}

245
static int (*_inits[])(void) __initdata = {
L
Linus Torvalds 已提交
246 247 248 249
	local_init,
	dm_target_init,
	dm_linear_init,
	dm_stripe_init,
250
	dm_kcopyd_init,
L
Linus Torvalds 已提交
251 252 253
	dm_interface_init,
};

254
static void (*_exits[])(void) = {
L
Linus Torvalds 已提交
255 256 257 258
	local_exit,
	dm_target_exit,
	dm_linear_exit,
	dm_stripe_exit,
259
	dm_kcopyd_exit,
L
Linus Torvalds 已提交
260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294
	dm_interface_exit,
};

static int __init dm_init(void)
{
	const int count = ARRAY_SIZE(_inits);

	int r, i;

	for (i = 0; i < count; i++) {
		r = _inits[i]();
		if (r)
			goto bad;
	}

	return 0;

      bad:
	while (i--)
		_exits[i]();

	return r;
}

static void __exit dm_exit(void)
{
	int i = ARRAY_SIZE(_exits);

	while (i--)
		_exits[i]();
}

/*
 * Block device functions
 */
A
Al Viro 已提交
295
static int dm_blk_open(struct block_device *bdev, fmode_t mode)
L
Linus Torvalds 已提交
296 297 298
{
	struct mapped_device *md;

J
Jeff Mahoney 已提交
299 300
	spin_lock(&_minor_lock);

A
Al Viro 已提交
301
	md = bdev->bd_disk->private_data;
J
Jeff Mahoney 已提交
302 303 304
	if (!md)
		goto out;

305 306
	if (test_bit(DMF_FREEING, &md->flags) ||
	    test_bit(DMF_DELETING, &md->flags)) {
J
Jeff Mahoney 已提交
307 308 309 310
		md = NULL;
		goto out;
	}

L
Linus Torvalds 已提交
311
	dm_get(md);
312
	atomic_inc(&md->open_count);
J
Jeff Mahoney 已提交
313 314 315 316 317

out:
	spin_unlock(&_minor_lock);

	return md ? 0 : -ENXIO;
L
Linus Torvalds 已提交
318 319
}

A
Al Viro 已提交
320
static int dm_blk_close(struct gendisk *disk, fmode_t mode)
L
Linus Torvalds 已提交
321
{
A
Al Viro 已提交
322
	struct mapped_device *md = disk->private_data;
323
	atomic_dec(&md->open_count);
L
Linus Torvalds 已提交
324 325 326 327
	dm_put(md);
	return 0;
}

328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351
int dm_open_count(struct mapped_device *md)
{
	return atomic_read(&md->open_count);
}

/*
 * Guarantees nothing is using the device before it's deleted.
 */
int dm_lock_for_deletion(struct mapped_device *md)
{
	int r = 0;

	spin_lock(&_minor_lock);

	if (dm_open_count(md))
		r = -EBUSY;
	else
		set_bit(DMF_DELETING, &md->flags);

	spin_unlock(&_minor_lock);

	return r;
}

D
Darrick J. Wong 已提交
352 353 354 355 356 357 358
static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
{
	struct mapped_device *md = bdev->bd_disk->private_data;

	return dm_get_geometry(md, geo);
}

A
Al Viro 已提交
359
static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
360 361
			unsigned int cmd, unsigned long arg)
{
A
Al Viro 已提交
362 363
	struct mapped_device *md = bdev->bd_disk->private_data;
	struct dm_table *map = dm_get_table(md);
364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381
	struct dm_target *tgt;
	int r = -ENOTTY;

	if (!map || !dm_table_get_size(map))
		goto out;

	/* We only support devices that have a single target */
	if (dm_table_get_num_targets(map) != 1)
		goto out;

	tgt = dm_table_get_target(map, 0);

	if (dm_suspended(md)) {
		r = -EAGAIN;
		goto out;
	}

	if (tgt->type->ioctl)
382
		r = tgt->type->ioctl(tgt, cmd, arg);
383 384 385 386 387 388 389

out:
	dm_table_put(map);

	return r;
}

A
Alasdair G Kergon 已提交
390
static struct dm_io *alloc_io(struct mapped_device *md)
L
Linus Torvalds 已提交
391 392 393 394
{
	return mempool_alloc(md->io_pool, GFP_NOIO);
}

A
Alasdair G Kergon 已提交
395
static void free_io(struct mapped_device *md, struct dm_io *io)
L
Linus Torvalds 已提交
396 397 398 399
{
	mempool_free(io, md->io_pool);
}

A
Alasdair G Kergon 已提交
400
static struct dm_target_io *alloc_tio(struct mapped_device *md)
L
Linus Torvalds 已提交
401 402 403 404
{
	return mempool_alloc(md->tio_pool, GFP_NOIO);
}

A
Alasdair G Kergon 已提交
405
static void free_tio(struct mapped_device *md, struct dm_target_io *tio)
L
Linus Torvalds 已提交
406 407 408 409
{
	mempool_free(tio, md->tio_pool);
}

410 411 412
static void start_io_acct(struct dm_io *io)
{
	struct mapped_device *md = io->md;
T
Tejun Heo 已提交
413
	int cpu;
414 415 416

	io->start_time = jiffies;

T
Tejun Heo 已提交
417 418 419 420
	cpu = part_stat_lock();
	part_round_stats(cpu, &dm_disk(md)->part0);
	part_stat_unlock();
	dm_disk(md)->part0.in_flight = atomic_inc_return(&md->pending);
421 422
}

423
static void end_io_acct(struct dm_io *io)
424 425 426 427
{
	struct mapped_device *md = io->md;
	struct bio *bio = io->bio;
	unsigned long duration = jiffies - io->start_time;
T
Tejun Heo 已提交
428
	int pending, cpu;
429 430
	int rw = bio_data_dir(bio);

T
Tejun Heo 已提交
431 432 433 434
	cpu = part_stat_lock();
	part_round_stats(cpu, &dm_disk(md)->part0);
	part_stat_add(cpu, &dm_disk(md)->part0, ticks[rw], duration);
	part_stat_unlock();
435

T
Tejun Heo 已提交
436 437
	dm_disk(md)->part0.in_flight = pending =
		atomic_dec_return(&md->pending);
438

439 440 441
	/* nudge anyone waiting on suspend queue */
	if (!pending)
		wake_up(&md->wait);
442 443
}

L
Linus Torvalds 已提交
444 445 446 447 448
/*
 * Add the bio to the list of deferred io.
 */
static int queue_io(struct mapped_device *md, struct bio *bio)
{
449
	down_write(&md->io_lock);
L
Linus Torvalds 已提交
450 451

	if (!test_bit(DMF_BLOCK_IO, &md->flags)) {
452
		up_write(&md->io_lock);
L
Linus Torvalds 已提交
453 454 455 456 457
		return 1;
	}

	bio_list_add(&md->deferred, bio);

458
	up_write(&md->io_lock);
L
Linus Torvalds 已提交
459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479
	return 0;		/* deferred successfully */
}

/*
 * Everyone (including functions in this file), should use this
 * function to access the md->map field, and make sure they call
 * dm_table_put() when finished.
 */
struct dm_table *dm_get_table(struct mapped_device *md)
{
	struct dm_table *t;

	read_lock(&md->map_lock);
	t = md->map;
	if (t)
		dm_table_get(t);
	read_unlock(&md->map_lock);

	return t;
}

D
Darrick J. Wong 已提交
480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506
/*
 * Get the geometry associated with a dm device
 */
int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
{
	*geo = md->geometry;

	return 0;
}

/*
 * Set the geometry of a device.
 */
int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
{
	sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;

	if (geo->start > sz) {
		DMWARN("Start sector is beyond the geometry limits.");
		return -EINVAL;
	}

	md->geometry = *geo;

	return 0;
}

L
Linus Torvalds 已提交
507 508 509 510 511 512 513 514 515
/*-----------------------------------------------------------------
 * CRUD START:
 *   A more elegant soln is in the works that uses the queue
 *   merge fn, unfortunately there are a couple of changes to
 *   the block layer that I want to make for this.  So in the
 *   interests of getting something for people to use I give
 *   you this clearly demarcated crap.
 *---------------------------------------------------------------*/

516 517 518 519 520
static int __noflush_suspending(struct mapped_device *md)
{
	return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
}

L
Linus Torvalds 已提交
521 522 523 524
/*
 * Decrements the number of outstanding ios that a bio has been
 * cloned into, completing the original io if necc.
 */
525
static void dec_pending(struct dm_io *io, int error)
L
Linus Torvalds 已提交
526
{
527 528 529 530
	unsigned long flags;

	/* Push-back supersedes any I/O errors */
	if (error && !(io->error > 0 && __noflush_suspending(io->md)))
L
Linus Torvalds 已提交
531 532 533
		io->error = error;

	if (atomic_dec_and_test(&io->io_count)) {
534 535 536 537 538 539 540 541 542 543 544 545 546 547 548
		if (io->error == DM_ENDIO_REQUEUE) {
			/*
			 * Target requested pushing back the I/O.
			 * This must be handled before the sleeper on
			 * suspend queue merges the pushback list.
			 */
			spin_lock_irqsave(&io->md->pushback_lock, flags);
			if (__noflush_suspending(io->md))
				bio_list_add(&io->md->pushback, io->bio);
			else
				/* noflush suspend was interrupted. */
				io->error = -EIO;
			spin_unlock_irqrestore(&io->md->pushback_lock, flags);
		}

549
		end_io_acct(io);
L
Linus Torvalds 已提交
550

551
		if (io->error != DM_ENDIO_REQUEUE) {
552
			trace_block_bio_complete(io->md->queue, io->bio);
553

554
			bio_endio(io->bio, io->error);
555
		}
556

L
Linus Torvalds 已提交
557 558 559 560
		free_io(io->md, io);
	}
}

561
static void clone_endio(struct bio *bio, int error)
L
Linus Torvalds 已提交
562 563
{
	int r = 0;
A
Alasdair G Kergon 已提交
564
	struct dm_target_io *tio = bio->bi_private;
S
Stefan Bader 已提交
565
	struct mapped_device *md = tio->io->md;
L
Linus Torvalds 已提交
566 567 568 569 570 571 572
	dm_endio_fn endio = tio->ti->type->end_io;

	if (!bio_flagged(bio, BIO_UPTODATE) && !error)
		error = -EIO;

	if (endio) {
		r = endio(tio->ti, bio, error, &tio->info);
573 574 575 576 577
		if (r < 0 || r == DM_ENDIO_REQUEUE)
			/*
			 * error and requeue request are handled
			 * in dec_pending().
			 */
L
Linus Torvalds 已提交
578
			error = r;
579 580
		else if (r == DM_ENDIO_INCOMPLETE)
			/* The target will handle the io */
581
			return;
582 583 584 585
		else if (r) {
			DMWARN("unimplemented target endio return value: %d", r);
			BUG();
		}
L
Linus Torvalds 已提交
586 587
	}

S
Stefan Bader 已提交
588 589 590 591 592 593 594
	dec_pending(tio->io, error);

	/*
	 * Store md for cleanup instead of tio which is about to get freed.
	 */
	bio->bi_private = md->bs;

L
Linus Torvalds 已提交
595
	bio_put(bio);
S
Stefan Bader 已提交
596
	free_tio(md, tio);
L
Linus Torvalds 已提交
597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619
}

static sector_t max_io_len(struct mapped_device *md,
			   sector_t sector, struct dm_target *ti)
{
	sector_t offset = sector - ti->begin;
	sector_t len = ti->len - offset;

	/*
	 * Does the target need to split even further ?
	 */
	if (ti->split_io) {
		sector_t boundary;
		boundary = ((offset + ti->split_io) & ~(ti->split_io - 1))
			   - offset;
		if (len > boundary)
			len = boundary;
	}

	return len;
}

static void __map_bio(struct dm_target *ti, struct bio *clone,
A
Alasdair G Kergon 已提交
620
		      struct dm_target_io *tio)
L
Linus Torvalds 已提交
621 622
{
	int r;
623
	sector_t sector;
S
Stefan Bader 已提交
624
	struct mapped_device *md;
L
Linus Torvalds 已提交
625 626 627 628 629 630 631 632 633 634 635 636 637 638 639

	/*
	 * Sanity checks.
	 */
	BUG_ON(!clone->bi_size);

	clone->bi_end_io = clone_endio;
	clone->bi_private = tio;

	/*
	 * Map the clone.  If r == 0 we don't need to do
	 * anything, the target has assumed ownership of
	 * this io.
	 */
	atomic_inc(&tio->io->io_count);
640
	sector = clone->bi_sector;
L
Linus Torvalds 已提交
641
	r = ti->type->map(ti, clone, &tio->info);
642
	if (r == DM_MAPIO_REMAPPED) {
L
Linus Torvalds 已提交
643
		/* the bio has been remapped so dispatch it */
644

645
		trace_block_remap(bdev_get_queue(clone->bi_bdev), clone,
646 647
				    tio->io->bio->bi_bdev->bd_dev,
				    clone->bi_sector, sector);
648

L
Linus Torvalds 已提交
649
		generic_make_request(clone);
650 651
	} else if (r < 0 || r == DM_MAPIO_REQUEUE) {
		/* error the io and bail out, or requeue it if needed */
S
Stefan Bader 已提交
652 653 654 655 656 657
		md = tio->io->md;
		dec_pending(tio->io, r);
		/*
		 * Store bio_set for cleanup.
		 */
		clone->bi_private = md->bs;
L
Linus Torvalds 已提交
658
		bio_put(clone);
S
Stefan Bader 已提交
659
		free_tio(md, tio);
660 661 662
	} else if (r) {
		DMWARN("unimplemented target map return value: %d", r);
		BUG();
L
Linus Torvalds 已提交
663 664 665 666 667 668 669 670 671 672 673 674 675
	}
}

struct clone_info {
	struct mapped_device *md;
	struct dm_table *map;
	struct bio *bio;
	struct dm_io *io;
	sector_t sector;
	sector_t sector_count;
	unsigned short idx;
};

P
Peter Osterlund 已提交
676 677
static void dm_bio_destructor(struct bio *bio)
{
S
Stefan Bader 已提交
678 679 680
	struct bio_set *bs = bio->bi_private;

	bio_free(bio, bs);
P
Peter Osterlund 已提交
681 682
}

L
Linus Torvalds 已提交
683 684 685 686 687
/*
 * Creates a little bio that is just does part of a bvec.
 */
static struct bio *split_bvec(struct bio *bio, sector_t sector,
			      unsigned short idx, unsigned int offset,
S
Stefan Bader 已提交
688
			      unsigned int len, struct bio_set *bs)
L
Linus Torvalds 已提交
689 690 691 692
{
	struct bio *clone;
	struct bio_vec *bv = bio->bi_io_vec + idx;

S
Stefan Bader 已提交
693
	clone = bio_alloc_bioset(GFP_NOIO, 1, bs);
P
Peter Osterlund 已提交
694
	clone->bi_destructor = dm_bio_destructor;
L
Linus Torvalds 已提交
695 696 697 698 699 700 701 702 703
	*clone->bi_io_vec = *bv;

	clone->bi_sector = sector;
	clone->bi_bdev = bio->bi_bdev;
	clone->bi_rw = bio->bi_rw;
	clone->bi_vcnt = 1;
	clone->bi_size = to_bytes(len);
	clone->bi_io_vec->bv_offset = offset;
	clone->bi_io_vec->bv_len = clone->bi_size;
M
Martin K. Petersen 已提交
704
	clone->bi_flags |= 1 << BIO_CLONED;
L
Linus Torvalds 已提交
705 706 707 708 709 710 711 712 713

	return clone;
}

/*
 * Creates a bio that consists of range of complete bvecs.
 */
static struct bio *clone_bio(struct bio *bio, sector_t sector,
			     unsigned short idx, unsigned short bv_count,
S
Stefan Bader 已提交
714
			     unsigned int len, struct bio_set *bs)
L
Linus Torvalds 已提交
715 716 717
{
	struct bio *clone;

S
Stefan Bader 已提交
718 719 720
	clone = bio_alloc_bioset(GFP_NOIO, bio->bi_max_vecs, bs);
	__bio_clone(clone, bio);
	clone->bi_destructor = dm_bio_destructor;
L
Linus Torvalds 已提交
721 722 723 724 725 726 727 728 729
	clone->bi_sector = sector;
	clone->bi_idx = idx;
	clone->bi_vcnt = idx + bv_count;
	clone->bi_size = to_bytes(len);
	clone->bi_flags &= ~(1 << BIO_SEG_VALID);

	return clone;
}

730
static int __clone_and_map(struct clone_info *ci)
L
Linus Torvalds 已提交
731 732
{
	struct bio *clone, *bio = ci->bio;
733 734
	struct dm_target *ti;
	sector_t len = 0, max;
A
Alasdair G Kergon 已提交
735
	struct dm_target_io *tio;
L
Linus Torvalds 已提交
736

737 738 739 740 741 742
	ti = dm_table_find_target(ci->map, ci->sector);
	if (!dm_target_is_valid(ti))
		return -EIO;

	max = max_io_len(ci->md, ci->sector, ti);

L
Linus Torvalds 已提交
743 744 745 746 747 748 749 750 751 752 753 754 755 756
	/*
	 * Allocate a target io object.
	 */
	tio = alloc_tio(ci->md);
	tio->io = ci->io;
	tio->ti = ti;
	memset(&tio->info, 0, sizeof(tio->info));

	if (ci->sector_count <= max) {
		/*
		 * Optimise for the simple case where we can do all of
		 * the remaining io with a single clone.
		 */
		clone = clone_bio(bio, ci->sector, ci->idx,
S
Stefan Bader 已提交
757 758
				  bio->bi_vcnt - ci->idx, ci->sector_count,
				  ci->md->bs);
L
Linus Torvalds 已提交
759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780
		__map_bio(ti, clone, tio);
		ci->sector_count = 0;

	} else if (to_sector(bio->bi_io_vec[ci->idx].bv_len) <= max) {
		/*
		 * There are some bvecs that don't span targets.
		 * Do as many of these as possible.
		 */
		int i;
		sector_t remaining = max;
		sector_t bv_len;

		for (i = ci->idx; remaining && (i < bio->bi_vcnt); i++) {
			bv_len = to_sector(bio->bi_io_vec[i].bv_len);

			if (bv_len > remaining)
				break;

			remaining -= bv_len;
			len += bv_len;
		}

S
Stefan Bader 已提交
781 782
		clone = clone_bio(bio, ci->sector, ci->idx, i - ci->idx, len,
				  ci->md->bs);
L
Linus Torvalds 已提交
783 784 785 786 787 788 789 790
		__map_bio(ti, clone, tio);

		ci->sector += len;
		ci->sector_count -= len;
		ci->idx = i;

	} else {
		/*
791
		 * Handle a bvec that must be split between two or more targets.
L
Linus Torvalds 已提交
792 793
		 */
		struct bio_vec *bv = bio->bi_io_vec + ci->idx;
794 795
		sector_t remaining = to_sector(bv->bv_len);
		unsigned int offset = 0;
L
Linus Torvalds 已提交
796

797 798 799
		do {
			if (offset) {
				ti = dm_table_find_target(ci->map, ci->sector);
800 801 802
				if (!dm_target_is_valid(ti))
					return -EIO;

803
				max = max_io_len(ci->md, ci->sector, ti);
L
Linus Torvalds 已提交
804

805 806 807 808 809 810 811 812 813
				tio = alloc_tio(ci->md);
				tio->io = ci->io;
				tio->ti = ti;
				memset(&tio->info, 0, sizeof(tio->info));
			}

			len = min(remaining, max);

			clone = split_bvec(bio, ci->sector, ci->idx,
S
Stefan Bader 已提交
814 815
					   bv->bv_offset + offset, len,
					   ci->md->bs);
816 817 818 819 820 821 822

			__map_bio(ti, clone, tio);

			ci->sector += len;
			ci->sector_count -= len;
			offset += to_bytes(len);
		} while (remaining -= len);
L
Linus Torvalds 已提交
823 824 825

		ci->idx++;
	}
826 827

	return 0;
L
Linus Torvalds 已提交
828 829 830 831 832
}

/*
 * Split the bio into several clones.
 */
M
Milan Broz 已提交
833
static int __split_bio(struct mapped_device *md, struct bio *bio)
L
Linus Torvalds 已提交
834 835
{
	struct clone_info ci;
836
	int error = 0;
L
Linus Torvalds 已提交
837 838

	ci.map = dm_get_table(md);
M
Milan Broz 已提交
839 840
	if (unlikely(!ci.map))
		return -EIO;
841 842 843 844 845
	if (unlikely(bio_barrier(bio) && !dm_table_barrier_ok(ci.map))) {
		dm_table_put(ci.map);
		bio_endio(bio, -EOPNOTSUPP);
		return 0;
	}
L
Linus Torvalds 已提交
846 847 848 849 850 851 852 853 854 855 856
	ci.md = md;
	ci.bio = bio;
	ci.io = alloc_io(md);
	ci.io->error = 0;
	atomic_set(&ci.io->io_count, 1);
	ci.io->bio = bio;
	ci.io->md = md;
	ci.sector = bio->bi_sector;
	ci.sector_count = bio_sectors(bio);
	ci.idx = bio->bi_idx;

857
	start_io_acct(ci.io);
858 859
	while (ci.sector_count && !error)
		error = __clone_and_map(&ci);
L
Linus Torvalds 已提交
860 861

	/* drop the extra reference count */
862
	dec_pending(ci.io, error);
L
Linus Torvalds 已提交
863
	dm_table_put(ci.map);
M
Milan Broz 已提交
864 865

	return 0;
L
Linus Torvalds 已提交
866 867 868 869 870
}
/*-----------------------------------------------------------------
 * CRUD END
 *---------------------------------------------------------------*/

M
Milan Broz 已提交
871 872 873 874 875 876 877 878
static int dm_merge_bvec(struct request_queue *q,
			 struct bvec_merge_data *bvm,
			 struct bio_vec *biovec)
{
	struct mapped_device *md = q->queuedata;
	struct dm_table *map = dm_get_table(md);
	struct dm_target *ti;
	sector_t max_sectors;
879
	int max_size = 0;
M
Milan Broz 已提交
880 881

	if (unlikely(!map))
882
		goto out;
M
Milan Broz 已提交
883 884

	ti = dm_table_find_target(map, bvm->bi_sector);
885 886
	if (!dm_target_is_valid(ti))
		goto out_table;
M
Milan Broz 已提交
887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904

	/*
	 * Find maximum amount of I/O that won't need splitting
	 */
	max_sectors = min(max_io_len(md, bvm->bi_sector, ti),
			  (sector_t) BIO_MAX_SECTORS);
	max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size;
	if (max_size < 0)
		max_size = 0;

	/*
	 * merge_bvec_fn() returns number of bytes
	 * it can accept at this offset
	 * max is precomputed maximal io size
	 */
	if (max_size && ti->type->merge)
		max_size = ti->type->merge(ti, bvm, biovec, max_size);

905
out_table:
906 907 908
	dm_table_put(map);

out:
M
Milan Broz 已提交
909 910 911 912 913 914 915 916 917
	/*
	 * Always allow an entire first page
	 */
	if (max_size <= biovec->bv_len && !(bvm->bi_size >> SECTOR_SHIFT))
		max_size = biovec->bv_len;

	return max_size;
}

L
Linus Torvalds 已提交
918 919 920 921
/*
 * The request function that just remaps the bio built up by
 * dm_merge_bvec.
 */
922
static int dm_request(struct request_queue *q, struct bio *bio)
L
Linus Torvalds 已提交
923
{
M
Milan Broz 已提交
924
	int r = -EIO;
925
	int rw = bio_data_dir(bio);
L
Linus Torvalds 已提交
926
	struct mapped_device *md = q->queuedata;
T
Tejun Heo 已提交
927
	int cpu;
L
Linus Torvalds 已提交
928

929
	down_read(&md->io_lock);
L
Linus Torvalds 已提交
930

T
Tejun Heo 已提交
931 932 933 934
	cpu = part_stat_lock();
	part_stat_inc(cpu, &dm_disk(md)->part0, ios[rw]);
	part_stat_add(cpu, &dm_disk(md)->part0, sectors[rw], bio_sectors(bio));
	part_stat_unlock();
935

L
Linus Torvalds 已提交
936 937 938 939 940
	/*
	 * If we're suspended we have to queue
	 * this io for later.
	 */
	while (test_bit(DMF_BLOCK_IO, &md->flags)) {
941
		up_read(&md->io_lock);
L
Linus Torvalds 已提交
942

M
Milan Broz 已提交
943 944
		if (bio_rw(bio) != READA)
			r = queue_io(md, bio);
L
Linus Torvalds 已提交
945

M
Milan Broz 已提交
946 947
		if (r <= 0)
			goto out_req;
L
Linus Torvalds 已提交
948 949 950 951 952

		/*
		 * We're in a while loop, because someone could suspend
		 * before we get to the following read lock.
		 */
953
		down_read(&md->io_lock);
L
Linus Torvalds 已提交
954 955
	}

M
Milan Broz 已提交
956
	r = __split_bio(md, bio);
957
	up_read(&md->io_lock);
M
Milan Broz 已提交
958 959 960 961 962

out_req:
	if (r < 0)
		bio_io_error(bio);

L
Linus Torvalds 已提交
963 964 965
	return 0;
}

966
static void dm_unplug_all(struct request_queue *q)
L
Linus Torvalds 已提交
967 968 969 970 971 972 973 974 975 976 977 978
{
	struct mapped_device *md = q->queuedata;
	struct dm_table *map = dm_get_table(md);

	if (map) {
		dm_table_unplug_all(map);
		dm_table_put(map);
	}
}

static int dm_any_congested(void *congested_data, int bdi_bits)
{
979 980 981
	int r = bdi_bits;
	struct mapped_device *md = congested_data;
	struct dm_table *map;
L
Linus Torvalds 已提交
982

983 984 985 986 987 988 989 990
	if (!test_bit(DMF_BLOCK_IO, &md->flags)) {
		map = dm_get_table(md);
		if (map) {
			r = dm_table_any_congested(map, bdi_bits);
			dm_table_put(map);
		}
	}

L
Linus Torvalds 已提交
991 992 993 994 995 996 997 998
	return r;
}

/*-----------------------------------------------------------------
 * An IDR is used to keep track of allocated minor numbers.
 *---------------------------------------------------------------*/
static DEFINE_IDR(_minor_idr);

999
static void free_minor(int minor)
L
Linus Torvalds 已提交
1000
{
1001
	spin_lock(&_minor_lock);
L
Linus Torvalds 已提交
1002
	idr_remove(&_minor_idr, minor);
1003
	spin_unlock(&_minor_lock);
L
Linus Torvalds 已提交
1004 1005 1006 1007 1008
}

/*
 * See if the device with a specific minor # is free.
 */
1009
static int specific_minor(int minor)
L
Linus Torvalds 已提交
1010 1011 1012 1013 1014 1015
{
	int r, m;

	if (minor >= (1 << MINORBITS))
		return -EINVAL;

J
Jeff Mahoney 已提交
1016 1017 1018 1019
	r = idr_pre_get(&_minor_idr, GFP_KERNEL);
	if (!r)
		return -ENOMEM;

1020
	spin_lock(&_minor_lock);
L
Linus Torvalds 已提交
1021 1022 1023 1024 1025 1026

	if (idr_find(&_minor_idr, minor)) {
		r = -EBUSY;
		goto out;
	}

1027
	r = idr_get_new_above(&_minor_idr, MINOR_ALLOCED, minor, &m);
J
Jeff Mahoney 已提交
1028
	if (r)
L
Linus Torvalds 已提交
1029 1030 1031 1032 1033 1034 1035 1036 1037
		goto out;

	if (m != minor) {
		idr_remove(&_minor_idr, m);
		r = -EBUSY;
		goto out;
	}

out:
1038
	spin_unlock(&_minor_lock);
L
Linus Torvalds 已提交
1039 1040 1041
	return r;
}

1042
static int next_free_minor(int *minor)
L
Linus Torvalds 已提交
1043
{
1044
	int r, m;
L
Linus Torvalds 已提交
1045 1046

	r = idr_pre_get(&_minor_idr, GFP_KERNEL);
J
Jeff Mahoney 已提交
1047 1048 1049
	if (!r)
		return -ENOMEM;

1050
	spin_lock(&_minor_lock);
L
Linus Torvalds 已提交
1051

1052
	r = idr_get_new(&_minor_idr, MINOR_ALLOCED, &m);
1053
	if (r)
L
Linus Torvalds 已提交
1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064
		goto out;

	if (m >= (1 << MINORBITS)) {
		idr_remove(&_minor_idr, m);
		r = -ENOSPC;
		goto out;
	}

	*minor = m;

out:
1065
	spin_unlock(&_minor_lock);
L
Linus Torvalds 已提交
1066 1067 1068 1069 1070 1071 1072 1073
	return r;
}

static struct block_device_operations dm_blk_dops;

/*
 * Allocate and initialise a blank device with a given minor.
 */
1074
static struct mapped_device *alloc_dev(int minor)
L
Linus Torvalds 已提交
1075 1076
{
	int r;
1077
	struct mapped_device *md = kzalloc(sizeof(*md), GFP_KERNEL);
1078
	void *old_md;
L
Linus Torvalds 已提交
1079 1080 1081 1082 1083 1084

	if (!md) {
		DMWARN("unable to allocate device, out of memory.");
		return NULL;
	}

1085
	if (!try_module_get(THIS_MODULE))
M
Milan Broz 已提交
1086
		goto bad_module_get;
1087

L
Linus Torvalds 已提交
1088
	/* get a minor number for the dev */
1089
	if (minor == DM_ANY_MINOR)
1090
		r = next_free_minor(&minor);
1091
	else
1092
		r = specific_minor(minor);
L
Linus Torvalds 已提交
1093
	if (r < 0)
M
Milan Broz 已提交
1094
		goto bad_minor;
L
Linus Torvalds 已提交
1095

1096
	init_rwsem(&md->io_lock);
1097
	mutex_init(&md->suspend_lock);
1098
	spin_lock_init(&md->pushback_lock);
L
Linus Torvalds 已提交
1099 1100
	rwlock_init(&md->map_lock);
	atomic_set(&md->holders, 1);
1101
	atomic_set(&md->open_count, 0);
L
Linus Torvalds 已提交
1102
	atomic_set(&md->event_nr, 0);
M
Mike Anderson 已提交
1103 1104 1105
	atomic_set(&md->uevent_seq, 0);
	INIT_LIST_HEAD(&md->uevent_list);
	spin_lock_init(&md->uevent_lock);
L
Linus Torvalds 已提交
1106 1107 1108

	md->queue = blk_alloc_queue(GFP_KERNEL);
	if (!md->queue)
M
Milan Broz 已提交
1109
		goto bad_queue;
L
Linus Torvalds 已提交
1110 1111 1112 1113 1114

	md->queue->queuedata = md;
	md->queue->backing_dev_info.congested_fn = dm_any_congested;
	md->queue->backing_dev_info.congested_data = md;
	blk_queue_make_request(md->queue, dm_request);
1115
	blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);
L
Linus Torvalds 已提交
1116
	md->queue->unplug_fn = dm_unplug_all;
M
Milan Broz 已提交
1117
	blk_queue_merge_bvec(md->queue, dm_merge_bvec);
L
Linus Torvalds 已提交
1118

1119
	md->io_pool = mempool_create_slab_pool(MIN_IOS, _io_cache);
K
Kiyoshi Ueda 已提交
1120
	if (!md->io_pool)
M
Milan Broz 已提交
1121
		goto bad_io_pool;
L
Linus Torvalds 已提交
1122

1123
	md->tio_pool = mempool_create_slab_pool(MIN_IOS, _tio_cache);
L
Linus Torvalds 已提交
1124
	if (!md->tio_pool)
M
Milan Broz 已提交
1125
		goto bad_tio_pool;
L
Linus Torvalds 已提交
1126

1127
	md->bs = bioset_create(16, 0);
S
Stefan Bader 已提交
1128 1129 1130
	if (!md->bs)
		goto bad_no_bioset;

L
Linus Torvalds 已提交
1131 1132
	md->disk = alloc_disk(1);
	if (!md->disk)
M
Milan Broz 已提交
1133
		goto bad_disk;
L
Linus Torvalds 已提交
1134

1135 1136 1137 1138
	atomic_set(&md->pending, 0);
	init_waitqueue_head(&md->wait);
	init_waitqueue_head(&md->eventq);

L
Linus Torvalds 已提交
1139 1140 1141 1142 1143 1144 1145
	md->disk->major = _major;
	md->disk->first_minor = minor;
	md->disk->fops = &dm_blk_dops;
	md->disk->queue = md->queue;
	md->disk->private_data = md;
	sprintf(md->disk->disk_name, "dm-%d", minor);
	add_disk(md->disk);
M
Mike Anderson 已提交
1146
	format_dev_t(md->name, MKDEV(_major, minor));
L
Linus Torvalds 已提交
1147

1148 1149 1150 1151
	md->wq = create_singlethread_workqueue("kdmflush");
	if (!md->wq)
		goto bad_thread;

1152
	/* Populate the mapping, nobody knows we exist yet */
1153
	spin_lock(&_minor_lock);
1154
	old_md = idr_replace(&_minor_idr, md, minor);
1155
	spin_unlock(&_minor_lock);
1156 1157 1158

	BUG_ON(old_md != MINOR_ALLOCED);

L
Linus Torvalds 已提交
1159 1160
	return md;

1161 1162
bad_thread:
	put_disk(md->disk);
M
Milan Broz 已提交
1163
bad_disk:
S
Stefan Bader 已提交
1164
	bioset_free(md->bs);
M
Milan Broz 已提交
1165
bad_no_bioset:
L
Linus Torvalds 已提交
1166
	mempool_destroy(md->tio_pool);
M
Milan Broz 已提交
1167
bad_tio_pool:
L
Linus Torvalds 已提交
1168
	mempool_destroy(md->io_pool);
M
Milan Broz 已提交
1169
bad_io_pool:
1170
	blk_cleanup_queue(md->queue);
M
Milan Broz 已提交
1171
bad_queue:
L
Linus Torvalds 已提交
1172
	free_minor(minor);
M
Milan Broz 已提交
1173
bad_minor:
1174
	module_put(THIS_MODULE);
M
Milan Broz 已提交
1175
bad_module_get:
L
Linus Torvalds 已提交
1176 1177 1178 1179
	kfree(md);
	return NULL;
}

J
Jun'ichi Nomura 已提交
1180 1181
static void unlock_fs(struct mapped_device *md);

L
Linus Torvalds 已提交
1182 1183
static void free_dev(struct mapped_device *md)
{
1184
	int minor = MINOR(disk_devt(md->disk));
1185

1186
	if (md->suspended_bdev) {
J
Jun'ichi Nomura 已提交
1187
		unlock_fs(md);
1188 1189
		bdput(md->suspended_bdev);
	}
1190
	destroy_workqueue(md->wq);
L
Linus Torvalds 已提交
1191 1192
	mempool_destroy(md->tio_pool);
	mempool_destroy(md->io_pool);
S
Stefan Bader 已提交
1193
	bioset_free(md->bs);
L
Linus Torvalds 已提交
1194
	del_gendisk(md->disk);
1195
	free_minor(minor);
J
Jeff Mahoney 已提交
1196 1197 1198 1199 1200

	spin_lock(&_minor_lock);
	md->disk->private_data = NULL;
	spin_unlock(&_minor_lock);

L
Linus Torvalds 已提交
1201
	put_disk(md->disk);
1202
	blk_cleanup_queue(md->queue);
1203
	module_put(THIS_MODULE);
L
Linus Torvalds 已提交
1204 1205 1206 1207 1208 1209 1210 1211
	kfree(md);
}

/*
 * Bind a table to the device.
 */
static void event_callback(void *context)
{
M
Mike Anderson 已提交
1212 1213
	unsigned long flags;
	LIST_HEAD(uevents);
L
Linus Torvalds 已提交
1214 1215
	struct mapped_device *md = (struct mapped_device *) context;

M
Mike Anderson 已提交
1216 1217 1218 1219
	spin_lock_irqsave(&md->uevent_lock, flags);
	list_splice_init(&md->uevent_list, &uevents);
	spin_unlock_irqrestore(&md->uevent_lock, flags);

1220
	dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
M
Mike Anderson 已提交
1221

L
Linus Torvalds 已提交
1222 1223 1224 1225
	atomic_inc(&md->event_nr);
	wake_up(&md->eventq);
}

1226
static void __set_size(struct mapped_device *md, sector_t size)
L
Linus Torvalds 已提交
1227
{
1228
	set_capacity(md->disk, size);
L
Linus Torvalds 已提交
1229

1230
	mutex_lock(&md->suspended_bdev->bd_inode->i_mutex);
1231
	i_size_write(md->suspended_bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
1232
	mutex_unlock(&md->suspended_bdev->bd_inode->i_mutex);
L
Linus Torvalds 已提交
1233 1234 1235 1236
}

static int __bind(struct mapped_device *md, struct dm_table *t)
{
1237
	struct request_queue *q = md->queue;
L
Linus Torvalds 已提交
1238 1239 1240
	sector_t size;

	size = dm_table_get_size(t);
D
Darrick J. Wong 已提交
1241 1242 1243 1244 1245 1246 1247

	/*
	 * Wipe any geometry if the size of the table changed.
	 */
	if (size != get_capacity(md->disk))
		memset(&md->geometry, 0, sizeof(md->geometry));

1248 1249
	if (md->suspended_bdev)
		__set_size(md, size);
1250 1251 1252

	if (!size) {
		dm_table_destroy(t);
L
Linus Torvalds 已提交
1253
		return 0;
1254
	}
L
Linus Torvalds 已提交
1255

1256 1257
	dm_table_event_callback(t, event_callback, md);

L
Linus Torvalds 已提交
1258 1259
	write_lock(&md->map_lock);
	md->map = t;
1260
	dm_table_set_restrictions(t, q);
L
Linus Torvalds 已提交
1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276
	write_unlock(&md->map_lock);

	return 0;
}

static void __unbind(struct mapped_device *md)
{
	struct dm_table *map = md->map;

	if (!map)
		return;

	dm_table_event_callback(map, NULL, NULL);
	write_lock(&md->map_lock);
	md->map = NULL;
	write_unlock(&md->map_lock);
1277
	dm_table_destroy(map);
L
Linus Torvalds 已提交
1278 1279 1280 1281 1282
}

/*
 * Constructor for a new device.
 */
1283
int dm_create(int minor, struct mapped_device **result)
L
Linus Torvalds 已提交
1284 1285 1286
{
	struct mapped_device *md;

1287
	md = alloc_dev(minor);
L
Linus Torvalds 已提交
1288 1289 1290
	if (!md)
		return -ENXIO;

M
Milan Broz 已提交
1291 1292
	dm_sysfs_init(md);

L
Linus Torvalds 已提交
1293 1294 1295 1296
	*result = md;
	return 0;
}

1297
static struct mapped_device *dm_find_md(dev_t dev)
L
Linus Torvalds 已提交
1298 1299 1300 1301 1302 1303 1304
{
	struct mapped_device *md;
	unsigned minor = MINOR(dev);

	if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
		return NULL;

1305
	spin_lock(&_minor_lock);
L
Linus Torvalds 已提交
1306 1307

	md = idr_find(&_minor_idr, minor);
J
Jeff Mahoney 已提交
1308
	if (md && (md == MINOR_ALLOCED ||
1309
		   (MINOR(disk_devt(dm_disk(md))) != minor) ||
A
Alasdair G Kergon 已提交
1310
		   test_bit(DMF_FREEING, &md->flags))) {
1311
		md = NULL;
J
Jeff Mahoney 已提交
1312 1313
		goto out;
	}
L
Linus Torvalds 已提交
1314

J
Jeff Mahoney 已提交
1315
out:
1316
	spin_unlock(&_minor_lock);
L
Linus Torvalds 已提交
1317

1318 1319 1320
	return md;
}

1321 1322 1323 1324 1325 1326 1327 1328 1329 1330
struct mapped_device *dm_get_md(dev_t dev)
{
	struct mapped_device *md = dm_find_md(dev);

	if (md)
		dm_get(md);

	return md;
}

A
Alasdair G Kergon 已提交
1331
void *dm_get_mdptr(struct mapped_device *md)
1332
{
A
Alasdair G Kergon 已提交
1333
	return md->interface_ptr;
L
Linus Torvalds 已提交
1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345
}

void dm_set_mdptr(struct mapped_device *md, void *ptr)
{
	md->interface_ptr = ptr;
}

void dm_get(struct mapped_device *md)
{
	atomic_inc(&md->holders);
}

1346 1347 1348 1349 1350 1351
const char *dm_device_name(struct mapped_device *md)
{
	return md->name;
}
EXPORT_SYMBOL_GPL(dm_device_name);

L
Linus Torvalds 已提交
1352 1353
void dm_put(struct mapped_device *md)
{
M
Mike Anderson 已提交
1354
	struct dm_table *map;
L
Linus Torvalds 已提交
1355

J
Jeff Mahoney 已提交
1356 1357
	BUG_ON(test_bit(DMF_FREEING, &md->flags));

1358
	if (atomic_dec_and_lock(&md->holders, &_minor_lock)) {
M
Mike Anderson 已提交
1359
		map = dm_get_table(md);
1360 1361
		idr_replace(&_minor_idr, MINOR_ALLOCED,
			    MINOR(disk_devt(dm_disk(md))));
J
Jeff Mahoney 已提交
1362
		set_bit(DMF_FREEING, &md->flags);
1363
		spin_unlock(&_minor_lock);
1364
		if (!dm_suspended(md)) {
L
Linus Torvalds 已提交
1365 1366 1367
			dm_table_presuspend_targets(map);
			dm_table_postsuspend_targets(map);
		}
M
Milan Broz 已提交
1368
		dm_sysfs_exit(md);
M
Mike Anderson 已提交
1369
		dm_table_put(map);
1370
		__unbind(md);
L
Linus Torvalds 已提交
1371 1372 1373
		free_dev(md);
	}
}
E
Edward Goggin 已提交
1374
EXPORT_SYMBOL_GPL(dm_put);
L
Linus Torvalds 已提交
1375

1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398
static int dm_wait_for_completion(struct mapped_device *md)
{
	int r = 0;

	while (1) {
		set_current_state(TASK_INTERRUPTIBLE);

		smp_mb();
		if (!atomic_read(&md->pending))
			break;

		if (signal_pending(current)) {
			r = -EINTR;
			break;
		}

		io_schedule();
	}
	set_current_state(TASK_RUNNING);

	return r;
}

L
Linus Torvalds 已提交
1399 1400 1401
/*
 * Process the deferred bios
 */
1402
static void __flush_deferred_io(struct mapped_device *md)
L
Linus Torvalds 已提交
1403
{
1404
	struct bio *c;
L
Linus Torvalds 已提交
1405

1406
	while ((c = bio_list_pop(&md->deferred))) {
M
Milan Broz 已提交
1407 1408
		if (__split_bio(md, c))
			bio_io_error(c);
L
Linus Torvalds 已提交
1409
	}
M
Milan Broz 已提交
1410 1411

	clear_bit(DMF_BLOCK_IO, &md->flags);
L
Linus Torvalds 已提交
1412 1413
}

1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424
static void __merge_pushback_list(struct mapped_device *md)
{
	unsigned long flags;

	spin_lock_irqsave(&md->pushback_lock, flags);
	clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
	bio_list_merge_head(&md->deferred, &md->pushback);
	bio_list_init(&md->pushback);
	spin_unlock_irqrestore(&md->pushback_lock, flags);
}

1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459
static void dm_wq_work(struct work_struct *work)
{
	struct dm_wq_req *req = container_of(work, struct dm_wq_req, work);
	struct mapped_device *md = req->md;

	down_write(&md->io_lock);
	switch (req->type) {
	case DM_WQ_FLUSH_DEFERRED:
		__flush_deferred_io(md);
		break;
	default:
		DMERR("dm_wq_work: unrecognised work type %d", req->type);
		BUG();
	}
	up_write(&md->io_lock);
}

static void dm_wq_queue(struct mapped_device *md, int type, void *context,
			struct dm_wq_req *req)
{
	req->type = type;
	req->md = md;
	req->context = context;
	INIT_WORK(&req->work, dm_wq_work);
	queue_work(md->wq, &req->work);
}

static void dm_queue_flush(struct mapped_device *md, int type, void *context)
{
	struct dm_wq_req req;

	dm_wq_queue(md, type, context, &req);
	flush_workqueue(md->wq);
}

L
Linus Torvalds 已提交
1460 1461 1462 1463 1464
/*
 * Swap in a new table (destroying old one).
 */
int dm_swap_table(struct mapped_device *md, struct dm_table *table)
{
1465
	int r = -EINVAL;
L
Linus Torvalds 已提交
1466

1467
	mutex_lock(&md->suspend_lock);
L
Linus Torvalds 已提交
1468 1469

	/* device must be suspended */
1470
	if (!dm_suspended(md))
1471
		goto out;
L
Linus Torvalds 已提交
1472

1473 1474 1475 1476 1477
	/* without bdev, the device size cannot be changed */
	if (!md->suspended_bdev)
		if (get_capacity(md->disk) != dm_table_get_size(table))
			goto out;

L
Linus Torvalds 已提交
1478 1479 1480
	__unbind(md);
	r = __bind(md, table);

1481
out:
1482
	mutex_unlock(&md->suspend_lock);
1483
	return r;
L
Linus Torvalds 已提交
1484 1485 1486 1487 1488 1489
}

/*
 * Functions to lock and unlock any filesystem running on the
 * device.
 */
1490
static int lock_fs(struct mapped_device *md)
L
Linus Torvalds 已提交
1491
{
1492
	int r;
L
Linus Torvalds 已提交
1493 1494

	WARN_ON(md->frozen_sb);
1495

1496
	md->frozen_sb = freeze_bdev(md->suspended_bdev);
1497
	if (IS_ERR(md->frozen_sb)) {
1498
		r = PTR_ERR(md->frozen_sb);
1499 1500
		md->frozen_sb = NULL;
		return r;
1501 1502
	}

1503 1504
	set_bit(DMF_FROZEN, &md->flags);

L
Linus Torvalds 已提交
1505
	/* don't bdput right now, we don't want the bdev
1506
	 * to go away while it is locked.
L
Linus Torvalds 已提交
1507 1508 1509 1510
	 */
	return 0;
}

1511
static void unlock_fs(struct mapped_device *md)
L
Linus Torvalds 已提交
1512
{
1513 1514 1515
	if (!test_bit(DMF_FROZEN, &md->flags))
		return;

1516
	thaw_bdev(md->suspended_bdev, md->frozen_sb);
L
Linus Torvalds 已提交
1517
	md->frozen_sb = NULL;
1518
	clear_bit(DMF_FROZEN, &md->flags);
L
Linus Torvalds 已提交
1519 1520 1521 1522 1523 1524 1525 1526 1527
}

/*
 * We need to be able to change a mapping table under a mounted
 * filesystem.  For example we might want to move some data in
 * the background.  Before the table can be swapped with
 * dm_bind_table, dm_suspend must be called to flush any in
 * flight bios and ensure that any further io gets deferred.
 */
1528
int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
L
Linus Torvalds 已提交
1529
{
1530
	struct dm_table *map = NULL;
L
Linus Torvalds 已提交
1531
	DECLARE_WAITQUEUE(wait, current);
1532
	int r = 0;
1533
	int do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG ? 1 : 0;
1534
	int noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG ? 1 : 0;
L
Linus Torvalds 已提交
1535

1536
	mutex_lock(&md->suspend_lock);
1537

M
Milan Broz 已提交
1538 1539
	if (dm_suspended(md)) {
		r = -EINVAL;
1540
		goto out_unlock;
M
Milan Broz 已提交
1541
	}
L
Linus Torvalds 已提交
1542 1543 1544

	map = dm_get_table(md);

1545 1546 1547 1548 1549 1550 1551
	/*
	 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
	 * This flag is cleared before dm_suspend returns.
	 */
	if (noflush)
		set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);

1552 1553 1554
	/* This does not get reverted if there's an error later. */
	dm_table_presuspend_targets(map);

1555 1556 1557 1558 1559 1560
	/* bdget() can stall if the pending I/Os are not flushed */
	if (!noflush) {
		md->suspended_bdev = bdget_disk(md->disk, 0);
		if (!md->suspended_bdev) {
			DMWARN("bdget failed in dm_suspend");
			r = -ENOMEM;
K
Kiyoshi Ueda 已提交
1561
			goto out;
1562
		}
1563

1564 1565 1566 1567 1568 1569 1570 1571 1572
		/*
		 * Flush I/O to the device. noflush supersedes do_lockfs,
		 * because lock_fs() needs to flush I/Os.
		 */
		if (do_lockfs) {
			r = lock_fs(md);
			if (r)
				goto out;
		}
1573
	}
L
Linus Torvalds 已提交
1574 1575

	/*
1576
	 * First we set the BLOCK_IO flag so no more ios will be mapped.
L
Linus Torvalds 已提交
1577
	 */
1578 1579
	down_write(&md->io_lock);
	set_bit(DMF_BLOCK_IO, &md->flags);
L
Linus Torvalds 已提交
1580 1581

	add_wait_queue(&md->wait, &wait);
1582
	up_write(&md->io_lock);
L
Linus Torvalds 已提交
1583 1584

	/* unplug */
1585
	if (map)
L
Linus Torvalds 已提交
1586 1587 1588
		dm_table_unplug_all(map);

	/*
1589
	 * Wait for the already-mapped ios to complete.
L
Linus Torvalds 已提交
1590
	 */
1591
	r = dm_wait_for_completion(md);
L
Linus Torvalds 已提交
1592

1593
	down_write(&md->io_lock);
L
Linus Torvalds 已提交
1594 1595
	remove_wait_queue(&md->wait, &wait);

1596 1597
	if (noflush)
		__merge_pushback_list(md);
1598
	up_write(&md->io_lock);
1599

L
Linus Torvalds 已提交
1600
	/* were we interrupted ? */
1601
	if (r < 0) {
1602
		dm_queue_flush(md, DM_WQ_FLUSH_DEFERRED, NULL);
M
Milan Broz 已提交
1603

1604
		unlock_fs(md);
1605
		goto out; /* pushback list is already flushed, so skip flush */
1606
	}
L
Linus Torvalds 已提交
1607

1608
	dm_table_postsuspend_targets(map);
L
Linus Torvalds 已提交
1609

1610
	set_bit(DMF_SUSPENDED, &md->flags);
1611

1612
out:
1613 1614 1615 1616 1617
	if (r && md->suspended_bdev) {
		bdput(md->suspended_bdev);
		md->suspended_bdev = NULL;
	}

1618
	dm_table_put(map);
1619 1620

out_unlock:
1621
	mutex_unlock(&md->suspend_lock);
1622
	return r;
L
Linus Torvalds 已提交
1623 1624 1625 1626
}

int dm_resume(struct mapped_device *md)
{
1627 1628
	int r = -EINVAL;
	struct dm_table *map = NULL;
L
Linus Torvalds 已提交
1629

1630
	mutex_lock(&md->suspend_lock);
1631
	if (!dm_suspended(md))
1632 1633 1634
		goto out;

	map = dm_get_table(md);
1635
	if (!map || !dm_table_get_size(map))
1636
		goto out;
L
Linus Torvalds 已提交
1637

1638 1639 1640
	r = dm_table_resume_targets(map);
	if (r)
		goto out;
1641

1642
	dm_queue_flush(md, DM_WQ_FLUSH_DEFERRED, NULL);
1643 1644 1645

	unlock_fs(md);

1646 1647 1648 1649
	if (md->suspended_bdev) {
		bdput(md->suspended_bdev);
		md->suspended_bdev = NULL;
	}
1650

1651 1652
	clear_bit(DMF_SUSPENDED, &md->flags);

L
Linus Torvalds 已提交
1653 1654
	dm_table_unplug_all(map);

1655
	dm_kobject_uevent(md);
1656

1657
	r = 0;
1658

1659 1660
out:
	dm_table_put(map);
1661
	mutex_unlock(&md->suspend_lock);
1662

1663
	return r;
L
Linus Torvalds 已提交
1664 1665 1666 1667 1668
}

/*-----------------------------------------------------------------
 * Event notification.
 *---------------------------------------------------------------*/
1669 1670
void dm_kobject_uevent(struct mapped_device *md)
{
1671
	kobject_uevent(&disk_to_dev(md->disk)->kobj, KOBJ_CHANGE);
1672 1673
}

M
Mike Anderson 已提交
1674 1675 1676 1677 1678
uint32_t dm_next_uevent_seq(struct mapped_device *md)
{
	return atomic_add_return(1, &md->uevent_seq);
}

L
Linus Torvalds 已提交
1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689
uint32_t dm_get_event_nr(struct mapped_device *md)
{
	return atomic_read(&md->event_nr);
}

int dm_wait_event(struct mapped_device *md, int event_nr)
{
	return wait_event_interruptible(md->eventq,
			(event_nr != atomic_read(&md->event_nr)));
}

M
Mike Anderson 已提交
1690 1691 1692 1693 1694 1695 1696 1697 1698
void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
{
	unsigned long flags;

	spin_lock_irqsave(&md->uevent_lock, flags);
	list_add(elist, &md->uevent_list);
	spin_unlock_irqrestore(&md->uevent_lock, flags);
}

L
Linus Torvalds 已提交
1699 1700 1701 1702 1703 1704 1705 1706 1707
/*
 * The gendisk is only valid as long as you have a reference
 * count on 'md'.
 */
struct gendisk *dm_disk(struct mapped_device *md)
{
	return md->disk;
}

M
Milan Broz 已提交
1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728
struct kobject *dm_kobject(struct mapped_device *md)
{
	return &md->kobj;
}

/*
 * struct mapped_device should not be exported outside of dm.c
 * so use this check to verify that kobj is part of md structure
 */
struct mapped_device *dm_get_from_kobject(struct kobject *kobj)
{
	struct mapped_device *md;

	md = container_of(kobj, struct mapped_device, kobj);
	if (&md->kobj != kobj)
		return NULL;

	dm_get(md);
	return md;
}

L
Linus Torvalds 已提交
1729 1730 1731 1732 1733
int dm_suspended(struct mapped_device *md)
{
	return test_bit(DMF_SUSPENDED, &md->flags);
}

1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744
int dm_noflush_suspending(struct dm_target *ti)
{
	struct mapped_device *md = dm_table_get_md(ti->table);
	int r = __noflush_suspending(md);

	dm_put(md);

	return r;
}
EXPORT_SYMBOL_GPL(dm_noflush_suspending);

L
Linus Torvalds 已提交
1745 1746 1747
static struct block_device_operations dm_blk_dops = {
	.open = dm_blk_open,
	.release = dm_blk_close,
1748
	.ioctl = dm_blk_ioctl,
D
Darrick J. Wong 已提交
1749
	.getgeo = dm_blk_getgeo,
L
Linus Torvalds 已提交
1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765
	.owner = THIS_MODULE
};

EXPORT_SYMBOL(dm_get_mapinfo);

/*
 * module hooks
 */
module_init(dm_init);
module_exit(dm_exit);

module_param(major, uint, 0);
MODULE_PARM_DESC(major, "The major number of the device mapper");
MODULE_DESCRIPTION(DM_NAME " driver");
MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
MODULE_LICENSE("GPL");