dm.c 38.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2
/*
 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
M
Milan Broz 已提交
3
 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
L
Linus Torvalds 已提交
4 5 6 7 8
 *
 * This file is released under the GPL.
 */

#include "dm.h"
M
Mike Anderson 已提交
9
#include "dm-uevent.h"
L
Linus Torvalds 已提交
10 11 12

#include <linux/init.h>
#include <linux/module.h>
A
Arjan van de Ven 已提交
13
#include <linux/mutex.h>
L
Linus Torvalds 已提交
14 15 16 17 18 19 20
#include <linux/moduleparam.h>
#include <linux/blkpg.h>
#include <linux/bio.h>
#include <linux/buffer_head.h>
#include <linux/mempool.h>
#include <linux/slab.h>
#include <linux/idr.h>
D
Darrick J. Wong 已提交
21
#include <linux/hdreg.h>
22 23

#include <trace/events/block.h>
L
Linus Torvalds 已提交
24

25 26
#define DM_MSG_PREFIX "core"

L
Linus Torvalds 已提交
27 28 29 30 31
static const char *_name = DM_NAME;

static unsigned int major = 0;
static unsigned int _major = 0;

32
static DEFINE_SPINLOCK(_minor_lock);
L
Linus Torvalds 已提交
33
/*
K
Kiyoshi Ueda 已提交
34
 * For bio-based dm.
L
Linus Torvalds 已提交
35 36 37 38 39 40
 * One of these is allocated per bio.
 */
struct dm_io {
	struct mapped_device *md;
	int error;
	atomic_t io_count;
R
Richard Kennedy 已提交
41
	struct bio *bio;
42
	unsigned long start_time;
L
Linus Torvalds 已提交
43 44 45
};

/*
K
Kiyoshi Ueda 已提交
46
 * For bio-based dm.
L
Linus Torvalds 已提交
47 48 49
 * One of these is allocated per target within a bio.  Hopefully
 * this will be simplified out one day.
 */
A
Alasdair G Kergon 已提交
50
struct dm_target_io {
L
Linus Torvalds 已提交
51 52 53 54 55
	struct dm_io *io;
	struct dm_target *ti;
	union map_info info;
};

K
Kiyoshi Ueda 已提交
56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76
/*
 * For request-based dm.
 * One of these is allocated per request.
 */
struct dm_rq_target_io {
	struct mapped_device *md;
	struct dm_target *ti;
	struct request *orig, clone;
	int error;
	union map_info info;
};

/*
 * For request-based dm.
 * One of these is allocated per bio.
 */
struct dm_rq_clone_bio_info {
	struct bio *orig;
	struct request *rq;
};

L
Linus Torvalds 已提交
77 78
union map_info *dm_get_mapinfo(struct bio *bio)
{
A
Alasdair G Kergon 已提交
79
	if (bio && bio->bi_private)
A
Alasdair G Kergon 已提交
80
		return &((struct dm_target_io *)bio->bi_private)->info;
A
Alasdair G Kergon 已提交
81
	return NULL;
L
Linus Torvalds 已提交
82 83
}

84 85
#define MINOR_ALLOCED ((void *)-1)

L
Linus Torvalds 已提交
86 87 88
/*
 * Bits for the md->flags field.
 */
89
#define DMF_BLOCK_IO_FOR_SUSPEND 0
L
Linus Torvalds 已提交
90
#define DMF_SUSPENDED 1
91
#define DMF_FROZEN 2
J
Jeff Mahoney 已提交
92
#define DMF_FREEING 3
93
#define DMF_DELETING 4
94
#define DMF_NOFLUSH_SUSPENDING 5
95
#define DMF_QUEUE_IO_TO_THREAD 6
L
Linus Torvalds 已提交
96

97 98 99
/*
 * Work processed by per-device workqueue.
 */
L
Linus Torvalds 已提交
100
struct mapped_device {
101
	struct rw_semaphore io_lock;
102
	struct mutex suspend_lock;
L
Linus Torvalds 已提交
103 104
	rwlock_t map_lock;
	atomic_t holders;
105
	atomic_t open_count;
L
Linus Torvalds 已提交
106 107 108

	unsigned long flags;

109
	struct request_queue *queue;
L
Linus Torvalds 已提交
110
	struct gendisk *disk;
M
Mike Anderson 已提交
111
	char name[16];
L
Linus Torvalds 已提交
112 113 114 115 116 117 118 119

	void *interface_ptr;

	/*
	 * A list of ios that arrived while we were suspended.
	 */
	atomic_t pending;
	wait_queue_head_t wait;
120
	struct work_struct work;
K
Kiyoshi Ueda 已提交
121
	struct bio_list deferred;
122
	spinlock_t deferred_lock;
L
Linus Torvalds 已提交
123

124 125 126 127 128
	/*
	 * An error from the barrier request currently being processed.
	 */
	int barrier_error;

129 130 131 132 133
	/*
	 * Processing queue (flush/barriers)
	 */
	struct workqueue_struct *wq;

L
Linus Torvalds 已提交
134 135 136 137 138 139 140 141 142 143 144
	/*
	 * The current mapping.
	 */
	struct dm_table *map;

	/*
	 * io objects are allocated from here.
	 */
	mempool_t *io_pool;
	mempool_t *tio_pool;

S
Stefan Bader 已提交
145 146
	struct bio_set *bs;

L
Linus Torvalds 已提交
147 148 149 150 151
	/*
	 * Event handling.
	 */
	atomic_t event_nr;
	wait_queue_head_t eventq;
M
Mike Anderson 已提交
152 153 154
	atomic_t uevent_seq;
	struct list_head uevent_list;
	spinlock_t uevent_lock; /* Protect access to uevent_list */
L
Linus Torvalds 已提交
155 156 157 158 159

	/*
	 * freeze/thaw support require holding onto a super block
	 */
	struct super_block *frozen_sb;
160
	struct block_device *bdev;
D
Darrick J. Wong 已提交
161 162 163

	/* forced geometry settings */
	struct hd_geometry geometry;
M
Milan Broz 已提交
164 165 166

	/* sysfs handle */
	struct kobject kobj;
L
Linus Torvalds 已提交
167 168 169
};

#define MIN_IOS 256
170 171
static struct kmem_cache *_io_cache;
static struct kmem_cache *_tio_cache;
K
Kiyoshi Ueda 已提交
172 173
static struct kmem_cache *_rq_tio_cache;
static struct kmem_cache *_rq_bio_info_cache;
L
Linus Torvalds 已提交
174 175 176

static int __init local_init(void)
{
K
Kiyoshi Ueda 已提交
177
	int r = -ENOMEM;
L
Linus Torvalds 已提交
178 179

	/* allocate a slab for the dm_ios */
A
Alasdair G Kergon 已提交
180
	_io_cache = KMEM_CACHE(dm_io, 0);
L
Linus Torvalds 已提交
181
	if (!_io_cache)
K
Kiyoshi Ueda 已提交
182
		return r;
L
Linus Torvalds 已提交
183 184

	/* allocate a slab for the target ios */
A
Alasdair G Kergon 已提交
185
	_tio_cache = KMEM_CACHE(dm_target_io, 0);
K
Kiyoshi Ueda 已提交
186 187
	if (!_tio_cache)
		goto out_free_io_cache;
L
Linus Torvalds 已提交
188

K
Kiyoshi Ueda 已提交
189 190 191 192 193 194 195 196
	_rq_tio_cache = KMEM_CACHE(dm_rq_target_io, 0);
	if (!_rq_tio_cache)
		goto out_free_tio_cache;

	_rq_bio_info_cache = KMEM_CACHE(dm_rq_clone_bio_info, 0);
	if (!_rq_bio_info_cache)
		goto out_free_rq_tio_cache;

M
Mike Anderson 已提交
197
	r = dm_uevent_init();
K
Kiyoshi Ueda 已提交
198
	if (r)
K
Kiyoshi Ueda 已提交
199
		goto out_free_rq_bio_info_cache;
M
Mike Anderson 已提交
200

L
Linus Torvalds 已提交
201 202
	_major = major;
	r = register_blkdev(_major, _name);
K
Kiyoshi Ueda 已提交
203 204
	if (r < 0)
		goto out_uevent_exit;
L
Linus Torvalds 已提交
205 206 207 208 209

	if (!_major)
		_major = r;

	return 0;
K
Kiyoshi Ueda 已提交
210 211 212

out_uevent_exit:
	dm_uevent_exit();
K
Kiyoshi Ueda 已提交
213 214 215 216
out_free_rq_bio_info_cache:
	kmem_cache_destroy(_rq_bio_info_cache);
out_free_rq_tio_cache:
	kmem_cache_destroy(_rq_tio_cache);
K
Kiyoshi Ueda 已提交
217 218 219 220 221 222
out_free_tio_cache:
	kmem_cache_destroy(_tio_cache);
out_free_io_cache:
	kmem_cache_destroy(_io_cache);

	return r;
L
Linus Torvalds 已提交
223 224 225 226
}

static void local_exit(void)
{
K
Kiyoshi Ueda 已提交
227 228
	kmem_cache_destroy(_rq_bio_info_cache);
	kmem_cache_destroy(_rq_tio_cache);
L
Linus Torvalds 已提交
229 230
	kmem_cache_destroy(_tio_cache);
	kmem_cache_destroy(_io_cache);
231
	unregister_blkdev(_major, _name);
M
Mike Anderson 已提交
232
	dm_uevent_exit();
L
Linus Torvalds 已提交
233 234 235 236 237 238

	_major = 0;

	DMINFO("cleaned up");
}

239
static int (*_inits[])(void) __initdata = {
L
Linus Torvalds 已提交
240 241 242 243
	local_init,
	dm_target_init,
	dm_linear_init,
	dm_stripe_init,
244
	dm_kcopyd_init,
L
Linus Torvalds 已提交
245 246 247
	dm_interface_init,
};

248
static void (*_exits[])(void) = {
L
Linus Torvalds 已提交
249 250 251 252
	local_exit,
	dm_target_exit,
	dm_linear_exit,
	dm_stripe_exit,
253
	dm_kcopyd_exit,
L
Linus Torvalds 已提交
254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288
	dm_interface_exit,
};

static int __init dm_init(void)
{
	const int count = ARRAY_SIZE(_inits);

	int r, i;

	for (i = 0; i < count; i++) {
		r = _inits[i]();
		if (r)
			goto bad;
	}

	return 0;

      bad:
	while (i--)
		_exits[i]();

	return r;
}

static void __exit dm_exit(void)
{
	int i = ARRAY_SIZE(_exits);

	while (i--)
		_exits[i]();
}

/*
 * Block device functions
 */
A
Al Viro 已提交
289
static int dm_blk_open(struct block_device *bdev, fmode_t mode)
L
Linus Torvalds 已提交
290 291 292
{
	struct mapped_device *md;

J
Jeff Mahoney 已提交
293 294
	spin_lock(&_minor_lock);

A
Al Viro 已提交
295
	md = bdev->bd_disk->private_data;
J
Jeff Mahoney 已提交
296 297 298
	if (!md)
		goto out;

299 300
	if (test_bit(DMF_FREEING, &md->flags) ||
	    test_bit(DMF_DELETING, &md->flags)) {
J
Jeff Mahoney 已提交
301 302 303 304
		md = NULL;
		goto out;
	}

L
Linus Torvalds 已提交
305
	dm_get(md);
306
	atomic_inc(&md->open_count);
J
Jeff Mahoney 已提交
307 308 309 310 311

out:
	spin_unlock(&_minor_lock);

	return md ? 0 : -ENXIO;
L
Linus Torvalds 已提交
312 313
}

A
Al Viro 已提交
314
static int dm_blk_close(struct gendisk *disk, fmode_t mode)
L
Linus Torvalds 已提交
315
{
A
Al Viro 已提交
316
	struct mapped_device *md = disk->private_data;
317
	atomic_dec(&md->open_count);
L
Linus Torvalds 已提交
318 319 320 321
	dm_put(md);
	return 0;
}

322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345
int dm_open_count(struct mapped_device *md)
{
	return atomic_read(&md->open_count);
}

/*
 * Guarantees nothing is using the device before it's deleted.
 */
int dm_lock_for_deletion(struct mapped_device *md)
{
	int r = 0;

	spin_lock(&_minor_lock);

	if (dm_open_count(md))
		r = -EBUSY;
	else
		set_bit(DMF_DELETING, &md->flags);

	spin_unlock(&_minor_lock);

	return r;
}

D
Darrick J. Wong 已提交
346 347 348 349 350 351 352
static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
{
	struct mapped_device *md = bdev->bd_disk->private_data;

	return dm_get_geometry(md, geo);
}

A
Al Viro 已提交
353
static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
354 355
			unsigned int cmd, unsigned long arg)
{
A
Al Viro 已提交
356 357
	struct mapped_device *md = bdev->bd_disk->private_data;
	struct dm_table *map = dm_get_table(md);
358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375
	struct dm_target *tgt;
	int r = -ENOTTY;

	if (!map || !dm_table_get_size(map))
		goto out;

	/* We only support devices that have a single target */
	if (dm_table_get_num_targets(map) != 1)
		goto out;

	tgt = dm_table_get_target(map, 0);

	if (dm_suspended(md)) {
		r = -EAGAIN;
		goto out;
	}

	if (tgt->type->ioctl)
376
		r = tgt->type->ioctl(tgt, cmd, arg);
377 378 379 380 381 382 383

out:
	dm_table_put(map);

	return r;
}

A
Alasdair G Kergon 已提交
384
static struct dm_io *alloc_io(struct mapped_device *md)
L
Linus Torvalds 已提交
385 386 387 388
{
	return mempool_alloc(md->io_pool, GFP_NOIO);
}

A
Alasdair G Kergon 已提交
389
static void free_io(struct mapped_device *md, struct dm_io *io)
L
Linus Torvalds 已提交
390 391 392 393
{
	mempool_free(io, md->io_pool);
}

A
Alasdair G Kergon 已提交
394
static struct dm_target_io *alloc_tio(struct mapped_device *md)
L
Linus Torvalds 已提交
395 396 397 398
{
	return mempool_alloc(md->tio_pool, GFP_NOIO);
}

A
Alasdair G Kergon 已提交
399
static void free_tio(struct mapped_device *md, struct dm_target_io *tio)
L
Linus Torvalds 已提交
400 401 402 403
{
	mempool_free(tio, md->tio_pool);
}

404 405 406
static void start_io_acct(struct dm_io *io)
{
	struct mapped_device *md = io->md;
T
Tejun Heo 已提交
407
	int cpu;
408 409 410

	io->start_time = jiffies;

T
Tejun Heo 已提交
411 412 413 414
	cpu = part_stat_lock();
	part_round_stats(cpu, &dm_disk(md)->part0);
	part_stat_unlock();
	dm_disk(md)->part0.in_flight = atomic_inc_return(&md->pending);
415 416
}

417
static void end_io_acct(struct dm_io *io)
418 419 420 421
{
	struct mapped_device *md = io->md;
	struct bio *bio = io->bio;
	unsigned long duration = jiffies - io->start_time;
T
Tejun Heo 已提交
422
	int pending, cpu;
423 424
	int rw = bio_data_dir(bio);

T
Tejun Heo 已提交
425 426 427 428
	cpu = part_stat_lock();
	part_round_stats(cpu, &dm_disk(md)->part0);
	part_stat_add(cpu, &dm_disk(md)->part0, ticks[rw], duration);
	part_stat_unlock();
429

430 431 432 433
	/*
	 * After this is decremented the bio must not be touched if it is
	 * a barrier.
	 */
T
Tejun Heo 已提交
434 435
	dm_disk(md)->part0.in_flight = pending =
		atomic_dec_return(&md->pending);
436

437 438 439
	/* nudge anyone waiting on suspend queue */
	if (!pending)
		wake_up(&md->wait);
440 441
}

L
Linus Torvalds 已提交
442 443 444
/*
 * Add the bio to the list of deferred io.
 */
M
Mikulas Patocka 已提交
445
static void queue_io(struct mapped_device *md, struct bio *bio)
L
Linus Torvalds 已提交
446
{
447
	down_write(&md->io_lock);
L
Linus Torvalds 已提交
448

449
	spin_lock_irq(&md->deferred_lock);
L
Linus Torvalds 已提交
450
	bio_list_add(&md->deferred, bio);
451
	spin_unlock_irq(&md->deferred_lock);
L
Linus Torvalds 已提交
452

M
Mikulas Patocka 已提交
453 454 455
	if (!test_and_set_bit(DMF_QUEUE_IO_TO_THREAD, &md->flags))
		queue_work(md->wq, &md->work);

456
	up_write(&md->io_lock);
L
Linus Torvalds 已提交
457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476
}

/*
 * Everyone (including functions in this file), should use this
 * function to access the md->map field, and make sure they call
 * dm_table_put() when finished.
 */
struct dm_table *dm_get_table(struct mapped_device *md)
{
	struct dm_table *t;

	read_lock(&md->map_lock);
	t = md->map;
	if (t)
		dm_table_get(t);
	read_unlock(&md->map_lock);

	return t;
}

D
Darrick J. Wong 已提交
477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503
/*
 * Get the geometry associated with a dm device
 */
int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
{
	*geo = md->geometry;

	return 0;
}

/*
 * Set the geometry of a device.
 */
int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
{
	sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;

	if (geo->start > sz) {
		DMWARN("Start sector is beyond the geometry limits.");
		return -EINVAL;
	}

	md->geometry = *geo;

	return 0;
}

L
Linus Torvalds 已提交
504 505 506 507 508 509 510 511 512
/*-----------------------------------------------------------------
 * CRUD START:
 *   A more elegant soln is in the works that uses the queue
 *   merge fn, unfortunately there are a couple of changes to
 *   the block layer that I want to make for this.  So in the
 *   interests of getting something for people to use I give
 *   you this clearly demarcated crap.
 *---------------------------------------------------------------*/

513 514 515 516 517
static int __noflush_suspending(struct mapped_device *md)
{
	return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
}

L
Linus Torvalds 已提交
518 519 520 521
/*
 * Decrements the number of outstanding ios that a bio has been
 * cloned into, completing the original io if necc.
 */
522
static void dec_pending(struct dm_io *io, int error)
L
Linus Torvalds 已提交
523
{
524
	unsigned long flags;
525 526 527
	int io_error;
	struct bio *bio;
	struct mapped_device *md = io->md;
528 529

	/* Push-back supersedes any I/O errors */
530
	if (error && !(io->error > 0 && __noflush_suspending(md)))
L
Linus Torvalds 已提交
531 532 533
		io->error = error;

	if (atomic_dec_and_test(&io->io_count)) {
534 535 536 537
		if (io->error == DM_ENDIO_REQUEUE) {
			/*
			 * Target requested pushing back the I/O.
			 */
538
			spin_lock_irqsave(&md->deferred_lock, flags);
539 540 541 542 543
			if (__noflush_suspending(md)) {
				if (!bio_barrier(io->bio))
					bio_list_add_head(&md->deferred,
							  io->bio);
			} else
544 545
				/* noflush suspend was interrupted. */
				io->error = -EIO;
546
			spin_unlock_irqrestore(&md->deferred_lock, flags);
547 548
		}

549 550
		io_error = io->error;
		bio = io->bio;
551

552 553 554 555 556 557
		if (bio_barrier(bio)) {
			/*
			 * There can be just one barrier request so we use
			 * a per-device variable for error reporting.
			 * Note that you can't touch the bio after end_io_acct
			 */
558
			if (!md->barrier_error && io_error != -EOPNOTSUPP)
559
				md->barrier_error = io_error;
560 561 562
			end_io_acct(io);
		} else {
			end_io_acct(io);
563

564 565
			if (io_error != DM_ENDIO_REQUEUE) {
				trace_block_bio_complete(md->queue, bio);
566

567 568
				bio_endio(bio, io_error);
			}
569
		}
570 571

		free_io(md, io);
L
Linus Torvalds 已提交
572 573 574
	}
}

575
static void clone_endio(struct bio *bio, int error)
L
Linus Torvalds 已提交
576 577
{
	int r = 0;
A
Alasdair G Kergon 已提交
578
	struct dm_target_io *tio = bio->bi_private;
579
	struct dm_io *io = tio->io;
S
Stefan Bader 已提交
580
	struct mapped_device *md = tio->io->md;
L
Linus Torvalds 已提交
581 582 583 584 585 586 587
	dm_endio_fn endio = tio->ti->type->end_io;

	if (!bio_flagged(bio, BIO_UPTODATE) && !error)
		error = -EIO;

	if (endio) {
		r = endio(tio->ti, bio, error, &tio->info);
588 589 590 591 592
		if (r < 0 || r == DM_ENDIO_REQUEUE)
			/*
			 * error and requeue request are handled
			 * in dec_pending().
			 */
L
Linus Torvalds 已提交
593
			error = r;
594 595
		else if (r == DM_ENDIO_INCOMPLETE)
			/* The target will handle the io */
596
			return;
597 598 599 600
		else if (r) {
			DMWARN("unimplemented target endio return value: %d", r);
			BUG();
		}
L
Linus Torvalds 已提交
601 602
	}

S
Stefan Bader 已提交
603 604 605 606 607 608
	/*
	 * Store md for cleanup instead of tio which is about to get freed.
	 */
	bio->bi_private = md->bs;

	free_tio(md, tio);
609 610
	bio_put(bio);
	dec_pending(io, error);
L
Linus Torvalds 已提交
611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633
}

static sector_t max_io_len(struct mapped_device *md,
			   sector_t sector, struct dm_target *ti)
{
	sector_t offset = sector - ti->begin;
	sector_t len = ti->len - offset;

	/*
	 * Does the target need to split even further ?
	 */
	if (ti->split_io) {
		sector_t boundary;
		boundary = ((offset + ti->split_io) & ~(ti->split_io - 1))
			   - offset;
		if (len > boundary)
			len = boundary;
	}

	return len;
}

static void __map_bio(struct dm_target *ti, struct bio *clone,
A
Alasdair G Kergon 已提交
634
		      struct dm_target_io *tio)
L
Linus Torvalds 已提交
635 636
{
	int r;
637
	sector_t sector;
S
Stefan Bader 已提交
638
	struct mapped_device *md;
L
Linus Torvalds 已提交
639 640 641 642 643 644 645 646 647 648

	clone->bi_end_io = clone_endio;
	clone->bi_private = tio;

	/*
	 * Map the clone.  If r == 0 we don't need to do
	 * anything, the target has assumed ownership of
	 * this io.
	 */
	atomic_inc(&tio->io->io_count);
649
	sector = clone->bi_sector;
L
Linus Torvalds 已提交
650
	r = ti->type->map(ti, clone, &tio->info);
651
	if (r == DM_MAPIO_REMAPPED) {
L
Linus Torvalds 已提交
652
		/* the bio has been remapped so dispatch it */
653

654
		trace_block_remap(bdev_get_queue(clone->bi_bdev), clone,
655
				    tio->io->bio->bi_bdev->bd_dev, sector);
656

L
Linus Torvalds 已提交
657
		generic_make_request(clone);
658 659
	} else if (r < 0 || r == DM_MAPIO_REQUEUE) {
		/* error the io and bail out, or requeue it if needed */
S
Stefan Bader 已提交
660 661 662 663 664 665
		md = tio->io->md;
		dec_pending(tio->io, r);
		/*
		 * Store bio_set for cleanup.
		 */
		clone->bi_private = md->bs;
L
Linus Torvalds 已提交
666
		bio_put(clone);
S
Stefan Bader 已提交
667
		free_tio(md, tio);
668 669 670
	} else if (r) {
		DMWARN("unimplemented target map return value: %d", r);
		BUG();
L
Linus Torvalds 已提交
671 672 673 674 675 676 677 678 679 680 681 682 683
	}
}

struct clone_info {
	struct mapped_device *md;
	struct dm_table *map;
	struct bio *bio;
	struct dm_io *io;
	sector_t sector;
	sector_t sector_count;
	unsigned short idx;
};

P
Peter Osterlund 已提交
684 685
static void dm_bio_destructor(struct bio *bio)
{
S
Stefan Bader 已提交
686 687 688
	struct bio_set *bs = bio->bi_private;

	bio_free(bio, bs);
P
Peter Osterlund 已提交
689 690
}

L
Linus Torvalds 已提交
691 692 693 694 695
/*
 * Creates a little bio that is just does part of a bvec.
 */
static struct bio *split_bvec(struct bio *bio, sector_t sector,
			      unsigned short idx, unsigned int offset,
S
Stefan Bader 已提交
696
			      unsigned int len, struct bio_set *bs)
L
Linus Torvalds 已提交
697 698 699 700
{
	struct bio *clone;
	struct bio_vec *bv = bio->bi_io_vec + idx;

S
Stefan Bader 已提交
701
	clone = bio_alloc_bioset(GFP_NOIO, 1, bs);
P
Peter Osterlund 已提交
702
	clone->bi_destructor = dm_bio_destructor;
L
Linus Torvalds 已提交
703 704 705 706
	*clone->bi_io_vec = *bv;

	clone->bi_sector = sector;
	clone->bi_bdev = bio->bi_bdev;
707
	clone->bi_rw = bio->bi_rw & ~(1 << BIO_RW_BARRIER);
L
Linus Torvalds 已提交
708 709 710 711
	clone->bi_vcnt = 1;
	clone->bi_size = to_bytes(len);
	clone->bi_io_vec->bv_offset = offset;
	clone->bi_io_vec->bv_len = clone->bi_size;
M
Martin K. Petersen 已提交
712
	clone->bi_flags |= 1 << BIO_CLONED;
L
Linus Torvalds 已提交
713

M
Martin K. Petersen 已提交
714 715 716 717 718 719
	if (bio_integrity(bio)) {
		bio_integrity_clone(clone, bio, GFP_NOIO);
		bio_integrity_trim(clone,
				   bio_sector_offset(bio, idx, offset), len);
	}

L
Linus Torvalds 已提交
720 721 722 723 724 725 726 727
	return clone;
}

/*
 * Creates a bio that consists of range of complete bvecs.
 */
static struct bio *clone_bio(struct bio *bio, sector_t sector,
			     unsigned short idx, unsigned short bv_count,
S
Stefan Bader 已提交
728
			     unsigned int len, struct bio_set *bs)
L
Linus Torvalds 已提交
729 730 731
{
	struct bio *clone;

S
Stefan Bader 已提交
732 733
	clone = bio_alloc_bioset(GFP_NOIO, bio->bi_max_vecs, bs);
	__bio_clone(clone, bio);
734
	clone->bi_rw &= ~(1 << BIO_RW_BARRIER);
S
Stefan Bader 已提交
735
	clone->bi_destructor = dm_bio_destructor;
L
Linus Torvalds 已提交
736 737 738 739 740 741
	clone->bi_sector = sector;
	clone->bi_idx = idx;
	clone->bi_vcnt = idx + bv_count;
	clone->bi_size = to_bytes(len);
	clone->bi_flags &= ~(1 << BIO_SEG_VALID);

M
Martin K. Petersen 已提交
742 743 744 745 746 747 748 749
	if (bio_integrity(bio)) {
		bio_integrity_clone(clone, bio, GFP_NOIO);

		if (idx != bio->bi_idx || clone->bi_size < bio->bi_size)
			bio_integrity_trim(clone,
					   bio_sector_offset(bio, idx, 0), len);
	}

L
Linus Torvalds 已提交
750 751 752
	return clone;
}

753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786
static void __flush_target(struct clone_info *ci, struct dm_target *ti,
			  unsigned flush_nr)
{
	struct dm_target_io *tio = alloc_tio(ci->md);
	struct bio *clone;

	tio->io = ci->io;
	tio->ti = ti;

	memset(&tio->info, 0, sizeof(tio->info));
	tio->info.flush_request = flush_nr;

	clone = bio_alloc_bioset(GFP_NOIO, 0, ci->md->bs);
	__bio_clone(clone, ci->bio);
	clone->bi_destructor = dm_bio_destructor;

	__map_bio(ti, clone, tio);
}

static int __clone_and_map_empty_barrier(struct clone_info *ci)
{
	unsigned target_nr = 0, flush_nr;
	struct dm_target *ti;

	while ((ti = dm_table_get_target(ci->map, target_nr++)))
		for (flush_nr = 0; flush_nr < ti->num_flush_requests;
		     flush_nr++)
			__flush_target(ci, ti, flush_nr);

	ci->sector_count = 0;

	return 0;
}

787
static int __clone_and_map(struct clone_info *ci)
L
Linus Torvalds 已提交
788 789
{
	struct bio *clone, *bio = ci->bio;
790 791
	struct dm_target *ti;
	sector_t len = 0, max;
A
Alasdair G Kergon 已提交
792
	struct dm_target_io *tio;
L
Linus Torvalds 已提交
793

794 795 796
	if (unlikely(bio_empty_barrier(bio)))
		return __clone_and_map_empty_barrier(ci);

797 798 799 800 801 802
	ti = dm_table_find_target(ci->map, ci->sector);
	if (!dm_target_is_valid(ti))
		return -EIO;

	max = max_io_len(ci->md, ci->sector, ti);

L
Linus Torvalds 已提交
803 804 805 806 807 808 809 810 811 812 813 814 815 816
	/*
	 * Allocate a target io object.
	 */
	tio = alloc_tio(ci->md);
	tio->io = ci->io;
	tio->ti = ti;
	memset(&tio->info, 0, sizeof(tio->info));

	if (ci->sector_count <= max) {
		/*
		 * Optimise for the simple case where we can do all of
		 * the remaining io with a single clone.
		 */
		clone = clone_bio(bio, ci->sector, ci->idx,
S
Stefan Bader 已提交
817 818
				  bio->bi_vcnt - ci->idx, ci->sector_count,
				  ci->md->bs);
L
Linus Torvalds 已提交
819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840
		__map_bio(ti, clone, tio);
		ci->sector_count = 0;

	} else if (to_sector(bio->bi_io_vec[ci->idx].bv_len) <= max) {
		/*
		 * There are some bvecs that don't span targets.
		 * Do as many of these as possible.
		 */
		int i;
		sector_t remaining = max;
		sector_t bv_len;

		for (i = ci->idx; remaining && (i < bio->bi_vcnt); i++) {
			bv_len = to_sector(bio->bi_io_vec[i].bv_len);

			if (bv_len > remaining)
				break;

			remaining -= bv_len;
			len += bv_len;
		}

S
Stefan Bader 已提交
841 842
		clone = clone_bio(bio, ci->sector, ci->idx, i - ci->idx, len,
				  ci->md->bs);
L
Linus Torvalds 已提交
843 844 845 846 847 848 849 850
		__map_bio(ti, clone, tio);

		ci->sector += len;
		ci->sector_count -= len;
		ci->idx = i;

	} else {
		/*
851
		 * Handle a bvec that must be split between two or more targets.
L
Linus Torvalds 已提交
852 853
		 */
		struct bio_vec *bv = bio->bi_io_vec + ci->idx;
854 855
		sector_t remaining = to_sector(bv->bv_len);
		unsigned int offset = 0;
L
Linus Torvalds 已提交
856

857 858 859
		do {
			if (offset) {
				ti = dm_table_find_target(ci->map, ci->sector);
860 861 862
				if (!dm_target_is_valid(ti))
					return -EIO;

863
				max = max_io_len(ci->md, ci->sector, ti);
L
Linus Torvalds 已提交
864

865 866 867 868 869 870 871 872 873
				tio = alloc_tio(ci->md);
				tio->io = ci->io;
				tio->ti = ti;
				memset(&tio->info, 0, sizeof(tio->info));
			}

			len = min(remaining, max);

			clone = split_bvec(bio, ci->sector, ci->idx,
S
Stefan Bader 已提交
874 875
					   bv->bv_offset + offset, len,
					   ci->md->bs);
876 877 878 879 880 881 882

			__map_bio(ti, clone, tio);

			ci->sector += len;
			ci->sector_count -= len;
			offset += to_bytes(len);
		} while (remaining -= len);
L
Linus Torvalds 已提交
883 884 885

		ci->idx++;
	}
886 887

	return 0;
L
Linus Torvalds 已提交
888 889 890
}

/*
M
Mikulas Patocka 已提交
891
 * Split the bio into several clones and submit it to targets.
L
Linus Torvalds 已提交
892
 */
893
static void __split_and_process_bio(struct mapped_device *md, struct bio *bio)
L
Linus Torvalds 已提交
894 895
{
	struct clone_info ci;
896
	int error = 0;
L
Linus Torvalds 已提交
897 898

	ci.map = dm_get_table(md);
899
	if (unlikely(!ci.map)) {
900 901 902
		if (!bio_barrier(bio))
			bio_io_error(bio);
		else
903 904
			if (!md->barrier_error)
				md->barrier_error = -EIO;
905 906
		return;
	}
907

L
Linus Torvalds 已提交
908 909 910 911 912 913 914 915 916
	ci.md = md;
	ci.bio = bio;
	ci.io = alloc_io(md);
	ci.io->error = 0;
	atomic_set(&ci.io->io_count, 1);
	ci.io->bio = bio;
	ci.io->md = md;
	ci.sector = bio->bi_sector;
	ci.sector_count = bio_sectors(bio);
917 918
	if (unlikely(bio_empty_barrier(bio)))
		ci.sector_count = 1;
L
Linus Torvalds 已提交
919 920
	ci.idx = bio->bi_idx;

921
	start_io_acct(ci.io);
922 923
	while (ci.sector_count && !error)
		error = __clone_and_map(&ci);
L
Linus Torvalds 已提交
924 925

	/* drop the extra reference count */
926
	dec_pending(ci.io, error);
L
Linus Torvalds 已提交
927 928 929 930 931 932
	dm_table_put(ci.map);
}
/*-----------------------------------------------------------------
 * CRUD END
 *---------------------------------------------------------------*/

M
Milan Broz 已提交
933 934 935 936 937 938 939 940
static int dm_merge_bvec(struct request_queue *q,
			 struct bvec_merge_data *bvm,
			 struct bio_vec *biovec)
{
	struct mapped_device *md = q->queuedata;
	struct dm_table *map = dm_get_table(md);
	struct dm_target *ti;
	sector_t max_sectors;
941
	int max_size = 0;
M
Milan Broz 已提交
942 943

	if (unlikely(!map))
944
		goto out;
M
Milan Broz 已提交
945 946

	ti = dm_table_find_target(map, bvm->bi_sector);
947 948
	if (!dm_target_is_valid(ti))
		goto out_table;
M
Milan Broz 已提交
949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965

	/*
	 * Find maximum amount of I/O that won't need splitting
	 */
	max_sectors = min(max_io_len(md, bvm->bi_sector, ti),
			  (sector_t) BIO_MAX_SECTORS);
	max_size = (max_sectors << SECTOR_SHIFT) - bvm->bi_size;
	if (max_size < 0)
		max_size = 0;

	/*
	 * merge_bvec_fn() returns number of bytes
	 * it can accept at this offset
	 * max is precomputed maximal io size
	 */
	if (max_size && ti->type->merge)
		max_size = ti->type->merge(ti, bvm, biovec, max_size);
966 967 968 969 970 971 972 973 974 975
	/*
	 * If the target doesn't support merge method and some of the devices
	 * provided their merge_bvec method (we know this by looking at
	 * queue_max_hw_sectors), then we can't allow bios with multiple vector
	 * entries.  So always set max_size to 0, and the code below allows
	 * just one page.
	 */
	else if (queue_max_hw_sectors(q) <= PAGE_SIZE >> 9)

		max_size = 0;
M
Milan Broz 已提交
976

977
out_table:
978 979 980
	dm_table_put(map);

out:
M
Milan Broz 已提交
981 982 983 984 985 986 987 988 989
	/*
	 * Always allow an entire first page
	 */
	if (max_size <= biovec->bv_len && !(bvm->bi_size >> SECTOR_SHIFT))
		max_size = biovec->bv_len;

	return max_size;
}

L
Linus Torvalds 已提交
990 991 992 993
/*
 * The request function that just remaps the bio built up by
 * dm_merge_bvec.
 */
994
static int dm_request(struct request_queue *q, struct bio *bio)
L
Linus Torvalds 已提交
995
{
996
	int rw = bio_data_dir(bio);
L
Linus Torvalds 已提交
997
	struct mapped_device *md = q->queuedata;
T
Tejun Heo 已提交
998
	int cpu;
L
Linus Torvalds 已提交
999

1000
	down_read(&md->io_lock);
L
Linus Torvalds 已提交
1001

T
Tejun Heo 已提交
1002 1003 1004 1005
	cpu = part_stat_lock();
	part_stat_inc(cpu, &dm_disk(md)->part0, ios[rw]);
	part_stat_add(cpu, &dm_disk(md)->part0, sectors[rw], bio_sectors(bio));
	part_stat_unlock();
1006

L
Linus Torvalds 已提交
1007
	/*
1008 1009
	 * If we're suspended or the thread is processing barriers
	 * we have to queue this io for later.
L
Linus Torvalds 已提交
1010
	 */
1011 1012
	if (unlikely(test_bit(DMF_QUEUE_IO_TO_THREAD, &md->flags)) ||
	    unlikely(bio_barrier(bio))) {
1013
		up_read(&md->io_lock);
L
Linus Torvalds 已提交
1014

A
Alasdair G Kergon 已提交
1015 1016 1017 1018 1019
		if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) &&
		    bio_rw(bio) == READA) {
			bio_io_error(bio);
			return 0;
		}
L
Linus Torvalds 已提交
1020

M
Mikulas Patocka 已提交
1021
		queue_io(md, bio);
L
Linus Torvalds 已提交
1022

M
Mikulas Patocka 已提交
1023
		return 0;
L
Linus Torvalds 已提交
1024 1025
	}

1026
	__split_and_process_bio(md, bio);
1027
	up_read(&md->io_lock);
1028
	return 0;
L
Linus Torvalds 已提交
1029 1030
}

1031
static void dm_unplug_all(struct request_queue *q)
L
Linus Torvalds 已提交
1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043
{
	struct mapped_device *md = q->queuedata;
	struct dm_table *map = dm_get_table(md);

	if (map) {
		dm_table_unplug_all(map);
		dm_table_put(map);
	}
}

static int dm_any_congested(void *congested_data, int bdi_bits)
{
1044 1045 1046
	int r = bdi_bits;
	struct mapped_device *md = congested_data;
	struct dm_table *map;
L
Linus Torvalds 已提交
1047

1048
	if (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
1049 1050 1051 1052 1053 1054 1055
		map = dm_get_table(md);
		if (map) {
			r = dm_table_any_congested(map, bdi_bits);
			dm_table_put(map);
		}
	}

L
Linus Torvalds 已提交
1056 1057 1058 1059 1060 1061 1062 1063
	return r;
}

/*-----------------------------------------------------------------
 * An IDR is used to keep track of allocated minor numbers.
 *---------------------------------------------------------------*/
static DEFINE_IDR(_minor_idr);

1064
static void free_minor(int minor)
L
Linus Torvalds 已提交
1065
{
1066
	spin_lock(&_minor_lock);
L
Linus Torvalds 已提交
1067
	idr_remove(&_minor_idr, minor);
1068
	spin_unlock(&_minor_lock);
L
Linus Torvalds 已提交
1069 1070 1071 1072 1073
}

/*
 * See if the device with a specific minor # is free.
 */
1074
static int specific_minor(int minor)
L
Linus Torvalds 已提交
1075 1076 1077 1078 1079 1080
{
	int r, m;

	if (minor >= (1 << MINORBITS))
		return -EINVAL;

J
Jeff Mahoney 已提交
1081 1082 1083 1084
	r = idr_pre_get(&_minor_idr, GFP_KERNEL);
	if (!r)
		return -ENOMEM;

1085
	spin_lock(&_minor_lock);
L
Linus Torvalds 已提交
1086 1087 1088 1089 1090 1091

	if (idr_find(&_minor_idr, minor)) {
		r = -EBUSY;
		goto out;
	}

1092
	r = idr_get_new_above(&_minor_idr, MINOR_ALLOCED, minor, &m);
J
Jeff Mahoney 已提交
1093
	if (r)
L
Linus Torvalds 已提交
1094 1095 1096 1097 1098 1099 1100 1101 1102
		goto out;

	if (m != minor) {
		idr_remove(&_minor_idr, m);
		r = -EBUSY;
		goto out;
	}

out:
1103
	spin_unlock(&_minor_lock);
L
Linus Torvalds 已提交
1104 1105 1106
	return r;
}

1107
static int next_free_minor(int *minor)
L
Linus Torvalds 已提交
1108
{
1109
	int r, m;
L
Linus Torvalds 已提交
1110 1111

	r = idr_pre_get(&_minor_idr, GFP_KERNEL);
J
Jeff Mahoney 已提交
1112 1113 1114
	if (!r)
		return -ENOMEM;

1115
	spin_lock(&_minor_lock);
L
Linus Torvalds 已提交
1116

1117
	r = idr_get_new(&_minor_idr, MINOR_ALLOCED, &m);
1118
	if (r)
L
Linus Torvalds 已提交
1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129
		goto out;

	if (m >= (1 << MINORBITS)) {
		idr_remove(&_minor_idr, m);
		r = -ENOSPC;
		goto out;
	}

	*minor = m;

out:
1130
	spin_unlock(&_minor_lock);
L
Linus Torvalds 已提交
1131 1132 1133 1134 1135
	return r;
}

static struct block_device_operations dm_blk_dops;

1136 1137
static void dm_wq_work(struct work_struct *work);

L
Linus Torvalds 已提交
1138 1139 1140
/*
 * Allocate and initialise a blank device with a given minor.
 */
1141
static struct mapped_device *alloc_dev(int minor)
L
Linus Torvalds 已提交
1142 1143
{
	int r;
1144
	struct mapped_device *md = kzalloc(sizeof(*md), GFP_KERNEL);
1145
	void *old_md;
L
Linus Torvalds 已提交
1146 1147 1148 1149 1150 1151

	if (!md) {
		DMWARN("unable to allocate device, out of memory.");
		return NULL;
	}

1152
	if (!try_module_get(THIS_MODULE))
M
Milan Broz 已提交
1153
		goto bad_module_get;
1154

L
Linus Torvalds 已提交
1155
	/* get a minor number for the dev */
1156
	if (minor == DM_ANY_MINOR)
1157
		r = next_free_minor(&minor);
1158
	else
1159
		r = specific_minor(minor);
L
Linus Torvalds 已提交
1160
	if (r < 0)
M
Milan Broz 已提交
1161
		goto bad_minor;
L
Linus Torvalds 已提交
1162

1163
	init_rwsem(&md->io_lock);
1164
	mutex_init(&md->suspend_lock);
1165
	spin_lock_init(&md->deferred_lock);
L
Linus Torvalds 已提交
1166 1167
	rwlock_init(&md->map_lock);
	atomic_set(&md->holders, 1);
1168
	atomic_set(&md->open_count, 0);
L
Linus Torvalds 已提交
1169
	atomic_set(&md->event_nr, 0);
M
Mike Anderson 已提交
1170 1171 1172
	atomic_set(&md->uevent_seq, 0);
	INIT_LIST_HEAD(&md->uevent_list);
	spin_lock_init(&md->uevent_lock);
L
Linus Torvalds 已提交
1173 1174 1175

	md->queue = blk_alloc_queue(GFP_KERNEL);
	if (!md->queue)
M
Milan Broz 已提交
1176
		goto bad_queue;
L
Linus Torvalds 已提交
1177 1178 1179 1180 1181

	md->queue->queuedata = md;
	md->queue->backing_dev_info.congested_fn = dm_any_congested;
	md->queue->backing_dev_info.congested_data = md;
	blk_queue_make_request(md->queue, dm_request);
M
Mikulas Patocka 已提交
1182
	blk_queue_ordered(md->queue, QUEUE_ORDERED_DRAIN, NULL);
1183
	blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);
L
Linus Torvalds 已提交
1184
	md->queue->unplug_fn = dm_unplug_all;
M
Milan Broz 已提交
1185
	blk_queue_merge_bvec(md->queue, dm_merge_bvec);
L
Linus Torvalds 已提交
1186

1187
	md->io_pool = mempool_create_slab_pool(MIN_IOS, _io_cache);
K
Kiyoshi Ueda 已提交
1188
	if (!md->io_pool)
M
Milan Broz 已提交
1189
		goto bad_io_pool;
L
Linus Torvalds 已提交
1190

1191
	md->tio_pool = mempool_create_slab_pool(MIN_IOS, _tio_cache);
L
Linus Torvalds 已提交
1192
	if (!md->tio_pool)
M
Milan Broz 已提交
1193
		goto bad_tio_pool;
L
Linus Torvalds 已提交
1194

1195
	md->bs = bioset_create(16, 0);
S
Stefan Bader 已提交
1196 1197 1198
	if (!md->bs)
		goto bad_no_bioset;

L
Linus Torvalds 已提交
1199 1200
	md->disk = alloc_disk(1);
	if (!md->disk)
M
Milan Broz 已提交
1201
		goto bad_disk;
L
Linus Torvalds 已提交
1202

1203 1204
	atomic_set(&md->pending, 0);
	init_waitqueue_head(&md->wait);
1205
	INIT_WORK(&md->work, dm_wq_work);
1206 1207
	init_waitqueue_head(&md->eventq);

L
Linus Torvalds 已提交
1208 1209 1210 1211 1212 1213 1214
	md->disk->major = _major;
	md->disk->first_minor = minor;
	md->disk->fops = &dm_blk_dops;
	md->disk->queue = md->queue;
	md->disk->private_data = md;
	sprintf(md->disk->disk_name, "dm-%d", minor);
	add_disk(md->disk);
M
Mike Anderson 已提交
1215
	format_dev_t(md->name, MKDEV(_major, minor));
L
Linus Torvalds 已提交
1216

1217 1218 1219 1220
	md->wq = create_singlethread_workqueue("kdmflush");
	if (!md->wq)
		goto bad_thread;

M
Mikulas Patocka 已提交
1221 1222 1223 1224
	md->bdev = bdget_disk(md->disk, 0);
	if (!md->bdev)
		goto bad_bdev;

1225
	/* Populate the mapping, nobody knows we exist yet */
1226
	spin_lock(&_minor_lock);
1227
	old_md = idr_replace(&_minor_idr, md, minor);
1228
	spin_unlock(&_minor_lock);
1229 1230 1231

	BUG_ON(old_md != MINOR_ALLOCED);

L
Linus Torvalds 已提交
1232 1233
	return md;

M
Mikulas Patocka 已提交
1234 1235
bad_bdev:
	destroy_workqueue(md->wq);
1236 1237
bad_thread:
	put_disk(md->disk);
M
Milan Broz 已提交
1238
bad_disk:
S
Stefan Bader 已提交
1239
	bioset_free(md->bs);
M
Milan Broz 已提交
1240
bad_no_bioset:
L
Linus Torvalds 已提交
1241
	mempool_destroy(md->tio_pool);
M
Milan Broz 已提交
1242
bad_tio_pool:
L
Linus Torvalds 已提交
1243
	mempool_destroy(md->io_pool);
M
Milan Broz 已提交
1244
bad_io_pool:
1245
	blk_cleanup_queue(md->queue);
M
Milan Broz 已提交
1246
bad_queue:
L
Linus Torvalds 已提交
1247
	free_minor(minor);
M
Milan Broz 已提交
1248
bad_minor:
1249
	module_put(THIS_MODULE);
M
Milan Broz 已提交
1250
bad_module_get:
L
Linus Torvalds 已提交
1251 1252 1253 1254
	kfree(md);
	return NULL;
}

J
Jun'ichi Nomura 已提交
1255 1256
static void unlock_fs(struct mapped_device *md);

L
Linus Torvalds 已提交
1257 1258
static void free_dev(struct mapped_device *md)
{
1259
	int minor = MINOR(disk_devt(md->disk));
1260

M
Mikulas Patocka 已提交
1261 1262
	unlock_fs(md);
	bdput(md->bdev);
1263
	destroy_workqueue(md->wq);
L
Linus Torvalds 已提交
1264 1265
	mempool_destroy(md->tio_pool);
	mempool_destroy(md->io_pool);
S
Stefan Bader 已提交
1266
	bioset_free(md->bs);
M
Martin K. Petersen 已提交
1267
	blk_integrity_unregister(md->disk);
L
Linus Torvalds 已提交
1268
	del_gendisk(md->disk);
1269
	free_minor(minor);
J
Jeff Mahoney 已提交
1270 1271 1272 1273 1274

	spin_lock(&_minor_lock);
	md->disk->private_data = NULL;
	spin_unlock(&_minor_lock);

L
Linus Torvalds 已提交
1275
	put_disk(md->disk);
1276
	blk_cleanup_queue(md->queue);
1277
	module_put(THIS_MODULE);
L
Linus Torvalds 已提交
1278 1279 1280 1281 1282 1283 1284 1285
	kfree(md);
}

/*
 * Bind a table to the device.
 */
static void event_callback(void *context)
{
M
Mike Anderson 已提交
1286 1287
	unsigned long flags;
	LIST_HEAD(uevents);
L
Linus Torvalds 已提交
1288 1289
	struct mapped_device *md = (struct mapped_device *) context;

M
Mike Anderson 已提交
1290 1291 1292 1293
	spin_lock_irqsave(&md->uevent_lock, flags);
	list_splice_init(&md->uevent_list, &uevents);
	spin_unlock_irqrestore(&md->uevent_lock, flags);

1294
	dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
M
Mike Anderson 已提交
1295

L
Linus Torvalds 已提交
1296 1297 1298 1299
	atomic_inc(&md->event_nr);
	wake_up(&md->eventq);
}

1300
static void __set_size(struct mapped_device *md, sector_t size)
L
Linus Torvalds 已提交
1301
{
1302
	set_capacity(md->disk, size);
L
Linus Torvalds 已提交
1303

1304 1305 1306
	mutex_lock(&md->bdev->bd_inode->i_mutex);
	i_size_write(md->bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
	mutex_unlock(&md->bdev->bd_inode->i_mutex);
L
Linus Torvalds 已提交
1307 1308 1309 1310
}

static int __bind(struct mapped_device *md, struct dm_table *t)
{
1311
	struct request_queue *q = md->queue;
L
Linus Torvalds 已提交
1312 1313 1314
	sector_t size;

	size = dm_table_get_size(t);
D
Darrick J. Wong 已提交
1315 1316 1317 1318 1319 1320 1321

	/*
	 * Wipe any geometry if the size of the table changed.
	 */
	if (size != get_capacity(md->disk))
		memset(&md->geometry, 0, sizeof(md->geometry));

M
Mikulas Patocka 已提交
1322
	__set_size(md, size);
1323 1324 1325

	if (!size) {
		dm_table_destroy(t);
L
Linus Torvalds 已提交
1326
		return 0;
1327
	}
L
Linus Torvalds 已提交
1328

1329 1330
	dm_table_event_callback(t, event_callback, md);

L
Linus Torvalds 已提交
1331 1332
	write_lock(&md->map_lock);
	md->map = t;
1333
	dm_table_set_restrictions(t, q);
L
Linus Torvalds 已提交
1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349
	write_unlock(&md->map_lock);

	return 0;
}

static void __unbind(struct mapped_device *md)
{
	struct dm_table *map = md->map;

	if (!map)
		return;

	dm_table_event_callback(map, NULL, NULL);
	write_lock(&md->map_lock);
	md->map = NULL;
	write_unlock(&md->map_lock);
1350
	dm_table_destroy(map);
L
Linus Torvalds 已提交
1351 1352 1353 1354 1355
}

/*
 * Constructor for a new device.
 */
1356
int dm_create(int minor, struct mapped_device **result)
L
Linus Torvalds 已提交
1357 1358 1359
{
	struct mapped_device *md;

1360
	md = alloc_dev(minor);
L
Linus Torvalds 已提交
1361 1362 1363
	if (!md)
		return -ENXIO;

M
Milan Broz 已提交
1364 1365
	dm_sysfs_init(md);

L
Linus Torvalds 已提交
1366 1367 1368 1369
	*result = md;
	return 0;
}

1370
static struct mapped_device *dm_find_md(dev_t dev)
L
Linus Torvalds 已提交
1371 1372 1373 1374 1375 1376 1377
{
	struct mapped_device *md;
	unsigned minor = MINOR(dev);

	if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
		return NULL;

1378
	spin_lock(&_minor_lock);
L
Linus Torvalds 已提交
1379 1380

	md = idr_find(&_minor_idr, minor);
J
Jeff Mahoney 已提交
1381
	if (md && (md == MINOR_ALLOCED ||
1382
		   (MINOR(disk_devt(dm_disk(md))) != minor) ||
A
Alasdair G Kergon 已提交
1383
		   test_bit(DMF_FREEING, &md->flags))) {
1384
		md = NULL;
J
Jeff Mahoney 已提交
1385 1386
		goto out;
	}
L
Linus Torvalds 已提交
1387

J
Jeff Mahoney 已提交
1388
out:
1389
	spin_unlock(&_minor_lock);
L
Linus Torvalds 已提交
1390

1391 1392 1393
	return md;
}

1394 1395 1396 1397 1398 1399 1400 1401 1402 1403
struct mapped_device *dm_get_md(dev_t dev)
{
	struct mapped_device *md = dm_find_md(dev);

	if (md)
		dm_get(md);

	return md;
}

A
Alasdair G Kergon 已提交
1404
void *dm_get_mdptr(struct mapped_device *md)
1405
{
A
Alasdair G Kergon 已提交
1406
	return md->interface_ptr;
L
Linus Torvalds 已提交
1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418
}

void dm_set_mdptr(struct mapped_device *md, void *ptr)
{
	md->interface_ptr = ptr;
}

void dm_get(struct mapped_device *md)
{
	atomic_inc(&md->holders);
}

1419 1420 1421 1422 1423 1424
const char *dm_device_name(struct mapped_device *md)
{
	return md->name;
}
EXPORT_SYMBOL_GPL(dm_device_name);

L
Linus Torvalds 已提交
1425 1426
void dm_put(struct mapped_device *md)
{
M
Mike Anderson 已提交
1427
	struct dm_table *map;
L
Linus Torvalds 已提交
1428

J
Jeff Mahoney 已提交
1429 1430
	BUG_ON(test_bit(DMF_FREEING, &md->flags));

1431
	if (atomic_dec_and_lock(&md->holders, &_minor_lock)) {
M
Mike Anderson 已提交
1432
		map = dm_get_table(md);
1433 1434
		idr_replace(&_minor_idr, MINOR_ALLOCED,
			    MINOR(disk_devt(dm_disk(md))));
J
Jeff Mahoney 已提交
1435
		set_bit(DMF_FREEING, &md->flags);
1436
		spin_unlock(&_minor_lock);
1437
		if (!dm_suspended(md)) {
L
Linus Torvalds 已提交
1438 1439 1440
			dm_table_presuspend_targets(map);
			dm_table_postsuspend_targets(map);
		}
M
Milan Broz 已提交
1441
		dm_sysfs_exit(md);
M
Mike Anderson 已提交
1442
		dm_table_put(map);
1443
		__unbind(md);
L
Linus Torvalds 已提交
1444 1445 1446
		free_dev(md);
	}
}
E
Edward Goggin 已提交
1447
EXPORT_SYMBOL_GPL(dm_put);
L
Linus Torvalds 已提交
1448

1449
static int dm_wait_for_completion(struct mapped_device *md, int interruptible)
1450 1451
{
	int r = 0;
1452 1453 1454 1455 1456
	DECLARE_WAITQUEUE(wait, current);

	dm_unplug_all(md->queue);

	add_wait_queue(&md->wait, &wait);
1457 1458

	while (1) {
1459
		set_current_state(interruptible);
1460 1461 1462 1463 1464

		smp_mb();
		if (!atomic_read(&md->pending))
			break;

1465 1466
		if (interruptible == TASK_INTERRUPTIBLE &&
		    signal_pending(current)) {
1467 1468 1469 1470 1471 1472 1473 1474
			r = -EINTR;
			break;
		}

		io_schedule();
	}
	set_current_state(TASK_RUNNING);

1475 1476
	remove_wait_queue(&md->wait, &wait);

1477 1478 1479
	return r;
}

M
Mikulas Patocka 已提交
1480
static void dm_flush(struct mapped_device *md)
1481 1482 1483 1484 1485 1486
{
	dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE);
}

static void process_barrier(struct mapped_device *md, struct bio *bio)
{
1487 1488
	md->barrier_error = 0;

M
Mikulas Patocka 已提交
1489
	dm_flush(md);
1490

1491 1492 1493
	if (!bio_empty_barrier(bio)) {
		__split_and_process_bio(md, bio);
		dm_flush(md);
1494 1495 1496
	}

	if (md->barrier_error != DM_ENDIO_REQUEUE)
M
Mikulas Patocka 已提交
1497
		bio_endio(bio, md->barrier_error);
1498 1499 1500 1501 1502
	else {
		spin_lock_irq(&md->deferred_lock);
		bio_list_add_head(&md->deferred, bio);
		spin_unlock_irq(&md->deferred_lock);
	}
1503 1504
}

L
Linus Torvalds 已提交
1505 1506 1507
/*
 * Process the deferred bios
 */
1508
static void dm_wq_work(struct work_struct *work)
L
Linus Torvalds 已提交
1509
{
1510 1511
	struct mapped_device *md = container_of(work, struct mapped_device,
						work);
1512
	struct bio *c;
L
Linus Torvalds 已提交
1513

1514 1515
	down_write(&md->io_lock);

1516
	while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
A
Alasdair G Kergon 已提交
1517 1518 1519 1520 1521
		spin_lock_irq(&md->deferred_lock);
		c = bio_list_pop(&md->deferred);
		spin_unlock_irq(&md->deferred_lock);

		if (!c) {
1522
			clear_bit(DMF_QUEUE_IO_TO_THREAD, &md->flags);
A
Alasdair G Kergon 已提交
1523 1524
			break;
		}
1525

1526 1527
		up_write(&md->io_lock);

1528 1529 1530 1531
		if (bio_barrier(c))
			process_barrier(md, c);
		else
			__split_and_process_bio(md, c);
1532 1533

		down_write(&md->io_lock);
1534
	}
M
Milan Broz 已提交
1535

1536
	up_write(&md->io_lock);
L
Linus Torvalds 已提交
1537 1538
}

1539
static void dm_queue_flush(struct mapped_device *md)
1540
{
1541 1542
	clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
	smp_mb__after_clear_bit();
1543
	queue_work(md->wq, &md->work);
1544 1545
}

L
Linus Torvalds 已提交
1546 1547 1548 1549 1550
/*
 * Swap in a new table (destroying old one).
 */
int dm_swap_table(struct mapped_device *md, struct dm_table *table)
{
1551
	int r = -EINVAL;
L
Linus Torvalds 已提交
1552

1553
	mutex_lock(&md->suspend_lock);
L
Linus Torvalds 已提交
1554 1555

	/* device must be suspended */
1556
	if (!dm_suspended(md))
1557
		goto out;
L
Linus Torvalds 已提交
1558 1559 1560 1561

	__unbind(md);
	r = __bind(md, table);

1562
out:
1563
	mutex_unlock(&md->suspend_lock);
1564
	return r;
L
Linus Torvalds 已提交
1565 1566 1567 1568 1569 1570
}

/*
 * Functions to lock and unlock any filesystem running on the
 * device.
 */
1571
static int lock_fs(struct mapped_device *md)
L
Linus Torvalds 已提交
1572
{
1573
	int r;
L
Linus Torvalds 已提交
1574 1575

	WARN_ON(md->frozen_sb);
1576

1577
	md->frozen_sb = freeze_bdev(md->bdev);
1578
	if (IS_ERR(md->frozen_sb)) {
1579
		r = PTR_ERR(md->frozen_sb);
1580 1581
		md->frozen_sb = NULL;
		return r;
1582 1583
	}

1584 1585
	set_bit(DMF_FROZEN, &md->flags);

L
Linus Torvalds 已提交
1586 1587 1588
	return 0;
}

1589
static void unlock_fs(struct mapped_device *md)
L
Linus Torvalds 已提交
1590
{
1591 1592 1593
	if (!test_bit(DMF_FROZEN, &md->flags))
		return;

1594
	thaw_bdev(md->bdev, md->frozen_sb);
L
Linus Torvalds 已提交
1595
	md->frozen_sb = NULL;
1596
	clear_bit(DMF_FROZEN, &md->flags);
L
Linus Torvalds 已提交
1597 1598 1599 1600 1601 1602 1603 1604 1605
}

/*
 * We need to be able to change a mapping table under a mounted
 * filesystem.  For example we might want to move some data in
 * the background.  Before the table can be swapped with
 * dm_bind_table, dm_suspend must be called to flush any in
 * flight bios and ensure that any further io gets deferred.
 */
1606
int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
L
Linus Torvalds 已提交
1607
{
1608
	struct dm_table *map = NULL;
1609
	int r = 0;
1610
	int do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG ? 1 : 0;
1611
	int noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG ? 1 : 0;
L
Linus Torvalds 已提交
1612

1613
	mutex_lock(&md->suspend_lock);
1614

M
Milan Broz 已提交
1615 1616
	if (dm_suspended(md)) {
		r = -EINVAL;
1617
		goto out_unlock;
M
Milan Broz 已提交
1618
	}
L
Linus Torvalds 已提交
1619 1620 1621

	map = dm_get_table(md);

1622 1623 1624 1625 1626 1627 1628
	/*
	 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
	 * This flag is cleared before dm_suspend returns.
	 */
	if (noflush)
		set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);

1629 1630 1631
	/* This does not get reverted if there's an error later. */
	dm_table_presuspend_targets(map);

M
Mikulas Patocka 已提交
1632 1633 1634 1635 1636 1637 1638
	/*
	 * Flush I/O to the device. noflush supersedes do_lockfs,
	 * because lock_fs() needs to flush I/Os.
	 */
	if (!noflush && do_lockfs) {
		r = lock_fs(md);
		if (r)
K
Kiyoshi Ueda 已提交
1639
			goto out;
1640
	}
L
Linus Torvalds 已提交
1641 1642

	/*
1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656
	 * Here we must make sure that no processes are submitting requests
	 * to target drivers i.e. no one may be executing
	 * __split_and_process_bio. This is called from dm_request and
	 * dm_wq_work.
	 *
	 * To get all processes out of __split_and_process_bio in dm_request,
	 * we take the write lock. To prevent any process from reentering
	 * __split_and_process_bio from dm_request, we set
	 * DMF_QUEUE_IO_TO_THREAD.
	 *
	 * To quiesce the thread (dm_wq_work), we set DMF_BLOCK_IO_FOR_SUSPEND
	 * and call flush_workqueue(md->wq). flush_workqueue will wait until
	 * dm_wq_work exits and DMF_BLOCK_IO_FOR_SUSPEND will prevent any
	 * further calls to __split_and_process_bio from dm_wq_work.
L
Linus Torvalds 已提交
1657
	 */
1658
	down_write(&md->io_lock);
1659 1660
	set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
	set_bit(DMF_QUEUE_IO_TO_THREAD, &md->flags);
1661
	up_write(&md->io_lock);
L
Linus Torvalds 已提交
1662

1663 1664
	flush_workqueue(md->wq);

L
Linus Torvalds 已提交
1665
	/*
1666 1667 1668
	 * At this point no more requests are entering target request routines.
	 * We call dm_wait_for_completion to wait for all existing requests
	 * to finish.
L
Linus Torvalds 已提交
1669
	 */
1670
	r = dm_wait_for_completion(md, TASK_INTERRUPTIBLE);
L
Linus Torvalds 已提交
1671

1672
	down_write(&md->io_lock);
1673
	if (noflush)
1674
		clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
1675
	up_write(&md->io_lock);
1676

L
Linus Torvalds 已提交
1677
	/* were we interrupted ? */
1678
	if (r < 0) {
1679
		dm_queue_flush(md);
M
Milan Broz 已提交
1680

1681
		unlock_fs(md);
1682
		goto out; /* pushback list is already flushed, so skip flush */
1683
	}
L
Linus Torvalds 已提交
1684

1685 1686 1687 1688 1689 1690
	/*
	 * If dm_wait_for_completion returned 0, the device is completely
	 * quiescent now. There is no request-processing activity. All new
	 * requests are being added to md->deferred list.
	 */

1691
	dm_table_postsuspend_targets(map);
L
Linus Torvalds 已提交
1692

1693
	set_bit(DMF_SUSPENDED, &md->flags);
1694

1695 1696
out:
	dm_table_put(map);
1697 1698

out_unlock:
1699
	mutex_unlock(&md->suspend_lock);
1700
	return r;
L
Linus Torvalds 已提交
1701 1702 1703 1704
}

int dm_resume(struct mapped_device *md)
{
1705 1706
	int r = -EINVAL;
	struct dm_table *map = NULL;
L
Linus Torvalds 已提交
1707

1708
	mutex_lock(&md->suspend_lock);
1709
	if (!dm_suspended(md))
1710 1711 1712
		goto out;

	map = dm_get_table(md);
1713
	if (!map || !dm_table_get_size(map))
1714
		goto out;
L
Linus Torvalds 已提交
1715

1716 1717 1718
	r = dm_table_resume_targets(map);
	if (r)
		goto out;
1719

1720
	dm_queue_flush(md);
1721 1722 1723 1724 1725

	unlock_fs(md);

	clear_bit(DMF_SUSPENDED, &md->flags);

L
Linus Torvalds 已提交
1726 1727
	dm_table_unplug_all(map);

1728
	dm_kobject_uevent(md);
1729

1730
	r = 0;
1731

1732 1733
out:
	dm_table_put(map);
1734
	mutex_unlock(&md->suspend_lock);
1735

1736
	return r;
L
Linus Torvalds 已提交
1737 1738 1739 1740 1741
}

/*-----------------------------------------------------------------
 * Event notification.
 *---------------------------------------------------------------*/
1742 1743
void dm_kobject_uevent(struct mapped_device *md)
{
1744
	kobject_uevent(&disk_to_dev(md->disk)->kobj, KOBJ_CHANGE);
1745 1746
}

M
Mike Anderson 已提交
1747 1748 1749 1750 1751
uint32_t dm_next_uevent_seq(struct mapped_device *md)
{
	return atomic_add_return(1, &md->uevent_seq);
}

L
Linus Torvalds 已提交
1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762
uint32_t dm_get_event_nr(struct mapped_device *md)
{
	return atomic_read(&md->event_nr);
}

int dm_wait_event(struct mapped_device *md, int event_nr)
{
	return wait_event_interruptible(md->eventq,
			(event_nr != atomic_read(&md->event_nr)));
}

M
Mike Anderson 已提交
1763 1764 1765 1766 1767 1768 1769 1770 1771
void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
{
	unsigned long flags;

	spin_lock_irqsave(&md->uevent_lock, flags);
	list_add(elist, &md->uevent_list);
	spin_unlock_irqrestore(&md->uevent_lock, flags);
}

L
Linus Torvalds 已提交
1772 1773 1774 1775 1776 1777 1778 1779 1780
/*
 * The gendisk is only valid as long as you have a reference
 * count on 'md'.
 */
struct gendisk *dm_disk(struct mapped_device *md)
{
	return md->disk;
}

M
Milan Broz 已提交
1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797
struct kobject *dm_kobject(struct mapped_device *md)
{
	return &md->kobj;
}

/*
 * struct mapped_device should not be exported outside of dm.c
 * so use this check to verify that kobj is part of md structure
 */
struct mapped_device *dm_get_from_kobject(struct kobject *kobj)
{
	struct mapped_device *md;

	md = container_of(kobj, struct mapped_device, kobj);
	if (&md->kobj != kobj)
		return NULL;

1798 1799 1800 1801
	if (test_bit(DMF_FREEING, &md->flags) ||
	    test_bit(DMF_DELETING, &md->flags))
		return NULL;

M
Milan Broz 已提交
1802 1803 1804 1805
	dm_get(md);
	return md;
}

L
Linus Torvalds 已提交
1806 1807 1808 1809 1810
int dm_suspended(struct mapped_device *md)
{
	return test_bit(DMF_SUSPENDED, &md->flags);
}

1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821
int dm_noflush_suspending(struct dm_target *ti)
{
	struct mapped_device *md = dm_table_get_md(ti->table);
	int r = __noflush_suspending(md);

	dm_put(md);

	return r;
}
EXPORT_SYMBOL_GPL(dm_noflush_suspending);

L
Linus Torvalds 已提交
1822 1823 1824
static struct block_device_operations dm_blk_dops = {
	.open = dm_blk_open,
	.release = dm_blk_close,
1825
	.ioctl = dm_blk_ioctl,
D
Darrick J. Wong 已提交
1826
	.getgeo = dm_blk_getgeo,
L
Linus Torvalds 已提交
1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842
	.owner = THIS_MODULE
};

EXPORT_SYMBOL(dm_get_mapinfo);

/*
 * module hooks
 */
module_init(dm_init);
module_exit(dm_exit);

module_param(major, uint, 0);
MODULE_PARM_DESC(major, "The major number of the device mapper");
MODULE_DESCRIPTION(DM_NAME " driver");
MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
MODULE_LICENSE("GPL");