dm-table.c 50.8 KB
Newer Older
L
Linus Torvalds 已提交
1 2
/*
 * Copyright (C) 2001 Sistina Software (UK) Limited.
3
 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
L
Linus Torvalds 已提交
4 5 6 7
 *
 * This file is released under the GPL.
 */

8
#include "dm-core.h"
L
Linus Torvalds 已提交
9 10 11 12 13 14

#include <linux/module.h>
#include <linux/vmalloc.h>
#include <linux/blkdev.h>
#include <linux/namei.h>
#include <linux/ctype.h>
15
#include <linux/string.h>
L
Linus Torvalds 已提交
16 17
#include <linux/slab.h>
#include <linux/interrupt.h>
A
Arjan van de Ven 已提交
18
#include <linux/mutex.h>
19
#include <linux/delay.h>
A
Arun Sharma 已提交
20
#include <linux/atomic.h>
21
#include <linux/blk-mq.h>
22
#include <linux/mount.h>
23
#include <linux/dax.h>
L
Linus Torvalds 已提交
24

25 26
#define DM_MSG_PREFIX "table"

L
Linus Torvalds 已提交
27 28 29 30 31 32
#define MAX_DEPTH 16
#define NODE_SIZE L1_CACHE_BYTES
#define KEYS_PER_NODE (NODE_SIZE / sizeof(sector_t))
#define CHILDREN_PER_NODE (KEYS_PER_NODE + 1)

struct dm_table {
M
Mike Anderson 已提交
33
	struct mapped_device *md;
34
	enum dm_queue_mode type;
L
Linus Torvalds 已提交
35 36 37 38 39 40 41 42 43 44 45

	/* btree table */
	unsigned int depth;
	unsigned int counts[MAX_DEPTH];	/* in nodes */
	sector_t *index[MAX_DEPTH];

	unsigned int num_targets;
	unsigned int num_allocated;
	sector_t *highs;
	struct dm_target *targets;

46
	struct target_type *immutable_target_type;
47 48 49 50

	bool integrity_supported:1;
	bool singleton:1;
	bool all_blk_mq:1;
51
	unsigned integrity_added:1;
M
Mike Snitzer 已提交
52

L
Linus Torvalds 已提交
53 54 55 56 57
	/*
	 * Indicates the rw permissions for the new logical
	 * device.  This should be a combination of FMODE_READ
	 * and FMODE_WRITE.
	 */
58
	fmode_t mode;
L
Linus Torvalds 已提交
59 60 61 62 63 64 65

	/* a list of devices used by this table */
	struct list_head devices;

	/* events get handed up using this callback */
	void (*event_fn)(void *);
	void *event_context;
K
Kiyoshi Ueda 已提交
66 67

	struct dm_md_mempools *mempools;
68 69

	struct list_head target_callbacks;
L
Linus Torvalds 已提交
70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149
};

/*
 * Similar to ceiling(log_size(n))
 */
static unsigned int int_log(unsigned int n, unsigned int base)
{
	int result = 0;

	while (n > 1) {
		n = dm_div_up(n, base);
		result++;
	}

	return result;
}

/*
 * Calculate the index of the child node of the n'th node k'th key.
 */
static inline unsigned int get_child(unsigned int n, unsigned int k)
{
	return (n * CHILDREN_PER_NODE) + k;
}

/*
 * Return the n'th node of level l from table t.
 */
static inline sector_t *get_node(struct dm_table *t,
				 unsigned int l, unsigned int n)
{
	return t->index[l] + (n * KEYS_PER_NODE);
}

/*
 * Return the highest key that you could lookup from the n'th
 * node on level l of the btree.
 */
static sector_t high(struct dm_table *t, unsigned int l, unsigned int n)
{
	for (; l < t->depth - 1; l++)
		n = get_child(n, CHILDREN_PER_NODE - 1);

	if (n >= t->counts[l])
		return (sector_t) - 1;

	return get_node(t, l, n)[KEYS_PER_NODE - 1];
}

/*
 * Fills in a level of the btree based on the highs of the level
 * below it.
 */
static int setup_btree_index(unsigned int l, struct dm_table *t)
{
	unsigned int n, k;
	sector_t *node;

	for (n = 0U; n < t->counts[l]; n++) {
		node = get_node(t, l, n);

		for (k = 0U; k < KEYS_PER_NODE; k++)
			node[k] = high(t, l + 1, get_child(n, k));
	}

	return 0;
}

void *dm_vcalloc(unsigned long nmemb, unsigned long elem_size)
{
	unsigned long size;
	void *addr;

	/*
	 * Check that we're not going to overflow.
	 */
	if (nmemb > (ULONG_MAX / elem_size))
		return NULL;

	size = nmemb * elem_size;
J
Joe Perches 已提交
150
	addr = vzalloc(size);
L
Linus Torvalds 已提交
151 152 153

	return addr;
}
154
EXPORT_SYMBOL(dm_vcalloc);
L
Linus Torvalds 已提交
155 156 157 158 159 160 161 162 163 164 165 166

/*
 * highs, and targets are managed as dynamic arrays during a
 * table load.
 */
static int alloc_targets(struct dm_table *t, unsigned int num)
{
	sector_t *n_highs;
	struct dm_target *n_targets;

	/*
	 * Allocate both the target array and offset array at once.
167 168
	 * Append an empty entry to catch sectors beyond the end of
	 * the device.
L
Linus Torvalds 已提交
169
	 */
170
	n_highs = (sector_t *) dm_vcalloc(num + 1, sizeof(struct dm_target) +
L
Linus Torvalds 已提交
171 172 173 174 175 176
					  sizeof(sector_t));
	if (!n_highs)
		return -ENOMEM;

	n_targets = (struct dm_target *) (n_highs + num);

177
	memset(n_highs, -1, sizeof(*n_highs) * num);
L
Linus Torvalds 已提交
178 179 180 181 182 183 184 185 186
	vfree(t->highs);

	t->num_allocated = num;
	t->highs = n_highs;
	t->targets = n_targets;

	return 0;
}

187
int dm_table_create(struct dm_table **result, fmode_t mode,
M
Mike Anderson 已提交
188
		    unsigned num_targets, struct mapped_device *md)
L
Linus Torvalds 已提交
189
{
D
Dmitry Monakhov 已提交
190
	struct dm_table *t = kzalloc(sizeof(*t), GFP_KERNEL);
L
Linus Torvalds 已提交
191 192 193 194 195

	if (!t)
		return -ENOMEM;

	INIT_LIST_HEAD(&t->devices);
196
	INIT_LIST_HEAD(&t->target_callbacks);
L
Linus Torvalds 已提交
197 198 199 200 201 202

	if (!num_targets)
		num_targets = KEYS_PER_NODE;

	num_targets = dm_round_up(num_targets, KEYS_PER_NODE);

203 204 205 206 207
	if (!num_targets) {
		kfree(t);
		return -ENOMEM;
	}

L
Linus Torvalds 已提交
208 209 210 211 212
	if (alloc_targets(t, num_targets)) {
		kfree(t);
		return -ENOMEM;
	}

213
	t->type = DM_TYPE_NONE;
L
Linus Torvalds 已提交
214
	t->mode = mode;
M
Mike Anderson 已提交
215
	t->md = md;
L
Linus Torvalds 已提交
216 217 218 219
	*result = t;
	return 0;
}

220
static void free_devices(struct list_head *devices, struct mapped_device *md)
L
Linus Torvalds 已提交
221 222 223
{
	struct list_head *tmp, *next;

P
Paul Jimenez 已提交
224
	list_for_each_safe(tmp, next, devices) {
M
Mikulas Patocka 已提交
225 226
		struct dm_dev_internal *dd =
		    list_entry(tmp, struct dm_dev_internal, list);
227 228 229
		DMWARN("%s: dm_table_destroy: dm_put_device call missing for %s",
		       dm_device_name(md), dd->dm_dev->name);
		dm_put_table_device(md, dd->dm_dev);
L
Linus Torvalds 已提交
230 231 232 233
		kfree(dd);
	}
}

234
void dm_table_destroy(struct dm_table *t)
L
Linus Torvalds 已提交
235 236 237
{
	unsigned int i;

238 239 240
	if (!t)
		return;

241
	/* free the indexes */
L
Linus Torvalds 已提交
242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257
	if (t->depth >= 2)
		vfree(t->index[t->depth - 2]);

	/* free the targets */
	for (i = 0; i < t->num_targets; i++) {
		struct dm_target *tgt = t->targets + i;

		if (tgt->type->dtr)
			tgt->type->dtr(tgt);

		dm_put_target_type(tgt->type);
	}

	vfree(t->highs);

	/* free the device list */
258
	free_devices(&t->devices, t->md);
L
Linus Torvalds 已提交
259

K
Kiyoshi Ueda 已提交
260 261
	dm_free_md_mempools(t->mempools);

L
Linus Torvalds 已提交
262 263 264 265 266 267
	kfree(t);
}

/*
 * See if we've already got a device in the list.
 */
M
Mikulas Patocka 已提交
268
static struct dm_dev_internal *find_device(struct list_head *l, dev_t dev)
L
Linus Torvalds 已提交
269
{
M
Mikulas Patocka 已提交
270
	struct dm_dev_internal *dd;
L
Linus Torvalds 已提交
271 272

	list_for_each_entry (dd, l, list)
273
		if (dd->dm_dev->bdev->bd_dev == dev)
L
Linus Torvalds 已提交
274 275 276 277 278 279
			return dd;

	return NULL;
}

/*
280
 * If possible, this checks an area of a destination device is invalid.
L
Linus Torvalds 已提交
281
 */
282 283
static int device_area_is_invalid(struct dm_target *ti, struct dm_dev *dev,
				  sector_t start, sector_t len, void *data)
L
Linus Torvalds 已提交
284
{
285
	struct request_queue *q;
286 287 288 289
	struct queue_limits *limits = data;
	struct block_device *bdev = dev->bdev;
	sector_t dev_size =
		i_size_read(bdev->bd_inode) >> SECTOR_SHIFT;
290
	unsigned short logical_block_size_sectors =
291
		limits->logical_block_size >> SECTOR_SHIFT;
292
	char b[BDEVNAME_SIZE];
M
Mike Anderson 已提交
293

294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309
	/*
	 * Some devices exist without request functions,
	 * such as loop devices not yet bound to backing files.
	 * Forbid the use of such devices.
	 */
	q = bdev_get_queue(bdev);
	if (!q || !q->make_request_fn) {
		DMWARN("%s: %s is not yet initialised: "
		       "start=%llu, len=%llu, dev_size=%llu",
		       dm_device_name(ti->table->md), bdevname(bdev, b),
		       (unsigned long long)start,
		       (unsigned long long)len,
		       (unsigned long long)dev_size);
		return 1;
	}

M
Mike Anderson 已提交
310
	if (!dev_size)
311
		return 0;
M
Mike Anderson 已提交
312

313
	if ((start >= dev_size) || (start + len > dev_size)) {
314 315 316 317 318 319
		DMWARN("%s: %s too small for target: "
		       "start=%llu, len=%llu, dev_size=%llu",
		       dm_device_name(ti->table->md), bdevname(bdev, b),
		       (unsigned long long)start,
		       (unsigned long long)len,
		       (unsigned long long)dev_size);
320
		return 1;
321 322
	}

323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355
	/*
	 * If the target is mapped to zoned block device(s), check
	 * that the zones are not partially mapped.
	 */
	if (bdev_zoned_model(bdev) != BLK_ZONED_NONE) {
		unsigned int zone_sectors = bdev_zone_sectors(bdev);

		if (start & (zone_sectors - 1)) {
			DMWARN("%s: start=%llu not aligned to h/w zone size %u of %s",
			       dm_device_name(ti->table->md),
			       (unsigned long long)start,
			       zone_sectors, bdevname(bdev, b));
			return 1;
		}

		/*
		 * Note: The last zone of a zoned block device may be smaller
		 * than other zones. So for a target mapping the end of a
		 * zoned block device with such a zone, len would not be zone
		 * aligned. We do not allow such last smaller zone to be part
		 * of the mapping here to ensure that mappings with multiple
		 * devices do not end up with a smaller zone in the middle of
		 * the sector range.
		 */
		if (len & (zone_sectors - 1)) {
			DMWARN("%s: len=%llu not aligned to h/w zone size %u of %s",
			       dm_device_name(ti->table->md),
			       (unsigned long long)len,
			       zone_sectors, bdevname(bdev, b));
			return 1;
		}
	}

356
	if (logical_block_size_sectors <= 1)
357
		return 0;
358 359 360

	if (start & (logical_block_size_sectors - 1)) {
		DMWARN("%s: start=%llu not aligned to h/w "
361
		       "logical block size %u of %s",
362 363
		       dm_device_name(ti->table->md),
		       (unsigned long long)start,
364
		       limits->logical_block_size, bdevname(bdev, b));
365
		return 1;
366 367
	}

368
	if (len & (logical_block_size_sectors - 1)) {
369
		DMWARN("%s: len=%llu not aligned to h/w "
370
		       "logical block size %u of %s",
371
		       dm_device_name(ti->table->md),
372
		       (unsigned long long)len,
373
		       limits->logical_block_size, bdevname(bdev, b));
374
		return 1;
375 376
	}

377
	return 0;
L
Linus Torvalds 已提交
378 379 380
}

/*
381
 * This upgrades the mode on an already open dm_dev, being
L
Linus Torvalds 已提交
382
 * careful to leave things as they were if we fail to reopen the
383 384
 * device and not to touch the existing bdev field in case
 * it is accessed concurrently inside dm_table_any_congested().
L
Linus Torvalds 已提交
385
 */
386
static int upgrade_mode(struct dm_dev_internal *dd, fmode_t new_mode,
M
Mikulas Patocka 已提交
387
			struct mapped_device *md)
L
Linus Torvalds 已提交
388 389
{
	int r;
390
	struct dm_dev *old_dev, *new_dev;
L
Linus Torvalds 已提交
391

392
	old_dev = dd->dm_dev;
393

394 395
	r = dm_get_table_device(md, dd->dm_dev->bdev->bd_dev,
				dd->dm_dev->mode | new_mode, &new_dev);
396 397
	if (r)
		return r;
L
Linus Torvalds 已提交
398

399 400
	dd->dm_dev = new_dev;
	dm_put_table_device(md, old_dev);
L
Linus Torvalds 已提交
401

402
	return 0;
L
Linus Torvalds 已提交
403 404
}

405 406 407 408 409
/*
 * Convert the path to a device
 */
dev_t dm_get_dev_t(const char *path)
{
410
	dev_t dev;
411 412 413 414 415 416 417 418 419 420 421 422 423 424
	struct block_device *bdev;

	bdev = lookup_bdev(path);
	if (IS_ERR(bdev))
		dev = name_to_dev_t(path);
	else {
		dev = bdev->bd_dev;
		bdput(bdev);
	}

	return dev;
}
EXPORT_SYMBOL_GPL(dm_get_dev_t);

L
Linus Torvalds 已提交
425 426 427 428
/*
 * Add a device to the list, or just increment the usage count if
 * it's already present.
 */
429 430
int dm_get_device(struct dm_target *ti, const char *path, fmode_t mode,
		  struct dm_dev **result)
L
Linus Torvalds 已提交
431 432
{
	int r;
433
	dev_t dev;
M
Mikulas Patocka 已提交
434
	struct dm_dev_internal *dd;
435
	struct dm_table *t = ti->table;
L
Linus Torvalds 已提交
436

437
	BUG_ON(!t);
L
Linus Torvalds 已提交
438

439 440 441
	dev = dm_get_dev_t(path);
	if (!dev)
		return -ENODEV;
L
Linus Torvalds 已提交
442 443 444 445 446 447 448

	dd = find_device(&t->devices, dev);
	if (!dd) {
		dd = kmalloc(sizeof(*dd), GFP_KERNEL);
		if (!dd)
			return -ENOMEM;

449
		if ((r = dm_get_table_device(t->md, dev, mode, &dd->dm_dev))) {
L
Linus Torvalds 已提交
450 451 452 453
			kfree(dd);
			return r;
		}

454
		refcount_set(&dd->count, 1);
L
Linus Torvalds 已提交
455
		list_add(&dd->list, &t->devices);
456
		goto out;
L
Linus Torvalds 已提交
457

458
	} else if (dd->dm_dev->mode != (mode | dd->dm_dev->mode)) {
459
		r = upgrade_mode(dd, mode, t->md);
L
Linus Torvalds 已提交
460 461 462
		if (r)
			return r;
	}
463 464
	refcount_inc(&dd->count);
out:
465
	*result = dd->dm_dev;
L
Linus Torvalds 已提交
466 467
	return 0;
}
468
EXPORT_SYMBOL(dm_get_device);
L
Linus Torvalds 已提交
469

470 471
static int dm_set_device_limits(struct dm_target *ti, struct dm_dev *dev,
				sector_t start, sector_t len, void *data)
L
Linus Torvalds 已提交
472
{
473 474
	struct queue_limits *limits = data;
	struct block_device *bdev = dev->bdev;
475
	struct request_queue *q = bdev_get_queue(bdev);
A
Alasdair G Kergon 已提交
476 477 478 479 480
	char b[BDEVNAME_SIZE];

	if (unlikely(!q)) {
		DMWARN("%s: Cannot set limits for nonexistent device %s",
		       dm_device_name(ti->table->md), bdevname(bdev, b));
481
		return 0;
A
Alasdair G Kergon 已提交
482
	}
483

484 485
	if (bdev_stack_limits(limits, bdev, start) < 0)
		DMWARN("%s: adding target device %s caused an alignment inconsistency: "
486 487 488 489 490 491
		       "physical_block_size=%u, logical_block_size=%u, "
		       "alignment_offset=%u, start=%llu",
		       dm_device_name(ti->table->md), bdevname(bdev, b),
		       q->limits.physical_block_size,
		       q->limits.logical_block_size,
		       q->limits.alignment_offset,
492
		       (unsigned long long) start << SECTOR_SHIFT);
493

494 495
	limits->zoned = blk_queue_zoned_model(q);

496
	return 0;
497
}
498

L
Linus Torvalds 已提交
499
/*
500
 * Decrement a device's use count and remove it if necessary.
L
Linus Torvalds 已提交
501
 */
M
Mikulas Patocka 已提交
502
void dm_put_device(struct dm_target *ti, struct dm_dev *d)
L
Linus Torvalds 已提交
503
{
504 505 506
	int found = 0;
	struct list_head *devices = &ti->table->devices;
	struct dm_dev_internal *dd;
M
Mikulas Patocka 已提交
507

508 509 510 511 512 513 514 515 516 517 518
	list_for_each_entry(dd, devices, list) {
		if (dd->dm_dev == d) {
			found = 1;
			break;
		}
	}
	if (!found) {
		DMWARN("%s: device %s not in table devices list",
		       dm_device_name(ti->table->md), d->name);
		return;
	}
519
	if (refcount_dec_and_test(&dd->count)) {
520
		dm_put_table_device(ti->table->md, d);
L
Linus Torvalds 已提交
521 522 523 524
		list_del(&dd->list);
		kfree(dd);
	}
}
525
EXPORT_SYMBOL(dm_put_device);
L
Linus Torvalds 已提交
526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542

/*
 * Checks to see if the target joins onto the end of the table.
 */
static int adjoin(struct dm_table *table, struct dm_target *ti)
{
	struct dm_target *prev;

	if (!table->num_targets)
		return !ti->begin;

	prev = &table->targets[table->num_targets - 1];
	return (ti->begin == (prev->begin + prev->len));
}

/*
 * Used to dynamically allocate the arg array.
543 544 545 546 547 548 549
 *
 * We do first allocation with GFP_NOIO because dm-mpath and dm-thin must
 * process messages even if some device is suspended. These messages have a
 * small fixed number of arguments.
 *
 * On the other hand, dm-switch needs to process bulk data using messages and
 * excessive use of GFP_NOIO could cause trouble.
L
Linus Torvalds 已提交
550
 */
551
static char **realloc_argv(unsigned *size, char **old_argv)
L
Linus Torvalds 已提交
552 553 554
{
	char **argv;
	unsigned new_size;
555
	gfp_t gfp;
L
Linus Torvalds 已提交
556

557 558
	if (*size) {
		new_size = *size * 2;
559 560 561 562 563
		gfp = GFP_KERNEL;
	} else {
		new_size = 8;
		gfp = GFP_NOIO;
	}
564
	argv = kmalloc_array(new_size, sizeof(*argv), gfp);
565
	if (argv && old_argv) {
566 567
		memcpy(argv, old_argv, *size * sizeof(*argv));
		*size = new_size;
L
Linus Torvalds 已提交
568 569 570 571 572 573 574 575 576 577 578 579 580 581 582
	}

	kfree(old_argv);
	return argv;
}

/*
 * Destructively splits up the argument list to pass to ctr.
 */
int dm_split_args(int *argc, char ***argvp, char *input)
{
	char *start, *end = input, *out, **argv = NULL;
	unsigned array_size = 0;

	*argc = 0;
583 584 585 586 587 588

	if (!input) {
		*argvp = NULL;
		return 0;
	}

L
Linus Torvalds 已提交
589 590 591 592 593 594
	argv = realloc_argv(&array_size, argv);
	if (!argv)
		return -ENOMEM;

	while (1) {
		/* Skip whitespace */
595
		start = skip_spaces(end);
L
Linus Torvalds 已提交
596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636

		if (!*start)
			break;	/* success, we hit the end */

		/* 'out' is used to remove any back-quotes */
		end = out = start;
		while (*end) {
			/* Everything apart from '\0' can be quoted */
			if (*end == '\\' && *(end + 1)) {
				*out++ = *(end + 1);
				end += 2;
				continue;
			}

			if (isspace(*end))
				break;	/* end of token */

			*out++ = *end++;
		}

		/* have we already filled the array ? */
		if ((*argc + 1) > array_size) {
			argv = realloc_argv(&array_size, argv);
			if (!argv)
				return -ENOMEM;
		}

		/* we know this is whitespace */
		if (*end)
			end++;

		/* terminate the string and put it in the array */
		*out = '\0';
		argv[*argc] = start;
		(*argc)++;
	}

	*argvp = argv;
	return 0;
}

637 638 639 640 641 642 643
/*
 * Impose necessary and sufficient conditions on a devices's table such
 * that any incoming bio which respects its logical_block_size can be
 * processed successfully.  If it falls across the boundary between
 * two or more targets, the size of each piece it gets split into must
 * be compatible with the logical_block_size of the target processing it.
 */
644 645
static int validate_hardware_logical_block_alignment(struct dm_table *table,
						 struct queue_limits *limits)
646 647 648 649 650 651
{
	/*
	 * This function uses arithmetic modulo the logical_block_size
	 * (in units of 512-byte sectors).
	 */
	unsigned short device_logical_block_size_sects =
652
		limits->logical_block_size >> SECTOR_SHIFT;
653 654 655 656 657 658 659 660 661 662 663 664 665

	/*
	 * Offset of the start of the next table entry, mod logical_block_size.
	 */
	unsigned short next_target_start = 0;

	/*
	 * Given an aligned bio that extends beyond the end of a
	 * target, how many sectors must the next target handle?
	 */
	unsigned short remaining = 0;

	struct dm_target *uninitialized_var(ti);
666
	struct queue_limits ti_limits;
667
	unsigned i;
668 669 670 671

	/*
	 * Check each entry in the table in turn.
	 */
672 673
	for (i = 0; i < dm_table_get_num_targets(table); i++) {
		ti = dm_table_get_target(table, i);
674

675
		blk_set_stacking_limits(&ti_limits);
676 677 678 679 680 681

		/* combine all target devices' limits */
		if (ti->type->iterate_devices)
			ti->type->iterate_devices(ti, dm_set_device_limits,
						  &ti_limits);

682 683 684 685 686
		/*
		 * If the remaining sectors fall entirely within this
		 * table entry are they compatible with its logical_block_size?
		 */
		if (remaining < ti->len &&
687
		    remaining & ((ti_limits.logical_block_size >>
688 689 690 691 692 693 694 695 696 697 698 699
				  SECTOR_SHIFT) - 1))
			break;	/* Error */

		next_target_start =
		    (unsigned short) ((next_target_start + ti->len) &
				      (device_logical_block_size_sects - 1));
		remaining = next_target_start ?
		    device_logical_block_size_sects - next_target_start : 0;
	}

	if (remaining) {
		DMWARN("%s: table line %u (start sect %llu len %llu) "
700
		       "not aligned to h/w logical block size %u",
701 702 703
		       dm_device_name(table->md), i,
		       (unsigned long long) ti->begin,
		       (unsigned long long) ti->len,
704
		       limits->logical_block_size);
705 706 707 708 709 710
		return -EINVAL;
	}

	return 0;
}

L
Linus Torvalds 已提交
711 712 713 714 715 716 717
int dm_table_add_target(struct dm_table *t, const char *type,
			sector_t start, sector_t len, char *params)
{
	int r = -EINVAL, argc;
	char **argv;
	struct dm_target *tgt;

718 719 720 721 722 723
	if (t->singleton) {
		DMERR("%s: target type %s must appear alone in table",
		      dm_device_name(t->md), t->targets->type->name);
		return -EINVAL;
	}

724
	BUG_ON(t->num_targets >= t->num_allocated);
L
Linus Torvalds 已提交
725 726 727 728 729

	tgt = t->targets + t->num_targets;
	memset(tgt, 0, sizeof(*tgt));

	if (!len) {
730
		DMERR("%s: zero-length target", dm_device_name(t->md));
L
Linus Torvalds 已提交
731 732 733 734 735
		return -EINVAL;
	}

	tgt->type = dm_get_target_type(type);
	if (!tgt->type) {
736
		DMERR("%s: %s: unknown target type", dm_device_name(t->md), type);
L
Linus Torvalds 已提交
737 738 739
		return -EINVAL;
	}

740 741
	if (dm_target_needs_singleton(tgt->type)) {
		if (t->num_targets) {
742 743
			tgt->error = "singleton target type must appear alone in table";
			goto bad;
744
		}
745
		t->singleton = true;
746 747
	}

748
	if (dm_target_always_writeable(tgt->type) && !(t->mode & FMODE_WRITE)) {
749 750
		tgt->error = "target type may not be included in a read-only table";
		goto bad;
751 752
	}

753 754
	if (t->immutable_target_type) {
		if (t->immutable_target_type != tgt->type) {
755 756
			tgt->error = "immutable target type cannot be mixed with other target types";
			goto bad;
757 758 759
		}
	} else if (dm_target_is_immutable(tgt->type)) {
		if (t->num_targets) {
760 761
			tgt->error = "immutable target type cannot be mixed with other target types";
			goto bad;
762 763 764 765
		}
		t->immutable_target_type = tgt->type;
	}

766 767 768
	if (dm_target_has_integrity(tgt->type))
		t->integrity_added = 1;

L
Linus Torvalds 已提交
769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794
	tgt->table = t;
	tgt->begin = start;
	tgt->len = len;
	tgt->error = "Unknown error";

	/*
	 * Does this target adjoin the previous one ?
	 */
	if (!adjoin(t, tgt)) {
		tgt->error = "Gap in table";
		goto bad;
	}

	r = dm_split_args(&argc, &argv, params);
	if (r) {
		tgt->error = "couldn't split parameters (insufficient memory)";
		goto bad;
	}

	r = tgt->type->ctr(tgt, argc, argv);
	kfree(argv);
	if (r)
		goto bad;

	t->highs[t->num_targets++] = tgt->begin + tgt->len - 1;

795 796
	if (!tgt->num_discard_bios && tgt->discards_supported)
		DMWARN("%s: %s: ignoring discards_supported because num_discard_bios is zero.",
M
Mike Snitzer 已提交
797
		       dm_device_name(t->md), type);
M
Mike Snitzer 已提交
798

L
Linus Torvalds 已提交
799 800 801
	return 0;

 bad:
802
	DMERR("%s: %s: %s", dm_device_name(t->md), type, tgt->error);
L
Linus Torvalds 已提交
803 804 805 806
	dm_put_target_type(tgt->type);
	return r;
}

807 808 809
/*
 * Target argument parsing helpers.
 */
E
Eric Biggers 已提交
810 811
static int validate_next_arg(const struct dm_arg *arg,
			     struct dm_arg_set *arg_set,
812 813 814
			     unsigned *value, char **error, unsigned grouped)
{
	const char *arg_str = dm_shift_arg(arg_set);
815
	char dummy;
816 817

	if (!arg_str ||
818
	    (sscanf(arg_str, "%u%c", value, &dummy) != 1) ||
819 820 821 822 823 824 825 826 827 828
	    (*value < arg->min) ||
	    (*value > arg->max) ||
	    (grouped && arg_set->argc < *value)) {
		*error = arg->error;
		return -EINVAL;
	}

	return 0;
}

E
Eric Biggers 已提交
829
int dm_read_arg(const struct dm_arg *arg, struct dm_arg_set *arg_set,
830 831 832 833 834 835
		unsigned *value, char **error)
{
	return validate_next_arg(arg, arg_set, value, error, 0);
}
EXPORT_SYMBOL(dm_read_arg);

E
Eric Biggers 已提交
836
int dm_read_arg_group(const struct dm_arg *arg, struct dm_arg_set *arg_set,
837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865
		      unsigned *value, char **error)
{
	return validate_next_arg(arg, arg_set, value, error, 1);
}
EXPORT_SYMBOL(dm_read_arg_group);

const char *dm_shift_arg(struct dm_arg_set *as)
{
	char *r;

	if (as->argc) {
		as->argc--;
		r = *as->argv;
		as->argv++;
		return r;
	}

	return NULL;
}
EXPORT_SYMBOL(dm_shift_arg);

void dm_consume_args(struct dm_arg_set *as, unsigned num_args)
{
	BUG_ON(as->argc < num_args);
	as->argc -= num_args;
	as->argv += num_args;
}
EXPORT_SYMBOL(dm_consume_args);

866
static bool __table_type_bio_based(enum dm_queue_mode table_type)
867 868
{
	return (table_type == DM_TYPE_BIO_BASED ||
869 870
		table_type == DM_TYPE_DAX_BIO_BASED ||
		table_type == DM_TYPE_NVME_BIO_BASED);
871 872
}

873
static bool __table_type_request_based(enum dm_queue_mode table_type)
874 875 876 877 878
{
	return (table_type == DM_TYPE_REQUEST_BASED ||
		table_type == DM_TYPE_MQ_REQUEST_BASED);
}

879
void dm_table_set_type(struct dm_table *t, enum dm_queue_mode type)
880 881 882 883 884
{
	t->type = type;
}
EXPORT_SYMBOL_GPL(dm_table_set_type);

885 886 887
static int device_supports_dax(struct dm_target *ti, struct dm_dev *dev,
			       sector_t start, sector_t len, void *data)
{
888
	return bdev_dax_supported(dev->bdev, PAGE_SIZE);
889 890 891 892 893
}

static bool dm_table_supports_dax(struct dm_table *t)
{
	struct dm_target *ti;
894
	unsigned i;
895 896

	/* Ensure that all targets support DAX. */
897 898
	for (i = 0; i < dm_table_get_num_targets(t); i++) {
		ti = dm_table_get_target(t, i);
899 900 901 902 903 904 905 906 907 908 909 910

		if (!ti->type->direct_access)
			return false;

		if (!ti->type->iterate_devices ||
		    !ti->type->iterate_devices(ti, device_supports_dax, NULL))
			return false;
	}

	return true;
}

911 912
static bool dm_table_does_not_support_partial_completion(struct dm_table *t);

913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931
struct verify_rq_based_data {
	unsigned sq_count;
	unsigned mq_count;
};

static int device_is_rq_based(struct dm_target *ti, struct dm_dev *dev,
			      sector_t start, sector_t len, void *data)
{
	struct request_queue *q = bdev_get_queue(dev->bdev);
	struct verify_rq_based_data *v = data;

	if (q->mq_ops)
		v->mq_count++;
	else
		v->sq_count++;

	return queue_is_rq_based(q);
}

932
static int dm_table_determine_type(struct dm_table *t)
K
Kiyoshi Ueda 已提交
933 934
{
	unsigned i;
935
	unsigned bio_based = 0, request_based = 0, hybrid = 0;
936
	struct verify_rq_based_data v = {.sq_count = 0, .mq_count = 0};
K
Kiyoshi Ueda 已提交
937
	struct dm_target *tgt;
938
	struct list_head *devices = dm_table_get_devices(t);
939
	enum dm_queue_mode live_md_type = dm_get_md_type(t->md);
K
Kiyoshi Ueda 已提交
940

941 942
	if (t->type != DM_TYPE_NONE) {
		/* target already set the table's type */
943 944 945
		if (t->type == DM_TYPE_BIO_BASED) {
			/* possibly upgrade to a variant of bio-based */
			goto verify_bio_based;
946
		}
947
		BUG_ON(t->type == DM_TYPE_DAX_BIO_BASED);
948
		BUG_ON(t->type == DM_TYPE_NVME_BIO_BASED);
949 950 951
		goto verify_rq_based;
	}

K
Kiyoshi Ueda 已提交
952 953
	for (i = 0; i < t->num_targets; i++) {
		tgt = t->targets + i;
954 955 956
		if (dm_target_hybrid(tgt))
			hybrid = 1;
		else if (dm_target_request_based(tgt))
K
Kiyoshi Ueda 已提交
957 958 959 960 961
			request_based = 1;
		else
			bio_based = 1;

		if (bio_based && request_based) {
962 963
			DMERR("Inconsistent table: different target types"
			      " can't be mixed up");
K
Kiyoshi Ueda 已提交
964 965 966 967
			return -EINVAL;
		}
	}

968 969 970 971 972 973
	if (hybrid && !bio_based && !request_based) {
		/*
		 * The targets can work either way.
		 * Determine the type from the live device.
		 * Default to bio-based if device is new.
		 */
974
		if (__table_type_request_based(live_md_type))
975 976 977 978 979
			request_based = 1;
		else
			bio_based = 1;
	}

K
Kiyoshi Ueda 已提交
980
	if (bio_based) {
981
verify_bio_based:
K
Kiyoshi Ueda 已提交
982 983
		/* We must use this table as bio-based */
		t->type = DM_TYPE_BIO_BASED;
M
Mike Snitzer 已提交
984
		if (dm_table_supports_dax(t) ||
985
		    (list_empty(devices) && live_md_type == DM_TYPE_DAX_BIO_BASED)) {
986
			t->type = DM_TYPE_DAX_BIO_BASED;
987 988 989 990 991 992 993 994 995
		} else {
			/* Check if upgrading to NVMe bio-based is valid or required */
			tgt = dm_table_get_immutable_target(t);
			if (tgt && !tgt->max_io_len && dm_table_does_not_support_partial_completion(t)) {
				t->type = DM_TYPE_NVME_BIO_BASED;
				goto verify_rq_based; /* must be stacked directly on NVMe (blk-mq) */
			} else if (list_empty(devices) && live_md_type == DM_TYPE_NVME_BIO_BASED) {
				t->type = DM_TYPE_NVME_BIO_BASED;
			}
996
		}
K
Kiyoshi Ueda 已提交
997 998 999 1000 1001
		return 0;
	}

	BUG_ON(!request_based); /* No targets in this table */

1002 1003 1004 1005 1006 1007 1008
	/*
	 * The only way to establish DM_TYPE_MQ_REQUEST_BASED is by
	 * having a compatible target use dm_table_set_type.
	 */
	t->type = DM_TYPE_REQUEST_BASED;

verify_rq_based:
1009 1010 1011 1012 1013 1014 1015
	/*
	 * Request-based dm supports only tables that have a single target now.
	 * To support multiple targets, request splitting support is needed,
	 * and that needs lots of changes in the block-layer.
	 * (e.g. request completion process for partial completion.)
	 */
	if (t->num_targets > 1) {
1016 1017
		DMERR("%s DM doesn't support multiple targets",
		      t->type == DM_TYPE_NVME_BIO_BASED ? "nvme bio-based" : "request-based");
1018 1019 1020
		return -EINVAL;
	}

1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033
	if (list_empty(devices)) {
		int srcu_idx;
		struct dm_table *live_table = dm_get_live_table(t->md, &srcu_idx);

		/* inherit live table's type and all_blk_mq */
		if (live_table) {
			t->type = live_table->type;
			t->all_blk_mq = live_table->all_blk_mq;
		}
		dm_put_live_table(t->md, srcu_idx);
		return 0;
	}

1034 1035 1036 1037 1038 1039 1040 1041 1042
	tgt = dm_table_get_immutable_target(t);
	if (!tgt) {
		DMERR("table load rejected: immutable target is required");
		return -EINVAL;
	} else if (tgt->max_io_len) {
		DMERR("table load rejected: immutable target that splits IO is not supported");
		return -EINVAL;
	}

K
Kiyoshi Ueda 已提交
1043
	/* Non-request-stackable devices can't be used for request-based dm */
1044 1045 1046 1047
	if (!tgt->type->iterate_devices ||
	    !tgt->type->iterate_devices(tgt, device_is_rq_based, &v)) {
		DMERR("table load rejected: including non-request-stackable devices");
		return -EINVAL;
1048
	}
1049
	if (v.sq_count && v.mq_count) {
1050 1051
		DMERR("table load rejected: not all devices are blk-mq request-stackable");
		return -EINVAL;
1052
	}
1053
	t->all_blk_mq = v.mq_count > 0;
K
Kiyoshi Ueda 已提交
1054

1055 1056
	if (!t->all_blk_mq &&
	    (t->type == DM_TYPE_MQ_REQUEST_BASED || t->type == DM_TYPE_NVME_BIO_BASED)) {
1057 1058 1059 1060
		DMERR("table load rejected: all devices are not blk-mq request-stackable");
		return -EINVAL;
	}

K
Kiyoshi Ueda 已提交
1061 1062 1063
	return 0;
}

1064
enum dm_queue_mode dm_table_get_type(struct dm_table *t)
K
Kiyoshi Ueda 已提交
1065 1066 1067 1068
{
	return t->type;
}

1069 1070 1071 1072 1073
struct target_type *dm_table_get_immutable_target_type(struct dm_table *t)
{
	return t->immutable_target_type;
}

M
Mike Snitzer 已提交
1074 1075 1076 1077 1078 1079 1080 1081 1082 1083
struct dm_target *dm_table_get_immutable_target(struct dm_table *t)
{
	/* Immutable target is implicitly a singleton */
	if (t->num_targets > 1 ||
	    !dm_target_is_immutable(t->targets[0].type))
		return NULL;

	return t->targets;
}

1084 1085
struct dm_target *dm_table_get_wildcard_target(struct dm_table *t)
{
1086 1087
	struct dm_target *ti;
	unsigned i;
1088

1089 1090
	for (i = 0; i < dm_table_get_num_targets(t); i++) {
		ti = dm_table_get_target(t, i);
1091 1092 1093 1094 1095 1096 1097
		if (dm_target_is_wildcard(ti->type))
			return ti;
	}

	return NULL;
}

1098 1099 1100 1101 1102
bool dm_table_bio_based(struct dm_table *t)
{
	return __table_type_bio_based(dm_table_get_type(t));
}

K
Kiyoshi Ueda 已提交
1103 1104
bool dm_table_request_based(struct dm_table *t)
{
1105
	return __table_type_request_based(dm_table_get_type(t));
1106 1107
}

1108
bool dm_table_all_blk_mq_devices(struct dm_table *t)
1109
{
1110
	return t->all_blk_mq;
K
Kiyoshi Ueda 已提交
1111 1112
}

1113
static int dm_table_alloc_md_mempools(struct dm_table *t, struct mapped_device *md)
K
Kiyoshi Ueda 已提交
1114
{
1115
	enum dm_queue_mode type = dm_table_get_type(t);
1116
	unsigned per_io_data_size = 0;
1117 1118
	unsigned min_pool_size = 0;
	struct dm_target *ti;
M
Mikulas Patocka 已提交
1119
	unsigned i;
K
Kiyoshi Ueda 已提交
1120

1121
	if (unlikely(type == DM_TYPE_NONE)) {
K
Kiyoshi Ueda 已提交
1122 1123 1124 1125
		DMWARN("no table type is set, can't allocate mempools");
		return -EINVAL;
	}

1126
	if (__table_type_bio_based(type))
1127
		for (i = 0; i < t->num_targets; i++) {
1128 1129 1130
			ti = t->targets + i;
			per_io_data_size = max(per_io_data_size, ti->per_io_data_size);
			min_pool_size = max(min_pool_size, ti->num_flush_bios);
1131 1132
		}

1133 1134
	t->mempools = dm_alloc_md_mempools(md, type, t->integrity_supported,
					   per_io_data_size, min_pool_size);
1135 1136
	if (!t->mempools)
		return -ENOMEM;
K
Kiyoshi Ueda 已提交
1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151

	return 0;
}

void dm_table_free_md_mempools(struct dm_table *t)
{
	dm_free_md_mempools(t->mempools);
	t->mempools = NULL;
}

struct dm_md_mempools *dm_table_get_md_mempools(struct dm_table *t)
{
	return t->mempools;
}

L
Linus Torvalds 已提交
1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168
static int setup_indexes(struct dm_table *t)
{
	int i;
	unsigned int total = 0;
	sector_t *indexes;

	/* allocate the space for *all* the indexes */
	for (i = t->depth - 2; i >= 0; i--) {
		t->counts[i] = dm_div_up(t->counts[i + 1], CHILDREN_PER_NODE);
		total += t->counts[i];
	}

	indexes = (sector_t *) dm_vcalloc(total, (unsigned long) NODE_SIZE);
	if (!indexes)
		return -ENOMEM;

	/* set up internal nodes, bottom-up */
J
Jun'ichi Nomura 已提交
1169
	for (i = t->depth - 2; i >= 0; i--) {
L
Linus Torvalds 已提交
1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180
		t->index[i] = indexes;
		indexes += (KEYS_PER_NODE * t->counts[i]);
		setup_btree_index(i, t);
	}

	return 0;
}

/*
 * Builds the btree to index the map.
 */
1181
static int dm_table_build_index(struct dm_table *t)
L
Linus Torvalds 已提交
1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199
{
	int r = 0;
	unsigned int leaf_nodes;

	/* how many indexes will the btree have ? */
	leaf_nodes = dm_div_up(t->num_targets, KEYS_PER_NODE);
	t->depth = 1 + int_log(leaf_nodes, CHILDREN_PER_NODE);

	/* leaf layer has already been set up */
	t->counts[t->depth - 1] = leaf_nodes;
	t->index[t->depth - 1] = t->highs;

	if (t->depth >= 2)
		r = setup_indexes(t);

	return r;
}

1200 1201 1202 1203 1204
static bool integrity_profile_exists(struct gendisk *disk)
{
	return !!blk_get_integrity(disk);
}

1205 1206 1207 1208
/*
 * Get a disk whose integrity profile reflects the table's profile.
 * Returns NULL if integrity support was inconsistent or unavailable.
 */
1209
static struct gendisk * dm_table_get_integrity_disk(struct dm_table *t)
1210 1211 1212 1213
{
	struct list_head *devices = dm_table_get_devices(t);
	struct dm_dev_internal *dd = NULL;
	struct gendisk *prev_disk = NULL, *template_disk = NULL;
1214 1215 1216 1217 1218 1219 1220
	unsigned i;

	for (i = 0; i < dm_table_get_num_targets(t); i++) {
		struct dm_target *ti = dm_table_get_target(t, i);
		if (!dm_target_passes_integrity(ti->type))
			goto no_integrity;
	}
1221 1222

	list_for_each_entry(dd, devices, list) {
1223
		template_disk = dd->dm_dev->bdev->bd_disk;
1224
		if (!integrity_profile_exists(template_disk))
1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242
			goto no_integrity;
		else if (prev_disk &&
			 blk_integrity_compare(prev_disk, template_disk) < 0)
			goto no_integrity;
		prev_disk = template_disk;
	}

	return template_disk;

no_integrity:
	if (prev_disk)
		DMWARN("%s: integrity not set: %s and %s profile mismatch",
		       dm_device_name(t->md),
		       prev_disk->disk_name,
		       template_disk->disk_name);
	return NULL;
}

1243
/*
1244 1245 1246
 * Register the mapped device for blk_integrity support if the
 * underlying devices have an integrity profile.  But all devices may
 * not have matching profiles (checking all devices isn't reliable
1247
 * during table load because this table may use other DM device(s) which
1248 1249 1250 1251
 * must be resumed before they will have an initialized integity
 * profile).  Consequently, stacked DM devices force a 2 stage integrity
 * profile validation: First pass during table load, final pass during
 * resume.
1252
 */
1253
static int dm_table_register_integrity(struct dm_table *t)
1254
{
1255
	struct mapped_device *md = t->md;
1256
	struct gendisk *template_disk = NULL;
1257

1258 1259 1260 1261
	/* If target handles integrity itself do not register it here. */
	if (t->integrity_added)
		return 0;

1262
	template_disk = dm_table_get_integrity_disk(t);
1263 1264
	if (!template_disk)
		return 0;
1265

1266
	if (!integrity_profile_exists(dm_disk(md))) {
1267
		t->integrity_supported = true;
1268 1269 1270 1271 1272 1273 1274
		/*
		 * Register integrity profile during table load; we can do
		 * this because the final profile must match during resume.
		 */
		blk_integrity_register(dm_disk(md),
				       blk_get_integrity(template_disk));
		return 0;
1275 1276 1277
	}

	/*
1278
	 * If DM device already has an initialized integrity
1279 1280
	 * profile the new profile should not conflict.
	 */
1281
	if (blk_integrity_compare(dm_disk(md), template_disk) < 0) {
1282 1283 1284 1285 1286 1287 1288
		DMWARN("%s: conflict with existing integrity profile: "
		       "%s profile mismatch",
		       dm_device_name(t->md),
		       template_disk->disk_name);
		return 1;
	}

1289
	/* Preserve existing integrity profile */
1290
	t->integrity_supported = true;
1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301
	return 0;
}

/*
 * Prepares the table for use by building the indices,
 * setting the type, and allocating mempools.
 */
int dm_table_complete(struct dm_table *t)
{
	int r;

1302
	r = dm_table_determine_type(t);
1303
	if (r) {
1304
		DMERR("unable to determine table type");
1305 1306 1307 1308 1309 1310 1311 1312 1313
		return r;
	}

	r = dm_table_build_index(t);
	if (r) {
		DMERR("unable to build btrees");
		return r;
	}

1314
	r = dm_table_register_integrity(t);
1315 1316 1317 1318 1319
	if (r) {
		DMERR("could not register integrity profile.");
		return r;
	}

1320
	r = dm_table_alloc_md_mempools(t, t->md);
1321 1322 1323 1324 1325 1326
	if (r)
		DMERR("unable to allocate mempools");

	return r;
}

A
Arjan van de Ven 已提交
1327
static DEFINE_MUTEX(_event_lock);
L
Linus Torvalds 已提交
1328 1329 1330
void dm_table_event_callback(struct dm_table *t,
			     void (*fn)(void *), void *context)
{
A
Arjan van de Ven 已提交
1331
	mutex_lock(&_event_lock);
L
Linus Torvalds 已提交
1332 1333
	t->event_fn = fn;
	t->event_context = context;
A
Arjan van de Ven 已提交
1334
	mutex_unlock(&_event_lock);
L
Linus Torvalds 已提交
1335 1336 1337 1338 1339 1340 1341 1342 1343 1344
}

void dm_table_event(struct dm_table *t)
{
	/*
	 * You can no longer call dm_table_event() from interrupt
	 * context, use a bottom half instead.
	 */
	BUG_ON(in_interrupt());

A
Arjan van de Ven 已提交
1345
	mutex_lock(&_event_lock);
L
Linus Torvalds 已提交
1346 1347
	if (t->event_fn)
		t->event_fn(t->event_context);
A
Arjan van de Ven 已提交
1348
	mutex_unlock(&_event_lock);
L
Linus Torvalds 已提交
1349
}
1350
EXPORT_SYMBOL(dm_table_event);
L
Linus Torvalds 已提交
1351

1352
inline sector_t dm_table_get_size(struct dm_table *t)
L
Linus Torvalds 已提交
1353 1354 1355
{
	return t->num_targets ? (t->highs[t->num_targets - 1] + 1) : 0;
}
1356
EXPORT_SYMBOL(dm_table_get_size);
L
Linus Torvalds 已提交
1357 1358 1359

struct dm_target *dm_table_get_target(struct dm_table *t, unsigned int index)
{
1360
	if (index >= t->num_targets)
L
Linus Torvalds 已提交
1361 1362 1363 1364 1365 1366 1367
		return NULL;

	return t->targets + index;
}

/*
 * Search the btree for the correct target.
1368 1369 1370
 *
 * Caller should check returned pointer with dm_target_is_valid()
 * to trap I/O beyond end of device.
L
Linus Torvalds 已提交
1371 1372 1373 1374 1375 1376
 */
struct dm_target *dm_table_find_target(struct dm_table *t, sector_t sector)
{
	unsigned int l, n = 0, k = 0;
	sector_t *node;

1377 1378 1379
	if (unlikely(sector >= dm_table_get_size(t)))
		return &t->targets[t->num_targets];

L
Linus Torvalds 已提交
1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391
	for (l = 0; l < t->depth; l++) {
		n = get_child(n, k);
		node = get_node(t, l, n);

		for (k = 0; k < KEYS_PER_NODE; k++)
			if (node[k] >= sector)
				break;
	}

	return &t->targets[(KEYS_PER_NODE * n) + k];
}

1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409
static int count_device(struct dm_target *ti, struct dm_dev *dev,
			sector_t start, sector_t len, void *data)
{
	unsigned *num_devices = data;

	(*num_devices)++;

	return 0;
}

/*
 * Check whether a table has no data devices attached using each
 * target's iterate_devices method.
 * Returns false if the result is unknown because a target doesn't
 * support iterate_devices.
 */
bool dm_table_has_no_data_devices(struct dm_table *table)
{
1410 1411
	struct dm_target *ti;
	unsigned i, num_devices;
1412

1413 1414
	for (i = 0; i < dm_table_get_num_targets(table); i++) {
		ti = dm_table_get_target(table, i);
1415 1416 1417 1418

		if (!ti->type->iterate_devices)
			return false;

1419
		num_devices = 0;
1420 1421 1422 1423 1424 1425 1426 1427
		ti->type->iterate_devices(ti, count_device, &num_devices);
		if (num_devices)
			return false;
	}

	return true;
}

1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509
static int device_is_zoned_model(struct dm_target *ti, struct dm_dev *dev,
				 sector_t start, sector_t len, void *data)
{
	struct request_queue *q = bdev_get_queue(dev->bdev);
	enum blk_zoned_model *zoned_model = data;

	return q && blk_queue_zoned_model(q) == *zoned_model;
}

static bool dm_table_supports_zoned_model(struct dm_table *t,
					  enum blk_zoned_model zoned_model)
{
	struct dm_target *ti;
	unsigned i;

	for (i = 0; i < dm_table_get_num_targets(t); i++) {
		ti = dm_table_get_target(t, i);

		if (zoned_model == BLK_ZONED_HM &&
		    !dm_target_supports_zoned_hm(ti->type))
			return false;

		if (!ti->type->iterate_devices ||
		    !ti->type->iterate_devices(ti, device_is_zoned_model, &zoned_model))
			return false;
	}

	return true;
}

static int device_matches_zone_sectors(struct dm_target *ti, struct dm_dev *dev,
				       sector_t start, sector_t len, void *data)
{
	struct request_queue *q = bdev_get_queue(dev->bdev);
	unsigned int *zone_sectors = data;

	return q && blk_queue_zone_sectors(q) == *zone_sectors;
}

static bool dm_table_matches_zone_sectors(struct dm_table *t,
					  unsigned int zone_sectors)
{
	struct dm_target *ti;
	unsigned i;

	for (i = 0; i < dm_table_get_num_targets(t); i++) {
		ti = dm_table_get_target(t, i);

		if (!ti->type->iterate_devices ||
		    !ti->type->iterate_devices(ti, device_matches_zone_sectors, &zone_sectors))
			return false;
	}

	return true;
}

static int validate_hardware_zoned_model(struct dm_table *table,
					 enum blk_zoned_model zoned_model,
					 unsigned int zone_sectors)
{
	if (zoned_model == BLK_ZONED_NONE)
		return 0;

	if (!dm_table_supports_zoned_model(table, zoned_model)) {
		DMERR("%s: zoned model is not consistent across all devices",
		      dm_device_name(table->md));
		return -EINVAL;
	}

	/* Check zone size validity and compatibility */
	if (!zone_sectors || !is_power_of_2(zone_sectors))
		return -EINVAL;

	if (!dm_table_matches_zone_sectors(table, zone_sectors)) {
		DMERR("%s: zone sectors is not consistent across all devices",
		      dm_device_name(table->md));
		return -EINVAL;
	}

	return 0;
}

1510 1511 1512 1513 1514 1515
/*
 * Establish the new table's queue_limits and validate them.
 */
int dm_calculate_queue_limits(struct dm_table *table,
			      struct queue_limits *limits)
{
1516
	struct dm_target *ti;
1517
	struct queue_limits ti_limits;
1518
	unsigned i;
1519 1520
	enum blk_zoned_model zoned_model = BLK_ZONED_NONE;
	unsigned int zone_sectors = 0;
1521

1522
	blk_set_stacking_limits(limits);
1523

1524
	for (i = 0; i < dm_table_get_num_targets(table); i++) {
1525
		blk_set_stacking_limits(&ti_limits);
1526

1527
		ti = dm_table_get_target(table, i);
1528 1529 1530 1531 1532 1533 1534 1535 1536 1537

		if (!ti->type->iterate_devices)
			goto combine_limits;

		/*
		 * Combine queue limits of all the devices this target uses.
		 */
		ti->type->iterate_devices(ti, dm_set_device_limits,
					  &ti_limits);

1538 1539 1540 1541 1542 1543 1544 1545 1546
		if (zoned_model == BLK_ZONED_NONE && ti_limits.zoned != BLK_ZONED_NONE) {
			/*
			 * After stacking all limits, validate all devices
			 * in table support this zoned model and zone sectors.
			 */
			zoned_model = ti_limits.zoned;
			zone_sectors = ti_limits.chunk_sectors;
		}

1547 1548 1549 1550
		/* Set I/O hints portion of queue limits */
		if (ti->type->io_hints)
			ti->type->io_hints(ti, &ti_limits);

1551 1552 1553 1554
		/*
		 * Check each device area is consistent with the target's
		 * overall queue limits.
		 */
1555 1556
		if (ti->type->iterate_devices(ti, device_area_is_invalid,
					      &ti_limits))
1557 1558 1559 1560 1561 1562 1563 1564
			return -EINVAL;

combine_limits:
		/*
		 * Merge this target's queue limits into the overall limits
		 * for the table.
		 */
		if (blk_stack_limits(limits, &ti_limits, 0) < 0)
1565
			DMWARN("%s: adding target device "
1566
			       "(start sect %llu len %llu) "
1567
			       "caused an alignment inconsistency",
1568 1569 1570
			       dm_device_name(table->md),
			       (unsigned long long) ti->begin,
			       (unsigned long long) ti->len);
1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586

		/*
		 * FIXME: this should likely be moved to blk_stack_limits(), would
		 * also eliminate limits->zoned stacking hack in dm_set_device_limits()
		 */
		if (limits->zoned == BLK_ZONED_NONE && ti_limits.zoned != BLK_ZONED_NONE) {
			/*
			 * By default, the stacked limits zoned model is set to
			 * BLK_ZONED_NONE in blk_set_stacking_limits(). Update
			 * this model using the first target model reported
			 * that is not BLK_ZONED_NONE. This will be either the
			 * first target device zoned model or the model reported
			 * by the target .io_hints.
			 */
			limits->zoned = ti_limits.zoned;
		}
1587 1588
	}

1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606
	/*
	 * Verify that the zoned model and zone sectors, as determined before
	 * any .io_hints override, are the same across all devices in the table.
	 * - this is especially relevant if .io_hints is emulating a disk-managed
	 *   zoned model (aka BLK_ZONED_NONE) on host-managed zoned block devices.
	 * BUT...
	 */
	if (limits->zoned != BLK_ZONED_NONE) {
		/*
		 * ...IF the above limits stacking determined a zoned model
		 * validate that all of the table's devices conform to it.
		 */
		zoned_model = limits->zoned;
		zone_sectors = limits->chunk_sectors;
	}
	if (validate_hardware_zoned_model(table, zoned_model, zone_sectors))
		return -EINVAL;

1607 1608 1609
	return validate_hardware_logical_block_alignment(table, limits);
}

M
Martin K. Petersen 已提交
1610
/*
1611 1612 1613
 * Verify that all devices have an integrity profile that matches the
 * DM device's registered integrity profile.  If the profiles don't
 * match then unregister the DM device's integrity profile.
M
Martin K. Petersen 已提交
1614
 */
1615
static void dm_table_verify_integrity(struct dm_table *t)
M
Martin K. Petersen 已提交
1616
{
1617
	struct gendisk *template_disk = NULL;
M
Martin K. Petersen 已提交
1618

1619 1620 1621
	if (t->integrity_added)
		return;

1622 1623 1624 1625 1626 1627 1628 1629 1630 1631
	if (t->integrity_supported) {
		/*
		 * Verify that the original integrity profile
		 * matches all the devices in this table.
		 */
		template_disk = dm_table_get_integrity_disk(t);
		if (template_disk &&
		    blk_integrity_compare(dm_disk(t->md), template_disk) >= 0)
			return;
	}
M
Martin K. Petersen 已提交
1632

1633
	if (integrity_profile_exists(dm_disk(t->md))) {
1634 1635
		DMWARN("%s: unable to establish an integrity profile",
		       dm_device_name(t->md));
1636 1637
		blk_integrity_unregister(dm_disk(t->md));
	}
M
Martin K. Petersen 已提交
1638 1639
}

1640 1641 1642
static int device_flush_capable(struct dm_target *ti, struct dm_dev *dev,
				sector_t start, sector_t len, void *data)
{
J
Jens Axboe 已提交
1643
	unsigned long flush = (unsigned long) data;
1644 1645
	struct request_queue *q = bdev_get_queue(dev->bdev);

J
Jens Axboe 已提交
1646
	return q && (q->queue_flags & flush);
1647 1648
}

J
Jens Axboe 已提交
1649
static bool dm_table_supports_flush(struct dm_table *t, unsigned long flush)
1650 1651
{
	struct dm_target *ti;
1652
	unsigned i;
1653 1654 1655 1656 1657 1658 1659

	/*
	 * Require at least one underlying device to support flushes.
	 * t->devices includes internal dm devices such as mirror logs
	 * so we need to use iterate_devices here, which targets
	 * supporting flushes must provide.
	 */
1660 1661
	for (i = 0; i < dm_table_get_num_targets(t); i++) {
		ti = dm_table_get_target(t, i);
1662

1663
		if (!ti->num_flush_bios)
1664 1665
			continue;

1666
		if (ti->flush_supported)
1667
			return true;
1668

1669
		if (ti->type->iterate_devices &&
J
Jens Axboe 已提交
1670
		    ti->type->iterate_devices(ti, device_flush_capable, (void *) flush))
1671
			return true;
1672 1673
	}

1674
	return false;
1675 1676
}

1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707
static int device_dax_write_cache_enabled(struct dm_target *ti,
					  struct dm_dev *dev, sector_t start,
					  sector_t len, void *data)
{
	struct dax_device *dax_dev = dev->dax_dev;

	if (!dax_dev)
		return false;

	if (dax_write_cache_enabled(dax_dev))
		return true;
	return false;
}

static int dm_table_supports_dax_write_cache(struct dm_table *t)
{
	struct dm_target *ti;
	unsigned i;

	for (i = 0; i < dm_table_get_num_targets(t); i++) {
		ti = dm_table_get_target(t, i);

		if (ti->type->iterate_devices &&
		    ti->type->iterate_devices(ti,
				device_dax_write_cache_enabled, NULL))
			return true;
	}

	return false;
}

1708 1709 1710 1711 1712 1713 1714 1715
static int device_is_nonrot(struct dm_target *ti, struct dm_dev *dev,
			    sector_t start, sector_t len, void *data)
{
	struct request_queue *q = bdev_get_queue(dev->bdev);

	return q && blk_queue_nonrot(q);
}

1716 1717 1718 1719 1720 1721 1722 1723
static int device_is_not_random(struct dm_target *ti, struct dm_dev *dev,
			     sector_t start, sector_t len, void *data)
{
	struct request_queue *q = bdev_get_queue(dev->bdev);

	return q && !blk_queue_add_random(q);
}

1724 1725 1726 1727 1728 1729 1730 1731
static int queue_supports_sg_merge(struct dm_target *ti, struct dm_dev *dev,
				   sector_t start, sector_t len, void *data)
{
	struct request_queue *q = bdev_get_queue(dev->bdev);

	return q && !test_bit(QUEUE_FLAG_NO_SG_MERGE, &q->queue_flags);
}

1732 1733
static bool dm_table_all_devices_attribute(struct dm_table *t,
					   iterate_devices_callout_fn func)
1734 1735
{
	struct dm_target *ti;
1736
	unsigned i;
1737

1738 1739
	for (i = 0; i < dm_table_get_num_targets(t); i++) {
		ti = dm_table_get_target(t, i);
1740 1741

		if (!ti->type->iterate_devices ||
1742
		    !ti->type->iterate_devices(ti, func, NULL))
1743
			return false;
1744 1745
	}

1746
	return true;
1747 1748
}

1749 1750 1751 1752 1753 1754
static int device_no_partial_completion(struct dm_target *ti, struct dm_dev *dev,
					sector_t start, sector_t len, void *data)
{
	char b[BDEVNAME_SIZE];

	/* For now, NVMe devices are the only devices of this class */
M
Mikulas Patocka 已提交
1755
	return (strncmp(bdevname(dev->bdev, b), "nvme", 4) == 0);
1756 1757 1758 1759 1760 1761 1762
}

static bool dm_table_does_not_support_partial_completion(struct dm_table *t)
{
	return dm_table_all_devices_attribute(t, device_no_partial_completion);
}

M
Mike Snitzer 已提交
1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773
static int device_not_write_same_capable(struct dm_target *ti, struct dm_dev *dev,
					 sector_t start, sector_t len, void *data)
{
	struct request_queue *q = bdev_get_queue(dev->bdev);

	return q && !q->limits.max_write_same_sectors;
}

static bool dm_table_supports_write_same(struct dm_table *t)
{
	struct dm_target *ti;
1774
	unsigned i;
M
Mike Snitzer 已提交
1775

1776 1777
	for (i = 0; i < dm_table_get_num_targets(t); i++) {
		ti = dm_table_get_target(t, i);
M
Mike Snitzer 已提交
1778

1779
		if (!ti->num_write_same_bios)
M
Mike Snitzer 已提交
1780 1781 1782
			return false;

		if (!ti->type->iterate_devices ||
M
Mike Snitzer 已提交
1783
		    ti->type->iterate_devices(ti, device_not_write_same_capable, NULL))
M
Mike Snitzer 已提交
1784 1785 1786 1787 1788 1789
			return false;
	}

	return true;
}

1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816
static int device_not_write_zeroes_capable(struct dm_target *ti, struct dm_dev *dev,
					   sector_t start, sector_t len, void *data)
{
	struct request_queue *q = bdev_get_queue(dev->bdev);

	return q && !q->limits.max_write_zeroes_sectors;
}

static bool dm_table_supports_write_zeroes(struct dm_table *t)
{
	struct dm_target *ti;
	unsigned i = 0;

	while (i < dm_table_get_num_targets(t)) {
		ti = dm_table_get_target(t, i++);

		if (!ti->num_write_zeroes_bios)
			return false;

		if (!ti->type->iterate_devices ||
		    ti->type->iterate_devices(ti, device_not_write_zeroes_capable, NULL))
			return false;
	}

	return true;
}

1817 1818
static int device_not_discard_capable(struct dm_target *ti, struct dm_dev *dev,
				      sector_t start, sector_t len, void *data)
1819 1820 1821
{
	struct request_queue *q = bdev_get_queue(dev->bdev);

1822
	return q && !blk_queue_discard(q);
1823 1824 1825 1826 1827
}

static bool dm_table_supports_discards(struct dm_table *t)
{
	struct dm_target *ti;
1828
	unsigned i;
1829

1830 1831
	for (i = 0; i < dm_table_get_num_targets(t); i++) {
		ti = dm_table_get_target(t, i);
1832 1833

		if (!ti->num_discard_bios)
1834
			return false;
1835

1836 1837 1838 1839 1840 1841 1842 1843 1844
		/*
		 * Either the target provides discard support (as implied by setting
		 * 'discards_supported') or it relies on _all_ data devices having
		 * discard support.
		 */
		if (!ti->discards_supported &&
		    (!ti->type->iterate_devices ||
		     ti->type->iterate_devices(ti, device_not_discard_capable, NULL)))
			return false;
1845 1846
	}

1847
	return true;
1848 1849
}

1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877
static int device_not_secure_erase_capable(struct dm_target *ti,
					   struct dm_dev *dev, sector_t start,
					   sector_t len, void *data)
{
	struct request_queue *q = bdev_get_queue(dev->bdev);

	return q && !blk_queue_secure_erase(q);
}

static bool dm_table_supports_secure_erase(struct dm_table *t)
{
	struct dm_target *ti;
	unsigned int i;

	for (i = 0; i < dm_table_get_num_targets(t); i++) {
		ti = dm_table_get_target(t, i);

		if (!ti->num_secure_erase_bios)
			return false;

		if (!ti->type->iterate_devices ||
		    ti->type->iterate_devices(ti, device_not_secure_erase_capable, NULL))
			return false;
	}

	return true;
}

1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907
static int device_requires_stable_pages(struct dm_target *ti,
					struct dm_dev *dev, sector_t start,
					sector_t len, void *data)
{
	struct request_queue *q = bdev_get_queue(dev->bdev);

	return q && bdi_cap_stable_pages_required(q->backing_dev_info);
}

/*
 * If any underlying device requires stable pages, a table must require
 * them as well.  Only targets that support iterate_devices are considered:
 * don't want error, zero, etc to require stable pages.
 */
static bool dm_table_requires_stable_pages(struct dm_table *t)
{
	struct dm_target *ti;
	unsigned i;

	for (i = 0; i < dm_table_get_num_targets(t); i++) {
		ti = dm_table_get_target(t, i);

		if (ti->type->iterate_devices &&
		    ti->type->iterate_devices(ti, device_requires_stable_pages, NULL))
			return true;
	}

	return false;
}

1908 1909
void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q,
			       struct queue_limits *limits)
L
Linus Torvalds 已提交
1910
{
1911
	bool wc = false, fua = false;
1912

L
Linus Torvalds 已提交
1913
	/*
1914
	 * Copy table's limits to the DM device's request_queue
L
Linus Torvalds 已提交
1915
	 */
1916
	q->limits = *limits;
1917

1918
	if (!dm_table_supports_discards(t)) {
1919
		blk_queue_flag_clear(QUEUE_FLAG_DISCARD, q);
1920 1921 1922 1923 1924 1925 1926
		/* Must also clear discard limits... */
		q->limits.max_discard_sectors = 0;
		q->limits.max_hw_discard_sectors = 0;
		q->limits.discard_granularity = 0;
		q->limits.discard_alignment = 0;
		q->limits.discard_misaligned = 0;
	} else
1927
		blk_queue_flag_set(QUEUE_FLAG_DISCARD, q);
M
Mike Snitzer 已提交
1928

1929
	if (dm_table_supports_secure_erase(t))
1930
		blk_queue_flag_set(QUEUE_FLAG_SECERASE, q);
1931

J
Jens Axboe 已提交
1932
	if (dm_table_supports_flush(t, (1UL << QUEUE_FLAG_WC))) {
1933
		wc = true;
J
Jens Axboe 已提交
1934
		if (dm_table_supports_flush(t, (1UL << QUEUE_FLAG_FUA)))
1935
			fua = true;
1936
	}
1937
	blk_queue_write_cache(q, wc, fua);
1938

1939
	if (dm_table_supports_dax(t))
1940
		blk_queue_flag_set(QUEUE_FLAG_DAX, q);
1941 1942 1943
	else
		blk_queue_flag_clear(QUEUE_FLAG_DAX, q);

1944 1945 1946
	if (dm_table_supports_dax_write_cache(t))
		dax_write_cache(t->md->dax_dev, true);

1947 1948
	/* Ensure that all underlying devices are non-rotational. */
	if (dm_table_all_devices_attribute(t, device_is_nonrot))
1949
		blk_queue_flag_set(QUEUE_FLAG_NONROT, q);
1950
	else
1951
		blk_queue_flag_clear(QUEUE_FLAG_NONROT, q);
1952

M
Mike Snitzer 已提交
1953 1954
	if (!dm_table_supports_write_same(t))
		q->limits.max_write_same_sectors = 0;
1955 1956
	if (!dm_table_supports_write_zeroes(t))
		q->limits.max_write_zeroes_sectors = 0;
M
Mike Snitzer 已提交
1957

1958
	if (dm_table_all_devices_attribute(t, queue_supports_sg_merge))
1959
		blk_queue_flag_clear(QUEUE_FLAG_NO_SG_MERGE, q);
1960
	else
1961
		blk_queue_flag_set(QUEUE_FLAG_NO_SG_MERGE, q);
1962

1963
	dm_table_verify_integrity(t);
K
Kiyoshi Ueda 已提交
1964

1965 1966 1967 1968 1969 1970 1971 1972 1973
	/*
	 * Some devices don't use blk_integrity but still want stable pages
	 * because they do their own checksumming.
	 */
	if (dm_table_requires_stable_pages(t))
		q->backing_dev_info->capabilities |= BDI_CAP_STABLE_WRITES;
	else
		q->backing_dev_info->capabilities &= ~BDI_CAP_STABLE_WRITES;

1974 1975 1976 1977 1978 1979 1980
	/*
	 * Determine whether or not this queue's I/O timings contribute
	 * to the entropy pool, Only request-based targets use this.
	 * Clear QUEUE_FLAG_ADD_RANDOM if any underlying device does not
	 * have it set.
	 */
	if (blk_queue_add_random(q) && dm_table_all_devices_attribute(t, device_is_not_random))
1981
		blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, q);
L
Linus Torvalds 已提交
1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993
}

unsigned int dm_table_get_num_targets(struct dm_table *t)
{
	return t->num_targets;
}

struct list_head *dm_table_get_devices(struct dm_table *t)
{
	return &t->devices;
}

1994
fmode_t dm_table_get_mode(struct dm_table *t)
L
Linus Torvalds 已提交
1995 1996 1997
{
	return t->mode;
}
1998
EXPORT_SYMBOL(dm_table_get_mode);
L
Linus Torvalds 已提交
1999

2000 2001 2002 2003 2004 2005 2006
enum suspend_mode {
	PRESUSPEND,
	PRESUSPEND_UNDO,
	POSTSUSPEND,
};

static void suspend_targets(struct dm_table *t, enum suspend_mode mode)
L
Linus Torvalds 已提交
2007 2008 2009 2010
{
	int i = t->num_targets;
	struct dm_target *ti = t->targets;

2011 2012
	lockdep_assert_held(&t->md->suspend_lock);

L
Linus Torvalds 已提交
2013
	while (i--) {
2014 2015 2016 2017 2018 2019 2020 2021 2022 2023
		switch (mode) {
		case PRESUSPEND:
			if (ti->type->presuspend)
				ti->type->presuspend(ti);
			break;
		case PRESUSPEND_UNDO:
			if (ti->type->presuspend_undo)
				ti->type->presuspend_undo(ti);
			break;
		case POSTSUSPEND:
L
Linus Torvalds 已提交
2024 2025
			if (ti->type->postsuspend)
				ti->type->postsuspend(ti);
2026 2027
			break;
		}
L
Linus Torvalds 已提交
2028 2029 2030 2031 2032 2033
		ti++;
	}
}

void dm_table_presuspend_targets(struct dm_table *t)
{
2034 2035 2036
	if (!t)
		return;

2037 2038 2039 2040 2041 2042 2043 2044 2045
	suspend_targets(t, PRESUSPEND);
}

void dm_table_presuspend_undo_targets(struct dm_table *t)
{
	if (!t)
		return;

	suspend_targets(t, PRESUSPEND_UNDO);
L
Linus Torvalds 已提交
2046 2047 2048 2049
}

void dm_table_postsuspend_targets(struct dm_table *t)
{
2050 2051 2052
	if (!t)
		return;

2053
	suspend_targets(t, POSTSUSPEND);
L
Linus Torvalds 已提交
2054 2055
}

2056
int dm_table_resume_targets(struct dm_table *t)
L
Linus Torvalds 已提交
2057
{
2058 2059
	int i, r = 0;

2060 2061
	lockdep_assert_held(&t->md->suspend_lock);

2062 2063 2064 2065 2066 2067 2068
	for (i = 0; i < t->num_targets; i++) {
		struct dm_target *ti = t->targets + i;

		if (!ti->type->preresume)
			continue;

		r = ti->type->preresume(ti);
2069 2070 2071
		if (r) {
			DMERR("%s: %s: preresume failed, error = %d",
			      dm_device_name(t->md), ti->type->name, r);
2072
			return r;
2073
		}
2074
	}
L
Linus Torvalds 已提交
2075 2076 2077 2078 2079 2080 2081

	for (i = 0; i < t->num_targets; i++) {
		struct dm_target *ti = t->targets + i;

		if (ti->type->resume)
			ti->type->resume(ti);
	}
2082 2083

	return 0;
L
Linus Torvalds 已提交
2084 2085
}

2086 2087 2088 2089 2090 2091
void dm_table_add_target_callbacks(struct dm_table *t, struct dm_target_callbacks *cb)
{
	list_add(&cb->list, &t->target_callbacks);
}
EXPORT_SYMBOL_GPL(dm_table_add_target_callbacks);

L
Linus Torvalds 已提交
2092 2093
int dm_table_any_congested(struct dm_table *t, int bdi_bits)
{
M
Mikulas Patocka 已提交
2094
	struct dm_dev_internal *dd;
P
Paul Jimenez 已提交
2095
	struct list_head *devices = dm_table_get_devices(t);
2096
	struct dm_target_callbacks *cb;
L
Linus Torvalds 已提交
2097 2098
	int r = 0;

P
Paul Jimenez 已提交
2099
	list_for_each_entry(dd, devices, list) {
2100
		struct request_queue *q = bdev_get_queue(dd->dm_dev->bdev);
A
Alasdair G Kergon 已提交
2101 2102 2103
		char b[BDEVNAME_SIZE];

		if (likely(q))
2104
			r |= bdi_congested(q->backing_dev_info, bdi_bits);
A
Alasdair G Kergon 已提交
2105 2106 2107
		else
			DMWARN_LIMIT("%s: any_congested: nonexistent device %s",
				     dm_device_name(t->md),
2108
				     bdevname(dd->dm_dev->bdev, b));
L
Linus Torvalds 已提交
2109 2110
	}

2111 2112 2113 2114
	list_for_each_entry(cb, &t->target_callbacks, list)
		if (cb->congested_fn)
			r |= cb->congested_fn(cb, bdi_bits);

L
Linus Torvalds 已提交
2115 2116 2117
	return r;
}

M
Mike Anderson 已提交
2118 2119 2120 2121
struct mapped_device *dm_table_get_md(struct dm_table *t)
{
	return t->md;
}
2122
EXPORT_SYMBOL(dm_table_get_md);
M
Mike Anderson 已提交
2123

2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135
void dm_table_run_md_queue_async(struct dm_table *t)
{
	struct mapped_device *md;
	struct request_queue *queue;
	unsigned long flags;

	if (!dm_table_request_based(t))
		return;

	md = dm_table_get_md(t);
	queue = dm_get_md_queue(md);
	if (queue) {
2136 2137 2138 2139 2140 2141 2142
		if (queue->mq_ops)
			blk_mq_run_hw_queues(queue, true);
		else {
			spin_lock_irqsave(queue->queue_lock, flags);
			blk_run_queue_async(queue);
			spin_unlock_irqrestore(queue->queue_lock, flags);
		}
2143 2144 2145 2146
	}
}
EXPORT_SYMBOL(dm_table_run_md_queue_async);