dm-table.c 47.4 KB
Newer Older
L
Linus Torvalds 已提交
1 2
/*
 * Copyright (C) 2001 Sistina Software (UK) Limited.
3
 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
L
Linus Torvalds 已提交
4 5 6 7
 *
 * This file is released under the GPL.
 */

8
#include "dm-core.h"
L
Linus Torvalds 已提交
9 10 11 12 13 14

#include <linux/module.h>
#include <linux/vmalloc.h>
#include <linux/blkdev.h>
#include <linux/namei.h>
#include <linux/ctype.h>
15
#include <linux/string.h>
L
Linus Torvalds 已提交
16 17
#include <linux/slab.h>
#include <linux/interrupt.h>
A
Arjan van de Ven 已提交
18
#include <linux/mutex.h>
19
#include <linux/delay.h>
A
Arun Sharma 已提交
20
#include <linux/atomic.h>
21
#include <linux/blk-mq.h>
22
#include <linux/mount.h>
23
#include <linux/dax.h>
L
Linus Torvalds 已提交
24

25 26
#define DM_MSG_PREFIX "table"

L
Linus Torvalds 已提交
27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108
#define NODE_SIZE L1_CACHE_BYTES
#define KEYS_PER_NODE (NODE_SIZE / sizeof(sector_t))
#define CHILDREN_PER_NODE (KEYS_PER_NODE + 1)

/*
 * Similar to ceiling(log_size(n))
 */
static unsigned int int_log(unsigned int n, unsigned int base)
{
	int result = 0;

	while (n > 1) {
		n = dm_div_up(n, base);
		result++;
	}

	return result;
}

/*
 * Calculate the index of the child node of the n'th node k'th key.
 */
static inline unsigned int get_child(unsigned int n, unsigned int k)
{
	return (n * CHILDREN_PER_NODE) + k;
}

/*
 * Return the n'th node of level l from table t.
 */
static inline sector_t *get_node(struct dm_table *t,
				 unsigned int l, unsigned int n)
{
	return t->index[l] + (n * KEYS_PER_NODE);
}

/*
 * Return the highest key that you could lookup from the n'th
 * node on level l of the btree.
 */
static sector_t high(struct dm_table *t, unsigned int l, unsigned int n)
{
	for (; l < t->depth - 1; l++)
		n = get_child(n, CHILDREN_PER_NODE - 1);

	if (n >= t->counts[l])
		return (sector_t) - 1;

	return get_node(t, l, n)[KEYS_PER_NODE - 1];
}

/*
 * Fills in a level of the btree based on the highs of the level
 * below it.
 */
static int setup_btree_index(unsigned int l, struct dm_table *t)
{
	unsigned int n, k;
	sector_t *node;

	for (n = 0U; n < t->counts[l]; n++) {
		node = get_node(t, l, n);

		for (k = 0U; k < KEYS_PER_NODE; k++)
			node[k] = high(t, l + 1, get_child(n, k));
	}

	return 0;
}

void *dm_vcalloc(unsigned long nmemb, unsigned long elem_size)
{
	unsigned long size;
	void *addr;

	/*
	 * Check that we're not going to overflow.
	 */
	if (nmemb > (ULONG_MAX / elem_size))
		return NULL;

	size = nmemb * elem_size;
J
Joe Perches 已提交
109
	addr = vzalloc(size);
L
Linus Torvalds 已提交
110 111 112

	return addr;
}
113
EXPORT_SYMBOL(dm_vcalloc);
L
Linus Torvalds 已提交
114 115 116 117 118 119 120 121 122 123 124 125 126

/*
 * highs, and targets are managed as dynamic arrays during a
 * table load.
 */
static int alloc_targets(struct dm_table *t, unsigned int num)
{
	sector_t *n_highs;
	struct dm_target *n_targets;

	/*
	 * Allocate both the target array and offset array at once.
	 */
127
	n_highs = (sector_t *) dm_vcalloc(num, sizeof(struct dm_target) +
L
Linus Torvalds 已提交
128 129 130 131 132 133
					  sizeof(sector_t));
	if (!n_highs)
		return -ENOMEM;

	n_targets = (struct dm_target *) (n_highs + num);

134
	memset(n_highs, -1, sizeof(*n_highs) * num);
L
Linus Torvalds 已提交
135 136 137 138 139 140 141 142 143
	vfree(t->highs);

	t->num_allocated = num;
	t->highs = n_highs;
	t->targets = n_targets;

	return 0;
}

144
int dm_table_create(struct dm_table **result, fmode_t mode,
M
Mike Anderson 已提交
145
		    unsigned num_targets, struct mapped_device *md)
L
Linus Torvalds 已提交
146
{
D
Dmitry Monakhov 已提交
147
	struct dm_table *t = kzalloc(sizeof(*t), GFP_KERNEL);
L
Linus Torvalds 已提交
148 149 150 151 152 153 154 155 156 157 158

	if (!t)
		return -ENOMEM;

	INIT_LIST_HEAD(&t->devices);

	if (!num_targets)
		num_targets = KEYS_PER_NODE;

	num_targets = dm_round_up(num_targets, KEYS_PER_NODE);

159 160 161 162 163
	if (!num_targets) {
		kfree(t);
		return -ENOMEM;
	}

L
Linus Torvalds 已提交
164 165 166 167 168
	if (alloc_targets(t, num_targets)) {
		kfree(t);
		return -ENOMEM;
	}

169
	t->type = DM_TYPE_NONE;
L
Linus Torvalds 已提交
170
	t->mode = mode;
M
Mike Anderson 已提交
171
	t->md = md;
L
Linus Torvalds 已提交
172 173 174 175
	*result = t;
	return 0;
}

176
static void free_devices(struct list_head *devices, struct mapped_device *md)
L
Linus Torvalds 已提交
177 178 179
{
	struct list_head *tmp, *next;

P
Paul Jimenez 已提交
180
	list_for_each_safe(tmp, next, devices) {
M
Mikulas Patocka 已提交
181 182
		struct dm_dev_internal *dd =
		    list_entry(tmp, struct dm_dev_internal, list);
183 184 185
		DMWARN("%s: dm_table_destroy: dm_put_device call missing for %s",
		       dm_device_name(md), dd->dm_dev->name);
		dm_put_table_device(md, dd->dm_dev);
L
Linus Torvalds 已提交
186 187 188 189
		kfree(dd);
	}
}

190
void dm_table_destroy(struct dm_table *t)
L
Linus Torvalds 已提交
191 192 193
{
	unsigned int i;

194 195 196
	if (!t)
		return;

197
	/* free the indexes */
L
Linus Torvalds 已提交
198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213
	if (t->depth >= 2)
		vfree(t->index[t->depth - 2]);

	/* free the targets */
	for (i = 0; i < t->num_targets; i++) {
		struct dm_target *tgt = t->targets + i;

		if (tgt->type->dtr)
			tgt->type->dtr(tgt);

		dm_put_target_type(tgt->type);
	}

	vfree(t->highs);

	/* free the device list */
214
	free_devices(&t->devices, t->md);
L
Linus Torvalds 已提交
215

K
Kiyoshi Ueda 已提交
216 217
	dm_free_md_mempools(t->mempools);

L
Linus Torvalds 已提交
218 219 220 221 222 223
	kfree(t);
}

/*
 * See if we've already got a device in the list.
 */
M
Mikulas Patocka 已提交
224
static struct dm_dev_internal *find_device(struct list_head *l, dev_t dev)
L
Linus Torvalds 已提交
225
{
M
Mikulas Patocka 已提交
226
	struct dm_dev_internal *dd;
L
Linus Torvalds 已提交
227 228

	list_for_each_entry (dd, l, list)
229
		if (dd->dm_dev->bdev->bd_dev == dev)
L
Linus Torvalds 已提交
230 231 232 233 234 235
			return dd;

	return NULL;
}

/*
236
 * If possible, this checks an area of a destination device is invalid.
L
Linus Torvalds 已提交
237
 */
238 239
static int device_area_is_invalid(struct dm_target *ti, struct dm_dev *dev,
				  sector_t start, sector_t len, void *data)
L
Linus Torvalds 已提交
240
{
241 242 243 244
	struct queue_limits *limits = data;
	struct block_device *bdev = dev->bdev;
	sector_t dev_size =
		i_size_read(bdev->bd_inode) >> SECTOR_SHIFT;
245
	unsigned short logical_block_size_sectors =
246
		limits->logical_block_size >> SECTOR_SHIFT;
247
	char b[BDEVNAME_SIZE];
M
Mike Anderson 已提交
248 249

	if (!dev_size)
250
		return 0;
M
Mike Anderson 已提交
251

252
	if ((start >= dev_size) || (start + len > dev_size)) {
253 254 255 256 257 258
		DMWARN("%s: %s too small for target: "
		       "start=%llu, len=%llu, dev_size=%llu",
		       dm_device_name(ti->table->md), bdevname(bdev, b),
		       (unsigned long long)start,
		       (unsigned long long)len,
		       (unsigned long long)dev_size);
259
		return 1;
260 261
	}

262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294
	/*
	 * If the target is mapped to zoned block device(s), check
	 * that the zones are not partially mapped.
	 */
	if (bdev_zoned_model(bdev) != BLK_ZONED_NONE) {
		unsigned int zone_sectors = bdev_zone_sectors(bdev);

		if (start & (zone_sectors - 1)) {
			DMWARN("%s: start=%llu not aligned to h/w zone size %u of %s",
			       dm_device_name(ti->table->md),
			       (unsigned long long)start,
			       zone_sectors, bdevname(bdev, b));
			return 1;
		}

		/*
		 * Note: The last zone of a zoned block device may be smaller
		 * than other zones. So for a target mapping the end of a
		 * zoned block device with such a zone, len would not be zone
		 * aligned. We do not allow such last smaller zone to be part
		 * of the mapping here to ensure that mappings with multiple
		 * devices do not end up with a smaller zone in the middle of
		 * the sector range.
		 */
		if (len & (zone_sectors - 1)) {
			DMWARN("%s: len=%llu not aligned to h/w zone size %u of %s",
			       dm_device_name(ti->table->md),
			       (unsigned long long)len,
			       zone_sectors, bdevname(bdev, b));
			return 1;
		}
	}

295
	if (logical_block_size_sectors <= 1)
296
		return 0;
297 298 299

	if (start & (logical_block_size_sectors - 1)) {
		DMWARN("%s: start=%llu not aligned to h/w "
300
		       "logical block size %u of %s",
301 302
		       dm_device_name(ti->table->md),
		       (unsigned long long)start,
303
		       limits->logical_block_size, bdevname(bdev, b));
304
		return 1;
305 306
	}

307
	if (len & (logical_block_size_sectors - 1)) {
308
		DMWARN("%s: len=%llu not aligned to h/w "
309
		       "logical block size %u of %s",
310
		       dm_device_name(ti->table->md),
311
		       (unsigned long long)len,
312
		       limits->logical_block_size, bdevname(bdev, b));
313
		return 1;
314 315
	}

316
	return 0;
L
Linus Torvalds 已提交
317 318 319
}

/*
320
 * This upgrades the mode on an already open dm_dev, being
L
Linus Torvalds 已提交
321
 * careful to leave things as they were if we fail to reopen the
322
 * device and not to touch the existing bdev field in case
323
 * it is accessed concurrently.
L
Linus Torvalds 已提交
324
 */
325
static int upgrade_mode(struct dm_dev_internal *dd, fmode_t new_mode,
M
Mikulas Patocka 已提交
326
			struct mapped_device *md)
L
Linus Torvalds 已提交
327 328
{
	int r;
329
	struct dm_dev *old_dev, *new_dev;
L
Linus Torvalds 已提交
330

331
	old_dev = dd->dm_dev;
332

333 334
	r = dm_get_table_device(md, dd->dm_dev->bdev->bd_dev,
				dd->dm_dev->mode | new_mode, &new_dev);
335 336
	if (r)
		return r;
L
Linus Torvalds 已提交
337

338 339
	dd->dm_dev = new_dev;
	dm_put_table_device(md, old_dev);
L
Linus Torvalds 已提交
340

341
	return 0;
L
Linus Torvalds 已提交
342 343
}

344 345 346 347 348
/*
 * Convert the path to a device
 */
dev_t dm_get_dev_t(const char *path)
{
349
	dev_t dev;
350

C
Christoph Hellwig 已提交
351
	if (lookup_bdev(path, &dev))
352 353 354 355 356
		dev = name_to_dev_t(path);
	return dev;
}
EXPORT_SYMBOL_GPL(dm_get_dev_t);

L
Linus Torvalds 已提交
357 358 359 360
/*
 * Add a device to the list, or just increment the usage count if
 * it's already present.
 */
361 362
int dm_get_device(struct dm_target *ti, const char *path, fmode_t mode,
		  struct dm_dev **result)
L
Linus Torvalds 已提交
363 364
{
	int r;
365
	dev_t dev;
366 367
	unsigned int major, minor;
	char dummy;
M
Mikulas Patocka 已提交
368
	struct dm_dev_internal *dd;
369
	struct dm_table *t = ti->table;
L
Linus Torvalds 已提交
370

371
	BUG_ON(!t);
L
Linus Torvalds 已提交
372

373 374 375 376 377 378 379 380 381 382
	if (sscanf(path, "%u:%u%c", &major, &minor, &dummy) == 2) {
		/* Extract the major/minor numbers */
		dev = MKDEV(major, minor);
		if (MAJOR(dev) != major || MINOR(dev) != minor)
			return -EOVERFLOW;
	} else {
		dev = dm_get_dev_t(path);
		if (!dev)
			return -ENODEV;
	}
L
Linus Torvalds 已提交
383 384 385 386 387 388 389

	dd = find_device(&t->devices, dev);
	if (!dd) {
		dd = kmalloc(sizeof(*dd), GFP_KERNEL);
		if (!dd)
			return -ENOMEM;

390
		if ((r = dm_get_table_device(t->md, dev, mode, &dd->dm_dev))) {
L
Linus Torvalds 已提交
391 392 393 394
			kfree(dd);
			return r;
		}

395
		refcount_set(&dd->count, 1);
L
Linus Torvalds 已提交
396
		list_add(&dd->list, &t->devices);
397
		goto out;
L
Linus Torvalds 已提交
398

399
	} else if (dd->dm_dev->mode != (mode | dd->dm_dev->mode)) {
400
		r = upgrade_mode(dd, mode, t->md);
L
Linus Torvalds 已提交
401 402 403
		if (r)
			return r;
	}
404 405
	refcount_inc(&dd->count);
out:
406
	*result = dd->dm_dev;
L
Linus Torvalds 已提交
407 408
	return 0;
}
409
EXPORT_SYMBOL(dm_get_device);
L
Linus Torvalds 已提交
410

411 412
static int dm_set_device_limits(struct dm_target *ti, struct dm_dev *dev,
				sector_t start, sector_t len, void *data)
L
Linus Torvalds 已提交
413
{
414 415
	struct queue_limits *limits = data;
	struct block_device *bdev = dev->bdev;
416
	struct request_queue *q = bdev_get_queue(bdev);
A
Alasdair G Kergon 已提交
417 418 419 420 421
	char b[BDEVNAME_SIZE];

	if (unlikely(!q)) {
		DMWARN("%s: Cannot set limits for nonexistent device %s",
		       dm_device_name(ti->table->md), bdevname(bdev, b));
422
		return 0;
A
Alasdair G Kergon 已提交
423
	}
424

425 426
	if (blk_stack_limits(limits, &q->limits,
			get_start_sect(bdev) + start) < 0)
427
		DMWARN("%s: adding target device %s caused an alignment inconsistency: "
428 429 430 431 432 433
		       "physical_block_size=%u, logical_block_size=%u, "
		       "alignment_offset=%u, start=%llu",
		       dm_device_name(ti->table->md), bdevname(bdev, b),
		       q->limits.physical_block_size,
		       q->limits.logical_block_size,
		       q->limits.alignment_offset,
434
		       (unsigned long long) start << SECTOR_SHIFT);
435
	return 0;
436
}
437

L
Linus Torvalds 已提交
438
/*
439
 * Decrement a device's use count and remove it if necessary.
L
Linus Torvalds 已提交
440
 */
M
Mikulas Patocka 已提交
441
void dm_put_device(struct dm_target *ti, struct dm_dev *d)
L
Linus Torvalds 已提交
442
{
443 444 445
	int found = 0;
	struct list_head *devices = &ti->table->devices;
	struct dm_dev_internal *dd;
M
Mikulas Patocka 已提交
446

447 448 449 450 451 452 453 454 455 456 457
	list_for_each_entry(dd, devices, list) {
		if (dd->dm_dev == d) {
			found = 1;
			break;
		}
	}
	if (!found) {
		DMWARN("%s: device %s not in table devices list",
		       dm_device_name(ti->table->md), d->name);
		return;
	}
458
	if (refcount_dec_and_test(&dd->count)) {
459
		dm_put_table_device(ti->table->md, d);
L
Linus Torvalds 已提交
460 461 462 463
		list_del(&dd->list);
		kfree(dd);
	}
}
464
EXPORT_SYMBOL(dm_put_device);
L
Linus Torvalds 已提交
465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481

/*
 * Checks to see if the target joins onto the end of the table.
 */
static int adjoin(struct dm_table *table, struct dm_target *ti)
{
	struct dm_target *prev;

	if (!table->num_targets)
		return !ti->begin;

	prev = &table->targets[table->num_targets - 1];
	return (ti->begin == (prev->begin + prev->len));
}

/*
 * Used to dynamically allocate the arg array.
482 483 484 485 486 487 488
 *
 * We do first allocation with GFP_NOIO because dm-mpath and dm-thin must
 * process messages even if some device is suspended. These messages have a
 * small fixed number of arguments.
 *
 * On the other hand, dm-switch needs to process bulk data using messages and
 * excessive use of GFP_NOIO could cause trouble.
L
Linus Torvalds 已提交
489
 */
490
static char **realloc_argv(unsigned *size, char **old_argv)
L
Linus Torvalds 已提交
491 492 493
{
	char **argv;
	unsigned new_size;
494
	gfp_t gfp;
L
Linus Torvalds 已提交
495

496 497
	if (*size) {
		new_size = *size * 2;
498 499 500 501 502
		gfp = GFP_KERNEL;
	} else {
		new_size = 8;
		gfp = GFP_NOIO;
	}
503
	argv = kmalloc_array(new_size, sizeof(*argv), gfp);
504
	if (argv && old_argv) {
505 506
		memcpy(argv, old_argv, *size * sizeof(*argv));
		*size = new_size;
L
Linus Torvalds 已提交
507 508 509 510 511 512 513 514 515 516 517 518 519 520 521
	}

	kfree(old_argv);
	return argv;
}

/*
 * Destructively splits up the argument list to pass to ctr.
 */
int dm_split_args(int *argc, char ***argvp, char *input)
{
	char *start, *end = input, *out, **argv = NULL;
	unsigned array_size = 0;

	*argc = 0;
522 523 524 525 526 527

	if (!input) {
		*argvp = NULL;
		return 0;
	}

L
Linus Torvalds 已提交
528 529 530 531 532 533
	argv = realloc_argv(&array_size, argv);
	if (!argv)
		return -ENOMEM;

	while (1) {
		/* Skip whitespace */
534
		start = skip_spaces(end);
L
Linus Torvalds 已提交
535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575

		if (!*start)
			break;	/* success, we hit the end */

		/* 'out' is used to remove any back-quotes */
		end = out = start;
		while (*end) {
			/* Everything apart from '\0' can be quoted */
			if (*end == '\\' && *(end + 1)) {
				*out++ = *(end + 1);
				end += 2;
				continue;
			}

			if (isspace(*end))
				break;	/* end of token */

			*out++ = *end++;
		}

		/* have we already filled the array ? */
		if ((*argc + 1) > array_size) {
			argv = realloc_argv(&array_size, argv);
			if (!argv)
				return -ENOMEM;
		}

		/* we know this is whitespace */
		if (*end)
			end++;

		/* terminate the string and put it in the array */
		*out = '\0';
		argv[*argc] = start;
		(*argc)++;
	}

	*argvp = argv;
	return 0;
}

576 577 578 579 580 581 582
/*
 * Impose necessary and sufficient conditions on a devices's table such
 * that any incoming bio which respects its logical_block_size can be
 * processed successfully.  If it falls across the boundary between
 * two or more targets, the size of each piece it gets split into must
 * be compatible with the logical_block_size of the target processing it.
 */
583 584
static int validate_hardware_logical_block_alignment(struct dm_table *table,
						 struct queue_limits *limits)
585 586 587 588 589 590
{
	/*
	 * This function uses arithmetic modulo the logical_block_size
	 * (in units of 512-byte sectors).
	 */
	unsigned short device_logical_block_size_sects =
591
		limits->logical_block_size >> SECTOR_SHIFT;
592 593 594 595 596 597 598 599 600 601 602 603

	/*
	 * Offset of the start of the next table entry, mod logical_block_size.
	 */
	unsigned short next_target_start = 0;

	/*
	 * Given an aligned bio that extends beyond the end of a
	 * target, how many sectors must the next target handle?
	 */
	unsigned short remaining = 0;

604
	struct dm_target *ti;
605
	struct queue_limits ti_limits;
606
	unsigned i;
607 608 609 610

	/*
	 * Check each entry in the table in turn.
	 */
611 612
	for (i = 0; i < dm_table_get_num_targets(table); i++) {
		ti = dm_table_get_target(table, i);
613

614
		blk_set_stacking_limits(&ti_limits);
615 616 617 618 619 620

		/* combine all target devices' limits */
		if (ti->type->iterate_devices)
			ti->type->iterate_devices(ti, dm_set_device_limits,
						  &ti_limits);

621 622 623 624 625
		/*
		 * If the remaining sectors fall entirely within this
		 * table entry are they compatible with its logical_block_size?
		 */
		if (remaining < ti->len &&
626
		    remaining & ((ti_limits.logical_block_size >>
627 628 629 630 631 632 633 634 635 636 637 638
				  SECTOR_SHIFT) - 1))
			break;	/* Error */

		next_target_start =
		    (unsigned short) ((next_target_start + ti->len) &
				      (device_logical_block_size_sects - 1));
		remaining = next_target_start ?
		    device_logical_block_size_sects - next_target_start : 0;
	}

	if (remaining) {
		DMWARN("%s: table line %u (start sect %llu len %llu) "
639
		       "not aligned to h/w logical block size %u",
640 641 642
		       dm_device_name(table->md), i,
		       (unsigned long long) ti->begin,
		       (unsigned long long) ti->len,
643
		       limits->logical_block_size);
644 645 646 647 648 649
		return -EINVAL;
	}

	return 0;
}

L
Linus Torvalds 已提交
650 651 652 653 654 655 656
int dm_table_add_target(struct dm_table *t, const char *type,
			sector_t start, sector_t len, char *params)
{
	int r = -EINVAL, argc;
	char **argv;
	struct dm_target *tgt;

657 658 659 660 661 662
	if (t->singleton) {
		DMERR("%s: target type %s must appear alone in table",
		      dm_device_name(t->md), t->targets->type->name);
		return -EINVAL;
	}

663
	BUG_ON(t->num_targets >= t->num_allocated);
L
Linus Torvalds 已提交
664 665 666 667 668

	tgt = t->targets + t->num_targets;
	memset(tgt, 0, sizeof(*tgt));

	if (!len) {
669
		DMERR("%s: zero-length target", dm_device_name(t->md));
L
Linus Torvalds 已提交
670 671 672 673 674
		return -EINVAL;
	}

	tgt->type = dm_get_target_type(type);
	if (!tgt->type) {
675
		DMERR("%s: %s: unknown target type", dm_device_name(t->md), type);
L
Linus Torvalds 已提交
676 677 678
		return -EINVAL;
	}

679 680
	if (dm_target_needs_singleton(tgt->type)) {
		if (t->num_targets) {
681 682
			tgt->error = "singleton target type must appear alone in table";
			goto bad;
683
		}
684
		t->singleton = true;
685 686
	}

687
	if (dm_target_always_writeable(tgt->type) && !(t->mode & FMODE_WRITE)) {
688 689
		tgt->error = "target type may not be included in a read-only table";
		goto bad;
690 691
	}

692 693
	if (t->immutable_target_type) {
		if (t->immutable_target_type != tgt->type) {
694 695
			tgt->error = "immutable target type cannot be mixed with other target types";
			goto bad;
696 697 698
		}
	} else if (dm_target_is_immutable(tgt->type)) {
		if (t->num_targets) {
699 700
			tgt->error = "immutable target type cannot be mixed with other target types";
			goto bad;
701 702 703 704
		}
		t->immutable_target_type = tgt->type;
	}

705 706 707
	if (dm_target_has_integrity(tgt->type))
		t->integrity_added = 1;

L
Linus Torvalds 已提交
708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733
	tgt->table = t;
	tgt->begin = start;
	tgt->len = len;
	tgt->error = "Unknown error";

	/*
	 * Does this target adjoin the previous one ?
	 */
	if (!adjoin(t, tgt)) {
		tgt->error = "Gap in table";
		goto bad;
	}

	r = dm_split_args(&argc, &argv, params);
	if (r) {
		tgt->error = "couldn't split parameters (insufficient memory)";
		goto bad;
	}

	r = tgt->type->ctr(tgt, argc, argv);
	kfree(argv);
	if (r)
		goto bad;

	t->highs[t->num_targets++] = tgt->begin + tgt->len - 1;

734 735
	if (!tgt->num_discard_bios && tgt->discards_supported)
		DMWARN("%s: %s: ignoring discards_supported because num_discard_bios is zero.",
M
Mike Snitzer 已提交
736
		       dm_device_name(t->md), type);
M
Mike Snitzer 已提交
737

L
Linus Torvalds 已提交
738 739 740
	return 0;

 bad:
741
	DMERR("%s: %s: %s", dm_device_name(t->md), type, tgt->error);
L
Linus Torvalds 已提交
742 743 744 745
	dm_put_target_type(tgt->type);
	return r;
}

746 747 748
/*
 * Target argument parsing helpers.
 */
E
Eric Biggers 已提交
749 750
static int validate_next_arg(const struct dm_arg *arg,
			     struct dm_arg_set *arg_set,
751 752 753
			     unsigned *value, char **error, unsigned grouped)
{
	const char *arg_str = dm_shift_arg(arg_set);
754
	char dummy;
755 756

	if (!arg_str ||
757
	    (sscanf(arg_str, "%u%c", value, &dummy) != 1) ||
758 759 760 761 762 763 764 765 766 767
	    (*value < arg->min) ||
	    (*value > arg->max) ||
	    (grouped && arg_set->argc < *value)) {
		*error = arg->error;
		return -EINVAL;
	}

	return 0;
}

E
Eric Biggers 已提交
768
int dm_read_arg(const struct dm_arg *arg, struct dm_arg_set *arg_set,
769 770 771 772 773 774
		unsigned *value, char **error)
{
	return validate_next_arg(arg, arg_set, value, error, 0);
}
EXPORT_SYMBOL(dm_read_arg);

E
Eric Biggers 已提交
775
int dm_read_arg_group(const struct dm_arg *arg, struct dm_arg_set *arg_set,
776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804
		      unsigned *value, char **error)
{
	return validate_next_arg(arg, arg_set, value, error, 1);
}
EXPORT_SYMBOL(dm_read_arg_group);

const char *dm_shift_arg(struct dm_arg_set *as)
{
	char *r;

	if (as->argc) {
		as->argc--;
		r = *as->argv;
		as->argv++;
		return r;
	}

	return NULL;
}
EXPORT_SYMBOL(dm_shift_arg);

void dm_consume_args(struct dm_arg_set *as, unsigned num_args)
{
	BUG_ON(as->argc < num_args);
	as->argc -= num_args;
	as->argv += num_args;
}
EXPORT_SYMBOL(dm_consume_args);

805
static bool __table_type_bio_based(enum dm_queue_mode table_type)
806 807
{
	return (table_type == DM_TYPE_BIO_BASED ||
808
		table_type == DM_TYPE_DAX_BIO_BASED);
809 810
}

811
static bool __table_type_request_based(enum dm_queue_mode table_type)
812
{
813
	return table_type == DM_TYPE_REQUEST_BASED;
814 815
}

816
void dm_table_set_type(struct dm_table *t, enum dm_queue_mode type)
817 818 819 820 821
{
	t->type = type;
}
EXPORT_SYMBOL_GPL(dm_table_set_type);

822
/* validate the dax capability of the target device span */
823
int device_not_dax_capable(struct dm_target *ti, struct dm_dev *dev,
824
			sector_t start, sector_t len, void *data)
825
{
826 827
	int blocksize = *(int *) data, id;
	bool rc;
828

829
	id = dax_read_lock();
830
	rc = !dax_supported(dev->dax_dev, dev->bdev, blocksize, start, len);
831 832 833
	dax_read_unlock(id);

	return rc;
834 835
}

P
Pankaj Gupta 已提交
836
/* Check devices support synchronous DAX */
837 838
static int device_not_dax_synchronous_capable(struct dm_target *ti, struct dm_dev *dev,
					      sector_t start, sector_t len, void *data)
P
Pankaj Gupta 已提交
839
{
840
	return !dev->dax_dev || !dax_synchronous(dev->dax_dev);
P
Pankaj Gupta 已提交
841 842 843
}

bool dm_table_supports_dax(struct dm_table *t,
844
			   iterate_devices_callout_fn iterate_fn, int *blocksize)
845 846
{
	struct dm_target *ti;
847
	unsigned i;
848 849

	/* Ensure that all targets support DAX. */
850 851
	for (i = 0; i < dm_table_get_num_targets(t); i++) {
		ti = dm_table_get_target(t, i);
852 853 854 855 856

		if (!ti->type->direct_access)
			return false;

		if (!ti->type->iterate_devices ||
857
		    ti->type->iterate_devices(ti, iterate_fn, blocksize))
858 859 860 861 862 863
			return false;
	}

	return true;
}

864 865
static int device_is_rq_stackable(struct dm_target *ti, struct dm_dev *dev,
				  sector_t start, sector_t len, void *data)
866
{
867 868
	struct block_device *bdev = dev->bdev;
	struct request_queue *q = bdev_get_queue(bdev);
869

870
	/* request-based cannot stack on partitions! */
871
	if (bdev_is_partition(bdev))
872
		return false;
873

J
Jens Axboe 已提交
874
	return queue_is_mq(q);
875 876
}

877
static int dm_table_determine_type(struct dm_table *t)
K
Kiyoshi Ueda 已提交
878 879
{
	unsigned i;
880
	unsigned bio_based = 0, request_based = 0, hybrid = 0;
K
Kiyoshi Ueda 已提交
881
	struct dm_target *tgt;
882
	struct list_head *devices = dm_table_get_devices(t);
883
	enum dm_queue_mode live_md_type = dm_get_md_type(t->md);
P
Pankaj Gupta 已提交
884
	int page_size = PAGE_SIZE;
K
Kiyoshi Ueda 已提交
885

886 887
	if (t->type != DM_TYPE_NONE) {
		/* target already set the table's type */
888 889 890
		if (t->type == DM_TYPE_BIO_BASED) {
			/* possibly upgrade to a variant of bio-based */
			goto verify_bio_based;
891
		}
892
		BUG_ON(t->type == DM_TYPE_DAX_BIO_BASED);
893 894 895
		goto verify_rq_based;
	}

K
Kiyoshi Ueda 已提交
896 897
	for (i = 0; i < t->num_targets; i++) {
		tgt = t->targets + i;
898 899 900
		if (dm_target_hybrid(tgt))
			hybrid = 1;
		else if (dm_target_request_based(tgt))
K
Kiyoshi Ueda 已提交
901 902 903 904 905
			request_based = 1;
		else
			bio_based = 1;

		if (bio_based && request_based) {
906 907
			DMERR("Inconsistent table: different target types"
			      " can't be mixed up");
K
Kiyoshi Ueda 已提交
908 909 910 911
			return -EINVAL;
		}
	}

912 913 914 915 916 917
	if (hybrid && !bio_based && !request_based) {
		/*
		 * The targets can work either way.
		 * Determine the type from the live device.
		 * Default to bio-based if device is new.
		 */
918
		if (__table_type_request_based(live_md_type))
919 920 921 922 923
			request_based = 1;
		else
			bio_based = 1;
	}

K
Kiyoshi Ueda 已提交
924
	if (bio_based) {
925
verify_bio_based:
K
Kiyoshi Ueda 已提交
926 927
		/* We must use this table as bio-based */
		t->type = DM_TYPE_BIO_BASED;
928
		if (dm_table_supports_dax(t, device_not_dax_capable, &page_size) ||
929
		    (list_empty(devices) && live_md_type == DM_TYPE_DAX_BIO_BASED)) {
930
			t->type = DM_TYPE_DAX_BIO_BASED;
931
		}
K
Kiyoshi Ueda 已提交
932 933 934 935 936
		return 0;
	}

	BUG_ON(!request_based); /* No targets in this table */

937 938 939
	t->type = DM_TYPE_REQUEST_BASED;

verify_rq_based:
940 941 942 943 944 945 946
	/*
	 * Request-based dm supports only tables that have a single target now.
	 * To support multiple targets, request splitting support is needed,
	 * and that needs lots of changes in the block-layer.
	 * (e.g. request completion process for partial completion.)
	 */
	if (t->num_targets > 1) {
947
		DMERR("request-based DM doesn't support multiple targets");
948 949 950
		return -EINVAL;
	}

951 952 953 954
	if (list_empty(devices)) {
		int srcu_idx;
		struct dm_table *live_table = dm_get_live_table(t->md, &srcu_idx);

955 956
		/* inherit live table's type */
		if (live_table)
957 958 959 960 961
			t->type = live_table->type;
		dm_put_live_table(t->md, srcu_idx);
		return 0;
	}

962 963 964 965 966 967 968 969 970
	tgt = dm_table_get_immutable_target(t);
	if (!tgt) {
		DMERR("table load rejected: immutable target is required");
		return -EINVAL;
	} else if (tgt->max_io_len) {
		DMERR("table load rejected: immutable target that splits IO is not supported");
		return -EINVAL;
	}

K
Kiyoshi Ueda 已提交
971
	/* Non-request-stackable devices can't be used for request-based dm */
972
	if (!tgt->type->iterate_devices ||
973
	    !tgt->type->iterate_devices(tgt, device_is_rq_stackable, NULL)) {
974 975
		DMERR("table load rejected: including non-request-stackable devices");
		return -EINVAL;
976
	}
977

K
Kiyoshi Ueda 已提交
978 979 980
	return 0;
}

981
enum dm_queue_mode dm_table_get_type(struct dm_table *t)
K
Kiyoshi Ueda 已提交
982 983 984 985
{
	return t->type;
}

986 987 988 989 990
struct target_type *dm_table_get_immutable_target_type(struct dm_table *t)
{
	return t->immutable_target_type;
}

M
Mike Snitzer 已提交
991 992 993 994 995 996 997 998 999 1000
struct dm_target *dm_table_get_immutable_target(struct dm_table *t)
{
	/* Immutable target is implicitly a singleton */
	if (t->num_targets > 1 ||
	    !dm_target_is_immutable(t->targets[0].type))
		return NULL;

	return t->targets;
}

1001 1002
struct dm_target *dm_table_get_wildcard_target(struct dm_table *t)
{
1003 1004
	struct dm_target *ti;
	unsigned i;
1005

1006 1007
	for (i = 0; i < dm_table_get_num_targets(t); i++) {
		ti = dm_table_get_target(t, i);
1008 1009 1010 1011 1012 1013 1014
		if (dm_target_is_wildcard(ti->type))
			return ti;
	}

	return NULL;
}

1015 1016 1017 1018 1019
bool dm_table_bio_based(struct dm_table *t)
{
	return __table_type_bio_based(dm_table_get_type(t));
}

K
Kiyoshi Ueda 已提交
1020 1021
bool dm_table_request_based(struct dm_table *t)
{
1022
	return __table_type_request_based(dm_table_get_type(t));
1023 1024
}

1025
static int dm_table_alloc_md_mempools(struct dm_table *t, struct mapped_device *md)
K
Kiyoshi Ueda 已提交
1026
{
1027
	enum dm_queue_mode type = dm_table_get_type(t);
1028
	unsigned per_io_data_size = 0;
1029 1030
	unsigned min_pool_size = 0;
	struct dm_target *ti;
M
Mikulas Patocka 已提交
1031
	unsigned i;
K
Kiyoshi Ueda 已提交
1032

1033
	if (unlikely(type == DM_TYPE_NONE)) {
K
Kiyoshi Ueda 已提交
1034 1035 1036 1037
		DMWARN("no table type is set, can't allocate mempools");
		return -EINVAL;
	}

1038
	if (__table_type_bio_based(type))
1039
		for (i = 0; i < t->num_targets; i++) {
1040 1041 1042
			ti = t->targets + i;
			per_io_data_size = max(per_io_data_size, ti->per_io_data_size);
			min_pool_size = max(min_pool_size, ti->num_flush_bios);
1043 1044
		}

1045 1046
	t->mempools = dm_alloc_md_mempools(md, type, t->integrity_supported,
					   per_io_data_size, min_pool_size);
1047 1048
	if (!t->mempools)
		return -ENOMEM;
K
Kiyoshi Ueda 已提交
1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063

	return 0;
}

void dm_table_free_md_mempools(struct dm_table *t)
{
	dm_free_md_mempools(t->mempools);
	t->mempools = NULL;
}

struct dm_md_mempools *dm_table_get_md_mempools(struct dm_table *t)
{
	return t->mempools;
}

L
Linus Torvalds 已提交
1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080
static int setup_indexes(struct dm_table *t)
{
	int i;
	unsigned int total = 0;
	sector_t *indexes;

	/* allocate the space for *all* the indexes */
	for (i = t->depth - 2; i >= 0; i--) {
		t->counts[i] = dm_div_up(t->counts[i + 1], CHILDREN_PER_NODE);
		total += t->counts[i];
	}

	indexes = (sector_t *) dm_vcalloc(total, (unsigned long) NODE_SIZE);
	if (!indexes)
		return -ENOMEM;

	/* set up internal nodes, bottom-up */
J
Jun'ichi Nomura 已提交
1081
	for (i = t->depth - 2; i >= 0; i--) {
L
Linus Torvalds 已提交
1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092
		t->index[i] = indexes;
		indexes += (KEYS_PER_NODE * t->counts[i]);
		setup_btree_index(i, t);
	}

	return 0;
}

/*
 * Builds the btree to index the map.
 */
1093
static int dm_table_build_index(struct dm_table *t)
L
Linus Torvalds 已提交
1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111
{
	int r = 0;
	unsigned int leaf_nodes;

	/* how many indexes will the btree have ? */
	leaf_nodes = dm_div_up(t->num_targets, KEYS_PER_NODE);
	t->depth = 1 + int_log(leaf_nodes, CHILDREN_PER_NODE);

	/* leaf layer has already been set up */
	t->counts[t->depth - 1] = leaf_nodes;
	t->index[t->depth - 1] = t->highs;

	if (t->depth >= 2)
		r = setup_indexes(t);

	return r;
}

1112 1113 1114 1115 1116
static bool integrity_profile_exists(struct gendisk *disk)
{
	return !!blk_get_integrity(disk);
}

1117 1118 1119 1120
/*
 * Get a disk whose integrity profile reflects the table's profile.
 * Returns NULL if integrity support was inconsistent or unavailable.
 */
1121
static struct gendisk * dm_table_get_integrity_disk(struct dm_table *t)
1122 1123 1124 1125
{
	struct list_head *devices = dm_table_get_devices(t);
	struct dm_dev_internal *dd = NULL;
	struct gendisk *prev_disk = NULL, *template_disk = NULL;
1126 1127 1128 1129 1130 1131 1132
	unsigned i;

	for (i = 0; i < dm_table_get_num_targets(t); i++) {
		struct dm_target *ti = dm_table_get_target(t, i);
		if (!dm_target_passes_integrity(ti->type))
			goto no_integrity;
	}
1133 1134

	list_for_each_entry(dd, devices, list) {
1135
		template_disk = dd->dm_dev->bdev->bd_disk;
1136
		if (!integrity_profile_exists(template_disk))
1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154
			goto no_integrity;
		else if (prev_disk &&
			 blk_integrity_compare(prev_disk, template_disk) < 0)
			goto no_integrity;
		prev_disk = template_disk;
	}

	return template_disk;

no_integrity:
	if (prev_disk)
		DMWARN("%s: integrity not set: %s and %s profile mismatch",
		       dm_device_name(t->md),
		       prev_disk->disk_name,
		       template_disk->disk_name);
	return NULL;
}

1155
/*
1156 1157 1158
 * Register the mapped device for blk_integrity support if the
 * underlying devices have an integrity profile.  But all devices may
 * not have matching profiles (checking all devices isn't reliable
1159
 * during table load because this table may use other DM device(s) which
1160 1161 1162 1163
 * must be resumed before they will have an initialized integity
 * profile).  Consequently, stacked DM devices force a 2 stage integrity
 * profile validation: First pass during table load, final pass during
 * resume.
1164
 */
1165
static int dm_table_register_integrity(struct dm_table *t)
1166
{
1167
	struct mapped_device *md = t->md;
1168
	struct gendisk *template_disk = NULL;
1169

1170 1171 1172 1173
	/* If target handles integrity itself do not register it here. */
	if (t->integrity_added)
		return 0;

1174
	template_disk = dm_table_get_integrity_disk(t);
1175 1176
	if (!template_disk)
		return 0;
1177

1178
	if (!integrity_profile_exists(dm_disk(md))) {
1179
		t->integrity_supported = true;
1180 1181 1182 1183 1184 1185 1186
		/*
		 * Register integrity profile during table load; we can do
		 * this because the final profile must match during resume.
		 */
		blk_integrity_register(dm_disk(md),
				       blk_get_integrity(template_disk));
		return 0;
1187 1188 1189
	}

	/*
1190
	 * If DM device already has an initialized integrity
1191 1192
	 * profile the new profile should not conflict.
	 */
1193
	if (blk_integrity_compare(dm_disk(md), template_disk) < 0) {
1194 1195 1196 1197 1198 1199 1200
		DMWARN("%s: conflict with existing integrity profile: "
		       "%s profile mismatch",
		       dm_device_name(t->md),
		       template_disk->disk_name);
		return 1;
	}

1201
	/* Preserve existing integrity profile */
1202
	t->integrity_supported = true;
1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213
	return 0;
}

/*
 * Prepares the table for use by building the indices,
 * setting the type, and allocating mempools.
 */
int dm_table_complete(struct dm_table *t)
{
	int r;

1214
	r = dm_table_determine_type(t);
1215
	if (r) {
1216
		DMERR("unable to determine table type");
1217 1218 1219 1220 1221 1222 1223 1224 1225
		return r;
	}

	r = dm_table_build_index(t);
	if (r) {
		DMERR("unable to build btrees");
		return r;
	}

1226
	r = dm_table_register_integrity(t);
1227 1228 1229 1230 1231
	if (r) {
		DMERR("could not register integrity profile.");
		return r;
	}

1232
	r = dm_table_alloc_md_mempools(t, t->md);
1233 1234 1235 1236 1237 1238
	if (r)
		DMERR("unable to allocate mempools");

	return r;
}

A
Arjan van de Ven 已提交
1239
static DEFINE_MUTEX(_event_lock);
L
Linus Torvalds 已提交
1240 1241 1242
void dm_table_event_callback(struct dm_table *t,
			     void (*fn)(void *), void *context)
{
A
Arjan van de Ven 已提交
1243
	mutex_lock(&_event_lock);
L
Linus Torvalds 已提交
1244 1245
	t->event_fn = fn;
	t->event_context = context;
A
Arjan van de Ven 已提交
1246
	mutex_unlock(&_event_lock);
L
Linus Torvalds 已提交
1247 1248 1249 1250
}

void dm_table_event(struct dm_table *t)
{
A
Arjan van de Ven 已提交
1251
	mutex_lock(&_event_lock);
L
Linus Torvalds 已提交
1252 1253
	if (t->event_fn)
		t->event_fn(t->event_context);
A
Arjan van de Ven 已提交
1254
	mutex_unlock(&_event_lock);
L
Linus Torvalds 已提交
1255
}
1256
EXPORT_SYMBOL(dm_table_event);
L
Linus Torvalds 已提交
1257

1258
inline sector_t dm_table_get_size(struct dm_table *t)
L
Linus Torvalds 已提交
1259 1260 1261
{
	return t->num_targets ? (t->highs[t->num_targets - 1] + 1) : 0;
}
1262
EXPORT_SYMBOL(dm_table_get_size);
L
Linus Torvalds 已提交
1263 1264 1265

struct dm_target *dm_table_get_target(struct dm_table *t, unsigned int index)
{
1266
	if (index >= t->num_targets)
L
Linus Torvalds 已提交
1267 1268 1269 1270 1271 1272 1273
		return NULL;

	return t->targets + index;
}

/*
 * Search the btree for the correct target.
1274
 *
1275
 * Caller should check returned pointer for NULL
1276
 * to trap I/O beyond end of device.
L
Linus Torvalds 已提交
1277 1278 1279 1280 1281 1282
 */
struct dm_target *dm_table_find_target(struct dm_table *t, sector_t sector)
{
	unsigned int l, n = 0, k = 0;
	sector_t *node;

1283
	if (unlikely(sector >= dm_table_get_size(t)))
1284
		return NULL;
1285

L
Linus Torvalds 已提交
1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297
	for (l = 0; l < t->depth; l++) {
		n = get_child(n, k);
		node = get_node(t, l, n);

		for (k = 0; k < KEYS_PER_NODE; k++)
			if (node[k] >= sector)
				break;
	}

	return &t->targets[(KEYS_PER_NODE * n) + k];
}

1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318
/*
 * type->iterate_devices() should be called when the sanity check needs to
 * iterate and check all underlying data devices. iterate_devices() will
 * iterate all underlying data devices until it encounters a non-zero return
 * code, returned by whether the input iterate_devices_callout_fn, or
 * iterate_devices() itself internally.
 *
 * For some target type (e.g. dm-stripe), one call of iterate_devices() may
 * iterate multiple underlying devices internally, in which case a non-zero
 * return code returned by iterate_devices_callout_fn will stop the iteration
 * in advance.
 *
 * Cases requiring _any_ underlying device supporting some kind of attribute,
 * should use the iteration structure like dm_table_any_dev_attr(), or call
 * it directly. @func should handle semantics of positive examples, e.g.
 * capable of something.
 *
 * Cases requiring _all_ underlying devices supporting some kind of attribute,
 * should use the iteration structure like dm_table_supports_nowait() or
 * dm_table_supports_discards(). Or introduce dm_table_all_devs_attr() that
 * uses an @anti_func that handle semantics of counter examples, e.g. not
1319
 * capable of something. So: return !dm_table_any_dev_attr(t, anti_func, data);
1320 1321
 */
static bool dm_table_any_dev_attr(struct dm_table *t,
1322
				  iterate_devices_callout_fn func, void *data)
1323 1324 1325 1326 1327 1328 1329 1330
{
	struct dm_target *ti;
	unsigned int i;

	for (i = 0; i < dm_table_get_num_targets(t); i++) {
		ti = dm_table_get_target(t, i);

		if (ti->type->iterate_devices &&
1331
		    ti->type->iterate_devices(ti, func, data))
1332 1333 1334 1335 1336 1337
			return true;
        }

	return false;
}

1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355
static int count_device(struct dm_target *ti, struct dm_dev *dev,
			sector_t start, sector_t len, void *data)
{
	unsigned *num_devices = data;

	(*num_devices)++;

	return 0;
}

/*
 * Check whether a table has no data devices attached using each
 * target's iterate_devices method.
 * Returns false if the result is unknown because a target doesn't
 * support iterate_devices.
 */
bool dm_table_has_no_data_devices(struct dm_table *table)
{
1356 1357
	struct dm_target *ti;
	unsigned i, num_devices;
1358

1359 1360
	for (i = 0; i < dm_table_get_num_targets(table); i++) {
		ti = dm_table_get_target(table, i);
1361 1362 1363 1364

		if (!ti->type->iterate_devices)
			return false;

1365
		num_devices = 0;
1366 1367 1368 1369 1370 1371 1372 1373
		ti->type->iterate_devices(ti, count_device, &num_devices);
		if (num_devices)
			return false;
	}

	return true;
}

1374 1375
static int device_not_zoned_model(struct dm_target *ti, struct dm_dev *dev,
				  sector_t start, sector_t len, void *data)
1376 1377 1378 1379
{
	struct request_queue *q = bdev_get_queue(dev->bdev);
	enum blk_zoned_model *zoned_model = data;

1380
	return blk_queue_zoned_model(q) != *zoned_model;
1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396
}

static bool dm_table_supports_zoned_model(struct dm_table *t,
					  enum blk_zoned_model zoned_model)
{
	struct dm_target *ti;
	unsigned i;

	for (i = 0; i < dm_table_get_num_targets(t); i++) {
		ti = dm_table_get_target(t, i);

		if (zoned_model == BLK_ZONED_HM &&
		    !dm_target_supports_zoned_hm(ti->type))
			return false;

		if (!ti->type->iterate_devices ||
1397
		    ti->type->iterate_devices(ti, device_not_zoned_model, &zoned_model))
1398 1399 1400 1401 1402 1403
			return false;
	}

	return true;
}

1404 1405
static int device_not_matches_zone_sectors(struct dm_target *ti, struct dm_dev *dev,
					   sector_t start, sector_t len, void *data)
1406 1407 1408 1409
{
	struct request_queue *q = bdev_get_queue(dev->bdev);
	unsigned int *zone_sectors = data;

1410
	return blk_queue_zone_sectors(q) != *zone_sectors;
1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429
}

static int validate_hardware_zoned_model(struct dm_table *table,
					 enum blk_zoned_model zoned_model,
					 unsigned int zone_sectors)
{
	if (zoned_model == BLK_ZONED_NONE)
		return 0;

	if (!dm_table_supports_zoned_model(table, zoned_model)) {
		DMERR("%s: zoned model is not consistent across all devices",
		      dm_device_name(table->md));
		return -EINVAL;
	}

	/* Check zone size validity and compatibility */
	if (!zone_sectors || !is_power_of_2(zone_sectors))
		return -EINVAL;

1430
	if (dm_table_any_dev_attr(table, device_not_matches_zone_sectors, &zone_sectors)) {
1431 1432 1433 1434 1435 1436 1437 1438
		DMERR("%s: zone sectors is not consistent across all devices",
		      dm_device_name(table->md));
		return -EINVAL;
	}

	return 0;
}

1439 1440 1441 1442 1443 1444
/*
 * Establish the new table's queue_limits and validate them.
 */
int dm_calculate_queue_limits(struct dm_table *table,
			      struct queue_limits *limits)
{
1445
	struct dm_target *ti;
1446
	struct queue_limits ti_limits;
1447
	unsigned i;
1448 1449
	enum blk_zoned_model zoned_model = BLK_ZONED_NONE;
	unsigned int zone_sectors = 0;
1450

1451
	blk_set_stacking_limits(limits);
1452

1453
	for (i = 0; i < dm_table_get_num_targets(table); i++) {
1454
		blk_set_stacking_limits(&ti_limits);
1455

1456
		ti = dm_table_get_target(table, i);
1457 1458 1459 1460 1461 1462 1463 1464 1465 1466

		if (!ti->type->iterate_devices)
			goto combine_limits;

		/*
		 * Combine queue limits of all the devices this target uses.
		 */
		ti->type->iterate_devices(ti, dm_set_device_limits,
					  &ti_limits);

1467 1468 1469 1470 1471 1472 1473 1474 1475
		if (zoned_model == BLK_ZONED_NONE && ti_limits.zoned != BLK_ZONED_NONE) {
			/*
			 * After stacking all limits, validate all devices
			 * in table support this zoned model and zone sectors.
			 */
			zoned_model = ti_limits.zoned;
			zone_sectors = ti_limits.chunk_sectors;
		}

1476 1477 1478 1479
		/* Set I/O hints portion of queue limits */
		if (ti->type->io_hints)
			ti->type->io_hints(ti, &ti_limits);

1480 1481 1482 1483
		/*
		 * Check each device area is consistent with the target's
		 * overall queue limits.
		 */
1484 1485
		if (ti->type->iterate_devices(ti, device_area_is_invalid,
					      &ti_limits))
1486 1487 1488 1489 1490 1491 1492 1493
			return -EINVAL;

combine_limits:
		/*
		 * Merge this target's queue limits into the overall limits
		 * for the table.
		 */
		if (blk_stack_limits(limits, &ti_limits, 0) < 0)
1494
			DMWARN("%s: adding target device "
1495
			       "(start sect %llu len %llu) "
1496
			       "caused an alignment inconsistency",
1497 1498 1499 1500 1501
			       dm_device_name(table->md),
			       (unsigned long long) ti->begin,
			       (unsigned long long) ti->len);
	}

1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519
	/*
	 * Verify that the zoned model and zone sectors, as determined before
	 * any .io_hints override, are the same across all devices in the table.
	 * - this is especially relevant if .io_hints is emulating a disk-managed
	 *   zoned model (aka BLK_ZONED_NONE) on host-managed zoned block devices.
	 * BUT...
	 */
	if (limits->zoned != BLK_ZONED_NONE) {
		/*
		 * ...IF the above limits stacking determined a zoned model
		 * validate that all of the table's devices conform to it.
		 */
		zoned_model = limits->zoned;
		zone_sectors = limits->chunk_sectors;
	}
	if (validate_hardware_zoned_model(table, zoned_model, zone_sectors))
		return -EINVAL;

1520 1521 1522
	return validate_hardware_logical_block_alignment(table, limits);
}

M
Martin K. Petersen 已提交
1523
/*
1524 1525 1526
 * Verify that all devices have an integrity profile that matches the
 * DM device's registered integrity profile.  If the profiles don't
 * match then unregister the DM device's integrity profile.
M
Martin K. Petersen 已提交
1527
 */
1528
static void dm_table_verify_integrity(struct dm_table *t)
M
Martin K. Petersen 已提交
1529
{
1530
	struct gendisk *template_disk = NULL;
M
Martin K. Petersen 已提交
1531

1532 1533 1534
	if (t->integrity_added)
		return;

1535 1536 1537 1538 1539 1540 1541 1542 1543 1544
	if (t->integrity_supported) {
		/*
		 * Verify that the original integrity profile
		 * matches all the devices in this table.
		 */
		template_disk = dm_table_get_integrity_disk(t);
		if (template_disk &&
		    blk_integrity_compare(dm_disk(t->md), template_disk) >= 0)
			return;
	}
M
Martin K. Petersen 已提交
1545

1546
	if (integrity_profile_exists(dm_disk(t->md))) {
1547 1548
		DMWARN("%s: unable to establish an integrity profile",
		       dm_device_name(t->md));
1549 1550
		blk_integrity_unregister(dm_disk(t->md));
	}
M
Martin K. Petersen 已提交
1551 1552
}

1553 1554 1555
static int device_flush_capable(struct dm_target *ti, struct dm_dev *dev,
				sector_t start, sector_t len, void *data)
{
J
Jens Axboe 已提交
1556
	unsigned long flush = (unsigned long) data;
1557 1558
	struct request_queue *q = bdev_get_queue(dev->bdev);

1559
	return (q->queue_flags & flush);
1560 1561
}

J
Jens Axboe 已提交
1562
static bool dm_table_supports_flush(struct dm_table *t, unsigned long flush)
1563 1564
{
	struct dm_target *ti;
1565
	unsigned i;
1566 1567 1568 1569 1570 1571 1572

	/*
	 * Require at least one underlying device to support flushes.
	 * t->devices includes internal dm devices such as mirror logs
	 * so we need to use iterate_devices here, which targets
	 * supporting flushes must provide.
	 */
1573 1574
	for (i = 0; i < dm_table_get_num_targets(t); i++) {
		ti = dm_table_get_target(t, i);
1575

1576
		if (!ti->num_flush_bios)
1577 1578
			continue;

1579
		if (ti->flush_supported)
1580
			return true;
1581

1582
		if (ti->type->iterate_devices &&
J
Jens Axboe 已提交
1583
		    ti->type->iterate_devices(ti, device_flush_capable, (void *) flush))
1584
			return true;
1585 1586
	}

1587
	return false;
1588 1589
}

1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603
static int device_dax_write_cache_enabled(struct dm_target *ti,
					  struct dm_dev *dev, sector_t start,
					  sector_t len, void *data)
{
	struct dax_device *dax_dev = dev->dax_dev;

	if (!dax_dev)
		return false;

	if (dax_write_cache_enabled(dax_dev))
		return true;
	return false;
}

1604 1605
static int device_is_rotational(struct dm_target *ti, struct dm_dev *dev,
				sector_t start, sector_t len, void *data)
1606 1607 1608
{
	struct request_queue *q = bdev_get_queue(dev->bdev);

1609
	return !blk_queue_nonrot(q);
1610 1611
}

1612 1613 1614 1615 1616
static int device_is_not_random(struct dm_target *ti, struct dm_dev *dev,
			     sector_t start, sector_t len, void *data)
{
	struct request_queue *q = bdev_get_queue(dev->bdev);

1617
	return !blk_queue_add_random(q);
1618 1619
}

M
Mike Snitzer 已提交
1620 1621 1622 1623 1624
static int device_not_write_same_capable(struct dm_target *ti, struct dm_dev *dev,
					 sector_t start, sector_t len, void *data)
{
	struct request_queue *q = bdev_get_queue(dev->bdev);

1625
	return !q->limits.max_write_same_sectors;
M
Mike Snitzer 已提交
1626 1627 1628 1629 1630
}

static bool dm_table_supports_write_same(struct dm_table *t)
{
	struct dm_target *ti;
1631
	unsigned i;
M
Mike Snitzer 已提交
1632

1633 1634
	for (i = 0; i < dm_table_get_num_targets(t); i++) {
		ti = dm_table_get_target(t, i);
M
Mike Snitzer 已提交
1635

1636
		if (!ti->num_write_same_bios)
M
Mike Snitzer 已提交
1637 1638 1639
			return false;

		if (!ti->type->iterate_devices ||
M
Mike Snitzer 已提交
1640
		    ti->type->iterate_devices(ti, device_not_write_same_capable, NULL))
M
Mike Snitzer 已提交
1641 1642 1643 1644 1645 1646
			return false;
	}

	return true;
}

1647 1648 1649 1650 1651
static int device_not_write_zeroes_capable(struct dm_target *ti, struct dm_dev *dev,
					   sector_t start, sector_t len, void *data)
{
	struct request_queue *q = bdev_get_queue(dev->bdev);

1652
	return !q->limits.max_write_zeroes_sectors;
1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673
}

static bool dm_table_supports_write_zeroes(struct dm_table *t)
{
	struct dm_target *ti;
	unsigned i = 0;

	while (i < dm_table_get_num_targets(t)) {
		ti = dm_table_get_target(t, i++);

		if (!ti->num_write_zeroes_bios)
			return false;

		if (!ti->type->iterate_devices ||
		    ti->type->iterate_devices(ti, device_not_write_zeroes_capable, NULL))
			return false;
	}

	return true;
}

1674 1675 1676 1677 1678
static int device_not_nowait_capable(struct dm_target *ti, struct dm_dev *dev,
				     sector_t start, sector_t len, void *data)
{
	struct request_queue *q = bdev_get_queue(dev->bdev);

1679
	return !blk_queue_nowait(q);
1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700
}

static bool dm_table_supports_nowait(struct dm_table *t)
{
	struct dm_target *ti;
	unsigned i = 0;

	while (i < dm_table_get_num_targets(t)) {
		ti = dm_table_get_target(t, i++);

		if (!dm_target_supports_nowait(ti->type))
			return false;

		if (!ti->type->iterate_devices ||
		    ti->type->iterate_devices(ti, device_not_nowait_capable, NULL))
			return false;
	}

	return true;
}

1701 1702
static int device_not_discard_capable(struct dm_target *ti, struct dm_dev *dev,
				      sector_t start, sector_t len, void *data)
1703 1704 1705
{
	struct request_queue *q = bdev_get_queue(dev->bdev);

1706
	return !blk_queue_discard(q);
1707 1708 1709 1710 1711
}

static bool dm_table_supports_discards(struct dm_table *t)
{
	struct dm_target *ti;
1712
	unsigned i;
1713

1714 1715
	for (i = 0; i < dm_table_get_num_targets(t); i++) {
		ti = dm_table_get_target(t, i);
1716 1717

		if (!ti->num_discard_bios)
1718
			return false;
1719

1720 1721 1722 1723 1724 1725 1726 1727 1728
		/*
		 * Either the target provides discard support (as implied by setting
		 * 'discards_supported') or it relies on _all_ data devices having
		 * discard support.
		 */
		if (!ti->discards_supported &&
		    (!ti->type->iterate_devices ||
		     ti->type->iterate_devices(ti, device_not_discard_capable, NULL)))
			return false;
1729 1730
	}

1731
	return true;
1732 1733
}

1734 1735 1736 1737 1738 1739
static int device_not_secure_erase_capable(struct dm_target *ti,
					   struct dm_dev *dev, sector_t start,
					   sector_t len, void *data)
{
	struct request_queue *q = bdev_get_queue(dev->bdev);

1740
	return !blk_queue_secure_erase(q);
1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761
}

static bool dm_table_supports_secure_erase(struct dm_table *t)
{
	struct dm_target *ti;
	unsigned int i;

	for (i = 0; i < dm_table_get_num_targets(t); i++) {
		ti = dm_table_get_target(t, i);

		if (!ti->num_secure_erase_bios)
			return false;

		if (!ti->type->iterate_devices ||
		    ti->type->iterate_devices(ti, device_not_secure_erase_capable, NULL))
			return false;
	}

	return true;
}

1762 1763 1764 1765 1766 1767
static int device_requires_stable_pages(struct dm_target *ti,
					struct dm_dev *dev, sector_t start,
					sector_t len, void *data)
{
	struct request_queue *q = bdev_get_queue(dev->bdev);

1768
	return blk_queue_stable_writes(q);
1769 1770
}

1771 1772
void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q,
			       struct queue_limits *limits)
L
Linus Torvalds 已提交
1773
{
1774
	bool wc = false, fua = false;
P
Pankaj Gupta 已提交
1775
	int page_size = PAGE_SIZE;
1776

L
Linus Torvalds 已提交
1777
	/*
1778
	 * Copy table's limits to the DM device's request_queue
L
Linus Torvalds 已提交
1779
	 */
1780
	q->limits = *limits;
1781

1782 1783 1784 1785 1786
	if (dm_table_supports_nowait(t))
		blk_queue_flag_set(QUEUE_FLAG_NOWAIT, q);
	else
		blk_queue_flag_clear(QUEUE_FLAG_NOWAIT, q);

1787
	if (!dm_table_supports_discards(t)) {
1788
		blk_queue_flag_clear(QUEUE_FLAG_DISCARD, q);
1789 1790 1791 1792 1793 1794 1795
		/* Must also clear discard limits... */
		q->limits.max_discard_sectors = 0;
		q->limits.max_hw_discard_sectors = 0;
		q->limits.discard_granularity = 0;
		q->limits.discard_alignment = 0;
		q->limits.discard_misaligned = 0;
	} else
1796
		blk_queue_flag_set(QUEUE_FLAG_DISCARD, q);
M
Mike Snitzer 已提交
1797

1798
	if (dm_table_supports_secure_erase(t))
1799
		blk_queue_flag_set(QUEUE_FLAG_SECERASE, q);
1800

J
Jens Axboe 已提交
1801
	if (dm_table_supports_flush(t, (1UL << QUEUE_FLAG_WC))) {
1802
		wc = true;
J
Jens Axboe 已提交
1803
		if (dm_table_supports_flush(t, (1UL << QUEUE_FLAG_FUA)))
1804
			fua = true;
1805
	}
1806
	blk_queue_write_cache(q, wc, fua);
1807

1808
	if (dm_table_supports_dax(t, device_not_dax_capable, &page_size)) {
1809
		blk_queue_flag_set(QUEUE_FLAG_DAX, q);
1810
		if (dm_table_supports_dax(t, device_not_dax_synchronous_capable, NULL))
P
Pankaj Gupta 已提交
1811 1812
			set_dax_synchronous(t->md->dax_dev);
	}
1813 1814 1815
	else
		blk_queue_flag_clear(QUEUE_FLAG_DAX, q);

1816
	if (dm_table_any_dev_attr(t, device_dax_write_cache_enabled, NULL))
1817 1818
		dax_write_cache(t->md->dax_dev, true);

1819
	/* Ensure that all underlying devices are non-rotational. */
1820
	if (dm_table_any_dev_attr(t, device_is_rotational, NULL))
1821
		blk_queue_flag_clear(QUEUE_FLAG_NONROT, q);
1822 1823
	else
		blk_queue_flag_set(QUEUE_FLAG_NONROT, q);
1824

M
Mike Snitzer 已提交
1825 1826
	if (!dm_table_supports_write_same(t))
		q->limits.max_write_same_sectors = 0;
1827 1828
	if (!dm_table_supports_write_zeroes(t))
		q->limits.max_write_zeroes_sectors = 0;
M
Mike Snitzer 已提交
1829

1830
	dm_table_verify_integrity(t);
K
Kiyoshi Ueda 已提交
1831

1832 1833 1834
	/*
	 * Some devices don't use blk_integrity but still want stable pages
	 * because they do their own checksumming.
1835 1836 1837
	 * If any underlying device requires stable pages, a table must require
	 * them as well.  Only targets that support iterate_devices are considered:
	 * don't want error, zero, etc to require stable pages.
1838
	 */
1839
	if (dm_table_any_dev_attr(t, device_requires_stable_pages, NULL))
1840
		blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, q);
1841
	else
1842
		blk_queue_flag_clear(QUEUE_FLAG_STABLE_WRITES, q);
1843

1844 1845 1846 1847 1848 1849
	/*
	 * Determine whether or not this queue's I/O timings contribute
	 * to the entropy pool, Only request-based targets use this.
	 * Clear QUEUE_FLAG_ADD_RANDOM if any underlying device does not
	 * have it set.
	 */
1850 1851
	if (blk_queue_add_random(q) &&
	    dm_table_any_dev_attr(t, device_is_not_random, NULL))
1852
		blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, q);
1853 1854 1855 1856

	/*
	 * For a zoned target, the number of zones should be updated for the
	 * correct value to be exposed in sysfs queue/nr_zones. For a BIO based
1857
	 * target, this is all that is needed.
1858
	 */
1859 1860 1861 1862 1863 1864
#ifdef CONFIG_BLK_DEV_ZONED
	if (blk_queue_is_zoned(q)) {
		WARN_ON_ONCE(queue_is_mq(q));
		q->nr_zones = blkdev_nr_zones(t->md->disk);
	}
#endif
1865

1866
	blk_queue_update_readahead(q);
L
Linus Torvalds 已提交
1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878
}

unsigned int dm_table_get_num_targets(struct dm_table *t)
{
	return t->num_targets;
}

struct list_head *dm_table_get_devices(struct dm_table *t)
{
	return &t->devices;
}

1879
fmode_t dm_table_get_mode(struct dm_table *t)
L
Linus Torvalds 已提交
1880 1881 1882
{
	return t->mode;
}
1883
EXPORT_SYMBOL(dm_table_get_mode);
L
Linus Torvalds 已提交
1884

1885 1886 1887 1888 1889 1890 1891
enum suspend_mode {
	PRESUSPEND,
	PRESUSPEND_UNDO,
	POSTSUSPEND,
};

static void suspend_targets(struct dm_table *t, enum suspend_mode mode)
L
Linus Torvalds 已提交
1892 1893 1894 1895
{
	int i = t->num_targets;
	struct dm_target *ti = t->targets;

1896 1897
	lockdep_assert_held(&t->md->suspend_lock);

L
Linus Torvalds 已提交
1898
	while (i--) {
1899 1900 1901 1902 1903 1904 1905 1906 1907 1908
		switch (mode) {
		case PRESUSPEND:
			if (ti->type->presuspend)
				ti->type->presuspend(ti);
			break;
		case PRESUSPEND_UNDO:
			if (ti->type->presuspend_undo)
				ti->type->presuspend_undo(ti);
			break;
		case POSTSUSPEND:
L
Linus Torvalds 已提交
1909 1910
			if (ti->type->postsuspend)
				ti->type->postsuspend(ti);
1911 1912
			break;
		}
L
Linus Torvalds 已提交
1913 1914 1915 1916 1917 1918
		ti++;
	}
}

void dm_table_presuspend_targets(struct dm_table *t)
{
1919 1920 1921
	if (!t)
		return;

1922 1923 1924 1925 1926 1927 1928 1929 1930
	suspend_targets(t, PRESUSPEND);
}

void dm_table_presuspend_undo_targets(struct dm_table *t)
{
	if (!t)
		return;

	suspend_targets(t, PRESUSPEND_UNDO);
L
Linus Torvalds 已提交
1931 1932 1933 1934
}

void dm_table_postsuspend_targets(struct dm_table *t)
{
1935 1936 1937
	if (!t)
		return;

1938
	suspend_targets(t, POSTSUSPEND);
L
Linus Torvalds 已提交
1939 1940
}

1941
int dm_table_resume_targets(struct dm_table *t)
L
Linus Torvalds 已提交
1942
{
1943 1944
	int i, r = 0;

1945 1946
	lockdep_assert_held(&t->md->suspend_lock);

1947 1948 1949 1950 1951 1952 1953
	for (i = 0; i < t->num_targets; i++) {
		struct dm_target *ti = t->targets + i;

		if (!ti->type->preresume)
			continue;

		r = ti->type->preresume(ti);
1954 1955 1956
		if (r) {
			DMERR("%s: %s: preresume failed, error = %d",
			      dm_device_name(t->md), ti->type->name, r);
1957
			return r;
1958
		}
1959
	}
L
Linus Torvalds 已提交
1960 1961 1962 1963 1964 1965 1966

	for (i = 0; i < t->num_targets; i++) {
		struct dm_target *ti = t->targets + i;

		if (ti->type->resume)
			ti->type->resume(ti);
	}
1967 1968

	return 0;
L
Linus Torvalds 已提交
1969 1970
}

M
Mike Anderson 已提交
1971 1972 1973 1974
struct mapped_device *dm_table_get_md(struct dm_table *t)
{
	return t->md;
}
1975
EXPORT_SYMBOL(dm_table_get_md);
M
Mike Anderson 已提交
1976

M
Michał Mirosław 已提交
1977 1978 1979 1980 1981 1982
const char *dm_table_device_name(struct dm_table *t)
{
	return dm_device_name(t->md);
}
EXPORT_SYMBOL_GPL(dm_table_device_name);

1983 1984 1985 1986 1987
void dm_table_run_md_queue_async(struct dm_table *t)
{
	if (!dm_table_request_based(t))
		return;

1988 1989
	if (t->md->queue)
		blk_mq_run_hw_queues(t->md->queue, true);
1990 1991 1992
}
EXPORT_SYMBOL(dm_table_run_md_queue_async);