dm-mpath.c 39.8 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7
/*
 * Copyright (C) 2003 Sistina Software Limited.
 * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved.
 *
 * This file is released under the GPL.
 */

8 9
#include <linux/device-mapper.h>

L
Linus Torvalds 已提交
10
#include "dm-path-selector.h"
M
Mike Anderson 已提交
11
#include "dm-uevent.h"
L
Linus Torvalds 已提交
12 13 14 15 16 17 18 19 20

#include <linux/ctype.h>
#include <linux/init.h>
#include <linux/mempool.h>
#include <linux/module.h>
#include <linux/pagemap.h>
#include <linux/slab.h>
#include <linux/time.h>
#include <linux/workqueue.h>
21
#include <scsi/scsi_dh.h>
L
Linus Torvalds 已提交
22 23
#include <asm/atomic.h>

24
#define DM_MSG_PREFIX "multipath"
L
Linus Torvalds 已提交
25
#define MESG_STR(x) x, sizeof(x)
26 27
#define DM_PG_INIT_DELAY_MSECS 2000
#define DM_PG_INIT_DELAY_DEFAULT ((unsigned) -1)
L
Linus Torvalds 已提交
28 29 30 31 32 33

/* Path properties */
struct pgpath {
	struct list_head list;

	struct priority_group *pg;	/* Owning PG */
34
	unsigned is_active;		/* Path status */
L
Linus Torvalds 已提交
35 36
	unsigned fail_count;		/* Cumulative failure count */

37
	struct dm_path path;
38
	struct delayed_work activate_path;
L
Linus Torvalds 已提交
39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66
};

#define path_to_pgpath(__pgp) container_of((__pgp), struct pgpath, path)

/*
 * Paths are grouped into Priority Groups and numbered from 1 upwards.
 * Each has a path selector which controls which path gets used.
 */
struct priority_group {
	struct list_head list;

	struct multipath *m;		/* Owning multipath instance */
	struct path_selector ps;

	unsigned pg_num;		/* Reference number */
	unsigned bypassed;		/* Temporarily bypass this PG? */

	unsigned nr_pgpaths;		/* Number of paths in PG */
	struct list_head pgpaths;
};

/* Multipath context */
struct multipath {
	struct list_head list;
	struct dm_target *ti;

	spinlock_t lock;

67
	const char *hw_handler_name;
68
	char *hw_handler_params;
69

L
Linus Torvalds 已提交
70 71
	unsigned nr_priority_groups;
	struct list_head priority_groups;
72 73 74

	wait_queue_head_t pg_init_wait;	/* Wait for pg_init completion */

L
Linus Torvalds 已提交
75
	unsigned pg_init_required;	/* pg_init needs calling? */
76
	unsigned pg_init_in_progress;	/* Only one pg_init allowed at once */
77
	unsigned pg_init_delay_retry;	/* Delay pg_init retry? */
L
Linus Torvalds 已提交
78 79 80 81 82 83 84 85 86

	unsigned nr_valid_paths;	/* Total number of usable paths */
	struct pgpath *current_pgpath;
	struct priority_group *current_pg;
	struct priority_group *next_pg;	/* Switch to this PG if set */
	unsigned repeat_count;		/* I/Os left before calling PS again */

	unsigned queue_io;		/* Must we queue all I/O? */
	unsigned queue_if_no_path;	/* Queue I/O if last path fails? */
87
	unsigned saved_queue_if_no_path;/* Saved state during suspension */
D
Dave Wysochanski 已提交
88 89
	unsigned pg_init_retries;	/* Number of times to retry pg_init */
	unsigned pg_init_count;		/* Number of times pg_init called */
90
	unsigned pg_init_delay_msecs;	/* Number of msecs before pg_init retry */
L
Linus Torvalds 已提交
91 92

	struct work_struct process_queued_ios;
93
	struct list_head queued_ios;
L
Linus Torvalds 已提交
94 95 96 97 98
	unsigned queue_size;

	struct work_struct trigger_event;

	/*
A
Alasdair G Kergon 已提交
99
	 * We must use a mempool of dm_mpath_io structs so that we
L
Linus Torvalds 已提交
100 101 102
	 * can resubmit bios on error.
	 */
	mempool_t *mpio_pool;
103 104

	struct mutex work_mutex;
L
Linus Torvalds 已提交
105 106 107 108 109
};

/*
 * Context information attached to each bio we process.
 */
A
Alasdair G Kergon 已提交
110
struct dm_mpath_io {
L
Linus Torvalds 已提交
111
	struct pgpath *pgpath;
112
	size_t nr_bytes;
L
Linus Torvalds 已提交
113 114 115 116 117 118
};

typedef int (*action_fn) (struct pgpath *pgpath);

#define MIN_IOS 256	/* Mempool size */

119
static struct kmem_cache *_mpio_cache;
L
Linus Torvalds 已提交
120

121
static struct workqueue_struct *kmultipathd, *kmpath_handlerd;
D
David Howells 已提交
122 123
static void process_queued_ios(struct work_struct *work);
static void trigger_event(struct work_struct *work);
124
static void activate_path(struct work_struct *work);
L
Linus Torvalds 已提交
125 126 127 128 129 130 131 132


/*-----------------------------------------------
 * Allocation routines
 *-----------------------------------------------*/

static struct pgpath *alloc_pgpath(void)
{
M
Micha³ Miros³aw 已提交
133
	struct pgpath *pgpath = kzalloc(sizeof(*pgpath), GFP_KERNEL);
L
Linus Torvalds 已提交
134

135
	if (pgpath) {
136
		pgpath->is_active = 1;
137
		INIT_DELAYED_WORK(&pgpath->activate_path, activate_path);
138
	}
L
Linus Torvalds 已提交
139 140 141 142

	return pgpath;
}

A
Alasdair G Kergon 已提交
143
static void free_pgpath(struct pgpath *pgpath)
L
Linus Torvalds 已提交
144 145 146 147 148 149 150 151
{
	kfree(pgpath);
}

static struct priority_group *alloc_priority_group(void)
{
	struct priority_group *pg;

M
Micha³ Miros³aw 已提交
152
	pg = kzalloc(sizeof(*pg), GFP_KERNEL);
L
Linus Torvalds 已提交
153

M
Micha³ Miros³aw 已提交
154 155
	if (pg)
		INIT_LIST_HEAD(&pg->pgpaths);
L
Linus Torvalds 已提交
156 157 158 159 160 161 162

	return pg;
}

static void free_pgpaths(struct list_head *pgpaths, struct dm_target *ti)
{
	struct pgpath *pgpath, *tmp;
163
	struct multipath *m = ti->private;
L
Linus Torvalds 已提交
164 165 166

	list_for_each_entry_safe(pgpath, tmp, pgpaths, list) {
		list_del(&pgpath->list);
167 168
		if (m->hw_handler_name)
			scsi_dh_detach(bdev_get_queue(pgpath->path.dev->bdev));
L
Linus Torvalds 已提交
169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187
		dm_put_device(ti, pgpath->path.dev);
		free_pgpath(pgpath);
	}
}

static void free_priority_group(struct priority_group *pg,
				struct dm_target *ti)
{
	struct path_selector *ps = &pg->ps;

	if (ps->type) {
		ps->type->destroy(ps);
		dm_put_path_selector(ps->type);
	}

	free_pgpaths(&pg->pgpaths, ti);
	kfree(pg);
}

M
Micha³ Miros³aw 已提交
188
static struct multipath *alloc_multipath(struct dm_target *ti)
L
Linus Torvalds 已提交
189 190 191
{
	struct multipath *m;

M
Micha³ Miros³aw 已提交
192
	m = kzalloc(sizeof(*m), GFP_KERNEL);
L
Linus Torvalds 已提交
193 194
	if (m) {
		INIT_LIST_HEAD(&m->priority_groups);
195
		INIT_LIST_HEAD(&m->queued_ios);
L
Linus Torvalds 已提交
196 197
		spin_lock_init(&m->lock);
		m->queue_io = 1;
198
		m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT;
D
David Howells 已提交
199 200
		INIT_WORK(&m->process_queued_ios, process_queued_ios);
		INIT_WORK(&m->trigger_event, trigger_event);
201
		init_waitqueue_head(&m->pg_init_wait);
202
		mutex_init(&m->work_mutex);
203
		m->mpio_pool = mempool_create_slab_pool(MIN_IOS, _mpio_cache);
L
Linus Torvalds 已提交
204 205 206 207
		if (!m->mpio_pool) {
			kfree(m);
			return NULL;
		}
M
Micha³ Miros³aw 已提交
208 209
		m->ti = ti;
		ti->private = m;
L
Linus Torvalds 已提交
210 211 212 213 214 215 216 217 218 219 220 221 222 223
	}

	return m;
}

static void free_multipath(struct multipath *m)
{
	struct priority_group *pg, *tmp;

	list_for_each_entry_safe(pg, tmp, &m->priority_groups, list) {
		list_del(&pg->list);
		free_priority_group(pg, m->ti);
	}

224
	kfree(m->hw_handler_name);
225
	kfree(m->hw_handler_params);
L
Linus Torvalds 已提交
226 227 228 229 230 231 232 233 234
	mempool_destroy(m->mpio_pool);
	kfree(m);
}


/*-----------------------------------------------
 * Path selection
 *-----------------------------------------------*/

K
Kiyoshi Ueda 已提交
235 236 237
static void __pg_init_all_paths(struct multipath *m)
{
	struct pgpath *pgpath;
238
	unsigned long pg_init_delay = 0;
K
Kiyoshi Ueda 已提交
239 240 241

	m->pg_init_count++;
	m->pg_init_required = 0;
242 243 244
	if (m->pg_init_delay_retry)
		pg_init_delay = msecs_to_jiffies(m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT ?
						 m->pg_init_delay_msecs : DM_PG_INIT_DELAY_MSECS);
K
Kiyoshi Ueda 已提交
245 246 247 248
	list_for_each_entry(pgpath, &m->current_pg->pgpaths, list) {
		/* Skip failed paths */
		if (!pgpath->is_active)
			continue;
249 250
		if (queue_delayed_work(kmpath_handlerd, &pgpath->activate_path,
				       pg_init_delay))
K
Kiyoshi Ueda 已提交
251 252 253 254
			m->pg_init_in_progress++;
	}
}

L
Linus Torvalds 已提交
255 256 257 258 259
static void __switch_pg(struct multipath *m, struct pgpath *pgpath)
{
	m->current_pg = pgpath->pg;

	/* Must we initialise the PG first, and queue I/O till it's ready? */
260
	if (m->hw_handler_name) {
L
Linus Torvalds 已提交
261 262 263 264 265 266
		m->pg_init_required = 1;
		m->queue_io = 1;
	} else {
		m->pg_init_required = 0;
		m->queue_io = 0;
	}
D
Dave Wysochanski 已提交
267 268

	m->pg_init_count = 0;
L
Linus Torvalds 已提交
269 270
}

271 272
static int __choose_path_in_pg(struct multipath *m, struct priority_group *pg,
			       size_t nr_bytes)
L
Linus Torvalds 已提交
273
{
274
	struct dm_path *path;
L
Linus Torvalds 已提交
275

276
	path = pg->ps.type->select_path(&pg->ps, &m->repeat_count, nr_bytes);
L
Linus Torvalds 已提交
277 278 279 280 281 282 283 284 285 286 287
	if (!path)
		return -ENXIO;

	m->current_pgpath = path_to_pgpath(path);

	if (m->current_pg != pg)
		__switch_pg(m, m->current_pgpath);

	return 0;
}

288
static void __choose_pgpath(struct multipath *m, size_t nr_bytes)
L
Linus Torvalds 已提交
289 290 291 292 293 294 295 296 297 298 299
{
	struct priority_group *pg;
	unsigned bypassed = 1;

	if (!m->nr_valid_paths)
		goto failed;

	/* Were we instructed to switch PG? */
	if (m->next_pg) {
		pg = m->next_pg;
		m->next_pg = NULL;
300
		if (!__choose_path_in_pg(m, pg, nr_bytes))
L
Linus Torvalds 已提交
301 302 303 304
			return;
	}

	/* Don't change PG until it has no remaining paths */
305
	if (m->current_pg && !__choose_path_in_pg(m, m->current_pg, nr_bytes))
L
Linus Torvalds 已提交
306 307 308 309 310 311 312 313 314 315 316
		return;

	/*
	 * Loop through priority groups until we find a valid path.
	 * First time we skip PGs marked 'bypassed'.
	 * Second time we only try the ones we skipped.
	 */
	do {
		list_for_each_entry(pg, &m->priority_groups, list) {
			if (pg->bypassed == bypassed)
				continue;
317
			if (!__choose_path_in_pg(m, pg, nr_bytes))
L
Linus Torvalds 已提交
318 319 320 321 322 323 324 325 326
				return;
		}
	} while (bypassed--);

failed:
	m->current_pgpath = NULL;
	m->current_pg = NULL;
}

327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343
/*
 * Check whether bios must be queued in the device-mapper core rather
 * than here in the target.
 *
 * m->lock must be held on entry.
 *
 * If m->queue_if_no_path and m->saved_queue_if_no_path hold the
 * same value then we are not between multipath_presuspend()
 * and multipath_resume() calls and we have no need to check
 * for the DMF_NOFLUSH_SUSPENDING flag.
 */
static int __must_push_back(struct multipath *m)
{
	return (m->queue_if_no_path != m->saved_queue_if_no_path &&
		dm_noflush_suspending(m->ti));
}

344
static int map_io(struct multipath *m, struct request *clone,
A
Alasdair G Kergon 已提交
345
		  struct dm_mpath_io *mpio, unsigned was_queued)
L
Linus Torvalds 已提交
346
{
347
	int r = DM_MAPIO_REMAPPED;
348
	size_t nr_bytes = blk_rq_bytes(clone);
L
Linus Torvalds 已提交
349 350
	unsigned long flags;
	struct pgpath *pgpath;
351
	struct block_device *bdev;
L
Linus Torvalds 已提交
352 353 354 355 356 357

	spin_lock_irqsave(&m->lock, flags);

	/* Do we need to select a new pgpath? */
	if (!m->current_pgpath ||
	    (!m->queue_io && (m->repeat_count && --m->repeat_count == 0)))
358
		__choose_pgpath(m, nr_bytes);
L
Linus Torvalds 已提交
359 360 361 362 363 364 365

	pgpath = m->current_pgpath;

	if (was_queued)
		m->queue_size--;

	if ((pgpath && m->queue_io) ||
366
	    (!pgpath && m->queue_if_no_path)) {
L
Linus Torvalds 已提交
367
		/* Queue for the daemon to resubmit */
368
		list_add_tail(&clone->queuelist, &m->queued_ios);
L
Linus Torvalds 已提交
369
		m->queue_size++;
370 371
		if ((m->pg_init_required && !m->pg_init_in_progress) ||
		    !m->queue_io)
372
			queue_work(kmultipathd, &m->process_queued_ios);
L
Linus Torvalds 已提交
373
		pgpath = NULL;
374
		r = DM_MAPIO_SUBMITTED;
375 376 377 378 379
	} else if (pgpath) {
		bdev = pgpath->path.dev->bdev;
		clone->q = bdev_get_queue(bdev);
		clone->rq_disk = bdev->bd_disk;
	} else if (__must_push_back(m))
380 381 382
		r = DM_MAPIO_REQUEUE;
	else
		r = -EIO;	/* Failed */
L
Linus Torvalds 已提交
383 384

	mpio->pgpath = pgpath;
385 386 387 388 389
	mpio->nr_bytes = nr_bytes;

	if (r == DM_MAPIO_REMAPPED && pgpath->pg->ps.type->start_io)
		pgpath->pg->ps.type->start_io(&pgpath->pg->ps, &pgpath->path,
					      nr_bytes);
L
Linus Torvalds 已提交
390 391 392 393 394 395 396 397 398

	spin_unlock_irqrestore(&m->lock, flags);

	return r;
}

/*
 * If we run out of usable paths, should we queue I/O or error it?
 */
399 400
static int queue_if_no_path(struct multipath *m, unsigned queue_if_no_path,
			    unsigned save_old_value)
L
Linus Torvalds 已提交
401 402 403 404 405
{
	unsigned long flags;

	spin_lock_irqsave(&m->lock, flags);

406 407 408 409
	if (save_old_value)
		m->saved_queue_if_no_path = m->queue_if_no_path;
	else
		m->saved_queue_if_no_path = queue_if_no_path;
L
Linus Torvalds 已提交
410
	m->queue_if_no_path = queue_if_no_path;
411
	if (!m->queue_if_no_path && m->queue_size)
412
		queue_work(kmultipathd, &m->process_queued_ios);
L
Linus Torvalds 已提交
413 414 415 416 417 418 419 420 421 422 423 424 425 426

	spin_unlock_irqrestore(&m->lock, flags);

	return 0;
}

/*-----------------------------------------------------------------
 * The multipath daemon is responsible for resubmitting queued ios.
 *---------------------------------------------------------------*/

static void dispatch_queued_ios(struct multipath *m)
{
	int r;
	unsigned long flags;
A
Alasdair G Kergon 已提交
427
	struct dm_mpath_io *mpio;
L
Linus Torvalds 已提交
428
	union map_info *info;
429 430
	struct request *clone, *n;
	LIST_HEAD(cl);
L
Linus Torvalds 已提交
431 432

	spin_lock_irqsave(&m->lock, flags);
433
	list_splice_init(&m->queued_ios, &cl);
L
Linus Torvalds 已提交
434 435
	spin_unlock_irqrestore(&m->lock, flags);

436 437
	list_for_each_entry_safe(clone, n, &cl, queuelist) {
		list_del_init(&clone->queuelist);
L
Linus Torvalds 已提交
438

439
		info = dm_get_rq_mapinfo(clone);
L
Linus Torvalds 已提交
440 441
		mpio = info->ptr;

442 443 444 445 446 447 448 449 450 451
		r = map_io(m, clone, mpio, 1);
		if (r < 0) {
			mempool_free(mpio, m->mpio_pool);
			dm_kill_unmapped_request(clone, r);
		} else if (r == DM_MAPIO_REMAPPED)
			dm_dispatch_request(clone);
		else if (r == DM_MAPIO_REQUEUE) {
			mempool_free(mpio, m->mpio_pool);
			dm_requeue_unmapped_request(clone);
		}
L
Linus Torvalds 已提交
452 453 454
	}
}

D
David Howells 已提交
455
static void process_queued_ios(struct work_struct *work)
L
Linus Torvalds 已提交
456
{
D
David Howells 已提交
457 458
	struct multipath *m =
		container_of(work, struct multipath, process_queued_ios);
K
Kiyoshi Ueda 已提交
459
	struct pgpath *pgpath = NULL;
460
	unsigned must_queue = 1;
L
Linus Torvalds 已提交
461 462 463 464
	unsigned long flags;

	spin_lock_irqsave(&m->lock, flags);

465 466 467
	if (!m->queue_size)
		goto out;

L
Linus Torvalds 已提交
468
	if (!m->current_pgpath)
469
		__choose_pgpath(m, 0);
L
Linus Torvalds 已提交
470 471 472

	pgpath = m->current_pgpath;

473 474 475
	if ((pgpath && !m->queue_io) ||
	    (!pgpath && !m->queue_if_no_path))
		must_queue = 0;
L
Linus Torvalds 已提交
476

K
Kiyoshi Ueda 已提交
477 478 479
	if (m->pg_init_required && !m->pg_init_in_progress && pgpath)
		__pg_init_all_paths(m);

480
out:
L
Linus Torvalds 已提交
481 482 483 484 485 486 487 488 489
	spin_unlock_irqrestore(&m->lock, flags);
	if (!must_queue)
		dispatch_queued_ios(m);
}

/*
 * An event is triggered whenever a path is taken out of use.
 * Includes path failure and PG bypass.
 */
D
David Howells 已提交
490
static void trigger_event(struct work_struct *work)
L
Linus Torvalds 已提交
491
{
D
David Howells 已提交
492 493
	struct multipath *m =
		container_of(work, struct multipath, trigger_event);
L
Linus Torvalds 已提交
494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560

	dm_table_event(m->ti->table);
}

/*-----------------------------------------------------------------
 * Constructor/argument parsing:
 * <#multipath feature args> [<arg>]*
 * <#hw_handler args> [hw_handler [<arg>]*]
 * <#priority groups>
 * <initial priority group>
 *     [<selector> <#selector args> [<arg>]*
 *      <#paths> <#per-path selector args>
 *         [<path> [<arg>]* ]+ ]+
 *---------------------------------------------------------------*/
struct param {
	unsigned min;
	unsigned max;
	char *error;
};

static int read_param(struct param *param, char *str, unsigned *v, char **error)
{
	if (!str ||
	    (sscanf(str, "%u", v) != 1) ||
	    (*v < param->min) ||
	    (*v > param->max)) {
		*error = param->error;
		return -EINVAL;
	}

	return 0;
}

struct arg_set {
	unsigned argc;
	char **argv;
};

static char *shift(struct arg_set *as)
{
	char *r;

	if (as->argc) {
		as->argc--;
		r = *as->argv;
		as->argv++;
		return r;
	}

	return NULL;
}

static void consume(struct arg_set *as, unsigned n)
{
	BUG_ON (as->argc < n);
	as->argc -= n;
	as->argv += n;
}

static int parse_path_selector(struct arg_set *as, struct priority_group *pg,
			       struct dm_target *ti)
{
	int r;
	struct path_selector_type *pst;
	unsigned ps_argc;

	static struct param _params[] = {
561
		{0, 1024, "invalid number of path selector args"},
L
Linus Torvalds 已提交
562 563 564 565
	};

	pst = dm_get_path_selector(shift(as));
	if (!pst) {
566
		ti->error = "unknown path selector type";
L
Linus Torvalds 已提交
567 568 569 570
		return -EINVAL;
	}

	r = read_param(_params, shift(as), &ps_argc, &ti->error);
571 572
	if (r) {
		dm_put_path_selector(pst);
L
Linus Torvalds 已提交
573
		return -EINVAL;
574
	}
L
Linus Torvalds 已提交
575

576 577 578 579 580 581
	if (ps_argc > as->argc) {
		dm_put_path_selector(pst);
		ti->error = "not enough arguments for path selector";
		return -EINVAL;
	}

L
Linus Torvalds 已提交
582 583 584
	r = pst->create(&pg->ps, ps_argc, as->argv);
	if (r) {
		dm_put_path_selector(pst);
585
		ti->error = "path selector constructor failed";
L
Linus Torvalds 已提交
586 587 588 589 590 591 592 593 594 595 596 597 598 599
		return r;
	}

	pg->ps.type = pst;
	consume(as, ps_argc);

	return 0;
}

static struct pgpath *parse_path(struct arg_set *as, struct path_selector *ps,
			       struct dm_target *ti)
{
	int r;
	struct pgpath *p;
600
	struct multipath *m = ti->private;
L
Linus Torvalds 已提交
601 602 603

	/* we need at least a path arg */
	if (as->argc < 1) {
604
		ti->error = "no device given";
605
		return ERR_PTR(-EINVAL);
L
Linus Torvalds 已提交
606 607 608 609
	}

	p = alloc_pgpath();
	if (!p)
610
		return ERR_PTR(-ENOMEM);
L
Linus Torvalds 已提交
611

612 613
	r = dm_get_device(ti, shift(as), dm_table_get_mode(ti->table),
			  &p->path.dev);
L
Linus Torvalds 已提交
614
	if (r) {
615
		ti->error = "error getting device";
L
Linus Torvalds 已提交
616 617 618
		goto bad;
	}

619
	if (m->hw_handler_name) {
620 621 622 623 624 625 626 627 628 629 630 631
		struct request_queue *q = bdev_get_queue(p->path.dev->bdev);

		r = scsi_dh_attach(q, m->hw_handler_name);
		if (r == -EBUSY) {
			/*
			 * Already attached to different hw_handler,
			 * try to reattach with correct one.
			 */
			scsi_dh_detach(q);
			r = scsi_dh_attach(q, m->hw_handler_name);
		}

632
		if (r < 0) {
633
			ti->error = "error attaching hardware handler";
634 635 636
			dm_put_device(ti, p->path.dev);
			goto bad;
		}
637 638 639 640 641 642 643 644 645 646 647

		if (m->hw_handler_params) {
			r = scsi_dh_set_params(q, m->hw_handler_params);
			if (r < 0) {
				ti->error = "unable to set hardware "
							"handler parameters";
				scsi_dh_detach(q);
				dm_put_device(ti, p->path.dev);
				goto bad;
			}
		}
648 649
	}

L
Linus Torvalds 已提交
650 651 652 653 654 655 656 657 658 659
	r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error);
	if (r) {
		dm_put_device(ti, p->path.dev);
		goto bad;
	}

	return p;

 bad:
	free_pgpath(p);
660
	return ERR_PTR(r);
L
Linus Torvalds 已提交
661 662 663
}

static struct priority_group *parse_priority_group(struct arg_set *as,
M
Micha³ Miros³aw 已提交
664
						   struct multipath *m)
L
Linus Torvalds 已提交
665 666
{
	static struct param _params[] = {
667 668
		{1, 1024, "invalid number of paths"},
		{0, 1024, "invalid number of selector args"}
L
Linus Torvalds 已提交
669 670 671 672 673
	};

	int r;
	unsigned i, nr_selector_args, nr_params;
	struct priority_group *pg;
M
Micha³ Miros³aw 已提交
674
	struct dm_target *ti = m->ti;
L
Linus Torvalds 已提交
675 676 677

	if (as->argc < 2) {
		as->argc = 0;
678 679
		ti->error = "not enough priority group arguments";
		return ERR_PTR(-EINVAL);
L
Linus Torvalds 已提交
680 681 682 683
	}

	pg = alloc_priority_group();
	if (!pg) {
684
		ti->error = "couldn't allocate priority group";
685
		return ERR_PTR(-ENOMEM);
L
Linus Torvalds 已提交
686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708
	}
	pg->m = m;

	r = parse_path_selector(as, pg, ti);
	if (r)
		goto bad;

	/*
	 * read the paths
	 */
	r = read_param(_params, shift(as), &pg->nr_pgpaths, &ti->error);
	if (r)
		goto bad;

	r = read_param(_params + 1, shift(as), &nr_selector_args, &ti->error);
	if (r)
		goto bad;

	nr_params = 1 + nr_selector_args;
	for (i = 0; i < pg->nr_pgpaths; i++) {
		struct pgpath *pgpath;
		struct arg_set path_args;

709 710
		if (as->argc < nr_params) {
			ti->error = "not enough path parameters";
711
			r = -EINVAL;
L
Linus Torvalds 已提交
712
			goto bad;
713
		}
L
Linus Torvalds 已提交
714 715 716 717 718

		path_args.argc = nr_params;
		path_args.argv = as->argv;

		pgpath = parse_path(&path_args, &pg->ps, ti);
719 720
		if (IS_ERR(pgpath)) {
			r = PTR_ERR(pgpath);
L
Linus Torvalds 已提交
721
			goto bad;
722
		}
L
Linus Torvalds 已提交
723 724 725 726 727 728 729 730 731 732

		pgpath->pg = pg;
		list_add_tail(&pgpath->list, &pg->pgpaths);
		consume(as, nr_params);
	}

	return pg;

 bad:
	free_priority_group(pg, ti);
733
	return ERR_PTR(r);
L
Linus Torvalds 已提交
734 735
}

M
Micha³ Miros³aw 已提交
736
static int parse_hw_handler(struct arg_set *as, struct multipath *m)
L
Linus Torvalds 已提交
737 738
{
	unsigned hw_argc;
739
	int ret;
M
Micha³ Miros³aw 已提交
740
	struct dm_target *ti = m->ti;
L
Linus Torvalds 已提交
741 742

	static struct param _params[] = {
743
		{0, 1024, "invalid number of hardware handler args"},
L
Linus Torvalds 已提交
744 745
	};

746
	if (read_param(_params, shift(as), &hw_argc, &ti->error))
L
Linus Torvalds 已提交
747 748 749 750 751
		return -EINVAL;

	if (!hw_argc)
		return 0;

752 753 754 755 756
	if (hw_argc > as->argc) {
		ti->error = "not enough arguments for hardware handler";
		return -EINVAL;
	}

757 758 759
	m->hw_handler_name = kstrdup(shift(as), GFP_KERNEL);
	request_module("scsi_dh_%s", m->hw_handler_name);
	if (scsi_dh_handler_exist(m->hw_handler_name) == 0) {
760
		ti->error = "unknown hardware handler type";
761 762
		ret = -EINVAL;
		goto fail;
L
Linus Torvalds 已提交
763
	}
764

765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780
	if (hw_argc > 1) {
		char *p;
		int i, j, len = 4;

		for (i = 0; i <= hw_argc - 2; i++)
			len += strlen(as->argv[i]) + 1;
		p = m->hw_handler_params = kzalloc(len, GFP_KERNEL);
		if (!p) {
			ti->error = "memory allocation failed";
			ret = -ENOMEM;
			goto fail;
		}
		j = sprintf(p, "%d", hw_argc - 1);
		for (i = 0, p+=j+1; i <= hw_argc - 2; i++, p+=j+1)
			j = sprintf(p, "%s", as->argv[i]);
	}
L
Linus Torvalds 已提交
781 782 783
	consume(as, hw_argc - 1);

	return 0;
784 785 786 787
fail:
	kfree(m->hw_handler_name);
	m->hw_handler_name = NULL;
	return ret;
L
Linus Torvalds 已提交
788 789
}

M
Micha³ Miros³aw 已提交
790
static int parse_features(struct arg_set *as, struct multipath *m)
L
Linus Torvalds 已提交
791 792 793
{
	int r;
	unsigned argc;
M
Micha³ Miros³aw 已提交
794
	struct dm_target *ti = m->ti;
D
Dave Wysochanski 已提交
795
	const char *param_name;
L
Linus Torvalds 已提交
796 797

	static struct param _params[] = {
798
		{0, 5, "invalid number of feature args"},
D
Dave Wysochanski 已提交
799
		{1, 50, "pg_init_retries must be between 1 and 50"},
800
		{0, 60000, "pg_init_delay_msecs must be between 0 and 60000"},
L
Linus Torvalds 已提交
801 802 803 804 805 806 807 808 809
	};

	r = read_param(_params, shift(as), &argc, &ti->error);
	if (r)
		return -EINVAL;

	if (!argc)
		return 0;

D
Dave Wysochanski 已提交
810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826
	do {
		param_name = shift(as);
		argc--;

		if (!strnicmp(param_name, MESG_STR("queue_if_no_path"))) {
			r = queue_if_no_path(m, 1, 0);
			continue;
		}

		if (!strnicmp(param_name, MESG_STR("pg_init_retries")) &&
		    (argc >= 1)) {
			r = read_param(_params + 1, shift(as),
				       &m->pg_init_retries, &ti->error);
			argc--;
			continue;
		}

827 828 829 830 831 832 833 834
		if (!strnicmp(param_name, MESG_STR("pg_init_delay_msecs")) &&
		    (argc >= 1)) {
			r = read_param(_params + 2, shift(as),
				       &m->pg_init_delay_msecs, &ti->error);
			argc--;
			continue;
		}

L
Linus Torvalds 已提交
835
		ti->error = "Unrecognised multipath feature request";
D
Dave Wysochanski 已提交
836 837 838 839
		r = -EINVAL;
	} while (argc && !r);

	return r;
L
Linus Torvalds 已提交
840 841 842 843 844 845 846
}

static int multipath_ctr(struct dm_target *ti, unsigned int argc,
			 char **argv)
{
	/* target parameters */
	static struct param _params[] = {
847 848
		{1, 1024, "invalid number of priority groups"},
		{1, 1024, "invalid initial priority group number"},
L
Linus Torvalds 已提交
849 850 851 852 853 854 855 856 857 858 859
	};

	int r;
	struct multipath *m;
	struct arg_set as;
	unsigned pg_count = 0;
	unsigned next_pg_num;

	as.argc = argc;
	as.argv = argv;

M
Micha³ Miros³aw 已提交
860
	m = alloc_multipath(ti);
L
Linus Torvalds 已提交
861
	if (!m) {
862
		ti->error = "can't allocate multipath";
L
Linus Torvalds 已提交
863 864 865
		return -EINVAL;
	}

M
Micha³ Miros³aw 已提交
866
	r = parse_features(&as, m);
L
Linus Torvalds 已提交
867 868 869
	if (r)
		goto bad;

M
Micha³ Miros³aw 已提交
870
	r = parse_hw_handler(&as, m);
L
Linus Torvalds 已提交
871 872 873 874 875 876 877 878 879 880 881 882 883 884 885
	if (r)
		goto bad;

	r = read_param(_params, shift(&as), &m->nr_priority_groups, &ti->error);
	if (r)
		goto bad;

	r = read_param(_params + 1, shift(&as), &next_pg_num, &ti->error);
	if (r)
		goto bad;

	/* parse the priority groups */
	while (as.argc) {
		struct priority_group *pg;

M
Micha³ Miros³aw 已提交
886
		pg = parse_priority_group(&as, m);
887 888
		if (IS_ERR(pg)) {
			r = PTR_ERR(pg);
L
Linus Torvalds 已提交
889 890 891 892 893 894 895 896 897 898 899 900
			goto bad;
		}

		m->nr_valid_paths += pg->nr_pgpaths;
		list_add_tail(&pg->list, &m->priority_groups);
		pg_count++;
		pg->pg_num = pg_count;
		if (!--next_pg_num)
			m->next_pg = pg;
	}

	if (pg_count != m->nr_priority_groups) {
901
		ti->error = "priority group count mismatch";
L
Linus Torvalds 已提交
902 903 904 905
		r = -EINVAL;
		goto bad;
	}

M
Mikulas Patocka 已提交
906
	ti->num_flush_requests = 1;
M
Mike Snitzer 已提交
907
	ti->num_discard_requests = 1;
M
Mikulas Patocka 已提交
908

L
Linus Torvalds 已提交
909 910 911 912 913 914 915
	return 0;

 bad:
	free_multipath(m);
	return r;
}

916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940
static void multipath_wait_for_pg_init_completion(struct multipath *m)
{
	DECLARE_WAITQUEUE(wait, current);
	unsigned long flags;

	add_wait_queue(&m->pg_init_wait, &wait);

	while (1) {
		set_current_state(TASK_UNINTERRUPTIBLE);

		spin_lock_irqsave(&m->lock, flags);
		if (!m->pg_init_in_progress) {
			spin_unlock_irqrestore(&m->lock, flags);
			break;
		}
		spin_unlock_irqrestore(&m->lock, flags);

		io_schedule();
	}
	set_current_state(TASK_RUNNING);

	remove_wait_queue(&m->pg_init_wait, &wait);
}

static void flush_multipath_work(struct multipath *m)
L
Linus Torvalds 已提交
941
{
942
	flush_workqueue(kmpath_handlerd);
943
	multipath_wait_for_pg_init_completion(m);
944
	flush_workqueue(kmultipathd);
T
Tejun Heo 已提交
945
	flush_work_sync(&m->trigger_event);
946 947 948 949 950 951
}

static void multipath_dtr(struct dm_target *ti)
{
	struct multipath *m = ti->private;

952
	flush_multipath_work(m);
L
Linus Torvalds 已提交
953 954 955 956
	free_multipath(m);
}

/*
957
 * Map cloned requests
L
Linus Torvalds 已提交
958
 */
959
static int multipath_map(struct dm_target *ti, struct request *clone,
L
Linus Torvalds 已提交
960 961 962
			 union map_info *map_context)
{
	int r;
A
Alasdair G Kergon 已提交
963
	struct dm_mpath_io *mpio;
L
Linus Torvalds 已提交
964 965
	struct multipath *m = (struct multipath *) ti->private;

966 967 968 969 970
	mpio = mempool_alloc(m->mpio_pool, GFP_ATOMIC);
	if (!mpio)
		/* ENOMEM, requeue */
		return DM_MAPIO_REQUEUE;
	memset(mpio, 0, sizeof(*mpio));
L
Linus Torvalds 已提交
971 972

	map_context->ptr = mpio;
973 974
	clone->cmd_flags |= REQ_FAILFAST_TRANSPORT;
	r = map_io(m, clone, mpio, 0);
975
	if (r < 0 || r == DM_MAPIO_REQUEUE)
L
Linus Torvalds 已提交
976 977 978 979 980 981 982 983 984 985 986 987 988 989 990
		mempool_free(mpio, m->mpio_pool);

	return r;
}

/*
 * Take a path out of use.
 */
static int fail_path(struct pgpath *pgpath)
{
	unsigned long flags;
	struct multipath *m = pgpath->pg->m;

	spin_lock_irqsave(&m->lock, flags);

991
	if (!pgpath->is_active)
L
Linus Torvalds 已提交
992 993
		goto out;

994
	DMWARN("Failing path %s.", pgpath->path.dev->name);
L
Linus Torvalds 已提交
995 996

	pgpath->pg->ps.type->fail_path(&pgpath->pg->ps, &pgpath->path);
997
	pgpath->is_active = 0;
L
Linus Torvalds 已提交
998 999 1000 1001 1002 1003 1004
	pgpath->fail_count++;

	m->nr_valid_paths--;

	if (pgpath == m->current_pgpath)
		m->current_pgpath = NULL;

M
Mike Anderson 已提交
1005 1006 1007
	dm_path_uevent(DM_UEVENT_PATH_FAILED, m->ti,
		      pgpath->path.dev->name, m->nr_valid_paths);

1008
	schedule_work(&m->trigger_event);
L
Linus Torvalds 已提交
1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026

out:
	spin_unlock_irqrestore(&m->lock, flags);

	return 0;
}

/*
 * Reinstate a previously-failed path
 */
static int reinstate_path(struct pgpath *pgpath)
{
	int r = 0;
	unsigned long flags;
	struct multipath *m = pgpath->pg->m;

	spin_lock_irqsave(&m->lock, flags);

1027
	if (pgpath->is_active)
L
Linus Torvalds 已提交
1028 1029
		goto out;

1030
	if (!pgpath->pg->ps.type->reinstate_path) {
L
Linus Torvalds 已提交
1031 1032 1033 1034 1035 1036 1037 1038 1039 1040
		DMWARN("Reinstate path not supported by path selector %s",
		       pgpath->pg->ps.type->name);
		r = -EINVAL;
		goto out;
	}

	r = pgpath->pg->ps.type->reinstate_path(&pgpath->pg->ps, &pgpath->path);
	if (r)
		goto out;

1041
	pgpath->is_active = 1;
L
Linus Torvalds 已提交
1042

1043 1044
	if (!m->nr_valid_paths++ && m->queue_size) {
		m->current_pgpath = NULL;
1045
		queue_work(kmultipathd, &m->process_queued_ios);
1046
	} else if (m->hw_handler_name && (m->current_pg == pgpath->pg)) {
1047
		if (queue_work(kmpath_handlerd, &pgpath->activate_path.work))
1048 1049
			m->pg_init_in_progress++;
	}
L
Linus Torvalds 已提交
1050

M
Mike Anderson 已提交
1051 1052 1053
	dm_path_uevent(DM_UEVENT_PATH_REINSTATED, m->ti,
		      pgpath->path.dev->name, m->nr_valid_paths);

1054
	schedule_work(&m->trigger_event);
L
Linus Torvalds 已提交
1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097

out:
	spin_unlock_irqrestore(&m->lock, flags);

	return r;
}

/*
 * Fail or reinstate all paths that match the provided struct dm_dev.
 */
static int action_dev(struct multipath *m, struct dm_dev *dev,
		      action_fn action)
{
	int r = 0;
	struct pgpath *pgpath;
	struct priority_group *pg;

	list_for_each_entry(pg, &m->priority_groups, list) {
		list_for_each_entry(pgpath, &pg->pgpaths, list) {
			if (pgpath->path.dev == dev)
				r = action(pgpath);
		}
	}

	return r;
}

/*
 * Temporarily try to avoid having to use the specified PG
 */
static void bypass_pg(struct multipath *m, struct priority_group *pg,
		      int bypassed)
{
	unsigned long flags;

	spin_lock_irqsave(&m->lock, flags);

	pg->bypassed = bypassed;
	m->current_pgpath = NULL;
	m->current_pg = NULL;

	spin_unlock_irqrestore(&m->lock, flags);

1098
	schedule_work(&m->trigger_event);
L
Linus Torvalds 已提交
1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127
}

/*
 * Switch to using the specified PG from the next I/O that gets mapped
 */
static int switch_pg_num(struct multipath *m, const char *pgstr)
{
	struct priority_group *pg;
	unsigned pgnum;
	unsigned long flags;

	if (!pgstr || (sscanf(pgstr, "%u", &pgnum) != 1) || !pgnum ||
	    (pgnum > m->nr_priority_groups)) {
		DMWARN("invalid PG number supplied to switch_pg_num");
		return -EINVAL;
	}

	spin_lock_irqsave(&m->lock, flags);
	list_for_each_entry(pg, &m->priority_groups, list) {
		pg->bypassed = 0;
		if (--pgnum)
			continue;

		m->current_pgpath = NULL;
		m->current_pg = NULL;
		m->next_pg = pg;
	}
	spin_unlock_irqrestore(&m->lock, flags);

1128
	schedule_work(&m->trigger_event);
L
Linus Torvalds 已提交
1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155
	return 0;
}

/*
 * Set/clear bypassed status of a PG.
 * PGs are numbered upwards from 1 in the order they were declared.
 */
static int bypass_pg_num(struct multipath *m, const char *pgstr, int bypassed)
{
	struct priority_group *pg;
	unsigned pgnum;

	if (!pgstr || (sscanf(pgstr, "%u", &pgnum) != 1) || !pgnum ||
	    (pgnum > m->nr_priority_groups)) {
		DMWARN("invalid PG number supplied to bypass_pg");
		return -EINVAL;
	}

	list_for_each_entry(pg, &m->priority_groups, list) {
		if (!--pgnum)
			break;
	}

	bypass_pg(m, pg, bypassed);
	return 0;
}

D
Dave Wysochanski 已提交
1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175
/*
 * Should we retry pg_init immediately?
 */
static int pg_init_limit_reached(struct multipath *m, struct pgpath *pgpath)
{
	unsigned long flags;
	int limit_reached = 0;

	spin_lock_irqsave(&m->lock, flags);

	if (m->pg_init_count <= m->pg_init_retries)
		m->pg_init_required = 1;
	else
		limit_reached = 1;

	spin_unlock_irqrestore(&m->lock, flags);

	return limit_reached;
}

1176
static void pg_init_done(void *data, int errors)
1177
{
1178
	struct pgpath *pgpath = data;
1179 1180 1181
	struct priority_group *pg = pgpath->pg;
	struct multipath *m = pg->m;
	unsigned long flags;
1182
	unsigned delay_retry = 0;
1183 1184 1185 1186 1187 1188 1189 1190 1191 1192

	/* device or driver problems */
	switch (errors) {
	case SCSI_DH_OK:
		break;
	case SCSI_DH_NOSYS:
		if (!m->hw_handler_name) {
			errors = 0;
			break;
		}
1193 1194
		DMERR("Could not failover the device: Handler scsi_dh_%s "
		      "Error %d.", m->hw_handler_name, errors);
1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207
		/*
		 * Fail path for now, so we do not ping pong
		 */
		fail_path(pgpath);
		break;
	case SCSI_DH_DEV_TEMP_BUSY:
		/*
		 * Probably doing something like FW upgrade on the
		 * controller so try the other pg.
		 */
		bypass_pg(m, pg, 1);
		break;
	case SCSI_DH_RETRY:
1208 1209
		/* Wait before retrying. */
		delay_retry = 1;
1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226
	case SCSI_DH_IMM_RETRY:
	case SCSI_DH_RES_TEMP_UNAVAIL:
		if (pg_init_limit_reached(m, pgpath))
			fail_path(pgpath);
		errors = 0;
		break;
	default:
		/*
		 * We probably do not want to fail the path for a device
		 * error, but this is what the old dm did. In future
		 * patches we can do more advanced handling.
		 */
		fail_path(pgpath);
	}

	spin_lock_irqsave(&m->lock, flags);
	if (errors) {
1227 1228 1229 1230 1231
		if (pgpath == m->current_pgpath) {
			DMERR("Could not failover device. Error %d.", errors);
			m->current_pgpath = NULL;
			m->current_pg = NULL;
		}
1232
	} else if (!m->pg_init_required)
1233 1234
		pg->bypassed = 0;

1235 1236 1237 1238 1239 1240 1241
	if (--m->pg_init_in_progress)
		/* Activations of other paths are still on going */
		goto out;

	if (!m->pg_init_required)
		m->queue_io = 0;

1242
	m->pg_init_delay_retry = delay_retry;
1243 1244
	queue_work(kmultipathd, &m->process_queued_ios);

1245 1246 1247 1248 1249
	/*
	 * Wake up any thread waiting to suspend.
	 */
	wake_up(&m->pg_init_wait);

1250
out:
1251 1252 1253
	spin_unlock_irqrestore(&m->lock, flags);
}

1254 1255
static void activate_path(struct work_struct *work)
{
1256
	struct pgpath *pgpath =
1257
		container_of(work, struct pgpath, activate_path.work);
1258

1259
	scsi_dh_activate(bdev_get_queue(pgpath->path.dev->bdev),
1260
				pg_init_done, pgpath);
1261 1262
}

L
Linus Torvalds 已提交
1263 1264 1265
/*
 * end_io handling
 */
1266
static int do_end_io(struct multipath *m, struct request *clone,
A
Alasdair G Kergon 已提交
1267
		     int error, struct dm_mpath_io *mpio)
L
Linus Torvalds 已提交
1268
{
1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280
	/*
	 * We don't queue any clone request inside the multipath target
	 * during end I/O handling, since those clone requests don't have
	 * bio clones.  If we queue them inside the multipath target,
	 * we need to make bio clones, that requires memory allocation.
	 * (See drivers/md/dm.c:end_clone_bio() about why the clone requests
	 *  don't have bio clones.)
	 * Instead of queueing the clone request here, we queue the original
	 * request into dm core, which will remake a clone request and
	 * clone bios for it and resubmit it later.
	 */
	int r = DM_ENDIO_REQUEUE;
1281
	unsigned long flags;
L
Linus Torvalds 已提交
1282

1283
	if (!error && !clone->errors)
L
Linus Torvalds 已提交
1284 1285
		return 0;	/* I/O complete */

1286 1287 1288
	if (error == -EOPNOTSUPP)
		return error;

M
Mike Snitzer 已提交
1289 1290 1291 1292 1293 1294 1295 1296 1297
	if (clone->cmd_flags & REQ_DISCARD)
		/*
		 * Pass all discard request failures up.
		 * FIXME: only fail_path if the discard failed due to a
		 * transport problem.  This requires precise understanding
		 * of the underlying failure (e.g. the SCSI sense).
		 */
		return error;

1298 1299
	if (mpio->pgpath)
		fail_path(mpio->pgpath);
L
Linus Torvalds 已提交
1300

1301
	spin_lock_irqsave(&m->lock, flags);
1302 1303
	if (!m->nr_valid_paths && !m->queue_if_no_path && !__must_push_back(m))
		r = -EIO;
1304
	spin_unlock_irqrestore(&m->lock, flags);
L
Linus Torvalds 已提交
1305

1306
	return r;
L
Linus Torvalds 已提交
1307 1308
}

1309
static int multipath_end_io(struct dm_target *ti, struct request *clone,
L
Linus Torvalds 已提交
1310 1311
			    int error, union map_info *map_context)
{
A
Alasdair G Kergon 已提交
1312 1313
	struct multipath *m = ti->private;
	struct dm_mpath_io *mpio = map_context->ptr;
L
Linus Torvalds 已提交
1314 1315 1316 1317
	struct pgpath *pgpath = mpio->pgpath;
	struct path_selector *ps;
	int r;

1318
	r  = do_end_io(m, clone, error, mpio);
L
Linus Torvalds 已提交
1319 1320 1321
	if (pgpath) {
		ps = &pgpath->pg->ps;
		if (ps->type->end_io)
1322
			ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes);
L
Linus Torvalds 已提交
1323
	}
1324
	mempool_free(mpio, m->mpio_pool);
L
Linus Torvalds 已提交
1325 1326 1327 1328 1329 1330

	return r;
}

/*
 * Suspend can't complete until all the I/O is processed so if
1331 1332 1333
 * the last path fails we must error any remaining I/O.
 * Note that if the freeze_bdev fails while suspending, the
 * queue_if_no_path state is lost - userspace should reset it.
L
Linus Torvalds 已提交
1334 1335 1336 1337 1338
 */
static void multipath_presuspend(struct dm_target *ti)
{
	struct multipath *m = (struct multipath *) ti->private;

1339
	queue_if_no_path(m, 0, 1);
L
Linus Torvalds 已提交
1340 1341
}

1342 1343
static void multipath_postsuspend(struct dm_target *ti)
{
1344 1345 1346
	struct multipath *m = ti->private;

	mutex_lock(&m->work_mutex);
1347
	flush_multipath_work(m);
1348
	mutex_unlock(&m->work_mutex);
1349 1350
}

1351 1352 1353
/*
 * Restore the queue_if_no_path setting.
 */
L
Linus Torvalds 已提交
1354 1355 1356 1357 1358 1359
static void multipath_resume(struct dm_target *ti)
{
	struct multipath *m = (struct multipath *) ti->private;
	unsigned long flags;

	spin_lock_irqsave(&m->lock, flags);
1360
	m->queue_if_no_path = m->saved_queue_if_no_path;
L
Linus Torvalds 已提交
1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394
	spin_unlock_irqrestore(&m->lock, flags);
}

/*
 * Info output has the following format:
 * num_multipath_feature_args [multipath_feature_args]*
 * num_handler_status_args [handler_status_args]*
 * num_groups init_group_number
 *            [A|D|E num_ps_status_args [ps_status_args]*
 *             num_paths num_selector_args
 *             [path_dev A|F fail_count [selector_args]* ]+ ]+
 *
 * Table output has the following format (identical to the constructor string):
 * num_feature_args [features_args]*
 * num_handler_args hw_handler [hw_handler_args]*
 * num_groups init_group_number
 *     [priority selector-name num_ps_args [ps_args]*
 *      num_paths num_selector_args [path_dev [selector_args]* ]+ ]+
 */
static int multipath_status(struct dm_target *ti, status_type_t type,
			    char *result, unsigned int maxlen)
{
	int sz = 0;
	unsigned long flags;
	struct multipath *m = (struct multipath *) ti->private;
	struct priority_group *pg;
	struct pgpath *p;
	unsigned pg_num;
	char state;

	spin_lock_irqsave(&m->lock, flags);

	/* Features */
	if (type == STATUSTYPE_INFO)
D
Dave Wysochanski 已提交
1395 1396 1397
		DMEMIT("2 %u %u ", m->queue_size, m->pg_init_count);
	else {
		DMEMIT("%u ", m->queue_if_no_path +
1398 1399
			      (m->pg_init_retries > 0) * 2 +
			      (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT) * 2);
D
Dave Wysochanski 已提交
1400 1401 1402 1403
		if (m->queue_if_no_path)
			DMEMIT("queue_if_no_path ");
		if (m->pg_init_retries)
			DMEMIT("pg_init_retries %u ", m->pg_init_retries);
1404 1405
		if (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT)
			DMEMIT("pg_init_delay_msecs %u ", m->pg_init_delay_msecs);
D
Dave Wysochanski 已提交
1406
	}
L
Linus Torvalds 已提交
1407

1408
	if (!m->hw_handler_name || type == STATUSTYPE_INFO)
L
Linus Torvalds 已提交
1409 1410
		DMEMIT("0 ");
	else
1411
		DMEMIT("1 %s ", m->hw_handler_name);
L
Linus Torvalds 已提交
1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447

	DMEMIT("%u ", m->nr_priority_groups);

	if (m->next_pg)
		pg_num = m->next_pg->pg_num;
	else if (m->current_pg)
		pg_num = m->current_pg->pg_num;
	else
			pg_num = 1;

	DMEMIT("%u ", pg_num);

	switch (type) {
	case STATUSTYPE_INFO:
		list_for_each_entry(pg, &m->priority_groups, list) {
			if (pg->bypassed)
				state = 'D';	/* Disabled */
			else if (pg == m->current_pg)
				state = 'A';	/* Currently Active */
			else
				state = 'E';	/* Enabled */

			DMEMIT("%c ", state);

			if (pg->ps.type->status)
				sz += pg->ps.type->status(&pg->ps, NULL, type,
							  result + sz,
							  maxlen - sz);
			else
				DMEMIT("0 ");

			DMEMIT("%u %u ", pg->nr_pgpaths,
			       pg->ps.type->info_args);

			list_for_each_entry(p, &pg->pgpaths, list) {
				DMEMIT("%s %s %u ", p->path.dev->name,
1448
				       p->is_active ? "A" : "F",
L
Linus Torvalds 已提交
1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489
				       p->fail_count);
				if (pg->ps.type->status)
					sz += pg->ps.type->status(&pg->ps,
					      &p->path, type, result + sz,
					      maxlen - sz);
			}
		}
		break;

	case STATUSTYPE_TABLE:
		list_for_each_entry(pg, &m->priority_groups, list) {
			DMEMIT("%s ", pg->ps.type->name);

			if (pg->ps.type->status)
				sz += pg->ps.type->status(&pg->ps, NULL, type,
							  result + sz,
							  maxlen - sz);
			else
				DMEMIT("0 ");

			DMEMIT("%u %u ", pg->nr_pgpaths,
			       pg->ps.type->table_args);

			list_for_each_entry(p, &pg->pgpaths, list) {
				DMEMIT("%s ", p->path.dev->name);
				if (pg->ps.type->status)
					sz += pg->ps.type->status(&pg->ps,
					      &p->path, type, result + sz,
					      maxlen - sz);
			}
		}
		break;
	}

	spin_unlock_irqrestore(&m->lock, flags);

	return 0;
}

static int multipath_message(struct dm_target *ti, unsigned argc, char **argv)
{
1490
	int r = -EINVAL;
L
Linus Torvalds 已提交
1491 1492 1493 1494
	struct dm_dev *dev;
	struct multipath *m = (struct multipath *) ti->private;
	action_fn action;

1495 1496
	mutex_lock(&m->work_mutex);

1497 1498 1499 1500 1501
	if (dm_suspended(ti)) {
		r = -EBUSY;
		goto out;
	}

L
Linus Torvalds 已提交
1502
	if (argc == 1) {
1503 1504 1505 1506 1507 1508 1509
		if (!strnicmp(argv[0], MESG_STR("queue_if_no_path"))) {
			r = queue_if_no_path(m, 1, 0);
			goto out;
		} else if (!strnicmp(argv[0], MESG_STR("fail_if_no_path"))) {
			r = queue_if_no_path(m, 0, 0);
			goto out;
		}
L
Linus Torvalds 已提交
1510 1511
	}

1512 1513 1514 1515
	if (argc != 2) {
		DMWARN("Unrecognised multipath message received.");
		goto out;
	}
L
Linus Torvalds 已提交
1516

1517 1518 1519 1520 1521 1522 1523 1524 1525 1526
	if (!strnicmp(argv[0], MESG_STR("disable_group"))) {
		r = bypass_pg_num(m, argv[1], 1);
		goto out;
	} else if (!strnicmp(argv[0], MESG_STR("enable_group"))) {
		r = bypass_pg_num(m, argv[1], 0);
		goto out;
	} else if (!strnicmp(argv[0], MESG_STR("switch_group"))) {
		r = switch_pg_num(m, argv[1]);
		goto out;
	} else if (!strnicmp(argv[0], MESG_STR("reinstate_path")))
L
Linus Torvalds 已提交
1527 1528 1529
		action = reinstate_path;
	else if (!strnicmp(argv[0], MESG_STR("fail_path")))
		action = fail_path;
1530 1531 1532 1533
	else {
		DMWARN("Unrecognised multipath message received.");
		goto out;
	}
L
Linus Torvalds 已提交
1534

1535
	r = dm_get_device(ti, argv[1], dm_table_get_mode(ti->table), &dev);
L
Linus Torvalds 已提交
1536
	if (r) {
1537
		DMWARN("message: error getting device %s",
L
Linus Torvalds 已提交
1538
		       argv[1]);
1539
		goto out;
L
Linus Torvalds 已提交
1540 1541 1542 1543 1544 1545
	}

	r = action_dev(m, dev, action);

	dm_put_device(ti, dev);

1546 1547
out:
	mutex_unlock(&m->work_mutex);
L
Linus Torvalds 已提交
1548 1549 1550
	return r;
}

1551
static int multipath_ioctl(struct dm_target *ti, unsigned int cmd,
M
Milan Broz 已提交
1552 1553 1554 1555
			   unsigned long arg)
{
	struct multipath *m = (struct multipath *) ti->private;
	struct block_device *bdev = NULL;
1556
	fmode_t mode = 0;
M
Milan Broz 已提交
1557 1558 1559 1560 1561 1562
	unsigned long flags;
	int r = 0;

	spin_lock_irqsave(&m->lock, flags);

	if (!m->current_pgpath)
1563
		__choose_pgpath(m, 0);
M
Milan Broz 已提交
1564

1565
	if (m->current_pgpath) {
M
Milan Broz 已提交
1566
		bdev = m->current_pgpath->path.dev->bdev;
1567
		mode = m->current_pgpath->path.dev->mode;
1568
	}
M
Milan Broz 已提交
1569 1570 1571 1572 1573 1574 1575 1576

	if (m->queue_io)
		r = -EAGAIN;
	else if (!bdev)
		r = -EIO;

	spin_unlock_irqrestore(&m->lock, flags);

1577
	return r ? : __blkdev_driver_ioctl(bdev, mode, cmd, arg);
M
Milan Broz 已提交
1578 1579
}

1580 1581 1582 1583 1584 1585 1586 1587 1588 1589
static int multipath_iterate_devices(struct dm_target *ti,
				     iterate_devices_callout_fn fn, void *data)
{
	struct multipath *m = ti->private;
	struct priority_group *pg;
	struct pgpath *p;
	int ret = 0;

	list_for_each_entry(pg, &m->priority_groups, list) {
		list_for_each_entry(p, &pg->pgpaths, list) {
1590
			ret = fn(ti, p->path.dev, ti->begin, ti->len, data);
1591 1592 1593 1594 1595 1596 1597 1598 1599
			if (ret)
				goto out;
		}
	}

out:
	return ret;
}

1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668
static int __pgpath_busy(struct pgpath *pgpath)
{
	struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev);

	return dm_underlying_device_busy(q);
}

/*
 * We return "busy", only when we can map I/Os but underlying devices
 * are busy (so even if we map I/Os now, the I/Os will wait on
 * the underlying queue).
 * In other words, if we want to kill I/Os or queue them inside us
 * due to map unavailability, we don't return "busy".  Otherwise,
 * dm core won't give us the I/Os and we can't do what we want.
 */
static int multipath_busy(struct dm_target *ti)
{
	int busy = 0, has_active = 0;
	struct multipath *m = ti->private;
	struct priority_group *pg;
	struct pgpath *pgpath;
	unsigned long flags;

	spin_lock_irqsave(&m->lock, flags);

	/* Guess which priority_group will be used at next mapping time */
	if (unlikely(!m->current_pgpath && m->next_pg))
		pg = m->next_pg;
	else if (likely(m->current_pg))
		pg = m->current_pg;
	else
		/*
		 * We don't know which pg will be used at next mapping time.
		 * We don't call __choose_pgpath() here to avoid to trigger
		 * pg_init just by busy checking.
		 * So we don't know whether underlying devices we will be using
		 * at next mapping time are busy or not. Just try mapping.
		 */
		goto out;

	/*
	 * If there is one non-busy active path at least, the path selector
	 * will be able to select it. So we consider such a pg as not busy.
	 */
	busy = 1;
	list_for_each_entry(pgpath, &pg->pgpaths, list)
		if (pgpath->is_active) {
			has_active = 1;

			if (!__pgpath_busy(pgpath)) {
				busy = 0;
				break;
			}
		}

	if (!has_active)
		/*
		 * No active path in this pg, so this pg won't be used and
		 * the current_pg will be changed at next mapping time.
		 * We need to try mapping to determine it.
		 */
		busy = 0;

out:
	spin_unlock_irqrestore(&m->lock, flags);

	return busy;
}

L
Linus Torvalds 已提交
1669 1670 1671 1672 1673
/*-----------------------------------------------------------------
 * Module setup
 *---------------------------------------------------------------*/
static struct target_type multipath_target = {
	.name = "multipath",
1674
	.version = {1, 2, 0},
L
Linus Torvalds 已提交
1675 1676 1677
	.module = THIS_MODULE,
	.ctr = multipath_ctr,
	.dtr = multipath_dtr,
1678 1679
	.map_rq = multipath_map,
	.rq_end_io = multipath_end_io,
L
Linus Torvalds 已提交
1680
	.presuspend = multipath_presuspend,
1681
	.postsuspend = multipath_postsuspend,
L
Linus Torvalds 已提交
1682 1683 1684
	.resume = multipath_resume,
	.status = multipath_status,
	.message = multipath_message,
M
Milan Broz 已提交
1685
	.ioctl  = multipath_ioctl,
1686
	.iterate_devices = multipath_iterate_devices,
1687
	.busy = multipath_busy,
L
Linus Torvalds 已提交
1688 1689 1690 1691 1692 1693 1694
};

static int __init dm_multipath_init(void)
{
	int r;

	/* allocate a slab for the dm_ios */
A
Alasdair G Kergon 已提交
1695
	_mpio_cache = KMEM_CACHE(dm_mpath_io, 0);
L
Linus Torvalds 已提交
1696 1697 1698 1699 1700
	if (!_mpio_cache)
		return -ENOMEM;

	r = dm_register_target(&multipath_target);
	if (r < 0) {
1701
		DMERR("register failed %d", r);
L
Linus Torvalds 已提交
1702 1703 1704 1705
		kmem_cache_destroy(_mpio_cache);
		return -EINVAL;
	}

1706
	kmultipathd = alloc_workqueue("kmpathd", WQ_MEM_RECLAIM, 0);
1707
	if (!kmultipathd) {
1708
		DMERR("failed to create workqueue kmpathd");
1709 1710 1711 1712 1713
		dm_unregister_target(&multipath_target);
		kmem_cache_destroy(_mpio_cache);
		return -ENOMEM;
	}

1714 1715 1716 1717 1718 1719
	/*
	 * A separate workqueue is used to handle the device handlers
	 * to avoid overloading existing workqueue. Overloading the
	 * old workqueue would also create a bottleneck in the
	 * path of the storage hardware device activation.
	 */
1720 1721
	kmpath_handlerd = alloc_ordered_workqueue("kmpath_handlerd",
						  WQ_MEM_RECLAIM);
1722 1723 1724 1725 1726 1727 1728 1729
	if (!kmpath_handlerd) {
		DMERR("failed to create workqueue kmpath_handlerd");
		destroy_workqueue(kmultipathd);
		dm_unregister_target(&multipath_target);
		kmem_cache_destroy(_mpio_cache);
		return -ENOMEM;
	}

1730
	DMINFO("version %u.%u.%u loaded",
L
Linus Torvalds 已提交
1731 1732 1733 1734 1735 1736 1737 1738
	       multipath_target.version[0], multipath_target.version[1],
	       multipath_target.version[2]);

	return r;
}

static void __exit dm_multipath_exit(void)
{
1739
	destroy_workqueue(kmpath_handlerd);
1740 1741
	destroy_workqueue(kmultipathd);

1742
	dm_unregister_target(&multipath_target);
L
Linus Torvalds 已提交
1743 1744 1745 1746 1747 1748 1749 1750 1751
	kmem_cache_destroy(_mpio_cache);
}

module_init(dm_multipath_init);
module_exit(dm_multipath_exit);

MODULE_DESCRIPTION(DM_NAME " multipath target");
MODULE_AUTHOR("Sistina Software <dm-devel@redhat.com>");
MODULE_LICENSE("GPL");