raid5.c 167.2 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4
/*
 * raid5.c : Multiple Devices driver for Linux
 *	   Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman
 *	   Copyright (C) 1999, 2000 Ingo Molnar
5
 *	   Copyright (C) 2002, 2003 H. Peter Anvin
L
Linus Torvalds 已提交
6
 *
7 8 9
 * RAID-4/5/6 management functions.
 * Thanks to Penguin Computing for making the RAID-6 development possible
 * by donating a test server!
L
Linus Torvalds 已提交
10 11 12 13 14 15 16 17 18 19 20
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2, or (at your option)
 * any later version.
 *
 * You should have received a copy of the GNU General Public License
 * (for example /usr/src/linux/COPYING); if not, write to the Free
 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

21 22 23 24 25 26 27 28 29
/*
 * BITMAP UNPLUGGING:
 *
 * The sequencing for updating the bitmap reliably is a little
 * subtle (and I got it wrong the first time) so it deserves some
 * explanation.
 *
 * We group bitmap updates into batches.  Each batch has a number.
 * We may write out several batches at once, but that isn't very important.
30 31
 * conf->seq_write is the number of the last batch successfully written.
 * conf->seq_flush is the number of the last batch that was closed to
32 33 34
 *    new additions.
 * When we discover that we will need to write to any block in a stripe
 * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq
35
 * the number of the batch it will be in. This is seq_flush+1.
36 37 38 39 40 41 42 43 44
 * When we are ready to do a write, if that batch hasn't been written yet,
 *   we plug the array and queue the stripe for later.
 * When an unplug happens, we increment bm_flush, thus closing the current
 *   batch.
 * When we notice that bm_flush > bm_write, we write out all pending updates
 * to the bitmap, and advance bm_write to where bm_flush was.
 * This may occasionally write a bit out twice, but is sure never to
 * miss any bits.
 */
L
Linus Torvalds 已提交
45

46
#include <linux/blkdev.h>
47
#include <linux/kthread.h>
48
#include <linux/raid/pq.h>
49
#include <linux/async_tx.h>
50
#include <linux/async.h>
51
#include <linux/seq_file.h>
52
#include <linux/cpu.h>
53
#include <linux/slab.h>
54
#include "md.h"
55
#include "raid5.h"
56
#include "raid0.h"
57
#include "bitmap.h"
58

L
Linus Torvalds 已提交
59 60 61 62 63 64 65 66 67
/*
 * Stripe cache
 */

#define NR_STRIPES		256
#define STRIPE_SIZE		PAGE_SIZE
#define STRIPE_SHIFT		(PAGE_SHIFT - 9)
#define STRIPE_SECTORS		(STRIPE_SIZE>>9)
#define	IO_THRESHOLD		1
68
#define BYPASS_THRESHOLD	1
69
#define NR_HASH			(PAGE_SIZE / sizeof(struct hlist_head))
L
Linus Torvalds 已提交
70 71
#define HASH_MASK		(NR_HASH - 1)

72
#define stripe_hash(conf, sect)	(&((conf)->stripe_hashtbl[((sect) >> STRIPE_SHIFT) & HASH_MASK]))
L
Linus Torvalds 已提交
73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93

/* bio's attached to a stripe+device for I/O are linked together in bi_sector
 * order without overlap.  There may be several bio's per stripe+device, and
 * a bio could span several devices.
 * When walking this list for a particular stripe+device, we must never proceed
 * beyond a bio that extends past this device, as the next bio might no longer
 * be valid.
 * This macro is used to determine the 'next' bio in the list, given the sector
 * of the current stripe+device
 */
#define r5_next_bio(bio, sect) ( ( (bio)->bi_sector + ((bio)->bi_size>>9) < sect + STRIPE_SECTORS) ? (bio)->bi_next : NULL)
/*
 * The following can be used to debug the driver
 */
#define RAID5_PARANOIA	1
#if RAID5_PARANOIA && defined(CONFIG_SMP)
# define CHECK_DEVLOCK() assert_spin_locked(&conf->device_lock)
#else
# define CHECK_DEVLOCK()
#endif

94
#ifdef DEBUG
L
Linus Torvalds 已提交
95 96 97 98
#define inline
#define __inline__
#endif

99 100
#define printk_rl(args...) ((void) (printk_ratelimit() && printk(args)))

101
/*
102 103
 * We maintain a biased count of active stripes in the bottom 16 bits of
 * bi_phys_segments, and a count of processed stripes in the upper 16 bits
104 105 106
 */
static inline int raid5_bi_phys_segments(struct bio *bio)
{
107
	return bio->bi_phys_segments & 0xffff;
108 109 110 111
}

static inline int raid5_bi_hw_segments(struct bio *bio)
{
112
	return (bio->bi_phys_segments >> 16) & 0xffff;
113 114 115 116 117 118 119 120 121 122 123 124 125
}

static inline int raid5_dec_bi_phys_segments(struct bio *bio)
{
	--bio->bi_phys_segments;
	return raid5_bi_phys_segments(bio);
}

static inline int raid5_dec_bi_hw_segments(struct bio *bio)
{
	unsigned short val = raid5_bi_hw_segments(bio);

	--val;
126
	bio->bi_phys_segments = (val << 16) | raid5_bi_phys_segments(bio);
127 128 129 130 131
	return val;
}

static inline void raid5_set_bi_hw_segments(struct bio *bio, unsigned int cnt)
{
132
	bio->bi_phys_segments = raid5_bi_phys_segments(bio) | (cnt << 16);
133 134
}

135 136 137
/* Find first data disk in a raid6 stripe */
static inline int raid6_d0(struct stripe_head *sh)
{
138 139 140 141
	if (sh->ddf_layout)
		/* ddf always start from first device */
		return 0;
	/* md starts just after Q block */
142 143 144 145 146
	if (sh->qd_idx == sh->disks - 1)
		return 0;
	else
		return sh->qd_idx + 1;
}
147 148 149 150 151
static inline int raid6_next_disk(int disk, int raid_disks)
{
	disk++;
	return (disk < raid_disks) ? disk : 0;
}
152

153 154 155 156 157
/* When walking through the disks in a raid5, starting at raid6_d0,
 * We need to map each disk to a 'slot', where the data disks are slot
 * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk
 * is raid_disks-1.  This help does that mapping.
 */
158 159
static int raid6_idx_to_slot(int idx, struct stripe_head *sh,
			     int *count, int syndrome_disks)
160
{
161
	int slot = *count;
162

163
	if (sh->ddf_layout)
164
		(*count)++;
165
	if (idx == sh->pd_idx)
166
		return syndrome_disks;
167
	if (idx == sh->qd_idx)
168
		return syndrome_disks + 1;
169
	if (!sh->ddf_layout)
170
		(*count)++;
171 172 173
	return slot;
}

174 175 176 177 178 179 180 181
static void return_io(struct bio *return_bi)
{
	struct bio *bi = return_bi;
	while (bi) {

		return_bi = bi->bi_next;
		bi->bi_next = NULL;
		bi->bi_size = 0;
182
		bio_endio(bi, 0);
183 184 185 186
		bi = return_bi;
	}
}

L
Linus Torvalds 已提交
187 188
static void print_raid5_conf (raid5_conf_t *conf);

189 190 191 192 193 194 195
static int stripe_operations_active(struct stripe_head *sh)
{
	return sh->check_state || sh->reconstruct_state ||
	       test_bit(STRIPE_BIOFILL_RUN, &sh->state) ||
	       test_bit(STRIPE_COMPUTE_RUN, &sh->state);
}

196
static void __release_stripe(raid5_conf_t *conf, struct stripe_head *sh)
L
Linus Torvalds 已提交
197 198
{
	if (atomic_dec_and_test(&sh->count)) {
199 200
		BUG_ON(!list_empty(&sh->lru));
		BUG_ON(atomic_read(&conf->active_stripes)==0);
L
Linus Torvalds 已提交
201
		if (test_bit(STRIPE_HANDLE, &sh->state)) {
N
NeilBrown 已提交
202
			if (test_bit(STRIPE_DELAYED, &sh->state))
L
Linus Torvalds 已提交
203
				list_add_tail(&sh->lru, &conf->delayed_list);
N
NeilBrown 已提交
204 205
			else if (test_bit(STRIPE_BIT_DELAY, &sh->state) &&
				   sh->bm_seq - conf->seq_write > 0)
206
				list_add_tail(&sh->lru, &conf->bitmap_list);
N
NeilBrown 已提交
207
			else {
208
				clear_bit(STRIPE_BIT_DELAY, &sh->state);
L
Linus Torvalds 已提交
209
				list_add_tail(&sh->lru, &conf->handle_list);
210
			}
L
Linus Torvalds 已提交
211 212
			md_wakeup_thread(conf->mddev->thread);
		} else {
213
			BUG_ON(stripe_operations_active(sh));
L
Linus Torvalds 已提交
214 215 216 217 218 219
			if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
				atomic_dec(&conf->preread_active_stripes);
				if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD)
					md_wakeup_thread(conf->mddev->thread);
			}
			atomic_dec(&conf->active_stripes);
220 221
			if (!test_bit(STRIPE_EXPANDING, &sh->state)) {
				list_add_tail(&sh->lru, &conf->inactive_list);
L
Linus Torvalds 已提交
222
				wake_up(&conf->wait_for_stripe);
223 224
				if (conf->retry_read_aligned)
					md_wakeup_thread(conf->mddev->thread);
225
			}
L
Linus Torvalds 已提交
226 227 228
		}
	}
}
229

L
Linus Torvalds 已提交
230 231 232 233
static void release_stripe(struct stripe_head *sh)
{
	raid5_conf_t *conf = sh->raid_conf;
	unsigned long flags;
234

L
Linus Torvalds 已提交
235 236 237 238 239
	spin_lock_irqsave(&conf->device_lock, flags);
	__release_stripe(conf, sh);
	spin_unlock_irqrestore(&conf->device_lock, flags);
}

240
static inline void remove_hash(struct stripe_head *sh)
L
Linus Torvalds 已提交
241
{
242 243
	pr_debug("remove_hash(), stripe %llu\n",
		(unsigned long long)sh->sector);
L
Linus Torvalds 已提交
244

245
	hlist_del_init(&sh->hash);
L
Linus Torvalds 已提交
246 247
}

248
static inline void insert_hash(raid5_conf_t *conf, struct stripe_head *sh)
L
Linus Torvalds 已提交
249
{
250
	struct hlist_head *hp = stripe_hash(conf, sh->sector);
L
Linus Torvalds 已提交
251

252 253
	pr_debug("insert_hash(), stripe %llu\n",
		(unsigned long long)sh->sector);
L
Linus Torvalds 已提交
254 255

	CHECK_DEVLOCK();
256
	hlist_add_head(&sh->hash, hp);
L
Linus Torvalds 已提交
257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277
}


/* find an idle stripe, make sure it is unhashed, and return it. */
static struct stripe_head *get_free_stripe(raid5_conf_t *conf)
{
	struct stripe_head *sh = NULL;
	struct list_head *first;

	CHECK_DEVLOCK();
	if (list_empty(&conf->inactive_list))
		goto out;
	first = conf->inactive_list.next;
	sh = list_entry(first, struct stripe_head, lru);
	list_del_init(first);
	remove_hash(sh);
	atomic_inc(&conf->active_stripes);
out:
	return sh;
}

278
static void shrink_buffers(struct stripe_head *sh)
L
Linus Torvalds 已提交
279 280 281
{
	struct page *p;
	int i;
282
	int num = sh->raid_conf->pool_size;
L
Linus Torvalds 已提交
283

284
	for (i = 0; i < num ; i++) {
L
Linus Torvalds 已提交
285 286 287 288
		p = sh->dev[i].page;
		if (!p)
			continue;
		sh->dev[i].page = NULL;
289
		put_page(p);
L
Linus Torvalds 已提交
290 291 292
	}
}

293
static int grow_buffers(struct stripe_head *sh)
L
Linus Torvalds 已提交
294 295
{
	int i;
296
	int num = sh->raid_conf->pool_size;
L
Linus Torvalds 已提交
297

298
	for (i = 0; i < num; i++) {
L
Linus Torvalds 已提交
299 300 301 302 303 304 305 306 307 308
		struct page *page;

		if (!(page = alloc_page(GFP_KERNEL))) {
			return 1;
		}
		sh->dev[i].page = page;
	}
	return 0;
}

309
static void raid5_build_block(struct stripe_head *sh, int i, int previous);
310 311
static void stripe_set_idx(sector_t stripe, raid5_conf_t *conf, int previous,
			    struct stripe_head *sh);
L
Linus Torvalds 已提交
312

313
static void init_stripe(struct stripe_head *sh, sector_t sector, int previous)
L
Linus Torvalds 已提交
314 315
{
	raid5_conf_t *conf = sh->raid_conf;
316
	int i;
L
Linus Torvalds 已提交
317

318 319
	BUG_ON(atomic_read(&sh->count) != 0);
	BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));
320
	BUG_ON(stripe_operations_active(sh));
321

L
Linus Torvalds 已提交
322
	CHECK_DEVLOCK();
323
	pr_debug("init_stripe called, stripe %llu\n",
L
Linus Torvalds 已提交
324 325 326
		(unsigned long long)sh->sector);

	remove_hash(sh);
327

328
	sh->generation = conf->generation - previous;
329
	sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks;
L
Linus Torvalds 已提交
330
	sh->sector = sector;
331
	stripe_set_idx(sector, conf, previous, sh);
L
Linus Torvalds 已提交
332 333
	sh->state = 0;

334 335

	for (i = sh->disks; i--; ) {
L
Linus Torvalds 已提交
336 337
		struct r5dev *dev = &sh->dev[i];

338
		if (dev->toread || dev->read || dev->towrite || dev->written ||
L
Linus Torvalds 已提交
339
		    test_bit(R5_LOCKED, &dev->flags)) {
340
			printk(KERN_ERR "sector=%llx i=%d %p %p %p %p %d\n",
L
Linus Torvalds 已提交
341
			       (unsigned long long)sh->sector, i, dev->toread,
342
			       dev->read, dev->towrite, dev->written,
L
Linus Torvalds 已提交
343 344 345 346
			       test_bit(R5_LOCKED, &dev->flags));
			BUG();
		}
		dev->flags = 0;
347
		raid5_build_block(sh, i, previous);
L
Linus Torvalds 已提交
348 349 350 351
	}
	insert_hash(conf, sh);
}

352 353
static struct stripe_head *__find_stripe(raid5_conf_t *conf, sector_t sector,
					 short generation)
L
Linus Torvalds 已提交
354 355
{
	struct stripe_head *sh;
356
	struct hlist_node *hn;
L
Linus Torvalds 已提交
357 358

	CHECK_DEVLOCK();
359
	pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector);
360
	hlist_for_each_entry(sh, hn, stripe_hash(conf, sector), hash)
361
		if (sh->sector == sector && sh->generation == generation)
L
Linus Torvalds 已提交
362
			return sh;
363
	pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector);
L
Linus Torvalds 已提交
364 365 366
	return NULL;
}

367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433
/*
 * Need to check if array has failed when deciding whether to:
 *  - start an array
 *  - remove non-faulty devices
 *  - add a spare
 *  - allow a reshape
 * This determination is simple when no reshape is happening.
 * However if there is a reshape, we need to carefully check
 * both the before and after sections.
 * This is because some failed devices may only affect one
 * of the two sections, and some non-in_sync devices may
 * be insync in the section most affected by failed devices.
 */
static int has_failed(raid5_conf_t *conf)
{
	int degraded;
	int i;
	if (conf->mddev->reshape_position == MaxSector)
		return conf->mddev->degraded > conf->max_degraded;

	rcu_read_lock();
	degraded = 0;
	for (i = 0; i < conf->previous_raid_disks; i++) {
		mdk_rdev_t *rdev = rcu_dereference(conf->disks[i].rdev);
		if (!rdev || test_bit(Faulty, &rdev->flags))
			degraded++;
		else if (test_bit(In_sync, &rdev->flags))
			;
		else
			/* not in-sync or faulty.
			 * If the reshape increases the number of devices,
			 * this is being recovered by the reshape, so
			 * this 'previous' section is not in_sync.
			 * If the number of devices is being reduced however,
			 * the device can only be part of the array if
			 * we are reverting a reshape, so this section will
			 * be in-sync.
			 */
			if (conf->raid_disks >= conf->previous_raid_disks)
				degraded++;
	}
	rcu_read_unlock();
	if (degraded > conf->max_degraded)
		return 1;
	rcu_read_lock();
	degraded = 0;
	for (i = 0; i < conf->raid_disks; i++) {
		mdk_rdev_t *rdev = rcu_dereference(conf->disks[i].rdev);
		if (!rdev || test_bit(Faulty, &rdev->flags))
			degraded++;
		else if (test_bit(In_sync, &rdev->flags))
			;
		else
			/* not in-sync or faulty.
			 * If reshape increases the number of devices, this
			 * section has already been recovered, else it
			 * almost certainly hasn't.
			 */
			if (conf->raid_disks <= conf->previous_raid_disks)
				degraded++;
	}
	rcu_read_unlock();
	if (degraded > conf->max_degraded)
		return 1;
	return 0;
}

434 435
static struct stripe_head *
get_active_stripe(raid5_conf_t *conf, sector_t sector,
436
		  int previous, int noblock, int noquiesce)
L
Linus Torvalds 已提交
437 438 439
{
	struct stripe_head *sh;

440
	pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector);
L
Linus Torvalds 已提交
441 442 443 444

	spin_lock_irq(&conf->device_lock);

	do {
445
		wait_event_lock_irq(conf->wait_for_stripe,
446
				    conf->quiesce == 0 || noquiesce,
447
				    conf->device_lock, /* nothing */);
448
		sh = __find_stripe(conf, sector, conf->generation - previous);
L
Linus Torvalds 已提交
449 450 451 452 453 454 455 456 457
		if (!sh) {
			if (!conf->inactive_blocked)
				sh = get_free_stripe(conf);
			if (noblock && sh == NULL)
				break;
			if (!sh) {
				conf->inactive_blocked = 1;
				wait_event_lock_irq(conf->wait_for_stripe,
						    !list_empty(&conf->inactive_list) &&
458 459
						    (atomic_read(&conf->active_stripes)
						     < (conf->max_nr_stripes *3/4)
L
Linus Torvalds 已提交
460 461
						     || !conf->inactive_blocked),
						    conf->device_lock,
462
						    );
L
Linus Torvalds 已提交
463 464
				conf->inactive_blocked = 0;
			} else
465
				init_stripe(sh, sector, previous);
L
Linus Torvalds 已提交
466 467
		} else {
			if (atomic_read(&sh->count)) {
468 469
				BUG_ON(!list_empty(&sh->lru)
				    && !test_bit(STRIPE_EXPANDING, &sh->state));
L
Linus Torvalds 已提交
470 471 472
			} else {
				if (!test_bit(STRIPE_HANDLE, &sh->state))
					atomic_inc(&conf->active_stripes);
473 474
				if (list_empty(&sh->lru) &&
				    !test_bit(STRIPE_EXPANDING, &sh->state))
475 476
					BUG();
				list_del_init(&sh->lru);
L
Linus Torvalds 已提交
477 478 479 480 481 482 483 484 485 486 487
			}
		}
	} while (sh == NULL);

	if (sh)
		atomic_inc(&sh->count);

	spin_unlock_irq(&conf->device_lock);
	return sh;
}

488 489 490 491
static void
raid5_end_read_request(struct bio *bi, int error);
static void
raid5_end_write_request(struct bio *bi, int error);
492

493
static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)
494 495 496 497 498 499 500 501 502 503
{
	raid5_conf_t *conf = sh->raid_conf;
	int i, disks = sh->disks;

	might_sleep();

	for (i = disks; i--; ) {
		int rw;
		struct bio *bi;
		mdk_rdev_t *rdev;
T
Tejun Heo 已提交
504 505 506 507 508 509
		if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) {
			if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags))
				rw = WRITE_FUA;
			else
				rw = WRITE;
		} else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags))
510 511 512 513 514 515 516
			rw = READ;
		else
			continue;

		bi = &sh->dev[i].req;

		bi->bi_rw = rw;
517
		if (rw & WRITE)
518 519 520 521 522 523 524 525 526 527 528 529 530
			bi->bi_end_io = raid5_end_write_request;
		else
			bi->bi_end_io = raid5_end_read_request;

		rcu_read_lock();
		rdev = rcu_dereference(conf->disks[i].rdev);
		if (rdev && test_bit(Faulty, &rdev->flags))
			rdev = NULL;
		if (rdev)
			atomic_inc(&rdev->nr_pending);
		rcu_read_unlock();

		if (rdev) {
531
			if (s->syncing || s->expanding || s->expanded)
532 533
				md_sync_acct(rdev->bdev, STRIPE_SECTORS);

D
Dan Williams 已提交
534 535
			set_bit(STRIPE_IO_STARTED, &sh->state);

536 537
			bi->bi_bdev = rdev->bdev;
			pr_debug("%s: for %llu schedule op %ld on disc %d\n",
538
				__func__, (unsigned long long)sh->sector,
539 540 541 542 543 544 545 546 547 548 549 550
				bi->bi_rw, i);
			atomic_inc(&sh->count);
			bi->bi_sector = sh->sector + rdev->data_offset;
			bi->bi_flags = 1 << BIO_UPTODATE;
			bi->bi_vcnt = 1;
			bi->bi_max_vecs = 1;
			bi->bi_idx = 0;
			bi->bi_io_vec = &sh->dev[i].vec;
			bi->bi_io_vec[0].bv_len = STRIPE_SIZE;
			bi->bi_io_vec[0].bv_offset = 0;
			bi->bi_size = STRIPE_SIZE;
			bi->bi_next = NULL;
551
			if ((rw & WRITE) &&
552 553 554 555 556
			    test_bit(R5_ReWrite, &sh->dev[i].flags))
				atomic_add(STRIPE_SECTORS,
					&rdev->corrected_errors);
			generic_make_request(bi);
		} else {
557
			if (rw & WRITE)
558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574
				set_bit(STRIPE_DEGRADED, &sh->state);
			pr_debug("skip op %ld on disc %d for sector %llu\n",
				bi->bi_rw, i, (unsigned long long)sh->sector);
			clear_bit(R5_LOCKED, &sh->dev[i].flags);
			set_bit(STRIPE_HANDLE, &sh->state);
		}
	}
}

static struct dma_async_tx_descriptor *
async_copy_data(int frombio, struct bio *bio, struct page *page,
	sector_t sector, struct dma_async_tx_descriptor *tx)
{
	struct bio_vec *bvl;
	struct page *bio_page;
	int i;
	int page_offset;
575
	struct async_submit_ctl submit;
D
Dan Williams 已提交
576
	enum async_tx_flags flags = 0;
577 578 579 580 581

	if (bio->bi_sector >= sector)
		page_offset = (signed)(bio->bi_sector - sector) * 512;
	else
		page_offset = (signed)(sector - bio->bi_sector) * -512;
582

D
Dan Williams 已提交
583 584 585 586
	if (frombio)
		flags |= ASYNC_TX_FENCE;
	init_async_submit(&submit, flags, tx, NULL, NULL, NULL);

587
	bio_for_each_segment(bvl, bio, i) {
588
		int len = bvl->bv_len;
589 590 591 592 593 594 595 596 597 598 599 600 601 602 603
		int clen;
		int b_offset = 0;

		if (page_offset < 0) {
			b_offset = -page_offset;
			page_offset += b_offset;
			len -= b_offset;
		}

		if (len > 0 && page_offset + len > STRIPE_SIZE)
			clen = STRIPE_SIZE - page_offset;
		else
			clen = len;

		if (clen > 0) {
604 605
			b_offset += bvl->bv_offset;
			bio_page = bvl->bv_page;
606 607
			if (frombio)
				tx = async_memcpy(page, bio_page, page_offset,
608
						  b_offset, clen, &submit);
609 610
			else
				tx = async_memcpy(bio_page, page, b_offset,
611
						  page_offset, clen, &submit);
612
		}
613 614 615
		/* chain the operations */
		submit.depend_tx = tx;

616 617 618 619 620 621 622 623 624 625 626 627 628
		if (clen < len) /* hit end of page */
			break;
		page_offset +=  len;
	}

	return tx;
}

static void ops_complete_biofill(void *stripe_head_ref)
{
	struct stripe_head *sh = stripe_head_ref;
	struct bio *return_bi = NULL;
	raid5_conf_t *conf = sh->raid_conf;
629
	int i;
630

631
	pr_debug("%s: stripe %llu\n", __func__,
632 633 634
		(unsigned long long)sh->sector);

	/* clear completed biofills */
635
	spin_lock_irq(&conf->device_lock);
636 637 638 639
	for (i = sh->disks; i--; ) {
		struct r5dev *dev = &sh->dev[i];

		/* acknowledge completion of a biofill operation */
640 641
		/* and check if we need to reply to a read request,
		 * new R5_Wantfill requests are held off until
642
		 * !STRIPE_BIOFILL_RUN
643 644
		 */
		if (test_and_clear_bit(R5_Wantfill, &dev->flags)) {
645 646 647 648 649 650 651 652
			struct bio *rbi, *rbi2;

			BUG_ON(!dev->read);
			rbi = dev->read;
			dev->read = NULL;
			while (rbi && rbi->bi_sector <
				dev->sector + STRIPE_SECTORS) {
				rbi2 = r5_next_bio(rbi, dev->sector);
653
				if (!raid5_dec_bi_phys_segments(rbi)) {
654 655 656 657 658 659 660
					rbi->bi_next = return_bi;
					return_bi = rbi;
				}
				rbi = rbi2;
			}
		}
	}
661 662
	spin_unlock_irq(&conf->device_lock);
	clear_bit(STRIPE_BIOFILL_RUN, &sh->state);
663 664 665

	return_io(return_bi);

666
	set_bit(STRIPE_HANDLE, &sh->state);
667 668 669 670 671 672 673
	release_stripe(sh);
}

static void ops_run_biofill(struct stripe_head *sh)
{
	struct dma_async_tx_descriptor *tx = NULL;
	raid5_conf_t *conf = sh->raid_conf;
674
	struct async_submit_ctl submit;
675 676
	int i;

677
	pr_debug("%s: stripe %llu\n", __func__,
678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697
		(unsigned long long)sh->sector);

	for (i = sh->disks; i--; ) {
		struct r5dev *dev = &sh->dev[i];
		if (test_bit(R5_Wantfill, &dev->flags)) {
			struct bio *rbi;
			spin_lock_irq(&conf->device_lock);
			dev->read = rbi = dev->toread;
			dev->toread = NULL;
			spin_unlock_irq(&conf->device_lock);
			while (rbi && rbi->bi_sector <
				dev->sector + STRIPE_SECTORS) {
				tx = async_copy_data(0, rbi, dev->page,
					dev->sector, tx);
				rbi = r5_next_bio(rbi, dev->sector);
			}
		}
	}

	atomic_inc(&sh->count);
698 699
	init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL);
	async_trigger_callback(&submit);
700 701
}

702
static void mark_target_uptodate(struct stripe_head *sh, int target)
703
{
704
	struct r5dev *tgt;
705

706 707
	if (target < 0)
		return;
708

709
	tgt = &sh->dev[target];
710 711 712
	set_bit(R5_UPTODATE, &tgt->flags);
	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
	clear_bit(R5_Wantcompute, &tgt->flags);
713 714
}

715
static void ops_complete_compute(void *stripe_head_ref)
716 717 718
{
	struct stripe_head *sh = stripe_head_ref;

719
	pr_debug("%s: stripe %llu\n", __func__,
720 721
		(unsigned long long)sh->sector);

722
	/* mark the computed target(s) as uptodate */
723
	mark_target_uptodate(sh, sh->ops.target);
724
	mark_target_uptodate(sh, sh->ops.target2);
725

726 727 728
	clear_bit(STRIPE_COMPUTE_RUN, &sh->state);
	if (sh->check_state == check_state_compute_run)
		sh->check_state = check_state_compute_result;
729 730 731 732
	set_bit(STRIPE_HANDLE, &sh->state);
	release_stripe(sh);
}

733 734 735 736 737 738 739 740 741
/* return a pointer to the address conversion region of the scribble buffer */
static addr_conv_t *to_addr_conv(struct stripe_head *sh,
				 struct raid5_percpu *percpu)
{
	return percpu->scribble + sizeof(struct page *) * (sh->disks + 2);
}

static struct dma_async_tx_descriptor *
ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)
742 743
{
	int disks = sh->disks;
744
	struct page **xor_srcs = percpu->scribble;
745 746 747 748 749
	int target = sh->ops.target;
	struct r5dev *tgt = &sh->dev[target];
	struct page *xor_dest = tgt->page;
	int count = 0;
	struct dma_async_tx_descriptor *tx;
750
	struct async_submit_ctl submit;
751 752 753
	int i;

	pr_debug("%s: stripe %llu block: %d\n",
754
		__func__, (unsigned long long)sh->sector, target);
755 756 757 758 759 760 761 762
	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));

	for (i = disks; i--; )
		if (i != target)
			xor_srcs[count++] = sh->dev[i].page;

	atomic_inc(&sh->count);

D
Dan Williams 已提交
763
	init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL,
764
			  ops_complete_compute, sh, to_addr_conv(sh, percpu));
765
	if (unlikely(count == 1))
766
		tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
767
	else
768
		tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
769 770 771 772

	return tx;
}

773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790
/* set_syndrome_sources - populate source buffers for gen_syndrome
 * @srcs - (struct page *) array of size sh->disks
 * @sh - stripe_head to parse
 *
 * Populates srcs in proper layout order for the stripe and returns the
 * 'count' of sources to be used in a call to async_gen_syndrome.  The P
 * destination buffer is recorded in srcs[count] and the Q destination
 * is recorded in srcs[count+1]].
 */
static int set_syndrome_sources(struct page **srcs, struct stripe_head *sh)
{
	int disks = sh->disks;
	int syndrome_disks = sh->ddf_layout ? disks : (disks - 2);
	int d0_idx = raid6_d0(sh);
	int count;
	int i;

	for (i = 0; i < disks; i++)
791
		srcs[i] = NULL;
792 793 794 795 796 797 798 799 800 801

	count = 0;
	i = d0_idx;
	do {
		int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);

		srcs[slot] = sh->dev[i].page;
		i = raid6_next_disk(i, disks);
	} while (i != d0_idx);

802
	return syndrome_disks;
803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822
}

static struct dma_async_tx_descriptor *
ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)
{
	int disks = sh->disks;
	struct page **blocks = percpu->scribble;
	int target;
	int qd_idx = sh->qd_idx;
	struct dma_async_tx_descriptor *tx;
	struct async_submit_ctl submit;
	struct r5dev *tgt;
	struct page *dest;
	int i;
	int count;

	if (sh->ops.target < 0)
		target = sh->ops.target2;
	else if (sh->ops.target2 < 0)
		target = sh->ops.target;
823
	else
824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839
		/* we should only have one valid target */
		BUG();
	BUG_ON(target < 0);
	pr_debug("%s: stripe %llu block: %d\n",
		__func__, (unsigned long long)sh->sector, target);

	tgt = &sh->dev[target];
	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
	dest = tgt->page;

	atomic_inc(&sh->count);

	if (target == qd_idx) {
		count = set_syndrome_sources(blocks, sh);
		blocks[count] = NULL; /* regenerating p is not necessary */
		BUG_ON(blocks[count+1] != dest); /* q should already be set */
D
Dan Williams 已提交
840 841
		init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
				  ops_complete_compute, sh,
842 843 844 845 846 847 848 849 850 851 852
				  to_addr_conv(sh, percpu));
		tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
	} else {
		/* Compute any data- or p-drive using XOR */
		count = 0;
		for (i = disks; i-- ; ) {
			if (i == target || i == qd_idx)
				continue;
			blocks[count++] = sh->dev[i].page;
		}

D
Dan Williams 已提交
853 854
		init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
				  NULL, ops_complete_compute, sh,
855 856 857
				  to_addr_conv(sh, percpu));
		tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit);
	}
858 859 860 861

	return tx;
}

862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882
static struct dma_async_tx_descriptor *
ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
{
	int i, count, disks = sh->disks;
	int syndrome_disks = sh->ddf_layout ? disks : disks-2;
	int d0_idx = raid6_d0(sh);
	int faila = -1, failb = -1;
	int target = sh->ops.target;
	int target2 = sh->ops.target2;
	struct r5dev *tgt = &sh->dev[target];
	struct r5dev *tgt2 = &sh->dev[target2];
	struct dma_async_tx_descriptor *tx;
	struct page **blocks = percpu->scribble;
	struct async_submit_ctl submit;

	pr_debug("%s: stripe %llu block1: %d block2: %d\n",
		 __func__, (unsigned long long)sh->sector, target, target2);
	BUG_ON(target < 0 || target2 < 0);
	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
	BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags));

883
	/* we need to open-code set_syndrome_sources to handle the
884 885 886
	 * slot number conversion for 'faila' and 'failb'
	 */
	for (i = 0; i < disks ; i++)
887
		blocks[i] = NULL;
888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913
	count = 0;
	i = d0_idx;
	do {
		int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);

		blocks[slot] = sh->dev[i].page;

		if (i == target)
			faila = slot;
		if (i == target2)
			failb = slot;
		i = raid6_next_disk(i, disks);
	} while (i != d0_idx);

	BUG_ON(faila == failb);
	if (failb < faila)
		swap(faila, failb);
	pr_debug("%s: stripe: %llu faila: %d failb: %d\n",
		 __func__, (unsigned long long)sh->sector, faila, failb);

	atomic_inc(&sh->count);

	if (failb == syndrome_disks+1) {
		/* Q disk is one of the missing disks */
		if (faila == syndrome_disks) {
			/* Missing P+Q, just recompute */
D
Dan Williams 已提交
914 915 916
			init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
					  ops_complete_compute, sh,
					  to_addr_conv(sh, percpu));
917
			return async_gen_syndrome(blocks, 0, syndrome_disks+2,
918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936
						  STRIPE_SIZE, &submit);
		} else {
			struct page *dest;
			int data_target;
			int qd_idx = sh->qd_idx;

			/* Missing D+Q: recompute D from P, then recompute Q */
			if (target == qd_idx)
				data_target = target2;
			else
				data_target = target;

			count = 0;
			for (i = disks; i-- ; ) {
				if (i == data_target || i == qd_idx)
					continue;
				blocks[count++] = sh->dev[i].page;
			}
			dest = sh->dev[data_target].page;
D
Dan Williams 已提交
937 938 939 940
			init_async_submit(&submit,
					  ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
					  NULL, NULL, NULL,
					  to_addr_conv(sh, percpu));
941 942 943 944
			tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE,
				       &submit);

			count = set_syndrome_sources(blocks, sh);
D
Dan Williams 已提交
945 946 947
			init_async_submit(&submit, ASYNC_TX_FENCE, tx,
					  ops_complete_compute, sh,
					  to_addr_conv(sh, percpu));
948 949 950 951
			return async_gen_syndrome(blocks, 0, count+2,
						  STRIPE_SIZE, &submit);
		}
	} else {
952 953 954 955 956 957 958 959 960 961 962 963 964 965
		init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
				  ops_complete_compute, sh,
				  to_addr_conv(sh, percpu));
		if (failb == syndrome_disks) {
			/* We're missing D+P. */
			return async_raid6_datap_recov(syndrome_disks+2,
						       STRIPE_SIZE, faila,
						       blocks, &submit);
		} else {
			/* We're missing D+D. */
			return async_raid6_2data_recov(syndrome_disks+2,
						       STRIPE_SIZE, faila, failb,
						       blocks, &submit);
		}
966 967 968 969
	}
}


970 971 972 973
static void ops_complete_prexor(void *stripe_head_ref)
{
	struct stripe_head *sh = stripe_head_ref;

974
	pr_debug("%s: stripe %llu\n", __func__,
975 976 977 978
		(unsigned long long)sh->sector);
}

static struct dma_async_tx_descriptor *
979 980
ops_run_prexor(struct stripe_head *sh, struct raid5_percpu *percpu,
	       struct dma_async_tx_descriptor *tx)
981 982
{
	int disks = sh->disks;
983
	struct page **xor_srcs = percpu->scribble;
984
	int count = 0, pd_idx = sh->pd_idx, i;
985
	struct async_submit_ctl submit;
986 987 988 989

	/* existing parity data subtracted */
	struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;

990
	pr_debug("%s: stripe %llu\n", __func__,
991 992 993 994 995
		(unsigned long long)sh->sector);

	for (i = disks; i--; ) {
		struct r5dev *dev = &sh->dev[i];
		/* Only process blocks that are known to be uptodate */
996
		if (test_bit(R5_Wantdrain, &dev->flags))
997 998 999
			xor_srcs[count++] = dev->page;
	}

D
Dan Williams 已提交
1000
	init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
1001
			  ops_complete_prexor, sh, to_addr_conv(sh, percpu));
1002
	tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
1003 1004 1005 1006 1007

	return tx;
}

static struct dma_async_tx_descriptor *
1008
ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
1009 1010
{
	int disks = sh->disks;
1011
	int i;
1012

1013
	pr_debug("%s: stripe %llu\n", __func__,
1014 1015 1016 1017 1018 1019
		(unsigned long long)sh->sector);

	for (i = disks; i--; ) {
		struct r5dev *dev = &sh->dev[i];
		struct bio *chosen;

1020
		if (test_and_clear_bit(R5_Wantdrain, &dev->flags)) {
1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031
			struct bio *wbi;

			spin_lock(&sh->lock);
			chosen = dev->towrite;
			dev->towrite = NULL;
			BUG_ON(dev->written);
			wbi = dev->written = chosen;
			spin_unlock(&sh->lock);

			while (wbi && wbi->bi_sector <
				dev->sector + STRIPE_SECTORS) {
T
Tejun Heo 已提交
1032 1033
				if (wbi->bi_rw & REQ_FUA)
					set_bit(R5_WantFUA, &dev->flags);
1034 1035 1036 1037 1038 1039 1040 1041 1042 1043
				tx = async_copy_data(1, wbi, dev->page,
					dev->sector, tx);
				wbi = r5_next_bio(wbi, dev->sector);
			}
		}
	}

	return tx;
}

1044
static void ops_complete_reconstruct(void *stripe_head_ref)
1045 1046
{
	struct stripe_head *sh = stripe_head_ref;
1047 1048 1049 1050
	int disks = sh->disks;
	int pd_idx = sh->pd_idx;
	int qd_idx = sh->qd_idx;
	int i;
T
Tejun Heo 已提交
1051
	bool fua = false;
1052

1053
	pr_debug("%s: stripe %llu\n", __func__,
1054 1055
		(unsigned long long)sh->sector);

T
Tejun Heo 已提交
1056 1057 1058
	for (i = disks; i--; )
		fua |= test_bit(R5_WantFUA, &sh->dev[i].flags);

1059 1060
	for (i = disks; i--; ) {
		struct r5dev *dev = &sh->dev[i];
1061

T
Tejun Heo 已提交
1062
		if (dev->written || i == pd_idx || i == qd_idx) {
1063
			set_bit(R5_UPTODATE, &dev->flags);
T
Tejun Heo 已提交
1064 1065 1066
			if (fua)
				set_bit(R5_WantFUA, &dev->flags);
		}
1067 1068
	}

1069 1070 1071 1072 1073 1074 1075 1076
	if (sh->reconstruct_state == reconstruct_state_drain_run)
		sh->reconstruct_state = reconstruct_state_drain_result;
	else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run)
		sh->reconstruct_state = reconstruct_state_prexor_drain_result;
	else {
		BUG_ON(sh->reconstruct_state != reconstruct_state_run);
		sh->reconstruct_state = reconstruct_state_result;
	}
1077 1078 1079 1080 1081 1082

	set_bit(STRIPE_HANDLE, &sh->state);
	release_stripe(sh);
}

static void
1083 1084
ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu,
		     struct dma_async_tx_descriptor *tx)
1085 1086
{
	int disks = sh->disks;
1087
	struct page **xor_srcs = percpu->scribble;
1088
	struct async_submit_ctl submit;
1089 1090
	int count = 0, pd_idx = sh->pd_idx, i;
	struct page *xor_dest;
1091
	int prexor = 0;
1092 1093
	unsigned long flags;

1094
	pr_debug("%s: stripe %llu\n", __func__,
1095 1096 1097 1098 1099
		(unsigned long long)sh->sector);

	/* check if prexor is active which means only process blocks
	 * that are part of a read-modify-write (written)
	 */
1100 1101
	if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
		prexor = 1;
1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121
		xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if (dev->written)
				xor_srcs[count++] = dev->page;
		}
	} else {
		xor_dest = sh->dev[pd_idx].page;
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if (i != pd_idx)
				xor_srcs[count++] = dev->page;
		}
	}

	/* 1/ if we prexor'd then the dest is reused as a source
	 * 2/ if we did not prexor then we are redoing the parity
	 * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST
	 * for the synchronous xor case
	 */
1122
	flags = ASYNC_TX_ACK |
1123 1124 1125 1126
		(prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST);

	atomic_inc(&sh->count);

1127
	init_async_submit(&submit, flags, tx, ops_complete_reconstruct, sh,
1128
			  to_addr_conv(sh, percpu));
1129 1130 1131 1132
	if (unlikely(count == 1))
		tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
	else
		tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
1133 1134
}

1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151
static void
ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,
		     struct dma_async_tx_descriptor *tx)
{
	struct async_submit_ctl submit;
	struct page **blocks = percpu->scribble;
	int count;

	pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector);

	count = set_syndrome_sources(blocks, sh);

	atomic_inc(&sh->count);

	init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_reconstruct,
			  sh, to_addr_conv(sh, percpu));
	async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE,  &submit);
1152 1153 1154 1155 1156 1157
}

static void ops_complete_check(void *stripe_head_ref)
{
	struct stripe_head *sh = stripe_head_ref;

1158
	pr_debug("%s: stripe %llu\n", __func__,
1159 1160
		(unsigned long long)sh->sector);

1161
	sh->check_state = check_state_check_result;
1162 1163 1164 1165
	set_bit(STRIPE_HANDLE, &sh->state);
	release_stripe(sh);
}

1166
static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu)
1167 1168
{
	int disks = sh->disks;
1169 1170 1171
	int pd_idx = sh->pd_idx;
	int qd_idx = sh->qd_idx;
	struct page *xor_dest;
1172
	struct page **xor_srcs = percpu->scribble;
1173
	struct dma_async_tx_descriptor *tx;
1174
	struct async_submit_ctl submit;
1175 1176
	int count;
	int i;
1177

1178
	pr_debug("%s: stripe %llu\n", __func__,
1179 1180
		(unsigned long long)sh->sector);

1181 1182 1183
	count = 0;
	xor_dest = sh->dev[pd_idx].page;
	xor_srcs[count++] = xor_dest;
1184
	for (i = disks; i--; ) {
1185 1186 1187
		if (i == pd_idx || i == qd_idx)
			continue;
		xor_srcs[count++] = sh->dev[i].page;
1188 1189
	}

1190 1191
	init_async_submit(&submit, 0, NULL, NULL, NULL,
			  to_addr_conv(sh, percpu));
D
Dan Williams 已提交
1192
	tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE,
1193
			   &sh->ops.zero_sum_result, &submit);
1194 1195

	atomic_inc(&sh->count);
1196 1197
	init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL);
	tx = async_trigger_callback(&submit);
1198 1199
}

1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211
static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp)
{
	struct page **srcs = percpu->scribble;
	struct async_submit_ctl submit;
	int count;

	pr_debug("%s: stripe %llu checkp: %d\n", __func__,
		(unsigned long long)sh->sector, checkp);

	count = set_syndrome_sources(srcs, sh);
	if (!checkp)
		srcs[count] = NULL;
1212 1213

	atomic_inc(&sh->count);
1214 1215 1216 1217
	init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check,
			  sh, to_addr_conv(sh, percpu));
	async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE,
			   &sh->ops.zero_sum_result, percpu->spare_page, &submit);
1218 1219
}

1220
static void __raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
1221 1222 1223
{
	int overlap_clear = 0, i, disks = sh->disks;
	struct dma_async_tx_descriptor *tx = NULL;
1224
	raid5_conf_t *conf = sh->raid_conf;
1225
	int level = conf->level;
1226 1227
	struct raid5_percpu *percpu;
	unsigned long cpu;
1228

1229 1230
	cpu = get_cpu();
	percpu = per_cpu_ptr(conf->percpu, cpu);
1231
	if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) {
1232 1233 1234 1235
		ops_run_biofill(sh);
		overlap_clear++;
	}

1236
	if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) {
1237 1238 1239 1240 1241 1242 1243 1244 1245 1246
		if (level < 6)
			tx = ops_run_compute5(sh, percpu);
		else {
			if (sh->ops.target2 < 0 || sh->ops.target < 0)
				tx = ops_run_compute6_1(sh, percpu);
			else
				tx = ops_run_compute6_2(sh, percpu);
		}
		/* terminate the chain if reconstruct is not set to be run */
		if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request))
1247 1248
			async_tx_ack(tx);
	}
1249

1250
	if (test_bit(STRIPE_OP_PREXOR, &ops_request))
1251
		tx = ops_run_prexor(sh, percpu, tx);
1252

1253
	if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) {
1254
		tx = ops_run_biodrain(sh, tx);
1255 1256 1257
		overlap_clear++;
	}

1258 1259 1260 1261 1262 1263
	if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) {
		if (level < 6)
			ops_run_reconstruct5(sh, percpu, tx);
		else
			ops_run_reconstruct6(sh, percpu, tx);
	}
1264

1265 1266 1267 1268 1269 1270 1271 1272 1273 1274
	if (test_bit(STRIPE_OP_CHECK, &ops_request)) {
		if (sh->check_state == check_state_run)
			ops_run_check_p(sh, percpu);
		else if (sh->check_state == check_state_run_q)
			ops_run_check_pq(sh, percpu, 0);
		else if (sh->check_state == check_state_run_pq)
			ops_run_check_pq(sh, percpu, 1);
		else
			BUG();
	}
1275 1276 1277 1278 1279 1280 1281

	if (overlap_clear)
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if (test_and_clear_bit(R5_Overlap, &dev->flags))
				wake_up(&sh->raid_conf->wait_for_overlap);
		}
1282
	put_cpu();
1283 1284
}

1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314
#ifdef CONFIG_MULTICORE_RAID456
static void async_run_ops(void *param, async_cookie_t cookie)
{
	struct stripe_head *sh = param;
	unsigned long ops_request = sh->ops.request;

	clear_bit_unlock(STRIPE_OPS_REQ_PENDING, &sh->state);
	wake_up(&sh->ops.wait_for_ops);

	__raid_run_ops(sh, ops_request);
	release_stripe(sh);
}

static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
{
	/* since handle_stripe can be called outside of raid5d context
	 * we need to ensure sh->ops.request is de-staged before another
	 * request arrives
	 */
	wait_event(sh->ops.wait_for_ops,
		   !test_and_set_bit_lock(STRIPE_OPS_REQ_PENDING, &sh->state));
	sh->ops.request = ops_request;

	atomic_inc(&sh->count);
	async_schedule(async_run_ops, sh);
}
#else
#define raid_run_ops __raid_run_ops
#endif

1315
static int grow_one_stripe(raid5_conf_t *conf)
L
Linus Torvalds 已提交
1316 1317
{
	struct stripe_head *sh;
N
Namhyung Kim 已提交
1318
	sh = kmem_cache_zalloc(conf->slab_cache, GFP_KERNEL);
1319 1320
	if (!sh)
		return 0;
N
Namhyung Kim 已提交
1321

1322 1323
	sh->raid_conf = conf;
	spin_lock_init(&sh->lock);
1324 1325 1326
	#ifdef CONFIG_MULTICORE_RAID456
	init_waitqueue_head(&sh->ops.wait_for_ops);
	#endif
1327

1328 1329
	if (grow_buffers(sh)) {
		shrink_buffers(sh);
1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342
		kmem_cache_free(conf->slab_cache, sh);
		return 0;
	}
	/* we just created an active stripe so... */
	atomic_set(&sh->count, 1);
	atomic_inc(&conf->active_stripes);
	INIT_LIST_HEAD(&sh->lru);
	release_stripe(sh);
	return 1;
}

static int grow_stripes(raid5_conf_t *conf, int num)
{
1343
	struct kmem_cache *sc;
1344
	int devs = max(conf->raid_disks, conf->previous_raid_disks);
L
Linus Torvalds 已提交
1345

1346 1347 1348 1349 1350 1351 1352 1353
	if (conf->mddev->gendisk)
		sprintf(conf->cache_name[0],
			"raid%d-%s", conf->level, mdname(conf->mddev));
	else
		sprintf(conf->cache_name[0],
			"raid%d-%p", conf->level, conf->mddev);
	sprintf(conf->cache_name[1], "%s-alt", conf->cache_name[0]);

1354 1355
	conf->active_name = 0;
	sc = kmem_cache_create(conf->cache_name[conf->active_name],
L
Linus Torvalds 已提交
1356
			       sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev),
1357
			       0, 0, NULL);
L
Linus Torvalds 已提交
1358 1359 1360
	if (!sc)
		return 1;
	conf->slab_cache = sc;
1361
	conf->pool_size = devs;
1362
	while (num--)
1363
		if (!grow_one_stripe(conf))
L
Linus Torvalds 已提交
1364 1365 1366
			return 1;
	return 0;
}
1367

1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389
/**
 * scribble_len - return the required size of the scribble region
 * @num - total number of disks in the array
 *
 * The size must be enough to contain:
 * 1/ a struct page pointer for each device in the array +2
 * 2/ room to convert each entry in (1) to its corresponding dma
 *    (dma_map_page()) or page (page_address()) address.
 *
 * Note: the +2 is for the destination buffers of the ddf/raid6 case where we
 * calculate over all devices (not just the data blocks), using zeros in place
 * of the P and Q blocks.
 */
static size_t scribble_len(int num)
{
	size_t len;

	len = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2);

	return len;
}

1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417
static int resize_stripes(raid5_conf_t *conf, int newsize)
{
	/* Make all the stripes able to hold 'newsize' devices.
	 * New slots in each stripe get 'page' set to a new page.
	 *
	 * This happens in stages:
	 * 1/ create a new kmem_cache and allocate the required number of
	 *    stripe_heads.
	 * 2/ gather all the old stripe_heads and tranfer the pages across
	 *    to the new stripe_heads.  This will have the side effect of
	 *    freezing the array as once all stripe_heads have been collected,
	 *    no IO will be possible.  Old stripe heads are freed once their
	 *    pages have been transferred over, and the old kmem_cache is
	 *    freed when all stripes are done.
	 * 3/ reallocate conf->disks to be suitable bigger.  If this fails,
	 *    we simple return a failre status - no need to clean anything up.
	 * 4/ allocate new pages for the new slots in the new stripe_heads.
	 *    If this fails, we don't bother trying the shrink the
	 *    stripe_heads down again, we just leave them as they are.
	 *    As each stripe_head is processed the new one is released into
	 *    active service.
	 *
	 * Once step2 is started, we cannot afford to wait for a write,
	 * so we use GFP_NOIO allocations.
	 */
	struct stripe_head *osh, *nsh;
	LIST_HEAD(newstripes);
	struct disk_info *ndisks;
1418
	unsigned long cpu;
1419
	int err;
1420
	struct kmem_cache *sc;
1421 1422 1423 1424 1425
	int i;

	if (newsize <= conf->pool_size)
		return 0; /* never bother to shrink */

1426 1427 1428
	err = md_allow_write(conf->mddev);
	if (err)
		return err;
1429

1430 1431 1432
	/* Step 1 */
	sc = kmem_cache_create(conf->cache_name[1-conf->active_name],
			       sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev),
1433
			       0, 0, NULL);
1434 1435 1436 1437
	if (!sc)
		return -ENOMEM;

	for (i = conf->max_nr_stripes; i; i--) {
N
Namhyung Kim 已提交
1438
		nsh = kmem_cache_zalloc(sc, GFP_KERNEL);
1439 1440 1441 1442 1443
		if (!nsh)
			break;

		nsh->raid_conf = conf;
		spin_lock_init(&nsh->lock);
1444 1445 1446
		#ifdef CONFIG_MULTICORE_RAID456
		init_waitqueue_head(&nsh->ops.wait_for_ops);
		#endif
1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468

		list_add(&nsh->lru, &newstripes);
	}
	if (i) {
		/* didn't get enough, give up */
		while (!list_empty(&newstripes)) {
			nsh = list_entry(newstripes.next, struct stripe_head, lru);
			list_del(&nsh->lru);
			kmem_cache_free(sc, nsh);
		}
		kmem_cache_destroy(sc);
		return -ENOMEM;
	}
	/* Step 2 - Must use GFP_NOIO now.
	 * OK, we have enough stripes, start collecting inactive
	 * stripes and copying them over
	 */
	list_for_each_entry(nsh, &newstripes, lru) {
		spin_lock_irq(&conf->device_lock);
		wait_event_lock_irq(conf->wait_for_stripe,
				    !list_empty(&conf->inactive_list),
				    conf->device_lock,
N
NeilBrown 已提交
1469
				    );
1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483
		osh = get_free_stripe(conf);
		spin_unlock_irq(&conf->device_lock);
		atomic_set(&nsh->count, 1);
		for(i=0; i<conf->pool_size; i++)
			nsh->dev[i].page = osh->dev[i].page;
		for( ; i<newsize; i++)
			nsh->dev[i].page = NULL;
		kmem_cache_free(conf->slab_cache, osh);
	}
	kmem_cache_destroy(conf->slab_cache);

	/* Step 3.
	 * At this point, we are holding all the stripes so the array
	 * is completely stalled, so now is a good time to resize
1484
	 * conf->disks and the scribble region
1485 1486 1487 1488 1489 1490 1491 1492 1493 1494
	 */
	ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO);
	if (ndisks) {
		for (i=0; i<conf->raid_disks; i++)
			ndisks[i] = conf->disks[i];
		kfree(conf->disks);
		conf->disks = ndisks;
	} else
		err = -ENOMEM;

1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513
	get_online_cpus();
	conf->scribble_len = scribble_len(newsize);
	for_each_present_cpu(cpu) {
		struct raid5_percpu *percpu;
		void *scribble;

		percpu = per_cpu_ptr(conf->percpu, cpu);
		scribble = kmalloc(conf->scribble_len, GFP_NOIO);

		if (scribble) {
			kfree(percpu->scribble);
			percpu->scribble = scribble;
		} else {
			err = -ENOMEM;
			break;
		}
	}
	put_online_cpus();

1514 1515 1516 1517
	/* Step 4, return new stripes to service */
	while(!list_empty(&newstripes)) {
		nsh = list_entry(newstripes.next, struct stripe_head, lru);
		list_del_init(&nsh->lru);
1518

1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534
		for (i=conf->raid_disks; i < newsize; i++)
			if (nsh->dev[i].page == NULL) {
				struct page *p = alloc_page(GFP_NOIO);
				nsh->dev[i].page = p;
				if (!p)
					err = -ENOMEM;
			}
		release_stripe(nsh);
	}
	/* critical section pass, GFP_NOIO no longer needed */

	conf->slab_cache = sc;
	conf->active_name = 1-conf->active_name;
	conf->pool_size = newsize;
	return err;
}
L
Linus Torvalds 已提交
1535

1536
static int drop_one_stripe(raid5_conf_t *conf)
L
Linus Torvalds 已提交
1537 1538 1539
{
	struct stripe_head *sh;

1540 1541 1542 1543 1544
	spin_lock_irq(&conf->device_lock);
	sh = get_free_stripe(conf);
	spin_unlock_irq(&conf->device_lock);
	if (!sh)
		return 0;
1545
	BUG_ON(atomic_read(&sh->count));
1546
	shrink_buffers(sh);
1547 1548 1549 1550 1551 1552 1553 1554 1555 1556
	kmem_cache_free(conf->slab_cache, sh);
	atomic_dec(&conf->active_stripes);
	return 1;
}

static void shrink_stripes(raid5_conf_t *conf)
{
	while (drop_one_stripe(conf))
		;

N
NeilBrown 已提交
1557 1558
	if (conf->slab_cache)
		kmem_cache_destroy(conf->slab_cache);
L
Linus Torvalds 已提交
1559 1560 1561
	conf->slab_cache = NULL;
}

1562
static void raid5_end_read_request(struct bio * bi, int error)
L
Linus Torvalds 已提交
1563
{
1564
	struct stripe_head *sh = bi->bi_private;
L
Linus Torvalds 已提交
1565
	raid5_conf_t *conf = sh->raid_conf;
1566
	int disks = sh->disks, i;
L
Linus Torvalds 已提交
1567
	int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
1568 1569
	char b[BDEVNAME_SIZE];
	mdk_rdev_t *rdev;
L
Linus Torvalds 已提交
1570 1571 1572 1573 1574 1575


	for (i=0 ; i<disks; i++)
		if (bi == &sh->dev[i].req)
			break;

1576 1577
	pr_debug("end_read_request %llu/%d, count: %d, uptodate %d.\n",
		(unsigned long long)sh->sector, i, atomic_read(&sh->count),
L
Linus Torvalds 已提交
1578 1579 1580
		uptodate);
	if (i == disks) {
		BUG();
1581
		return;
L
Linus Torvalds 已提交
1582 1583 1584 1585
	}

	if (uptodate) {
		set_bit(R5_UPTODATE, &sh->dev[i].flags);
1586
		if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
1587
			rdev = conf->disks[i].rdev;
1588
			printk_rl(KERN_INFO "md/raid:%s: read error corrected"
1589 1590 1591 1592 1593
				  " (%lu sectors at %llu on %s)\n",
				  mdname(conf->mddev), STRIPE_SECTORS,
				  (unsigned long long)(sh->sector
						       + rdev->data_offset),
				  bdevname(rdev->bdev, b));
1594 1595 1596
			clear_bit(R5_ReadError, &sh->dev[i].flags);
			clear_bit(R5_ReWrite, &sh->dev[i].flags);
		}
1597 1598
		if (atomic_read(&conf->disks[i].rdev->read_errors))
			atomic_set(&conf->disks[i].rdev->read_errors, 0);
L
Linus Torvalds 已提交
1599
	} else {
1600
		const char *bdn = bdevname(conf->disks[i].rdev->bdev, b);
1601
		int retry = 0;
1602 1603
		rdev = conf->disks[i].rdev;

L
Linus Torvalds 已提交
1604
		clear_bit(R5_UPTODATE, &sh->dev[i].flags);
1605
		atomic_inc(&rdev->read_errors);
1606
		if (conf->mddev->degraded >= conf->max_degraded)
1607
			printk_rl(KERN_WARNING
1608
				  "md/raid:%s: read error not correctable "
1609 1610 1611 1612 1613
				  "(sector %llu on %s).\n",
				  mdname(conf->mddev),
				  (unsigned long long)(sh->sector
						       + rdev->data_offset),
				  bdn);
1614
		else if (test_bit(R5_ReWrite, &sh->dev[i].flags))
1615
			/* Oh, no!!! */
1616
			printk_rl(KERN_WARNING
1617
				  "md/raid:%s: read error NOT corrected!! "
1618 1619 1620 1621 1622
				  "(sector %llu on %s).\n",
				  mdname(conf->mddev),
				  (unsigned long long)(sh->sector
						       + rdev->data_offset),
				  bdn);
1623
		else if (atomic_read(&rdev->read_errors)
1624
			 > conf->max_nr_stripes)
N
NeilBrown 已提交
1625
			printk(KERN_WARNING
1626
			       "md/raid:%s: Too many read errors, failing device %s.\n",
1627
			       mdname(conf->mddev), bdn);
1628 1629 1630 1631 1632
		else
			retry = 1;
		if (retry)
			set_bit(R5_ReadError, &sh->dev[i].flags);
		else {
1633 1634
			clear_bit(R5_ReadError, &sh->dev[i].flags);
			clear_bit(R5_ReWrite, &sh->dev[i].flags);
1635
			md_error(conf->mddev, rdev);
1636
		}
L
Linus Torvalds 已提交
1637 1638 1639 1640 1641 1642 1643
	}
	rdev_dec_pending(conf->disks[i].rdev, conf->mddev);
	clear_bit(R5_LOCKED, &sh->dev[i].flags);
	set_bit(STRIPE_HANDLE, &sh->state);
	release_stripe(sh);
}

1644
static void raid5_end_write_request(struct bio *bi, int error)
L
Linus Torvalds 已提交
1645
{
1646
	struct stripe_head *sh = bi->bi_private;
L
Linus Torvalds 已提交
1647
	raid5_conf_t *conf = sh->raid_conf;
1648
	int disks = sh->disks, i;
L
Linus Torvalds 已提交
1649 1650 1651 1652 1653 1654
	int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);

	for (i=0 ; i<disks; i++)
		if (bi == &sh->dev[i].req)
			break;

1655
	pr_debug("end_write_request %llu/%d, count %d, uptodate: %d.\n",
L
Linus Torvalds 已提交
1656 1657 1658 1659
		(unsigned long long)sh->sector, i, atomic_read(&sh->count),
		uptodate);
	if (i == disks) {
		BUG();
1660
		return;
L
Linus Torvalds 已提交
1661 1662 1663 1664 1665 1666 1667 1668 1669
	}

	if (!uptodate)
		md_error(conf->mddev, conf->disks[i].rdev);

	rdev_dec_pending(conf->disks[i].rdev, conf->mddev);
	
	clear_bit(R5_LOCKED, &sh->dev[i].flags);
	set_bit(STRIPE_HANDLE, &sh->state);
1670
	release_stripe(sh);
L
Linus Torvalds 已提交
1671 1672 1673
}


1674
static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous);
L
Linus Torvalds 已提交
1675
	
1676
static void raid5_build_block(struct stripe_head *sh, int i, int previous)
L
Linus Torvalds 已提交
1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691
{
	struct r5dev *dev = &sh->dev[i];

	bio_init(&dev->req);
	dev->req.bi_io_vec = &dev->vec;
	dev->req.bi_vcnt++;
	dev->req.bi_max_vecs++;
	dev->vec.bv_page = dev->page;
	dev->vec.bv_len = STRIPE_SIZE;
	dev->vec.bv_offset = 0;

	dev->req.bi_sector = sh->sector;
	dev->req.bi_private = sh;

	dev->flags = 0;
1692
	dev->sector = compute_blocknr(sh, i, previous);
L
Linus Torvalds 已提交
1693 1694 1695 1696 1697
}

static void error(mddev_t *mddev, mdk_rdev_t *rdev)
{
	char b[BDEVNAME_SIZE];
1698
	raid5_conf_t *conf = mddev->private;
1699
	pr_debug("raid456: error called\n");
L
Linus Torvalds 已提交
1700

1701 1702 1703 1704 1705 1706 1707 1708 1709
	if (test_and_clear_bit(In_sync, &rdev->flags)) {
		unsigned long flags;
		spin_lock_irqsave(&conf->device_lock, flags);
		mddev->degraded++;
		spin_unlock_irqrestore(&conf->device_lock, flags);
		/*
		 * if recovery was running, make sure it aborts.
		 */
		set_bit(MD_RECOVERY_INTR, &mddev->recovery);
L
Linus Torvalds 已提交
1710
	}
1711 1712 1713 1714 1715 1716 1717 1718 1719
	set_bit(Faulty, &rdev->flags);
	set_bit(MD_CHANGE_DEVS, &mddev->flags);
	printk(KERN_ALERT
	       "md/raid:%s: Disk failure on %s, disabling device.\n"
	       "md/raid:%s: Operation continuing on %d devices.\n",
	       mdname(mddev),
	       bdevname(rdev->bdev, b),
	       mdname(mddev),
	       conf->raid_disks - mddev->degraded);
1720
}
L
Linus Torvalds 已提交
1721 1722 1723 1724 1725

/*
 * Input: a 'big' sector number,
 * Output: index of the data and parity disk, and the sector # in them.
 */
1726
static sector_t raid5_compute_sector(raid5_conf_t *conf, sector_t r_sector,
1727 1728
				     int previous, int *dd_idx,
				     struct stripe_head *sh)
L
Linus Torvalds 已提交
1729
{
N
NeilBrown 已提交
1730
	sector_t stripe, stripe2;
1731
	sector_t chunk_number;
L
Linus Torvalds 已提交
1732
	unsigned int chunk_offset;
1733
	int pd_idx, qd_idx;
1734
	int ddf_layout = 0;
L
Linus Torvalds 已提交
1735
	sector_t new_sector;
1736 1737
	int algorithm = previous ? conf->prev_algo
				 : conf->algorithm;
1738 1739
	int sectors_per_chunk = previous ? conf->prev_chunk_sectors
					 : conf->chunk_sectors;
1740 1741 1742
	int raid_disks = previous ? conf->previous_raid_disks
				  : conf->raid_disks;
	int data_disks = raid_disks - conf->max_degraded;
L
Linus Torvalds 已提交
1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754

	/* First compute the information on this sector */

	/*
	 * Compute the chunk number and the sector offset inside the chunk
	 */
	chunk_offset = sector_div(r_sector, sectors_per_chunk);
	chunk_number = r_sector;

	/*
	 * Compute the stripe number
	 */
1755 1756
	stripe = chunk_number;
	*dd_idx = sector_div(stripe, data_disks);
N
NeilBrown 已提交
1757
	stripe2 = stripe;
L
Linus Torvalds 已提交
1758 1759 1760
	/*
	 * Select the parity disk based on the user selected algorithm.
	 */
1761
	pd_idx = qd_idx = ~0;
1762 1763
	switch(conf->level) {
	case 4:
1764
		pd_idx = data_disks;
1765 1766
		break;
	case 5:
1767
		switch (algorithm) {
L
Linus Torvalds 已提交
1768
		case ALGORITHM_LEFT_ASYMMETRIC:
N
NeilBrown 已提交
1769
			pd_idx = data_disks - sector_div(stripe2, raid_disks);
1770
			if (*dd_idx >= pd_idx)
L
Linus Torvalds 已提交
1771 1772 1773
				(*dd_idx)++;
			break;
		case ALGORITHM_RIGHT_ASYMMETRIC:
N
NeilBrown 已提交
1774
			pd_idx = sector_div(stripe2, raid_disks);
1775
			if (*dd_idx >= pd_idx)
L
Linus Torvalds 已提交
1776 1777 1778
				(*dd_idx)++;
			break;
		case ALGORITHM_LEFT_SYMMETRIC:
N
NeilBrown 已提交
1779
			pd_idx = data_disks - sector_div(stripe2, raid_disks);
1780
			*dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
L
Linus Torvalds 已提交
1781 1782
			break;
		case ALGORITHM_RIGHT_SYMMETRIC:
N
NeilBrown 已提交
1783
			pd_idx = sector_div(stripe2, raid_disks);
1784
			*dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
L
Linus Torvalds 已提交
1785
			break;
1786 1787 1788 1789 1790 1791 1792
		case ALGORITHM_PARITY_0:
			pd_idx = 0;
			(*dd_idx)++;
			break;
		case ALGORITHM_PARITY_N:
			pd_idx = data_disks;
			break;
L
Linus Torvalds 已提交
1793
		default:
1794
			BUG();
1795 1796 1797 1798
		}
		break;
	case 6:

1799
		switch (algorithm) {
1800
		case ALGORITHM_LEFT_ASYMMETRIC:
N
NeilBrown 已提交
1801
			pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
1802 1803
			qd_idx = pd_idx + 1;
			if (pd_idx == raid_disks-1) {
1804
				(*dd_idx)++;	/* Q D D D P */
1805 1806
				qd_idx = 0;
			} else if (*dd_idx >= pd_idx)
1807 1808 1809
				(*dd_idx) += 2; /* D D P Q D */
			break;
		case ALGORITHM_RIGHT_ASYMMETRIC:
N
NeilBrown 已提交
1810
			pd_idx = sector_div(stripe2, raid_disks);
1811 1812
			qd_idx = pd_idx + 1;
			if (pd_idx == raid_disks-1) {
1813
				(*dd_idx)++;	/* Q D D D P */
1814 1815
				qd_idx = 0;
			} else if (*dd_idx >= pd_idx)
1816 1817 1818
				(*dd_idx) += 2; /* D D P Q D */
			break;
		case ALGORITHM_LEFT_SYMMETRIC:
N
NeilBrown 已提交
1819
			pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
1820 1821
			qd_idx = (pd_idx + 1) % raid_disks;
			*dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
1822 1823
			break;
		case ALGORITHM_RIGHT_SYMMETRIC:
N
NeilBrown 已提交
1824
			pd_idx = sector_div(stripe2, raid_disks);
1825 1826
			qd_idx = (pd_idx + 1) % raid_disks;
			*dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
1827
			break;
1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842

		case ALGORITHM_PARITY_0:
			pd_idx = 0;
			qd_idx = 1;
			(*dd_idx) += 2;
			break;
		case ALGORITHM_PARITY_N:
			pd_idx = data_disks;
			qd_idx = data_disks + 1;
			break;

		case ALGORITHM_ROTATING_ZERO_RESTART:
			/* Exactly the same as RIGHT_ASYMMETRIC, but or
			 * of blocks for computing Q is different.
			 */
N
NeilBrown 已提交
1843
			pd_idx = sector_div(stripe2, raid_disks);
1844 1845 1846 1847 1848 1849
			qd_idx = pd_idx + 1;
			if (pd_idx == raid_disks-1) {
				(*dd_idx)++;	/* Q D D D P */
				qd_idx = 0;
			} else if (*dd_idx >= pd_idx)
				(*dd_idx) += 2; /* D D P Q D */
1850
			ddf_layout = 1;
1851 1852 1853 1854 1855 1856 1857
			break;

		case ALGORITHM_ROTATING_N_RESTART:
			/* Same a left_asymmetric, by first stripe is
			 * D D D P Q  rather than
			 * Q D D D P
			 */
N
NeilBrown 已提交
1858 1859
			stripe2 += 1;
			pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
1860 1861 1862 1863 1864 1865
			qd_idx = pd_idx + 1;
			if (pd_idx == raid_disks-1) {
				(*dd_idx)++;	/* Q D D D P */
				qd_idx = 0;
			} else if (*dd_idx >= pd_idx)
				(*dd_idx) += 2; /* D D P Q D */
1866
			ddf_layout = 1;
1867 1868 1869 1870
			break;

		case ALGORITHM_ROTATING_N_CONTINUE:
			/* Same as left_symmetric but Q is before P */
N
NeilBrown 已提交
1871
			pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
1872 1873
			qd_idx = (pd_idx + raid_disks - 1) % raid_disks;
			*dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
1874
			ddf_layout = 1;
1875 1876 1877 1878
			break;

		case ALGORITHM_LEFT_ASYMMETRIC_6:
			/* RAID5 left_asymmetric, with Q on last device */
N
NeilBrown 已提交
1879
			pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
1880 1881 1882 1883 1884 1885
			if (*dd_idx >= pd_idx)
				(*dd_idx)++;
			qd_idx = raid_disks - 1;
			break;

		case ALGORITHM_RIGHT_ASYMMETRIC_6:
N
NeilBrown 已提交
1886
			pd_idx = sector_div(stripe2, raid_disks-1);
1887 1888 1889 1890 1891 1892
			if (*dd_idx >= pd_idx)
				(*dd_idx)++;
			qd_idx = raid_disks - 1;
			break;

		case ALGORITHM_LEFT_SYMMETRIC_6:
N
NeilBrown 已提交
1893
			pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
1894 1895 1896 1897 1898
			*dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
			qd_idx = raid_disks - 1;
			break;

		case ALGORITHM_RIGHT_SYMMETRIC_6:
N
NeilBrown 已提交
1899
			pd_idx = sector_div(stripe2, raid_disks-1);
1900 1901 1902 1903 1904 1905 1906 1907 1908 1909
			*dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
			qd_idx = raid_disks - 1;
			break;

		case ALGORITHM_PARITY_0_6:
			pd_idx = 0;
			(*dd_idx)++;
			qd_idx = raid_disks - 1;
			break;

1910
		default:
1911
			BUG();
1912 1913
		}
		break;
L
Linus Torvalds 已提交
1914 1915
	}

1916 1917 1918
	if (sh) {
		sh->pd_idx = pd_idx;
		sh->qd_idx = qd_idx;
1919
		sh->ddf_layout = ddf_layout;
1920
	}
L
Linus Torvalds 已提交
1921 1922 1923 1924 1925 1926 1927 1928
	/*
	 * Finally, compute the new sector number
	 */
	new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset;
	return new_sector;
}


1929
static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous)
L
Linus Torvalds 已提交
1930 1931
{
	raid5_conf_t *conf = sh->raid_conf;
1932 1933
	int raid_disks = sh->disks;
	int data_disks = raid_disks - conf->max_degraded;
L
Linus Torvalds 已提交
1934
	sector_t new_sector = sh->sector, check;
1935 1936
	int sectors_per_chunk = previous ? conf->prev_chunk_sectors
					 : conf->chunk_sectors;
1937 1938
	int algorithm = previous ? conf->prev_algo
				 : conf->algorithm;
L
Linus Torvalds 已提交
1939 1940
	sector_t stripe;
	int chunk_offset;
1941 1942
	sector_t chunk_number;
	int dummy1, dd_idx = i;
L
Linus Torvalds 已提交
1943
	sector_t r_sector;
1944
	struct stripe_head sh2;
L
Linus Torvalds 已提交
1945

1946

L
Linus Torvalds 已提交
1947 1948 1949
	chunk_offset = sector_div(new_sector, sectors_per_chunk);
	stripe = new_sector;

1950 1951 1952 1953 1954
	if (i == sh->pd_idx)
		return 0;
	switch(conf->level) {
	case 4: break;
	case 5:
1955
		switch (algorithm) {
L
Linus Torvalds 已提交
1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966
		case ALGORITHM_LEFT_ASYMMETRIC:
		case ALGORITHM_RIGHT_ASYMMETRIC:
			if (i > sh->pd_idx)
				i--;
			break;
		case ALGORITHM_LEFT_SYMMETRIC:
		case ALGORITHM_RIGHT_SYMMETRIC:
			if (i < sh->pd_idx)
				i += raid_disks;
			i -= (sh->pd_idx + 1);
			break;
1967 1968 1969 1970 1971
		case ALGORITHM_PARITY_0:
			i -= 1;
			break;
		case ALGORITHM_PARITY_N:
			break;
L
Linus Torvalds 已提交
1972
		default:
1973
			BUG();
1974 1975 1976
		}
		break;
	case 6:
1977
		if (i == sh->qd_idx)
1978
			return 0; /* It is the Q disk */
1979
		switch (algorithm) {
1980 1981
		case ALGORITHM_LEFT_ASYMMETRIC:
		case ALGORITHM_RIGHT_ASYMMETRIC:
1982 1983 1984 1985
		case ALGORITHM_ROTATING_ZERO_RESTART:
		case ALGORITHM_ROTATING_N_RESTART:
			if (sh->pd_idx == raid_disks-1)
				i--;	/* Q D D D P */
1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999
			else if (i > sh->pd_idx)
				i -= 2; /* D D P Q D */
			break;
		case ALGORITHM_LEFT_SYMMETRIC:
		case ALGORITHM_RIGHT_SYMMETRIC:
			if (sh->pd_idx == raid_disks-1)
				i--; /* Q D D D P */
			else {
				/* D D P Q D */
				if (i < sh->pd_idx)
					i += raid_disks;
				i -= (sh->pd_idx + 2);
			}
			break;
2000 2001 2002 2003 2004 2005
		case ALGORITHM_PARITY_0:
			i -= 2;
			break;
		case ALGORITHM_PARITY_N:
			break;
		case ALGORITHM_ROTATING_N_CONTINUE:
2006
			/* Like left_symmetric, but P is before Q */
2007 2008
			if (sh->pd_idx == 0)
				i--;	/* P D D D Q */
2009 2010 2011 2012 2013 2014
			else {
				/* D D Q P D */
				if (i < sh->pd_idx)
					i += raid_disks;
				i -= (sh->pd_idx + 1);
			}
2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029
			break;
		case ALGORITHM_LEFT_ASYMMETRIC_6:
		case ALGORITHM_RIGHT_ASYMMETRIC_6:
			if (i > sh->pd_idx)
				i--;
			break;
		case ALGORITHM_LEFT_SYMMETRIC_6:
		case ALGORITHM_RIGHT_SYMMETRIC_6:
			if (i < sh->pd_idx)
				i += data_disks + 1;
			i -= (sh->pd_idx + 1);
			break;
		case ALGORITHM_PARITY_0_6:
			i -= 1;
			break;
2030
		default:
2031
			BUG();
2032 2033
		}
		break;
L
Linus Torvalds 已提交
2034 2035 2036
	}

	chunk_number = stripe * data_disks + i;
2037
	r_sector = chunk_number * sectors_per_chunk + chunk_offset;
L
Linus Torvalds 已提交
2038

2039
	check = raid5_compute_sector(conf, r_sector,
2040
				     previous, &dummy1, &sh2);
2041 2042
	if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx
		|| sh2.qd_idx != sh->qd_idx) {
2043 2044
		printk(KERN_ERR "md/raid:%s: compute_blocknr: map not correct\n",
		       mdname(conf->mddev));
L
Linus Torvalds 已提交
2045 2046 2047 2048 2049 2050
		return 0;
	}
	return r_sector;
}


2051
static void
2052
schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s,
2053
			 int rcw, int expand)
2054 2055
{
	int i, pd_idx = sh->pd_idx, disks = sh->disks;
2056 2057
	raid5_conf_t *conf = sh->raid_conf;
	int level = conf->level;
2058 2059 2060 2061 2062 2063 2064

	if (rcw) {
		/* if we are not expanding this is a proper write request, and
		 * there will be bios with new data to be drained into the
		 * stripe cache
		 */
		if (!expand) {
2065 2066 2067 2068
			sh->reconstruct_state = reconstruct_state_drain_run;
			set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
		} else
			sh->reconstruct_state = reconstruct_state_run;
2069

2070
		set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
2071 2072 2073 2074 2075 2076

		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];

			if (dev->towrite) {
				set_bit(R5_LOCKED, &dev->flags);
2077
				set_bit(R5_Wantdrain, &dev->flags);
2078 2079
				if (!expand)
					clear_bit(R5_UPTODATE, &dev->flags);
2080
				s->locked++;
2081 2082
			}
		}
2083
		if (s->locked + conf->max_degraded == disks)
2084
			if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state))
2085
				atomic_inc(&conf->pending_full_writes);
2086
	} else {
2087
		BUG_ON(level == 6);
2088 2089 2090
		BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) ||
			test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags)));

2091
		sh->reconstruct_state = reconstruct_state_prexor_drain_run;
2092 2093
		set_bit(STRIPE_OP_PREXOR, &s->ops_request);
		set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
2094
		set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
2095 2096 2097 2098 2099 2100 2101 2102

		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if (i == pd_idx)
				continue;

			if (dev->towrite &&
			    (test_bit(R5_UPTODATE, &dev->flags) ||
2103 2104
			     test_bit(R5_Wantcompute, &dev->flags))) {
				set_bit(R5_Wantdrain, &dev->flags);
2105 2106
				set_bit(R5_LOCKED, &dev->flags);
				clear_bit(R5_UPTODATE, &dev->flags);
2107
				s->locked++;
2108 2109 2110 2111
			}
		}
	}

2112
	/* keep the parity disk(s) locked while asynchronous operations
2113 2114 2115 2116
	 * are in flight
	 */
	set_bit(R5_LOCKED, &sh->dev[pd_idx].flags);
	clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
2117
	s->locked++;
2118

2119 2120 2121 2122 2123 2124 2125 2126 2127
	if (level == 6) {
		int qd_idx = sh->qd_idx;
		struct r5dev *dev = &sh->dev[qd_idx];

		set_bit(R5_LOCKED, &dev->flags);
		clear_bit(R5_UPTODATE, &dev->flags);
		s->locked++;
	}

2128
	pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n",
2129
		__func__, (unsigned long long)sh->sector,
2130
		s->locked, s->ops_request);
2131
}
2132

L
Linus Torvalds 已提交
2133 2134
/*
 * Each stripe/dev can have one or more bion attached.
2135
 * toread/towrite point to the first in a chain.
L
Linus Torvalds 已提交
2136 2137 2138 2139 2140 2141
 * The bi_next chain must be in order.
 */
static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx, int forwrite)
{
	struct bio **bip;
	raid5_conf_t *conf = sh->raid_conf;
2142
	int firstwrite=0;
L
Linus Torvalds 已提交
2143

2144
	pr_debug("adding bh b#%llu to stripe s#%llu\n",
L
Linus Torvalds 已提交
2145 2146 2147 2148 2149 2150
		(unsigned long long)bi->bi_sector,
		(unsigned long long)sh->sector);


	spin_lock(&sh->lock);
	spin_lock_irq(&conf->device_lock);
2151
	if (forwrite) {
L
Linus Torvalds 已提交
2152
		bip = &sh->dev[dd_idx].towrite;
2153 2154 2155
		if (*bip == NULL && sh->dev[dd_idx].written == NULL)
			firstwrite = 1;
	} else
L
Linus Torvalds 已提交
2156 2157 2158 2159 2160 2161 2162 2163 2164
		bip = &sh->dev[dd_idx].toread;
	while (*bip && (*bip)->bi_sector < bi->bi_sector) {
		if ((*bip)->bi_sector + ((*bip)->bi_size >> 9) > bi->bi_sector)
			goto overlap;
		bip = & (*bip)->bi_next;
	}
	if (*bip && (*bip)->bi_sector < bi->bi_sector + ((bi->bi_size)>>9))
		goto overlap;

2165
	BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next);
L
Linus Torvalds 已提交
2166 2167 2168
	if (*bip)
		bi->bi_next = *bip;
	*bip = bi;
2169
	bi->bi_phys_segments++;
L
Linus Torvalds 已提交
2170 2171 2172
	spin_unlock_irq(&conf->device_lock);
	spin_unlock(&sh->lock);

2173
	pr_debug("added bi b#%llu to stripe s#%llu, disk %d.\n",
L
Linus Torvalds 已提交
2174 2175 2176
		(unsigned long long)bi->bi_sector,
		(unsigned long long)sh->sector, dd_idx);

2177 2178 2179
	if (conf->mddev->bitmap && firstwrite) {
		bitmap_startwrite(conf->mddev->bitmap, sh->sector,
				  STRIPE_SECTORS, 0);
2180
		sh->bm_seq = conf->seq_flush+1;
2181 2182 2183
		set_bit(STRIPE_BIT_DELAY, &sh->state);
	}

L
Linus Torvalds 已提交
2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205
	if (forwrite) {
		/* check if page is covered */
		sector_t sector = sh->dev[dd_idx].sector;
		for (bi=sh->dev[dd_idx].towrite;
		     sector < sh->dev[dd_idx].sector + STRIPE_SECTORS &&
			     bi && bi->bi_sector <= sector;
		     bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) {
			if (bi->bi_sector + (bi->bi_size>>9) >= sector)
				sector = bi->bi_sector + (bi->bi_size>>9);
		}
		if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS)
			set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags);
	}
	return 1;

 overlap:
	set_bit(R5_Overlap, &sh->dev[dd_idx].flags);
	spin_unlock_irq(&conf->device_lock);
	spin_unlock(&sh->lock);
	return 0;
}

2206 2207
static void end_reshape(raid5_conf_t *conf);

2208 2209
static void stripe_set_idx(sector_t stripe, raid5_conf_t *conf, int previous,
			    struct stripe_head *sh)
2210
{
2211
	int sectors_per_chunk =
2212
		previous ? conf->prev_chunk_sectors : conf->chunk_sectors;
2213
	int dd_idx;
2214
	int chunk_offset = sector_div(stripe, sectors_per_chunk);
2215
	int disks = previous ? conf->previous_raid_disks : conf->raid_disks;
2216

2217 2218
	raid5_compute_sector(conf,
			     stripe * (disks - conf->max_degraded)
2219
			     *sectors_per_chunk + chunk_offset,
2220
			     previous,
2221
			     &dd_idx, sh);
2222 2223
}

2224
static void
2225
handle_failed_stripe(raid5_conf_t *conf, struct stripe_head *sh,
2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258
				struct stripe_head_state *s, int disks,
				struct bio **return_bi)
{
	int i;
	for (i = disks; i--; ) {
		struct bio *bi;
		int bitmap_end = 0;

		if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
			mdk_rdev_t *rdev;
			rcu_read_lock();
			rdev = rcu_dereference(conf->disks[i].rdev);
			if (rdev && test_bit(In_sync, &rdev->flags))
				/* multiple read failures in one stripe */
				md_error(conf->mddev, rdev);
			rcu_read_unlock();
		}
		spin_lock_irq(&conf->device_lock);
		/* fail all writes first */
		bi = sh->dev[i].towrite;
		sh->dev[i].towrite = NULL;
		if (bi) {
			s->to_write--;
			bitmap_end = 1;
		}

		if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
			wake_up(&conf->wait_for_overlap);

		while (bi && bi->bi_sector <
			sh->dev[i].sector + STRIPE_SECTORS) {
			struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector);
			clear_bit(BIO_UPTODATE, &bi->bi_flags);
2259
			if (!raid5_dec_bi_phys_segments(bi)) {
2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273
				md_write_end(conf->mddev);
				bi->bi_next = *return_bi;
				*return_bi = bi;
			}
			bi = nextbi;
		}
		/* and fail all 'written' */
		bi = sh->dev[i].written;
		sh->dev[i].written = NULL;
		if (bi) bitmap_end = 1;
		while (bi && bi->bi_sector <
		       sh->dev[i].sector + STRIPE_SECTORS) {
			struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector);
			clear_bit(BIO_UPTODATE, &bi->bi_flags);
2274
			if (!raid5_dec_bi_phys_segments(bi)) {
2275 2276 2277 2278 2279 2280 2281
				md_write_end(conf->mddev);
				bi->bi_next = *return_bi;
				*return_bi = bi;
			}
			bi = bi2;
		}

2282 2283 2284 2285 2286 2287
		/* fail any reads if this device is non-operational and
		 * the data has not reached the cache yet.
		 */
		if (!test_bit(R5_Wantfill, &sh->dev[i].flags) &&
		    (!test_bit(R5_Insync, &sh->dev[i].flags) ||
		      test_bit(R5_ReadError, &sh->dev[i].flags))) {
2288 2289 2290 2291 2292 2293 2294 2295 2296 2297
			bi = sh->dev[i].toread;
			sh->dev[i].toread = NULL;
			if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
				wake_up(&conf->wait_for_overlap);
			if (bi) s->to_read--;
			while (bi && bi->bi_sector <
			       sh->dev[i].sector + STRIPE_SECTORS) {
				struct bio *nextbi =
					r5_next_bio(bi, sh->dev[i].sector);
				clear_bit(BIO_UPTODATE, &bi->bi_flags);
2298
				if (!raid5_dec_bi_phys_segments(bi)) {
2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310
					bi->bi_next = *return_bi;
					*return_bi = bi;
				}
				bi = nextbi;
			}
		}
		spin_unlock_irq(&conf->device_lock);
		if (bitmap_end)
			bitmap_endwrite(conf->mddev->bitmap, sh->sector,
					STRIPE_SECTORS, 0, 0);
	}

2311 2312 2313
	if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
		if (atomic_dec_and_test(&conf->pending_full_writes))
			md_wakeup_thread(conf->mddev->thread);
2314 2315
}

2316 2317 2318 2319 2320
/* fetch_block5 - checks the given member device to see if its data needs
 * to be read or computed to satisfy a request.
 *
 * Returns 1 when no more member devices need to be checked, otherwise returns
 * 0 to tell the loop in handle_stripe_fill5 to continue
2321
 */
2322 2323
static int fetch_block5(struct stripe_head *sh, struct stripe_head_state *s,
			int disk_idx, int disks)
2324 2325 2326 2327 2328 2329
{
	struct r5dev *dev = &sh->dev[disk_idx];
	struct r5dev *failed_dev = &sh->dev[s->failed_num];

	/* is the data in this block needed, and can we get it? */
	if (!test_bit(R5_LOCKED, &dev->flags) &&
2330 2331 2332 2333 2334 2335 2336 2337
	    !test_bit(R5_UPTODATE, &dev->flags) &&
	    (dev->toread ||
	     (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)) ||
	     s->syncing || s->expanding ||
	     (s->failed &&
	      (failed_dev->toread ||
	       (failed_dev->towrite &&
		!test_bit(R5_OVERWRITE, &failed_dev->flags)))))) {
2338 2339
		/* We would like to get this block, possibly by computing it,
		 * otherwise read it if the backing disk is insync
2340 2341
		 */
		if ((s->uptodate == disks - 1) &&
2342
		    (s->failed && disk_idx == s->failed_num)) {
2343 2344
			set_bit(STRIPE_COMPUTE_RUN, &sh->state);
			set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
2345 2346
			set_bit(R5_Wantcompute, &dev->flags);
			sh->ops.target = disk_idx;
2347
			sh->ops.target2 = -1;
2348 2349
			s->req_compute = 1;
			/* Careful: from this point on 'uptodate' is in the eye
2350
			 * of raid_run_ops which services 'compute' operations
2351 2352 2353 2354 2355
			 * before writes. R5_Wantcompute flags a block that will
			 * be R5_UPTODATE by the time it is needed for a
			 * subsequent operation.
			 */
			s->uptodate++;
2356
			return 1; /* uptodate + compute == disks */
2357
		} else if (test_bit(R5_Insync, &dev->flags)) {
2358 2359 2360 2361 2362 2363 2364 2365
			set_bit(R5_LOCKED, &dev->flags);
			set_bit(R5_Wantread, &dev->flags);
			s->locked++;
			pr_debug("Reading block %d (sync=%d)\n", disk_idx,
				s->syncing);
		}
	}

2366
	return 0;
2367 2368
}

2369 2370 2371 2372
/**
 * handle_stripe_fill5 - read or compute data to satisfy pending requests.
 */
static void handle_stripe_fill5(struct stripe_head *sh,
2373 2374 2375
			struct stripe_head_state *s, int disks)
{
	int i;
2376 2377 2378 2379 2380

	/* look for blocks to read/compute, skip this if a compute
	 * is already in flight, or if the stripe contents are in the
	 * midst of changing due to a write
	 */
2381
	if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state &&
2382
	    !sh->reconstruct_state)
2383
		for (i = disks; i--; )
2384
			if (fetch_block5(sh, s, i, disks))
2385
				break;
2386 2387 2388
	set_bit(STRIPE_HANDLE, &sh->state);
}

2389 2390 2391 2392 2393 2394 2395 2396
/* fetch_block6 - checks the given member device to see if its data needs
 * to be read or computed to satisfy a request.
 *
 * Returns 1 when no more member devices need to be checked, otherwise returns
 * 0 to tell the loop in handle_stripe_fill6 to continue
 */
static int fetch_block6(struct stripe_head *sh, struct stripe_head_state *s,
			 struct r6_state *r6s, int disk_idx, int disks)
2397
{
2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420
	struct r5dev *dev = &sh->dev[disk_idx];
	struct r5dev *fdev[2] = { &sh->dev[r6s->failed_num[0]],
				  &sh->dev[r6s->failed_num[1]] };

	if (!test_bit(R5_LOCKED, &dev->flags) &&
	    !test_bit(R5_UPTODATE, &dev->flags) &&
	    (dev->toread ||
	     (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)) ||
	     s->syncing || s->expanding ||
	     (s->failed >= 1 &&
	      (fdev[0]->toread || s->to_write)) ||
	     (s->failed >= 2 &&
	      (fdev[1]->toread || s->to_write)))) {
		/* we would like to get this block, possibly by computing it,
		 * otherwise read it if the backing disk is insync
		 */
		BUG_ON(test_bit(R5_Wantcompute, &dev->flags));
		BUG_ON(test_bit(R5_Wantread, &dev->flags));
		if ((s->uptodate == disks - 1) &&
		    (s->failed && (disk_idx == r6s->failed_num[0] ||
				   disk_idx == r6s->failed_num[1]))) {
			/* have disk failed, and we're requested to fetch it;
			 * do compute it
2421
			 */
2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442
			pr_debug("Computing stripe %llu block %d\n",
			       (unsigned long long)sh->sector, disk_idx);
			set_bit(STRIPE_COMPUTE_RUN, &sh->state);
			set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
			set_bit(R5_Wantcompute, &dev->flags);
			sh->ops.target = disk_idx;
			sh->ops.target2 = -1; /* no 2nd target */
			s->req_compute = 1;
			s->uptodate++;
			return 1;
		} else if (s->uptodate == disks-2 && s->failed >= 2) {
			/* Computing 2-failure is *very* expensive; only
			 * do it if failed >= 2
			 */
			int other;
			for (other = disks; other--; ) {
				if (other == disk_idx)
					continue;
				if (!test_bit(R5_UPTODATE,
				      &sh->dev[other].flags))
					break;
2443
			}
2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462
			BUG_ON(other < 0);
			pr_debug("Computing stripe %llu blocks %d,%d\n",
			       (unsigned long long)sh->sector,
			       disk_idx, other);
			set_bit(STRIPE_COMPUTE_RUN, &sh->state);
			set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
			set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags);
			set_bit(R5_Wantcompute, &sh->dev[other].flags);
			sh->ops.target = disk_idx;
			sh->ops.target2 = other;
			s->uptodate += 2;
			s->req_compute = 1;
			return 1;
		} else if (test_bit(R5_Insync, &dev->flags)) {
			set_bit(R5_LOCKED, &dev->flags);
			set_bit(R5_Wantread, &dev->flags);
			s->locked++;
			pr_debug("Reading block %d (sync=%d)\n",
				disk_idx, s->syncing);
2463 2464
		}
	}
2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486

	return 0;
}

/**
 * handle_stripe_fill6 - read or compute data to satisfy pending requests.
 */
static void handle_stripe_fill6(struct stripe_head *sh,
			struct stripe_head_state *s, struct r6_state *r6s,
			int disks)
{
	int i;

	/* look for blocks to read/compute, skip this if a compute
	 * is already in flight, or if the stripe contents are in the
	 * midst of changing due to a write
	 */
	if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state &&
	    !sh->reconstruct_state)
		for (i = disks; i--; )
			if (fetch_block6(sh, s, r6s, i, disks))
				break;
2487 2488 2489 2490
	set_bit(STRIPE_HANDLE, &sh->state);
}


2491
/* handle_stripe_clean_event
2492 2493 2494 2495
 * any written block on an uptodate or failed drive can be returned.
 * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but
 * never LOCKED, so we don't need to test 'failed' directly.
 */
2496
static void handle_stripe_clean_event(raid5_conf_t *conf,
2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509
	struct stripe_head *sh, int disks, struct bio **return_bi)
{
	int i;
	struct r5dev *dev;

	for (i = disks; i--; )
		if (sh->dev[i].written) {
			dev = &sh->dev[i];
			if (!test_bit(R5_LOCKED, &dev->flags) &&
				test_bit(R5_UPTODATE, &dev->flags)) {
				/* We can return any write requests */
				struct bio *wbi, *wbi2;
				int bitmap_end = 0;
2510
				pr_debug("Return write for disc %d\n", i);
2511 2512 2513 2514 2515 2516
				spin_lock_irq(&conf->device_lock);
				wbi = dev->written;
				dev->written = NULL;
				while (wbi && wbi->bi_sector <
					dev->sector + STRIPE_SECTORS) {
					wbi2 = r5_next_bio(wbi, dev->sector);
2517
					if (!raid5_dec_bi_phys_segments(wbi)) {
2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534
						md_write_end(conf->mddev);
						wbi->bi_next = *return_bi;
						*return_bi = wbi;
					}
					wbi = wbi2;
				}
				if (dev->towrite == NULL)
					bitmap_end = 1;
				spin_unlock_irq(&conf->device_lock);
				if (bitmap_end)
					bitmap_endwrite(conf->mddev->bitmap,
							sh->sector,
							STRIPE_SECTORS,
					 !test_bit(STRIPE_DEGRADED, &sh->state),
							0);
			}
		}
2535 2536 2537 2538

	if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
		if (atomic_dec_and_test(&conf->pending_full_writes))
			md_wakeup_thread(conf->mddev->thread);
2539 2540
}

2541
static void handle_stripe_dirtying5(raid5_conf_t *conf,
2542 2543 2544 2545 2546 2547 2548 2549
		struct stripe_head *sh,	struct stripe_head_state *s, int disks)
{
	int rmw = 0, rcw = 0, i;
	for (i = disks; i--; ) {
		/* would I have to read this buffer for read_modify_write */
		struct r5dev *dev = &sh->dev[i];
		if ((dev->towrite || i == sh->pd_idx) &&
		    !test_bit(R5_LOCKED, &dev->flags) &&
2550 2551
		    !(test_bit(R5_UPTODATE, &dev->flags) ||
		      test_bit(R5_Wantcompute, &dev->flags))) {
2552 2553 2554 2555 2556 2557 2558 2559
			if (test_bit(R5_Insync, &dev->flags))
				rmw++;
			else
				rmw += 2*disks;  /* cannot read it */
		}
		/* Would I have to read this buffer for reconstruct_write */
		if (!test_bit(R5_OVERWRITE, &dev->flags) && i != sh->pd_idx &&
		    !test_bit(R5_LOCKED, &dev->flags) &&
2560 2561 2562
		    !(test_bit(R5_UPTODATE, &dev->flags) ||
		    test_bit(R5_Wantcompute, &dev->flags))) {
			if (test_bit(R5_Insync, &dev->flags)) rcw++;
2563 2564 2565 2566
			else
				rcw += 2*disks;
		}
	}
2567
	pr_debug("for sector %llu, rmw=%d rcw=%d\n",
2568 2569 2570 2571 2572 2573 2574 2575
		(unsigned long long)sh->sector, rmw, rcw);
	set_bit(STRIPE_HANDLE, &sh->state);
	if (rmw < rcw && rmw > 0)
		/* prefer read-modify-write, but need to get some data */
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if ((dev->towrite || i == sh->pd_idx) &&
			    !test_bit(R5_LOCKED, &dev->flags) &&
2576 2577
			    !(test_bit(R5_UPTODATE, &dev->flags) ||
			    test_bit(R5_Wantcompute, &dev->flags)) &&
2578 2579 2580
			    test_bit(R5_Insync, &dev->flags)) {
				if (
				  test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
2581
					pr_debug("Read_old block "
2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598
						"%d for r-m-w\n", i);
					set_bit(R5_LOCKED, &dev->flags);
					set_bit(R5_Wantread, &dev->flags);
					s->locked++;
				} else {
					set_bit(STRIPE_DELAYED, &sh->state);
					set_bit(STRIPE_HANDLE, &sh->state);
				}
			}
		}
	if (rcw <= rmw && rcw > 0)
		/* want reconstruct write, but need to get some data */
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if (!test_bit(R5_OVERWRITE, &dev->flags) &&
			    i != sh->pd_idx &&
			    !test_bit(R5_LOCKED, &dev->flags) &&
2599 2600
			    !(test_bit(R5_UPTODATE, &dev->flags) ||
			    test_bit(R5_Wantcompute, &dev->flags)) &&
2601 2602 2603
			    test_bit(R5_Insync, &dev->flags)) {
				if (
				  test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
2604
					pr_debug("Read_old block "
2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617
						"%d for Reconstruct\n", i);
					set_bit(R5_LOCKED, &dev->flags);
					set_bit(R5_Wantread, &dev->flags);
					s->locked++;
				} else {
					set_bit(STRIPE_DELAYED, &sh->state);
					set_bit(STRIPE_HANDLE, &sh->state);
				}
			}
		}
	/* now if nothing is locked, and if we have enough data,
	 * we can start a write request
	 */
2618 2619
	/* since handle_stripe can be called at any time we need to handle the
	 * case where a compute block operation has been submitted and then a
2620 2621
	 * subsequent call wants to start a write request.  raid_run_ops only
	 * handles the case where compute block and reconstruct are requested
2622 2623 2624
	 * simultaneously.  If this is not the case then new writes need to be
	 * held off until the compute completes.
	 */
2625 2626 2627
	if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) &&
	    (s->locked == 0 && (rcw == 0 || rmw == 0) &&
	    !test_bit(STRIPE_BIT_DELAY, &sh->state)))
2628
		schedule_reconstruction(sh, s, rcw == 0, 0);
2629 2630
}

2631
static void handle_stripe_dirtying6(raid5_conf_t *conf,
2632 2633 2634
		struct stripe_head *sh,	struct stripe_head_state *s,
		struct r6_state *r6s, int disks)
{
2635
	int rcw = 0, pd_idx = sh->pd_idx, i;
N
NeilBrown 已提交
2636
	int qd_idx = sh->qd_idx;
2637 2638

	set_bit(STRIPE_HANDLE, &sh->state);
2639 2640
	for (i = disks; i--; ) {
		struct r5dev *dev = &sh->dev[i];
2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664
		/* check if we haven't enough data */
		if (!test_bit(R5_OVERWRITE, &dev->flags) &&
		    i != pd_idx && i != qd_idx &&
		    !test_bit(R5_LOCKED, &dev->flags) &&
		    !(test_bit(R5_UPTODATE, &dev->flags) ||
		      test_bit(R5_Wantcompute, &dev->flags))) {
			rcw++;
			if (!test_bit(R5_Insync, &dev->flags))
				continue; /* it's a failed drive */

			if (
			  test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
				pr_debug("Read_old stripe %llu "
					"block %d for Reconstruct\n",
				     (unsigned long long)sh->sector, i);
				set_bit(R5_LOCKED, &dev->flags);
				set_bit(R5_Wantread, &dev->flags);
				s->locked++;
			} else {
				pr_debug("Request delayed stripe %llu "
					"block %d for Reconstruct\n",
				     (unsigned long long)sh->sector, i);
				set_bit(STRIPE_DELAYED, &sh->state);
				set_bit(STRIPE_HANDLE, &sh->state);
2665 2666 2667 2668 2669 2670
			}
		}
	}
	/* now if nothing is locked, and if we have enough data, we can start a
	 * write request
	 */
2671 2672
	if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) &&
	    s->locked == 0 && rcw == 0 &&
2673
	    !test_bit(STRIPE_BIT_DELAY, &sh->state)) {
2674
		schedule_reconstruction(sh, s, 1, 0);
2675 2676 2677 2678 2679 2680
	}
}

static void handle_parity_checks5(raid5_conf_t *conf, struct stripe_head *sh,
				struct stripe_head_state *s, int disks)
{
2681
	struct r5dev *dev = NULL;
2682

2683
	set_bit(STRIPE_HANDLE, &sh->state);
2684

2685 2686 2687
	switch (sh->check_state) {
	case check_state_idle:
		/* start a new check operation if there are no failures */
2688 2689
		if (s->failed == 0) {
			BUG_ON(s->uptodate != disks);
2690 2691
			sh->check_state = check_state_run;
			set_bit(STRIPE_OP_CHECK, &s->ops_request);
2692 2693
			clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
			s->uptodate--;
2694
			break;
2695
		}
2696 2697 2698 2699 2700 2701 2702 2703 2704 2705
		dev = &sh->dev[s->failed_num];
		/* fall through */
	case check_state_compute_result:
		sh->check_state = check_state_idle;
		if (!dev)
			dev = &sh->dev[sh->pd_idx];

		/* check that a write has not made the stripe insync */
		if (test_bit(STRIPE_INSYNC, &sh->state))
			break;
D
Dan Williams 已提交
2706

2707 2708 2709 2710 2711
		/* either failed parity check, or recovery is happening */
		BUG_ON(!test_bit(R5_UPTODATE, &dev->flags));
		BUG_ON(s->uptodate != disks);

		set_bit(R5_LOCKED, &dev->flags);
2712
		s->locked++;
2713
		set_bit(R5_Wantwrite, &dev->flags);
2714

2715 2716
		clear_bit(STRIPE_DEGRADED, &sh->state);
		set_bit(STRIPE_INSYNC, &sh->state);
2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732
		break;
	case check_state_run:
		break; /* we will be called again upon completion */
	case check_state_check_result:
		sh->check_state = check_state_idle;

		/* if a failure occurred during the check operation, leave
		 * STRIPE_INSYNC not set and let the stripe be handled again
		 */
		if (s->failed)
			break;

		/* handle a successful check operation, if parity is correct
		 * we are done.  Otherwise update the mismatch count and repair
		 * parity if !MD_RECOVERY_CHECK
		 */
D
Dan Williams 已提交
2733
		if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0)
2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744
			/* parity is correct (on disc,
			 * not in buffer any more)
			 */
			set_bit(STRIPE_INSYNC, &sh->state);
		else {
			conf->mddev->resync_mismatches += STRIPE_SECTORS;
			if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery))
				/* don't try to repair!! */
				set_bit(STRIPE_INSYNC, &sh->state);
			else {
				sh->check_state = check_state_compute_run;
2745
				set_bit(STRIPE_COMPUTE_RUN, &sh->state);
2746 2747 2748 2749
				set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
				set_bit(R5_Wantcompute,
					&sh->dev[sh->pd_idx].flags);
				sh->ops.target = sh->pd_idx;
2750
				sh->ops.target2 = -1;
2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761
				s->uptodate++;
			}
		}
		break;
	case check_state_compute_run:
		break;
	default:
		printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
		       __func__, sh->check_state,
		       (unsigned long long) sh->sector);
		BUG();
2762 2763 2764 2765 2766
	}
}


static void handle_parity_checks6(raid5_conf_t *conf, struct stripe_head *sh,
2767 2768
				  struct stripe_head_state *s,
				  struct r6_state *r6s, int disks)
2769 2770
{
	int pd_idx = sh->pd_idx;
N
NeilBrown 已提交
2771
	int qd_idx = sh->qd_idx;
2772
	struct r5dev *dev;
2773 2774 2775 2776

	set_bit(STRIPE_HANDLE, &sh->state);

	BUG_ON(s->failed > 2);
2777

2778 2779 2780 2781 2782 2783
	/* Want to check and possibly repair P and Q.
	 * However there could be one 'failed' device, in which
	 * case we can only check one of them, possibly using the
	 * other to generate missing data
	 */

2784 2785 2786
	switch (sh->check_state) {
	case check_state_idle:
		/* start a new check operation if there are < 2 failures */
2787
		if (s->failed == r6s->q_failed) {
2788
			/* The only possible failed device holds Q, so it
2789 2790 2791
			 * makes sense to check P (If anything else were failed,
			 * we would have used P to recreate it).
			 */
2792
			sh->check_state = check_state_run;
2793 2794
		}
		if (!r6s->q_failed && s->failed < 2) {
2795
			/* Q is not failed, and we didn't use it to generate
2796 2797
			 * anything, so it makes sense to check it
			 */
2798 2799 2800 2801
			if (sh->check_state == check_state_run)
				sh->check_state = check_state_run_pq;
			else
				sh->check_state = check_state_run_q;
2802 2803
		}

2804 2805
		/* discard potentially stale zero_sum_result */
		sh->ops.zero_sum_result = 0;
2806

2807 2808 2809 2810
		if (sh->check_state == check_state_run) {
			/* async_xor_zero_sum destroys the contents of P */
			clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
			s->uptodate--;
2811
		}
2812 2813 2814 2815 2816 2817 2818
		if (sh->check_state >= check_state_run &&
		    sh->check_state <= check_state_run_pq) {
			/* async_syndrome_zero_sum preserves P and Q, so
			 * no need to mark them !uptodate here
			 */
			set_bit(STRIPE_OP_CHECK, &s->ops_request);
			break;
2819 2820
		}

2821 2822 2823 2824 2825
		/* we have 2-disk failure */
		BUG_ON(s->failed != 2);
		/* fall through */
	case check_state_compute_result:
		sh->check_state = check_state_idle;
2826

2827 2828 2829
		/* check that a write has not made the stripe insync */
		if (test_bit(STRIPE_INSYNC, &sh->state))
			break;
2830 2831

		/* now write out any block on a failed drive,
2832
		 * or P or Q if they were recomputed
2833
		 */
2834
		BUG_ON(s->uptodate < disks - 1); /* We don't need Q to recover */
2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846
		if (s->failed == 2) {
			dev = &sh->dev[r6s->failed_num[1]];
			s->locked++;
			set_bit(R5_LOCKED, &dev->flags);
			set_bit(R5_Wantwrite, &dev->flags);
		}
		if (s->failed >= 1) {
			dev = &sh->dev[r6s->failed_num[0]];
			s->locked++;
			set_bit(R5_LOCKED, &dev->flags);
			set_bit(R5_Wantwrite, &dev->flags);
		}
2847
		if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
2848 2849 2850 2851 2852
			dev = &sh->dev[pd_idx];
			s->locked++;
			set_bit(R5_LOCKED, &dev->flags);
			set_bit(R5_Wantwrite, &dev->flags);
		}
2853
		if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
2854 2855 2856 2857 2858 2859 2860 2861
			dev = &sh->dev[qd_idx];
			s->locked++;
			set_bit(R5_LOCKED, &dev->flags);
			set_bit(R5_Wantwrite, &dev->flags);
		}
		clear_bit(STRIPE_DEGRADED, &sh->state);

		set_bit(STRIPE_INSYNC, &sh->state);
2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925
		break;
	case check_state_run:
	case check_state_run_q:
	case check_state_run_pq:
		break; /* we will be called again upon completion */
	case check_state_check_result:
		sh->check_state = check_state_idle;

		/* handle a successful check operation, if parity is correct
		 * we are done.  Otherwise update the mismatch count and repair
		 * parity if !MD_RECOVERY_CHECK
		 */
		if (sh->ops.zero_sum_result == 0) {
			/* both parities are correct */
			if (!s->failed)
				set_bit(STRIPE_INSYNC, &sh->state);
			else {
				/* in contrast to the raid5 case we can validate
				 * parity, but still have a failure to write
				 * back
				 */
				sh->check_state = check_state_compute_result;
				/* Returning at this point means that we may go
				 * off and bring p and/or q uptodate again so
				 * we make sure to check zero_sum_result again
				 * to verify if p or q need writeback
				 */
			}
		} else {
			conf->mddev->resync_mismatches += STRIPE_SECTORS;
			if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery))
				/* don't try to repair!! */
				set_bit(STRIPE_INSYNC, &sh->state);
			else {
				int *target = &sh->ops.target;

				sh->ops.target = -1;
				sh->ops.target2 = -1;
				sh->check_state = check_state_compute_run;
				set_bit(STRIPE_COMPUTE_RUN, &sh->state);
				set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
				if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
					set_bit(R5_Wantcompute,
						&sh->dev[pd_idx].flags);
					*target = pd_idx;
					target = &sh->ops.target2;
					s->uptodate++;
				}
				if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
					set_bit(R5_Wantcompute,
						&sh->dev[qd_idx].flags);
					*target = qd_idx;
					s->uptodate++;
				}
			}
		}
		break;
	case check_state_compute_run:
		break;
	default:
		printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
		       __func__, sh->check_state,
		       (unsigned long long) sh->sector);
		BUG();
2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936
	}
}

static void handle_stripe_expansion(raid5_conf_t *conf, struct stripe_head *sh,
				struct r6_state *r6s)
{
	int i;

	/* We have read all the blocks in this stripe and now we need to
	 * copy some of them into a target stripe for expand.
	 */
2937
	struct dma_async_tx_descriptor *tx = NULL;
2938 2939
	clear_bit(STRIPE_EXPAND_SOURCE, &sh->state);
	for (i = 0; i < sh->disks; i++)
N
NeilBrown 已提交
2940
		if (i != sh->pd_idx && i != sh->qd_idx) {
2941
			int dd_idx, j;
2942
			struct stripe_head *sh2;
2943
			struct async_submit_ctl submit;
2944

2945
			sector_t bn = compute_blocknr(sh, i, 1);
2946 2947
			sector_t s = raid5_compute_sector(conf, bn, 0,
							  &dd_idx, NULL);
2948
			sh2 = get_active_stripe(conf, s, 0, 1, 1);
2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960
			if (sh2 == NULL)
				/* so far only the early blocks of this stripe
				 * have been requested.  When later blocks
				 * get requested, we will try again
				 */
				continue;
			if (!test_bit(STRIPE_EXPANDING, &sh2->state) ||
			   test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) {
				/* must have already done this block */
				release_stripe(sh2);
				continue;
			}
2961 2962

			/* place all the copies on one channel */
2963
			init_async_submit(&submit, 0, tx, NULL, NULL, NULL);
2964
			tx = async_memcpy(sh2->dev[dd_idx].page,
2965
					  sh->dev[i].page, 0, 0, STRIPE_SIZE,
2966
					  &submit);
2967

2968 2969 2970 2971
			set_bit(R5_Expanded, &sh2->dev[dd_idx].flags);
			set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags);
			for (j = 0; j < conf->raid_disks; j++)
				if (j != sh2->pd_idx &&
2972
				    (!r6s || j != sh2->qd_idx) &&
2973 2974 2975 2976 2977 2978 2979
				    !test_bit(R5_Expanded, &sh2->dev[j].flags))
					break;
			if (j == conf->raid_disks) {
				set_bit(STRIPE_EXPAND_READY, &sh2->state);
				set_bit(STRIPE_HANDLE, &sh2->state);
			}
			release_stripe(sh2);
2980

2981
		}
2982 2983 2984 2985 2986
	/* done submitting copies, wait for them to complete */
	if (tx) {
		async_tx_ack(tx);
		dma_wait_for_async_tx(tx);
	}
2987
}
L
Linus Torvalds 已提交
2988

2989

L
Linus Torvalds 已提交
2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005
/*
 * handle_stripe - do things to a stripe.
 *
 * We lock the stripe and then examine the state of various bits
 * to see what needs to be done.
 * Possible results:
 *    return some read request which now have data
 *    return some write requests which are safely on disc
 *    schedule a read on some buffers
 *    schedule a write of some buffers
 *    return confirmation of parity correctness
 *
 * buffers are taken off read_list or write_list, and bh_cache buffers
 * get BH_Lock set before the stripe lock is released.
 *
 */
3006

3007
static void handle_stripe5(struct stripe_head *sh)
L
Linus Torvalds 已提交
3008 3009
{
	raid5_conf_t *conf = sh->raid_conf;
3010 3011 3012
	int disks = sh->disks, i;
	struct bio *return_bi = NULL;
	struct stripe_head_state s;
L
Linus Torvalds 已提交
3013
	struct r5dev *dev;
3014
	mdk_rdev_t *blocked_rdev = NULL;
3015
	int prexor;
3016
	int dec_preread_active = 0;
L
Linus Torvalds 已提交
3017

3018
	memset(&s, 0, sizeof(s));
3019 3020 3021 3022
	pr_debug("handling stripe %llu, state=%#lx cnt=%d, pd_idx=%d check:%d "
		 "reconstruct:%d\n", (unsigned long long)sh->sector, sh->state,
		 atomic_read(&sh->count), sh->pd_idx, sh->check_state,
		 sh->reconstruct_state);
L
Linus Torvalds 已提交
3023 3024

	spin_lock(&sh->lock);
3025 3026 3027 3028
	if (test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) {
		set_bit(STRIPE_SYNCING, &sh->state);
		clear_bit(STRIPE_INSYNC, &sh->state);
	}
L
Linus Torvalds 已提交
3029 3030 3031
	clear_bit(STRIPE_HANDLE, &sh->state);
	clear_bit(STRIPE_DELAYED, &sh->state);

3032 3033 3034
	s.syncing = test_bit(STRIPE_SYNCING, &sh->state);
	s.expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state);
	s.expanded = test_bit(STRIPE_EXPAND_READY, &sh->state);
N
Neil Brown 已提交
3035

3036
	/* Now to look around and see what can be done */
3037
	rcu_read_lock();
L
Linus Torvalds 已提交
3038 3039
	for (i=disks; i--; ) {
		mdk_rdev_t *rdev;
3040 3041

		dev = &sh->dev[i];
L
Linus Torvalds 已提交
3042

3043 3044 3045 3046 3047 3048 3049
		pr_debug("check %d: state 0x%lx toread %p read %p write %p "
			"written %p\n",	i, dev->flags, dev->toread, dev->read,
			dev->towrite, dev->written);

		/* maybe we can request a biofill operation
		 *
		 * new wantfill requests are only permitted while
3050
		 * ops_complete_biofill is guaranteed to be inactive
3051 3052
		 */
		if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread &&
3053
		    !test_bit(STRIPE_BIOFILL_RUN, &sh->state))
3054
			set_bit(R5_Wantfill, &dev->flags);
L
Linus Torvalds 已提交
3055 3056

		/* now count some things */
3057 3058
		if (test_bit(R5_LOCKED, &dev->flags)) s.locked++;
		if (test_bit(R5_UPTODATE, &dev->flags)) s.uptodate++;
3059
		if (test_bit(R5_Wantcompute, &dev->flags)) s.compute++;
L
Linus Torvalds 已提交
3060

3061 3062 3063
		if (test_bit(R5_Wantfill, &dev->flags))
			s.to_fill++;
		else if (dev->toread)
3064
			s.to_read++;
L
Linus Torvalds 已提交
3065
		if (dev->towrite) {
3066
			s.to_write++;
L
Linus Torvalds 已提交
3067
			if (!test_bit(R5_OVERWRITE, &dev->flags))
3068
				s.non_overwrite++;
L
Linus Torvalds 已提交
3069
		}
3070 3071
		if (dev->written)
			s.written++;
3072
		rdev = rcu_dereference(conf->disks[i].rdev);
3073 3074
		if (blocked_rdev == NULL &&
		    rdev && unlikely(test_bit(Blocked, &rdev->flags))) {
3075 3076 3077
			blocked_rdev = rdev;
			atomic_inc(&rdev->nr_pending);
		}
3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088
		clear_bit(R5_Insync, &dev->flags);
		if (!rdev)
			/* Not in-sync */;
		else if (test_bit(In_sync, &rdev->flags))
			set_bit(R5_Insync, &dev->flags);
		else {
			/* could be in-sync depending on recovery/reshape status */
			if (sh->sector + STRIPE_SECTORS <= rdev->recovery_offset)
				set_bit(R5_Insync, &dev->flags);
		}
		if (!test_bit(R5_Insync, &dev->flags)) {
N
NeilBrown 已提交
3089
			/* The ReadError flag will just be confusing now */
3090 3091 3092
			clear_bit(R5_ReadError, &dev->flags);
			clear_bit(R5_ReWrite, &dev->flags);
		}
3093 3094 3095
		if (test_bit(R5_ReadError, &dev->flags))
			clear_bit(R5_Insync, &dev->flags);
		if (!test_bit(R5_Insync, &dev->flags)) {
3096 3097
			s.failed++;
			s.failed_num = i;
3098
		}
L
Linus Torvalds 已提交
3099
	}
3100
	rcu_read_unlock();
3101

3102
	if (unlikely(blocked_rdev)) {
3103 3104 3105 3106 3107 3108 3109 3110
		if (s.syncing || s.expanding || s.expanded ||
		    s.to_write || s.written) {
			set_bit(STRIPE_HANDLE, &sh->state);
			goto unlock;
		}
		/* There is nothing for the blocked_rdev to block */
		rdev_dec_pending(blocked_rdev, conf->mddev);
		blocked_rdev = NULL;
3111 3112
	}

3113 3114 3115 3116
	if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) {
		set_bit(STRIPE_OP_BIOFILL, &s.ops_request);
		set_bit(STRIPE_BIOFILL_RUN, &sh->state);
	}
3117

3118
	pr_debug("locked=%d uptodate=%d to_read=%d"
L
Linus Torvalds 已提交
3119
		" to_write=%d failed=%d failed_num=%d\n",
3120 3121
		s.locked, s.uptodate, s.to_read, s.to_write,
		s.failed, s.failed_num);
L
Linus Torvalds 已提交
3122 3123 3124
	/* check if the array has lost two devices and, if so, some requests might
	 * need to be failed
	 */
3125
	if (s.failed > 1 && s.to_read+s.to_write+s.written)
3126
		handle_failed_stripe(conf, sh, &s, disks, &return_bi);
3127
	if (s.failed > 1 && s.syncing) {
L
Linus Torvalds 已提交
3128 3129
		md_done_sync(conf->mddev, STRIPE_SECTORS,0);
		clear_bit(STRIPE_SYNCING, &sh->state);
3130
		s.syncing = 0;
L
Linus Torvalds 已提交
3131 3132 3133 3134 3135 3136
	}

	/* might be able to return some write requests if the parity block
	 * is safe, or on a failed drive
	 */
	dev = &sh->dev[sh->pd_idx];
3137 3138 3139 3140 3141
	if ( s.written &&
	     ((test_bit(R5_Insync, &dev->flags) &&
	       !test_bit(R5_LOCKED, &dev->flags) &&
	       test_bit(R5_UPTODATE, &dev->flags)) ||
	       (s.failed == 1 && s.failed_num == sh->pd_idx)))
3142
		handle_stripe_clean_event(conf, sh, disks, &return_bi);
L
Linus Torvalds 已提交
3143 3144 3145 3146 3147

	/* Now we might consider reading some blocks, either to check/generate
	 * parity, or to satisfy requests
	 * or to load a block that is being partially written.
	 */
3148
	if (s.to_read || s.non_overwrite ||
3149
	    (s.syncing && (s.uptodate + s.compute < disks)) || s.expanding)
3150
		handle_stripe_fill5(sh, &s, disks);
L
Linus Torvalds 已提交
3151

3152 3153 3154
	/* Now we check to see if any write operations have recently
	 * completed
	 */
3155
	prexor = 0;
3156
	if (sh->reconstruct_state == reconstruct_state_prexor_drain_result)
3157
		prexor = 1;
3158 3159
	if (sh->reconstruct_state == reconstruct_state_drain_result ||
	    sh->reconstruct_state == reconstruct_state_prexor_drain_result) {
3160
		sh->reconstruct_state = reconstruct_state_idle;
3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171

		/* All the 'written' buffers and the parity block are ready to
		 * be written back to disk
		 */
		BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags));
		for (i = disks; i--; ) {
			dev = &sh->dev[i];
			if (test_bit(R5_LOCKED, &dev->flags) &&
				(i == sh->pd_idx || dev->written)) {
				pr_debug("Writing block %d\n", i);
				set_bit(R5_Wantwrite, &dev->flags);
3172 3173
				if (prexor)
					continue;
3174 3175 3176 3177 3178
				if (!test_bit(R5_Insync, &dev->flags) ||
				    (i == sh->pd_idx && s.failed == 0))
					set_bit(STRIPE_INSYNC, &sh->state);
			}
		}
3179 3180
		if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
			dec_preread_active = 1;
3181 3182 3183 3184 3185 3186 3187 3188
	}

	/* Now to consider new write requests and what else, if anything
	 * should be read.  We do not handle new writes when:
	 * 1/ A 'write' operation (copy+xor) is already in flight.
	 * 2/ A 'check' operation is in flight, as it may clobber the parity
	 *    block.
	 */
3189
	if (s.to_write && !sh->reconstruct_state && !sh->check_state)
3190
		handle_stripe_dirtying5(conf, sh, &s, disks);
L
Linus Torvalds 已提交
3191 3192

	/* maybe we need to check and possibly fix the parity for this stripe
3193 3194 3195
	 * Any reads will already have been scheduled, so we just see if enough
	 * data is available.  The parity check is held off while parity
	 * dependent operations are in flight.
L
Linus Torvalds 已提交
3196
	 */
3197 3198
	if (sh->check_state ||
	    (s.syncing && s.locked == 0 &&
3199
	     !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
3200
	     !test_bit(STRIPE_INSYNC, &sh->state)))
3201
		handle_parity_checks5(conf, sh, &s, disks);
3202

3203
	if (s.syncing && s.locked == 0 && test_bit(STRIPE_INSYNC, &sh->state)) {
L
Linus Torvalds 已提交
3204 3205 3206
		md_done_sync(conf->mddev, STRIPE_SECTORS,1);
		clear_bit(STRIPE_SYNCING, &sh->state);
	}
3207 3208 3209 3210

	/* If the failed drive is just a ReadError, then we might need to progress
	 * the repair/check process
	 */
3211 3212 3213 3214
	if (s.failed == 1 && !conf->mddev->ro &&
	    test_bit(R5_ReadError, &sh->dev[s.failed_num].flags)
	    && !test_bit(R5_LOCKED, &sh->dev[s.failed_num].flags)
	    && test_bit(R5_UPTODATE, &sh->dev[s.failed_num].flags)
3215
		) {
3216
		dev = &sh->dev[s.failed_num];
3217 3218 3219 3220
		if (!test_bit(R5_ReWrite, &dev->flags)) {
			set_bit(R5_Wantwrite, &dev->flags);
			set_bit(R5_ReWrite, &dev->flags);
			set_bit(R5_LOCKED, &dev->flags);
3221
			s.locked++;
3222 3223 3224 3225
		} else {
			/* let's read it back */
			set_bit(R5_Wantread, &dev->flags);
			set_bit(R5_LOCKED, &dev->flags);
3226
			s.locked++;
3227 3228 3229
		}
	}

3230 3231
	/* Finish reconstruct operations initiated by the expansion process */
	if (sh->reconstruct_state == reconstruct_state_result) {
3232
		struct stripe_head *sh2
3233
			= get_active_stripe(conf, sh->sector, 1, 1, 1);
3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248
		if (sh2 && test_bit(STRIPE_EXPAND_SOURCE, &sh2->state)) {
			/* sh cannot be written until sh2 has been read.
			 * so arrange for sh to be delayed a little
			 */
			set_bit(STRIPE_DELAYED, &sh->state);
			set_bit(STRIPE_HANDLE, &sh->state);
			if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE,
					      &sh2->state))
				atomic_inc(&conf->preread_active_stripes);
			release_stripe(sh2);
			goto unlock;
		}
		if (sh2)
			release_stripe(sh2);

3249
		sh->reconstruct_state = reconstruct_state_idle;
3250
		clear_bit(STRIPE_EXPANDING, &sh->state);
D
Dan Williams 已提交
3251
		for (i = conf->raid_disks; i--; ) {
3252
			set_bit(R5_Wantwrite, &sh->dev[i].flags);
D
Dan Williams 已提交
3253
			set_bit(R5_LOCKED, &sh->dev[i].flags);
3254
			s.locked++;
D
Dan Williams 已提交
3255
		}
3256 3257 3258
	}

	if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) &&
3259
	    !sh->reconstruct_state) {
3260 3261
		/* Need to write out all blocks after computing parity */
		sh->disks = conf->raid_disks;
3262
		stripe_set_idx(sh->sector, conf, 0, sh);
3263
		schedule_reconstruction(sh, &s, 1, 1);
3264
	} else if (s.expanded && !sh->reconstruct_state && s.locked == 0) {
3265
		clear_bit(STRIPE_EXPAND_READY, &sh->state);
3266
		atomic_dec(&conf->reshape_stripes);
3267 3268 3269 3270
		wake_up(&conf->wait_for_overlap);
		md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
	}

3271
	if (s.expanding && s.locked == 0 &&
3272
	    !test_bit(STRIPE_COMPUTE_RUN, &sh->state))
3273
		handle_stripe_expansion(conf, sh, NULL);
3274

3275
 unlock:
L
Linus Torvalds 已提交
3276 3277
	spin_unlock(&sh->lock);

3278 3279 3280 3281
	/* wait for this device to become unblocked */
	if (unlikely(blocked_rdev))
		md_wait_for_blocked_rdev(blocked_rdev, conf->mddev);

3282
	if (s.ops_request)
3283
		raid_run_ops(sh, s.ops_request);
3284

3285
	ops_run_io(sh, &s);
L
Linus Torvalds 已提交
3286

3287 3288
	if (dec_preread_active) {
		/* We delay this until after ops_run_io so that if make_request
T
Tejun Heo 已提交
3289
		 * is waiting on a flush, it won't continue until the writes
3290 3291 3292 3293 3294 3295 3296
		 * have actually been submitted.
		 */
		atomic_dec(&conf->preread_active_stripes);
		if (atomic_read(&conf->preread_active_stripes) <
		    IO_THRESHOLD)
			md_wakeup_thread(conf->mddev->thread);
	}
3297
	return_io(return_bi);
L
Linus Torvalds 已提交
3298 3299
}

3300
static void handle_stripe6(struct stripe_head *sh)
L
Linus Torvalds 已提交
3301
{
3302
	raid5_conf_t *conf = sh->raid_conf;
3303
	int disks = sh->disks;
3304
	struct bio *return_bi = NULL;
N
NeilBrown 已提交
3305
	int i, pd_idx = sh->pd_idx, qd_idx = sh->qd_idx;
3306 3307
	struct stripe_head_state s;
	struct r6_state r6s;
3308
	struct r5dev *dev, *pdev, *qdev;
3309
	mdk_rdev_t *blocked_rdev = NULL;
3310
	int dec_preread_active = 0;
L
Linus Torvalds 已提交
3311

3312
	pr_debug("handling stripe %llu, state=%#lx cnt=%d, "
3313
		"pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n",
3314
	       (unsigned long long)sh->sector, sh->state,
3315 3316
	       atomic_read(&sh->count), pd_idx, qd_idx,
	       sh->check_state, sh->reconstruct_state);
3317
	memset(&s, 0, sizeof(s));
3318

3319
	spin_lock(&sh->lock);
3320 3321 3322 3323
	if (test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) {
		set_bit(STRIPE_SYNCING, &sh->state);
		clear_bit(STRIPE_INSYNC, &sh->state);
	}
3324 3325 3326
	clear_bit(STRIPE_HANDLE, &sh->state);
	clear_bit(STRIPE_DELAYED, &sh->state);

3327 3328 3329
	s.syncing = test_bit(STRIPE_SYNCING, &sh->state);
	s.expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state);
	s.expanded = test_bit(STRIPE_EXPAND_READY, &sh->state);
3330
	/* Now to look around and see what can be done */
L
Linus Torvalds 已提交
3331 3332

	rcu_read_lock();
3333 3334 3335
	for (i=disks; i--; ) {
		mdk_rdev_t *rdev;
		dev = &sh->dev[i];
L
Linus Torvalds 已提交
3336

3337
		pr_debug("check %d: state 0x%lx read %p write %p written %p\n",
3338
			i, dev->flags, dev->toread, dev->towrite, dev->written);
3339 3340 3341 3342 3343 3344 3345 3346
		/* maybe we can reply to a read
		 *
		 * new wantfill requests are only permitted while
		 * ops_complete_biofill is guaranteed to be inactive
		 */
		if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread &&
		    !test_bit(STRIPE_BIOFILL_RUN, &sh->state))
			set_bit(R5_Wantfill, &dev->flags);
L
Linus Torvalds 已提交
3347

3348
		/* now count some things */
3349 3350
		if (test_bit(R5_LOCKED, &dev->flags)) s.locked++;
		if (test_bit(R5_UPTODATE, &dev->flags)) s.uptodate++;
3351 3352 3353 3354
		if (test_bit(R5_Wantcompute, &dev->flags)) {
			s.compute++;
			BUG_ON(s.compute > 2);
		}
L
Linus Torvalds 已提交
3355

3356 3357 3358
		if (test_bit(R5_Wantfill, &dev->flags)) {
			s.to_fill++;
		} else if (dev->toread)
3359
			s.to_read++;
3360
		if (dev->towrite) {
3361
			s.to_write++;
3362
			if (!test_bit(R5_OVERWRITE, &dev->flags))
3363
				s.non_overwrite++;
3364
		}
3365 3366
		if (dev->written)
			s.written++;
3367
		rdev = rcu_dereference(conf->disks[i].rdev);
3368 3369
		if (blocked_rdev == NULL &&
		    rdev && unlikely(test_bit(Blocked, &rdev->flags))) {
3370 3371 3372
			blocked_rdev = rdev;
			atomic_inc(&rdev->nr_pending);
		}
3373 3374 3375 3376 3377 3378 3379 3380 3381 3382 3383
		clear_bit(R5_Insync, &dev->flags);
		if (!rdev)
			/* Not in-sync */;
		else if (test_bit(In_sync, &rdev->flags))
			set_bit(R5_Insync, &dev->flags);
		else {
			/* in sync if before recovery_offset */
			if (sh->sector + STRIPE_SECTORS <= rdev->recovery_offset)
				set_bit(R5_Insync, &dev->flags);
		}
		if (!test_bit(R5_Insync, &dev->flags)) {
3384 3385 3386
			/* The ReadError flag will just be confusing now */
			clear_bit(R5_ReadError, &dev->flags);
			clear_bit(R5_ReWrite, &dev->flags);
L
Linus Torvalds 已提交
3387
		}
3388 3389 3390
		if (test_bit(R5_ReadError, &dev->flags))
			clear_bit(R5_Insync, &dev->flags);
		if (!test_bit(R5_Insync, &dev->flags)) {
3391 3392 3393
			if (s.failed < 2)
				r6s.failed_num[s.failed] = i;
			s.failed++;
3394
		}
L
Linus Torvalds 已提交
3395 3396
	}
	rcu_read_unlock();
3397 3398

	if (unlikely(blocked_rdev)) {
3399 3400 3401 3402 3403 3404 3405 3406
		if (s.syncing || s.expanding || s.expanded ||
		    s.to_write || s.written) {
			set_bit(STRIPE_HANDLE, &sh->state);
			goto unlock;
		}
		/* There is nothing for the blocked_rdev to block */
		rdev_dec_pending(blocked_rdev, conf->mddev);
		blocked_rdev = NULL;
3407
	}
3408

3409 3410 3411 3412 3413
	if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) {
		set_bit(STRIPE_OP_BIOFILL, &s.ops_request);
		set_bit(STRIPE_BIOFILL_RUN, &sh->state);
	}

3414
	pr_debug("locked=%d uptodate=%d to_read=%d"
3415
	       " to_write=%d failed=%d failed_num=%d,%d\n",
3416 3417 3418 3419
	       s.locked, s.uptodate, s.to_read, s.to_write, s.failed,
	       r6s.failed_num[0], r6s.failed_num[1]);
	/* check if the array has lost >2 devices and, if so, some requests
	 * might need to be failed
3420
	 */
3421
	if (s.failed > 2 && s.to_read+s.to_write+s.written)
3422
		handle_failed_stripe(conf, sh, &s, disks, &return_bi);
3423
	if (s.failed > 2 && s.syncing) {
3424 3425
		md_done_sync(conf->mddev, STRIPE_SECTORS,0);
		clear_bit(STRIPE_SYNCING, &sh->state);
3426
		s.syncing = 0;
3427 3428 3429 3430 3431 3432 3433
	}

	/*
	 * might be able to return some write requests if the parity blocks
	 * are safe, or on a failed drive
	 */
	pdev = &sh->dev[pd_idx];
3434 3435
	r6s.p_failed = (s.failed >= 1 && r6s.failed_num[0] == pd_idx)
		|| (s.failed >= 2 && r6s.failed_num[1] == pd_idx);
N
NeilBrown 已提交
3436 3437 3438
	qdev = &sh->dev[qd_idx];
	r6s.q_failed = (s.failed >= 1 && r6s.failed_num[0] == qd_idx)
		|| (s.failed >= 2 && r6s.failed_num[1] == qd_idx);
3439 3440 3441

	if ( s.written &&
	     ( r6s.p_failed || ((test_bit(R5_Insync, &pdev->flags)
3442
			     && !test_bit(R5_LOCKED, &pdev->flags)
3443 3444
			     && test_bit(R5_UPTODATE, &pdev->flags)))) &&
	     ( r6s.q_failed || ((test_bit(R5_Insync, &qdev->flags)
3445
			     && !test_bit(R5_LOCKED, &qdev->flags)
3446
			     && test_bit(R5_UPTODATE, &qdev->flags)))))
3447
		handle_stripe_clean_event(conf, sh, disks, &return_bi);
3448 3449 3450 3451 3452

	/* Now we might consider reading some blocks, either to check/generate
	 * parity, or to satisfy requests
	 * or to load a block that is being partially written.
	 */
3453
	if (s.to_read || s.non_overwrite || (s.to_write && s.failed) ||
3454
	    (s.syncing && (s.uptodate + s.compute < disks)) || s.expanding)
3455
		handle_stripe_fill6(sh, &s, &r6s, disks);
3456

3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481
	/* Now we check to see if any write operations have recently
	 * completed
	 */
	if (sh->reconstruct_state == reconstruct_state_drain_result) {

		sh->reconstruct_state = reconstruct_state_idle;
		/* All the 'written' buffers and the parity blocks are ready to
		 * be written back to disk
		 */
		BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags));
		BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[qd_idx].flags));
		for (i = disks; i--; ) {
			dev = &sh->dev[i];
			if (test_bit(R5_LOCKED, &dev->flags) &&
			    (i == sh->pd_idx || i == qd_idx ||
			     dev->written)) {
				pr_debug("Writing block %d\n", i);
				BUG_ON(!test_bit(R5_UPTODATE, &dev->flags));
				set_bit(R5_Wantwrite, &dev->flags);
				if (!test_bit(R5_Insync, &dev->flags) ||
				    ((i == sh->pd_idx || i == qd_idx) &&
				      s.failed == 0))
					set_bit(STRIPE_INSYNC, &sh->state);
			}
		}
3482 3483
		if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
			dec_preread_active = 1;
3484 3485
	}

3486 3487 3488 3489 3490 3491 3492
	/* Now to consider new write requests and what else, if anything
	 * should be read.  We do not handle new writes when:
	 * 1/ A 'write' operation (copy+gen_syndrome) is already in flight.
	 * 2/ A 'check' operation is in flight, as it may clobber the parity
	 *    block.
	 */
	if (s.to_write && !sh->reconstruct_state && !sh->check_state)
3493
		handle_stripe_dirtying6(conf, sh, &s, &r6s, disks);
3494 3495

	/* maybe we need to check and possibly fix the parity for this stripe
3496
	 * Any reads will already have been scheduled, so we just see if enough
3497 3498
	 * data is available.  The parity check is held off while parity
	 * dependent operations are in flight.
3499
	 */
3500 3501 3502 3503
	if (sh->check_state ||
	    (s.syncing && s.locked == 0 &&
	     !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
	     !test_bit(STRIPE_INSYNC, &sh->state)))
3504
		handle_parity_checks6(conf, sh, &s, &r6s, disks);
3505

3506
	if (s.syncing && s.locked == 0 && test_bit(STRIPE_INSYNC, &sh->state)) {
3507 3508 3509 3510 3511 3512 3513
		md_done_sync(conf->mddev, STRIPE_SECTORS,1);
		clear_bit(STRIPE_SYNCING, &sh->state);
	}

	/* If the failed drives are just a ReadError, then we might need
	 * to progress the repair/check process
	 */
3514 3515 3516
	if (s.failed <= 2 && !conf->mddev->ro)
		for (i = 0; i < s.failed; i++) {
			dev = &sh->dev[r6s.failed_num[i]];
3517 3518 3519 3520 3521 3522 3523 3524
			if (test_bit(R5_ReadError, &dev->flags)
			    && !test_bit(R5_LOCKED, &dev->flags)
			    && test_bit(R5_UPTODATE, &dev->flags)
				) {
				if (!test_bit(R5_ReWrite, &dev->flags)) {
					set_bit(R5_Wantwrite, &dev->flags);
					set_bit(R5_ReWrite, &dev->flags);
					set_bit(R5_LOCKED, &dev->flags);
3525
					s.locked++;
3526 3527 3528 3529
				} else {
					/* let's read it back */
					set_bit(R5_Wantread, &dev->flags);
					set_bit(R5_LOCKED, &dev->flags);
3530
					s.locked++;
3531 3532 3533
				}
			}
		}
3534

3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547
	/* Finish reconstruct operations initiated by the expansion process */
	if (sh->reconstruct_state == reconstruct_state_result) {
		sh->reconstruct_state = reconstruct_state_idle;
		clear_bit(STRIPE_EXPANDING, &sh->state);
		for (i = conf->raid_disks; i--; ) {
			set_bit(R5_Wantwrite, &sh->dev[i].flags);
			set_bit(R5_LOCKED, &sh->dev[i].flags);
			s.locked++;
		}
	}

	if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) &&
	    !sh->reconstruct_state) {
3548
		struct stripe_head *sh2
3549
			= get_active_stripe(conf, sh->sector, 1, 1, 1);
3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564
		if (sh2 && test_bit(STRIPE_EXPAND_SOURCE, &sh2->state)) {
			/* sh cannot be written until sh2 has been read.
			 * so arrange for sh to be delayed a little
			 */
			set_bit(STRIPE_DELAYED, &sh->state);
			set_bit(STRIPE_HANDLE, &sh->state);
			if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE,
					      &sh2->state))
				atomic_inc(&conf->preread_active_stripes);
			release_stripe(sh2);
			goto unlock;
		}
		if (sh2)
			release_stripe(sh2);

3565 3566
		/* Need to write out all blocks after computing P&Q */
		sh->disks = conf->raid_disks;
3567
		stripe_set_idx(sh->sector, conf, 0, sh);
3568 3569
		schedule_reconstruction(sh, &s, 1, 1);
	} else if (s.expanded && !sh->reconstruct_state && s.locked == 0) {
3570 3571 3572 3573 3574 3575
		clear_bit(STRIPE_EXPAND_READY, &sh->state);
		atomic_dec(&conf->reshape_stripes);
		wake_up(&conf->wait_for_overlap);
		md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
	}

3576
	if (s.expanding && s.locked == 0 &&
3577
	    !test_bit(STRIPE_COMPUTE_RUN, &sh->state))
3578
		handle_stripe_expansion(conf, sh, &r6s);
3579

3580
 unlock:
3581 3582
	spin_unlock(&sh->lock);

3583 3584 3585 3586
	/* wait for this device to become unblocked */
	if (unlikely(blocked_rdev))
		md_wait_for_blocked_rdev(blocked_rdev, conf->mddev);

3587 3588 3589
	if (s.ops_request)
		raid_run_ops(sh, s.ops_request);

D
Dan Williams 已提交
3590
	ops_run_io(sh, &s);
3591

3592 3593 3594

	if (dec_preread_active) {
		/* We delay this until after ops_run_io so that if make_request
T
Tejun Heo 已提交
3595
		 * is waiting on a flush, it won't continue until the writes
3596 3597 3598 3599 3600 3601 3602 3603
		 * have actually been submitted.
		 */
		atomic_dec(&conf->preread_active_stripes);
		if (atomic_read(&conf->preread_active_stripes) <
		    IO_THRESHOLD)
			md_wakeup_thread(conf->mddev->thread);
	}

D
Dan Williams 已提交
3604
	return_io(return_bi);
3605 3606
}

3607
static void handle_stripe(struct stripe_head *sh)
3608 3609
{
	if (sh->raid_conf->level == 6)
3610
		handle_stripe6(sh);
3611
	else
3612
		handle_stripe5(sh);
3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625
}

static void raid5_activate_delayed(raid5_conf_t *conf)
{
	if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) {
		while (!list_empty(&conf->delayed_list)) {
			struct list_head *l = conf->delayed_list.next;
			struct stripe_head *sh;
			sh = list_entry(l, struct stripe_head, lru);
			list_del_init(l);
			clear_bit(STRIPE_DELAYED, &sh->state);
			if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
				atomic_inc(&conf->preread_active_stripes);
3626
			list_add_tail(&sh->lru, &conf->hold_list);
3627
		}
N
NeilBrown 已提交
3628
	}
3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644
}

static void activate_bit_delay(raid5_conf_t *conf)
{
	/* device_lock is held */
	struct list_head head;
	list_add(&head, &conf->bitmap_list);
	list_del_init(&conf->bitmap_list);
	while (!list_empty(&head)) {
		struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru);
		list_del_init(&sh->lru);
		atomic_inc(&sh->count);
		__release_stripe(conf, sh);
	}
}

N
NeilBrown 已提交
3645
int md_raid5_congested(mddev_t *mddev, int bits)
3646
{
3647
	raid5_conf_t *conf = mddev->private;
3648 3649 3650 3651

	/* No difference between reads and writes.  Just check
	 * how busy the stripe_cache is
	 */
3652

3653 3654 3655 3656 3657 3658 3659 3660 3661
	if (conf->inactive_blocked)
		return 1;
	if (conf->quiesce)
		return 1;
	if (list_empty_careful(&conf->inactive_list))
		return 1;

	return 0;
}
N
NeilBrown 已提交
3662 3663 3664 3665 3666 3667 3668 3669 3670
EXPORT_SYMBOL_GPL(md_raid5_congested);

static int raid5_congested(void *data, int bits)
{
	mddev_t *mddev = data;

	return mddev_congested(mddev, bits) ||
		md_raid5_congested(mddev, bits);
}
3671

3672 3673 3674
/* We want read requests to align with chunks where possible,
 * but write requests don't need to.
 */
3675 3676 3677
static int raid5_mergeable_bvec(struct request_queue *q,
				struct bvec_merge_data *bvm,
				struct bio_vec *biovec)
3678 3679
{
	mddev_t *mddev = q->queuedata;
3680
	sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev);
3681
	int max;
3682
	unsigned int chunk_sectors = mddev->chunk_sectors;
3683
	unsigned int bio_sectors = bvm->bi_size >> 9;
3684

3685
	if ((bvm->bi_rw & 1) == WRITE)
3686 3687
		return biovec->bv_len; /* always allow writes to be mergeable */

3688 3689
	if (mddev->new_chunk_sectors < mddev->chunk_sectors)
		chunk_sectors = mddev->new_chunk_sectors;
3690 3691 3692 3693 3694 3695 3696 3697
	max =  (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9;
	if (max < 0) max = 0;
	if (max <= biovec->bv_len && bio_sectors == 0)
		return biovec->bv_len;
	else
		return max;
}

3698 3699 3700 3701

static int in_chunk_boundary(mddev_t *mddev, struct bio *bio)
{
	sector_t sector = bio->bi_sector + get_start_sect(bio->bi_bdev);
3702
	unsigned int chunk_sectors = mddev->chunk_sectors;
3703 3704
	unsigned int bio_sectors = bio->bi_size >> 9;

3705 3706
	if (mddev->new_chunk_sectors < mddev->chunk_sectors)
		chunk_sectors = mddev->new_chunk_sectors;
3707 3708 3709 3710
	return  chunk_sectors >=
		((sector & (chunk_sectors - 1)) + bio_sectors);
}

3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739
/*
 *  add bio to the retry LIFO  ( in O(1) ... we are in interrupt )
 *  later sampled by raid5d.
 */
static void add_bio_to_retry(struct bio *bi,raid5_conf_t *conf)
{
	unsigned long flags;

	spin_lock_irqsave(&conf->device_lock, flags);

	bi->bi_next = conf->retry_read_aligned_list;
	conf->retry_read_aligned_list = bi;

	spin_unlock_irqrestore(&conf->device_lock, flags);
	md_wakeup_thread(conf->mddev->thread);
}


static struct bio *remove_bio_from_retry(raid5_conf_t *conf)
{
	struct bio *bi;

	bi = conf->retry_read_aligned;
	if (bi) {
		conf->retry_read_aligned = NULL;
		return bi;
	}
	bi = conf->retry_read_aligned_list;
	if(bi) {
3740
		conf->retry_read_aligned_list = bi->bi_next;
3741
		bi->bi_next = NULL;
3742 3743 3744 3745
		/*
		 * this sets the active strip count to 1 and the processed
		 * strip count to zero (upper 8 bits)
		 */
3746 3747 3748 3749 3750 3751 3752
		bi->bi_phys_segments = 1; /* biased count of active stripes */
	}

	return bi;
}


3753 3754 3755 3756 3757 3758
/*
 *  The "raid5_align_endio" should check if the read succeeded and if it
 *  did, call bio_endio on the original bio (having bio_put the new bio
 *  first).
 *  If the read failed..
 */
3759
static void raid5_align_endio(struct bio *bi, int error)
3760 3761
{
	struct bio* raid_bi  = bi->bi_private;
3762 3763 3764 3765 3766
	mddev_t *mddev;
	raid5_conf_t *conf;
	int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
	mdk_rdev_t *rdev;

3767
	bio_put(bi);
3768 3769 3770

	rdev = (void*)raid_bi->bi_next;
	raid_bi->bi_next = NULL;
3771 3772
	mddev = rdev->mddev;
	conf = mddev->private;
3773 3774 3775 3776

	rdev_dec_pending(rdev, conf->mddev);

	if (!error && uptodate) {
3777
		bio_endio(raid_bi, 0);
3778 3779
		if (atomic_dec_and_test(&conf->active_aligned_reads))
			wake_up(&conf->wait_for_stripe);
3780
		return;
3781 3782 3783
	}


3784
	pr_debug("raid5_align_endio : io error...handing IO for a retry\n");
3785 3786

	add_bio_to_retry(raid_bi, conf);
3787 3788
}

3789 3790
static int bio_fits_rdev(struct bio *bi)
{
3791
	struct request_queue *q = bdev_get_queue(bi->bi_bdev);
3792

3793
	if ((bi->bi_size>>9) > queue_max_sectors(q))
3794 3795
		return 0;
	blk_recount_segments(q, bi);
3796
	if (bi->bi_phys_segments > queue_max_segments(q))
3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808
		return 0;

	if (q->merge_bvec_fn)
		/* it's too hard to apply the merge_bvec_fn at this stage,
		 * just just give up
		 */
		return 0;

	return 1;
}


3809
static int chunk_aligned_read(mddev_t *mddev, struct bio * raid_bio)
3810
{
3811
	raid5_conf_t *conf = mddev->private;
N
NeilBrown 已提交
3812
	int dd_idx;
3813 3814 3815 3816
	struct bio* align_bi;
	mdk_rdev_t *rdev;

	if (!in_chunk_boundary(mddev, raid_bio)) {
3817
		pr_debug("chunk_aligned_read : non aligned\n");
3818 3819 3820
		return 0;
	}
	/*
3821
	 * use bio_clone_mddev to make a copy of the bio
3822
	 */
3823
	align_bi = bio_clone_mddev(raid_bio, GFP_NOIO, mddev);
3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834
	if (!align_bi)
		return 0;
	/*
	 *   set bi_end_io to a new function, and set bi_private to the
	 *     original bio.
	 */
	align_bi->bi_end_io  = raid5_align_endio;
	align_bi->bi_private = raid_bio;
	/*
	 *	compute position
	 */
3835 3836
	align_bi->bi_sector =  raid5_compute_sector(conf, raid_bio->bi_sector,
						    0,
3837
						    &dd_idx, NULL);
3838 3839 3840 3841 3842 3843

	rcu_read_lock();
	rdev = rcu_dereference(conf->disks[dd_idx].rdev);
	if (rdev && test_bit(In_sync, &rdev->flags)) {
		atomic_inc(&rdev->nr_pending);
		rcu_read_unlock();
3844 3845 3846 3847 3848
		raid_bio->bi_next = (void*)rdev;
		align_bi->bi_bdev =  rdev->bdev;
		align_bi->bi_flags &= ~(1 << BIO_SEG_VALID);
		align_bi->bi_sector += rdev->data_offset;

3849 3850 3851 3852 3853 3854 3855
		if (!bio_fits_rdev(align_bi)) {
			/* too big in some way */
			bio_put(align_bi);
			rdev_dec_pending(rdev, mddev);
			return 0;
		}

3856 3857 3858 3859 3860 3861 3862
		spin_lock_irq(&conf->device_lock);
		wait_event_lock_irq(conf->wait_for_stripe,
				    conf->quiesce == 0,
				    conf->device_lock, /* nothing */);
		atomic_inc(&conf->active_aligned_reads);
		spin_unlock_irq(&conf->device_lock);

3863 3864 3865 3866
		generic_make_request(align_bi);
		return 1;
	} else {
		rcu_read_unlock();
3867
		bio_put(align_bi);
3868 3869 3870 3871
		return 0;
	}
}

3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923
/* __get_priority_stripe - get the next stripe to process
 *
 * Full stripe writes are allowed to pass preread active stripes up until
 * the bypass_threshold is exceeded.  In general the bypass_count
 * increments when the handle_list is handled before the hold_list; however, it
 * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a
 * stripe with in flight i/o.  The bypass_count will be reset when the
 * head of the hold_list has changed, i.e. the head was promoted to the
 * handle_list.
 */
static struct stripe_head *__get_priority_stripe(raid5_conf_t *conf)
{
	struct stripe_head *sh;

	pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n",
		  __func__,
		  list_empty(&conf->handle_list) ? "empty" : "busy",
		  list_empty(&conf->hold_list) ? "empty" : "busy",
		  atomic_read(&conf->pending_full_writes), conf->bypass_count);

	if (!list_empty(&conf->handle_list)) {
		sh = list_entry(conf->handle_list.next, typeof(*sh), lru);

		if (list_empty(&conf->hold_list))
			conf->bypass_count = 0;
		else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) {
			if (conf->hold_list.next == conf->last_hold)
				conf->bypass_count++;
			else {
				conf->last_hold = conf->hold_list.next;
				conf->bypass_count -= conf->bypass_threshold;
				if (conf->bypass_count < 0)
					conf->bypass_count = 0;
			}
		}
	} else if (!list_empty(&conf->hold_list) &&
		   ((conf->bypass_threshold &&
		     conf->bypass_count > conf->bypass_threshold) ||
		    atomic_read(&conf->pending_full_writes) == 0)) {
		sh = list_entry(conf->hold_list.next,
				typeof(*sh), lru);
		conf->bypass_count -= conf->bypass_threshold;
		if (conf->bypass_count < 0)
			conf->bypass_count = 0;
	} else
		return NULL;

	list_del_init(&sh->lru);
	atomic_inc(&sh->count);
	BUG_ON(atomic_read(&sh->count) != 1);
	return sh;
}
3924

3925
static int make_request(mddev_t *mddev, struct bio * bi)
L
Linus Torvalds 已提交
3926
{
3927
	raid5_conf_t *conf = mddev->private;
3928
	int dd_idx;
L
Linus Torvalds 已提交
3929 3930 3931
	sector_t new_sector;
	sector_t logical_sector, last_sector;
	struct stripe_head *sh;
3932
	const int rw = bio_data_dir(bi);
3933
	int remaining;
3934
	int plugged;
L
Linus Torvalds 已提交
3935

T
Tejun Heo 已提交
3936 3937
	if (unlikely(bi->bi_rw & REQ_FLUSH)) {
		md_flush_request(mddev, bi);
3938 3939 3940
		return 0;
	}

3941
	md_write_start(mddev, bi);
3942

3943
	if (rw == READ &&
3944
	     mddev->reshape_position == MaxSector &&
3945
	     chunk_aligned_read(mddev,bi))
3946
		return 0;
3947

L
Linus Torvalds 已提交
3948 3949 3950 3951
	logical_sector = bi->bi_sector & ~((sector_t)STRIPE_SECTORS-1);
	last_sector = bi->bi_sector + (bi->bi_size>>9);
	bi->bi_next = NULL;
	bi->bi_phys_segments = 1;	/* over-loaded to count active stripes */
3952

3953
	plugged = mddev_check_plugged(mddev);
L
Linus Torvalds 已提交
3954 3955
	for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) {
		DEFINE_WAIT(w);
3956
		int disks, data_disks;
3957
		int previous;
3958

3959
	retry:
3960
		previous = 0;
3961
		disks = conf->raid_disks;
3962
		prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE);
3963
		if (unlikely(conf->reshape_progress != MaxSector)) {
3964
			/* spinlock is needed as reshape_progress may be
3965 3966
			 * 64bit on a 32bit platform, and so it might be
			 * possible to see a half-updated value
3967
			 * Of course reshape_progress could change after
3968 3969 3970 3971
			 * the lock is dropped, so once we get a reference
			 * to the stripe that we think it is, we will have
			 * to check again.
			 */
3972
			spin_lock_irq(&conf->device_lock);
3973 3974 3975
			if (mddev->delta_disks < 0
			    ? logical_sector < conf->reshape_progress
			    : logical_sector >= conf->reshape_progress) {
3976
				disks = conf->previous_raid_disks;
3977 3978
				previous = 1;
			} else {
3979 3980 3981
				if (mddev->delta_disks < 0
				    ? logical_sector < conf->reshape_safe
				    : logical_sector >= conf->reshape_safe) {
3982 3983 3984 3985 3986
					spin_unlock_irq(&conf->device_lock);
					schedule();
					goto retry;
				}
			}
3987 3988
			spin_unlock_irq(&conf->device_lock);
		}
3989 3990
		data_disks = disks - conf->max_degraded;

3991 3992
		new_sector = raid5_compute_sector(conf, logical_sector,
						  previous,
3993
						  &dd_idx, NULL);
3994
		pr_debug("raid456: make_request, sector %llu logical %llu\n",
L
Linus Torvalds 已提交
3995 3996 3997
			(unsigned long long)new_sector, 
			(unsigned long long)logical_sector);

3998
		sh = get_active_stripe(conf, new_sector, previous,
3999
				       (bi->bi_rw&RWA_MASK), 0);
L
Linus Torvalds 已提交
4000
		if (sh) {
4001
			if (unlikely(previous)) {
4002
				/* expansion might have moved on while waiting for a
4003 4004 4005 4006 4007 4008
				 * stripe, so we must do the range check again.
				 * Expansion could still move past after this
				 * test, but as we are holding a reference to
				 * 'sh', we know that if that happens,
				 *  STRIPE_EXPANDING will get set and the expansion
				 * won't proceed until we finish with the stripe.
4009 4010 4011
				 */
				int must_retry = 0;
				spin_lock_irq(&conf->device_lock);
4012 4013 4014
				if (mddev->delta_disks < 0
				    ? logical_sector >= conf->reshape_progress
				    : logical_sector < conf->reshape_progress)
4015 4016 4017 4018 4019
					/* mismatch, need to try again */
					must_retry = 1;
				spin_unlock_irq(&conf->device_lock);
				if (must_retry) {
					release_stripe(sh);
4020
					schedule();
4021 4022 4023
					goto retry;
				}
			}
4024

4025
			if (rw == WRITE &&
4026
			    logical_sector >= mddev->suspend_lo &&
4027 4028
			    logical_sector < mddev->suspend_hi) {
				release_stripe(sh);
4029 4030 4031 4032 4033 4034 4035 4036 4037 4038
				/* As the suspend_* range is controlled by
				 * userspace, we want an interruptible
				 * wait.
				 */
				flush_signals(current);
				prepare_to_wait(&conf->wait_for_overlap,
						&w, TASK_INTERRUPTIBLE);
				if (logical_sector >= mddev->suspend_lo &&
				    logical_sector < mddev->suspend_hi)
					schedule();
4039 4040
				goto retry;
			}
4041 4042

			if (test_bit(STRIPE_EXPANDING, &sh->state) ||
4043
			    !add_stripe_bio(sh, bi, dd_idx, rw)) {
4044 4045
				/* Stripe is busy expanding or
				 * add failed due to overlap.  Flush everything
L
Linus Torvalds 已提交
4046 4047
				 * and wait a while
				 */
N
NeilBrown 已提交
4048
				md_wakeup_thread(mddev->thread);
L
Linus Torvalds 已提交
4049 4050 4051 4052 4053
				release_stripe(sh);
				schedule();
				goto retry;
			}
			finish_wait(&conf->wait_for_overlap, &w);
4054 4055
			set_bit(STRIPE_HANDLE, &sh->state);
			clear_bit(STRIPE_DELAYED, &sh->state);
T
Tejun Heo 已提交
4056
			if ((bi->bi_rw & REQ_SYNC) &&
4057 4058
			    !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
				atomic_inc(&conf->preread_active_stripes);
L
Linus Torvalds 已提交
4059 4060 4061 4062 4063 4064 4065 4066 4067
			release_stripe(sh);
		} else {
			/* cannot get stripe for read-ahead, just give-up */
			clear_bit(BIO_UPTODATE, &bi->bi_flags);
			finish_wait(&conf->wait_for_overlap, &w);
			break;
		}
			
	}
4068 4069 4070
	if (!plugged)
		md_wakeup_thread(mddev->thread);

L
Linus Torvalds 已提交
4071
	spin_lock_irq(&conf->device_lock);
4072
	remaining = raid5_dec_bi_phys_segments(bi);
4073 4074
	spin_unlock_irq(&conf->device_lock);
	if (remaining == 0) {
L
Linus Torvalds 已提交
4075

4076
		if ( rw == WRITE )
L
Linus Torvalds 已提交
4077
			md_write_end(mddev);
4078

4079
		bio_endio(bi, 0);
L
Linus Torvalds 已提交
4080
	}
4081

L
Linus Torvalds 已提交
4082 4083 4084
	return 0;
}

D
Dan Williams 已提交
4085 4086
static sector_t raid5_size(mddev_t *mddev, sector_t sectors, int raid_disks);

4087
static sector_t reshape_request(mddev_t *mddev, sector_t sector_nr, int *skipped)
L
Linus Torvalds 已提交
4088
{
4089 4090 4091 4092 4093 4094 4095 4096 4097
	/* reshaping is quite different to recovery/resync so it is
	 * handled quite separately ... here.
	 *
	 * On each call to sync_request, we gather one chunk worth of
	 * destination stripes and flag them as expanding.
	 * Then we find all the source stripes and request reads.
	 * As the reads complete, handle_stripe will copy the data
	 * into the destination stripe and release that stripe.
	 */
4098
	raid5_conf_t *conf = mddev->private;
L
Linus Torvalds 已提交
4099
	struct stripe_head *sh;
4100
	sector_t first_sector, last_sector;
4101 4102 4103
	int raid_disks = conf->previous_raid_disks;
	int data_disks = raid_disks - conf->max_degraded;
	int new_data_disks = conf->raid_disks - conf->max_degraded;
4104 4105
	int i;
	int dd_idx;
4106
	sector_t writepos, readpos, safepos;
4107
	sector_t stripe_addr;
4108
	int reshape_sectors;
4109
	struct list_head stripes;
4110

4111 4112 4113 4114 4115 4116
	if (sector_nr == 0) {
		/* If restarting in the middle, skip the initial sectors */
		if (mddev->delta_disks < 0 &&
		    conf->reshape_progress < raid5_size(mddev, 0, 0)) {
			sector_nr = raid5_size(mddev, 0, 0)
				- conf->reshape_progress;
4117
		} else if (mddev->delta_disks >= 0 &&
4118 4119
			   conf->reshape_progress > 0)
			sector_nr = conf->reshape_progress;
4120
		sector_div(sector_nr, new_data_disks);
4121
		if (sector_nr) {
4122 4123
			mddev->curr_resync_completed = sector_nr;
			sysfs_notify(&mddev->kobj, NULL, "sync_completed");
4124 4125 4126
			*skipped = 1;
			return sector_nr;
		}
4127 4128
	}

4129 4130 4131 4132
	/* We need to process a full chunk at a time.
	 * If old and new chunk sizes differ, we need to process the
	 * largest of these
	 */
4133 4134
	if (mddev->new_chunk_sectors > mddev->chunk_sectors)
		reshape_sectors = mddev->new_chunk_sectors;
4135
	else
4136
		reshape_sectors = mddev->chunk_sectors;
4137

4138 4139 4140 4141 4142
	/* we update the metadata when there is more than 3Meg
	 * in the block range (that is rather arbitrary, should
	 * probably be time based) or when the data about to be
	 * copied would over-write the source of the data at
	 * the front of the range.
4143 4144
	 * i.e. one new_stripe along from reshape_progress new_maps
	 * to after where reshape_safe old_maps to
4145
	 */
4146
	writepos = conf->reshape_progress;
4147
	sector_div(writepos, new_data_disks);
4148 4149
	readpos = conf->reshape_progress;
	sector_div(readpos, data_disks);
4150
	safepos = conf->reshape_safe;
4151
	sector_div(safepos, data_disks);
4152
	if (mddev->delta_disks < 0) {
4153
		writepos -= min_t(sector_t, reshape_sectors, writepos);
4154
		readpos += reshape_sectors;
4155
		safepos += reshape_sectors;
4156
	} else {
4157
		writepos += reshape_sectors;
4158 4159
		readpos -= min_t(sector_t, reshape_sectors, readpos);
		safepos -= min_t(sector_t, reshape_sectors, safepos);
4160
	}
4161

4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178
	/* 'writepos' is the most advanced device address we might write.
	 * 'readpos' is the least advanced device address we might read.
	 * 'safepos' is the least address recorded in the metadata as having
	 *     been reshaped.
	 * If 'readpos' is behind 'writepos', then there is no way that we can
	 * ensure safety in the face of a crash - that must be done by userspace
	 * making a backup of the data.  So in that case there is no particular
	 * rush to update metadata.
	 * Otherwise if 'safepos' is behind 'writepos', then we really need to
	 * update the metadata to advance 'safepos' to match 'readpos' so that
	 * we can be safe in the event of a crash.
	 * So we insist on updating metadata if safepos is behind writepos and
	 * readpos is beyond writepos.
	 * In any case, update the metadata every 10 seconds.
	 * Maybe that number should be configurable, but I'm not sure it is
	 * worth it.... maybe it could be a multiple of safemode_delay???
	 */
4179
	if ((mddev->delta_disks < 0
4180 4181 4182
	     ? (safepos > writepos && readpos < writepos)
	     : (safepos < writepos && readpos > writepos)) ||
	    time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
4183 4184 4185
		/* Cannot proceed until we've updated the superblock... */
		wait_event(conf->wait_for_overlap,
			   atomic_read(&conf->reshape_stripes)==0);
4186
		mddev->reshape_position = conf->reshape_progress;
4187
		mddev->curr_resync_completed = sector_nr;
4188
		conf->reshape_checkpoint = jiffies;
4189
		set_bit(MD_CHANGE_DEVS, &mddev->flags);
4190
		md_wakeup_thread(mddev->thread);
4191
		wait_event(mddev->sb_wait, mddev->flags == 0 ||
4192 4193
			   kthread_should_stop());
		spin_lock_irq(&conf->device_lock);
4194
		conf->reshape_safe = mddev->reshape_position;
4195 4196
		spin_unlock_irq(&conf->device_lock);
		wake_up(&conf->wait_for_overlap);
4197
		sysfs_notify(&mddev->kobj, NULL, "sync_completed");
4198 4199
	}

4200 4201 4202 4203
	if (mddev->delta_disks < 0) {
		BUG_ON(conf->reshape_progress == 0);
		stripe_addr = writepos;
		BUG_ON((mddev->dev_sectors &
4204 4205
			~((sector_t)reshape_sectors - 1))
		       - reshape_sectors - stripe_addr
4206 4207
		       != sector_nr);
	} else {
4208
		BUG_ON(writepos != sector_nr + reshape_sectors);
4209 4210
		stripe_addr = sector_nr;
	}
4211
	INIT_LIST_HEAD(&stripes);
4212
	for (i = 0; i < reshape_sectors; i += STRIPE_SECTORS) {
4213
		int j;
4214
		int skipped_disk = 0;
4215
		sh = get_active_stripe(conf, stripe_addr+i, 0, 0, 1);
4216 4217 4218 4219 4220 4221 4222 4223 4224
		set_bit(STRIPE_EXPANDING, &sh->state);
		atomic_inc(&conf->reshape_stripes);
		/* If any of this stripe is beyond the end of the old
		 * array, then we need to zero those blocks
		 */
		for (j=sh->disks; j--;) {
			sector_t s;
			if (j == sh->pd_idx)
				continue;
4225
			if (conf->level == 6 &&
4226
			    j == sh->qd_idx)
4227
				continue;
4228
			s = compute_blocknr(sh, j, 0);
D
Dan Williams 已提交
4229
			if (s < raid5_size(mddev, 0, 0)) {
4230
				skipped_disk = 1;
4231 4232 4233 4234 4235 4236
				continue;
			}
			memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE);
			set_bit(R5_Expanded, &sh->dev[j].flags);
			set_bit(R5_UPTODATE, &sh->dev[j].flags);
		}
4237
		if (!skipped_disk) {
4238 4239 4240
			set_bit(STRIPE_EXPAND_READY, &sh->state);
			set_bit(STRIPE_HANDLE, &sh->state);
		}
4241
		list_add(&sh->lru, &stripes);
4242 4243
	}
	spin_lock_irq(&conf->device_lock);
4244
	if (mddev->delta_disks < 0)
4245
		conf->reshape_progress -= reshape_sectors * new_data_disks;
4246
	else
4247
		conf->reshape_progress += reshape_sectors * new_data_disks;
4248 4249 4250 4251 4252 4253 4254
	spin_unlock_irq(&conf->device_lock);
	/* Ok, those stripe are ready. We can start scheduling
	 * reads on the source stripes.
	 * The source stripes are determined by mapping the first and last
	 * block on the destination stripes.
	 */
	first_sector =
4255
		raid5_compute_sector(conf, stripe_addr*(new_data_disks),
4256
				     1, &dd_idx, NULL);
4257
	last_sector =
4258
		raid5_compute_sector(conf, ((stripe_addr+reshape_sectors)
4259
					    * new_data_disks - 1),
4260
				     1, &dd_idx, NULL);
A
Andre Noll 已提交
4261 4262
	if (last_sector >= mddev->dev_sectors)
		last_sector = mddev->dev_sectors - 1;
4263
	while (first_sector <= last_sector) {
4264
		sh = get_active_stripe(conf, first_sector, 1, 0, 1);
4265 4266 4267 4268 4269
		set_bit(STRIPE_EXPAND_SOURCE, &sh->state);
		set_bit(STRIPE_HANDLE, &sh->state);
		release_stripe(sh);
		first_sector += STRIPE_SECTORS;
	}
4270 4271 4272 4273 4274 4275 4276 4277
	/* Now that the sources are clearly marked, we can release
	 * the destination stripes
	 */
	while (!list_empty(&stripes)) {
		sh = list_entry(stripes.next, struct stripe_head, lru);
		list_del_init(&sh->lru);
		release_stripe(sh);
	}
4278 4279 4280
	/* If this takes us to the resync_max point where we have to pause,
	 * then we need to write out the superblock.
	 */
4281
	sector_nr += reshape_sectors;
4282 4283
	if ((sector_nr - mddev->curr_resync_completed) * 2
	    >= mddev->resync_max - mddev->curr_resync_completed) {
4284 4285 4286
		/* Cannot proceed until we've updated the superblock... */
		wait_event(conf->wait_for_overlap,
			   atomic_read(&conf->reshape_stripes) == 0);
4287
		mddev->reshape_position = conf->reshape_progress;
4288
		mddev->curr_resync_completed = sector_nr;
4289
		conf->reshape_checkpoint = jiffies;
4290 4291 4292 4293 4294 4295
		set_bit(MD_CHANGE_DEVS, &mddev->flags);
		md_wakeup_thread(mddev->thread);
		wait_event(mddev->sb_wait,
			   !test_bit(MD_CHANGE_DEVS, &mddev->flags)
			   || kthread_should_stop());
		spin_lock_irq(&conf->device_lock);
4296
		conf->reshape_safe = mddev->reshape_position;
4297 4298
		spin_unlock_irq(&conf->device_lock);
		wake_up(&conf->wait_for_overlap);
4299
		sysfs_notify(&mddev->kobj, NULL, "sync_completed");
4300
	}
4301
	return reshape_sectors;
4302 4303 4304 4305 4306
}

/* FIXME go_faster isn't used */
static inline sector_t sync_request(mddev_t *mddev, sector_t sector_nr, int *skipped, int go_faster)
{
4307
	raid5_conf_t *conf = mddev->private;
4308
	struct stripe_head *sh;
A
Andre Noll 已提交
4309
	sector_t max_sector = mddev->dev_sectors;
N
NeilBrown 已提交
4310
	sector_t sync_blocks;
4311 4312
	int still_degraded = 0;
	int i;
L
Linus Torvalds 已提交
4313

4314
	if (sector_nr >= max_sector) {
L
Linus Torvalds 已提交
4315
		/* just being told to finish up .. nothing much to do */
4316

4317 4318 4319 4320
		if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {
			end_reshape(conf);
			return 0;
		}
4321 4322 4323 4324

		if (mddev->curr_resync < max_sector) /* aborted */
			bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
					&sync_blocks, 1);
4325
		else /* completed sync */
4326 4327 4328
			conf->fullsync = 0;
		bitmap_close_sync(mddev->bitmap);

L
Linus Torvalds 已提交
4329 4330
		return 0;
	}
4331

4332 4333 4334
	/* Allow raid5_quiesce to complete */
	wait_event(conf->wait_for_overlap, conf->quiesce != 2);

4335 4336
	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
		return reshape_request(mddev, sector_nr, skipped);
4337

4338 4339 4340 4341 4342 4343
	/* No need to check resync_max as we never do more than one
	 * stripe, and as resync_max will always be on a chunk boundary,
	 * if the check in md_do_sync didn't fire, there is no chance
	 * of overstepping resync_max here
	 */

4344
	/* if there is too many failed drives and we are trying
L
Linus Torvalds 已提交
4345 4346 4347
	 * to resync, then assert that we are finished, because there is
	 * nothing we can do.
	 */
4348
	if (mddev->degraded >= conf->max_degraded &&
4349
	    test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
A
Andre Noll 已提交
4350
		sector_t rv = mddev->dev_sectors - sector_nr;
4351
		*skipped = 1;
L
Linus Torvalds 已提交
4352 4353
		return rv;
	}
4354
	if (!bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) &&
4355
	    !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
4356 4357 4358 4359 4360 4361
	    !conf->fullsync && sync_blocks >= STRIPE_SECTORS) {
		/* we can skip this block, and probably more */
		sync_blocks /= STRIPE_SECTORS;
		*skipped = 1;
		return sync_blocks * STRIPE_SECTORS; /* keep things rounded to whole stripes */
	}
L
Linus Torvalds 已提交
4362

N
NeilBrown 已提交
4363 4364 4365

	bitmap_cond_end_sync(mddev->bitmap, sector_nr);

4366
	sh = get_active_stripe(conf, sector_nr, 0, 1, 0);
L
Linus Torvalds 已提交
4367
	if (sh == NULL) {
4368
		sh = get_active_stripe(conf, sector_nr, 0, 0, 0);
L
Linus Torvalds 已提交
4369
		/* make sure we don't swamp the stripe cache if someone else
4370
		 * is trying to get access
L
Linus Torvalds 已提交
4371
		 */
4372
		schedule_timeout_uninterruptible(1);
L
Linus Torvalds 已提交
4373
	}
4374 4375 4376 4377
	/* Need to check if array will still be degraded after recovery/resync
	 * We don't need to check the 'failed' flag as when that gets set,
	 * recovery aborts.
	 */
4378
	for (i = 0; i < conf->raid_disks; i++)
4379 4380 4381 4382 4383
		if (conf->disks[i].rdev == NULL)
			still_degraded = 1;

	bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, still_degraded);

4384
	set_bit(STRIPE_SYNC_REQUESTED, &sh->state);
L
Linus Torvalds 已提交
4385

4386
	handle_stripe(sh);
L
Linus Torvalds 已提交
4387 4388 4389 4390 4391
	release_stripe(sh);

	return STRIPE_SECTORS;
}

4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404
static int  retry_aligned_read(raid5_conf_t *conf, struct bio *raid_bio)
{
	/* We may not be able to submit a whole bio at once as there
	 * may not be enough stripe_heads available.
	 * We cannot pre-allocate enough stripe_heads as we may need
	 * more than exist in the cache (if we allow ever large chunks).
	 * So we do one stripe head at a time and record in
	 * ->bi_hw_segments how many have been done.
	 *
	 * We *know* that this entire raid_bio is in one chunk, so
	 * it will be only one 'dd_idx' and only need one call to raid5_compute_sector.
	 */
	struct stripe_head *sh;
4405
	int dd_idx;
4406 4407 4408 4409 4410 4411
	sector_t sector, logical_sector, last_sector;
	int scnt = 0;
	int remaining;
	int handled = 0;

	logical_sector = raid_bio->bi_sector & ~((sector_t)STRIPE_SECTORS-1);
4412
	sector = raid5_compute_sector(conf, logical_sector,
4413
				      0, &dd_idx, NULL);
4414 4415 4416
	last_sector = raid_bio->bi_sector + (raid_bio->bi_size>>9);

	for (; logical_sector < last_sector;
4417 4418 4419
	     logical_sector += STRIPE_SECTORS,
		     sector += STRIPE_SECTORS,
		     scnt++) {
4420

4421
		if (scnt < raid5_bi_hw_segments(raid_bio))
4422 4423 4424
			/* already done this stripe */
			continue;

4425
		sh = get_active_stripe(conf, sector, 0, 1, 0);
4426 4427 4428

		if (!sh) {
			/* failed to get a stripe - must wait */
4429
			raid5_set_bi_hw_segments(raid_bio, scnt);
4430 4431 4432 4433 4434
			conf->retry_read_aligned = raid_bio;
			return handled;
		}

		set_bit(R5_ReadError, &sh->dev[dd_idx].flags);
4435 4436
		if (!add_stripe_bio(sh, raid_bio, dd_idx, 0)) {
			release_stripe(sh);
4437
			raid5_set_bi_hw_segments(raid_bio, scnt);
4438 4439 4440 4441
			conf->retry_read_aligned = raid_bio;
			return handled;
		}

4442
		handle_stripe(sh);
4443 4444 4445 4446
		release_stripe(sh);
		handled++;
	}
	spin_lock_irq(&conf->device_lock);
4447
	remaining = raid5_dec_bi_phys_segments(raid_bio);
4448
	spin_unlock_irq(&conf->device_lock);
4449 4450
	if (remaining == 0)
		bio_endio(raid_bio, 0);
4451 4452 4453 4454 4455 4456
	if (atomic_dec_and_test(&conf->active_aligned_reads))
		wake_up(&conf->wait_for_stripe);
	return handled;
}


L
Linus Torvalds 已提交
4457 4458 4459 4460 4461 4462 4463
/*
 * This is our raid5 kernel thread.
 *
 * We scan the hash table for stripes which can be handled now.
 * During the scan, completed stripes are saved for us by the interrupt
 * handler, so that they will not have to wait for our next wakeup.
 */
4464
static void raid5d(mddev_t *mddev)
L
Linus Torvalds 已提交
4465 4466
{
	struct stripe_head *sh;
4467
	raid5_conf_t *conf = mddev->private;
L
Linus Torvalds 已提交
4468
	int handled;
4469
	struct blk_plug plug;
L
Linus Torvalds 已提交
4470

4471
	pr_debug("+++ raid5d active\n");
L
Linus Torvalds 已提交
4472 4473 4474

	md_check_recovery(mddev);

4475
	blk_start_plug(&plug);
L
Linus Torvalds 已提交
4476 4477 4478
	handled = 0;
	spin_lock_irq(&conf->device_lock);
	while (1) {
4479
		struct bio *bio;
L
Linus Torvalds 已提交
4480

4481 4482 4483 4484
		if (atomic_read(&mddev->plug_cnt) == 0 &&
		    !list_empty(&conf->bitmap_list)) {
			/* Now is a good time to flush some bitmap updates */
			conf->seq_flush++;
4485
			spin_unlock_irq(&conf->device_lock);
4486
			bitmap_unplug(mddev->bitmap);
4487
			spin_lock_irq(&conf->device_lock);
4488
			conf->seq_write = conf->seq_flush;
4489 4490
			activate_bit_delay(conf);
		}
4491 4492
		if (atomic_read(&mddev->plug_cnt) == 0)
			raid5_activate_delayed(conf);
4493

4494 4495 4496 4497 4498 4499 4500 4501 4502 4503
		while ((bio = remove_bio_from_retry(conf))) {
			int ok;
			spin_unlock_irq(&conf->device_lock);
			ok = retry_aligned_read(conf, bio);
			spin_lock_irq(&conf->device_lock);
			if (!ok)
				break;
			handled++;
		}

4504 4505
		sh = __get_priority_stripe(conf);

4506
		if (!sh)
L
Linus Torvalds 已提交
4507 4508 4509 4510
			break;
		spin_unlock_irq(&conf->device_lock);
		
		handled++;
4511 4512 4513
		handle_stripe(sh);
		release_stripe(sh);
		cond_resched();
L
Linus Torvalds 已提交
4514 4515 4516

		spin_lock_irq(&conf->device_lock);
	}
4517
	pr_debug("%d stripes handled\n", handled);
L
Linus Torvalds 已提交
4518 4519 4520

	spin_unlock_irq(&conf->device_lock);

4521
	async_tx_issue_pending_all();
4522
	blk_finish_plug(&plug);
L
Linus Torvalds 已提交
4523

4524
	pr_debug("--- raid5d inactive\n");
L
Linus Torvalds 已提交
4525 4526
}

4527
static ssize_t
4528
raid5_show_stripe_cache_size(mddev_t *mddev, char *page)
4529
{
4530
	raid5_conf_t *conf = mddev->private;
4531 4532 4533 4534
	if (conf)
		return sprintf(page, "%d\n", conf->max_nr_stripes);
	else
		return 0;
4535 4536
}

4537 4538
int
raid5_set_cache_size(mddev_t *mddev, int size)
4539
{
4540
	raid5_conf_t *conf = mddev->private;
4541 4542
	int err;

4543
	if (size <= 16 || size > 32768)
4544
		return -EINVAL;
4545
	while (size < conf->max_nr_stripes) {
4546 4547 4548 4549 4550
		if (drop_one_stripe(conf))
			conf->max_nr_stripes--;
		else
			break;
	}
4551 4552 4553
	err = md_allow_write(mddev);
	if (err)
		return err;
4554
	while (size > conf->max_nr_stripes) {
4555 4556 4557 4558
		if (grow_one_stripe(conf))
			conf->max_nr_stripes++;
		else break;
	}
4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579
	return 0;
}
EXPORT_SYMBOL(raid5_set_cache_size);

static ssize_t
raid5_store_stripe_cache_size(mddev_t *mddev, const char *page, size_t len)
{
	raid5_conf_t *conf = mddev->private;
	unsigned long new;
	int err;

	if (len >= PAGE_SIZE)
		return -EINVAL;
	if (!conf)
		return -ENODEV;

	if (strict_strtoul(page, 10, &new))
		return -EINVAL;
	err = raid5_set_cache_size(mddev, new);
	if (err)
		return err;
4580 4581
	return len;
}
4582

4583 4584 4585 4586
static struct md_sysfs_entry
raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR,
				raid5_show_stripe_cache_size,
				raid5_store_stripe_cache_size);
4587

4588 4589 4590
static ssize_t
raid5_show_preread_threshold(mddev_t *mddev, char *page)
{
4591
	raid5_conf_t *conf = mddev->private;
4592 4593 4594 4595 4596 4597 4598 4599 4600
	if (conf)
		return sprintf(page, "%d\n", conf->bypass_threshold);
	else
		return 0;
}

static ssize_t
raid5_store_preread_threshold(mddev_t *mddev, const char *page, size_t len)
{
4601
	raid5_conf_t *conf = mddev->private;
4602
	unsigned long new;
4603 4604 4605 4606 4607
	if (len >= PAGE_SIZE)
		return -EINVAL;
	if (!conf)
		return -ENODEV;

4608
	if (strict_strtoul(page, 10, &new))
4609
		return -EINVAL;
4610
	if (new > conf->max_nr_stripes)
4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621
		return -EINVAL;
	conf->bypass_threshold = new;
	return len;
}

static struct md_sysfs_entry
raid5_preread_bypass_threshold = __ATTR(preread_bypass_threshold,
					S_IRUGO | S_IWUSR,
					raid5_show_preread_threshold,
					raid5_store_preread_threshold);

4622
static ssize_t
4623
stripe_cache_active_show(mddev_t *mddev, char *page)
4624
{
4625
	raid5_conf_t *conf = mddev->private;
4626 4627 4628 4629
	if (conf)
		return sprintf(page, "%d\n", atomic_read(&conf->active_stripes));
	else
		return 0;
4630 4631
}

4632 4633
static struct md_sysfs_entry
raid5_stripecache_active = __ATTR_RO(stripe_cache_active);
4634

4635
static struct attribute *raid5_attrs[] =  {
4636 4637
	&raid5_stripecache_size.attr,
	&raid5_stripecache_active.attr,
4638
	&raid5_preread_bypass_threshold.attr,
4639 4640
	NULL,
};
4641 4642 4643
static struct attribute_group raid5_attrs_group = {
	.name = NULL,
	.attrs = raid5_attrs,
4644 4645
};

4646 4647 4648
static sector_t
raid5_size(mddev_t *mddev, sector_t sectors, int raid_disks)
{
4649
	raid5_conf_t *conf = mddev->private;
4650 4651 4652

	if (!sectors)
		sectors = mddev->dev_sectors;
4653
	if (!raid_disks)
4654
		/* size is defined by the smallest of previous and new size */
4655
		raid_disks = min(conf->raid_disks, conf->previous_raid_disks);
4656

4657
	sectors &= ~((sector_t)mddev->chunk_sectors - 1);
4658
	sectors &= ~((sector_t)mddev->new_chunk_sectors - 1);
4659 4660 4661
	return sectors * (raid_disks - conf->max_degraded);
}

4662 4663 4664 4665 4666 4667 4668 4669 4670 4671 4672 4673
static void raid5_free_percpu(raid5_conf_t *conf)
{
	struct raid5_percpu *percpu;
	unsigned long cpu;

	if (!conf->percpu)
		return;

	get_online_cpus();
	for_each_possible_cpu(cpu) {
		percpu = per_cpu_ptr(conf->percpu, cpu);
		safe_put_page(percpu->spare_page);
4674
		kfree(percpu->scribble);
4675 4676 4677 4678 4679 4680 4681 4682 4683
	}
#ifdef CONFIG_HOTPLUG_CPU
	unregister_cpu_notifier(&conf->cpu_notify);
#endif
	put_online_cpus();

	free_percpu(conf->percpu);
}

4684 4685 4686
static void free_conf(raid5_conf_t *conf)
{
	shrink_stripes(conf);
4687
	raid5_free_percpu(conf);
4688 4689 4690 4691 4692
	kfree(conf->disks);
	kfree(conf->stripe_hashtbl);
	kfree(conf);
}

4693 4694 4695 4696 4697 4698 4699 4700 4701 4702 4703
#ifdef CONFIG_HOTPLUG_CPU
static int raid456_cpu_notify(struct notifier_block *nfb, unsigned long action,
			      void *hcpu)
{
	raid5_conf_t *conf = container_of(nfb, raid5_conf_t, cpu_notify);
	long cpu = (long)hcpu;
	struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu);

	switch (action) {
	case CPU_UP_PREPARE:
	case CPU_UP_PREPARE_FROZEN:
4704
		if (conf->level == 6 && !percpu->spare_page)
4705
			percpu->spare_page = alloc_page(GFP_KERNEL);
4706 4707 4708 4709 4710 4711 4712
		if (!percpu->scribble)
			percpu->scribble = kmalloc(conf->scribble_len, GFP_KERNEL);

		if (!percpu->scribble ||
		    (conf->level == 6 && !percpu->spare_page)) {
			safe_put_page(percpu->spare_page);
			kfree(percpu->scribble);
4713 4714
			pr_err("%s: failed memory allocation for cpu%ld\n",
			       __func__, cpu);
4715
			return notifier_from_errno(-ENOMEM);
4716 4717 4718 4719 4720
		}
		break;
	case CPU_DEAD:
	case CPU_DEAD_FROZEN:
		safe_put_page(percpu->spare_page);
4721
		kfree(percpu->scribble);
4722
		percpu->spare_page = NULL;
4723
		percpu->scribble = NULL;
4724 4725 4726 4727 4728 4729 4730 4731 4732 4733 4734 4735
		break;
	default:
		break;
	}
	return NOTIFY_OK;
}
#endif

static int raid5_alloc_percpu(raid5_conf_t *conf)
{
	unsigned long cpu;
	struct page *spare_page;
4736
	struct raid5_percpu __percpu *allcpus;
4737
	void *scribble;
4738 4739 4740 4741 4742 4743 4744 4745 4746 4747
	int err;

	allcpus = alloc_percpu(struct raid5_percpu);
	if (!allcpus)
		return -ENOMEM;
	conf->percpu = allcpus;

	get_online_cpus();
	err = 0;
	for_each_present_cpu(cpu) {
4748 4749 4750 4751 4752 4753 4754 4755
		if (conf->level == 6) {
			spare_page = alloc_page(GFP_KERNEL);
			if (!spare_page) {
				err = -ENOMEM;
				break;
			}
			per_cpu_ptr(conf->percpu, cpu)->spare_page = spare_page;
		}
4756
		scribble = kmalloc(conf->scribble_len, GFP_KERNEL);
4757
		if (!scribble) {
4758 4759 4760
			err = -ENOMEM;
			break;
		}
4761
		per_cpu_ptr(conf->percpu, cpu)->scribble = scribble;
4762 4763 4764 4765 4766 4767 4768 4769 4770 4771 4772 4773
	}
#ifdef CONFIG_HOTPLUG_CPU
	conf->cpu_notify.notifier_call = raid456_cpu_notify;
	conf->cpu_notify.priority = 0;
	if (err == 0)
		err = register_cpu_notifier(&conf->cpu_notify);
#endif
	put_online_cpus();

	return err;
}

N
NeilBrown 已提交
4774
static raid5_conf_t *setup_conf(mddev_t *mddev)
L
Linus Torvalds 已提交
4775 4776
{
	raid5_conf_t *conf;
4777
	int raid_disk, memory, max_disks;
L
Linus Torvalds 已提交
4778 4779 4780
	mdk_rdev_t *rdev;
	struct disk_info *disk;

N
NeilBrown 已提交
4781 4782 4783
	if (mddev->new_level != 5
	    && mddev->new_level != 4
	    && mddev->new_level != 6) {
4784
		printk(KERN_ERR "md/raid:%s: raid level not set to 4/5/6 (%d)\n",
N
NeilBrown 已提交
4785 4786
		       mdname(mddev), mddev->new_level);
		return ERR_PTR(-EIO);
L
Linus Torvalds 已提交
4787
	}
N
NeilBrown 已提交
4788 4789 4790 4791
	if ((mddev->new_level == 5
	     && !algorithm_valid_raid5(mddev->new_layout)) ||
	    (mddev->new_level == 6
	     && !algorithm_valid_raid6(mddev->new_layout))) {
4792
		printk(KERN_ERR "md/raid:%s: layout %d not supported\n",
N
NeilBrown 已提交
4793 4794
		       mdname(mddev), mddev->new_layout);
		return ERR_PTR(-EIO);
4795
	}
N
NeilBrown 已提交
4796
	if (mddev->new_level == 6 && mddev->raid_disks < 4) {
4797
		printk(KERN_ERR "md/raid:%s: not enough configured devices (%d, minimum 4)\n",
N
NeilBrown 已提交
4798 4799
		       mdname(mddev), mddev->raid_disks);
		return ERR_PTR(-EINVAL);
4800 4801
	}

4802 4803 4804
	if (!mddev->new_chunk_sectors ||
	    (mddev->new_chunk_sectors << 9) % PAGE_SIZE ||
	    !is_power_of_2(mddev->new_chunk_sectors)) {
4805 4806
		printk(KERN_ERR "md/raid:%s: invalid chunk size %d\n",
		       mdname(mddev), mddev->new_chunk_sectors << 9);
N
NeilBrown 已提交
4807
		return ERR_PTR(-EINVAL);
4808 4809
	}

N
NeilBrown 已提交
4810 4811
	conf = kzalloc(sizeof(raid5_conf_t), GFP_KERNEL);
	if (conf == NULL)
L
Linus Torvalds 已提交
4812
		goto abort;
4813 4814 4815 4816 4817 4818 4819 4820 4821 4822 4823 4824
	spin_lock_init(&conf->device_lock);
	init_waitqueue_head(&conf->wait_for_stripe);
	init_waitqueue_head(&conf->wait_for_overlap);
	INIT_LIST_HEAD(&conf->handle_list);
	INIT_LIST_HEAD(&conf->hold_list);
	INIT_LIST_HEAD(&conf->delayed_list);
	INIT_LIST_HEAD(&conf->bitmap_list);
	INIT_LIST_HEAD(&conf->inactive_list);
	atomic_set(&conf->active_stripes, 0);
	atomic_set(&conf->preread_active_stripes, 0);
	atomic_set(&conf->active_aligned_reads, 0);
	conf->bypass_threshold = BYPASS_THRESHOLD;
N
NeilBrown 已提交
4825 4826 4827 4828 4829

	conf->raid_disks = mddev->raid_disks;
	if (mddev->reshape_position == MaxSector)
		conf->previous_raid_disks = mddev->raid_disks;
	else
4830
		conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks;
4831 4832
	max_disks = max(conf->raid_disks, conf->previous_raid_disks);
	conf->scribble_len = scribble_len(max_disks);
4833

4834
	conf->disks = kzalloc(max_disks * sizeof(struct disk_info),
4835 4836 4837
			      GFP_KERNEL);
	if (!conf->disks)
		goto abort;
4838

L
Linus Torvalds 已提交
4839 4840
	conf->mddev = mddev;

4841
	if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL)
L
Linus Torvalds 已提交
4842 4843
		goto abort;

4844 4845 4846 4847
	conf->level = mddev->new_level;
	if (raid5_alloc_percpu(conf) != 0)
		goto abort;

4848
	pr_debug("raid456: run(%s) called.\n", mdname(mddev));
L
Linus Torvalds 已提交
4849

4850
	list_for_each_entry(rdev, &mddev->disks, same_set) {
L
Linus Torvalds 已提交
4851
		raid_disk = rdev->raid_disk;
4852
		if (raid_disk >= max_disks
L
Linus Torvalds 已提交
4853 4854 4855 4856 4857 4858
		    || raid_disk < 0)
			continue;
		disk = conf->disks + raid_disk;

		disk->rdev = rdev;

4859
		if (test_bit(In_sync, &rdev->flags)) {
L
Linus Torvalds 已提交
4860
			char b[BDEVNAME_SIZE];
4861 4862 4863
			printk(KERN_INFO "md/raid:%s: device %s operational as raid"
			       " disk %d\n",
			       mdname(mddev), bdevname(rdev->bdev, b), raid_disk);
J
Jonathan Brassow 已提交
4864
		} else if (rdev->saved_raid_disk != raid_disk)
4865 4866
			/* Cannot rely on bitmap to complete recovery */
			conf->fullsync = 1;
L
Linus Torvalds 已提交
4867 4868
	}

4869
	conf->chunk_sectors = mddev->new_chunk_sectors;
N
NeilBrown 已提交
4870
	conf->level = mddev->new_level;
4871 4872 4873 4874
	if (conf->level == 6)
		conf->max_degraded = 2;
	else
		conf->max_degraded = 1;
N
NeilBrown 已提交
4875
	conf->algorithm = mddev->new_layout;
L
Linus Torvalds 已提交
4876
	conf->max_nr_stripes = NR_STRIPES;
4877
	conf->reshape_progress = mddev->reshape_position;
4878
	if (conf->reshape_progress != MaxSector) {
4879
		conf->prev_chunk_sectors = mddev->chunk_sectors;
4880 4881
		conf->prev_algo = mddev->layout;
	}
L
Linus Torvalds 已提交
4882

N
NeilBrown 已提交
4883
	memory = conf->max_nr_stripes * (sizeof(struct stripe_head) +
4884
		 max_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024;
N
NeilBrown 已提交
4885 4886
	if (grow_stripes(conf, conf->max_nr_stripes)) {
		printk(KERN_ERR
4887 4888
		       "md/raid:%s: couldn't allocate %dkB for buffers\n",
		       mdname(mddev), memory);
N
NeilBrown 已提交
4889 4890
		goto abort;
	} else
4891 4892
		printk(KERN_INFO "md/raid:%s: allocated %dkB\n",
		       mdname(mddev), memory);
L
Linus Torvalds 已提交
4893

4894
	conf->thread = md_register_thread(raid5d, mddev, NULL);
N
NeilBrown 已提交
4895 4896
	if (!conf->thread) {
		printk(KERN_ERR
4897
		       "md/raid:%s: couldn't allocate thread.\n",
N
NeilBrown 已提交
4898
		       mdname(mddev));
4899 4900
		goto abort;
	}
N
NeilBrown 已提交
4901 4902 4903 4904 4905

	return conf;

 abort:
	if (conf) {
4906
		free_conf(conf);
N
NeilBrown 已提交
4907 4908 4909 4910 4911
		return ERR_PTR(-EIO);
	} else
		return ERR_PTR(-ENOMEM);
}

4912 4913 4914 4915 4916 4917 4918 4919 4920 4921 4922 4923 4924 4925 4926 4927 4928 4929 4930 4931 4932 4933 4934 4935 4936 4937 4938

static int only_parity(int raid_disk, int algo, int raid_disks, int max_degraded)
{
	switch (algo) {
	case ALGORITHM_PARITY_0:
		if (raid_disk < max_degraded)
			return 1;
		break;
	case ALGORITHM_PARITY_N:
		if (raid_disk >= raid_disks - max_degraded)
			return 1;
		break;
	case ALGORITHM_PARITY_0_6:
		if (raid_disk == 0 || 
		    raid_disk == raid_disks - 1)
			return 1;
		break;
	case ALGORITHM_LEFT_ASYMMETRIC_6:
	case ALGORITHM_RIGHT_ASYMMETRIC_6:
	case ALGORITHM_LEFT_SYMMETRIC_6:
	case ALGORITHM_RIGHT_SYMMETRIC_6:
		if (raid_disk == raid_disks - 1)
			return 1;
	}
	return 0;
}

N
NeilBrown 已提交
4939 4940 4941
static int run(mddev_t *mddev)
{
	raid5_conf_t *conf;
4942
	int working_disks = 0;
4943
	int dirty_parity_disks = 0;
N
NeilBrown 已提交
4944
	mdk_rdev_t *rdev;
4945
	sector_t reshape_offset = 0;
N
NeilBrown 已提交
4946

4947
	if (mddev->recovery_cp != MaxSector)
4948
		printk(KERN_NOTICE "md/raid:%s: not clean"
4949 4950
		       " -- starting background reconstruction\n",
		       mdname(mddev));
N
NeilBrown 已提交
4951 4952 4953 4954 4955 4956 4957 4958
	if (mddev->reshape_position != MaxSector) {
		/* Check that we can continue the reshape.
		 * Currently only disks can change, it must
		 * increase, and we must be past the point where
		 * a stripe over-writes itself
		 */
		sector_t here_new, here_old;
		int old_disks;
4959
		int max_degraded = (mddev->level == 6 ? 2 : 1);
N
NeilBrown 已提交
4960

4961
		if (mddev->new_level != mddev->level) {
4962
			printk(KERN_ERR "md/raid:%s: unsupported reshape "
N
NeilBrown 已提交
4963 4964 4965 4966 4967 4968 4969 4970 4971 4972
			       "required - aborting.\n",
			       mdname(mddev));
			return -EINVAL;
		}
		old_disks = mddev->raid_disks - mddev->delta_disks;
		/* reshape_position must be on a new-stripe boundary, and one
		 * further up in new geometry must map after here in old
		 * geometry.
		 */
		here_new = mddev->reshape_position;
4973
		if (sector_div(here_new, mddev->new_chunk_sectors *
N
NeilBrown 已提交
4974
			       (mddev->raid_disks - max_degraded))) {
4975 4976
			printk(KERN_ERR "md/raid:%s: reshape_position not "
			       "on a stripe boundary\n", mdname(mddev));
N
NeilBrown 已提交
4977 4978
			return -EINVAL;
		}
4979
		reshape_offset = here_new * mddev->new_chunk_sectors;
N
NeilBrown 已提交
4980 4981
		/* here_new is the stripe we will write to */
		here_old = mddev->reshape_position;
4982
		sector_div(here_old, mddev->chunk_sectors *
N
NeilBrown 已提交
4983 4984 4985
			   (old_disks-max_degraded));
		/* here_old is the first stripe that we might need to read
		 * from */
4986 4987 4988 4989 4990 4991 4992 4993 4994 4995 4996
		if (mddev->delta_disks == 0) {
			/* We cannot be sure it is safe to start an in-place
			 * reshape.  It is only safe if user-space if monitoring
			 * and taking constant backups.
			 * mdadm always starts a situation like this in
			 * readonly mode so it can take control before
			 * allowing any writes.  So just check for that.
			 */
			if ((here_new * mddev->new_chunk_sectors != 
			     here_old * mddev->chunk_sectors) ||
			    mddev->ro == 0) {
4997 4998 4999
				printk(KERN_ERR "md/raid:%s: in-place reshape must be started"
				       " in read-only mode - aborting\n",
				       mdname(mddev));
5000 5001 5002 5003 5004 5005 5006
				return -EINVAL;
			}
		} else if (mddev->delta_disks < 0
		    ? (here_new * mddev->new_chunk_sectors <=
		       here_old * mddev->chunk_sectors)
		    : (here_new * mddev->new_chunk_sectors >=
		       here_old * mddev->chunk_sectors)) {
N
NeilBrown 已提交
5007
			/* Reading from the same stripe as writing to - bad */
5008 5009 5010
			printk(KERN_ERR "md/raid:%s: reshape_position too early for "
			       "auto-recovery - aborting.\n",
			       mdname(mddev));
N
NeilBrown 已提交
5011 5012
			return -EINVAL;
		}
5013 5014
		printk(KERN_INFO "md/raid:%s: reshape will continue\n",
		       mdname(mddev));
N
NeilBrown 已提交
5015 5016 5017 5018
		/* OK, we should be able to continue; */
	} else {
		BUG_ON(mddev->level != mddev->new_level);
		BUG_ON(mddev->layout != mddev->new_layout);
5019
		BUG_ON(mddev->chunk_sectors != mddev->new_chunk_sectors);
N
NeilBrown 已提交
5020
		BUG_ON(mddev->delta_disks != 0);
L
Linus Torvalds 已提交
5021
	}
N
NeilBrown 已提交
5022

5023 5024 5025 5026 5027
	if (mddev->private == NULL)
		conf = setup_conf(mddev);
	else
		conf = mddev->private;

N
NeilBrown 已提交
5028 5029 5030 5031 5032 5033 5034 5035 5036 5037
	if (IS_ERR(conf))
		return PTR_ERR(conf);

	mddev->thread = conf->thread;
	conf->thread = NULL;
	mddev->private = conf;

	/*
	 * 0 for a fully functional array, 1 or 2 for a degraded array.
	 */
5038 5039 5040
	list_for_each_entry(rdev, &mddev->disks, same_set) {
		if (rdev->raid_disk < 0)
			continue;
5041
		if (test_bit(In_sync, &rdev->flags)) {
N
NeilBrown 已提交
5042
			working_disks++;
5043 5044
			continue;
		}
5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057 5058 5059 5060 5061 5062 5063 5064 5065 5066 5067 5068 5069 5070 5071 5072
		/* This disc is not fully in-sync.  However if it
		 * just stored parity (beyond the recovery_offset),
		 * when we don't need to be concerned about the
		 * array being dirty.
		 * When reshape goes 'backwards', we never have
		 * partially completed devices, so we only need
		 * to worry about reshape going forwards.
		 */
		/* Hack because v0.91 doesn't store recovery_offset properly. */
		if (mddev->major_version == 0 &&
		    mddev->minor_version > 90)
			rdev->recovery_offset = reshape_offset;
			
		if (rdev->recovery_offset < reshape_offset) {
			/* We need to check old and new layout */
			if (!only_parity(rdev->raid_disk,
					 conf->algorithm,
					 conf->raid_disks,
					 conf->max_degraded))
				continue;
		}
		if (!only_parity(rdev->raid_disk,
				 conf->prev_algo,
				 conf->previous_raid_disks,
				 conf->max_degraded))
			continue;
		dirty_parity_disks++;
	}
N
NeilBrown 已提交
5073

5074 5075
	mddev->degraded = (max(conf->raid_disks, conf->previous_raid_disks)
			   - working_disks);
N
NeilBrown 已提交
5076

5077
	if (has_failed(conf)) {
5078
		printk(KERN_ERR "md/raid:%s: not enough operational devices"
L
Linus Torvalds 已提交
5079
			" (%d/%d failed)\n",
5080
			mdname(mddev), mddev->degraded, conf->raid_disks);
L
Linus Torvalds 已提交
5081 5082 5083
		goto abort;
	}

N
NeilBrown 已提交
5084
	/* device size must be a multiple of chunk size */
5085
	mddev->dev_sectors &= ~(mddev->chunk_sectors - 1);
N
NeilBrown 已提交
5086 5087
	mddev->resync_max_sectors = mddev->dev_sectors;

5088
	if (mddev->degraded > dirty_parity_disks &&
L
Linus Torvalds 已提交
5089
	    mddev->recovery_cp != MaxSector) {
5090 5091
		if (mddev->ok_start_degraded)
			printk(KERN_WARNING
5092 5093
			       "md/raid:%s: starting dirty degraded array"
			       " - data corruption possible.\n",
5094 5095 5096
			       mdname(mddev));
		else {
			printk(KERN_ERR
5097
			       "md/raid:%s: cannot start dirty degraded array.\n",
5098 5099 5100
			       mdname(mddev));
			goto abort;
		}
L
Linus Torvalds 已提交
5101 5102 5103
	}

	if (mddev->degraded == 0)
5104 5105
		printk(KERN_INFO "md/raid:%s: raid level %d active with %d out of %d"
		       " devices, algorithm %d\n", mdname(mddev), conf->level,
5106 5107
		       mddev->raid_disks-mddev->degraded, mddev->raid_disks,
		       mddev->new_layout);
L
Linus Torvalds 已提交
5108
	else
5109 5110 5111 5112 5113
		printk(KERN_ALERT "md/raid:%s: raid level %d active with %d"
		       " out of %d devices, algorithm %d\n",
		       mdname(mddev), conf->level,
		       mddev->raid_disks - mddev->degraded,
		       mddev->raid_disks, mddev->new_layout);
L
Linus Torvalds 已提交
5114 5115 5116

	print_raid5_conf(conf);

5117 5118
	if (conf->reshape_progress != MaxSector) {
		conf->reshape_safe = conf->reshape_progress;
5119 5120 5121 5122 5123 5124
		atomic_set(&conf->reshape_stripes, 0);
		clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
		clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
		set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
		set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
		mddev->sync_thread = md_register_thread(md_do_sync, mddev,
5125
							"reshape");
5126 5127
	}

L
Linus Torvalds 已提交
5128 5129

	/* Ok, everything is just fine now */
5130 5131
	if (mddev->to_remove == &raid5_attrs_group)
		mddev->to_remove = NULL;
N
NeilBrown 已提交
5132 5133
	else if (mddev->kobj.sd &&
	    sysfs_create_group(&mddev->kobj, &raid5_attrs_group))
5134
		printk(KERN_WARNING
5135
		       "raid5: failed to create sysfs attributes for %s\n",
5136
		       mdname(mddev));
5137
	md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
5138

5139
	if (mddev->queue) {
5140
		int chunk_size;
5141 5142 5143 5144 5145 5146 5147 5148 5149
		/* read-ahead size must cover two whole stripes, which
		 * is 2 * (datadisks) * chunksize where 'n' is the
		 * number of raid devices
		 */
		int data_disks = conf->previous_raid_disks - conf->max_degraded;
		int stripe = data_disks *
			((mddev->chunk_sectors << 9) / PAGE_SIZE);
		if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
			mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
N
NeilBrown 已提交
5150

5151
		blk_queue_merge_bvec(mddev->queue, raid5_mergeable_bvec);
5152

N
NeilBrown 已提交
5153 5154
		mddev->queue->backing_dev_info.congested_data = mddev;
		mddev->queue->backing_dev_info.congested_fn = raid5_congested;
5155

5156 5157 5158 5159
		chunk_size = mddev->chunk_sectors << 9;
		blk_queue_io_min(mddev->queue, chunk_size);
		blk_queue_io_opt(mddev->queue, chunk_size *
				 (conf->raid_disks - conf->max_degraded));
5160

5161 5162 5163 5164
		list_for_each_entry(rdev, &mddev->disks, same_set)
			disk_stack_limits(mddev->gendisk, rdev->bdev,
					  rdev->data_offset << 9);
	}
5165

L
Linus Torvalds 已提交
5166 5167
	return 0;
abort:
5168
	md_unregister_thread(mddev->thread);
N
NeilBrown 已提交
5169
	mddev->thread = NULL;
L
Linus Torvalds 已提交
5170 5171
	if (conf) {
		print_raid5_conf(conf);
5172
		free_conf(conf);
L
Linus Torvalds 已提交
5173 5174
	}
	mddev->private = NULL;
5175
	printk(KERN_ALERT "md/raid:%s: failed to run raid set.\n", mdname(mddev));
L
Linus Torvalds 已提交
5176 5177 5178
	return -EIO;
}

5179
static int stop(mddev_t *mddev)
L
Linus Torvalds 已提交
5180
{
5181
	raid5_conf_t *conf = mddev->private;
L
Linus Torvalds 已提交
5182 5183 5184

	md_unregister_thread(mddev->thread);
	mddev->thread = NULL;
N
NeilBrown 已提交
5185 5186
	if (mddev->queue)
		mddev->queue->backing_dev_info.congested_fn = NULL;
5187
	free_conf(conf);
5188 5189
	mddev->private = NULL;
	mddev->to_remove = &raid5_attrs_group;
L
Linus Torvalds 已提交
5190 5191 5192
	return 0;
}

5193
#ifdef DEBUG
5194
static void print_sh(struct seq_file *seq, struct stripe_head *sh)
L
Linus Torvalds 已提交
5195 5196 5197
{
	int i;

5198 5199 5200 5201 5202
	seq_printf(seq, "sh %llu, pd_idx %d, state %ld.\n",
		   (unsigned long long)sh->sector, sh->pd_idx, sh->state);
	seq_printf(seq, "sh %llu,  count %d.\n",
		   (unsigned long long)sh->sector, atomic_read(&sh->count));
	seq_printf(seq, "sh %llu, ", (unsigned long long)sh->sector);
5203
	for (i = 0; i < sh->disks; i++) {
5204 5205
		seq_printf(seq, "(cache%d: %p %ld) ",
			   i, sh->dev[i].page, sh->dev[i].flags);
L
Linus Torvalds 已提交
5206
	}
5207
	seq_printf(seq, "\n");
L
Linus Torvalds 已提交
5208 5209
}

5210
static void printall(struct seq_file *seq, raid5_conf_t *conf)
L
Linus Torvalds 已提交
5211 5212
{
	struct stripe_head *sh;
5213
	struct hlist_node *hn;
L
Linus Torvalds 已提交
5214 5215 5216 5217
	int i;

	spin_lock_irq(&conf->device_lock);
	for (i = 0; i < NR_HASH; i++) {
5218
		hlist_for_each_entry(sh, hn, &conf->stripe_hashtbl[i], hash) {
L
Linus Torvalds 已提交
5219 5220
			if (sh->raid_conf != conf)
				continue;
5221
			print_sh(seq, sh);
L
Linus Torvalds 已提交
5222 5223 5224 5225 5226 5227
		}
	}
	spin_unlock_irq(&conf->device_lock);
}
#endif

5228
static void status(struct seq_file *seq, mddev_t *mddev)
L
Linus Torvalds 已提交
5229
{
5230
	raid5_conf_t *conf = mddev->private;
L
Linus Torvalds 已提交
5231 5232
	int i;

5233 5234
	seq_printf(seq, " level %d, %dk chunk, algorithm %d", mddev->level,
		mddev->chunk_sectors / 2, mddev->layout);
5235
	seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded);
L
Linus Torvalds 已提交
5236 5237 5238
	for (i = 0; i < conf->raid_disks; i++)
		seq_printf (seq, "%s",
			       conf->disks[i].rdev &&
5239
			       test_bit(In_sync, &conf->disks[i].rdev->flags) ? "U" : "_");
L
Linus Torvalds 已提交
5240
	seq_printf (seq, "]");
5241
#ifdef DEBUG
5242 5243
	seq_printf (seq, "\n");
	printall(seq, conf);
L
Linus Torvalds 已提交
5244 5245 5246 5247 5248 5249 5250 5251
#endif
}

static void print_raid5_conf (raid5_conf_t *conf)
{
	int i;
	struct disk_info *tmp;

5252
	printk(KERN_DEBUG "RAID conf printout:\n");
L
Linus Torvalds 已提交
5253 5254 5255 5256
	if (!conf) {
		printk("(conf==NULL)\n");
		return;
	}
5257 5258 5259
	printk(KERN_DEBUG " --- level:%d rd:%d wd:%d\n", conf->level,
	       conf->raid_disks,
	       conf->raid_disks - conf->mddev->degraded);
L
Linus Torvalds 已提交
5260 5261 5262 5263 5264

	for (i = 0; i < conf->raid_disks; i++) {
		char b[BDEVNAME_SIZE];
		tmp = conf->disks + i;
		if (tmp->rdev)
5265 5266 5267
			printk(KERN_DEBUG " disk %d, o:%d, dev:%s\n",
			       i, !test_bit(Faulty, &tmp->rdev->flags),
			       bdevname(tmp->rdev->bdev, b));
L
Linus Torvalds 已提交
5268 5269 5270 5271 5272 5273 5274 5275
	}
}

static int raid5_spare_active(mddev_t *mddev)
{
	int i;
	raid5_conf_t *conf = mddev->private;
	struct disk_info *tmp;
5276 5277
	int count = 0;
	unsigned long flags;
L
Linus Torvalds 已提交
5278 5279 5280 5281

	for (i = 0; i < conf->raid_disks; i++) {
		tmp = conf->disks + i;
		if (tmp->rdev
5282
		    && tmp->rdev->recovery_offset == MaxSector
5283
		    && !test_bit(Faulty, &tmp->rdev->flags)
5284
		    && !test_and_set_bit(In_sync, &tmp->rdev->flags)) {
5285
			count++;
5286
			sysfs_notify_dirent_safe(tmp->rdev->sysfs_state);
L
Linus Torvalds 已提交
5287 5288
		}
	}
5289 5290 5291
	spin_lock_irqsave(&conf->device_lock, flags);
	mddev->degraded -= count;
	spin_unlock_irqrestore(&conf->device_lock, flags);
L
Linus Torvalds 已提交
5292
	print_raid5_conf(conf);
5293
	return count;
L
Linus Torvalds 已提交
5294 5295 5296 5297 5298 5299 5300 5301 5302 5303 5304 5305
}

static int raid5_remove_disk(mddev_t *mddev, int number)
{
	raid5_conf_t *conf = mddev->private;
	int err = 0;
	mdk_rdev_t *rdev;
	struct disk_info *p = conf->disks + number;

	print_raid5_conf(conf);
	rdev = p->rdev;
	if (rdev) {
5306 5307 5308 5309
		if (number >= conf->raid_disks &&
		    conf->reshape_progress == MaxSector)
			clear_bit(In_sync, &rdev->flags);

5310
		if (test_bit(In_sync, &rdev->flags) ||
L
Linus Torvalds 已提交
5311 5312 5313 5314
		    atomic_read(&rdev->nr_pending)) {
			err = -EBUSY;
			goto abort;
		}
5315 5316 5317 5318
		/* Only remove non-faulty devices if recovery
		 * isn't possible.
		 */
		if (!test_bit(Faulty, &rdev->flags) &&
5319
		    !has_failed(conf) &&
5320
		    number < conf->raid_disks) {
5321 5322 5323
			err = -EBUSY;
			goto abort;
		}
L
Linus Torvalds 已提交
5324
		p->rdev = NULL;
5325
		synchronize_rcu();
L
Linus Torvalds 已提交
5326 5327 5328 5329 5330 5331 5332 5333 5334 5335 5336 5337 5338 5339 5340
		if (atomic_read(&rdev->nr_pending)) {
			/* lost the race, try later */
			err = -EBUSY;
			p->rdev = rdev;
		}
	}
abort:

	print_raid5_conf(conf);
	return err;
}

static int raid5_add_disk(mddev_t *mddev, mdk_rdev_t *rdev)
{
	raid5_conf_t *conf = mddev->private;
5341
	int err = -EEXIST;
L
Linus Torvalds 已提交
5342 5343
	int disk;
	struct disk_info *p;
5344 5345
	int first = 0;
	int last = conf->raid_disks - 1;
L
Linus Torvalds 已提交
5346

5347
	if (has_failed(conf))
L
Linus Torvalds 已提交
5348
		/* no point adding a device */
5349
		return -EINVAL;
L
Linus Torvalds 已提交
5350

5351 5352
	if (rdev->raid_disk >= 0)
		first = last = rdev->raid_disk;
L
Linus Torvalds 已提交
5353 5354

	/*
5355 5356
	 * find the disk ... but prefer rdev->saved_raid_disk
	 * if possible.
L
Linus Torvalds 已提交
5357
	 */
5358
	if (rdev->saved_raid_disk >= 0 &&
5359
	    rdev->saved_raid_disk >= first &&
5360 5361 5362
	    conf->disks[rdev->saved_raid_disk].rdev == NULL)
		disk = rdev->saved_raid_disk;
	else
5363 5364
		disk = first;
	for ( ; disk <= last ; disk++)
L
Linus Torvalds 已提交
5365
		if ((p=conf->disks + disk)->rdev == NULL) {
5366
			clear_bit(In_sync, &rdev->flags);
L
Linus Torvalds 已提交
5367
			rdev->raid_disk = disk;
5368
			err = 0;
5369 5370
			if (rdev->saved_raid_disk != disk)
				conf->fullsync = 1;
5371
			rcu_assign_pointer(p->rdev, rdev);
L
Linus Torvalds 已提交
5372 5373 5374
			break;
		}
	print_raid5_conf(conf);
5375
	return err;
L
Linus Torvalds 已提交
5376 5377 5378 5379 5380 5381 5382 5383 5384 5385 5386
}

static int raid5_resize(mddev_t *mddev, sector_t sectors)
{
	/* no resync is happening, and there is enough space
	 * on all devices, so we can resize.
	 * We need to make sure resync covers any new space.
	 * If the array is shrinking we should possibly wait until
	 * any io in the removed space completes, but it hardly seems
	 * worth it.
	 */
5387
	sectors &= ~((sector_t)mddev->chunk_sectors - 1);
5388 5389
	md_set_array_sectors(mddev, raid5_size(mddev, sectors,
					       mddev->raid_disks));
D
Dan Williams 已提交
5390 5391 5392
	if (mddev->array_sectors >
	    raid5_size(mddev, sectors, mddev->raid_disks))
		return -EINVAL;
5393
	set_capacity(mddev->gendisk, mddev->array_sectors);
5394
	revalidate_disk(mddev->gendisk);
5395 5396
	if (sectors > mddev->dev_sectors &&
	    mddev->recovery_cp > mddev->dev_sectors) {
A
Andre Noll 已提交
5397
		mddev->recovery_cp = mddev->dev_sectors;
L
Linus Torvalds 已提交
5398 5399
		set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
	}
A
Andre Noll 已提交
5400
	mddev->dev_sectors = sectors;
5401
	mddev->resync_max_sectors = sectors;
L
Linus Torvalds 已提交
5402 5403 5404
	return 0;
}

5405 5406 5407 5408 5409 5410 5411 5412 5413 5414 5415 5416 5417 5418 5419
static int check_stripe_cache(mddev_t *mddev)
{
	/* Can only proceed if there are plenty of stripe_heads.
	 * We need a minimum of one full stripe,, and for sensible progress
	 * it is best to have about 4 times that.
	 * If we require 4 times, then the default 256 4K stripe_heads will
	 * allow for chunk sizes up to 256K, which is probably OK.
	 * If the chunk size is greater, user-space should request more
	 * stripe_heads first.
	 */
	raid5_conf_t *conf = mddev->private;
	if (((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4
	    > conf->max_nr_stripes ||
	    ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4
	    > conf->max_nr_stripes) {
5420 5421
		printk(KERN_WARNING "md/raid:%s: reshape: not enough stripes.  Needed %lu\n",
		       mdname(mddev),
5422 5423 5424 5425 5426 5427 5428
		       ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9)
			/ STRIPE_SIZE)*4);
		return 0;
	}
	return 1;
}

5429
static int check_reshape(mddev_t *mddev)
5430
{
5431
	raid5_conf_t *conf = mddev->private;
5432

5433 5434
	if (mddev->delta_disks == 0 &&
	    mddev->new_layout == mddev->layout &&
5435
	    mddev->new_chunk_sectors == mddev->chunk_sectors)
5436
		return 0; /* nothing to do */
5437 5438 5439
	if (mddev->bitmap)
		/* Cannot grow a bitmap yet */
		return -EBUSY;
5440
	if (has_failed(conf))
5441 5442 5443 5444 5445 5446 5447 5448 5449 5450 5451 5452 5453
		return -EINVAL;
	if (mddev->delta_disks < 0) {
		/* We might be able to shrink, but the devices must
		 * be made bigger first.
		 * For raid6, 4 is the minimum size.
		 * Otherwise 2 is the minimum
		 */
		int min = 2;
		if (mddev->level == 6)
			min = 4;
		if (mddev->raid_disks + mddev->delta_disks < min)
			return -EINVAL;
	}
5454

5455
	if (!check_stripe_cache(mddev))
5456 5457
		return -ENOSPC;

5458
	return resize_stripes(conf, conf->raid_disks + mddev->delta_disks);
5459 5460 5461 5462
}

static int raid5_start_reshape(mddev_t *mddev)
{
5463
	raid5_conf_t *conf = mddev->private;
5464 5465
	mdk_rdev_t *rdev;
	int spares = 0;
5466
	unsigned long flags;
5467

5468
	if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
5469 5470
		return -EBUSY;

5471 5472 5473
	if (!check_stripe_cache(mddev))
		return -ENOSPC;

5474
	list_for_each_entry(rdev, &mddev->disks, same_set)
5475 5476
		if (!test_bit(In_sync, &rdev->flags)
		    && !test_bit(Faulty, &rdev->flags))
5477
			spares++;
5478

5479
	if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded)
5480 5481 5482 5483 5484
		/* Not enough devices even to make a degraded array
		 * of that size
		 */
		return -EINVAL;

5485 5486 5487 5488 5489 5490
	/* Refuse to reduce size of the array.  Any reductions in
	 * array size must be through explicit setting of array_size
	 * attribute.
	 */
	if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks)
	    < mddev->array_sectors) {
5491
		printk(KERN_ERR "md/raid:%s: array size must be reduced "
5492 5493 5494 5495
		       "before number of disks\n", mdname(mddev));
		return -EINVAL;
	}

5496
	atomic_set(&conf->reshape_stripes, 0);
5497 5498
	spin_lock_irq(&conf->device_lock);
	conf->previous_raid_disks = conf->raid_disks;
5499
	conf->raid_disks += mddev->delta_disks;
5500 5501
	conf->prev_chunk_sectors = conf->chunk_sectors;
	conf->chunk_sectors = mddev->new_chunk_sectors;
5502 5503
	conf->prev_algo = conf->algorithm;
	conf->algorithm = mddev->new_layout;
5504 5505 5506 5507 5508
	if (mddev->delta_disks < 0)
		conf->reshape_progress = raid5_size(mddev, 0, 0);
	else
		conf->reshape_progress = 0;
	conf->reshape_safe = conf->reshape_progress;
5509
	conf->generation++;
5510 5511 5512 5513
	spin_unlock_irq(&conf->device_lock);

	/* Add some new drives, as many as will fit.
	 * We know there are enough to make the newly sized array work.
5514 5515 5516 5517
	 * Don't add devices if we are reducing the number of
	 * devices in the array.  This is because it is not possible
	 * to correctly record the "partially reconstructed" state of
	 * such devices during the reshape and confusion could result.
5518
	 */
5519 5520 5521 5522 5523 5524 5525 5526 5527 5528 5529 5530 5531 5532 5533 5534 5535
	if (mddev->delta_disks >= 0) {
		int added_devices = 0;
		list_for_each_entry(rdev, &mddev->disks, same_set)
			if (rdev->raid_disk < 0 &&
			    !test_bit(Faulty, &rdev->flags)) {
				if (raid5_add_disk(mddev, rdev) == 0) {
					char nm[20];
					if (rdev->raid_disk
					    >= conf->previous_raid_disks) {
						set_bit(In_sync, &rdev->flags);
						added_devices++;
					} else
						rdev->recovery_offset = 0;
					sprintf(nm, "rd%d", rdev->raid_disk);
					if (sysfs_create_link(&mddev->kobj,
							      &rdev->kobj, nm))
						/* Failure here is OK */;
5536
				}
5537 5538 5539 5540 5541 5542
			} else if (rdev->raid_disk >= conf->previous_raid_disks
				   && !test_bit(Faulty, &rdev->flags)) {
				/* This is a spare that was manually added */
				set_bit(In_sync, &rdev->flags);
				added_devices++;
			}
5543

5544 5545 5546 5547
		/* When a reshape changes the number of devices,
		 * ->degraded is measured against the larger of the
		 * pre and post number of devices.
		 */
5548
		spin_lock_irqsave(&conf->device_lock, flags);
5549
		mddev->degraded += (conf->raid_disks - conf->previous_raid_disks)
5550 5551 5552
			- added_devices;
		spin_unlock_irqrestore(&conf->device_lock, flags);
	}
5553
	mddev->raid_disks = conf->raid_disks;
5554
	mddev->reshape_position = conf->reshape_progress;
5555
	set_bit(MD_CHANGE_DEVS, &mddev->flags);
5556

5557 5558 5559 5560 5561
	clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
	clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
	set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
	set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
	mddev->sync_thread = md_register_thread(md_do_sync, mddev,
5562
						"reshape");
5563 5564 5565 5566
	if (!mddev->sync_thread) {
		mddev->recovery = 0;
		spin_lock_irq(&conf->device_lock);
		mddev->raid_disks = conf->raid_disks = conf->previous_raid_disks;
5567
		conf->reshape_progress = MaxSector;
5568 5569 5570
		spin_unlock_irq(&conf->device_lock);
		return -EAGAIN;
	}
5571
	conf->reshape_checkpoint = jiffies;
5572 5573 5574 5575 5576
	md_wakeup_thread(mddev->sync_thread);
	md_new_event(mddev);
	return 0;
}

5577 5578 5579
/* This is called from the reshape thread and should make any
 * changes needed in 'conf'
 */
5580 5581 5582
static void end_reshape(raid5_conf_t *conf)
{

5583 5584 5585
	if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) {

		spin_lock_irq(&conf->device_lock);
5586
		conf->previous_raid_disks = conf->raid_disks;
5587
		conf->reshape_progress = MaxSector;
5588
		spin_unlock_irq(&conf->device_lock);
5589
		wake_up(&conf->wait_for_overlap);
5590 5591 5592 5593

		/* read-ahead size must cover two whole stripes, which is
		 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices
		 */
5594
		if (conf->mddev->queue) {
5595
			int data_disks = conf->raid_disks - conf->max_degraded;
5596
			int stripe = data_disks * ((conf->chunk_sectors << 9)
5597
						   / PAGE_SIZE);
5598 5599 5600
			if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
				conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
		}
5601 5602 5603
	}
}

5604 5605 5606
/* This is called from the raid5d thread with mddev_lock held.
 * It makes config changes to the device.
 */
5607 5608
static void raid5_finish_reshape(mddev_t *mddev)
{
5609
	raid5_conf_t *conf = mddev->private;
5610 5611 5612

	if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {

5613 5614 5615
		if (mddev->delta_disks > 0) {
			md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
			set_capacity(mddev->gendisk, mddev->array_sectors);
5616
			revalidate_disk(mddev->gendisk);
5617 5618 5619 5620 5621 5622 5623 5624 5625 5626
		} else {
			int d;
			mddev->degraded = conf->raid_disks;
			for (d = 0; d < conf->raid_disks ; d++)
				if (conf->disks[d].rdev &&
				    test_bit(In_sync,
					     &conf->disks[d].rdev->flags))
					mddev->degraded--;
			for (d = conf->raid_disks ;
			     d < conf->raid_disks - mddev->delta_disks;
5627 5628 5629 5630 5631 5632 5633 5634 5635
			     d++) {
				mdk_rdev_t *rdev = conf->disks[d].rdev;
				if (rdev && raid5_remove_disk(mddev, d) == 0) {
					char nm[20];
					sprintf(nm, "rd%d", rdev->raid_disk);
					sysfs_remove_link(&mddev->kobj, nm);
					rdev->raid_disk = -1;
				}
			}
5636
		}
5637
		mddev->layout = conf->algorithm;
5638
		mddev->chunk_sectors = conf->chunk_sectors;
5639 5640
		mddev->reshape_position = MaxSector;
		mddev->delta_disks = 0;
5641 5642 5643
	}
}

5644 5645
static void raid5_quiesce(mddev_t *mddev, int state)
{
5646
	raid5_conf_t *conf = mddev->private;
5647 5648

	switch(state) {
5649 5650 5651 5652
	case 2: /* resume for a suspend */
		wake_up(&conf->wait_for_overlap);
		break;

5653 5654
	case 1: /* stop all writes */
		spin_lock_irq(&conf->device_lock);
5655 5656 5657 5658
		/* '2' tells resync/reshape to pause so that all
		 * active stripes can drain
		 */
		conf->quiesce = 2;
5659
		wait_event_lock_irq(conf->wait_for_stripe,
5660 5661
				    atomic_read(&conf->active_stripes) == 0 &&
				    atomic_read(&conf->active_aligned_reads) == 0,
5662
				    conf->device_lock, /* nothing */);
5663
		conf->quiesce = 1;
5664
		spin_unlock_irq(&conf->device_lock);
5665 5666
		/* allow reshape to continue */
		wake_up(&conf->wait_for_overlap);
5667 5668 5669 5670 5671 5672
		break;

	case 0: /* re-enable writes */
		spin_lock_irq(&conf->device_lock);
		conf->quiesce = 0;
		wake_up(&conf->wait_for_stripe);
5673
		wake_up(&conf->wait_for_overlap);
5674 5675 5676 5677
		spin_unlock_irq(&conf->device_lock);
		break;
	}
}
5678

5679

D
Dan Williams 已提交
5680
static void *raid45_takeover_raid0(mddev_t *mddev, int level)
5681
{
D
Dan Williams 已提交
5682
	struct raid0_private_data *raid0_priv = mddev->private;
5683
	sector_t sectors;
5684

D
Dan Williams 已提交
5685 5686
	/* for raid0 takeover only one zone is supported */
	if (raid0_priv->nr_strip_zones > 1) {
5687 5688
		printk(KERN_ERR "md/raid:%s: cannot takeover raid0 with more than one zone.\n",
		       mdname(mddev));
D
Dan Williams 已提交
5689 5690 5691
		return ERR_PTR(-EINVAL);
	}

5692 5693 5694
	sectors = raid0_priv->strip_zone[0].zone_end;
	sector_div(sectors, raid0_priv->strip_zone[0].nb_dev);
	mddev->dev_sectors = sectors;
D
Dan Williams 已提交
5695
	mddev->new_level = level;
5696 5697 5698 5699 5700 5701 5702 5703 5704 5705 5706
	mddev->new_layout = ALGORITHM_PARITY_N;
	mddev->new_chunk_sectors = mddev->chunk_sectors;
	mddev->raid_disks += 1;
	mddev->delta_disks = 1;
	/* make sure it will be not marked as dirty */
	mddev->recovery_cp = MaxSector;

	return setup_conf(mddev);
}


5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718 5719 5720 5721 5722 5723 5724 5725 5726 5727 5728
static void *raid5_takeover_raid1(mddev_t *mddev)
{
	int chunksect;

	if (mddev->raid_disks != 2 ||
	    mddev->degraded > 1)
		return ERR_PTR(-EINVAL);

	/* Should check if there are write-behind devices? */

	chunksect = 64*2; /* 64K by default */

	/* The array must be an exact multiple of chunksize */
	while (chunksect && (mddev->array_sectors & (chunksect-1)))
		chunksect >>= 1;

	if ((chunksect<<9) < STRIPE_SIZE)
		/* array size does not allow a suitable chunk size */
		return ERR_PTR(-EINVAL);

	mddev->new_level = 5;
	mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC;
5729
	mddev->new_chunk_sectors = chunksect;
5730 5731 5732 5733

	return setup_conf(mddev);
}

5734 5735 5736 5737 5738 5739 5740 5741 5742 5743 5744 5745 5746 5747 5748 5749 5750 5751 5752 5753 5754 5755 5756 5757 5758 5759 5760 5761 5762 5763 5764 5765 5766
static void *raid5_takeover_raid6(mddev_t *mddev)
{
	int new_layout;

	switch (mddev->layout) {
	case ALGORITHM_LEFT_ASYMMETRIC_6:
		new_layout = ALGORITHM_LEFT_ASYMMETRIC;
		break;
	case ALGORITHM_RIGHT_ASYMMETRIC_6:
		new_layout = ALGORITHM_RIGHT_ASYMMETRIC;
		break;
	case ALGORITHM_LEFT_SYMMETRIC_6:
		new_layout = ALGORITHM_LEFT_SYMMETRIC;
		break;
	case ALGORITHM_RIGHT_SYMMETRIC_6:
		new_layout = ALGORITHM_RIGHT_SYMMETRIC;
		break;
	case ALGORITHM_PARITY_0_6:
		new_layout = ALGORITHM_PARITY_0;
		break;
	case ALGORITHM_PARITY_N:
		new_layout = ALGORITHM_PARITY_N;
		break;
	default:
		return ERR_PTR(-EINVAL);
	}
	mddev->new_level = 5;
	mddev->new_layout = new_layout;
	mddev->delta_disks = -1;
	mddev->raid_disks -= 1;
	return setup_conf(mddev);
}

5767

5768
static int raid5_check_reshape(mddev_t *mddev)
5769
{
5770 5771 5772 5773
	/* For a 2-drive array, the layout and chunk size can be changed
	 * immediately as not restriping is needed.
	 * For larger arrays we record the new value - after validation
	 * to be used by a reshape pass.
5774
	 */
5775
	raid5_conf_t *conf = mddev->private;
5776
	int new_chunk = mddev->new_chunk_sectors;
5777

5778
	if (mddev->new_layout >= 0 && !algorithm_valid_raid5(mddev->new_layout))
5779 5780
		return -EINVAL;
	if (new_chunk > 0) {
5781
		if (!is_power_of_2(new_chunk))
5782
			return -EINVAL;
5783
		if (new_chunk < (PAGE_SIZE>>9))
5784
			return -EINVAL;
5785
		if (mddev->array_sectors & (new_chunk-1))
5786 5787 5788 5789 5790 5791
			/* not factor of array size */
			return -EINVAL;
	}

	/* They look valid */

5792
	if (mddev->raid_disks == 2) {
5793 5794 5795 5796
		/* can make the change immediately */
		if (mddev->new_layout >= 0) {
			conf->algorithm = mddev->new_layout;
			mddev->layout = mddev->new_layout;
5797 5798
		}
		if (new_chunk > 0) {
5799 5800
			conf->chunk_sectors = new_chunk ;
			mddev->chunk_sectors = new_chunk;
5801 5802 5803
		}
		set_bit(MD_CHANGE_DEVS, &mddev->flags);
		md_wakeup_thread(mddev->thread);
5804
	}
5805
	return check_reshape(mddev);
5806 5807
}

5808
static int raid6_check_reshape(mddev_t *mddev)
5809
{
5810
	int new_chunk = mddev->new_chunk_sectors;
5811

5812
	if (mddev->new_layout >= 0 && !algorithm_valid_raid6(mddev->new_layout))
5813
		return -EINVAL;
5814
	if (new_chunk > 0) {
5815
		if (!is_power_of_2(new_chunk))
5816
			return -EINVAL;
5817
		if (new_chunk < (PAGE_SIZE >> 9))
5818
			return -EINVAL;
5819
		if (mddev->array_sectors & (new_chunk-1))
5820 5821
			/* not factor of array size */
			return -EINVAL;
5822
	}
5823 5824

	/* They look valid */
5825
	return check_reshape(mddev);
5826 5827
}

5828 5829 5830
static void *raid5_takeover(mddev_t *mddev)
{
	/* raid5 can take over:
D
Dan Williams 已提交
5831
	 *  raid0 - if there is only one strip zone - make it a raid4 layout
5832 5833 5834 5835
	 *  raid1 - if there are two drives.  We need to know the chunk size
	 *  raid4 - trivial - just use a raid4 layout.
	 *  raid6 - Providing it is a *_6 layout
	 */
D
Dan Williams 已提交
5836 5837
	if (mddev->level == 0)
		return raid45_takeover_raid0(mddev, 5);
5838 5839
	if (mddev->level == 1)
		return raid5_takeover_raid1(mddev);
5840 5841 5842 5843 5844
	if (mddev->level == 4) {
		mddev->new_layout = ALGORITHM_PARITY_N;
		mddev->new_level = 5;
		return setup_conf(mddev);
	}
5845 5846
	if (mddev->level == 6)
		return raid5_takeover_raid6(mddev);
5847 5848 5849 5850

	return ERR_PTR(-EINVAL);
}

5851 5852
static void *raid4_takeover(mddev_t *mddev)
{
D
Dan Williams 已提交
5853 5854 5855
	/* raid4 can take over:
	 *  raid0 - if there is only one strip zone
	 *  raid5 - if layout is right
5856
	 */
D
Dan Williams 已提交
5857 5858
	if (mddev->level == 0)
		return raid45_takeover_raid0(mddev, 4);
5859 5860 5861 5862 5863 5864 5865 5866
	if (mddev->level == 5 &&
	    mddev->layout == ALGORITHM_PARITY_N) {
		mddev->new_layout = 0;
		mddev->new_level = 4;
		return setup_conf(mddev);
	}
	return ERR_PTR(-EINVAL);
}
5867

5868 5869 5870 5871 5872 5873 5874 5875 5876 5877 5878 5879 5880 5881 5882 5883 5884 5885 5886 5887 5888 5889 5890 5891 5892 5893 5894 5895 5896 5897 5898 5899 5900 5901 5902 5903 5904 5905 5906 5907 5908 5909 5910 5911 5912 5913 5914 5915 5916
static struct mdk_personality raid5_personality;

static void *raid6_takeover(mddev_t *mddev)
{
	/* Currently can only take over a raid5.  We map the
	 * personality to an equivalent raid6 personality
	 * with the Q block at the end.
	 */
	int new_layout;

	if (mddev->pers != &raid5_personality)
		return ERR_PTR(-EINVAL);
	if (mddev->degraded > 1)
		return ERR_PTR(-EINVAL);
	if (mddev->raid_disks > 253)
		return ERR_PTR(-EINVAL);
	if (mddev->raid_disks < 3)
		return ERR_PTR(-EINVAL);

	switch (mddev->layout) {
	case ALGORITHM_LEFT_ASYMMETRIC:
		new_layout = ALGORITHM_LEFT_ASYMMETRIC_6;
		break;
	case ALGORITHM_RIGHT_ASYMMETRIC:
		new_layout = ALGORITHM_RIGHT_ASYMMETRIC_6;
		break;
	case ALGORITHM_LEFT_SYMMETRIC:
		new_layout = ALGORITHM_LEFT_SYMMETRIC_6;
		break;
	case ALGORITHM_RIGHT_SYMMETRIC:
		new_layout = ALGORITHM_RIGHT_SYMMETRIC_6;
		break;
	case ALGORITHM_PARITY_0:
		new_layout = ALGORITHM_PARITY_0_6;
		break;
	case ALGORITHM_PARITY_N:
		new_layout = ALGORITHM_PARITY_N;
		break;
	default:
		return ERR_PTR(-EINVAL);
	}
	mddev->new_level = 6;
	mddev->new_layout = new_layout;
	mddev->delta_disks = 1;
	mddev->raid_disks += 1;
	return setup_conf(mddev);
}


5917 5918 5919 5920 5921 5922 5923 5924 5925 5926 5927 5928 5929 5930 5931
static struct mdk_personality raid6_personality =
{
	.name		= "raid6",
	.level		= 6,
	.owner		= THIS_MODULE,
	.make_request	= make_request,
	.run		= run,
	.stop		= stop,
	.status		= status,
	.error_handler	= error,
	.hot_add_disk	= raid5_add_disk,
	.hot_remove_disk= raid5_remove_disk,
	.spare_active	= raid5_spare_active,
	.sync_request	= sync_request,
	.resize		= raid5_resize,
5932
	.size		= raid5_size,
5933
	.check_reshape	= raid6_check_reshape,
5934
	.start_reshape  = raid5_start_reshape,
5935
	.finish_reshape = raid5_finish_reshape,
5936
	.quiesce	= raid5_quiesce,
5937
	.takeover	= raid6_takeover,
5938
};
5939
static struct mdk_personality raid5_personality =
L
Linus Torvalds 已提交
5940 5941
{
	.name		= "raid5",
5942
	.level		= 5,
L
Linus Torvalds 已提交
5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953
	.owner		= THIS_MODULE,
	.make_request	= make_request,
	.run		= run,
	.stop		= stop,
	.status		= status,
	.error_handler	= error,
	.hot_add_disk	= raid5_add_disk,
	.hot_remove_disk= raid5_remove_disk,
	.spare_active	= raid5_spare_active,
	.sync_request	= sync_request,
	.resize		= raid5_resize,
5954
	.size		= raid5_size,
5955 5956
	.check_reshape	= raid5_check_reshape,
	.start_reshape  = raid5_start_reshape,
5957
	.finish_reshape = raid5_finish_reshape,
5958
	.quiesce	= raid5_quiesce,
5959
	.takeover	= raid5_takeover,
L
Linus Torvalds 已提交
5960 5961
};

5962
static struct mdk_personality raid4_personality =
L
Linus Torvalds 已提交
5963
{
5964 5965 5966 5967 5968 5969 5970 5971 5972 5973 5974 5975 5976
	.name		= "raid4",
	.level		= 4,
	.owner		= THIS_MODULE,
	.make_request	= make_request,
	.run		= run,
	.stop		= stop,
	.status		= status,
	.error_handler	= error,
	.hot_add_disk	= raid5_add_disk,
	.hot_remove_disk= raid5_remove_disk,
	.spare_active	= raid5_spare_active,
	.sync_request	= sync_request,
	.resize		= raid5_resize,
5977
	.size		= raid5_size,
5978 5979
	.check_reshape	= raid5_check_reshape,
	.start_reshape  = raid5_start_reshape,
5980
	.finish_reshape = raid5_finish_reshape,
5981
	.quiesce	= raid5_quiesce,
5982
	.takeover	= raid4_takeover,
5983 5984 5985 5986
};

static int __init raid5_init(void)
{
5987
	register_md_personality(&raid6_personality);
5988 5989 5990
	register_md_personality(&raid5_personality);
	register_md_personality(&raid4_personality);
	return 0;
L
Linus Torvalds 已提交
5991 5992
}

5993
static void raid5_exit(void)
L
Linus Torvalds 已提交
5994
{
5995
	unregister_md_personality(&raid6_personality);
5996 5997
	unregister_md_personality(&raid5_personality);
	unregister_md_personality(&raid4_personality);
L
Linus Torvalds 已提交
5998 5999 6000 6001 6002
}

module_init(raid5_init);
module_exit(raid5_exit);
MODULE_LICENSE("GPL");
6003
MODULE_DESCRIPTION("RAID4/5/6 (striping with parity) personality for MD");
L
Linus Torvalds 已提交
6004
MODULE_ALIAS("md-personality-4"); /* RAID5 */
6005 6006
MODULE_ALIAS("md-raid5");
MODULE_ALIAS("md-raid4");
6007 6008
MODULE_ALIAS("md-level-5");
MODULE_ALIAS("md-level-4");
6009 6010 6011 6012 6013 6014 6015
MODULE_ALIAS("md-personality-8"); /* RAID6 */
MODULE_ALIAS("md-raid6");
MODULE_ALIAS("md-level-6");

/* This used to be two separate modules, they were: */
MODULE_ALIAS("raid5");
MODULE_ALIAS("raid6");