raid5.c 170.9 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4
/*
 * raid5.c : Multiple Devices driver for Linux
 *	   Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman
 *	   Copyright (C) 1999, 2000 Ingo Molnar
5
 *	   Copyright (C) 2002, 2003 H. Peter Anvin
L
Linus Torvalds 已提交
6
 *
7 8 9
 * RAID-4/5/6 management functions.
 * Thanks to Penguin Computing for making the RAID-6 development possible
 * by donating a test server!
L
Linus Torvalds 已提交
10 11 12 13 14 15 16 17 18 19 20
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2, or (at your option)
 * any later version.
 *
 * You should have received a copy of the GNU General Public License
 * (for example /usr/src/linux/COPYING); if not, write to the Free
 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

21 22 23 24 25 26 27 28 29
/*
 * BITMAP UNPLUGGING:
 *
 * The sequencing for updating the bitmap reliably is a little
 * subtle (and I got it wrong the first time) so it deserves some
 * explanation.
 *
 * We group bitmap updates into batches.  Each batch has a number.
 * We may write out several batches at once, but that isn't very important.
30 31
 * conf->seq_write is the number of the last batch successfully written.
 * conf->seq_flush is the number of the last batch that was closed to
32 33 34
 *    new additions.
 * When we discover that we will need to write to any block in a stripe
 * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq
35
 * the number of the batch it will be in. This is seq_flush+1.
36 37 38 39 40 41 42 43 44
 * When we are ready to do a write, if that batch hasn't been written yet,
 *   we plug the array and queue the stripe for later.
 * When an unplug happens, we increment bm_flush, thus closing the current
 *   batch.
 * When we notice that bm_flush > bm_write, we write out all pending updates
 * to the bitmap, and advance bm_write to where bm_flush was.
 * This may occasionally write a bit out twice, but is sure never to
 * miss any bits.
 */
L
Linus Torvalds 已提交
45

46
#include <linux/blkdev.h>
47
#include <linux/kthread.h>
48
#include <linux/raid/pq.h>
49
#include <linux/async_tx.h>
50
#include <linux/module.h>
51
#include <linux/async.h>
52
#include <linux/seq_file.h>
53
#include <linux/cpu.h>
54
#include <linux/slab.h>
55
#include <linux/ratelimit.h>
56
#include "md.h"
57
#include "raid5.h"
58
#include "raid0.h"
59
#include "bitmap.h"
60

L
Linus Torvalds 已提交
61 62 63 64 65 66 67 68 69
/*
 * Stripe cache
 */

#define NR_STRIPES		256
#define STRIPE_SIZE		PAGE_SIZE
#define STRIPE_SHIFT		(PAGE_SHIFT - 9)
#define STRIPE_SECTORS		(STRIPE_SIZE>>9)
#define	IO_THRESHOLD		1
70
#define BYPASS_THRESHOLD	1
71
#define NR_HASH			(PAGE_SIZE / sizeof(struct hlist_head))
L
Linus Torvalds 已提交
72 73
#define HASH_MASK		(NR_HASH - 1)

74
static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect)
75 76 77 78
{
	int hash = (sect >> STRIPE_SHIFT) & HASH_MASK;
	return &conf->stripe_hashtbl[hash];
}
L
Linus Torvalds 已提交
79 80 81 82 83 84 85

/* bio's attached to a stripe+device for I/O are linked together in bi_sector
 * order without overlap.  There may be several bio's per stripe+device, and
 * a bio could span several devices.
 * When walking this list for a particular stripe+device, we must never proceed
 * beyond a bio that extends past this device, as the next bio might no longer
 * be valid.
86
 * This function is used to determine the 'next' bio in the list, given the sector
L
Linus Torvalds 已提交
87 88
 * of the current stripe+device
 */
89 90 91 92 93 94 95 96
static inline struct bio *r5_next_bio(struct bio *bio, sector_t sector)
{
	int sectors = bio->bi_size >> 9;
	if (bio->bi_sector + sectors < sector + STRIPE_SECTORS)
		return bio->bi_next;
	else
		return NULL;
}
L
Linus Torvalds 已提交
97

98
/*
99 100
 * We maintain a biased count of active stripes in the bottom 16 bits of
 * bi_phys_segments, and a count of processed stripes in the upper 16 bits
101 102 103
 */
static inline int raid5_bi_phys_segments(struct bio *bio)
{
104
	return bio->bi_phys_segments & 0xffff;
105 106 107 108
}

static inline int raid5_bi_hw_segments(struct bio *bio)
{
109
	return (bio->bi_phys_segments >> 16) & 0xffff;
110 111 112 113 114 115 116 117 118 119 120 121 122
}

static inline int raid5_dec_bi_phys_segments(struct bio *bio)
{
	--bio->bi_phys_segments;
	return raid5_bi_phys_segments(bio);
}

static inline int raid5_dec_bi_hw_segments(struct bio *bio)
{
	unsigned short val = raid5_bi_hw_segments(bio);

	--val;
123
	bio->bi_phys_segments = (val << 16) | raid5_bi_phys_segments(bio);
124 125 126 127 128
	return val;
}

static inline void raid5_set_bi_hw_segments(struct bio *bio, unsigned int cnt)
{
129
	bio->bi_phys_segments = raid5_bi_phys_segments(bio) | (cnt << 16);
130 131
}

132 133 134
/* Find first data disk in a raid6 stripe */
static inline int raid6_d0(struct stripe_head *sh)
{
135 136 137 138
	if (sh->ddf_layout)
		/* ddf always start from first device */
		return 0;
	/* md starts just after Q block */
139 140 141 142 143
	if (sh->qd_idx == sh->disks - 1)
		return 0;
	else
		return sh->qd_idx + 1;
}
144 145 146 147 148
static inline int raid6_next_disk(int disk, int raid_disks)
{
	disk++;
	return (disk < raid_disks) ? disk : 0;
}
149

150 151 152 153 154
/* When walking through the disks in a raid5, starting at raid6_d0,
 * We need to map each disk to a 'slot', where the data disks are slot
 * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk
 * is raid_disks-1.  This help does that mapping.
 */
155 156
static int raid6_idx_to_slot(int idx, struct stripe_head *sh,
			     int *count, int syndrome_disks)
157
{
158
	int slot = *count;
159

160
	if (sh->ddf_layout)
161
		(*count)++;
162
	if (idx == sh->pd_idx)
163
		return syndrome_disks;
164
	if (idx == sh->qd_idx)
165
		return syndrome_disks + 1;
166
	if (!sh->ddf_layout)
167
		(*count)++;
168 169 170
	return slot;
}

171 172 173 174 175 176 177 178
static void return_io(struct bio *return_bi)
{
	struct bio *bi = return_bi;
	while (bi) {

		return_bi = bi->bi_next;
		bi->bi_next = NULL;
		bi->bi_size = 0;
179
		bio_endio(bi, 0);
180 181 182 183
		bi = return_bi;
	}
}

184
static void print_raid5_conf (struct r5conf *conf);
L
Linus Torvalds 已提交
185

186 187 188 189 190 191 192
static int stripe_operations_active(struct stripe_head *sh)
{
	return sh->check_state || sh->reconstruct_state ||
	       test_bit(STRIPE_BIOFILL_RUN, &sh->state) ||
	       test_bit(STRIPE_COMPUTE_RUN, &sh->state);
}

193
static void __release_stripe(struct r5conf *conf, struct stripe_head *sh)
L
Linus Torvalds 已提交
194 195
{
	if (atomic_dec_and_test(&sh->count)) {
196 197
		BUG_ON(!list_empty(&sh->lru));
		BUG_ON(atomic_read(&conf->active_stripes)==0);
L
Linus Torvalds 已提交
198
		if (test_bit(STRIPE_HANDLE, &sh->state)) {
N
NeilBrown 已提交
199
			if (test_bit(STRIPE_DELAYED, &sh->state))
L
Linus Torvalds 已提交
200
				list_add_tail(&sh->lru, &conf->delayed_list);
N
NeilBrown 已提交
201 202
			else if (test_bit(STRIPE_BIT_DELAY, &sh->state) &&
				   sh->bm_seq - conf->seq_write > 0)
203
				list_add_tail(&sh->lru, &conf->bitmap_list);
N
NeilBrown 已提交
204
			else {
205
				clear_bit(STRIPE_BIT_DELAY, &sh->state);
L
Linus Torvalds 已提交
206
				list_add_tail(&sh->lru, &conf->handle_list);
207
			}
L
Linus Torvalds 已提交
208 209
			md_wakeup_thread(conf->mddev->thread);
		} else {
210
			BUG_ON(stripe_operations_active(sh));
211 212 213
			if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
				if (atomic_dec_return(&conf->preread_active_stripes)
				    < IO_THRESHOLD)
L
Linus Torvalds 已提交
214 215
					md_wakeup_thread(conf->mddev->thread);
			atomic_dec(&conf->active_stripes);
216 217
			if (!test_bit(STRIPE_EXPANDING, &sh->state)) {
				list_add_tail(&sh->lru, &conf->inactive_list);
L
Linus Torvalds 已提交
218
				wake_up(&conf->wait_for_stripe);
219 220
				if (conf->retry_read_aligned)
					md_wakeup_thread(conf->mddev->thread);
221
			}
L
Linus Torvalds 已提交
222 223 224
		}
	}
}
225

L
Linus Torvalds 已提交
226 227
static void release_stripe(struct stripe_head *sh)
{
228
	struct r5conf *conf = sh->raid_conf;
L
Linus Torvalds 已提交
229
	unsigned long flags;
230

L
Linus Torvalds 已提交
231 232 233 234 235
	spin_lock_irqsave(&conf->device_lock, flags);
	__release_stripe(conf, sh);
	spin_unlock_irqrestore(&conf->device_lock, flags);
}

236
static inline void remove_hash(struct stripe_head *sh)
L
Linus Torvalds 已提交
237
{
238 239
	pr_debug("remove_hash(), stripe %llu\n",
		(unsigned long long)sh->sector);
L
Linus Torvalds 已提交
240

241
	hlist_del_init(&sh->hash);
L
Linus Torvalds 已提交
242 243
}

244
static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh)
L
Linus Torvalds 已提交
245
{
246
	struct hlist_head *hp = stripe_hash(conf, sh->sector);
L
Linus Torvalds 已提交
247

248 249
	pr_debug("insert_hash(), stripe %llu\n",
		(unsigned long long)sh->sector);
L
Linus Torvalds 已提交
250

251
	hlist_add_head(&sh->hash, hp);
L
Linus Torvalds 已提交
252 253 254 255
}


/* find an idle stripe, make sure it is unhashed, and return it. */
256
static struct stripe_head *get_free_stripe(struct r5conf *conf)
L
Linus Torvalds 已提交
257 258 259 260 261 262 263 264 265 266 267 268 269 270 271
{
	struct stripe_head *sh = NULL;
	struct list_head *first;

	if (list_empty(&conf->inactive_list))
		goto out;
	first = conf->inactive_list.next;
	sh = list_entry(first, struct stripe_head, lru);
	list_del_init(first);
	remove_hash(sh);
	atomic_inc(&conf->active_stripes);
out:
	return sh;
}

272
static void shrink_buffers(struct stripe_head *sh)
L
Linus Torvalds 已提交
273 274 275
{
	struct page *p;
	int i;
276
	int num = sh->raid_conf->pool_size;
L
Linus Torvalds 已提交
277

278
	for (i = 0; i < num ; i++) {
L
Linus Torvalds 已提交
279 280 281 282
		p = sh->dev[i].page;
		if (!p)
			continue;
		sh->dev[i].page = NULL;
283
		put_page(p);
L
Linus Torvalds 已提交
284 285 286
	}
}

287
static int grow_buffers(struct stripe_head *sh)
L
Linus Torvalds 已提交
288 289
{
	int i;
290
	int num = sh->raid_conf->pool_size;
L
Linus Torvalds 已提交
291

292
	for (i = 0; i < num; i++) {
L
Linus Torvalds 已提交
293 294 295 296 297 298 299 300 301 302
		struct page *page;

		if (!(page = alloc_page(GFP_KERNEL))) {
			return 1;
		}
		sh->dev[i].page = page;
	}
	return 0;
}

303
static void raid5_build_block(struct stripe_head *sh, int i, int previous);
304
static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,
305
			    struct stripe_head *sh);
L
Linus Torvalds 已提交
306

307
static void init_stripe(struct stripe_head *sh, sector_t sector, int previous)
L
Linus Torvalds 已提交
308
{
309
	struct r5conf *conf = sh->raid_conf;
310
	int i;
L
Linus Torvalds 已提交
311

312 313
	BUG_ON(atomic_read(&sh->count) != 0);
	BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));
314
	BUG_ON(stripe_operations_active(sh));
315

316
	pr_debug("init_stripe called, stripe %llu\n",
L
Linus Torvalds 已提交
317 318 319
		(unsigned long long)sh->sector);

	remove_hash(sh);
320

321
	sh->generation = conf->generation - previous;
322
	sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks;
L
Linus Torvalds 已提交
323
	sh->sector = sector;
324
	stripe_set_idx(sector, conf, previous, sh);
L
Linus Torvalds 已提交
325 326
	sh->state = 0;

327 328

	for (i = sh->disks; i--; ) {
L
Linus Torvalds 已提交
329 330
		struct r5dev *dev = &sh->dev[i];

331
		if (dev->toread || dev->read || dev->towrite || dev->written ||
L
Linus Torvalds 已提交
332
		    test_bit(R5_LOCKED, &dev->flags)) {
333
			printk(KERN_ERR "sector=%llx i=%d %p %p %p %p %d\n",
L
Linus Torvalds 已提交
334
			       (unsigned long long)sh->sector, i, dev->toread,
335
			       dev->read, dev->towrite, dev->written,
L
Linus Torvalds 已提交
336
			       test_bit(R5_LOCKED, &dev->flags));
337
			WARN_ON(1);
L
Linus Torvalds 已提交
338 339
		}
		dev->flags = 0;
340
		raid5_build_block(sh, i, previous);
L
Linus Torvalds 已提交
341 342 343 344
	}
	insert_hash(conf, sh);
}

345
static struct stripe_head *__find_stripe(struct r5conf *conf, sector_t sector,
346
					 short generation)
L
Linus Torvalds 已提交
347 348
{
	struct stripe_head *sh;
349
	struct hlist_node *hn;
L
Linus Torvalds 已提交
350

351
	pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector);
352
	hlist_for_each_entry(sh, hn, stripe_hash(conf, sector), hash)
353
		if (sh->sector == sector && sh->generation == generation)
L
Linus Torvalds 已提交
354
			return sh;
355
	pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector);
L
Linus Torvalds 已提交
356 357 358
	return NULL;
}

359 360 361 362 363 364 365 366 367 368 369 370 371
/*
 * Need to check if array has failed when deciding whether to:
 *  - start an array
 *  - remove non-faulty devices
 *  - add a spare
 *  - allow a reshape
 * This determination is simple when no reshape is happening.
 * However if there is a reshape, we need to carefully check
 * both the before and after sections.
 * This is because some failed devices may only affect one
 * of the two sections, and some non-in_sync devices may
 * be insync in the section most affected by failed devices.
 */
372
static int calc_degraded(struct r5conf *conf)
373
{
374
	int degraded, degraded2;
375 376 377 378 379
	int i;

	rcu_read_lock();
	degraded = 0;
	for (i = 0; i < conf->previous_raid_disks; i++) {
380
		struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398
		if (!rdev || test_bit(Faulty, &rdev->flags))
			degraded++;
		else if (test_bit(In_sync, &rdev->flags))
			;
		else
			/* not in-sync or faulty.
			 * If the reshape increases the number of devices,
			 * this is being recovered by the reshape, so
			 * this 'previous' section is not in_sync.
			 * If the number of devices is being reduced however,
			 * the device can only be part of the array if
			 * we are reverting a reshape, so this section will
			 * be in-sync.
			 */
			if (conf->raid_disks >= conf->previous_raid_disks)
				degraded++;
	}
	rcu_read_unlock();
399 400
	if (conf->raid_disks == conf->previous_raid_disks)
		return degraded;
401
	rcu_read_lock();
402
	degraded2 = 0;
403
	for (i = 0; i < conf->raid_disks; i++) {
404
		struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
405
		if (!rdev || test_bit(Faulty, &rdev->flags))
406
			degraded2++;
407 408 409 410 411 412 413 414 415
		else if (test_bit(In_sync, &rdev->flags))
			;
		else
			/* not in-sync or faulty.
			 * If reshape increases the number of devices, this
			 * section has already been recovered, else it
			 * almost certainly hasn't.
			 */
			if (conf->raid_disks <= conf->previous_raid_disks)
416
				degraded2++;
417 418
	}
	rcu_read_unlock();
419 420 421 422 423 424 425 426 427 428 429 430 431
	if (degraded2 > degraded)
		return degraded2;
	return degraded;
}

static int has_failed(struct r5conf *conf)
{
	int degraded;

	if (conf->mddev->reshape_position == MaxSector)
		return conf->mddev->degraded > conf->max_degraded;

	degraded = calc_degraded(conf);
432 433 434 435 436
	if (degraded > conf->max_degraded)
		return 1;
	return 0;
}

437
static struct stripe_head *
438
get_active_stripe(struct r5conf *conf, sector_t sector,
439
		  int previous, int noblock, int noquiesce)
L
Linus Torvalds 已提交
440 441 442
{
	struct stripe_head *sh;

443
	pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector);
L
Linus Torvalds 已提交
444 445 446 447

	spin_lock_irq(&conf->device_lock);

	do {
448
		wait_event_lock_irq(conf->wait_for_stripe,
449
				    conf->quiesce == 0 || noquiesce,
450
				    conf->device_lock, /* nothing */);
451
		sh = __find_stripe(conf, sector, conf->generation - previous);
L
Linus Torvalds 已提交
452 453 454 455 456 457 458 459 460
		if (!sh) {
			if (!conf->inactive_blocked)
				sh = get_free_stripe(conf);
			if (noblock && sh == NULL)
				break;
			if (!sh) {
				conf->inactive_blocked = 1;
				wait_event_lock_irq(conf->wait_for_stripe,
						    !list_empty(&conf->inactive_list) &&
461 462
						    (atomic_read(&conf->active_stripes)
						     < (conf->max_nr_stripes *3/4)
L
Linus Torvalds 已提交
463 464
						     || !conf->inactive_blocked),
						    conf->device_lock,
465
						    );
L
Linus Torvalds 已提交
466 467
				conf->inactive_blocked = 0;
			} else
468
				init_stripe(sh, sector, previous);
L
Linus Torvalds 已提交
469 470
		} else {
			if (atomic_read(&sh->count)) {
471 472
				BUG_ON(!list_empty(&sh->lru)
				    && !test_bit(STRIPE_EXPANDING, &sh->state));
L
Linus Torvalds 已提交
473 474 475
			} else {
				if (!test_bit(STRIPE_HANDLE, &sh->state))
					atomic_inc(&conf->active_stripes);
476 477
				if (list_empty(&sh->lru) &&
				    !test_bit(STRIPE_EXPANDING, &sh->state))
478 479
					BUG();
				list_del_init(&sh->lru);
L
Linus Torvalds 已提交
480 481 482 483 484 485 486 487 488 489 490
			}
		}
	} while (sh == NULL);

	if (sh)
		atomic_inc(&sh->count);

	spin_unlock_irq(&conf->device_lock);
	return sh;
}

491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511
/* Determine if 'data_offset' or 'new_data_offset' should be used
 * in this stripe_head.
 */
static int use_new_offset(struct r5conf *conf, struct stripe_head *sh)
{
	sector_t progress = conf->reshape_progress;
	/* Need a memory barrier to make sure we see the value
	 * of conf->generation, or ->data_offset that was set before
	 * reshape_progress was updated.
	 */
	smp_rmb();
	if (progress == MaxSector)
		return 0;
	if (sh->generation == conf->generation - 1)
		return 0;
	/* We are in a reshape, and this is a new-generation stripe,
	 * so use new_data_offset.
	 */
	return 1;
}

512 513 514 515
static void
raid5_end_read_request(struct bio *bi, int error);
static void
raid5_end_write_request(struct bio *bi, int error);
516

517
static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)
518
{
519
	struct r5conf *conf = sh->raid_conf;
520 521 522 523 524 525
	int i, disks = sh->disks;

	might_sleep();

	for (i = disks; i--; ) {
		int rw;
526
		int replace_only = 0;
527 528
		struct bio *bi, *rbi;
		struct md_rdev *rdev, *rrdev = NULL;
T
Tejun Heo 已提交
529 530 531 532 533 534
		if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) {
			if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags))
				rw = WRITE_FUA;
			else
				rw = WRITE;
		} else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags))
535
			rw = READ;
536 537 538 539 540
		else if (test_and_clear_bit(R5_WantReplace,
					    &sh->dev[i].flags)) {
			rw = WRITE;
			replace_only = 1;
		} else
541
			continue;
S
Shaohua Li 已提交
542 543
		if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags))
			rw |= REQ_SYNC;
544 545

		bi = &sh->dev[i].req;
546
		rbi = &sh->dev[i].rreq; /* For writing to replacement */
547 548

		bi->bi_rw = rw;
549 550
		rbi->bi_rw = rw;
		if (rw & WRITE) {
551
			bi->bi_end_io = raid5_end_write_request;
552 553
			rbi->bi_end_io = raid5_end_write_request;
		} else
554 555 556
			bi->bi_end_io = raid5_end_read_request;

		rcu_read_lock();
557
		rrdev = rcu_dereference(conf->disks[i].replacement);
558 559 560 561 562 563
		smp_mb(); /* Ensure that if rrdev is NULL, rdev won't be */
		rdev = rcu_dereference(conf->disks[i].rdev);
		if (!rdev) {
			rdev = rrdev;
			rrdev = NULL;
		}
564 565 566
		if (rw & WRITE) {
			if (replace_only)
				rdev = NULL;
567 568 569
			if (rdev == rrdev)
				/* We raced and saw duplicates */
				rrdev = NULL;
570
		} else {
571
			if (test_bit(R5_ReadRepl, &sh->dev[i].flags) && rrdev)
572 573 574
				rdev = rrdev;
			rrdev = NULL;
		}
575

576 577 578 579
		if (rdev && test_bit(Faulty, &rdev->flags))
			rdev = NULL;
		if (rdev)
			atomic_inc(&rdev->nr_pending);
580 581 582 583
		if (rrdev && test_bit(Faulty, &rrdev->flags))
			rrdev = NULL;
		if (rrdev)
			atomic_inc(&rrdev->nr_pending);
584 585
		rcu_read_unlock();

586
		/* We have already checked bad blocks for reads.  Now
587 588
		 * need to check for writes.  We never accept write errors
		 * on the replacement, so we don't to check rrdev.
589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616
		 */
		while ((rw & WRITE) && rdev &&
		       test_bit(WriteErrorSeen, &rdev->flags)) {
			sector_t first_bad;
			int bad_sectors;
			int bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS,
					      &first_bad, &bad_sectors);
			if (!bad)
				break;

			if (bad < 0) {
				set_bit(BlockedBadBlocks, &rdev->flags);
				if (!conf->mddev->external &&
				    conf->mddev->flags) {
					/* It is very unlikely, but we might
					 * still need to write out the
					 * bad block log - better give it
					 * a chance*/
					md_check_recovery(conf->mddev);
				}
				md_wait_for_blocked_rdev(rdev, conf->mddev);
			} else {
				/* Acknowledged bad block - skip the write */
				rdev_dec_pending(rdev, conf->mddev);
				rdev = NULL;
			}
		}

617
		if (rdev) {
618 619
			if (s->syncing || s->expanding || s->expanded
			    || s->replacing)
620 621
				md_sync_acct(rdev->bdev, STRIPE_SECTORS);

D
Dan Williams 已提交
622 623
			set_bit(STRIPE_IO_STARTED, &sh->state);

624 625
			bi->bi_bdev = rdev->bdev;
			pr_debug("%s: for %llu schedule op %ld on disc %d\n",
626
				__func__, (unsigned long long)sh->sector,
627 628
				bi->bi_rw, i);
			atomic_inc(&sh->count);
629 630 631 632 633 634
			if (use_new_offset(conf, sh))
				bi->bi_sector = (sh->sector
						 + rdev->new_data_offset);
			else
				bi->bi_sector = (sh->sector
						 + rdev->data_offset);
635 636 637 638 639 640
			bi->bi_flags = 1 << BIO_UPTODATE;
			bi->bi_idx = 0;
			bi->bi_io_vec[0].bv_len = STRIPE_SIZE;
			bi->bi_io_vec[0].bv_offset = 0;
			bi->bi_size = STRIPE_SIZE;
			bi->bi_next = NULL;
641 642
			if (rrdev)
				set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags);
643
			generic_make_request(bi);
644 645
		}
		if (rrdev) {
646 647
			if (s->syncing || s->expanding || s->expanded
			    || s->replacing)
648 649 650 651 652 653 654 655 656 657
				md_sync_acct(rrdev->bdev, STRIPE_SECTORS);

			set_bit(STRIPE_IO_STARTED, &sh->state);

			rbi->bi_bdev = rrdev->bdev;
			pr_debug("%s: for %llu schedule op %ld on "
				 "replacement disc %d\n",
				__func__, (unsigned long long)sh->sector,
				rbi->bi_rw, i);
			atomic_inc(&sh->count);
658 659 660 661 662 663
			if (use_new_offset(conf, sh))
				rbi->bi_sector = (sh->sector
						  + rrdev->new_data_offset);
			else
				rbi->bi_sector = (sh->sector
						  + rrdev->data_offset);
664 665 666 667 668 669 670 671 672
			rbi->bi_flags = 1 << BIO_UPTODATE;
			rbi->bi_idx = 0;
			rbi->bi_io_vec[0].bv_len = STRIPE_SIZE;
			rbi->bi_io_vec[0].bv_offset = 0;
			rbi->bi_size = STRIPE_SIZE;
			rbi->bi_next = NULL;
			generic_make_request(rbi);
		}
		if (!rdev && !rrdev) {
673
			if (rw & WRITE)
674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690
				set_bit(STRIPE_DEGRADED, &sh->state);
			pr_debug("skip op %ld on disc %d for sector %llu\n",
				bi->bi_rw, i, (unsigned long long)sh->sector);
			clear_bit(R5_LOCKED, &sh->dev[i].flags);
			set_bit(STRIPE_HANDLE, &sh->state);
		}
	}
}

static struct dma_async_tx_descriptor *
async_copy_data(int frombio, struct bio *bio, struct page *page,
	sector_t sector, struct dma_async_tx_descriptor *tx)
{
	struct bio_vec *bvl;
	struct page *bio_page;
	int i;
	int page_offset;
691
	struct async_submit_ctl submit;
D
Dan Williams 已提交
692
	enum async_tx_flags flags = 0;
693 694 695 696 697

	if (bio->bi_sector >= sector)
		page_offset = (signed)(bio->bi_sector - sector) * 512;
	else
		page_offset = (signed)(sector - bio->bi_sector) * -512;
698

D
Dan Williams 已提交
699 700 701 702
	if (frombio)
		flags |= ASYNC_TX_FENCE;
	init_async_submit(&submit, flags, tx, NULL, NULL, NULL);

703
	bio_for_each_segment(bvl, bio, i) {
704
		int len = bvl->bv_len;
705 706 707 708 709 710 711 712 713 714 715 716 717 718 719
		int clen;
		int b_offset = 0;

		if (page_offset < 0) {
			b_offset = -page_offset;
			page_offset += b_offset;
			len -= b_offset;
		}

		if (len > 0 && page_offset + len > STRIPE_SIZE)
			clen = STRIPE_SIZE - page_offset;
		else
			clen = len;

		if (clen > 0) {
720 721
			b_offset += bvl->bv_offset;
			bio_page = bvl->bv_page;
722 723
			if (frombio)
				tx = async_memcpy(page, bio_page, page_offset,
724
						  b_offset, clen, &submit);
725 726
			else
				tx = async_memcpy(bio_page, page, b_offset,
727
						  page_offset, clen, &submit);
728
		}
729 730 731
		/* chain the operations */
		submit.depend_tx = tx;

732 733 734 735 736 737 738 739 740 741 742 743
		if (clen < len) /* hit end of page */
			break;
		page_offset +=  len;
	}

	return tx;
}

static void ops_complete_biofill(void *stripe_head_ref)
{
	struct stripe_head *sh = stripe_head_ref;
	struct bio *return_bi = NULL;
744
	struct r5conf *conf = sh->raid_conf;
745
	int i;
746

747
	pr_debug("%s: stripe %llu\n", __func__,
748 749 750
		(unsigned long long)sh->sector);

	/* clear completed biofills */
751
	spin_lock_irq(&conf->device_lock);
752 753 754 755
	for (i = sh->disks; i--; ) {
		struct r5dev *dev = &sh->dev[i];

		/* acknowledge completion of a biofill operation */
756 757
		/* and check if we need to reply to a read request,
		 * new R5_Wantfill requests are held off until
758
		 * !STRIPE_BIOFILL_RUN
759 760
		 */
		if (test_and_clear_bit(R5_Wantfill, &dev->flags)) {
761 762 763 764 765 766 767 768
			struct bio *rbi, *rbi2;

			BUG_ON(!dev->read);
			rbi = dev->read;
			dev->read = NULL;
			while (rbi && rbi->bi_sector <
				dev->sector + STRIPE_SECTORS) {
				rbi2 = r5_next_bio(rbi, dev->sector);
769
				if (!raid5_dec_bi_phys_segments(rbi)) {
770 771 772 773 774 775 776
					rbi->bi_next = return_bi;
					return_bi = rbi;
				}
				rbi = rbi2;
			}
		}
	}
777 778
	spin_unlock_irq(&conf->device_lock);
	clear_bit(STRIPE_BIOFILL_RUN, &sh->state);
779 780 781

	return_io(return_bi);

782
	set_bit(STRIPE_HANDLE, &sh->state);
783 784 785 786 787 788
	release_stripe(sh);
}

static void ops_run_biofill(struct stripe_head *sh)
{
	struct dma_async_tx_descriptor *tx = NULL;
789
	struct r5conf *conf = sh->raid_conf;
790
	struct async_submit_ctl submit;
791 792
	int i;

793
	pr_debug("%s: stripe %llu\n", __func__,
794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813
		(unsigned long long)sh->sector);

	for (i = sh->disks; i--; ) {
		struct r5dev *dev = &sh->dev[i];
		if (test_bit(R5_Wantfill, &dev->flags)) {
			struct bio *rbi;
			spin_lock_irq(&conf->device_lock);
			dev->read = rbi = dev->toread;
			dev->toread = NULL;
			spin_unlock_irq(&conf->device_lock);
			while (rbi && rbi->bi_sector <
				dev->sector + STRIPE_SECTORS) {
				tx = async_copy_data(0, rbi, dev->page,
					dev->sector, tx);
				rbi = r5_next_bio(rbi, dev->sector);
			}
		}
	}

	atomic_inc(&sh->count);
814 815
	init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL);
	async_trigger_callback(&submit);
816 817
}

818
static void mark_target_uptodate(struct stripe_head *sh, int target)
819
{
820
	struct r5dev *tgt;
821

822 823
	if (target < 0)
		return;
824

825
	tgt = &sh->dev[target];
826 827 828
	set_bit(R5_UPTODATE, &tgt->flags);
	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
	clear_bit(R5_Wantcompute, &tgt->flags);
829 830
}

831
static void ops_complete_compute(void *stripe_head_ref)
832 833 834
{
	struct stripe_head *sh = stripe_head_ref;

835
	pr_debug("%s: stripe %llu\n", __func__,
836 837
		(unsigned long long)sh->sector);

838
	/* mark the computed target(s) as uptodate */
839
	mark_target_uptodate(sh, sh->ops.target);
840
	mark_target_uptodate(sh, sh->ops.target2);
841

842 843 844
	clear_bit(STRIPE_COMPUTE_RUN, &sh->state);
	if (sh->check_state == check_state_compute_run)
		sh->check_state = check_state_compute_result;
845 846 847 848
	set_bit(STRIPE_HANDLE, &sh->state);
	release_stripe(sh);
}

849 850 851 852 853 854 855 856 857
/* return a pointer to the address conversion region of the scribble buffer */
static addr_conv_t *to_addr_conv(struct stripe_head *sh,
				 struct raid5_percpu *percpu)
{
	return percpu->scribble + sizeof(struct page *) * (sh->disks + 2);
}

static struct dma_async_tx_descriptor *
ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)
858 859
{
	int disks = sh->disks;
860
	struct page **xor_srcs = percpu->scribble;
861 862 863 864 865
	int target = sh->ops.target;
	struct r5dev *tgt = &sh->dev[target];
	struct page *xor_dest = tgt->page;
	int count = 0;
	struct dma_async_tx_descriptor *tx;
866
	struct async_submit_ctl submit;
867 868 869
	int i;

	pr_debug("%s: stripe %llu block: %d\n",
870
		__func__, (unsigned long long)sh->sector, target);
871 872 873 874 875 876 877 878
	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));

	for (i = disks; i--; )
		if (i != target)
			xor_srcs[count++] = sh->dev[i].page;

	atomic_inc(&sh->count);

D
Dan Williams 已提交
879
	init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL,
880
			  ops_complete_compute, sh, to_addr_conv(sh, percpu));
881
	if (unlikely(count == 1))
882
		tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
883
	else
884
		tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
885 886 887 888

	return tx;
}

889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906
/* set_syndrome_sources - populate source buffers for gen_syndrome
 * @srcs - (struct page *) array of size sh->disks
 * @sh - stripe_head to parse
 *
 * Populates srcs in proper layout order for the stripe and returns the
 * 'count' of sources to be used in a call to async_gen_syndrome.  The P
 * destination buffer is recorded in srcs[count] and the Q destination
 * is recorded in srcs[count+1]].
 */
static int set_syndrome_sources(struct page **srcs, struct stripe_head *sh)
{
	int disks = sh->disks;
	int syndrome_disks = sh->ddf_layout ? disks : (disks - 2);
	int d0_idx = raid6_d0(sh);
	int count;
	int i;

	for (i = 0; i < disks; i++)
907
		srcs[i] = NULL;
908 909 910 911 912 913 914 915 916 917

	count = 0;
	i = d0_idx;
	do {
		int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);

		srcs[slot] = sh->dev[i].page;
		i = raid6_next_disk(i, disks);
	} while (i != d0_idx);

918
	return syndrome_disks;
919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938
}

static struct dma_async_tx_descriptor *
ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)
{
	int disks = sh->disks;
	struct page **blocks = percpu->scribble;
	int target;
	int qd_idx = sh->qd_idx;
	struct dma_async_tx_descriptor *tx;
	struct async_submit_ctl submit;
	struct r5dev *tgt;
	struct page *dest;
	int i;
	int count;

	if (sh->ops.target < 0)
		target = sh->ops.target2;
	else if (sh->ops.target2 < 0)
		target = sh->ops.target;
939
	else
940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955
		/* we should only have one valid target */
		BUG();
	BUG_ON(target < 0);
	pr_debug("%s: stripe %llu block: %d\n",
		__func__, (unsigned long long)sh->sector, target);

	tgt = &sh->dev[target];
	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
	dest = tgt->page;

	atomic_inc(&sh->count);

	if (target == qd_idx) {
		count = set_syndrome_sources(blocks, sh);
		blocks[count] = NULL; /* regenerating p is not necessary */
		BUG_ON(blocks[count+1] != dest); /* q should already be set */
D
Dan Williams 已提交
956 957
		init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
				  ops_complete_compute, sh,
958 959 960 961 962 963 964 965 966 967 968
				  to_addr_conv(sh, percpu));
		tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
	} else {
		/* Compute any data- or p-drive using XOR */
		count = 0;
		for (i = disks; i-- ; ) {
			if (i == target || i == qd_idx)
				continue;
			blocks[count++] = sh->dev[i].page;
		}

D
Dan Williams 已提交
969 970
		init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
				  NULL, ops_complete_compute, sh,
971 972 973
				  to_addr_conv(sh, percpu));
		tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit);
	}
974 975 976 977

	return tx;
}

978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998
static struct dma_async_tx_descriptor *
ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
{
	int i, count, disks = sh->disks;
	int syndrome_disks = sh->ddf_layout ? disks : disks-2;
	int d0_idx = raid6_d0(sh);
	int faila = -1, failb = -1;
	int target = sh->ops.target;
	int target2 = sh->ops.target2;
	struct r5dev *tgt = &sh->dev[target];
	struct r5dev *tgt2 = &sh->dev[target2];
	struct dma_async_tx_descriptor *tx;
	struct page **blocks = percpu->scribble;
	struct async_submit_ctl submit;

	pr_debug("%s: stripe %llu block1: %d block2: %d\n",
		 __func__, (unsigned long long)sh->sector, target, target2);
	BUG_ON(target < 0 || target2 < 0);
	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
	BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags));

999
	/* we need to open-code set_syndrome_sources to handle the
1000 1001 1002
	 * slot number conversion for 'faila' and 'failb'
	 */
	for (i = 0; i < disks ; i++)
1003
		blocks[i] = NULL;
1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029
	count = 0;
	i = d0_idx;
	do {
		int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);

		blocks[slot] = sh->dev[i].page;

		if (i == target)
			faila = slot;
		if (i == target2)
			failb = slot;
		i = raid6_next_disk(i, disks);
	} while (i != d0_idx);

	BUG_ON(faila == failb);
	if (failb < faila)
		swap(faila, failb);
	pr_debug("%s: stripe: %llu faila: %d failb: %d\n",
		 __func__, (unsigned long long)sh->sector, faila, failb);

	atomic_inc(&sh->count);

	if (failb == syndrome_disks+1) {
		/* Q disk is one of the missing disks */
		if (faila == syndrome_disks) {
			/* Missing P+Q, just recompute */
D
Dan Williams 已提交
1030 1031 1032
			init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
					  ops_complete_compute, sh,
					  to_addr_conv(sh, percpu));
1033
			return async_gen_syndrome(blocks, 0, syndrome_disks+2,
1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052
						  STRIPE_SIZE, &submit);
		} else {
			struct page *dest;
			int data_target;
			int qd_idx = sh->qd_idx;

			/* Missing D+Q: recompute D from P, then recompute Q */
			if (target == qd_idx)
				data_target = target2;
			else
				data_target = target;

			count = 0;
			for (i = disks; i-- ; ) {
				if (i == data_target || i == qd_idx)
					continue;
				blocks[count++] = sh->dev[i].page;
			}
			dest = sh->dev[data_target].page;
D
Dan Williams 已提交
1053 1054 1055 1056
			init_async_submit(&submit,
					  ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
					  NULL, NULL, NULL,
					  to_addr_conv(sh, percpu));
1057 1058 1059 1060
			tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE,
				       &submit);

			count = set_syndrome_sources(blocks, sh);
D
Dan Williams 已提交
1061 1062 1063
			init_async_submit(&submit, ASYNC_TX_FENCE, tx,
					  ops_complete_compute, sh,
					  to_addr_conv(sh, percpu));
1064 1065 1066 1067
			return async_gen_syndrome(blocks, 0, count+2,
						  STRIPE_SIZE, &submit);
		}
	} else {
1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081
		init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
				  ops_complete_compute, sh,
				  to_addr_conv(sh, percpu));
		if (failb == syndrome_disks) {
			/* We're missing D+P. */
			return async_raid6_datap_recov(syndrome_disks+2,
						       STRIPE_SIZE, faila,
						       blocks, &submit);
		} else {
			/* We're missing D+D. */
			return async_raid6_2data_recov(syndrome_disks+2,
						       STRIPE_SIZE, faila, failb,
						       blocks, &submit);
		}
1082 1083 1084 1085
	}
}


1086 1087 1088 1089
static void ops_complete_prexor(void *stripe_head_ref)
{
	struct stripe_head *sh = stripe_head_ref;

1090
	pr_debug("%s: stripe %llu\n", __func__,
1091 1092 1093 1094
		(unsigned long long)sh->sector);
}

static struct dma_async_tx_descriptor *
1095 1096
ops_run_prexor(struct stripe_head *sh, struct raid5_percpu *percpu,
	       struct dma_async_tx_descriptor *tx)
1097 1098
{
	int disks = sh->disks;
1099
	struct page **xor_srcs = percpu->scribble;
1100
	int count = 0, pd_idx = sh->pd_idx, i;
1101
	struct async_submit_ctl submit;
1102 1103 1104 1105

	/* existing parity data subtracted */
	struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;

1106
	pr_debug("%s: stripe %llu\n", __func__,
1107 1108 1109 1110 1111
		(unsigned long long)sh->sector);

	for (i = disks; i--; ) {
		struct r5dev *dev = &sh->dev[i];
		/* Only process blocks that are known to be uptodate */
1112
		if (test_bit(R5_Wantdrain, &dev->flags))
1113 1114 1115
			xor_srcs[count++] = dev->page;
	}

D
Dan Williams 已提交
1116
	init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
1117
			  ops_complete_prexor, sh, to_addr_conv(sh, percpu));
1118
	tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
1119 1120 1121 1122 1123

	return tx;
}

static struct dma_async_tx_descriptor *
1124
ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
1125 1126
{
	int disks = sh->disks;
1127
	int i;
1128

1129
	pr_debug("%s: stripe %llu\n", __func__,
1130 1131 1132 1133 1134 1135
		(unsigned long long)sh->sector);

	for (i = disks; i--; ) {
		struct r5dev *dev = &sh->dev[i];
		struct bio *chosen;

1136
		if (test_and_clear_bit(R5_Wantdrain, &dev->flags)) {
1137 1138
			struct bio *wbi;

1139
			spin_lock_irq(&sh->raid_conf->device_lock);
1140 1141 1142 1143
			chosen = dev->towrite;
			dev->towrite = NULL;
			BUG_ON(dev->written);
			wbi = dev->written = chosen;
1144
			spin_unlock_irq(&sh->raid_conf->device_lock);
1145 1146 1147

			while (wbi && wbi->bi_sector <
				dev->sector + STRIPE_SECTORS) {
T
Tejun Heo 已提交
1148 1149
				if (wbi->bi_rw & REQ_FUA)
					set_bit(R5_WantFUA, &dev->flags);
S
Shaohua Li 已提交
1150 1151
				if (wbi->bi_rw & REQ_SYNC)
					set_bit(R5_SyncIO, &dev->flags);
1152 1153 1154 1155 1156 1157 1158 1159 1160 1161
				tx = async_copy_data(1, wbi, dev->page,
					dev->sector, tx);
				wbi = r5_next_bio(wbi, dev->sector);
			}
		}
	}

	return tx;
}

1162
static void ops_complete_reconstruct(void *stripe_head_ref)
1163 1164
{
	struct stripe_head *sh = stripe_head_ref;
1165 1166 1167 1168
	int disks = sh->disks;
	int pd_idx = sh->pd_idx;
	int qd_idx = sh->qd_idx;
	int i;
S
Shaohua Li 已提交
1169
	bool fua = false, sync = false;
1170

1171
	pr_debug("%s: stripe %llu\n", __func__,
1172 1173
		(unsigned long long)sh->sector);

S
Shaohua Li 已提交
1174
	for (i = disks; i--; ) {
T
Tejun Heo 已提交
1175
		fua |= test_bit(R5_WantFUA, &sh->dev[i].flags);
S
Shaohua Li 已提交
1176 1177
		sync |= test_bit(R5_SyncIO, &sh->dev[i].flags);
	}
T
Tejun Heo 已提交
1178

1179 1180
	for (i = disks; i--; ) {
		struct r5dev *dev = &sh->dev[i];
1181

T
Tejun Heo 已提交
1182
		if (dev->written || i == pd_idx || i == qd_idx) {
1183
			set_bit(R5_UPTODATE, &dev->flags);
T
Tejun Heo 已提交
1184 1185
			if (fua)
				set_bit(R5_WantFUA, &dev->flags);
S
Shaohua Li 已提交
1186 1187
			if (sync)
				set_bit(R5_SyncIO, &dev->flags);
T
Tejun Heo 已提交
1188
		}
1189 1190
	}

1191 1192 1193 1194 1195 1196 1197 1198
	if (sh->reconstruct_state == reconstruct_state_drain_run)
		sh->reconstruct_state = reconstruct_state_drain_result;
	else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run)
		sh->reconstruct_state = reconstruct_state_prexor_drain_result;
	else {
		BUG_ON(sh->reconstruct_state != reconstruct_state_run);
		sh->reconstruct_state = reconstruct_state_result;
	}
1199 1200 1201 1202 1203 1204

	set_bit(STRIPE_HANDLE, &sh->state);
	release_stripe(sh);
}

static void
1205 1206
ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu,
		     struct dma_async_tx_descriptor *tx)
1207 1208
{
	int disks = sh->disks;
1209
	struct page **xor_srcs = percpu->scribble;
1210
	struct async_submit_ctl submit;
1211 1212
	int count = 0, pd_idx = sh->pd_idx, i;
	struct page *xor_dest;
1213
	int prexor = 0;
1214 1215
	unsigned long flags;

1216
	pr_debug("%s: stripe %llu\n", __func__,
1217 1218 1219 1220 1221
		(unsigned long long)sh->sector);

	/* check if prexor is active which means only process blocks
	 * that are part of a read-modify-write (written)
	 */
1222 1223
	if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
		prexor = 1;
1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243
		xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if (dev->written)
				xor_srcs[count++] = dev->page;
		}
	} else {
		xor_dest = sh->dev[pd_idx].page;
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if (i != pd_idx)
				xor_srcs[count++] = dev->page;
		}
	}

	/* 1/ if we prexor'd then the dest is reused as a source
	 * 2/ if we did not prexor then we are redoing the parity
	 * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST
	 * for the synchronous xor case
	 */
1244
	flags = ASYNC_TX_ACK |
1245 1246 1247 1248
		(prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST);

	atomic_inc(&sh->count);

1249
	init_async_submit(&submit, flags, tx, ops_complete_reconstruct, sh,
1250
			  to_addr_conv(sh, percpu));
1251 1252 1253 1254
	if (unlikely(count == 1))
		tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
	else
		tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
1255 1256
}

1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273
static void
ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,
		     struct dma_async_tx_descriptor *tx)
{
	struct async_submit_ctl submit;
	struct page **blocks = percpu->scribble;
	int count;

	pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector);

	count = set_syndrome_sources(blocks, sh);

	atomic_inc(&sh->count);

	init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_reconstruct,
			  sh, to_addr_conv(sh, percpu));
	async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE,  &submit);
1274 1275 1276 1277 1278 1279
}

static void ops_complete_check(void *stripe_head_ref)
{
	struct stripe_head *sh = stripe_head_ref;

1280
	pr_debug("%s: stripe %llu\n", __func__,
1281 1282
		(unsigned long long)sh->sector);

1283
	sh->check_state = check_state_check_result;
1284 1285 1286 1287
	set_bit(STRIPE_HANDLE, &sh->state);
	release_stripe(sh);
}

1288
static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu)
1289 1290
{
	int disks = sh->disks;
1291 1292 1293
	int pd_idx = sh->pd_idx;
	int qd_idx = sh->qd_idx;
	struct page *xor_dest;
1294
	struct page **xor_srcs = percpu->scribble;
1295
	struct dma_async_tx_descriptor *tx;
1296
	struct async_submit_ctl submit;
1297 1298
	int count;
	int i;
1299

1300
	pr_debug("%s: stripe %llu\n", __func__,
1301 1302
		(unsigned long long)sh->sector);

1303 1304 1305
	count = 0;
	xor_dest = sh->dev[pd_idx].page;
	xor_srcs[count++] = xor_dest;
1306
	for (i = disks; i--; ) {
1307 1308 1309
		if (i == pd_idx || i == qd_idx)
			continue;
		xor_srcs[count++] = sh->dev[i].page;
1310 1311
	}

1312 1313
	init_async_submit(&submit, 0, NULL, NULL, NULL,
			  to_addr_conv(sh, percpu));
D
Dan Williams 已提交
1314
	tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE,
1315
			   &sh->ops.zero_sum_result, &submit);
1316 1317

	atomic_inc(&sh->count);
1318 1319
	init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL);
	tx = async_trigger_callback(&submit);
1320 1321
}

1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333
static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp)
{
	struct page **srcs = percpu->scribble;
	struct async_submit_ctl submit;
	int count;

	pr_debug("%s: stripe %llu checkp: %d\n", __func__,
		(unsigned long long)sh->sector, checkp);

	count = set_syndrome_sources(srcs, sh);
	if (!checkp)
		srcs[count] = NULL;
1334 1335

	atomic_inc(&sh->count);
1336 1337 1338 1339
	init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check,
			  sh, to_addr_conv(sh, percpu));
	async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE,
			   &sh->ops.zero_sum_result, percpu->spare_page, &submit);
1340 1341
}

1342
static void __raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
1343 1344 1345
{
	int overlap_clear = 0, i, disks = sh->disks;
	struct dma_async_tx_descriptor *tx = NULL;
1346
	struct r5conf *conf = sh->raid_conf;
1347
	int level = conf->level;
1348 1349
	struct raid5_percpu *percpu;
	unsigned long cpu;
1350

1351 1352
	cpu = get_cpu();
	percpu = per_cpu_ptr(conf->percpu, cpu);
1353
	if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) {
1354 1355 1356 1357
		ops_run_biofill(sh);
		overlap_clear++;
	}

1358
	if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) {
1359 1360 1361 1362 1363 1364 1365 1366 1367 1368
		if (level < 6)
			tx = ops_run_compute5(sh, percpu);
		else {
			if (sh->ops.target2 < 0 || sh->ops.target < 0)
				tx = ops_run_compute6_1(sh, percpu);
			else
				tx = ops_run_compute6_2(sh, percpu);
		}
		/* terminate the chain if reconstruct is not set to be run */
		if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request))
1369 1370
			async_tx_ack(tx);
	}
1371

1372
	if (test_bit(STRIPE_OP_PREXOR, &ops_request))
1373
		tx = ops_run_prexor(sh, percpu, tx);
1374

1375
	if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) {
1376
		tx = ops_run_biodrain(sh, tx);
1377 1378 1379
		overlap_clear++;
	}

1380 1381 1382 1383 1384 1385
	if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) {
		if (level < 6)
			ops_run_reconstruct5(sh, percpu, tx);
		else
			ops_run_reconstruct6(sh, percpu, tx);
	}
1386

1387 1388 1389 1390 1391 1392 1393 1394 1395 1396
	if (test_bit(STRIPE_OP_CHECK, &ops_request)) {
		if (sh->check_state == check_state_run)
			ops_run_check_p(sh, percpu);
		else if (sh->check_state == check_state_run_q)
			ops_run_check_pq(sh, percpu, 0);
		else if (sh->check_state == check_state_run_pq)
			ops_run_check_pq(sh, percpu, 1);
		else
			BUG();
	}
1397 1398 1399 1400 1401 1402 1403

	if (overlap_clear)
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if (test_and_clear_bit(R5_Overlap, &dev->flags))
				wake_up(&sh->raid_conf->wait_for_overlap);
		}
1404
	put_cpu();
1405 1406
}

1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436
#ifdef CONFIG_MULTICORE_RAID456
static void async_run_ops(void *param, async_cookie_t cookie)
{
	struct stripe_head *sh = param;
	unsigned long ops_request = sh->ops.request;

	clear_bit_unlock(STRIPE_OPS_REQ_PENDING, &sh->state);
	wake_up(&sh->ops.wait_for_ops);

	__raid_run_ops(sh, ops_request);
	release_stripe(sh);
}

static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
{
	/* since handle_stripe can be called outside of raid5d context
	 * we need to ensure sh->ops.request is de-staged before another
	 * request arrives
	 */
	wait_event(sh->ops.wait_for_ops,
		   !test_and_set_bit_lock(STRIPE_OPS_REQ_PENDING, &sh->state));
	sh->ops.request = ops_request;

	atomic_inc(&sh->count);
	async_schedule(async_run_ops, sh);
}
#else
#define raid_run_ops __raid_run_ops
#endif

1437
static int grow_one_stripe(struct r5conf *conf)
L
Linus Torvalds 已提交
1438 1439
{
	struct stripe_head *sh;
N
Namhyung Kim 已提交
1440
	sh = kmem_cache_zalloc(conf->slab_cache, GFP_KERNEL);
1441 1442
	if (!sh)
		return 0;
N
Namhyung Kim 已提交
1443

1444
	sh->raid_conf = conf;
1445 1446 1447
	#ifdef CONFIG_MULTICORE_RAID456
	init_waitqueue_head(&sh->ops.wait_for_ops);
	#endif
1448

1449 1450
	if (grow_buffers(sh)) {
		shrink_buffers(sh);
1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461
		kmem_cache_free(conf->slab_cache, sh);
		return 0;
	}
	/* we just created an active stripe so... */
	atomic_set(&sh->count, 1);
	atomic_inc(&conf->active_stripes);
	INIT_LIST_HEAD(&sh->lru);
	release_stripe(sh);
	return 1;
}

1462
static int grow_stripes(struct r5conf *conf, int num)
1463
{
1464
	struct kmem_cache *sc;
1465
	int devs = max(conf->raid_disks, conf->previous_raid_disks);
L
Linus Torvalds 已提交
1466

1467 1468 1469 1470 1471 1472 1473 1474
	if (conf->mddev->gendisk)
		sprintf(conf->cache_name[0],
			"raid%d-%s", conf->level, mdname(conf->mddev));
	else
		sprintf(conf->cache_name[0],
			"raid%d-%p", conf->level, conf->mddev);
	sprintf(conf->cache_name[1], "%s-alt", conf->cache_name[0]);

1475 1476
	conf->active_name = 0;
	sc = kmem_cache_create(conf->cache_name[conf->active_name],
L
Linus Torvalds 已提交
1477
			       sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev),
1478
			       0, 0, NULL);
L
Linus Torvalds 已提交
1479 1480 1481
	if (!sc)
		return 1;
	conf->slab_cache = sc;
1482
	conf->pool_size = devs;
1483
	while (num--)
1484
		if (!grow_one_stripe(conf))
L
Linus Torvalds 已提交
1485 1486 1487
			return 1;
	return 0;
}
1488

1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510
/**
 * scribble_len - return the required size of the scribble region
 * @num - total number of disks in the array
 *
 * The size must be enough to contain:
 * 1/ a struct page pointer for each device in the array +2
 * 2/ room to convert each entry in (1) to its corresponding dma
 *    (dma_map_page()) or page (page_address()) address.
 *
 * Note: the +2 is for the destination buffers of the ddf/raid6 case where we
 * calculate over all devices (not just the data blocks), using zeros in place
 * of the P and Q blocks.
 */
static size_t scribble_len(int num)
{
	size_t len;

	len = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2);

	return len;
}

1511
static int resize_stripes(struct r5conf *conf, int newsize)
1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538
{
	/* Make all the stripes able to hold 'newsize' devices.
	 * New slots in each stripe get 'page' set to a new page.
	 *
	 * This happens in stages:
	 * 1/ create a new kmem_cache and allocate the required number of
	 *    stripe_heads.
	 * 2/ gather all the old stripe_heads and tranfer the pages across
	 *    to the new stripe_heads.  This will have the side effect of
	 *    freezing the array as once all stripe_heads have been collected,
	 *    no IO will be possible.  Old stripe heads are freed once their
	 *    pages have been transferred over, and the old kmem_cache is
	 *    freed when all stripes are done.
	 * 3/ reallocate conf->disks to be suitable bigger.  If this fails,
	 *    we simple return a failre status - no need to clean anything up.
	 * 4/ allocate new pages for the new slots in the new stripe_heads.
	 *    If this fails, we don't bother trying the shrink the
	 *    stripe_heads down again, we just leave them as they are.
	 *    As each stripe_head is processed the new one is released into
	 *    active service.
	 *
	 * Once step2 is started, we cannot afford to wait for a write,
	 * so we use GFP_NOIO allocations.
	 */
	struct stripe_head *osh, *nsh;
	LIST_HEAD(newstripes);
	struct disk_info *ndisks;
1539
	unsigned long cpu;
1540
	int err;
1541
	struct kmem_cache *sc;
1542 1543 1544 1545 1546
	int i;

	if (newsize <= conf->pool_size)
		return 0; /* never bother to shrink */

1547 1548 1549
	err = md_allow_write(conf->mddev);
	if (err)
		return err;
1550

1551 1552 1553
	/* Step 1 */
	sc = kmem_cache_create(conf->cache_name[1-conf->active_name],
			       sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev),
1554
			       0, 0, NULL);
1555 1556 1557 1558
	if (!sc)
		return -ENOMEM;

	for (i = conf->max_nr_stripes; i; i--) {
N
Namhyung Kim 已提交
1559
		nsh = kmem_cache_zalloc(sc, GFP_KERNEL);
1560 1561 1562 1563
		if (!nsh)
			break;

		nsh->raid_conf = conf;
1564 1565 1566
		#ifdef CONFIG_MULTICORE_RAID456
		init_waitqueue_head(&nsh->ops.wait_for_ops);
		#endif
1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588

		list_add(&nsh->lru, &newstripes);
	}
	if (i) {
		/* didn't get enough, give up */
		while (!list_empty(&newstripes)) {
			nsh = list_entry(newstripes.next, struct stripe_head, lru);
			list_del(&nsh->lru);
			kmem_cache_free(sc, nsh);
		}
		kmem_cache_destroy(sc);
		return -ENOMEM;
	}
	/* Step 2 - Must use GFP_NOIO now.
	 * OK, we have enough stripes, start collecting inactive
	 * stripes and copying them over
	 */
	list_for_each_entry(nsh, &newstripes, lru) {
		spin_lock_irq(&conf->device_lock);
		wait_event_lock_irq(conf->wait_for_stripe,
				    !list_empty(&conf->inactive_list),
				    conf->device_lock,
N
NeilBrown 已提交
1589
				    );
1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603
		osh = get_free_stripe(conf);
		spin_unlock_irq(&conf->device_lock);
		atomic_set(&nsh->count, 1);
		for(i=0; i<conf->pool_size; i++)
			nsh->dev[i].page = osh->dev[i].page;
		for( ; i<newsize; i++)
			nsh->dev[i].page = NULL;
		kmem_cache_free(conf->slab_cache, osh);
	}
	kmem_cache_destroy(conf->slab_cache);

	/* Step 3.
	 * At this point, we are holding all the stripes so the array
	 * is completely stalled, so now is a good time to resize
1604
	 * conf->disks and the scribble region
1605 1606 1607 1608 1609 1610 1611 1612 1613 1614
	 */
	ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO);
	if (ndisks) {
		for (i=0; i<conf->raid_disks; i++)
			ndisks[i] = conf->disks[i];
		kfree(conf->disks);
		conf->disks = ndisks;
	} else
		err = -ENOMEM;

1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633
	get_online_cpus();
	conf->scribble_len = scribble_len(newsize);
	for_each_present_cpu(cpu) {
		struct raid5_percpu *percpu;
		void *scribble;

		percpu = per_cpu_ptr(conf->percpu, cpu);
		scribble = kmalloc(conf->scribble_len, GFP_NOIO);

		if (scribble) {
			kfree(percpu->scribble);
			percpu->scribble = scribble;
		} else {
			err = -ENOMEM;
			break;
		}
	}
	put_online_cpus();

1634 1635 1636 1637
	/* Step 4, return new stripes to service */
	while(!list_empty(&newstripes)) {
		nsh = list_entry(newstripes.next, struct stripe_head, lru);
		list_del_init(&nsh->lru);
1638

1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654
		for (i=conf->raid_disks; i < newsize; i++)
			if (nsh->dev[i].page == NULL) {
				struct page *p = alloc_page(GFP_NOIO);
				nsh->dev[i].page = p;
				if (!p)
					err = -ENOMEM;
			}
		release_stripe(nsh);
	}
	/* critical section pass, GFP_NOIO no longer needed */

	conf->slab_cache = sc;
	conf->active_name = 1-conf->active_name;
	conf->pool_size = newsize;
	return err;
}
L
Linus Torvalds 已提交
1655

1656
static int drop_one_stripe(struct r5conf *conf)
L
Linus Torvalds 已提交
1657 1658 1659
{
	struct stripe_head *sh;

1660 1661 1662 1663 1664
	spin_lock_irq(&conf->device_lock);
	sh = get_free_stripe(conf);
	spin_unlock_irq(&conf->device_lock);
	if (!sh)
		return 0;
1665
	BUG_ON(atomic_read(&sh->count));
1666
	shrink_buffers(sh);
1667 1668 1669 1670 1671
	kmem_cache_free(conf->slab_cache, sh);
	atomic_dec(&conf->active_stripes);
	return 1;
}

1672
static void shrink_stripes(struct r5conf *conf)
1673 1674 1675 1676
{
	while (drop_one_stripe(conf))
		;

N
NeilBrown 已提交
1677 1678
	if (conf->slab_cache)
		kmem_cache_destroy(conf->slab_cache);
L
Linus Torvalds 已提交
1679 1680 1681
	conf->slab_cache = NULL;
}

1682
static void raid5_end_read_request(struct bio * bi, int error)
L
Linus Torvalds 已提交
1683
{
1684
	struct stripe_head *sh = bi->bi_private;
1685
	struct r5conf *conf = sh->raid_conf;
1686
	int disks = sh->disks, i;
L
Linus Torvalds 已提交
1687
	int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
1688
	char b[BDEVNAME_SIZE];
1689
	struct md_rdev *rdev = NULL;
1690
	sector_t s;
L
Linus Torvalds 已提交
1691 1692 1693 1694 1695

	for (i=0 ; i<disks; i++)
		if (bi == &sh->dev[i].req)
			break;

1696 1697
	pr_debug("end_read_request %llu/%d, count: %d, uptodate %d.\n",
		(unsigned long long)sh->sector, i, atomic_read(&sh->count),
L
Linus Torvalds 已提交
1698 1699 1700
		uptodate);
	if (i == disks) {
		BUG();
1701
		return;
L
Linus Torvalds 已提交
1702
	}
1703
	if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
1704 1705 1706 1707 1708
		/* If replacement finished while this request was outstanding,
		 * 'replacement' might be NULL already.
		 * In that case it moved down to 'rdev'.
		 * rdev is not removed until all requests are finished.
		 */
1709
		rdev = conf->disks[i].replacement;
1710
	if (!rdev)
1711
		rdev = conf->disks[i].rdev;
L
Linus Torvalds 已提交
1712

1713 1714 1715 1716
	if (use_new_offset(conf, sh))
		s = sh->sector + rdev->new_data_offset;
	else
		s = sh->sector + rdev->data_offset;
L
Linus Torvalds 已提交
1717 1718
	if (uptodate) {
		set_bit(R5_UPTODATE, &sh->dev[i].flags);
1719
		if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
1720 1721 1722 1723
			/* Note that this cannot happen on a
			 * replacement device.  We just fail those on
			 * any error
			 */
1724 1725 1726 1727 1728
			printk_ratelimited(
				KERN_INFO
				"md/raid:%s: read error corrected"
				" (%lu sectors at %llu on %s)\n",
				mdname(conf->mddev), STRIPE_SECTORS,
1729
				(unsigned long long)s,
1730
				bdevname(rdev->bdev, b));
1731
			atomic_add(STRIPE_SECTORS, &rdev->corrected_errors);
1732 1733 1734
			clear_bit(R5_ReadError, &sh->dev[i].flags);
			clear_bit(R5_ReWrite, &sh->dev[i].flags);
		}
1735 1736
		if (atomic_read(&rdev->read_errors))
			atomic_set(&rdev->read_errors, 0);
L
Linus Torvalds 已提交
1737
	} else {
1738
		const char *bdn = bdevname(rdev->bdev, b);
1739
		int retry = 0;
1740

L
Linus Torvalds 已提交
1741
		clear_bit(R5_UPTODATE, &sh->dev[i].flags);
1742
		atomic_inc(&rdev->read_errors);
1743 1744 1745 1746 1747 1748
		if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
			printk_ratelimited(
				KERN_WARNING
				"md/raid:%s: read error on replacement device "
				"(sector %llu on %s).\n",
				mdname(conf->mddev),
1749
				(unsigned long long)s,
1750 1751
				bdn);
		else if (conf->mddev->degraded >= conf->max_degraded)
1752 1753 1754 1755 1756
			printk_ratelimited(
				KERN_WARNING
				"md/raid:%s: read error not correctable "
				"(sector %llu on %s).\n",
				mdname(conf->mddev),
1757
				(unsigned long long)s,
1758
				bdn);
1759
		else if (test_bit(R5_ReWrite, &sh->dev[i].flags))
1760
			/* Oh, no!!! */
1761 1762 1763 1764 1765
			printk_ratelimited(
				KERN_WARNING
				"md/raid:%s: read error NOT corrected!! "
				"(sector %llu on %s).\n",
				mdname(conf->mddev),
1766
				(unsigned long long)s,
1767
				bdn);
1768
		else if (atomic_read(&rdev->read_errors)
1769
			 > conf->max_nr_stripes)
N
NeilBrown 已提交
1770
			printk(KERN_WARNING
1771
			       "md/raid:%s: Too many read errors, failing device %s.\n",
1772
			       mdname(conf->mddev), bdn);
1773 1774 1775 1776 1777
		else
			retry = 1;
		if (retry)
			set_bit(R5_ReadError, &sh->dev[i].flags);
		else {
1778 1779
			clear_bit(R5_ReadError, &sh->dev[i].flags);
			clear_bit(R5_ReWrite, &sh->dev[i].flags);
1780
			md_error(conf->mddev, rdev);
1781
		}
L
Linus Torvalds 已提交
1782
	}
1783
	rdev_dec_pending(rdev, conf->mddev);
L
Linus Torvalds 已提交
1784 1785 1786 1787 1788
	clear_bit(R5_LOCKED, &sh->dev[i].flags);
	set_bit(STRIPE_HANDLE, &sh->state);
	release_stripe(sh);
}

1789
static void raid5_end_write_request(struct bio *bi, int error)
L
Linus Torvalds 已提交
1790
{
1791
	struct stripe_head *sh = bi->bi_private;
1792
	struct r5conf *conf = sh->raid_conf;
1793
	int disks = sh->disks, i;
1794
	struct md_rdev *uninitialized_var(rdev);
L
Linus Torvalds 已提交
1795
	int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
1796 1797
	sector_t first_bad;
	int bad_sectors;
1798
	int replacement = 0;
L
Linus Torvalds 已提交
1799

1800 1801 1802
	for (i = 0 ; i < disks; i++) {
		if (bi == &sh->dev[i].req) {
			rdev = conf->disks[i].rdev;
L
Linus Torvalds 已提交
1803
			break;
1804 1805 1806
		}
		if (bi == &sh->dev[i].rreq) {
			rdev = conf->disks[i].replacement;
1807 1808 1809 1810 1811 1812 1813 1814
			if (rdev)
				replacement = 1;
			else
				/* rdev was removed and 'replacement'
				 * replaced it.  rdev is not removed
				 * until all requests are finished.
				 */
				rdev = conf->disks[i].rdev;
1815 1816 1817
			break;
		}
	}
1818
	pr_debug("end_write_request %llu/%d, count %d, uptodate: %d.\n",
L
Linus Torvalds 已提交
1819 1820 1821 1822
		(unsigned long long)sh->sector, i, atomic_read(&sh->count),
		uptodate);
	if (i == disks) {
		BUG();
1823
		return;
L
Linus Torvalds 已提交
1824 1825
	}

1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836
	if (replacement) {
		if (!uptodate)
			md_error(conf->mddev, rdev);
		else if (is_badblock(rdev, sh->sector,
				     STRIPE_SECTORS,
				     &first_bad, &bad_sectors))
			set_bit(R5_MadeGoodRepl, &sh->dev[i].flags);
	} else {
		if (!uptodate) {
			set_bit(WriteErrorSeen, &rdev->flags);
			set_bit(R5_WriteError, &sh->dev[i].flags);
1837 1838 1839
			if (!test_and_set_bit(WantReplacement, &rdev->flags))
				set_bit(MD_RECOVERY_NEEDED,
					&rdev->mddev->recovery);
1840 1841 1842 1843 1844 1845
		} else if (is_badblock(rdev, sh->sector,
				       STRIPE_SECTORS,
				       &first_bad, &bad_sectors))
			set_bit(R5_MadeGood, &sh->dev[i].flags);
	}
	rdev_dec_pending(rdev, conf->mddev);
L
Linus Torvalds 已提交
1846

1847 1848
	if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags))
		clear_bit(R5_LOCKED, &sh->dev[i].flags);
L
Linus Torvalds 已提交
1849
	set_bit(STRIPE_HANDLE, &sh->state);
1850
	release_stripe(sh);
L
Linus Torvalds 已提交
1851 1852
}

1853
static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous);
L
Linus Torvalds 已提交
1854
	
1855
static void raid5_build_block(struct stripe_head *sh, int i, int previous)
L
Linus Torvalds 已提交
1856 1857 1858 1859 1860 1861 1862 1863
{
	struct r5dev *dev = &sh->dev[i];

	bio_init(&dev->req);
	dev->req.bi_io_vec = &dev->vec;
	dev->req.bi_vcnt++;
	dev->req.bi_max_vecs++;
	dev->req.bi_private = sh;
1864
	dev->vec.bv_page = dev->page;
L
Linus Torvalds 已提交
1865

1866 1867 1868 1869 1870 1871 1872
	bio_init(&dev->rreq);
	dev->rreq.bi_io_vec = &dev->rvec;
	dev->rreq.bi_vcnt++;
	dev->rreq.bi_max_vecs++;
	dev->rreq.bi_private = sh;
	dev->rvec.bv_page = dev->page;

L
Linus Torvalds 已提交
1873
	dev->flags = 0;
1874
	dev->sector = compute_blocknr(sh, i, previous);
L
Linus Torvalds 已提交
1875 1876
}

1877
static void error(struct mddev *mddev, struct md_rdev *rdev)
L
Linus Torvalds 已提交
1878 1879
{
	char b[BDEVNAME_SIZE];
1880
	struct r5conf *conf = mddev->private;
1881
	unsigned long flags;
1882
	pr_debug("raid456: error called\n");
L
Linus Torvalds 已提交
1883

1884 1885 1886 1887 1888 1889
	spin_lock_irqsave(&conf->device_lock, flags);
	clear_bit(In_sync, &rdev->flags);
	mddev->degraded = calc_degraded(conf);
	spin_unlock_irqrestore(&conf->device_lock, flags);
	set_bit(MD_RECOVERY_INTR, &mddev->recovery);

1890
	set_bit(Blocked, &rdev->flags);
1891 1892 1893 1894 1895 1896 1897 1898 1899
	set_bit(Faulty, &rdev->flags);
	set_bit(MD_CHANGE_DEVS, &mddev->flags);
	printk(KERN_ALERT
	       "md/raid:%s: Disk failure on %s, disabling device.\n"
	       "md/raid:%s: Operation continuing on %d devices.\n",
	       mdname(mddev),
	       bdevname(rdev->bdev, b),
	       mdname(mddev),
	       conf->raid_disks - mddev->degraded);
1900
}
L
Linus Torvalds 已提交
1901 1902 1903 1904 1905

/*
 * Input: a 'big' sector number,
 * Output: index of the data and parity disk, and the sector # in them.
 */
1906
static sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector,
1907 1908
				     int previous, int *dd_idx,
				     struct stripe_head *sh)
L
Linus Torvalds 已提交
1909
{
N
NeilBrown 已提交
1910
	sector_t stripe, stripe2;
1911
	sector_t chunk_number;
L
Linus Torvalds 已提交
1912
	unsigned int chunk_offset;
1913
	int pd_idx, qd_idx;
1914
	int ddf_layout = 0;
L
Linus Torvalds 已提交
1915
	sector_t new_sector;
1916 1917
	int algorithm = previous ? conf->prev_algo
				 : conf->algorithm;
1918 1919
	int sectors_per_chunk = previous ? conf->prev_chunk_sectors
					 : conf->chunk_sectors;
1920 1921 1922
	int raid_disks = previous ? conf->previous_raid_disks
				  : conf->raid_disks;
	int data_disks = raid_disks - conf->max_degraded;
L
Linus Torvalds 已提交
1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934

	/* First compute the information on this sector */

	/*
	 * Compute the chunk number and the sector offset inside the chunk
	 */
	chunk_offset = sector_div(r_sector, sectors_per_chunk);
	chunk_number = r_sector;

	/*
	 * Compute the stripe number
	 */
1935 1936
	stripe = chunk_number;
	*dd_idx = sector_div(stripe, data_disks);
N
NeilBrown 已提交
1937
	stripe2 = stripe;
L
Linus Torvalds 已提交
1938 1939 1940
	/*
	 * Select the parity disk based on the user selected algorithm.
	 */
1941
	pd_idx = qd_idx = -1;
1942 1943
	switch(conf->level) {
	case 4:
1944
		pd_idx = data_disks;
1945 1946
		break;
	case 5:
1947
		switch (algorithm) {
L
Linus Torvalds 已提交
1948
		case ALGORITHM_LEFT_ASYMMETRIC:
N
NeilBrown 已提交
1949
			pd_idx = data_disks - sector_div(stripe2, raid_disks);
1950
			if (*dd_idx >= pd_idx)
L
Linus Torvalds 已提交
1951 1952 1953
				(*dd_idx)++;
			break;
		case ALGORITHM_RIGHT_ASYMMETRIC:
N
NeilBrown 已提交
1954
			pd_idx = sector_div(stripe2, raid_disks);
1955
			if (*dd_idx >= pd_idx)
L
Linus Torvalds 已提交
1956 1957 1958
				(*dd_idx)++;
			break;
		case ALGORITHM_LEFT_SYMMETRIC:
N
NeilBrown 已提交
1959
			pd_idx = data_disks - sector_div(stripe2, raid_disks);
1960
			*dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
L
Linus Torvalds 已提交
1961 1962
			break;
		case ALGORITHM_RIGHT_SYMMETRIC:
N
NeilBrown 已提交
1963
			pd_idx = sector_div(stripe2, raid_disks);
1964
			*dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
L
Linus Torvalds 已提交
1965
			break;
1966 1967 1968 1969 1970 1971 1972
		case ALGORITHM_PARITY_0:
			pd_idx = 0;
			(*dd_idx)++;
			break;
		case ALGORITHM_PARITY_N:
			pd_idx = data_disks;
			break;
L
Linus Torvalds 已提交
1973
		default:
1974
			BUG();
1975 1976 1977 1978
		}
		break;
	case 6:

1979
		switch (algorithm) {
1980
		case ALGORITHM_LEFT_ASYMMETRIC:
N
NeilBrown 已提交
1981
			pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
1982 1983
			qd_idx = pd_idx + 1;
			if (pd_idx == raid_disks-1) {
1984
				(*dd_idx)++;	/* Q D D D P */
1985 1986
				qd_idx = 0;
			} else if (*dd_idx >= pd_idx)
1987 1988 1989
				(*dd_idx) += 2; /* D D P Q D */
			break;
		case ALGORITHM_RIGHT_ASYMMETRIC:
N
NeilBrown 已提交
1990
			pd_idx = sector_div(stripe2, raid_disks);
1991 1992
			qd_idx = pd_idx + 1;
			if (pd_idx == raid_disks-1) {
1993
				(*dd_idx)++;	/* Q D D D P */
1994 1995
				qd_idx = 0;
			} else if (*dd_idx >= pd_idx)
1996 1997 1998
				(*dd_idx) += 2; /* D D P Q D */
			break;
		case ALGORITHM_LEFT_SYMMETRIC:
N
NeilBrown 已提交
1999
			pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
2000 2001
			qd_idx = (pd_idx + 1) % raid_disks;
			*dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
2002 2003
			break;
		case ALGORITHM_RIGHT_SYMMETRIC:
N
NeilBrown 已提交
2004
			pd_idx = sector_div(stripe2, raid_disks);
2005 2006
			qd_idx = (pd_idx + 1) % raid_disks;
			*dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
2007
			break;
2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022

		case ALGORITHM_PARITY_0:
			pd_idx = 0;
			qd_idx = 1;
			(*dd_idx) += 2;
			break;
		case ALGORITHM_PARITY_N:
			pd_idx = data_disks;
			qd_idx = data_disks + 1;
			break;

		case ALGORITHM_ROTATING_ZERO_RESTART:
			/* Exactly the same as RIGHT_ASYMMETRIC, but or
			 * of blocks for computing Q is different.
			 */
N
NeilBrown 已提交
2023
			pd_idx = sector_div(stripe2, raid_disks);
2024 2025 2026 2027 2028 2029
			qd_idx = pd_idx + 1;
			if (pd_idx == raid_disks-1) {
				(*dd_idx)++;	/* Q D D D P */
				qd_idx = 0;
			} else if (*dd_idx >= pd_idx)
				(*dd_idx) += 2; /* D D P Q D */
2030
			ddf_layout = 1;
2031 2032 2033 2034 2035 2036 2037
			break;

		case ALGORITHM_ROTATING_N_RESTART:
			/* Same a left_asymmetric, by first stripe is
			 * D D D P Q  rather than
			 * Q D D D P
			 */
N
NeilBrown 已提交
2038 2039
			stripe2 += 1;
			pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
2040 2041 2042 2043 2044 2045
			qd_idx = pd_idx + 1;
			if (pd_idx == raid_disks-1) {
				(*dd_idx)++;	/* Q D D D P */
				qd_idx = 0;
			} else if (*dd_idx >= pd_idx)
				(*dd_idx) += 2; /* D D P Q D */
2046
			ddf_layout = 1;
2047 2048 2049 2050
			break;

		case ALGORITHM_ROTATING_N_CONTINUE:
			/* Same as left_symmetric but Q is before P */
N
NeilBrown 已提交
2051
			pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
2052 2053
			qd_idx = (pd_idx + raid_disks - 1) % raid_disks;
			*dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
2054
			ddf_layout = 1;
2055 2056 2057 2058
			break;

		case ALGORITHM_LEFT_ASYMMETRIC_6:
			/* RAID5 left_asymmetric, with Q on last device */
N
NeilBrown 已提交
2059
			pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
2060 2061 2062 2063 2064 2065
			if (*dd_idx >= pd_idx)
				(*dd_idx)++;
			qd_idx = raid_disks - 1;
			break;

		case ALGORITHM_RIGHT_ASYMMETRIC_6:
N
NeilBrown 已提交
2066
			pd_idx = sector_div(stripe2, raid_disks-1);
2067 2068 2069 2070 2071 2072
			if (*dd_idx >= pd_idx)
				(*dd_idx)++;
			qd_idx = raid_disks - 1;
			break;

		case ALGORITHM_LEFT_SYMMETRIC_6:
N
NeilBrown 已提交
2073
			pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
2074 2075 2076 2077 2078
			*dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
			qd_idx = raid_disks - 1;
			break;

		case ALGORITHM_RIGHT_SYMMETRIC_6:
N
NeilBrown 已提交
2079
			pd_idx = sector_div(stripe2, raid_disks-1);
2080 2081 2082 2083 2084 2085 2086 2087 2088 2089
			*dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
			qd_idx = raid_disks - 1;
			break;

		case ALGORITHM_PARITY_0_6:
			pd_idx = 0;
			(*dd_idx)++;
			qd_idx = raid_disks - 1;
			break;

2090
		default:
2091
			BUG();
2092 2093
		}
		break;
L
Linus Torvalds 已提交
2094 2095
	}

2096 2097 2098
	if (sh) {
		sh->pd_idx = pd_idx;
		sh->qd_idx = qd_idx;
2099
		sh->ddf_layout = ddf_layout;
2100
	}
L
Linus Torvalds 已提交
2101 2102 2103 2104 2105 2106 2107 2108
	/*
	 * Finally, compute the new sector number
	 */
	new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset;
	return new_sector;
}


2109
static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous)
L
Linus Torvalds 已提交
2110
{
2111
	struct r5conf *conf = sh->raid_conf;
2112 2113
	int raid_disks = sh->disks;
	int data_disks = raid_disks - conf->max_degraded;
L
Linus Torvalds 已提交
2114
	sector_t new_sector = sh->sector, check;
2115 2116
	int sectors_per_chunk = previous ? conf->prev_chunk_sectors
					 : conf->chunk_sectors;
2117 2118
	int algorithm = previous ? conf->prev_algo
				 : conf->algorithm;
L
Linus Torvalds 已提交
2119 2120
	sector_t stripe;
	int chunk_offset;
2121 2122
	sector_t chunk_number;
	int dummy1, dd_idx = i;
L
Linus Torvalds 已提交
2123
	sector_t r_sector;
2124
	struct stripe_head sh2;
L
Linus Torvalds 已提交
2125

2126

L
Linus Torvalds 已提交
2127 2128 2129
	chunk_offset = sector_div(new_sector, sectors_per_chunk);
	stripe = new_sector;

2130 2131 2132 2133 2134
	if (i == sh->pd_idx)
		return 0;
	switch(conf->level) {
	case 4: break;
	case 5:
2135
		switch (algorithm) {
L
Linus Torvalds 已提交
2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146
		case ALGORITHM_LEFT_ASYMMETRIC:
		case ALGORITHM_RIGHT_ASYMMETRIC:
			if (i > sh->pd_idx)
				i--;
			break;
		case ALGORITHM_LEFT_SYMMETRIC:
		case ALGORITHM_RIGHT_SYMMETRIC:
			if (i < sh->pd_idx)
				i += raid_disks;
			i -= (sh->pd_idx + 1);
			break;
2147 2148 2149 2150 2151
		case ALGORITHM_PARITY_0:
			i -= 1;
			break;
		case ALGORITHM_PARITY_N:
			break;
L
Linus Torvalds 已提交
2152
		default:
2153
			BUG();
2154 2155 2156
		}
		break;
	case 6:
2157
		if (i == sh->qd_idx)
2158
			return 0; /* It is the Q disk */
2159
		switch (algorithm) {
2160 2161
		case ALGORITHM_LEFT_ASYMMETRIC:
		case ALGORITHM_RIGHT_ASYMMETRIC:
2162 2163 2164 2165
		case ALGORITHM_ROTATING_ZERO_RESTART:
		case ALGORITHM_ROTATING_N_RESTART:
			if (sh->pd_idx == raid_disks-1)
				i--;	/* Q D D D P */
2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179
			else if (i > sh->pd_idx)
				i -= 2; /* D D P Q D */
			break;
		case ALGORITHM_LEFT_SYMMETRIC:
		case ALGORITHM_RIGHT_SYMMETRIC:
			if (sh->pd_idx == raid_disks-1)
				i--; /* Q D D D P */
			else {
				/* D D P Q D */
				if (i < sh->pd_idx)
					i += raid_disks;
				i -= (sh->pd_idx + 2);
			}
			break;
2180 2181 2182 2183 2184 2185
		case ALGORITHM_PARITY_0:
			i -= 2;
			break;
		case ALGORITHM_PARITY_N:
			break;
		case ALGORITHM_ROTATING_N_CONTINUE:
2186
			/* Like left_symmetric, but P is before Q */
2187 2188
			if (sh->pd_idx == 0)
				i--;	/* P D D D Q */
2189 2190 2191 2192 2193 2194
			else {
				/* D D Q P D */
				if (i < sh->pd_idx)
					i += raid_disks;
				i -= (sh->pd_idx + 1);
			}
2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209
			break;
		case ALGORITHM_LEFT_ASYMMETRIC_6:
		case ALGORITHM_RIGHT_ASYMMETRIC_6:
			if (i > sh->pd_idx)
				i--;
			break;
		case ALGORITHM_LEFT_SYMMETRIC_6:
		case ALGORITHM_RIGHT_SYMMETRIC_6:
			if (i < sh->pd_idx)
				i += data_disks + 1;
			i -= (sh->pd_idx + 1);
			break;
		case ALGORITHM_PARITY_0_6:
			i -= 1;
			break;
2210
		default:
2211
			BUG();
2212 2213
		}
		break;
L
Linus Torvalds 已提交
2214 2215 2216
	}

	chunk_number = stripe * data_disks + i;
2217
	r_sector = chunk_number * sectors_per_chunk + chunk_offset;
L
Linus Torvalds 已提交
2218

2219
	check = raid5_compute_sector(conf, r_sector,
2220
				     previous, &dummy1, &sh2);
2221 2222
	if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx
		|| sh2.qd_idx != sh->qd_idx) {
2223 2224
		printk(KERN_ERR "md/raid:%s: compute_blocknr: map not correct\n",
		       mdname(conf->mddev));
L
Linus Torvalds 已提交
2225 2226 2227 2228 2229 2230
		return 0;
	}
	return r_sector;
}


2231
static void
2232
schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s,
2233
			 int rcw, int expand)
2234 2235
{
	int i, pd_idx = sh->pd_idx, disks = sh->disks;
2236
	struct r5conf *conf = sh->raid_conf;
2237
	int level = conf->level;
2238 2239 2240 2241 2242 2243 2244

	if (rcw) {
		/* if we are not expanding this is a proper write request, and
		 * there will be bios with new data to be drained into the
		 * stripe cache
		 */
		if (!expand) {
2245 2246 2247 2248
			sh->reconstruct_state = reconstruct_state_drain_run;
			set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
		} else
			sh->reconstruct_state = reconstruct_state_run;
2249

2250
		set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
2251 2252 2253 2254 2255 2256

		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];

			if (dev->towrite) {
				set_bit(R5_LOCKED, &dev->flags);
2257
				set_bit(R5_Wantdrain, &dev->flags);
2258 2259
				if (!expand)
					clear_bit(R5_UPTODATE, &dev->flags);
2260
				s->locked++;
2261 2262
			}
		}
2263
		if (s->locked + conf->max_degraded == disks)
2264
			if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state))
2265
				atomic_inc(&conf->pending_full_writes);
2266
	} else {
2267
		BUG_ON(level == 6);
2268 2269 2270
		BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) ||
			test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags)));

2271
		sh->reconstruct_state = reconstruct_state_prexor_drain_run;
2272 2273
		set_bit(STRIPE_OP_PREXOR, &s->ops_request);
		set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
2274
		set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
2275 2276 2277 2278 2279 2280 2281 2282

		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if (i == pd_idx)
				continue;

			if (dev->towrite &&
			    (test_bit(R5_UPTODATE, &dev->flags) ||
2283 2284
			     test_bit(R5_Wantcompute, &dev->flags))) {
				set_bit(R5_Wantdrain, &dev->flags);
2285 2286
				set_bit(R5_LOCKED, &dev->flags);
				clear_bit(R5_UPTODATE, &dev->flags);
2287
				s->locked++;
2288 2289 2290 2291
			}
		}
	}

2292
	/* keep the parity disk(s) locked while asynchronous operations
2293 2294 2295 2296
	 * are in flight
	 */
	set_bit(R5_LOCKED, &sh->dev[pd_idx].flags);
	clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
2297
	s->locked++;
2298

2299 2300 2301 2302 2303 2304 2305 2306 2307
	if (level == 6) {
		int qd_idx = sh->qd_idx;
		struct r5dev *dev = &sh->dev[qd_idx];

		set_bit(R5_LOCKED, &dev->flags);
		clear_bit(R5_UPTODATE, &dev->flags);
		s->locked++;
	}

2308
	pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n",
2309
		__func__, (unsigned long long)sh->sector,
2310
		s->locked, s->ops_request);
2311
}
2312

L
Linus Torvalds 已提交
2313 2314
/*
 * Each stripe/dev can have one or more bion attached.
2315
 * toread/towrite point to the first in a chain.
L
Linus Torvalds 已提交
2316 2317 2318 2319 2320
 * The bi_next chain must be in order.
 */
static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx, int forwrite)
{
	struct bio **bip;
2321
	struct r5conf *conf = sh->raid_conf;
2322
	int firstwrite=0;
L
Linus Torvalds 已提交
2323

2324
	pr_debug("adding bi b#%llu to stripe s#%llu\n",
L
Linus Torvalds 已提交
2325 2326 2327 2328 2329
		(unsigned long long)bi->bi_sector,
		(unsigned long long)sh->sector);


	spin_lock_irq(&conf->device_lock);
2330
	if (forwrite) {
L
Linus Torvalds 已提交
2331
		bip = &sh->dev[dd_idx].towrite;
2332 2333 2334
		if (*bip == NULL && sh->dev[dd_idx].written == NULL)
			firstwrite = 1;
	} else
L
Linus Torvalds 已提交
2335 2336 2337 2338 2339 2340 2341 2342 2343
		bip = &sh->dev[dd_idx].toread;
	while (*bip && (*bip)->bi_sector < bi->bi_sector) {
		if ((*bip)->bi_sector + ((*bip)->bi_size >> 9) > bi->bi_sector)
			goto overlap;
		bip = & (*bip)->bi_next;
	}
	if (*bip && (*bip)->bi_sector < bi->bi_sector + ((bi->bi_size)>>9))
		goto overlap;

2344
	BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next);
L
Linus Torvalds 已提交
2345 2346 2347
	if (*bip)
		bi->bi_next = *bip;
	*bip = bi;
2348
	bi->bi_phys_segments++;
2349

L
Linus Torvalds 已提交
2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362
	if (forwrite) {
		/* check if page is covered */
		sector_t sector = sh->dev[dd_idx].sector;
		for (bi=sh->dev[dd_idx].towrite;
		     sector < sh->dev[dd_idx].sector + STRIPE_SECTORS &&
			     bi && bi->bi_sector <= sector;
		     bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) {
			if (bi->bi_sector + (bi->bi_size>>9) >= sector)
				sector = bi->bi_sector + (bi->bi_size>>9);
		}
		if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS)
			set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags);
	}
2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374
	spin_unlock_irq(&conf->device_lock);

	pr_debug("added bi b#%llu to stripe s#%llu, disk %d.\n",
		(unsigned long long)(*bip)->bi_sector,
		(unsigned long long)sh->sector, dd_idx);

	if (conf->mddev->bitmap && firstwrite) {
		bitmap_startwrite(conf->mddev->bitmap, sh->sector,
				  STRIPE_SECTORS, 0);
		sh->bm_seq = conf->seq_flush+1;
		set_bit(STRIPE_BIT_DELAY, &sh->state);
	}
L
Linus Torvalds 已提交
2375 2376 2377 2378 2379 2380 2381 2382
	return 1;

 overlap:
	set_bit(R5_Overlap, &sh->dev[dd_idx].flags);
	spin_unlock_irq(&conf->device_lock);
	return 0;
}

2383
static void end_reshape(struct r5conf *conf);
2384

2385
static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,
2386
			    struct stripe_head *sh)
2387
{
2388
	int sectors_per_chunk =
2389
		previous ? conf->prev_chunk_sectors : conf->chunk_sectors;
2390
	int dd_idx;
2391
	int chunk_offset = sector_div(stripe, sectors_per_chunk);
2392
	int disks = previous ? conf->previous_raid_disks : conf->raid_disks;
2393

2394 2395
	raid5_compute_sector(conf,
			     stripe * (disks - conf->max_degraded)
2396
			     *sectors_per_chunk + chunk_offset,
2397
			     previous,
2398
			     &dd_idx, sh);
2399 2400
}

2401
static void
2402
handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh,
2403 2404 2405 2406 2407 2408 2409 2410 2411
				struct stripe_head_state *s, int disks,
				struct bio **return_bi)
{
	int i;
	for (i = disks; i--; ) {
		struct bio *bi;
		int bitmap_end = 0;

		if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
2412
			struct md_rdev *rdev;
2413 2414 2415
			rcu_read_lock();
			rdev = rcu_dereference(conf->disks[i].rdev);
			if (rdev && test_bit(In_sync, &rdev->flags))
2416 2417 2418
				atomic_inc(&rdev->nr_pending);
			else
				rdev = NULL;
2419
			rcu_read_unlock();
2420 2421 2422 2423 2424 2425 2426 2427
			if (rdev) {
				if (!rdev_set_badblocks(
					    rdev,
					    sh->sector,
					    STRIPE_SECTORS, 0))
					md_error(conf->mddev, rdev);
				rdev_dec_pending(rdev, conf->mddev);
			}
2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444
		}
		spin_lock_irq(&conf->device_lock);
		/* fail all writes first */
		bi = sh->dev[i].towrite;
		sh->dev[i].towrite = NULL;
		if (bi) {
			s->to_write--;
			bitmap_end = 1;
		}

		if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
			wake_up(&conf->wait_for_overlap);

		while (bi && bi->bi_sector <
			sh->dev[i].sector + STRIPE_SECTORS) {
			struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector);
			clear_bit(BIO_UPTODATE, &bi->bi_flags);
2445
			if (!raid5_dec_bi_phys_segments(bi)) {
2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459
				md_write_end(conf->mddev);
				bi->bi_next = *return_bi;
				*return_bi = bi;
			}
			bi = nextbi;
		}
		/* and fail all 'written' */
		bi = sh->dev[i].written;
		sh->dev[i].written = NULL;
		if (bi) bitmap_end = 1;
		while (bi && bi->bi_sector <
		       sh->dev[i].sector + STRIPE_SECTORS) {
			struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector);
			clear_bit(BIO_UPTODATE, &bi->bi_flags);
2460
			if (!raid5_dec_bi_phys_segments(bi)) {
2461 2462 2463 2464 2465 2466 2467
				md_write_end(conf->mddev);
				bi->bi_next = *return_bi;
				*return_bi = bi;
			}
			bi = bi2;
		}

2468 2469 2470 2471 2472 2473
		/* fail any reads if this device is non-operational and
		 * the data has not reached the cache yet.
		 */
		if (!test_bit(R5_Wantfill, &sh->dev[i].flags) &&
		    (!test_bit(R5_Insync, &sh->dev[i].flags) ||
		      test_bit(R5_ReadError, &sh->dev[i].flags))) {
2474 2475 2476 2477 2478 2479 2480 2481 2482 2483
			bi = sh->dev[i].toread;
			sh->dev[i].toread = NULL;
			if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
				wake_up(&conf->wait_for_overlap);
			if (bi) s->to_read--;
			while (bi && bi->bi_sector <
			       sh->dev[i].sector + STRIPE_SECTORS) {
				struct bio *nextbi =
					r5_next_bio(bi, sh->dev[i].sector);
				clear_bit(BIO_UPTODATE, &bi->bi_flags);
2484
				if (!raid5_dec_bi_phys_segments(bi)) {
2485 2486 2487 2488 2489 2490 2491 2492 2493 2494
					bi->bi_next = *return_bi;
					*return_bi = bi;
				}
				bi = nextbi;
			}
		}
		spin_unlock_irq(&conf->device_lock);
		if (bitmap_end)
			bitmap_endwrite(conf->mddev->bitmap, sh->sector,
					STRIPE_SECTORS, 0, 0);
2495 2496 2497 2498
		/* If we were in the middle of a write the parity block might
		 * still be locked - so just clear all R5_LOCKED flags
		 */
		clear_bit(R5_LOCKED, &sh->dev[i].flags);
2499 2500
	}

2501 2502 2503
	if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
		if (atomic_dec_and_test(&conf->pending_full_writes))
			md_wakeup_thread(conf->mddev->thread);
2504 2505
}

2506
static void
2507
handle_failed_sync(struct r5conf *conf, struct stripe_head *sh,
2508 2509 2510 2511 2512 2513 2514
		   struct stripe_head_state *s)
{
	int abort = 0;
	int i;

	clear_bit(STRIPE_SYNCING, &sh->state);
	s->syncing = 0;
2515
	s->replacing = 0;
2516
	/* There is nothing more to do for sync/check/repair.
2517 2518 2519
	 * Don't even need to abort as that is handled elsewhere
	 * if needed, and not always wanted e.g. if there is a known
	 * bad block here.
2520
	 * For recover/replace we need to record a bad block on all
2521 2522
	 * non-sync devices, or abort the recovery
	 */
2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545
	if (test_bit(MD_RECOVERY_RECOVER, &conf->mddev->recovery)) {
		/* During recovery devices cannot be removed, so
		 * locking and refcounting of rdevs is not needed
		 */
		for (i = 0; i < conf->raid_disks; i++) {
			struct md_rdev *rdev = conf->disks[i].rdev;
			if (rdev
			    && !test_bit(Faulty, &rdev->flags)
			    && !test_bit(In_sync, &rdev->flags)
			    && !rdev_set_badblocks(rdev, sh->sector,
						   STRIPE_SECTORS, 0))
				abort = 1;
			rdev = conf->disks[i].replacement;
			if (rdev
			    && !test_bit(Faulty, &rdev->flags)
			    && !test_bit(In_sync, &rdev->flags)
			    && !rdev_set_badblocks(rdev, sh->sector,
						   STRIPE_SECTORS, 0))
				abort = 1;
		}
		if (abort)
			conf->recovery_disabled =
				conf->mddev->recovery_disabled;
2546
	}
2547
	md_done_sync(conf->mddev, STRIPE_SECTORS, !abort);
2548 2549
}

2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565
static int want_replace(struct stripe_head *sh, int disk_idx)
{
	struct md_rdev *rdev;
	int rv = 0;
	/* Doing recovery so rcu locking not required */
	rdev = sh->raid_conf->disks[disk_idx].replacement;
	if (rdev
	    && !test_bit(Faulty, &rdev->flags)
	    && !test_bit(In_sync, &rdev->flags)
	    && (rdev->recovery_offset <= sh->sector
		|| rdev->mddev->recovery_cp <= sh->sector))
		rv = 1;

	return rv;
}

2566
/* fetch_block - checks the given member device to see if its data needs
2567 2568 2569
 * to be read or computed to satisfy a request.
 *
 * Returns 1 when no more member devices need to be checked, otherwise returns
2570
 * 0 to tell the loop in handle_stripe_fill to continue
2571
 */
2572 2573
static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s,
		       int disk_idx, int disks)
2574
{
2575
	struct r5dev *dev = &sh->dev[disk_idx];
2576 2577
	struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]],
				  &sh->dev[s->failed_num[1]] };
2578

2579
	/* is the data in this block needed, and can we get it? */
2580 2581 2582 2583 2584
	if (!test_bit(R5_LOCKED, &dev->flags) &&
	    !test_bit(R5_UPTODATE, &dev->flags) &&
	    (dev->toread ||
	     (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)) ||
	     s->syncing || s->expanding ||
2585
	     (s->replacing && want_replace(sh, disk_idx)) ||
2586 2587
	     (s->failed >= 1 && fdev[0]->toread) ||
	     (s->failed >= 2 && fdev[1]->toread) ||
2588 2589 2590
	     (sh->raid_conf->level <= 5 && s->failed && fdev[0]->towrite &&
	      !test_bit(R5_OVERWRITE, &fdev[0]->flags)) ||
	     (sh->raid_conf->level == 6 && s->failed && s->to_write))) {
2591 2592 2593 2594 2595 2596
		/* we would like to get this block, possibly by computing it,
		 * otherwise read it if the backing disk is insync
		 */
		BUG_ON(test_bit(R5_Wantcompute, &dev->flags));
		BUG_ON(test_bit(R5_Wantread, &dev->flags));
		if ((s->uptodate == disks - 1) &&
2597 2598
		    (s->failed && (disk_idx == s->failed_num[0] ||
				   disk_idx == s->failed_num[1]))) {
2599 2600
			/* have disk failed, and we're requested to fetch it;
			 * do compute it
2601
			 */
2602 2603 2604 2605 2606 2607 2608 2609
			pr_debug("Computing stripe %llu block %d\n",
			       (unsigned long long)sh->sector, disk_idx);
			set_bit(STRIPE_COMPUTE_RUN, &sh->state);
			set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
			set_bit(R5_Wantcompute, &dev->flags);
			sh->ops.target = disk_idx;
			sh->ops.target2 = -1; /* no 2nd target */
			s->req_compute = 1;
2610 2611 2612 2613 2614 2615
			/* Careful: from this point on 'uptodate' is in the eye
			 * of raid_run_ops which services 'compute' operations
			 * before writes. R5_Wantcompute flags a block that will
			 * be R5_UPTODATE by the time it is needed for a
			 * subsequent operation.
			 */
2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628
			s->uptodate++;
			return 1;
		} else if (s->uptodate == disks-2 && s->failed >= 2) {
			/* Computing 2-failure is *very* expensive; only
			 * do it if failed >= 2
			 */
			int other;
			for (other = disks; other--; ) {
				if (other == disk_idx)
					continue;
				if (!test_bit(R5_UPTODATE,
				      &sh->dev[other].flags))
					break;
2629
			}
2630 2631 2632 2633 2634 2635 2636 2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648
			BUG_ON(other < 0);
			pr_debug("Computing stripe %llu blocks %d,%d\n",
			       (unsigned long long)sh->sector,
			       disk_idx, other);
			set_bit(STRIPE_COMPUTE_RUN, &sh->state);
			set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
			set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags);
			set_bit(R5_Wantcompute, &sh->dev[other].flags);
			sh->ops.target = disk_idx;
			sh->ops.target2 = other;
			s->uptodate += 2;
			s->req_compute = 1;
			return 1;
		} else if (test_bit(R5_Insync, &dev->flags)) {
			set_bit(R5_LOCKED, &dev->flags);
			set_bit(R5_Wantread, &dev->flags);
			s->locked++;
			pr_debug("Reading block %d (sync=%d)\n",
				disk_idx, s->syncing);
2649 2650
		}
	}
2651 2652 2653 2654 2655

	return 0;
}

/**
2656
 * handle_stripe_fill - read or compute data to satisfy pending requests.
2657
 */
2658 2659 2660
static void handle_stripe_fill(struct stripe_head *sh,
			       struct stripe_head_state *s,
			       int disks)
2661 2662 2663 2664 2665 2666 2667 2668 2669 2670
{
	int i;

	/* look for blocks to read/compute, skip this if a compute
	 * is already in flight, or if the stripe contents are in the
	 * midst of changing due to a write
	 */
	if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state &&
	    !sh->reconstruct_state)
		for (i = disks; i--; )
2671
			if (fetch_block(sh, s, i, disks))
2672
				break;
2673 2674 2675 2676
	set_bit(STRIPE_HANDLE, &sh->state);
}


2677
/* handle_stripe_clean_event
2678 2679 2680 2681
 * any written block on an uptodate or failed drive can be returned.
 * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but
 * never LOCKED, so we don't need to test 'failed' directly.
 */
2682
static void handle_stripe_clean_event(struct r5conf *conf,
2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695
	struct stripe_head *sh, int disks, struct bio **return_bi)
{
	int i;
	struct r5dev *dev;

	for (i = disks; i--; )
		if (sh->dev[i].written) {
			dev = &sh->dev[i];
			if (!test_bit(R5_LOCKED, &dev->flags) &&
				test_bit(R5_UPTODATE, &dev->flags)) {
				/* We can return any write requests */
				struct bio *wbi, *wbi2;
				int bitmap_end = 0;
2696
				pr_debug("Return write for disc %d\n", i);
2697 2698 2699 2700 2701 2702
				spin_lock_irq(&conf->device_lock);
				wbi = dev->written;
				dev->written = NULL;
				while (wbi && wbi->bi_sector <
					dev->sector + STRIPE_SECTORS) {
					wbi2 = r5_next_bio(wbi, dev->sector);
2703
					if (!raid5_dec_bi_phys_segments(wbi)) {
2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720
						md_write_end(conf->mddev);
						wbi->bi_next = *return_bi;
						*return_bi = wbi;
					}
					wbi = wbi2;
				}
				if (dev->towrite == NULL)
					bitmap_end = 1;
				spin_unlock_irq(&conf->device_lock);
				if (bitmap_end)
					bitmap_endwrite(conf->mddev->bitmap,
							sh->sector,
							STRIPE_SECTORS,
					 !test_bit(STRIPE_DEGRADED, &sh->state),
							0);
			}
		}
2721 2722 2723 2724

	if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
		if (atomic_dec_and_test(&conf->pending_full_writes))
			md_wakeup_thread(conf->mddev->thread);
2725 2726
}

2727
static void handle_stripe_dirtying(struct r5conf *conf,
2728 2729 2730
				   struct stripe_head *sh,
				   struct stripe_head_state *s,
				   int disks)
2731 2732
{
	int rmw = 0, rcw = 0, i;
2733 2734 2735 2736 2737 2738 2739
	if (conf->max_degraded == 2) {
		/* RAID6 requires 'rcw' in current implementation
		 * Calculate the real rcw later - for now fake it
		 * look like rcw is cheaper
		 */
		rcw = 1; rmw = 2;
	} else for (i = disks; i--; ) {
2740 2741 2742 2743
		/* would I have to read this buffer for read_modify_write */
		struct r5dev *dev = &sh->dev[i];
		if ((dev->towrite || i == sh->pd_idx) &&
		    !test_bit(R5_LOCKED, &dev->flags) &&
2744 2745
		    !(test_bit(R5_UPTODATE, &dev->flags) ||
		      test_bit(R5_Wantcompute, &dev->flags))) {
2746 2747 2748 2749 2750 2751 2752 2753
			if (test_bit(R5_Insync, &dev->flags))
				rmw++;
			else
				rmw += 2*disks;  /* cannot read it */
		}
		/* Would I have to read this buffer for reconstruct_write */
		if (!test_bit(R5_OVERWRITE, &dev->flags) && i != sh->pd_idx &&
		    !test_bit(R5_LOCKED, &dev->flags) &&
2754 2755 2756
		    !(test_bit(R5_UPTODATE, &dev->flags) ||
		    test_bit(R5_Wantcompute, &dev->flags))) {
			if (test_bit(R5_Insync, &dev->flags)) rcw++;
2757 2758 2759 2760
			else
				rcw += 2*disks;
		}
	}
2761
	pr_debug("for sector %llu, rmw=%d rcw=%d\n",
2762 2763 2764 2765 2766 2767 2768 2769
		(unsigned long long)sh->sector, rmw, rcw);
	set_bit(STRIPE_HANDLE, &sh->state);
	if (rmw < rcw && rmw > 0)
		/* prefer read-modify-write, but need to get some data */
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if ((dev->towrite || i == sh->pd_idx) &&
			    !test_bit(R5_LOCKED, &dev->flags) &&
2770 2771
			    !(test_bit(R5_UPTODATE, &dev->flags) ||
			    test_bit(R5_Wantcompute, &dev->flags)) &&
2772 2773 2774
			    test_bit(R5_Insync, &dev->flags)) {
				if (
				  test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
2775
					pr_debug("Read_old block "
2776 2777 2778 2779 2780 2781 2782 2783 2784 2785
						"%d for r-m-w\n", i);
					set_bit(R5_LOCKED, &dev->flags);
					set_bit(R5_Wantread, &dev->flags);
					s->locked++;
				} else {
					set_bit(STRIPE_DELAYED, &sh->state);
					set_bit(STRIPE_HANDLE, &sh->state);
				}
			}
		}
2786
	if (rcw <= rmw && rcw > 0) {
2787
		/* want reconstruct write, but need to get some data */
2788
		rcw = 0;
2789 2790 2791
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if (!test_bit(R5_OVERWRITE, &dev->flags) &&
2792
			    i != sh->pd_idx && i != sh->qd_idx &&
2793
			    !test_bit(R5_LOCKED, &dev->flags) &&
2794
			    !(test_bit(R5_UPTODATE, &dev->flags) ||
2795 2796 2797 2798
			      test_bit(R5_Wantcompute, &dev->flags))) {
				rcw++;
				if (!test_bit(R5_Insync, &dev->flags))
					continue; /* it's a failed drive */
2799 2800
				if (
				  test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
2801
					pr_debug("Read_old block "
2802 2803 2804 2805 2806 2807 2808 2809 2810 2811
						"%d for Reconstruct\n", i);
					set_bit(R5_LOCKED, &dev->flags);
					set_bit(R5_Wantread, &dev->flags);
					s->locked++;
				} else {
					set_bit(STRIPE_DELAYED, &sh->state);
					set_bit(STRIPE_HANDLE, &sh->state);
				}
			}
		}
2812
	}
2813 2814 2815
	/* now if nothing is locked, and if we have enough data,
	 * we can start a write request
	 */
2816 2817
	/* since handle_stripe can be called at any time we need to handle the
	 * case where a compute block operation has been submitted and then a
2818 2819
	 * subsequent call wants to start a write request.  raid_run_ops only
	 * handles the case where compute block and reconstruct are requested
2820 2821 2822
	 * simultaneously.  If this is not the case then new writes need to be
	 * held off until the compute completes.
	 */
2823 2824 2825
	if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) &&
	    (s->locked == 0 && (rcw == 0 || rmw == 0) &&
	    !test_bit(STRIPE_BIT_DELAY, &sh->state)))
2826
		schedule_reconstruction(sh, s, rcw == 0, 0);
2827 2828
}

2829
static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh,
2830 2831
				struct stripe_head_state *s, int disks)
{
2832
	struct r5dev *dev = NULL;
2833

2834
	set_bit(STRIPE_HANDLE, &sh->state);
2835

2836 2837 2838
	switch (sh->check_state) {
	case check_state_idle:
		/* start a new check operation if there are no failures */
2839 2840
		if (s->failed == 0) {
			BUG_ON(s->uptodate != disks);
2841 2842
			sh->check_state = check_state_run;
			set_bit(STRIPE_OP_CHECK, &s->ops_request);
2843 2844
			clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
			s->uptodate--;
2845
			break;
2846
		}
2847
		dev = &sh->dev[s->failed_num[0]];
2848 2849 2850 2851 2852 2853 2854 2855 2856
		/* fall through */
	case check_state_compute_result:
		sh->check_state = check_state_idle;
		if (!dev)
			dev = &sh->dev[sh->pd_idx];

		/* check that a write has not made the stripe insync */
		if (test_bit(STRIPE_INSYNC, &sh->state))
			break;
D
Dan Williams 已提交
2857

2858 2859 2860 2861 2862
		/* either failed parity check, or recovery is happening */
		BUG_ON(!test_bit(R5_UPTODATE, &dev->flags));
		BUG_ON(s->uptodate != disks);

		set_bit(R5_LOCKED, &dev->flags);
2863
		s->locked++;
2864
		set_bit(R5_Wantwrite, &dev->flags);
2865

2866 2867
		clear_bit(STRIPE_DEGRADED, &sh->state);
		set_bit(STRIPE_INSYNC, &sh->state);
2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883
		break;
	case check_state_run:
		break; /* we will be called again upon completion */
	case check_state_check_result:
		sh->check_state = check_state_idle;

		/* if a failure occurred during the check operation, leave
		 * STRIPE_INSYNC not set and let the stripe be handled again
		 */
		if (s->failed)
			break;

		/* handle a successful check operation, if parity is correct
		 * we are done.  Otherwise update the mismatch count and repair
		 * parity if !MD_RECOVERY_CHECK
		 */
D
Dan Williams 已提交
2884
		if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0)
2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895
			/* parity is correct (on disc,
			 * not in buffer any more)
			 */
			set_bit(STRIPE_INSYNC, &sh->state);
		else {
			conf->mddev->resync_mismatches += STRIPE_SECTORS;
			if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery))
				/* don't try to repair!! */
				set_bit(STRIPE_INSYNC, &sh->state);
			else {
				sh->check_state = check_state_compute_run;
2896
				set_bit(STRIPE_COMPUTE_RUN, &sh->state);
2897 2898 2899 2900
				set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
				set_bit(R5_Wantcompute,
					&sh->dev[sh->pd_idx].flags);
				sh->ops.target = sh->pd_idx;
2901
				sh->ops.target2 = -1;
2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912
				s->uptodate++;
			}
		}
		break;
	case check_state_compute_run:
		break;
	default:
		printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
		       __func__, sh->check_state,
		       (unsigned long long) sh->sector);
		BUG();
2913 2914 2915 2916
	}
}


2917
static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh,
2918
				  struct stripe_head_state *s,
2919
				  int disks)
2920 2921
{
	int pd_idx = sh->pd_idx;
N
NeilBrown 已提交
2922
	int qd_idx = sh->qd_idx;
2923
	struct r5dev *dev;
2924 2925 2926 2927

	set_bit(STRIPE_HANDLE, &sh->state);

	BUG_ON(s->failed > 2);
2928

2929 2930 2931 2932 2933 2934
	/* Want to check and possibly repair P and Q.
	 * However there could be one 'failed' device, in which
	 * case we can only check one of them, possibly using the
	 * other to generate missing data
	 */

2935 2936 2937
	switch (sh->check_state) {
	case check_state_idle:
		/* start a new check operation if there are < 2 failures */
2938
		if (s->failed == s->q_failed) {
2939
			/* The only possible failed device holds Q, so it
2940 2941 2942
			 * makes sense to check P (If anything else were failed,
			 * we would have used P to recreate it).
			 */
2943
			sh->check_state = check_state_run;
2944
		}
2945
		if (!s->q_failed && s->failed < 2) {
2946
			/* Q is not failed, and we didn't use it to generate
2947 2948
			 * anything, so it makes sense to check it
			 */
2949 2950 2951 2952
			if (sh->check_state == check_state_run)
				sh->check_state = check_state_run_pq;
			else
				sh->check_state = check_state_run_q;
2953 2954
		}

2955 2956
		/* discard potentially stale zero_sum_result */
		sh->ops.zero_sum_result = 0;
2957

2958 2959 2960 2961
		if (sh->check_state == check_state_run) {
			/* async_xor_zero_sum destroys the contents of P */
			clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
			s->uptodate--;
2962
		}
2963 2964 2965 2966 2967 2968 2969
		if (sh->check_state >= check_state_run &&
		    sh->check_state <= check_state_run_pq) {
			/* async_syndrome_zero_sum preserves P and Q, so
			 * no need to mark them !uptodate here
			 */
			set_bit(STRIPE_OP_CHECK, &s->ops_request);
			break;
2970 2971
		}

2972 2973 2974 2975 2976
		/* we have 2-disk failure */
		BUG_ON(s->failed != 2);
		/* fall through */
	case check_state_compute_result:
		sh->check_state = check_state_idle;
2977

2978 2979 2980
		/* check that a write has not made the stripe insync */
		if (test_bit(STRIPE_INSYNC, &sh->state))
			break;
2981 2982

		/* now write out any block on a failed drive,
2983
		 * or P or Q if they were recomputed
2984
		 */
2985
		BUG_ON(s->uptodate < disks - 1); /* We don't need Q to recover */
2986
		if (s->failed == 2) {
2987
			dev = &sh->dev[s->failed_num[1]];
2988 2989 2990 2991 2992
			s->locked++;
			set_bit(R5_LOCKED, &dev->flags);
			set_bit(R5_Wantwrite, &dev->flags);
		}
		if (s->failed >= 1) {
2993
			dev = &sh->dev[s->failed_num[0]];
2994 2995 2996 2997
			s->locked++;
			set_bit(R5_LOCKED, &dev->flags);
			set_bit(R5_Wantwrite, &dev->flags);
		}
2998
		if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
2999 3000 3001 3002 3003
			dev = &sh->dev[pd_idx];
			s->locked++;
			set_bit(R5_LOCKED, &dev->flags);
			set_bit(R5_Wantwrite, &dev->flags);
		}
3004
		if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
3005 3006 3007 3008 3009 3010 3011 3012
			dev = &sh->dev[qd_idx];
			s->locked++;
			set_bit(R5_LOCKED, &dev->flags);
			set_bit(R5_Wantwrite, &dev->flags);
		}
		clear_bit(STRIPE_DEGRADED, &sh->state);

		set_bit(STRIPE_INSYNC, &sh->state);
3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076
		break;
	case check_state_run:
	case check_state_run_q:
	case check_state_run_pq:
		break; /* we will be called again upon completion */
	case check_state_check_result:
		sh->check_state = check_state_idle;

		/* handle a successful check operation, if parity is correct
		 * we are done.  Otherwise update the mismatch count and repair
		 * parity if !MD_RECOVERY_CHECK
		 */
		if (sh->ops.zero_sum_result == 0) {
			/* both parities are correct */
			if (!s->failed)
				set_bit(STRIPE_INSYNC, &sh->state);
			else {
				/* in contrast to the raid5 case we can validate
				 * parity, but still have a failure to write
				 * back
				 */
				sh->check_state = check_state_compute_result;
				/* Returning at this point means that we may go
				 * off and bring p and/or q uptodate again so
				 * we make sure to check zero_sum_result again
				 * to verify if p or q need writeback
				 */
			}
		} else {
			conf->mddev->resync_mismatches += STRIPE_SECTORS;
			if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery))
				/* don't try to repair!! */
				set_bit(STRIPE_INSYNC, &sh->state);
			else {
				int *target = &sh->ops.target;

				sh->ops.target = -1;
				sh->ops.target2 = -1;
				sh->check_state = check_state_compute_run;
				set_bit(STRIPE_COMPUTE_RUN, &sh->state);
				set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
				if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
					set_bit(R5_Wantcompute,
						&sh->dev[pd_idx].flags);
					*target = pd_idx;
					target = &sh->ops.target2;
					s->uptodate++;
				}
				if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
					set_bit(R5_Wantcompute,
						&sh->dev[qd_idx].flags);
					*target = qd_idx;
					s->uptodate++;
				}
			}
		}
		break;
	case check_state_compute_run:
		break;
	default:
		printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
		       __func__, sh->check_state,
		       (unsigned long long) sh->sector);
		BUG();
3077 3078 3079
	}
}

3080
static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh)
3081 3082 3083 3084 3085 3086
{
	int i;

	/* We have read all the blocks in this stripe and now we need to
	 * copy some of them into a target stripe for expand.
	 */
3087
	struct dma_async_tx_descriptor *tx = NULL;
3088 3089
	clear_bit(STRIPE_EXPAND_SOURCE, &sh->state);
	for (i = 0; i < sh->disks; i++)
N
NeilBrown 已提交
3090
		if (i != sh->pd_idx && i != sh->qd_idx) {
3091
			int dd_idx, j;
3092
			struct stripe_head *sh2;
3093
			struct async_submit_ctl submit;
3094

3095
			sector_t bn = compute_blocknr(sh, i, 1);
3096 3097
			sector_t s = raid5_compute_sector(conf, bn, 0,
							  &dd_idx, NULL);
3098
			sh2 = get_active_stripe(conf, s, 0, 1, 1);
3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110
			if (sh2 == NULL)
				/* so far only the early blocks of this stripe
				 * have been requested.  When later blocks
				 * get requested, we will try again
				 */
				continue;
			if (!test_bit(STRIPE_EXPANDING, &sh2->state) ||
			   test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) {
				/* must have already done this block */
				release_stripe(sh2);
				continue;
			}
3111 3112

			/* place all the copies on one channel */
3113
			init_async_submit(&submit, 0, tx, NULL, NULL, NULL);
3114
			tx = async_memcpy(sh2->dev[dd_idx].page,
3115
					  sh->dev[i].page, 0, 0, STRIPE_SIZE,
3116
					  &submit);
3117

3118 3119 3120 3121
			set_bit(R5_Expanded, &sh2->dev[dd_idx].flags);
			set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags);
			for (j = 0; j < conf->raid_disks; j++)
				if (j != sh2->pd_idx &&
3122
				    j != sh2->qd_idx &&
3123 3124 3125 3126 3127 3128 3129
				    !test_bit(R5_Expanded, &sh2->dev[j].flags))
					break;
			if (j == conf->raid_disks) {
				set_bit(STRIPE_EXPAND_READY, &sh2->state);
				set_bit(STRIPE_HANDLE, &sh2->state);
			}
			release_stripe(sh2);
3130

3131
		}
3132 3133 3134 3135 3136
	/* done submitting copies, wait for them to complete */
	if (tx) {
		async_tx_ack(tx);
		dma_wait_for_async_tx(tx);
	}
3137
}
L
Linus Torvalds 已提交
3138 3139 3140 3141

/*
 * handle_stripe - do things to a stripe.
 *
3142 3143
 * We lock the stripe by setting STRIPE_ACTIVE and then examine the
 * state of various bits to see what needs to be done.
L
Linus Torvalds 已提交
3144
 * Possible results:
3145 3146
 *    return some read requests which now have data
 *    return some write requests which are safely on storage
L
Linus Torvalds 已提交
3147 3148 3149 3150 3151
 *    schedule a read on some buffers
 *    schedule a write of some buffers
 *    return confirmation of parity correctness
 *
 */
3152

3153
static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)
L
Linus Torvalds 已提交
3154
{
3155
	struct r5conf *conf = sh->raid_conf;
3156
	int disks = sh->disks;
3157 3158
	struct r5dev *dev;
	int i;
3159
	int do_recovery = 0;
L
Linus Torvalds 已提交
3160

3161 3162 3163 3164 3165 3166
	memset(s, 0, sizeof(*s));

	s->expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state);
	s->expanded = test_bit(STRIPE_EXPAND_READY, &sh->state);
	s->failed_num[0] = -1;
	s->failed_num[1] = -1;
L
Linus Torvalds 已提交
3167

3168
	/* Now to look around and see what can be done */
L
Linus Torvalds 已提交
3169
	rcu_read_lock();
3170
	spin_lock_irq(&conf->device_lock);
3171
	for (i=disks; i--; ) {
3172
		struct md_rdev *rdev;
3173 3174 3175
		sector_t first_bad;
		int bad_sectors;
		int is_bad = 0;
3176

3177
		dev = &sh->dev[i];
L
Linus Torvalds 已提交
3178

3179
		pr_debug("check %d: state 0x%lx read %p write %p written %p\n",
3180 3181
			 i, dev->flags,
			 dev->toread, dev->towrite, dev->written);
3182 3183 3184 3185 3186 3187 3188 3189
		/* maybe we can reply to a read
		 *
		 * new wantfill requests are only permitted while
		 * ops_complete_biofill is guaranteed to be inactive
		 */
		if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread &&
		    !test_bit(STRIPE_BIOFILL_RUN, &sh->state))
			set_bit(R5_Wantfill, &dev->flags);
L
Linus Torvalds 已提交
3190

3191
		/* now count some things */
3192 3193 3194 3195
		if (test_bit(R5_LOCKED, &dev->flags))
			s->locked++;
		if (test_bit(R5_UPTODATE, &dev->flags))
			s->uptodate++;
3196
		if (test_bit(R5_Wantcompute, &dev->flags)) {
3197 3198
			s->compute++;
			BUG_ON(s->compute > 2);
3199
		}
L
Linus Torvalds 已提交
3200

3201
		if (test_bit(R5_Wantfill, &dev->flags))
3202
			s->to_fill++;
3203
		else if (dev->toread)
3204
			s->to_read++;
3205
		if (dev->towrite) {
3206
			s->to_write++;
3207
			if (!test_bit(R5_OVERWRITE, &dev->flags))
3208
				s->non_overwrite++;
3209
		}
3210
		if (dev->written)
3211
			s->written++;
3212 3213 3214 3215 3216 3217 3218 3219 3220 3221
		/* Prefer to use the replacement for reads, but only
		 * if it is recovered enough and has no bad blocks.
		 */
		rdev = rcu_dereference(conf->disks[i].replacement);
		if (rdev && !test_bit(Faulty, &rdev->flags) &&
		    rdev->recovery_offset >= sh->sector + STRIPE_SECTORS &&
		    !is_badblock(rdev, sh->sector, STRIPE_SECTORS,
				 &first_bad, &bad_sectors))
			set_bit(R5_ReadRepl, &dev->flags);
		else {
3222 3223
			if (rdev)
				set_bit(R5_NeedReplace, &dev->flags);
3224 3225 3226
			rdev = rcu_dereference(conf->disks[i].rdev);
			clear_bit(R5_ReadRepl, &dev->flags);
		}
3227 3228
		if (rdev && test_bit(Faulty, &rdev->flags))
			rdev = NULL;
3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240
		if (rdev) {
			is_bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS,
					     &first_bad, &bad_sectors);
			if (s->blocked_rdev == NULL
			    && (test_bit(Blocked, &rdev->flags)
				|| is_bad < 0)) {
				if (is_bad < 0)
					set_bit(BlockedBadBlocks,
						&rdev->flags);
				s->blocked_rdev = rdev;
				atomic_inc(&rdev->nr_pending);
			}
3241
		}
3242 3243 3244
		clear_bit(R5_Insync, &dev->flags);
		if (!rdev)
			/* Not in-sync */;
3245 3246
		else if (is_bad) {
			/* also not in-sync */
3247 3248
			if (!test_bit(WriteErrorSeen, &rdev->flags) &&
			    test_bit(R5_UPTODATE, &dev->flags)) {
3249 3250 3251 3252 3253 3254 3255
				/* treat as in-sync, but with a read error
				 * which we can now try to correct
				 */
				set_bit(R5_Insync, &dev->flags);
				set_bit(R5_ReadError, &dev->flags);
			}
		} else if (test_bit(In_sync, &rdev->flags))
3256
			set_bit(R5_Insync, &dev->flags);
3257
		else if (sh->sector + STRIPE_SECTORS <= rdev->recovery_offset)
3258
			/* in sync if before recovery_offset */
3259 3260 3261 3262 3263 3264 3265 3266 3267
			set_bit(R5_Insync, &dev->flags);
		else if (test_bit(R5_UPTODATE, &dev->flags) &&
			 test_bit(R5_Expanded, &dev->flags))
			/* If we've reshaped into here, we assume it is Insync.
			 * We will shortly update recovery_offset to make
			 * it official.
			 */
			set_bit(R5_Insync, &dev->flags);

A
Adam Kwolek 已提交
3268
		if (rdev && test_bit(R5_WriteError, &dev->flags)) {
3269 3270 3271 3272 3273 3274 3275
			/* This flag does not apply to '.replacement'
			 * only to .rdev, so make sure to check that*/
			struct md_rdev *rdev2 = rcu_dereference(
				conf->disks[i].rdev);
			if (rdev2 == rdev)
				clear_bit(R5_Insync, &dev->flags);
			if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
3276
				s->handle_bad_blocks = 1;
3277
				atomic_inc(&rdev2->nr_pending);
3278 3279 3280
			} else
				clear_bit(R5_WriteError, &dev->flags);
		}
A
Adam Kwolek 已提交
3281
		if (rdev && test_bit(R5_MadeGood, &dev->flags)) {
3282 3283 3284 3285 3286
			/* This flag does not apply to '.replacement'
			 * only to .rdev, so make sure to check that*/
			struct md_rdev *rdev2 = rcu_dereference(
				conf->disks[i].rdev);
			if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
3287
				s->handle_bad_blocks = 1;
3288
				atomic_inc(&rdev2->nr_pending);
3289 3290 3291
			} else
				clear_bit(R5_MadeGood, &dev->flags);
		}
3292 3293 3294 3295 3296 3297 3298 3299 3300
		if (test_bit(R5_MadeGoodRepl, &dev->flags)) {
			struct md_rdev *rdev2 = rcu_dereference(
				conf->disks[i].replacement);
			if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
				s->handle_bad_blocks = 1;
				atomic_inc(&rdev2->nr_pending);
			} else
				clear_bit(R5_MadeGoodRepl, &dev->flags);
		}
3301
		if (!test_bit(R5_Insync, &dev->flags)) {
3302 3303 3304
			/* The ReadError flag will just be confusing now */
			clear_bit(R5_ReadError, &dev->flags);
			clear_bit(R5_ReWrite, &dev->flags);
L
Linus Torvalds 已提交
3305
		}
3306 3307 3308
		if (test_bit(R5_ReadError, &dev->flags))
			clear_bit(R5_Insync, &dev->flags);
		if (!test_bit(R5_Insync, &dev->flags)) {
3309 3310 3311
			if (s->failed < 2)
				s->failed_num[s->failed] = i;
			s->failed++;
3312 3313
			if (rdev && !test_bit(Faulty, &rdev->flags))
				do_recovery = 1;
3314
		}
L
Linus Torvalds 已提交
3315
	}
3316
	spin_unlock_irq(&conf->device_lock);
3317 3318 3319 3320
	if (test_bit(STRIPE_SYNCING, &sh->state)) {
		/* If there is a failed device being replaced,
		 *     we must be recovering.
		 * else if we are after recovery_cp, we must be syncing
3321
		 * else if MD_RECOVERY_REQUESTED is set, we also are syncing.
3322 3323 3324 3325 3326
		 * else we can only be replacing
		 * sync and recovery both need to read all devices, and so
		 * use the same flag.
		 */
		if (do_recovery ||
3327 3328
		    sh->sector >= conf->mddev->recovery_cp ||
		    test_bit(MD_RECOVERY_REQUESTED, &(conf->mddev->recovery)))
3329 3330 3331 3332
			s->syncing = 1;
		else
			s->replacing = 1;
	}
L
Linus Torvalds 已提交
3333
	rcu_read_unlock();
3334 3335 3336 3337 3338
}

static void handle_stripe(struct stripe_head *sh)
{
	struct stripe_head_state s;
3339
	struct r5conf *conf = sh->raid_conf;
3340
	int i;
3341 3342
	int prexor;
	int disks = sh->disks;
3343
	struct r5dev *pdev, *qdev;
3344 3345

	clear_bit(STRIPE_HANDLE, &sh->state);
3346
	if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) {
3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363
		/* already being handled, ensure it gets handled
		 * again when current action finishes */
		set_bit(STRIPE_HANDLE, &sh->state);
		return;
	}

	if (test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) {
		set_bit(STRIPE_SYNCING, &sh->state);
		clear_bit(STRIPE_INSYNC, &sh->state);
	}
	clear_bit(STRIPE_DELAYED, &sh->state);

	pr_debug("handling stripe %llu, state=%#lx cnt=%d, "
		"pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n",
	       (unsigned long long)sh->sector, sh->state,
	       atomic_read(&sh->count), sh->pd_idx, sh->qd_idx,
	       sh->check_state, sh->reconstruct_state);
3364

3365
	analyse_stripe(sh, &s);
3366

3367 3368 3369 3370 3371
	if (s.handle_bad_blocks) {
		set_bit(STRIPE_HANDLE, &sh->state);
		goto finish;
	}

3372 3373
	if (unlikely(s.blocked_rdev)) {
		if (s.syncing || s.expanding || s.expanded ||
3374
		    s.replacing || s.to_write || s.written) {
3375 3376 3377 3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394
			set_bit(STRIPE_HANDLE, &sh->state);
			goto finish;
		}
		/* There is nothing for the blocked_rdev to block */
		rdev_dec_pending(s.blocked_rdev, conf->mddev);
		s.blocked_rdev = NULL;
	}

	if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) {
		set_bit(STRIPE_OP_BIOFILL, &s.ops_request);
		set_bit(STRIPE_BIOFILL_RUN, &sh->state);
	}

	pr_debug("locked=%d uptodate=%d to_read=%d"
	       " to_write=%d failed=%d failed_num=%d,%d\n",
	       s.locked, s.uptodate, s.to_read, s.to_write, s.failed,
	       s.failed_num[0], s.failed_num[1]);
	/* check if the array has lost more than max_degraded devices and,
	 * if so, some requests might need to be failed.
	 */
3395 3396 3397 3398 3399
	if (s.failed > conf->max_degraded) {
		sh->check_state = 0;
		sh->reconstruct_state = 0;
		if (s.to_read+s.to_write+s.written)
			handle_failed_stripe(conf, sh, &s, disks, &s.return_bi);
3400
		if (s.syncing + s.replacing)
3401 3402
			handle_failed_sync(conf, sh, &s);
	}
3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430

	/*
	 * might be able to return some write requests if the parity blocks
	 * are safe, or on a failed drive
	 */
	pdev = &sh->dev[sh->pd_idx];
	s.p_failed = (s.failed >= 1 && s.failed_num[0] == sh->pd_idx)
		|| (s.failed >= 2 && s.failed_num[1] == sh->pd_idx);
	qdev = &sh->dev[sh->qd_idx];
	s.q_failed = (s.failed >= 1 && s.failed_num[0] == sh->qd_idx)
		|| (s.failed >= 2 && s.failed_num[1] == sh->qd_idx)
		|| conf->level < 6;

	if (s.written &&
	    (s.p_failed || ((test_bit(R5_Insync, &pdev->flags)
			     && !test_bit(R5_LOCKED, &pdev->flags)
			     && test_bit(R5_UPTODATE, &pdev->flags)))) &&
	    (s.q_failed || ((test_bit(R5_Insync, &qdev->flags)
			     && !test_bit(R5_LOCKED, &qdev->flags)
			     && test_bit(R5_UPTODATE, &qdev->flags)))))
		handle_stripe_clean_event(conf, sh, disks, &s.return_bi);

	/* Now we might consider reading some blocks, either to check/generate
	 * parity, or to satisfy requests
	 * or to load a block that is being partially written.
	 */
	if (s.to_read || s.non_overwrite
	    || (conf->level == 6 && s.to_write && s.failed)
3431 3432 3433
	    || (s.syncing && (s.uptodate + s.compute < disks))
	    || s.replacing
	    || s.expanding)
3434 3435
		handle_stripe_fill(sh, &s, disks);

3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493
	/* Now we check to see if any write operations have recently
	 * completed
	 */
	prexor = 0;
	if (sh->reconstruct_state == reconstruct_state_prexor_drain_result)
		prexor = 1;
	if (sh->reconstruct_state == reconstruct_state_drain_result ||
	    sh->reconstruct_state == reconstruct_state_prexor_drain_result) {
		sh->reconstruct_state = reconstruct_state_idle;

		/* All the 'written' buffers and the parity block are ready to
		 * be written back to disk
		 */
		BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags));
		BUG_ON(sh->qd_idx >= 0 &&
		       !test_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags));
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if (test_bit(R5_LOCKED, &dev->flags) &&
				(i == sh->pd_idx || i == sh->qd_idx ||
				 dev->written)) {
				pr_debug("Writing block %d\n", i);
				set_bit(R5_Wantwrite, &dev->flags);
				if (prexor)
					continue;
				if (!test_bit(R5_Insync, &dev->flags) ||
				    ((i == sh->pd_idx || i == sh->qd_idx)  &&
				     s.failed == 0))
					set_bit(STRIPE_INSYNC, &sh->state);
			}
		}
		if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
			s.dec_preread_active = 1;
	}

	/* Now to consider new write requests and what else, if anything
	 * should be read.  We do not handle new writes when:
	 * 1/ A 'write' operation (copy+xor) is already in flight.
	 * 2/ A 'check' operation is in flight, as it may clobber the parity
	 *    block.
	 */
	if (s.to_write && !sh->reconstruct_state && !sh->check_state)
		handle_stripe_dirtying(conf, sh, &s, disks);

	/* maybe we need to check and possibly fix the parity for this stripe
	 * Any reads will already have been scheduled, so we just see if enough
	 * data is available.  The parity check is held off while parity
	 * dependent operations are in flight.
	 */
	if (sh->check_state ||
	    (s.syncing && s.locked == 0 &&
	     !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
	     !test_bit(STRIPE_INSYNC, &sh->state))) {
		if (conf->level == 6)
			handle_parity_checks6(conf, sh, &s, disks);
		else
			handle_parity_checks5(conf, sh, &s, disks);
	}
3494

3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508
	if (s.replacing && s.locked == 0
	    && !test_bit(STRIPE_INSYNC, &sh->state)) {
		/* Write out to replacement devices where possible */
		for (i = 0; i < conf->raid_disks; i++)
			if (test_bit(R5_UPTODATE, &sh->dev[i].flags) &&
			    test_bit(R5_NeedReplace, &sh->dev[i].flags)) {
				set_bit(R5_WantReplace, &sh->dev[i].flags);
				set_bit(R5_LOCKED, &sh->dev[i].flags);
				s.locked++;
			}
		set_bit(STRIPE_INSYNC, &sh->state);
	}
	if ((s.syncing || s.replacing) && s.locked == 0 &&
	    test_bit(STRIPE_INSYNC, &sh->state)) {
3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537
		md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
		clear_bit(STRIPE_SYNCING, &sh->state);
	}

	/* If the failed drives are just a ReadError, then we might need
	 * to progress the repair/check process
	 */
	if (s.failed <= conf->max_degraded && !conf->mddev->ro)
		for (i = 0; i < s.failed; i++) {
			struct r5dev *dev = &sh->dev[s.failed_num[i]];
			if (test_bit(R5_ReadError, &dev->flags)
			    && !test_bit(R5_LOCKED, &dev->flags)
			    && test_bit(R5_UPTODATE, &dev->flags)
				) {
				if (!test_bit(R5_ReWrite, &dev->flags)) {
					set_bit(R5_Wantwrite, &dev->flags);
					set_bit(R5_ReWrite, &dev->flags);
					set_bit(R5_LOCKED, &dev->flags);
					s.locked++;
				} else {
					/* let's read it back */
					set_bit(R5_Wantread, &dev->flags);
					set_bit(R5_LOCKED, &dev->flags);
					s.locked++;
				}
			}
		}


3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564
	/* Finish reconstruct operations initiated by the expansion process */
	if (sh->reconstruct_state == reconstruct_state_result) {
		struct stripe_head *sh_src
			= get_active_stripe(conf, sh->sector, 1, 1, 1);
		if (sh_src && test_bit(STRIPE_EXPAND_SOURCE, &sh_src->state)) {
			/* sh cannot be written until sh_src has been read.
			 * so arrange for sh to be delayed a little
			 */
			set_bit(STRIPE_DELAYED, &sh->state);
			set_bit(STRIPE_HANDLE, &sh->state);
			if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE,
					      &sh_src->state))
				atomic_inc(&conf->preread_active_stripes);
			release_stripe(sh_src);
			goto finish;
		}
		if (sh_src)
			release_stripe(sh_src);

		sh->reconstruct_state = reconstruct_state_idle;
		clear_bit(STRIPE_EXPANDING, &sh->state);
		for (i = conf->raid_disks; i--; ) {
			set_bit(R5_Wantwrite, &sh->dev[i].flags);
			set_bit(R5_LOCKED, &sh->dev[i].flags);
			s.locked++;
		}
	}
3565

3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577 3578 3579 3580 3581
	if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) &&
	    !sh->reconstruct_state) {
		/* Need to write out all blocks after computing parity */
		sh->disks = conf->raid_disks;
		stripe_set_idx(sh->sector, conf, 0, sh);
		schedule_reconstruction(sh, &s, 1, 1);
	} else if (s.expanded && !sh->reconstruct_state && s.locked == 0) {
		clear_bit(STRIPE_EXPAND_READY, &sh->state);
		atomic_dec(&conf->reshape_stripes);
		wake_up(&conf->wait_for_overlap);
		md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
	}

	if (s.expanding && s.locked == 0 &&
	    !test_bit(STRIPE_COMPUTE_RUN, &sh->state))
		handle_stripe_expansion(conf, sh);
3582

3583
finish:
3584
	/* wait for this device to become unblocked */
3585
	if (conf->mddev->external && unlikely(s.blocked_rdev))
3586
		md_wait_for_blocked_rdev(s.blocked_rdev, conf->mddev);
3587

3588 3589
	if (s.handle_bad_blocks)
		for (i = disks; i--; ) {
3590
			struct md_rdev *rdev;
3591 3592 3593 3594 3595 3596 3597 3598 3599
			struct r5dev *dev = &sh->dev[i];
			if (test_and_clear_bit(R5_WriteError, &dev->flags)) {
				/* We own a safe reference to the rdev */
				rdev = conf->disks[i].rdev;
				if (!rdev_set_badblocks(rdev, sh->sector,
							STRIPE_SECTORS, 0))
					md_error(conf->mddev, rdev);
				rdev_dec_pending(rdev, conf->mddev);
			}
3600 3601 3602
			if (test_and_clear_bit(R5_MadeGood, &dev->flags)) {
				rdev = conf->disks[i].rdev;
				rdev_clear_badblocks(rdev, sh->sector,
3603
						     STRIPE_SECTORS, 0);
3604 3605
				rdev_dec_pending(rdev, conf->mddev);
			}
3606 3607
			if (test_and_clear_bit(R5_MadeGoodRepl, &dev->flags)) {
				rdev = conf->disks[i].replacement;
3608 3609 3610
				if (!rdev)
					/* rdev have been moved down */
					rdev = conf->disks[i].rdev;
3611
				rdev_clear_badblocks(rdev, sh->sector,
3612
						     STRIPE_SECTORS, 0);
3613 3614
				rdev_dec_pending(rdev, conf->mddev);
			}
3615 3616
		}

3617 3618 3619
	if (s.ops_request)
		raid_run_ops(sh, s.ops_request);

D
Dan Williams 已提交
3620
	ops_run_io(sh, &s);
3621

3622
	if (s.dec_preread_active) {
3623
		/* We delay this until after ops_run_io so that if make_request
T
Tejun Heo 已提交
3624
		 * is waiting on a flush, it won't continue until the writes
3625 3626 3627 3628 3629 3630 3631 3632
		 * have actually been submitted.
		 */
		atomic_dec(&conf->preread_active_stripes);
		if (atomic_read(&conf->preread_active_stripes) <
		    IO_THRESHOLD)
			md_wakeup_thread(conf->mddev->thread);
	}

3633
	return_io(s.return_bi);
3634

3635
	clear_bit_unlock(STRIPE_ACTIVE, &sh->state);
3636 3637
}

3638
static void raid5_activate_delayed(struct r5conf *conf)
3639 3640 3641 3642 3643 3644 3645 3646 3647 3648
{
	if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) {
		while (!list_empty(&conf->delayed_list)) {
			struct list_head *l = conf->delayed_list.next;
			struct stripe_head *sh;
			sh = list_entry(l, struct stripe_head, lru);
			list_del_init(l);
			clear_bit(STRIPE_DELAYED, &sh->state);
			if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
				atomic_inc(&conf->preread_active_stripes);
3649
			list_add_tail(&sh->lru, &conf->hold_list);
3650
		}
N
NeilBrown 已提交
3651
	}
3652 3653
}

3654
static void activate_bit_delay(struct r5conf *conf)
3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667
{
	/* device_lock is held */
	struct list_head head;
	list_add(&head, &conf->bitmap_list);
	list_del_init(&conf->bitmap_list);
	while (!list_empty(&head)) {
		struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru);
		list_del_init(&sh->lru);
		atomic_inc(&sh->count);
		__release_stripe(conf, sh);
	}
}

3668
int md_raid5_congested(struct mddev *mddev, int bits)
3669
{
3670
	struct r5conf *conf = mddev->private;
3671 3672 3673 3674

	/* No difference between reads and writes.  Just check
	 * how busy the stripe_cache is
	 */
3675

3676 3677 3678 3679 3680 3681 3682 3683 3684
	if (conf->inactive_blocked)
		return 1;
	if (conf->quiesce)
		return 1;
	if (list_empty_careful(&conf->inactive_list))
		return 1;

	return 0;
}
N
NeilBrown 已提交
3685 3686 3687 3688
EXPORT_SYMBOL_GPL(md_raid5_congested);

static int raid5_congested(void *data, int bits)
{
3689
	struct mddev *mddev = data;
N
NeilBrown 已提交
3690 3691 3692 3693

	return mddev_congested(mddev, bits) ||
		md_raid5_congested(mddev, bits);
}
3694

3695 3696 3697
/* We want read requests to align with chunks where possible,
 * but write requests don't need to.
 */
3698 3699 3700
static int raid5_mergeable_bvec(struct request_queue *q,
				struct bvec_merge_data *bvm,
				struct bio_vec *biovec)
3701
{
3702
	struct mddev *mddev = q->queuedata;
3703
	sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev);
3704
	int max;
3705
	unsigned int chunk_sectors = mddev->chunk_sectors;
3706
	unsigned int bio_sectors = bvm->bi_size >> 9;
3707

3708
	if ((bvm->bi_rw & 1) == WRITE)
3709 3710
		return biovec->bv_len; /* always allow writes to be mergeable */

3711 3712
	if (mddev->new_chunk_sectors < mddev->chunk_sectors)
		chunk_sectors = mddev->new_chunk_sectors;
3713 3714 3715 3716 3717 3718 3719 3720
	max =  (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9;
	if (max < 0) max = 0;
	if (max <= biovec->bv_len && bio_sectors == 0)
		return biovec->bv_len;
	else
		return max;
}

3721

3722
static int in_chunk_boundary(struct mddev *mddev, struct bio *bio)
3723 3724
{
	sector_t sector = bio->bi_sector + get_start_sect(bio->bi_bdev);
3725
	unsigned int chunk_sectors = mddev->chunk_sectors;
3726 3727
	unsigned int bio_sectors = bio->bi_size >> 9;

3728 3729
	if (mddev->new_chunk_sectors < mddev->chunk_sectors)
		chunk_sectors = mddev->new_chunk_sectors;
3730 3731 3732 3733
	return  chunk_sectors >=
		((sector & (chunk_sectors - 1)) + bio_sectors);
}

3734 3735 3736 3737
/*
 *  add bio to the retry LIFO  ( in O(1) ... we are in interrupt )
 *  later sampled by raid5d.
 */
3738
static void add_bio_to_retry(struct bio *bi,struct r5conf *conf)
3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751
{
	unsigned long flags;

	spin_lock_irqsave(&conf->device_lock, flags);

	bi->bi_next = conf->retry_read_aligned_list;
	conf->retry_read_aligned_list = bi;

	spin_unlock_irqrestore(&conf->device_lock, flags);
	md_wakeup_thread(conf->mddev->thread);
}


3752
static struct bio *remove_bio_from_retry(struct r5conf *conf)
3753 3754 3755 3756 3757 3758 3759 3760 3761 3762
{
	struct bio *bi;

	bi = conf->retry_read_aligned;
	if (bi) {
		conf->retry_read_aligned = NULL;
		return bi;
	}
	bi = conf->retry_read_aligned_list;
	if(bi) {
3763
		conf->retry_read_aligned_list = bi->bi_next;
3764
		bi->bi_next = NULL;
3765 3766 3767 3768
		/*
		 * this sets the active strip count to 1 and the processed
		 * strip count to zero (upper 8 bits)
		 */
3769 3770 3771 3772 3773 3774 3775
		bi->bi_phys_segments = 1; /* biased count of active stripes */
	}

	return bi;
}


3776 3777 3778 3779 3780 3781
/*
 *  The "raid5_align_endio" should check if the read succeeded and if it
 *  did, call bio_endio on the original bio (having bio_put the new bio
 *  first).
 *  If the read failed..
 */
3782
static void raid5_align_endio(struct bio *bi, int error)
3783 3784
{
	struct bio* raid_bi  = bi->bi_private;
3785
	struct mddev *mddev;
3786
	struct r5conf *conf;
3787
	int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
3788
	struct md_rdev *rdev;
3789

3790
	bio_put(bi);
3791 3792 3793

	rdev = (void*)raid_bi->bi_next;
	raid_bi->bi_next = NULL;
3794 3795
	mddev = rdev->mddev;
	conf = mddev->private;
3796 3797 3798 3799

	rdev_dec_pending(rdev, conf->mddev);

	if (!error && uptodate) {
3800
		bio_endio(raid_bi, 0);
3801 3802
		if (atomic_dec_and_test(&conf->active_aligned_reads))
			wake_up(&conf->wait_for_stripe);
3803
		return;
3804 3805 3806
	}


3807
	pr_debug("raid5_align_endio : io error...handing IO for a retry\n");
3808 3809

	add_bio_to_retry(raid_bi, conf);
3810 3811
}

3812 3813
static int bio_fits_rdev(struct bio *bi)
{
3814
	struct request_queue *q = bdev_get_queue(bi->bi_bdev);
3815

3816
	if ((bi->bi_size>>9) > queue_max_sectors(q))
3817 3818
		return 0;
	blk_recount_segments(q, bi);
3819
	if (bi->bi_phys_segments > queue_max_segments(q))
3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831
		return 0;

	if (q->merge_bvec_fn)
		/* it's too hard to apply the merge_bvec_fn at this stage,
		 * just just give up
		 */
		return 0;

	return 1;
}


3832
static int chunk_aligned_read(struct mddev *mddev, struct bio * raid_bio)
3833
{
3834
	struct r5conf *conf = mddev->private;
N
NeilBrown 已提交
3835
	int dd_idx;
3836
	struct bio* align_bi;
3837
	struct md_rdev *rdev;
3838
	sector_t end_sector;
3839 3840

	if (!in_chunk_boundary(mddev, raid_bio)) {
3841
		pr_debug("chunk_aligned_read : non aligned\n");
3842 3843 3844
		return 0;
	}
	/*
3845
	 * use bio_clone_mddev to make a copy of the bio
3846
	 */
3847
	align_bi = bio_clone_mddev(raid_bio, GFP_NOIO, mddev);
3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858
	if (!align_bi)
		return 0;
	/*
	 *   set bi_end_io to a new function, and set bi_private to the
	 *     original bio.
	 */
	align_bi->bi_end_io  = raid5_align_endio;
	align_bi->bi_private = raid_bio;
	/*
	 *	compute position
	 */
3859 3860
	align_bi->bi_sector =  raid5_compute_sector(conf, raid_bio->bi_sector,
						    0,
3861
						    &dd_idx, NULL);
3862

3863
	end_sector = align_bi->bi_sector + (align_bi->bi_size >> 9);
3864
	rcu_read_lock();
3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875
	rdev = rcu_dereference(conf->disks[dd_idx].replacement);
	if (!rdev || test_bit(Faulty, &rdev->flags) ||
	    rdev->recovery_offset < end_sector) {
		rdev = rcu_dereference(conf->disks[dd_idx].rdev);
		if (rdev &&
		    (test_bit(Faulty, &rdev->flags) ||
		    !(test_bit(In_sync, &rdev->flags) ||
		      rdev->recovery_offset >= end_sector)))
			rdev = NULL;
	}
	if (rdev) {
3876 3877 3878
		sector_t first_bad;
		int bad_sectors;

3879 3880
		atomic_inc(&rdev->nr_pending);
		rcu_read_unlock();
3881 3882 3883
		raid_bio->bi_next = (void*)rdev;
		align_bi->bi_bdev =  rdev->bdev;
		align_bi->bi_flags &= ~(1 << BIO_SEG_VALID);
3884
		/* No reshape active, so we can trust rdev->data_offset */
3885 3886
		align_bi->bi_sector += rdev->data_offset;

3887 3888 3889 3890
		if (!bio_fits_rdev(align_bi) ||
		    is_badblock(rdev, align_bi->bi_sector, align_bi->bi_size>>9,
				&first_bad, &bad_sectors)) {
			/* too big in some way, or has a known bad block */
3891 3892 3893 3894 3895
			bio_put(align_bi);
			rdev_dec_pending(rdev, mddev);
			return 0;
		}

3896 3897 3898 3899 3900 3901 3902
		spin_lock_irq(&conf->device_lock);
		wait_event_lock_irq(conf->wait_for_stripe,
				    conf->quiesce == 0,
				    conf->device_lock, /* nothing */);
		atomic_inc(&conf->active_aligned_reads);
		spin_unlock_irq(&conf->device_lock);

3903 3904 3905 3906
		generic_make_request(align_bi);
		return 1;
	} else {
		rcu_read_unlock();
3907
		bio_put(align_bi);
3908 3909 3910 3911
		return 0;
	}
}

3912 3913 3914 3915 3916 3917 3918 3919 3920 3921
/* __get_priority_stripe - get the next stripe to process
 *
 * Full stripe writes are allowed to pass preread active stripes up until
 * the bypass_threshold is exceeded.  In general the bypass_count
 * increments when the handle_list is handled before the hold_list; however, it
 * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a
 * stripe with in flight i/o.  The bypass_count will be reset when the
 * head of the hold_list has changed, i.e. the head was promoted to the
 * handle_list.
 */
3922
static struct stripe_head *__get_priority_stripe(struct r5conf *conf)
3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963
{
	struct stripe_head *sh;

	pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n",
		  __func__,
		  list_empty(&conf->handle_list) ? "empty" : "busy",
		  list_empty(&conf->hold_list) ? "empty" : "busy",
		  atomic_read(&conf->pending_full_writes), conf->bypass_count);

	if (!list_empty(&conf->handle_list)) {
		sh = list_entry(conf->handle_list.next, typeof(*sh), lru);

		if (list_empty(&conf->hold_list))
			conf->bypass_count = 0;
		else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) {
			if (conf->hold_list.next == conf->last_hold)
				conf->bypass_count++;
			else {
				conf->last_hold = conf->hold_list.next;
				conf->bypass_count -= conf->bypass_threshold;
				if (conf->bypass_count < 0)
					conf->bypass_count = 0;
			}
		}
	} else if (!list_empty(&conf->hold_list) &&
		   ((conf->bypass_threshold &&
		     conf->bypass_count > conf->bypass_threshold) ||
		    atomic_read(&conf->pending_full_writes) == 0)) {
		sh = list_entry(conf->hold_list.next,
				typeof(*sh), lru);
		conf->bypass_count -= conf->bypass_threshold;
		if (conf->bypass_count < 0)
			conf->bypass_count = 0;
	} else
		return NULL;

	list_del_init(&sh->lru);
	atomic_inc(&sh->count);
	BUG_ON(atomic_read(&sh->count) != 1);
	return sh;
}
3964

3965
static void make_request(struct mddev *mddev, struct bio * bi)
L
Linus Torvalds 已提交
3966
{
3967
	struct r5conf *conf = mddev->private;
3968
	int dd_idx;
L
Linus Torvalds 已提交
3969 3970 3971
	sector_t new_sector;
	sector_t logical_sector, last_sector;
	struct stripe_head *sh;
3972
	const int rw = bio_data_dir(bi);
3973
	int remaining;
3974
	int plugged;
L
Linus Torvalds 已提交
3975

T
Tejun Heo 已提交
3976 3977
	if (unlikely(bi->bi_rw & REQ_FLUSH)) {
		md_flush_request(mddev, bi);
3978
		return;
3979 3980
	}

3981
	md_write_start(mddev, bi);
3982

3983
	if (rw == READ &&
3984
	     mddev->reshape_position == MaxSector &&
3985
	     chunk_aligned_read(mddev,bi))
3986
		return;
3987

L
Linus Torvalds 已提交
3988 3989 3990 3991
	logical_sector = bi->bi_sector & ~((sector_t)STRIPE_SECTORS-1);
	last_sector = bi->bi_sector + (bi->bi_size>>9);
	bi->bi_next = NULL;
	bi->bi_phys_segments = 1;	/* over-loaded to count active stripes */
3992

3993
	plugged = mddev_check_plugged(mddev);
L
Linus Torvalds 已提交
3994 3995
	for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) {
		DEFINE_WAIT(w);
3996
		int previous;
3997

3998
	retry:
3999
		previous = 0;
4000
		prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE);
4001
		if (unlikely(conf->reshape_progress != MaxSector)) {
4002
			/* spinlock is needed as reshape_progress may be
4003 4004
			 * 64bit on a 32bit platform, and so it might be
			 * possible to see a half-updated value
4005
			 * Of course reshape_progress could change after
4006 4007 4008 4009
			 * the lock is dropped, so once we get a reference
			 * to the stripe that we think it is, we will have
			 * to check again.
			 */
4010
			spin_lock_irq(&conf->device_lock);
4011
			if (mddev->reshape_backwards
4012 4013
			    ? logical_sector < conf->reshape_progress
			    : logical_sector >= conf->reshape_progress) {
4014 4015
				previous = 1;
			} else {
4016
				if (mddev->reshape_backwards
4017 4018
				    ? logical_sector < conf->reshape_safe
				    : logical_sector >= conf->reshape_safe) {
4019 4020 4021 4022 4023
					spin_unlock_irq(&conf->device_lock);
					schedule();
					goto retry;
				}
			}
4024 4025
			spin_unlock_irq(&conf->device_lock);
		}
4026

4027 4028
		new_sector = raid5_compute_sector(conf, logical_sector,
						  previous,
4029
						  &dd_idx, NULL);
4030
		pr_debug("raid456: make_request, sector %llu logical %llu\n",
L
Linus Torvalds 已提交
4031 4032 4033
			(unsigned long long)new_sector, 
			(unsigned long long)logical_sector);

4034
		sh = get_active_stripe(conf, new_sector, previous,
4035
				       (bi->bi_rw&RWA_MASK), 0);
L
Linus Torvalds 已提交
4036
		if (sh) {
4037
			if (unlikely(previous)) {
4038
				/* expansion might have moved on while waiting for a
4039 4040 4041 4042 4043 4044
				 * stripe, so we must do the range check again.
				 * Expansion could still move past after this
				 * test, but as we are holding a reference to
				 * 'sh', we know that if that happens,
				 *  STRIPE_EXPANDING will get set and the expansion
				 * won't proceed until we finish with the stripe.
4045 4046 4047
				 */
				int must_retry = 0;
				spin_lock_irq(&conf->device_lock);
4048
				if (mddev->reshape_backwards
4049 4050
				    ? logical_sector >= conf->reshape_progress
				    : logical_sector < conf->reshape_progress)
4051 4052 4053 4054 4055
					/* mismatch, need to try again */
					must_retry = 1;
				spin_unlock_irq(&conf->device_lock);
				if (must_retry) {
					release_stripe(sh);
4056
					schedule();
4057 4058 4059
					goto retry;
				}
			}
4060

4061
			if (rw == WRITE &&
4062
			    logical_sector >= mddev->suspend_lo &&
4063 4064
			    logical_sector < mddev->suspend_hi) {
				release_stripe(sh);
4065 4066 4067 4068 4069 4070 4071 4072 4073 4074
				/* As the suspend_* range is controlled by
				 * userspace, we want an interruptible
				 * wait.
				 */
				flush_signals(current);
				prepare_to_wait(&conf->wait_for_overlap,
						&w, TASK_INTERRUPTIBLE);
				if (logical_sector >= mddev->suspend_lo &&
				    logical_sector < mddev->suspend_hi)
					schedule();
4075 4076
				goto retry;
			}
4077 4078

			if (test_bit(STRIPE_EXPANDING, &sh->state) ||
4079
			    !add_stripe_bio(sh, bi, dd_idx, rw)) {
4080 4081
				/* Stripe is busy expanding or
				 * add failed due to overlap.  Flush everything
L
Linus Torvalds 已提交
4082 4083
				 * and wait a while
				 */
N
NeilBrown 已提交
4084
				md_wakeup_thread(mddev->thread);
L
Linus Torvalds 已提交
4085 4086 4087 4088 4089
				release_stripe(sh);
				schedule();
				goto retry;
			}
			finish_wait(&conf->wait_for_overlap, &w);
4090 4091
			set_bit(STRIPE_HANDLE, &sh->state);
			clear_bit(STRIPE_DELAYED, &sh->state);
T
Tejun Heo 已提交
4092
			if ((bi->bi_rw & REQ_SYNC) &&
4093 4094
			    !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
				atomic_inc(&conf->preread_active_stripes);
L
Linus Torvalds 已提交
4095 4096 4097 4098 4099 4100 4101 4102 4103
			release_stripe(sh);
		} else {
			/* cannot get stripe for read-ahead, just give-up */
			clear_bit(BIO_UPTODATE, &bi->bi_flags);
			finish_wait(&conf->wait_for_overlap, &w);
			break;
		}
			
	}
4104 4105 4106
	if (!plugged)
		md_wakeup_thread(mddev->thread);

L
Linus Torvalds 已提交
4107
	spin_lock_irq(&conf->device_lock);
4108
	remaining = raid5_dec_bi_phys_segments(bi);
4109 4110
	spin_unlock_irq(&conf->device_lock);
	if (remaining == 0) {
L
Linus Torvalds 已提交
4111

4112
		if ( rw == WRITE )
L
Linus Torvalds 已提交
4113
			md_write_end(mddev);
4114

4115
		bio_endio(bi, 0);
L
Linus Torvalds 已提交
4116 4117 4118
	}
}

4119
static sector_t raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks);
D
Dan Williams 已提交
4120

4121
static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *skipped)
L
Linus Torvalds 已提交
4122
{
4123 4124 4125 4126 4127 4128 4129 4130 4131
	/* reshaping is quite different to recovery/resync so it is
	 * handled quite separately ... here.
	 *
	 * On each call to sync_request, we gather one chunk worth of
	 * destination stripes and flag them as expanding.
	 * Then we find all the source stripes and request reads.
	 * As the reads complete, handle_stripe will copy the data
	 * into the destination stripe and release that stripe.
	 */
4132
	struct r5conf *conf = mddev->private;
L
Linus Torvalds 已提交
4133
	struct stripe_head *sh;
4134
	sector_t first_sector, last_sector;
4135 4136 4137
	int raid_disks = conf->previous_raid_disks;
	int data_disks = raid_disks - conf->max_degraded;
	int new_data_disks = conf->raid_disks - conf->max_degraded;
4138 4139
	int i;
	int dd_idx;
4140
	sector_t writepos, readpos, safepos;
4141
	sector_t stripe_addr;
4142
	int reshape_sectors;
4143
	struct list_head stripes;
4144

4145 4146
	if (sector_nr == 0) {
		/* If restarting in the middle, skip the initial sectors */
4147
		if (mddev->reshape_backwards &&
4148 4149 4150
		    conf->reshape_progress < raid5_size(mddev, 0, 0)) {
			sector_nr = raid5_size(mddev, 0, 0)
				- conf->reshape_progress;
4151
		} else if (!mddev->reshape_backwards &&
4152 4153
			   conf->reshape_progress > 0)
			sector_nr = conf->reshape_progress;
4154
		sector_div(sector_nr, new_data_disks);
4155
		if (sector_nr) {
4156 4157
			mddev->curr_resync_completed = sector_nr;
			sysfs_notify(&mddev->kobj, NULL, "sync_completed");
4158 4159 4160
			*skipped = 1;
			return sector_nr;
		}
4161 4162
	}

4163 4164 4165 4166
	/* We need to process a full chunk at a time.
	 * If old and new chunk sizes differ, we need to process the
	 * largest of these
	 */
4167 4168
	if (mddev->new_chunk_sectors > mddev->chunk_sectors)
		reshape_sectors = mddev->new_chunk_sectors;
4169
	else
4170
		reshape_sectors = mddev->chunk_sectors;
4171

4172 4173 4174 4175 4176
	/* We update the metadata at least every 10 seconds, or when
	 * the data about to be copied would over-write the source of
	 * the data at the front of the range.  i.e. one new_stripe
	 * along from reshape_progress new_maps to after where
	 * reshape_safe old_maps to
4177
	 */
4178
	writepos = conf->reshape_progress;
4179
	sector_div(writepos, new_data_disks);
4180 4181
	readpos = conf->reshape_progress;
	sector_div(readpos, data_disks);
4182
	safepos = conf->reshape_safe;
4183
	sector_div(safepos, data_disks);
4184
	if (mddev->reshape_backwards) {
4185
		writepos -= min_t(sector_t, reshape_sectors, writepos);
4186
		readpos += reshape_sectors;
4187
		safepos += reshape_sectors;
4188
	} else {
4189
		writepos += reshape_sectors;
4190 4191
		readpos -= min_t(sector_t, reshape_sectors, readpos);
		safepos -= min_t(sector_t, reshape_sectors, safepos);
4192
	}
4193

4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208
	/* Having calculated the 'writepos' possibly use it
	 * to set 'stripe_addr' which is where we will write to.
	 */
	if (mddev->reshape_backwards) {
		BUG_ON(conf->reshape_progress == 0);
		stripe_addr = writepos;
		BUG_ON((mddev->dev_sectors &
			~((sector_t)reshape_sectors - 1))
		       - reshape_sectors - stripe_addr
		       != sector_nr);
	} else {
		BUG_ON(writepos != sector_nr + reshape_sectors);
		stripe_addr = sector_nr;
	}

4209 4210 4211 4212
	/* 'writepos' is the most advanced device address we might write.
	 * 'readpos' is the least advanced device address we might read.
	 * 'safepos' is the least address recorded in the metadata as having
	 *     been reshaped.
4213 4214 4215 4216
	 * If there is a min_offset_diff, these are adjusted either by
	 * increasing the safepos/readpos if diff is negative, or
	 * increasing writepos if diff is positive.
	 * If 'readpos' is then behind 'writepos', there is no way that we can
4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228
	 * ensure safety in the face of a crash - that must be done by userspace
	 * making a backup of the data.  So in that case there is no particular
	 * rush to update metadata.
	 * Otherwise if 'safepos' is behind 'writepos', then we really need to
	 * update the metadata to advance 'safepos' to match 'readpos' so that
	 * we can be safe in the event of a crash.
	 * So we insist on updating metadata if safepos is behind writepos and
	 * readpos is beyond writepos.
	 * In any case, update the metadata every 10 seconds.
	 * Maybe that number should be configurable, but I'm not sure it is
	 * worth it.... maybe it could be a multiple of safemode_delay???
	 */
4229 4230 4231 4232 4233 4234
	if (conf->min_offset_diff < 0) {
		safepos += -conf->min_offset_diff;
		readpos += -conf->min_offset_diff;
	} else
		writepos += conf->min_offset_diff;

4235
	if ((mddev->reshape_backwards
4236 4237 4238
	     ? (safepos > writepos && readpos < writepos)
	     : (safepos < writepos && readpos > writepos)) ||
	    time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
4239 4240 4241
		/* Cannot proceed until we've updated the superblock... */
		wait_event(conf->wait_for_overlap,
			   atomic_read(&conf->reshape_stripes)==0);
4242
		mddev->reshape_position = conf->reshape_progress;
4243
		mddev->curr_resync_completed = sector_nr;
4244
		conf->reshape_checkpoint = jiffies;
4245
		set_bit(MD_CHANGE_DEVS, &mddev->flags);
4246
		md_wakeup_thread(mddev->thread);
4247
		wait_event(mddev->sb_wait, mddev->flags == 0 ||
4248 4249
			   kthread_should_stop());
		spin_lock_irq(&conf->device_lock);
4250
		conf->reshape_safe = mddev->reshape_position;
4251 4252
		spin_unlock_irq(&conf->device_lock);
		wake_up(&conf->wait_for_overlap);
4253
		sysfs_notify(&mddev->kobj, NULL, "sync_completed");
4254 4255
	}

4256
	INIT_LIST_HEAD(&stripes);
4257
	for (i = 0; i < reshape_sectors; i += STRIPE_SECTORS) {
4258
		int j;
4259
		int skipped_disk = 0;
4260
		sh = get_active_stripe(conf, stripe_addr+i, 0, 0, 1);
4261 4262 4263 4264 4265 4266 4267 4268 4269
		set_bit(STRIPE_EXPANDING, &sh->state);
		atomic_inc(&conf->reshape_stripes);
		/* If any of this stripe is beyond the end of the old
		 * array, then we need to zero those blocks
		 */
		for (j=sh->disks; j--;) {
			sector_t s;
			if (j == sh->pd_idx)
				continue;
4270
			if (conf->level == 6 &&
4271
			    j == sh->qd_idx)
4272
				continue;
4273
			s = compute_blocknr(sh, j, 0);
D
Dan Williams 已提交
4274
			if (s < raid5_size(mddev, 0, 0)) {
4275
				skipped_disk = 1;
4276 4277 4278 4279 4280 4281
				continue;
			}
			memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE);
			set_bit(R5_Expanded, &sh->dev[j].flags);
			set_bit(R5_UPTODATE, &sh->dev[j].flags);
		}
4282
		if (!skipped_disk) {
4283 4284 4285
			set_bit(STRIPE_EXPAND_READY, &sh->state);
			set_bit(STRIPE_HANDLE, &sh->state);
		}
4286
		list_add(&sh->lru, &stripes);
4287 4288
	}
	spin_lock_irq(&conf->device_lock);
4289
	if (mddev->reshape_backwards)
4290
		conf->reshape_progress -= reshape_sectors * new_data_disks;
4291
	else
4292
		conf->reshape_progress += reshape_sectors * new_data_disks;
4293 4294 4295 4296 4297 4298 4299
	spin_unlock_irq(&conf->device_lock);
	/* Ok, those stripe are ready. We can start scheduling
	 * reads on the source stripes.
	 * The source stripes are determined by mapping the first and last
	 * block on the destination stripes.
	 */
	first_sector =
4300
		raid5_compute_sector(conf, stripe_addr*(new_data_disks),
4301
				     1, &dd_idx, NULL);
4302
	last_sector =
4303
		raid5_compute_sector(conf, ((stripe_addr+reshape_sectors)
4304
					    * new_data_disks - 1),
4305
				     1, &dd_idx, NULL);
A
Andre Noll 已提交
4306 4307
	if (last_sector >= mddev->dev_sectors)
		last_sector = mddev->dev_sectors - 1;
4308
	while (first_sector <= last_sector) {
4309
		sh = get_active_stripe(conf, first_sector, 1, 0, 1);
4310 4311 4312 4313 4314
		set_bit(STRIPE_EXPAND_SOURCE, &sh->state);
		set_bit(STRIPE_HANDLE, &sh->state);
		release_stripe(sh);
		first_sector += STRIPE_SECTORS;
	}
4315 4316 4317 4318 4319 4320 4321 4322
	/* Now that the sources are clearly marked, we can release
	 * the destination stripes
	 */
	while (!list_empty(&stripes)) {
		sh = list_entry(stripes.next, struct stripe_head, lru);
		list_del_init(&sh->lru);
		release_stripe(sh);
	}
4323 4324 4325
	/* If this takes us to the resync_max point where we have to pause,
	 * then we need to write out the superblock.
	 */
4326
	sector_nr += reshape_sectors;
4327 4328
	if ((sector_nr - mddev->curr_resync_completed) * 2
	    >= mddev->resync_max - mddev->curr_resync_completed) {
4329 4330 4331
		/* Cannot proceed until we've updated the superblock... */
		wait_event(conf->wait_for_overlap,
			   atomic_read(&conf->reshape_stripes) == 0);
4332
		mddev->reshape_position = conf->reshape_progress;
4333
		mddev->curr_resync_completed = sector_nr;
4334
		conf->reshape_checkpoint = jiffies;
4335 4336 4337 4338 4339 4340
		set_bit(MD_CHANGE_DEVS, &mddev->flags);
		md_wakeup_thread(mddev->thread);
		wait_event(mddev->sb_wait,
			   !test_bit(MD_CHANGE_DEVS, &mddev->flags)
			   || kthread_should_stop());
		spin_lock_irq(&conf->device_lock);
4341
		conf->reshape_safe = mddev->reshape_position;
4342 4343
		spin_unlock_irq(&conf->device_lock);
		wake_up(&conf->wait_for_overlap);
4344
		sysfs_notify(&mddev->kobj, NULL, "sync_completed");
4345
	}
4346
	return reshape_sectors;
4347 4348 4349
}

/* FIXME go_faster isn't used */
4350
static inline sector_t sync_request(struct mddev *mddev, sector_t sector_nr, int *skipped, int go_faster)
4351
{
4352
	struct r5conf *conf = mddev->private;
4353
	struct stripe_head *sh;
A
Andre Noll 已提交
4354
	sector_t max_sector = mddev->dev_sectors;
N
NeilBrown 已提交
4355
	sector_t sync_blocks;
4356 4357
	int still_degraded = 0;
	int i;
L
Linus Torvalds 已提交
4358

4359
	if (sector_nr >= max_sector) {
L
Linus Torvalds 已提交
4360
		/* just being told to finish up .. nothing much to do */
4361

4362 4363 4364 4365
		if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {
			end_reshape(conf);
			return 0;
		}
4366 4367 4368 4369

		if (mddev->curr_resync < max_sector) /* aborted */
			bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
					&sync_blocks, 1);
4370
		else /* completed sync */
4371 4372 4373
			conf->fullsync = 0;
		bitmap_close_sync(mddev->bitmap);

L
Linus Torvalds 已提交
4374 4375
		return 0;
	}
4376

4377 4378 4379
	/* Allow raid5_quiesce to complete */
	wait_event(conf->wait_for_overlap, conf->quiesce != 2);

4380 4381
	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
		return reshape_request(mddev, sector_nr, skipped);
4382

4383 4384 4385 4386 4387 4388
	/* No need to check resync_max as we never do more than one
	 * stripe, and as resync_max will always be on a chunk boundary,
	 * if the check in md_do_sync didn't fire, there is no chance
	 * of overstepping resync_max here
	 */

4389
	/* if there is too many failed drives and we are trying
L
Linus Torvalds 已提交
4390 4391 4392
	 * to resync, then assert that we are finished, because there is
	 * nothing we can do.
	 */
4393
	if (mddev->degraded >= conf->max_degraded &&
4394
	    test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
A
Andre Noll 已提交
4395
		sector_t rv = mddev->dev_sectors - sector_nr;
4396
		*skipped = 1;
L
Linus Torvalds 已提交
4397 4398
		return rv;
	}
4399
	if (!bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) &&
4400
	    !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
4401 4402 4403 4404 4405 4406
	    !conf->fullsync && sync_blocks >= STRIPE_SECTORS) {
		/* we can skip this block, and probably more */
		sync_blocks /= STRIPE_SECTORS;
		*skipped = 1;
		return sync_blocks * STRIPE_SECTORS; /* keep things rounded to whole stripes */
	}
L
Linus Torvalds 已提交
4407

N
NeilBrown 已提交
4408 4409
	bitmap_cond_end_sync(mddev->bitmap, sector_nr);

4410
	sh = get_active_stripe(conf, sector_nr, 0, 1, 0);
L
Linus Torvalds 已提交
4411
	if (sh == NULL) {
4412
		sh = get_active_stripe(conf, sector_nr, 0, 0, 0);
L
Linus Torvalds 已提交
4413
		/* make sure we don't swamp the stripe cache if someone else
4414
		 * is trying to get access
L
Linus Torvalds 已提交
4415
		 */
4416
		schedule_timeout_uninterruptible(1);
L
Linus Torvalds 已提交
4417
	}
4418 4419 4420 4421
	/* Need to check if array will still be degraded after recovery/resync
	 * We don't need to check the 'failed' flag as when that gets set,
	 * recovery aborts.
	 */
4422
	for (i = 0; i < conf->raid_disks; i++)
4423 4424 4425 4426 4427
		if (conf->disks[i].rdev == NULL)
			still_degraded = 1;

	bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, still_degraded);

4428
	set_bit(STRIPE_SYNC_REQUESTED, &sh->state);
L
Linus Torvalds 已提交
4429

4430
	handle_stripe(sh);
L
Linus Torvalds 已提交
4431 4432 4433 4434 4435
	release_stripe(sh);

	return STRIPE_SECTORS;
}

4436
static int  retry_aligned_read(struct r5conf *conf, struct bio *raid_bio)
4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448
{
	/* We may not be able to submit a whole bio at once as there
	 * may not be enough stripe_heads available.
	 * We cannot pre-allocate enough stripe_heads as we may need
	 * more than exist in the cache (if we allow ever large chunks).
	 * So we do one stripe head at a time and record in
	 * ->bi_hw_segments how many have been done.
	 *
	 * We *know* that this entire raid_bio is in one chunk, so
	 * it will be only one 'dd_idx' and only need one call to raid5_compute_sector.
	 */
	struct stripe_head *sh;
4449
	int dd_idx;
4450 4451 4452 4453 4454 4455
	sector_t sector, logical_sector, last_sector;
	int scnt = 0;
	int remaining;
	int handled = 0;

	logical_sector = raid_bio->bi_sector & ~((sector_t)STRIPE_SECTORS-1);
4456
	sector = raid5_compute_sector(conf, logical_sector,
4457
				      0, &dd_idx, NULL);
4458 4459 4460
	last_sector = raid_bio->bi_sector + (raid_bio->bi_size>>9);

	for (; logical_sector < last_sector;
4461 4462 4463
	     logical_sector += STRIPE_SECTORS,
		     sector += STRIPE_SECTORS,
		     scnt++) {
4464

4465
		if (scnt < raid5_bi_hw_segments(raid_bio))
4466 4467 4468
			/* already done this stripe */
			continue;

4469
		sh = get_active_stripe(conf, sector, 0, 1, 0);
4470 4471 4472

		if (!sh) {
			/* failed to get a stripe - must wait */
4473
			raid5_set_bi_hw_segments(raid_bio, scnt);
4474 4475 4476 4477
			conf->retry_read_aligned = raid_bio;
			return handled;
		}

4478 4479
		if (!add_stripe_bio(sh, raid_bio, dd_idx, 0)) {
			release_stripe(sh);
4480
			raid5_set_bi_hw_segments(raid_bio, scnt);
4481 4482 4483 4484
			conf->retry_read_aligned = raid_bio;
			return handled;
		}

4485
		handle_stripe(sh);
4486 4487 4488 4489
		release_stripe(sh);
		handled++;
	}
	spin_lock_irq(&conf->device_lock);
4490
	remaining = raid5_dec_bi_phys_segments(raid_bio);
4491
	spin_unlock_irq(&conf->device_lock);
4492 4493
	if (remaining == 0)
		bio_endio(raid_bio, 0);
4494 4495 4496 4497 4498 4499
	if (atomic_dec_and_test(&conf->active_aligned_reads))
		wake_up(&conf->wait_for_stripe);
	return handled;
}


L
Linus Torvalds 已提交
4500 4501 4502 4503 4504 4505 4506
/*
 * This is our raid5 kernel thread.
 *
 * We scan the hash table for stripes which can be handled now.
 * During the scan, completed stripes are saved for us by the interrupt
 * handler, so that they will not have to wait for our next wakeup.
 */
4507
static void raid5d(struct mddev *mddev)
L
Linus Torvalds 已提交
4508 4509
{
	struct stripe_head *sh;
4510
	struct r5conf *conf = mddev->private;
L
Linus Torvalds 已提交
4511
	int handled;
4512
	struct blk_plug plug;
L
Linus Torvalds 已提交
4513

4514
	pr_debug("+++ raid5d active\n");
L
Linus Torvalds 已提交
4515 4516 4517

	md_check_recovery(mddev);

4518
	blk_start_plug(&plug);
L
Linus Torvalds 已提交
4519 4520 4521
	handled = 0;
	spin_lock_irq(&conf->device_lock);
	while (1) {
4522
		struct bio *bio;
L
Linus Torvalds 已提交
4523

4524 4525 4526 4527
		if (atomic_read(&mddev->plug_cnt) == 0 &&
		    !list_empty(&conf->bitmap_list)) {
			/* Now is a good time to flush some bitmap updates */
			conf->seq_flush++;
4528
			spin_unlock_irq(&conf->device_lock);
4529
			bitmap_unplug(mddev->bitmap);
4530
			spin_lock_irq(&conf->device_lock);
4531
			conf->seq_write = conf->seq_flush;
4532 4533
			activate_bit_delay(conf);
		}
4534 4535
		if (atomic_read(&mddev->plug_cnt) == 0)
			raid5_activate_delayed(conf);
4536

4537 4538 4539 4540 4541 4542 4543 4544 4545 4546
		while ((bio = remove_bio_from_retry(conf))) {
			int ok;
			spin_unlock_irq(&conf->device_lock);
			ok = retry_aligned_read(conf, bio);
			spin_lock_irq(&conf->device_lock);
			if (!ok)
				break;
			handled++;
		}

4547 4548
		sh = __get_priority_stripe(conf);

4549
		if (!sh)
L
Linus Torvalds 已提交
4550 4551 4552 4553
			break;
		spin_unlock_irq(&conf->device_lock);
		
		handled++;
4554 4555 4556
		handle_stripe(sh);
		release_stripe(sh);
		cond_resched();
L
Linus Torvalds 已提交
4557

4558 4559 4560
		if (mddev->flags & ~(1<<MD_CHANGE_PENDING))
			md_check_recovery(mddev);

L
Linus Torvalds 已提交
4561 4562
		spin_lock_irq(&conf->device_lock);
	}
4563
	pr_debug("%d stripes handled\n", handled);
L
Linus Torvalds 已提交
4564 4565 4566

	spin_unlock_irq(&conf->device_lock);

4567
	async_tx_issue_pending_all();
4568
	blk_finish_plug(&plug);
L
Linus Torvalds 已提交
4569

4570
	pr_debug("--- raid5d inactive\n");
L
Linus Torvalds 已提交
4571 4572
}

4573
static ssize_t
4574
raid5_show_stripe_cache_size(struct mddev *mddev, char *page)
4575
{
4576
	struct r5conf *conf = mddev->private;
4577 4578 4579 4580
	if (conf)
		return sprintf(page, "%d\n", conf->max_nr_stripes);
	else
		return 0;
4581 4582
}

4583
int
4584
raid5_set_cache_size(struct mddev *mddev, int size)
4585
{
4586
	struct r5conf *conf = mddev->private;
4587 4588
	int err;

4589
	if (size <= 16 || size > 32768)
4590
		return -EINVAL;
4591
	while (size < conf->max_nr_stripes) {
4592 4593 4594 4595 4596
		if (drop_one_stripe(conf))
			conf->max_nr_stripes--;
		else
			break;
	}
4597 4598 4599
	err = md_allow_write(mddev);
	if (err)
		return err;
4600
	while (size > conf->max_nr_stripes) {
4601 4602 4603 4604
		if (grow_one_stripe(conf))
			conf->max_nr_stripes++;
		else break;
	}
4605 4606 4607 4608 4609
	return 0;
}
EXPORT_SYMBOL(raid5_set_cache_size);

static ssize_t
4610
raid5_store_stripe_cache_size(struct mddev *mddev, const char *page, size_t len)
4611
{
4612
	struct r5conf *conf = mddev->private;
4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625
	unsigned long new;
	int err;

	if (len >= PAGE_SIZE)
		return -EINVAL;
	if (!conf)
		return -ENODEV;

	if (strict_strtoul(page, 10, &new))
		return -EINVAL;
	err = raid5_set_cache_size(mddev, new);
	if (err)
		return err;
4626 4627
	return len;
}
4628

4629 4630 4631 4632
static struct md_sysfs_entry
raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR,
				raid5_show_stripe_cache_size,
				raid5_store_stripe_cache_size);
4633

4634
static ssize_t
4635
raid5_show_preread_threshold(struct mddev *mddev, char *page)
4636
{
4637
	struct r5conf *conf = mddev->private;
4638 4639 4640 4641 4642 4643 4644
	if (conf)
		return sprintf(page, "%d\n", conf->bypass_threshold);
	else
		return 0;
}

static ssize_t
4645
raid5_store_preread_threshold(struct mddev *mddev, const char *page, size_t len)
4646
{
4647
	struct r5conf *conf = mddev->private;
4648
	unsigned long new;
4649 4650 4651 4652 4653
	if (len >= PAGE_SIZE)
		return -EINVAL;
	if (!conf)
		return -ENODEV;

4654
	if (strict_strtoul(page, 10, &new))
4655
		return -EINVAL;
4656
	if (new > conf->max_nr_stripes)
4657 4658 4659 4660 4661 4662 4663 4664 4665 4666 4667
		return -EINVAL;
	conf->bypass_threshold = new;
	return len;
}

static struct md_sysfs_entry
raid5_preread_bypass_threshold = __ATTR(preread_bypass_threshold,
					S_IRUGO | S_IWUSR,
					raid5_show_preread_threshold,
					raid5_store_preread_threshold);

4668
static ssize_t
4669
stripe_cache_active_show(struct mddev *mddev, char *page)
4670
{
4671
	struct r5conf *conf = mddev->private;
4672 4673 4674 4675
	if (conf)
		return sprintf(page, "%d\n", atomic_read(&conf->active_stripes));
	else
		return 0;
4676 4677
}

4678 4679
static struct md_sysfs_entry
raid5_stripecache_active = __ATTR_RO(stripe_cache_active);
4680

4681
static struct attribute *raid5_attrs[] =  {
4682 4683
	&raid5_stripecache_size.attr,
	&raid5_stripecache_active.attr,
4684
	&raid5_preread_bypass_threshold.attr,
4685 4686
	NULL,
};
4687 4688 4689
static struct attribute_group raid5_attrs_group = {
	.name = NULL,
	.attrs = raid5_attrs,
4690 4691
};

4692
static sector_t
4693
raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks)
4694
{
4695
	struct r5conf *conf = mddev->private;
4696 4697 4698

	if (!sectors)
		sectors = mddev->dev_sectors;
4699
	if (!raid_disks)
4700
		/* size is defined by the smallest of previous and new size */
4701
		raid_disks = min(conf->raid_disks, conf->previous_raid_disks);
4702

4703
	sectors &= ~((sector_t)mddev->chunk_sectors - 1);
4704
	sectors &= ~((sector_t)mddev->new_chunk_sectors - 1);
4705 4706 4707
	return sectors * (raid_disks - conf->max_degraded);
}

4708
static void raid5_free_percpu(struct r5conf *conf)
4709 4710 4711 4712 4713 4714 4715 4716 4717 4718 4719
{
	struct raid5_percpu *percpu;
	unsigned long cpu;

	if (!conf->percpu)
		return;

	get_online_cpus();
	for_each_possible_cpu(cpu) {
		percpu = per_cpu_ptr(conf->percpu, cpu);
		safe_put_page(percpu->spare_page);
4720
		kfree(percpu->scribble);
4721 4722 4723 4724 4725 4726 4727 4728 4729
	}
#ifdef CONFIG_HOTPLUG_CPU
	unregister_cpu_notifier(&conf->cpu_notify);
#endif
	put_online_cpus();

	free_percpu(conf->percpu);
}

4730
static void free_conf(struct r5conf *conf)
4731 4732
{
	shrink_stripes(conf);
4733
	raid5_free_percpu(conf);
4734 4735 4736 4737 4738
	kfree(conf->disks);
	kfree(conf->stripe_hashtbl);
	kfree(conf);
}

4739 4740 4741 4742
#ifdef CONFIG_HOTPLUG_CPU
static int raid456_cpu_notify(struct notifier_block *nfb, unsigned long action,
			      void *hcpu)
{
4743
	struct r5conf *conf = container_of(nfb, struct r5conf, cpu_notify);
4744 4745 4746 4747 4748 4749
	long cpu = (long)hcpu;
	struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu);

	switch (action) {
	case CPU_UP_PREPARE:
	case CPU_UP_PREPARE_FROZEN:
4750
		if (conf->level == 6 && !percpu->spare_page)
4751
			percpu->spare_page = alloc_page(GFP_KERNEL);
4752 4753 4754 4755 4756 4757 4758
		if (!percpu->scribble)
			percpu->scribble = kmalloc(conf->scribble_len, GFP_KERNEL);

		if (!percpu->scribble ||
		    (conf->level == 6 && !percpu->spare_page)) {
			safe_put_page(percpu->spare_page);
			kfree(percpu->scribble);
4759 4760
			pr_err("%s: failed memory allocation for cpu%ld\n",
			       __func__, cpu);
4761
			return notifier_from_errno(-ENOMEM);
4762 4763 4764 4765 4766
		}
		break;
	case CPU_DEAD:
	case CPU_DEAD_FROZEN:
		safe_put_page(percpu->spare_page);
4767
		kfree(percpu->scribble);
4768
		percpu->spare_page = NULL;
4769
		percpu->scribble = NULL;
4770 4771 4772 4773 4774 4775 4776 4777
		break;
	default:
		break;
	}
	return NOTIFY_OK;
}
#endif

4778
static int raid5_alloc_percpu(struct r5conf *conf)
4779 4780 4781
{
	unsigned long cpu;
	struct page *spare_page;
4782
	struct raid5_percpu __percpu *allcpus;
4783
	void *scribble;
4784 4785 4786 4787 4788 4789 4790 4791 4792 4793
	int err;

	allcpus = alloc_percpu(struct raid5_percpu);
	if (!allcpus)
		return -ENOMEM;
	conf->percpu = allcpus;

	get_online_cpus();
	err = 0;
	for_each_present_cpu(cpu) {
4794 4795 4796 4797 4798 4799 4800 4801
		if (conf->level == 6) {
			spare_page = alloc_page(GFP_KERNEL);
			if (!spare_page) {
				err = -ENOMEM;
				break;
			}
			per_cpu_ptr(conf->percpu, cpu)->spare_page = spare_page;
		}
4802
		scribble = kmalloc(conf->scribble_len, GFP_KERNEL);
4803
		if (!scribble) {
4804 4805 4806
			err = -ENOMEM;
			break;
		}
4807
		per_cpu_ptr(conf->percpu, cpu)->scribble = scribble;
4808 4809 4810 4811 4812 4813 4814 4815 4816 4817 4818 4819
	}
#ifdef CONFIG_HOTPLUG_CPU
	conf->cpu_notify.notifier_call = raid456_cpu_notify;
	conf->cpu_notify.priority = 0;
	if (err == 0)
		err = register_cpu_notifier(&conf->cpu_notify);
#endif
	put_online_cpus();

	return err;
}

4820
static struct r5conf *setup_conf(struct mddev *mddev)
L
Linus Torvalds 已提交
4821
{
4822
	struct r5conf *conf;
4823
	int raid_disk, memory, max_disks;
4824
	struct md_rdev *rdev;
L
Linus Torvalds 已提交
4825 4826
	struct disk_info *disk;

N
NeilBrown 已提交
4827 4828 4829
	if (mddev->new_level != 5
	    && mddev->new_level != 4
	    && mddev->new_level != 6) {
4830
		printk(KERN_ERR "md/raid:%s: raid level not set to 4/5/6 (%d)\n",
N
NeilBrown 已提交
4831 4832
		       mdname(mddev), mddev->new_level);
		return ERR_PTR(-EIO);
L
Linus Torvalds 已提交
4833
	}
N
NeilBrown 已提交
4834 4835 4836 4837
	if ((mddev->new_level == 5
	     && !algorithm_valid_raid5(mddev->new_layout)) ||
	    (mddev->new_level == 6
	     && !algorithm_valid_raid6(mddev->new_layout))) {
4838
		printk(KERN_ERR "md/raid:%s: layout %d not supported\n",
N
NeilBrown 已提交
4839 4840
		       mdname(mddev), mddev->new_layout);
		return ERR_PTR(-EIO);
4841
	}
N
NeilBrown 已提交
4842
	if (mddev->new_level == 6 && mddev->raid_disks < 4) {
4843
		printk(KERN_ERR "md/raid:%s: not enough configured devices (%d, minimum 4)\n",
N
NeilBrown 已提交
4844 4845
		       mdname(mddev), mddev->raid_disks);
		return ERR_PTR(-EINVAL);
4846 4847
	}

4848 4849 4850
	if (!mddev->new_chunk_sectors ||
	    (mddev->new_chunk_sectors << 9) % PAGE_SIZE ||
	    !is_power_of_2(mddev->new_chunk_sectors)) {
4851 4852
		printk(KERN_ERR "md/raid:%s: invalid chunk size %d\n",
		       mdname(mddev), mddev->new_chunk_sectors << 9);
N
NeilBrown 已提交
4853
		return ERR_PTR(-EINVAL);
4854 4855
	}

4856
	conf = kzalloc(sizeof(struct r5conf), GFP_KERNEL);
N
NeilBrown 已提交
4857
	if (conf == NULL)
L
Linus Torvalds 已提交
4858
		goto abort;
4859 4860 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870
	spin_lock_init(&conf->device_lock);
	init_waitqueue_head(&conf->wait_for_stripe);
	init_waitqueue_head(&conf->wait_for_overlap);
	INIT_LIST_HEAD(&conf->handle_list);
	INIT_LIST_HEAD(&conf->hold_list);
	INIT_LIST_HEAD(&conf->delayed_list);
	INIT_LIST_HEAD(&conf->bitmap_list);
	INIT_LIST_HEAD(&conf->inactive_list);
	atomic_set(&conf->active_stripes, 0);
	atomic_set(&conf->preread_active_stripes, 0);
	atomic_set(&conf->active_aligned_reads, 0);
	conf->bypass_threshold = BYPASS_THRESHOLD;
4871
	conf->recovery_disabled = mddev->recovery_disabled - 1;
N
NeilBrown 已提交
4872 4873 4874 4875 4876

	conf->raid_disks = mddev->raid_disks;
	if (mddev->reshape_position == MaxSector)
		conf->previous_raid_disks = mddev->raid_disks;
	else
4877
		conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks;
4878 4879
	max_disks = max(conf->raid_disks, conf->previous_raid_disks);
	conf->scribble_len = scribble_len(max_disks);
4880

4881
	conf->disks = kzalloc(max_disks * sizeof(struct disk_info),
4882 4883 4884
			      GFP_KERNEL);
	if (!conf->disks)
		goto abort;
4885

L
Linus Torvalds 已提交
4886 4887
	conf->mddev = mddev;

4888
	if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL)
L
Linus Torvalds 已提交
4889 4890
		goto abort;

4891 4892 4893 4894
	conf->level = mddev->new_level;
	if (raid5_alloc_percpu(conf) != 0)
		goto abort;

4895
	pr_debug("raid456: run(%s) called.\n", mdname(mddev));
L
Linus Torvalds 已提交
4896

N
NeilBrown 已提交
4897
	rdev_for_each(rdev, mddev) {
L
Linus Torvalds 已提交
4898
		raid_disk = rdev->raid_disk;
4899
		if (raid_disk >= max_disks
L
Linus Torvalds 已提交
4900 4901 4902 4903
		    || raid_disk < 0)
			continue;
		disk = conf->disks + raid_disk;

4904 4905 4906 4907 4908 4909 4910 4911 4912
		if (test_bit(Replacement, &rdev->flags)) {
			if (disk->replacement)
				goto abort;
			disk->replacement = rdev;
		} else {
			if (disk->rdev)
				goto abort;
			disk->rdev = rdev;
		}
L
Linus Torvalds 已提交
4913

4914
		if (test_bit(In_sync, &rdev->flags)) {
L
Linus Torvalds 已提交
4915
			char b[BDEVNAME_SIZE];
4916 4917 4918
			printk(KERN_INFO "md/raid:%s: device %s operational as raid"
			       " disk %d\n",
			       mdname(mddev), bdevname(rdev->bdev, b), raid_disk);
J
Jonathan Brassow 已提交
4919
		} else if (rdev->saved_raid_disk != raid_disk)
4920 4921
			/* Cannot rely on bitmap to complete recovery */
			conf->fullsync = 1;
L
Linus Torvalds 已提交
4922 4923
	}

4924
	conf->chunk_sectors = mddev->new_chunk_sectors;
N
NeilBrown 已提交
4925
	conf->level = mddev->new_level;
4926 4927 4928 4929
	if (conf->level == 6)
		conf->max_degraded = 2;
	else
		conf->max_degraded = 1;
N
NeilBrown 已提交
4930
	conf->algorithm = mddev->new_layout;
L
Linus Torvalds 已提交
4931
	conf->max_nr_stripes = NR_STRIPES;
4932
	conf->reshape_progress = mddev->reshape_position;
4933
	if (conf->reshape_progress != MaxSector) {
4934
		conf->prev_chunk_sectors = mddev->chunk_sectors;
4935 4936
		conf->prev_algo = mddev->layout;
	}
L
Linus Torvalds 已提交
4937

N
NeilBrown 已提交
4938
	memory = conf->max_nr_stripes * (sizeof(struct stripe_head) +
4939
		 max_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024;
N
NeilBrown 已提交
4940 4941
	if (grow_stripes(conf, conf->max_nr_stripes)) {
		printk(KERN_ERR
4942 4943
		       "md/raid:%s: couldn't allocate %dkB for buffers\n",
		       mdname(mddev), memory);
N
NeilBrown 已提交
4944 4945
		goto abort;
	} else
4946 4947
		printk(KERN_INFO "md/raid:%s: allocated %dkB\n",
		       mdname(mddev), memory);
L
Linus Torvalds 已提交
4948

4949
	conf->thread = md_register_thread(raid5d, mddev, NULL);
N
NeilBrown 已提交
4950 4951
	if (!conf->thread) {
		printk(KERN_ERR
4952
		       "md/raid:%s: couldn't allocate thread.\n",
N
NeilBrown 已提交
4953
		       mdname(mddev));
4954 4955
		goto abort;
	}
N
NeilBrown 已提交
4956 4957 4958 4959 4960

	return conf;

 abort:
	if (conf) {
4961
		free_conf(conf);
N
NeilBrown 已提交
4962 4963 4964 4965 4966
		return ERR_PTR(-EIO);
	} else
		return ERR_PTR(-ENOMEM);
}

4967 4968 4969 4970 4971 4972 4973 4974 4975 4976 4977 4978 4979 4980 4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991 4992 4993

static int only_parity(int raid_disk, int algo, int raid_disks, int max_degraded)
{
	switch (algo) {
	case ALGORITHM_PARITY_0:
		if (raid_disk < max_degraded)
			return 1;
		break;
	case ALGORITHM_PARITY_N:
		if (raid_disk >= raid_disks - max_degraded)
			return 1;
		break;
	case ALGORITHM_PARITY_0_6:
		if (raid_disk == 0 || 
		    raid_disk == raid_disks - 1)
			return 1;
		break;
	case ALGORITHM_LEFT_ASYMMETRIC_6:
	case ALGORITHM_RIGHT_ASYMMETRIC_6:
	case ALGORITHM_LEFT_SYMMETRIC_6:
	case ALGORITHM_RIGHT_SYMMETRIC_6:
		if (raid_disk == raid_disks - 1)
			return 1;
	}
	return 0;
}

4994
static int run(struct mddev *mddev)
N
NeilBrown 已提交
4995
{
4996
	struct r5conf *conf;
4997
	int working_disks = 0;
4998
	int dirty_parity_disks = 0;
4999
	struct md_rdev *rdev;
5000
	sector_t reshape_offset = 0;
5001
	int i;
5002 5003
	long long min_offset_diff = 0;
	int first = 1;
N
NeilBrown 已提交
5004

5005
	if (mddev->recovery_cp != MaxSector)
5006
		printk(KERN_NOTICE "md/raid:%s: not clean"
5007 5008
		       " -- starting background reconstruction\n",
		       mdname(mddev));
5009 5010 5011 5012 5013 5014 5015 5016 5017 5018 5019 5020 5021 5022 5023 5024 5025

	rdev_for_each(rdev, mddev) {
		long long diff;
		if (rdev->raid_disk < 0)
			continue;
		diff = (rdev->new_data_offset - rdev->data_offset);
		if (first) {
			min_offset_diff = diff;
			first = 0;
		} else if (mddev->reshape_backwards &&
			 diff < min_offset_diff)
			min_offset_diff = diff;
		else if (!mddev->reshape_backwards &&
			 diff > min_offset_diff)
			min_offset_diff = diff;
	}

N
NeilBrown 已提交
5026 5027
	if (mddev->reshape_position != MaxSector) {
		/* Check that we can continue the reshape.
5028 5029 5030 5031 5032 5033 5034 5035 5036 5037
		 * Difficulties arise if the stripe we would write to
		 * next is at or after the stripe we would read from next.
		 * For a reshape that changes the number of devices, this
		 * is only possible for a very short time, and mdadm makes
		 * sure that time appears to have past before assembling
		 * the array.  So we fail if that time hasn't passed.
		 * For a reshape that keeps the number of devices the same
		 * mdadm must be monitoring the reshape can keeping the
		 * critical areas read-only and backed up.  It will start
		 * the array in read-only mode, so we check for that.
N
NeilBrown 已提交
5038 5039 5040
		 */
		sector_t here_new, here_old;
		int old_disks;
5041
		int max_degraded = (mddev->level == 6 ? 2 : 1);
N
NeilBrown 已提交
5042

5043
		if (mddev->new_level != mddev->level) {
5044
			printk(KERN_ERR "md/raid:%s: unsupported reshape "
N
NeilBrown 已提交
5045 5046 5047 5048 5049 5050 5051 5052 5053 5054
			       "required - aborting.\n",
			       mdname(mddev));
			return -EINVAL;
		}
		old_disks = mddev->raid_disks - mddev->delta_disks;
		/* reshape_position must be on a new-stripe boundary, and one
		 * further up in new geometry must map after here in old
		 * geometry.
		 */
		here_new = mddev->reshape_position;
5055
		if (sector_div(here_new, mddev->new_chunk_sectors *
N
NeilBrown 已提交
5056
			       (mddev->raid_disks - max_degraded))) {
5057 5058
			printk(KERN_ERR "md/raid:%s: reshape_position not "
			       "on a stripe boundary\n", mdname(mddev));
N
NeilBrown 已提交
5059 5060
			return -EINVAL;
		}
5061
		reshape_offset = here_new * mddev->new_chunk_sectors;
N
NeilBrown 已提交
5062 5063
		/* here_new is the stripe we will write to */
		here_old = mddev->reshape_position;
5064
		sector_div(here_old, mddev->chunk_sectors *
N
NeilBrown 已提交
5065 5066 5067
			   (old_disks-max_degraded));
		/* here_old is the first stripe that we might need to read
		 * from */
5068
		if (mddev->delta_disks == 0) {
5069 5070 5071 5072 5073 5074
			if ((here_new * mddev->new_chunk_sectors !=
			     here_old * mddev->chunk_sectors)) {
				printk(KERN_ERR "md/raid:%s: reshape position is"
				       " confused - aborting\n", mdname(mddev));
				return -EINVAL;
			}
5075
			/* We cannot be sure it is safe to start an in-place
5076
			 * reshape.  It is only safe if user-space is monitoring
5077 5078 5079 5080 5081
			 * and taking constant backups.
			 * mdadm always starts a situation like this in
			 * readonly mode so it can take control before
			 * allowing any writes.  So just check for that.
			 */
5082 5083 5084 5085 5086 5087 5088
			if (abs(min_offset_diff) >= mddev->chunk_sectors &&
			    abs(min_offset_diff) >= mddev->new_chunk_sectors)
				/* not really in-place - so OK */;
			else if (mddev->ro == 0) {
				printk(KERN_ERR "md/raid:%s: in-place reshape "
				       "must be started in read-only mode "
				       "- aborting\n",
5089
				       mdname(mddev));
5090 5091
				return -EINVAL;
			}
5092
		} else if (mddev->reshape_backwards
5093
		    ? (here_new * mddev->new_chunk_sectors + min_offset_diff <=
5094 5095
		       here_old * mddev->chunk_sectors)
		    : (here_new * mddev->new_chunk_sectors >=
5096
		       here_old * mddev->chunk_sectors + (-min_offset_diff))) {
N
NeilBrown 已提交
5097
			/* Reading from the same stripe as writing to - bad */
5098 5099 5100
			printk(KERN_ERR "md/raid:%s: reshape_position too early for "
			       "auto-recovery - aborting.\n",
			       mdname(mddev));
N
NeilBrown 已提交
5101 5102
			return -EINVAL;
		}
5103 5104
		printk(KERN_INFO "md/raid:%s: reshape will continue\n",
		       mdname(mddev));
N
NeilBrown 已提交
5105 5106 5107 5108
		/* OK, we should be able to continue; */
	} else {
		BUG_ON(mddev->level != mddev->new_level);
		BUG_ON(mddev->layout != mddev->new_layout);
5109
		BUG_ON(mddev->chunk_sectors != mddev->new_chunk_sectors);
N
NeilBrown 已提交
5110
		BUG_ON(mddev->delta_disks != 0);
L
Linus Torvalds 已提交
5111
	}
N
NeilBrown 已提交
5112

5113 5114 5115 5116 5117
	if (mddev->private == NULL)
		conf = setup_conf(mddev);
	else
		conf = mddev->private;

N
NeilBrown 已提交
5118 5119 5120
	if (IS_ERR(conf))
		return PTR_ERR(conf);

5121
	conf->min_offset_diff = min_offset_diff;
N
NeilBrown 已提交
5122 5123 5124 5125
	mddev->thread = conf->thread;
	conf->thread = NULL;
	mddev->private = conf;

5126 5127 5128 5129 5130 5131 5132 5133 5134 5135 5136
	for (i = 0; i < conf->raid_disks && conf->previous_raid_disks;
	     i++) {
		rdev = conf->disks[i].rdev;
		if (!rdev && conf->disks[i].replacement) {
			/* The replacement is all we have yet */
			rdev = conf->disks[i].replacement;
			conf->disks[i].replacement = NULL;
			clear_bit(Replacement, &rdev->flags);
			conf->disks[i].rdev = rdev;
		}
		if (!rdev)
5137
			continue;
5138 5139 5140 5141 5142 5143 5144
		if (conf->disks[i].replacement &&
		    conf->reshape_progress != MaxSector) {
			/* replacements and reshape simply do not mix. */
			printk(KERN_ERR "md: cannot handle concurrent "
			       "replacement and reshape.\n");
			goto abort;
		}
5145
		if (test_bit(In_sync, &rdev->flags)) {
N
NeilBrown 已提交
5146
			working_disks++;
5147 5148
			continue;
		}
5149 5150 5151 5152 5153 5154 5155 5156 5157 5158 5159 5160 5161 5162 5163 5164 5165 5166 5167 5168 5169 5170 5171 5172 5173 5174 5175 5176
		/* This disc is not fully in-sync.  However if it
		 * just stored parity (beyond the recovery_offset),
		 * when we don't need to be concerned about the
		 * array being dirty.
		 * When reshape goes 'backwards', we never have
		 * partially completed devices, so we only need
		 * to worry about reshape going forwards.
		 */
		/* Hack because v0.91 doesn't store recovery_offset properly. */
		if (mddev->major_version == 0 &&
		    mddev->minor_version > 90)
			rdev->recovery_offset = reshape_offset;
			
		if (rdev->recovery_offset < reshape_offset) {
			/* We need to check old and new layout */
			if (!only_parity(rdev->raid_disk,
					 conf->algorithm,
					 conf->raid_disks,
					 conf->max_degraded))
				continue;
		}
		if (!only_parity(rdev->raid_disk,
				 conf->prev_algo,
				 conf->previous_raid_disks,
				 conf->max_degraded))
			continue;
		dirty_parity_disks++;
	}
N
NeilBrown 已提交
5177

5178 5179 5180
	/*
	 * 0 for a fully functional array, 1 or 2 for a degraded array.
	 */
5181
	mddev->degraded = calc_degraded(conf);
N
NeilBrown 已提交
5182

5183
	if (has_failed(conf)) {
5184
		printk(KERN_ERR "md/raid:%s: not enough operational devices"
L
Linus Torvalds 已提交
5185
			" (%d/%d failed)\n",
5186
			mdname(mddev), mddev->degraded, conf->raid_disks);
L
Linus Torvalds 已提交
5187 5188 5189
		goto abort;
	}

N
NeilBrown 已提交
5190
	/* device size must be a multiple of chunk size */
5191
	mddev->dev_sectors &= ~(mddev->chunk_sectors - 1);
N
NeilBrown 已提交
5192 5193
	mddev->resync_max_sectors = mddev->dev_sectors;

5194
	if (mddev->degraded > dirty_parity_disks &&
L
Linus Torvalds 已提交
5195
	    mddev->recovery_cp != MaxSector) {
5196 5197
		if (mddev->ok_start_degraded)
			printk(KERN_WARNING
5198 5199
			       "md/raid:%s: starting dirty degraded array"
			       " - data corruption possible.\n",
5200 5201 5202
			       mdname(mddev));
		else {
			printk(KERN_ERR
5203
			       "md/raid:%s: cannot start dirty degraded array.\n",
5204 5205 5206
			       mdname(mddev));
			goto abort;
		}
L
Linus Torvalds 已提交
5207 5208 5209
	}

	if (mddev->degraded == 0)
5210 5211
		printk(KERN_INFO "md/raid:%s: raid level %d active with %d out of %d"
		       " devices, algorithm %d\n", mdname(mddev), conf->level,
5212 5213
		       mddev->raid_disks-mddev->degraded, mddev->raid_disks,
		       mddev->new_layout);
L
Linus Torvalds 已提交
5214
	else
5215 5216 5217 5218 5219
		printk(KERN_ALERT "md/raid:%s: raid level %d active with %d"
		       " out of %d devices, algorithm %d\n",
		       mdname(mddev), conf->level,
		       mddev->raid_disks - mddev->degraded,
		       mddev->raid_disks, mddev->new_layout);
L
Linus Torvalds 已提交
5220 5221 5222

	print_raid5_conf(conf);

5223 5224
	if (conf->reshape_progress != MaxSector) {
		conf->reshape_safe = conf->reshape_progress;
5225 5226 5227 5228 5229 5230
		atomic_set(&conf->reshape_stripes, 0);
		clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
		clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
		set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
		set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
		mddev->sync_thread = md_register_thread(md_do_sync, mddev,
5231
							"reshape");
5232 5233
	}

L
Linus Torvalds 已提交
5234 5235

	/* Ok, everything is just fine now */
5236 5237
	if (mddev->to_remove == &raid5_attrs_group)
		mddev->to_remove = NULL;
N
NeilBrown 已提交
5238 5239
	else if (mddev->kobj.sd &&
	    sysfs_create_group(&mddev->kobj, &raid5_attrs_group))
5240
		printk(KERN_WARNING
5241
		       "raid5: failed to create sysfs attributes for %s\n",
5242
		       mdname(mddev));
5243
	md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
5244

5245
	if (mddev->queue) {
5246
		int chunk_size;
5247 5248 5249 5250 5251 5252 5253 5254 5255
		/* read-ahead size must cover two whole stripes, which
		 * is 2 * (datadisks) * chunksize where 'n' is the
		 * number of raid devices
		 */
		int data_disks = conf->previous_raid_disks - conf->max_degraded;
		int stripe = data_disks *
			((mddev->chunk_sectors << 9) / PAGE_SIZE);
		if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
			mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
N
NeilBrown 已提交
5256

5257
		blk_queue_merge_bvec(mddev->queue, raid5_mergeable_bvec);
5258

N
NeilBrown 已提交
5259 5260
		mddev->queue->backing_dev_info.congested_data = mddev;
		mddev->queue->backing_dev_info.congested_fn = raid5_congested;
5261

5262 5263 5264 5265
		chunk_size = mddev->chunk_sectors << 9;
		blk_queue_io_min(mddev->queue, chunk_size);
		blk_queue_io_opt(mddev->queue, chunk_size *
				 (conf->raid_disks - conf->max_degraded));
5266

5267
		rdev_for_each(rdev, mddev) {
5268 5269
			disk_stack_limits(mddev->gendisk, rdev->bdev,
					  rdev->data_offset << 9);
5270 5271 5272
			disk_stack_limits(mddev->gendisk, rdev->bdev,
					  rdev->new_data_offset << 9);
		}
5273
	}
5274

L
Linus Torvalds 已提交
5275 5276
	return 0;
abort:
5277
	md_unregister_thread(&mddev->thread);
N
NeilBrown 已提交
5278 5279
	print_raid5_conf(conf);
	free_conf(conf);
L
Linus Torvalds 已提交
5280
	mddev->private = NULL;
5281
	printk(KERN_ALERT "md/raid:%s: failed to run raid set.\n", mdname(mddev));
L
Linus Torvalds 已提交
5282 5283 5284
	return -EIO;
}

5285
static int stop(struct mddev *mddev)
L
Linus Torvalds 已提交
5286
{
5287
	struct r5conf *conf = mddev->private;
L
Linus Torvalds 已提交
5288

5289
	md_unregister_thread(&mddev->thread);
N
NeilBrown 已提交
5290 5291
	if (mddev->queue)
		mddev->queue->backing_dev_info.congested_fn = NULL;
5292
	free_conf(conf);
5293 5294
	mddev->private = NULL;
	mddev->to_remove = &raid5_attrs_group;
L
Linus Torvalds 已提交
5295 5296 5297
	return 0;
}

5298
static void status(struct seq_file *seq, struct mddev *mddev)
L
Linus Torvalds 已提交
5299
{
5300
	struct r5conf *conf = mddev->private;
L
Linus Torvalds 已提交
5301 5302
	int i;

5303 5304
	seq_printf(seq, " level %d, %dk chunk, algorithm %d", mddev->level,
		mddev->chunk_sectors / 2, mddev->layout);
5305
	seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded);
L
Linus Torvalds 已提交
5306 5307 5308
	for (i = 0; i < conf->raid_disks; i++)
		seq_printf (seq, "%s",
			       conf->disks[i].rdev &&
5309
			       test_bit(In_sync, &conf->disks[i].rdev->flags) ? "U" : "_");
L
Linus Torvalds 已提交
5310 5311 5312
	seq_printf (seq, "]");
}

5313
static void print_raid5_conf (struct r5conf *conf)
L
Linus Torvalds 已提交
5314 5315 5316 5317
{
	int i;
	struct disk_info *tmp;

5318
	printk(KERN_DEBUG "RAID conf printout:\n");
L
Linus Torvalds 已提交
5319 5320 5321 5322
	if (!conf) {
		printk("(conf==NULL)\n");
		return;
	}
5323 5324 5325
	printk(KERN_DEBUG " --- level:%d rd:%d wd:%d\n", conf->level,
	       conf->raid_disks,
	       conf->raid_disks - conf->mddev->degraded);
L
Linus Torvalds 已提交
5326 5327 5328 5329 5330

	for (i = 0; i < conf->raid_disks; i++) {
		char b[BDEVNAME_SIZE];
		tmp = conf->disks + i;
		if (tmp->rdev)
5331 5332 5333
			printk(KERN_DEBUG " disk %d, o:%d, dev:%s\n",
			       i, !test_bit(Faulty, &tmp->rdev->flags),
			       bdevname(tmp->rdev->bdev, b));
L
Linus Torvalds 已提交
5334 5335 5336
	}
}

5337
static int raid5_spare_active(struct mddev *mddev)
L
Linus Torvalds 已提交
5338 5339
{
	int i;
5340
	struct r5conf *conf = mddev->private;
L
Linus Torvalds 已提交
5341
	struct disk_info *tmp;
5342 5343
	int count = 0;
	unsigned long flags;
L
Linus Torvalds 已提交
5344 5345 5346

	for (i = 0; i < conf->raid_disks; i++) {
		tmp = conf->disks + i;
5347 5348 5349 5350 5351 5352 5353 5354 5355 5356 5357 5358 5359 5360 5361 5362 5363 5364 5365
		if (tmp->replacement
		    && tmp->replacement->recovery_offset == MaxSector
		    && !test_bit(Faulty, &tmp->replacement->flags)
		    && !test_and_set_bit(In_sync, &tmp->replacement->flags)) {
			/* Replacement has just become active. */
			if (!tmp->rdev
			    || !test_and_clear_bit(In_sync, &tmp->rdev->flags))
				count++;
			if (tmp->rdev) {
				/* Replaced device not technically faulty,
				 * but we need to be sure it gets removed
				 * and never re-added.
				 */
				set_bit(Faulty, &tmp->rdev->flags);
				sysfs_notify_dirent_safe(
					tmp->rdev->sysfs_state);
			}
			sysfs_notify_dirent_safe(tmp->replacement->sysfs_state);
		} else if (tmp->rdev
5366
		    && tmp->rdev->recovery_offset == MaxSector
5367
		    && !test_bit(Faulty, &tmp->rdev->flags)
5368
		    && !test_and_set_bit(In_sync, &tmp->rdev->flags)) {
5369
			count++;
5370
			sysfs_notify_dirent_safe(tmp->rdev->sysfs_state);
L
Linus Torvalds 已提交
5371 5372
		}
	}
5373
	spin_lock_irqsave(&conf->device_lock, flags);
5374
	mddev->degraded = calc_degraded(conf);
5375
	spin_unlock_irqrestore(&conf->device_lock, flags);
L
Linus Torvalds 已提交
5376
	print_raid5_conf(conf);
5377
	return count;
L
Linus Torvalds 已提交
5378 5379
}

5380
static int raid5_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
L
Linus Torvalds 已提交
5381
{
5382
	struct r5conf *conf = mddev->private;
L
Linus Torvalds 已提交
5383
	int err = 0;
5384
	int number = rdev->raid_disk;
5385
	struct md_rdev **rdevp;
L
Linus Torvalds 已提交
5386 5387 5388
	struct disk_info *p = conf->disks + number;

	print_raid5_conf(conf);
5389 5390 5391 5392 5393 5394 5395 5396 5397 5398 5399 5400 5401 5402 5403 5404 5405 5406 5407 5408 5409 5410
	if (rdev == p->rdev)
		rdevp = &p->rdev;
	else if (rdev == p->replacement)
		rdevp = &p->replacement;
	else
		return 0;

	if (number >= conf->raid_disks &&
	    conf->reshape_progress == MaxSector)
		clear_bit(In_sync, &rdev->flags);

	if (test_bit(In_sync, &rdev->flags) ||
	    atomic_read(&rdev->nr_pending)) {
		err = -EBUSY;
		goto abort;
	}
	/* Only remove non-faulty devices if recovery
	 * isn't possible.
	 */
	if (!test_bit(Faulty, &rdev->flags) &&
	    mddev->recovery_disabled != conf->recovery_disabled &&
	    !has_failed(conf) &&
5411
	    (!p->replacement || p->replacement == rdev) &&
5412 5413 5414 5415 5416 5417 5418 5419 5420 5421
	    number < conf->raid_disks) {
		err = -EBUSY;
		goto abort;
	}
	*rdevp = NULL;
	synchronize_rcu();
	if (atomic_read(&rdev->nr_pending)) {
		/* lost the race, try later */
		err = -EBUSY;
		*rdevp = rdev;
5422 5423 5424 5425 5426 5427 5428 5429 5430 5431 5432 5433 5434 5435
	} else if (p->replacement) {
		/* We must have just cleared 'rdev' */
		p->rdev = p->replacement;
		clear_bit(Replacement, &p->replacement->flags);
		smp_mb(); /* Make sure other CPUs may see both as identical
			   * but will never see neither - if they are careful
			   */
		p->replacement = NULL;
		clear_bit(WantReplacement, &rdev->flags);
	} else
		/* We might have just removed the Replacement as faulty-
		 * clear the bit just in case
		 */
		clear_bit(WantReplacement, &rdev->flags);
L
Linus Torvalds 已提交
5436 5437 5438 5439 5440 5441
abort:

	print_raid5_conf(conf);
	return err;
}

5442
static int raid5_add_disk(struct mddev *mddev, struct md_rdev *rdev)
L
Linus Torvalds 已提交
5443
{
5444
	struct r5conf *conf = mddev->private;
5445
	int err = -EEXIST;
L
Linus Torvalds 已提交
5446 5447
	int disk;
	struct disk_info *p;
5448 5449
	int first = 0;
	int last = conf->raid_disks - 1;
L
Linus Torvalds 已提交
5450

5451 5452 5453
	if (mddev->recovery_disabled == conf->recovery_disabled)
		return -EBUSY;

N
NeilBrown 已提交
5454
	if (rdev->saved_raid_disk < 0 && has_failed(conf))
L
Linus Torvalds 已提交
5455
		/* no point adding a device */
5456
		return -EINVAL;
L
Linus Torvalds 已提交
5457

5458 5459
	if (rdev->raid_disk >= 0)
		first = last = rdev->raid_disk;
L
Linus Torvalds 已提交
5460 5461

	/*
5462 5463
	 * find the disk ... but prefer rdev->saved_raid_disk
	 * if possible.
L
Linus Torvalds 已提交
5464
	 */
5465
	if (rdev->saved_raid_disk >= 0 &&
5466
	    rdev->saved_raid_disk >= first &&
5467 5468 5469
	    conf->disks[rdev->saved_raid_disk].rdev == NULL)
		disk = rdev->saved_raid_disk;
	else
5470
		disk = first;
5471 5472 5473
	for ( ; disk <= last ; disk++) {
		p = conf->disks + disk;
		if (p->rdev == NULL) {
5474
			clear_bit(In_sync, &rdev->flags);
L
Linus Torvalds 已提交
5475
			rdev->raid_disk = disk;
5476
			err = 0;
5477 5478
			if (rdev->saved_raid_disk != disk)
				conf->fullsync = 1;
5479
			rcu_assign_pointer(p->rdev, rdev);
L
Linus Torvalds 已提交
5480 5481
			break;
		}
5482 5483 5484 5485 5486 5487 5488 5489 5490 5491 5492
		if (test_bit(WantReplacement, &p->rdev->flags) &&
		    p->replacement == NULL) {
			clear_bit(In_sync, &rdev->flags);
			set_bit(Replacement, &rdev->flags);
			rdev->raid_disk = disk;
			err = 0;
			conf->fullsync = 1;
			rcu_assign_pointer(p->replacement, rdev);
			break;
		}
	}
L
Linus Torvalds 已提交
5493
	print_raid5_conf(conf);
5494
	return err;
L
Linus Torvalds 已提交
5495 5496
}

5497
static int raid5_resize(struct mddev *mddev, sector_t sectors)
L
Linus Torvalds 已提交
5498 5499 5500 5501 5502 5503 5504 5505
{
	/* no resync is happening, and there is enough space
	 * on all devices, so we can resize.
	 * We need to make sure resync covers any new space.
	 * If the array is shrinking we should possibly wait until
	 * any io in the removed space completes, but it hardly seems
	 * worth it.
	 */
5506
	sector_t newsize;
5507
	sectors &= ~((sector_t)mddev->chunk_sectors - 1);
5508 5509 5510
	newsize = raid5_size(mddev, sectors, mddev->raid_disks);
	if (mddev->external_size &&
	    mddev->array_sectors > newsize)
D
Dan Williams 已提交
5511
		return -EINVAL;
5512 5513 5514 5515 5516 5517
	if (mddev->bitmap) {
		int ret = bitmap_resize(mddev->bitmap, sectors, 0, 0);
		if (ret)
			return ret;
	}
	md_set_array_sectors(mddev, newsize);
5518
	set_capacity(mddev->gendisk, mddev->array_sectors);
5519
	revalidate_disk(mddev->gendisk);
5520 5521
	if (sectors > mddev->dev_sectors &&
	    mddev->recovery_cp > mddev->dev_sectors) {
A
Andre Noll 已提交
5522
		mddev->recovery_cp = mddev->dev_sectors;
L
Linus Torvalds 已提交
5523 5524
		set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
	}
A
Andre Noll 已提交
5525
	mddev->dev_sectors = sectors;
5526
	mddev->resync_max_sectors = sectors;
L
Linus Torvalds 已提交
5527 5528 5529
	return 0;
}

5530
static int check_stripe_cache(struct mddev *mddev)
5531 5532 5533 5534 5535 5536 5537 5538 5539
{
	/* Can only proceed if there are plenty of stripe_heads.
	 * We need a minimum of one full stripe,, and for sensible progress
	 * it is best to have about 4 times that.
	 * If we require 4 times, then the default 256 4K stripe_heads will
	 * allow for chunk sizes up to 256K, which is probably OK.
	 * If the chunk size is greater, user-space should request more
	 * stripe_heads first.
	 */
5540
	struct r5conf *conf = mddev->private;
5541 5542 5543 5544
	if (((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4
	    > conf->max_nr_stripes ||
	    ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4
	    > conf->max_nr_stripes) {
5545 5546
		printk(KERN_WARNING "md/raid:%s: reshape: not enough stripes.  Needed %lu\n",
		       mdname(mddev),
5547 5548 5549 5550 5551 5552 5553
		       ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9)
			/ STRIPE_SIZE)*4);
		return 0;
	}
	return 1;
}

5554
static int check_reshape(struct mddev *mddev)
5555
{
5556
	struct r5conf *conf = mddev->private;
5557

5558 5559
	if (mddev->delta_disks == 0 &&
	    mddev->new_layout == mddev->layout &&
5560
	    mddev->new_chunk_sectors == mddev->chunk_sectors)
5561
		return 0; /* nothing to do */
5562
	if (has_failed(conf))
5563 5564 5565 5566 5567 5568 5569 5570 5571 5572 5573 5574 5575
		return -EINVAL;
	if (mddev->delta_disks < 0) {
		/* We might be able to shrink, but the devices must
		 * be made bigger first.
		 * For raid6, 4 is the minimum size.
		 * Otherwise 2 is the minimum
		 */
		int min = 2;
		if (mddev->level == 6)
			min = 4;
		if (mddev->raid_disks + mddev->delta_disks < min)
			return -EINVAL;
	}
5576

5577
	if (!check_stripe_cache(mddev))
5578 5579
		return -ENOSPC;

5580
	return resize_stripes(conf, conf->raid_disks + mddev->delta_disks);
5581 5582
}

5583
static int raid5_start_reshape(struct mddev *mddev)
5584
{
5585
	struct r5conf *conf = mddev->private;
5586
	struct md_rdev *rdev;
5587
	int spares = 0;
5588
	unsigned long flags;
5589

5590
	if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
5591 5592
		return -EBUSY;

5593 5594 5595
	if (!check_stripe_cache(mddev))
		return -ENOSPC;

5596 5597 5598
	if (has_failed(conf))
		return -EINVAL;

5599
	rdev_for_each(rdev, mddev) {
5600 5601
		if (!test_bit(In_sync, &rdev->flags)
		    && !test_bit(Faulty, &rdev->flags))
5602
			spares++;
5603
	}
5604

5605
	if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded)
5606 5607 5608 5609 5610
		/* Not enough devices even to make a degraded array
		 * of that size
		 */
		return -EINVAL;

5611 5612 5613 5614 5615 5616
	/* Refuse to reduce size of the array.  Any reductions in
	 * array size must be through explicit setting of array_size
	 * attribute.
	 */
	if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks)
	    < mddev->array_sectors) {
5617
		printk(KERN_ERR "md/raid:%s: array size must be reduced "
5618 5619 5620 5621
		       "before number of disks\n", mdname(mddev));
		return -EINVAL;
	}

5622
	atomic_set(&conf->reshape_stripes, 0);
5623 5624
	spin_lock_irq(&conf->device_lock);
	conf->previous_raid_disks = conf->raid_disks;
5625
	conf->raid_disks += mddev->delta_disks;
5626 5627
	conf->prev_chunk_sectors = conf->chunk_sectors;
	conf->chunk_sectors = mddev->new_chunk_sectors;
5628 5629
	conf->prev_algo = conf->algorithm;
	conf->algorithm = mddev->new_layout;
5630 5631 5632 5633 5634
	conf->generation++;
	/* Code that selects data_offset needs to see the generation update
	 * if reshape_progress has been set - so a memory barrier needed.
	 */
	smp_mb();
5635
	if (mddev->reshape_backwards)
5636 5637 5638 5639
		conf->reshape_progress = raid5_size(mddev, 0, 0);
	else
		conf->reshape_progress = 0;
	conf->reshape_safe = conf->reshape_progress;
5640 5641 5642 5643
	spin_unlock_irq(&conf->device_lock);

	/* Add some new drives, as many as will fit.
	 * We know there are enough to make the newly sized array work.
5644 5645 5646 5647
	 * Don't add devices if we are reducing the number of
	 * devices in the array.  This is because it is not possible
	 * to correctly record the "partially reconstructed" state of
	 * such devices during the reshape and confusion could result.
5648
	 */
5649
	if (mddev->delta_disks >= 0) {
N
NeilBrown 已提交
5650
		rdev_for_each(rdev, mddev)
5651 5652 5653 5654
			if (rdev->raid_disk < 0 &&
			    !test_bit(Faulty, &rdev->flags)) {
				if (raid5_add_disk(mddev, rdev) == 0) {
					if (rdev->raid_disk
5655
					    >= conf->previous_raid_disks)
5656
						set_bit(In_sync, &rdev->flags);
5657
					else
5658
						rdev->recovery_offset = 0;
5659 5660

					if (sysfs_link_rdev(mddev, rdev))
5661
						/* Failure here is OK */;
5662
				}
5663 5664 5665 5666 5667
			} else if (rdev->raid_disk >= conf->previous_raid_disks
				   && !test_bit(Faulty, &rdev->flags)) {
				/* This is a spare that was manually added */
				set_bit(In_sync, &rdev->flags);
			}
5668

5669 5670 5671 5672
		/* When a reshape changes the number of devices,
		 * ->degraded is measured against the larger of the
		 * pre and post number of devices.
		 */
5673
		spin_lock_irqsave(&conf->device_lock, flags);
5674
		mddev->degraded = calc_degraded(conf);
5675 5676
		spin_unlock_irqrestore(&conf->device_lock, flags);
	}
5677
	mddev->raid_disks = conf->raid_disks;
5678
	mddev->reshape_position = conf->reshape_progress;
5679
	set_bit(MD_CHANGE_DEVS, &mddev->flags);
5680

5681 5682 5683 5684 5685
	clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
	clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
	set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
	set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
	mddev->sync_thread = md_register_thread(md_do_sync, mddev,
5686
						"reshape");
5687 5688 5689 5690
	if (!mddev->sync_thread) {
		mddev->recovery = 0;
		spin_lock_irq(&conf->device_lock);
		mddev->raid_disks = conf->raid_disks = conf->previous_raid_disks;
5691 5692 5693
		rdev_for_each(rdev, mddev)
			rdev->new_data_offset = rdev->data_offset;
		smp_wmb();
5694
		conf->reshape_progress = MaxSector;
5695
		mddev->reshape_position = MaxSector;
5696 5697 5698
		spin_unlock_irq(&conf->device_lock);
		return -EAGAIN;
	}
5699
	conf->reshape_checkpoint = jiffies;
5700 5701 5702 5703 5704
	md_wakeup_thread(mddev->sync_thread);
	md_new_event(mddev);
	return 0;
}

5705 5706 5707
/* This is called from the reshape thread and should make any
 * changes needed in 'conf'
 */
5708
static void end_reshape(struct r5conf *conf)
5709 5710
{

5711
	if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) {
5712
		struct md_rdev *rdev;
5713 5714

		spin_lock_irq(&conf->device_lock);
5715
		conf->previous_raid_disks = conf->raid_disks;
5716 5717 5718
		rdev_for_each(rdev, conf->mddev)
			rdev->data_offset = rdev->new_data_offset;
		smp_wmb();
5719
		conf->reshape_progress = MaxSector;
5720
		spin_unlock_irq(&conf->device_lock);
5721
		wake_up(&conf->wait_for_overlap);
5722 5723 5724 5725

		/* read-ahead size must cover two whole stripes, which is
		 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices
		 */
5726
		if (conf->mddev->queue) {
5727
			int data_disks = conf->raid_disks - conf->max_degraded;
5728
			int stripe = data_disks * ((conf->chunk_sectors << 9)
5729
						   / PAGE_SIZE);
5730 5731 5732
			if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
				conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
		}
5733 5734 5735
	}
}

5736 5737 5738
/* This is called from the raid5d thread with mddev_lock held.
 * It makes config changes to the device.
 */
5739
static void raid5_finish_reshape(struct mddev *mddev)
5740
{
5741
	struct r5conf *conf = mddev->private;
5742 5743 5744

	if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {

5745 5746 5747
		if (mddev->delta_disks > 0) {
			md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
			set_capacity(mddev->gendisk, mddev->array_sectors);
5748
			revalidate_disk(mddev->gendisk);
5749 5750
		} else {
			int d;
5751 5752 5753
			spin_lock_irq(&conf->device_lock);
			mddev->degraded = calc_degraded(conf);
			spin_unlock_irq(&conf->device_lock);
5754 5755
			for (d = conf->raid_disks ;
			     d < conf->raid_disks - mddev->delta_disks;
5756
			     d++) {
5757
				struct md_rdev *rdev = conf->disks[d].rdev;
5758 5759 5760 5761 5762
				if (rdev)
					clear_bit(In_sync, &rdev->flags);
				rdev = conf->disks[d].replacement;
				if (rdev)
					clear_bit(In_sync, &rdev->flags);
5763
			}
5764
		}
5765
		mddev->layout = conf->algorithm;
5766
		mddev->chunk_sectors = conf->chunk_sectors;
5767 5768
		mddev->reshape_position = MaxSector;
		mddev->delta_disks = 0;
5769
		mddev->reshape_backwards = 0;
5770 5771 5772
	}
}

5773
static void raid5_quiesce(struct mddev *mddev, int state)
5774
{
5775
	struct r5conf *conf = mddev->private;
5776 5777

	switch(state) {
5778 5779 5780 5781
	case 2: /* resume for a suspend */
		wake_up(&conf->wait_for_overlap);
		break;

5782 5783
	case 1: /* stop all writes */
		spin_lock_irq(&conf->device_lock);
5784 5785 5786 5787
		/* '2' tells resync/reshape to pause so that all
		 * active stripes can drain
		 */
		conf->quiesce = 2;
5788
		wait_event_lock_irq(conf->wait_for_stripe,
5789 5790
				    atomic_read(&conf->active_stripes) == 0 &&
				    atomic_read(&conf->active_aligned_reads) == 0,
5791
				    conf->device_lock, /* nothing */);
5792
		conf->quiesce = 1;
5793
		spin_unlock_irq(&conf->device_lock);
5794 5795
		/* allow reshape to continue */
		wake_up(&conf->wait_for_overlap);
5796 5797 5798 5799 5800 5801
		break;

	case 0: /* re-enable writes */
		spin_lock_irq(&conf->device_lock);
		conf->quiesce = 0;
		wake_up(&conf->wait_for_stripe);
5802
		wake_up(&conf->wait_for_overlap);
5803 5804 5805 5806
		spin_unlock_irq(&conf->device_lock);
		break;
	}
}
5807

5808

5809
static void *raid45_takeover_raid0(struct mddev *mddev, int level)
5810
{
5811
	struct r0conf *raid0_conf = mddev->private;
5812
	sector_t sectors;
5813

D
Dan Williams 已提交
5814
	/* for raid0 takeover only one zone is supported */
5815
	if (raid0_conf->nr_strip_zones > 1) {
5816 5817
		printk(KERN_ERR "md/raid:%s: cannot takeover raid0 with more than one zone.\n",
		       mdname(mddev));
D
Dan Williams 已提交
5818 5819 5820
		return ERR_PTR(-EINVAL);
	}

5821 5822
	sectors = raid0_conf->strip_zone[0].zone_end;
	sector_div(sectors, raid0_conf->strip_zone[0].nb_dev);
5823
	mddev->dev_sectors = sectors;
D
Dan Williams 已提交
5824
	mddev->new_level = level;
5825 5826 5827 5828 5829 5830 5831 5832 5833 5834 5835
	mddev->new_layout = ALGORITHM_PARITY_N;
	mddev->new_chunk_sectors = mddev->chunk_sectors;
	mddev->raid_disks += 1;
	mddev->delta_disks = 1;
	/* make sure it will be not marked as dirty */
	mddev->recovery_cp = MaxSector;

	return setup_conf(mddev);
}


5836
static void *raid5_takeover_raid1(struct mddev *mddev)
5837 5838 5839 5840 5841 5842 5843 5844 5845 5846 5847 5848 5849 5850 5851 5852 5853 5854 5855 5856 5857
{
	int chunksect;

	if (mddev->raid_disks != 2 ||
	    mddev->degraded > 1)
		return ERR_PTR(-EINVAL);

	/* Should check if there are write-behind devices? */

	chunksect = 64*2; /* 64K by default */

	/* The array must be an exact multiple of chunksize */
	while (chunksect && (mddev->array_sectors & (chunksect-1)))
		chunksect >>= 1;

	if ((chunksect<<9) < STRIPE_SIZE)
		/* array size does not allow a suitable chunk size */
		return ERR_PTR(-EINVAL);

	mddev->new_level = 5;
	mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC;
5858
	mddev->new_chunk_sectors = chunksect;
5859 5860 5861 5862

	return setup_conf(mddev);
}

5863
static void *raid5_takeover_raid6(struct mddev *mddev)
5864 5865 5866 5867 5868 5869 5870 5871 5872 5873 5874 5875 5876 5877 5878 5879 5880 5881 5882 5883 5884 5885 5886 5887 5888 5889 5890 5891 5892 5893 5894 5895
{
	int new_layout;

	switch (mddev->layout) {
	case ALGORITHM_LEFT_ASYMMETRIC_6:
		new_layout = ALGORITHM_LEFT_ASYMMETRIC;
		break;
	case ALGORITHM_RIGHT_ASYMMETRIC_6:
		new_layout = ALGORITHM_RIGHT_ASYMMETRIC;
		break;
	case ALGORITHM_LEFT_SYMMETRIC_6:
		new_layout = ALGORITHM_LEFT_SYMMETRIC;
		break;
	case ALGORITHM_RIGHT_SYMMETRIC_6:
		new_layout = ALGORITHM_RIGHT_SYMMETRIC;
		break;
	case ALGORITHM_PARITY_0_6:
		new_layout = ALGORITHM_PARITY_0;
		break;
	case ALGORITHM_PARITY_N:
		new_layout = ALGORITHM_PARITY_N;
		break;
	default:
		return ERR_PTR(-EINVAL);
	}
	mddev->new_level = 5;
	mddev->new_layout = new_layout;
	mddev->delta_disks = -1;
	mddev->raid_disks -= 1;
	return setup_conf(mddev);
}

5896

5897
static int raid5_check_reshape(struct mddev *mddev)
5898
{
5899 5900 5901 5902
	/* For a 2-drive array, the layout and chunk size can be changed
	 * immediately as not restriping is needed.
	 * For larger arrays we record the new value - after validation
	 * to be used by a reshape pass.
5903
	 */
5904
	struct r5conf *conf = mddev->private;
5905
	int new_chunk = mddev->new_chunk_sectors;
5906

5907
	if (mddev->new_layout >= 0 && !algorithm_valid_raid5(mddev->new_layout))
5908 5909
		return -EINVAL;
	if (new_chunk > 0) {
5910
		if (!is_power_of_2(new_chunk))
5911
			return -EINVAL;
5912
		if (new_chunk < (PAGE_SIZE>>9))
5913
			return -EINVAL;
5914
		if (mddev->array_sectors & (new_chunk-1))
5915 5916 5917 5918 5919 5920
			/* not factor of array size */
			return -EINVAL;
	}

	/* They look valid */

5921
	if (mddev->raid_disks == 2) {
5922 5923 5924 5925
		/* can make the change immediately */
		if (mddev->new_layout >= 0) {
			conf->algorithm = mddev->new_layout;
			mddev->layout = mddev->new_layout;
5926 5927
		}
		if (new_chunk > 0) {
5928 5929
			conf->chunk_sectors = new_chunk ;
			mddev->chunk_sectors = new_chunk;
5930 5931 5932
		}
		set_bit(MD_CHANGE_DEVS, &mddev->flags);
		md_wakeup_thread(mddev->thread);
5933
	}
5934
	return check_reshape(mddev);
5935 5936
}

5937
static int raid6_check_reshape(struct mddev *mddev)
5938
{
5939
	int new_chunk = mddev->new_chunk_sectors;
5940

5941
	if (mddev->new_layout >= 0 && !algorithm_valid_raid6(mddev->new_layout))
5942
		return -EINVAL;
5943
	if (new_chunk > 0) {
5944
		if (!is_power_of_2(new_chunk))
5945
			return -EINVAL;
5946
		if (new_chunk < (PAGE_SIZE >> 9))
5947
			return -EINVAL;
5948
		if (mddev->array_sectors & (new_chunk-1))
5949 5950
			/* not factor of array size */
			return -EINVAL;
5951
	}
5952 5953

	/* They look valid */
5954
	return check_reshape(mddev);
5955 5956
}

5957
static void *raid5_takeover(struct mddev *mddev)
5958 5959
{
	/* raid5 can take over:
D
Dan Williams 已提交
5960
	 *  raid0 - if there is only one strip zone - make it a raid4 layout
5961 5962 5963 5964
	 *  raid1 - if there are two drives.  We need to know the chunk size
	 *  raid4 - trivial - just use a raid4 layout.
	 *  raid6 - Providing it is a *_6 layout
	 */
D
Dan Williams 已提交
5965 5966
	if (mddev->level == 0)
		return raid45_takeover_raid0(mddev, 5);
5967 5968
	if (mddev->level == 1)
		return raid5_takeover_raid1(mddev);
5969 5970 5971 5972 5973
	if (mddev->level == 4) {
		mddev->new_layout = ALGORITHM_PARITY_N;
		mddev->new_level = 5;
		return setup_conf(mddev);
	}
5974 5975
	if (mddev->level == 6)
		return raid5_takeover_raid6(mddev);
5976 5977 5978 5979

	return ERR_PTR(-EINVAL);
}

5980
static void *raid4_takeover(struct mddev *mddev)
5981
{
D
Dan Williams 已提交
5982 5983 5984
	/* raid4 can take over:
	 *  raid0 - if there is only one strip zone
	 *  raid5 - if layout is right
5985
	 */
D
Dan Williams 已提交
5986 5987
	if (mddev->level == 0)
		return raid45_takeover_raid0(mddev, 4);
5988 5989 5990 5991 5992 5993 5994 5995
	if (mddev->level == 5 &&
	    mddev->layout == ALGORITHM_PARITY_N) {
		mddev->new_layout = 0;
		mddev->new_level = 4;
		return setup_conf(mddev);
	}
	return ERR_PTR(-EINVAL);
}
5996

5997
static struct md_personality raid5_personality;
5998

5999
static void *raid6_takeover(struct mddev *mddev)
6000 6001 6002 6003 6004 6005 6006 6007 6008 6009 6010 6011 6012 6013 6014 6015 6016 6017 6018 6019 6020 6021 6022 6023 6024 6025 6026 6027 6028 6029 6030 6031 6032 6033 6034 6035 6036 6037 6038 6039 6040 6041 6042 6043 6044 6045
{
	/* Currently can only take over a raid5.  We map the
	 * personality to an equivalent raid6 personality
	 * with the Q block at the end.
	 */
	int new_layout;

	if (mddev->pers != &raid5_personality)
		return ERR_PTR(-EINVAL);
	if (mddev->degraded > 1)
		return ERR_PTR(-EINVAL);
	if (mddev->raid_disks > 253)
		return ERR_PTR(-EINVAL);
	if (mddev->raid_disks < 3)
		return ERR_PTR(-EINVAL);

	switch (mddev->layout) {
	case ALGORITHM_LEFT_ASYMMETRIC:
		new_layout = ALGORITHM_LEFT_ASYMMETRIC_6;
		break;
	case ALGORITHM_RIGHT_ASYMMETRIC:
		new_layout = ALGORITHM_RIGHT_ASYMMETRIC_6;
		break;
	case ALGORITHM_LEFT_SYMMETRIC:
		new_layout = ALGORITHM_LEFT_SYMMETRIC_6;
		break;
	case ALGORITHM_RIGHT_SYMMETRIC:
		new_layout = ALGORITHM_RIGHT_SYMMETRIC_6;
		break;
	case ALGORITHM_PARITY_0:
		new_layout = ALGORITHM_PARITY_0_6;
		break;
	case ALGORITHM_PARITY_N:
		new_layout = ALGORITHM_PARITY_N;
		break;
	default:
		return ERR_PTR(-EINVAL);
	}
	mddev->new_level = 6;
	mddev->new_layout = new_layout;
	mddev->delta_disks = 1;
	mddev->raid_disks += 1;
	return setup_conf(mddev);
}


6046
static struct md_personality raid6_personality =
6047 6048 6049 6050 6051 6052 6053 6054 6055 6056 6057 6058 6059 6060
{
	.name		= "raid6",
	.level		= 6,
	.owner		= THIS_MODULE,
	.make_request	= make_request,
	.run		= run,
	.stop		= stop,
	.status		= status,
	.error_handler	= error,
	.hot_add_disk	= raid5_add_disk,
	.hot_remove_disk= raid5_remove_disk,
	.spare_active	= raid5_spare_active,
	.sync_request	= sync_request,
	.resize		= raid5_resize,
6061
	.size		= raid5_size,
6062
	.check_reshape	= raid6_check_reshape,
6063
	.start_reshape  = raid5_start_reshape,
6064
	.finish_reshape = raid5_finish_reshape,
6065
	.quiesce	= raid5_quiesce,
6066
	.takeover	= raid6_takeover,
6067
};
6068
static struct md_personality raid5_personality =
L
Linus Torvalds 已提交
6069 6070
{
	.name		= "raid5",
6071
	.level		= 5,
L
Linus Torvalds 已提交
6072 6073 6074 6075 6076 6077 6078 6079 6080 6081 6082
	.owner		= THIS_MODULE,
	.make_request	= make_request,
	.run		= run,
	.stop		= stop,
	.status		= status,
	.error_handler	= error,
	.hot_add_disk	= raid5_add_disk,
	.hot_remove_disk= raid5_remove_disk,
	.spare_active	= raid5_spare_active,
	.sync_request	= sync_request,
	.resize		= raid5_resize,
6083
	.size		= raid5_size,
6084 6085
	.check_reshape	= raid5_check_reshape,
	.start_reshape  = raid5_start_reshape,
6086
	.finish_reshape = raid5_finish_reshape,
6087
	.quiesce	= raid5_quiesce,
6088
	.takeover	= raid5_takeover,
L
Linus Torvalds 已提交
6089 6090
};

6091
static struct md_personality raid4_personality =
L
Linus Torvalds 已提交
6092
{
6093 6094 6095 6096 6097 6098 6099 6100 6101 6102 6103 6104 6105
	.name		= "raid4",
	.level		= 4,
	.owner		= THIS_MODULE,
	.make_request	= make_request,
	.run		= run,
	.stop		= stop,
	.status		= status,
	.error_handler	= error,
	.hot_add_disk	= raid5_add_disk,
	.hot_remove_disk= raid5_remove_disk,
	.spare_active	= raid5_spare_active,
	.sync_request	= sync_request,
	.resize		= raid5_resize,
6106
	.size		= raid5_size,
6107 6108
	.check_reshape	= raid5_check_reshape,
	.start_reshape  = raid5_start_reshape,
6109
	.finish_reshape = raid5_finish_reshape,
6110
	.quiesce	= raid5_quiesce,
6111
	.takeover	= raid4_takeover,
6112 6113 6114 6115
};

static int __init raid5_init(void)
{
6116
	register_md_personality(&raid6_personality);
6117 6118 6119
	register_md_personality(&raid5_personality);
	register_md_personality(&raid4_personality);
	return 0;
L
Linus Torvalds 已提交
6120 6121
}

6122
static void raid5_exit(void)
L
Linus Torvalds 已提交
6123
{
6124
	unregister_md_personality(&raid6_personality);
6125 6126
	unregister_md_personality(&raid5_personality);
	unregister_md_personality(&raid4_personality);
L
Linus Torvalds 已提交
6127 6128 6129 6130 6131
}

module_init(raid5_init);
module_exit(raid5_exit);
MODULE_LICENSE("GPL");
6132
MODULE_DESCRIPTION("RAID4/5/6 (striping with parity) personality for MD");
L
Linus Torvalds 已提交
6133
MODULE_ALIAS("md-personality-4"); /* RAID5 */
6134 6135
MODULE_ALIAS("md-raid5");
MODULE_ALIAS("md-raid4");
6136 6137
MODULE_ALIAS("md-level-5");
MODULE_ALIAS("md-level-4");
6138 6139 6140 6141 6142 6143 6144
MODULE_ALIAS("md-personality-8"); /* RAID6 */
MODULE_ALIAS("md-raid6");
MODULE_ALIAS("md-level-6");

/* This used to be two separate modules, they were: */
MODULE_ALIAS("raid5");
MODULE_ALIAS("raid6");