raid5.c 218.0 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4
/*
 * raid5.c : Multiple Devices driver for Linux
 *	   Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman
 *	   Copyright (C) 1999, 2000 Ingo Molnar
5
 *	   Copyright (C) 2002, 2003 H. Peter Anvin
L
Linus Torvalds 已提交
6
 *
7 8 9
 * RAID-4/5/6 management functions.
 * Thanks to Penguin Computing for making the RAID-6 development possible
 * by donating a test server!
L
Linus Torvalds 已提交
10 11 12 13 14 15 16 17 18 19 20
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2, or (at your option)
 * any later version.
 *
 * You should have received a copy of the GNU General Public License
 * (for example /usr/src/linux/COPYING); if not, write to the Free
 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

21 22 23 24 25 26 27 28 29
/*
 * BITMAP UNPLUGGING:
 *
 * The sequencing for updating the bitmap reliably is a little
 * subtle (and I got it wrong the first time) so it deserves some
 * explanation.
 *
 * We group bitmap updates into batches.  Each batch has a number.
 * We may write out several batches at once, but that isn't very important.
30 31
 * conf->seq_write is the number of the last batch successfully written.
 * conf->seq_flush is the number of the last batch that was closed to
32 33 34
 *    new additions.
 * When we discover that we will need to write to any block in a stripe
 * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq
35
 * the number of the batch it will be in. This is seq_flush+1.
36 37 38 39 40 41 42 43 44
 * When we are ready to do a write, if that batch hasn't been written yet,
 *   we plug the array and queue the stripe for later.
 * When an unplug happens, we increment bm_flush, thus closing the current
 *   batch.
 * When we notice that bm_flush > bm_write, we write out all pending updates
 * to the bitmap, and advance bm_write to where bm_flush was.
 * This may occasionally write a bit out twice, but is sure never to
 * miss any bits.
 */
L
Linus Torvalds 已提交
45

46
#include <linux/blkdev.h>
47
#include <linux/kthread.h>
48
#include <linux/raid/pq.h>
49
#include <linux/async_tx.h>
50
#include <linux/module.h>
51
#include <linux/async.h>
52
#include <linux/seq_file.h>
53
#include <linux/cpu.h>
54
#include <linux/slab.h>
55
#include <linux/ratelimit.h>
56
#include <linux/nodemask.h>
57
#include <linux/flex_array.h>
N
NeilBrown 已提交
58 59
#include <trace/events/block.h>

60
#include "md.h"
61
#include "raid5.h"
62
#include "raid0.h"
63
#include "bitmap.h"
64

65 66 67
#define cpu_to_group(cpu) cpu_to_node(cpu)
#define ANY_GROUP NUMA_NO_NODE

68 69 70 71
static bool devices_handle_discard_safely = false;
module_param(devices_handle_discard_safely, bool, 0644);
MODULE_PARM_DESC(devices_handle_discard_safely,
		 "Set to Y if all devices in each array reliably return zeroes on reads from discarded regions");
72
static struct workqueue_struct *raid5_wq;
L
Linus Torvalds 已提交
73 74 75 76 77 78 79 80 81
/*
 * Stripe cache
 */

#define NR_STRIPES		256
#define STRIPE_SIZE		PAGE_SIZE
#define STRIPE_SHIFT		(PAGE_SHIFT - 9)
#define STRIPE_SECTORS		(STRIPE_SIZE>>9)
#define	IO_THRESHOLD		1
82
#define BYPASS_THRESHOLD	1
83
#define NR_HASH			(PAGE_SIZE / sizeof(struct hlist_head))
L
Linus Torvalds 已提交
84
#define HASH_MASK		(NR_HASH - 1)
85
#define MAX_STRIPE_BATCH	8
L
Linus Torvalds 已提交
86

87
static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect)
88 89 90 91
{
	int hash = (sect >> STRIPE_SHIFT) & HASH_MASK;
	return &conf->stripe_hashtbl[hash];
}
L
Linus Torvalds 已提交
92

93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128
static inline int stripe_hash_locks_hash(sector_t sect)
{
	return (sect >> STRIPE_SHIFT) & STRIPE_HASH_LOCKS_MASK;
}

static inline void lock_device_hash_lock(struct r5conf *conf, int hash)
{
	spin_lock_irq(conf->hash_locks + hash);
	spin_lock(&conf->device_lock);
}

static inline void unlock_device_hash_lock(struct r5conf *conf, int hash)
{
	spin_unlock(&conf->device_lock);
	spin_unlock_irq(conf->hash_locks + hash);
}

static inline void lock_all_device_hash_locks_irq(struct r5conf *conf)
{
	int i;
	local_irq_disable();
	spin_lock(conf->hash_locks);
	for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++)
		spin_lock_nest_lock(conf->hash_locks + i, conf->hash_locks);
	spin_lock(&conf->device_lock);
}

static inline void unlock_all_device_hash_locks_irq(struct r5conf *conf)
{
	int i;
	spin_unlock(&conf->device_lock);
	for (i = NR_STRIPE_HASH_LOCKS; i; i--)
		spin_unlock(conf->hash_locks + i - 1);
	local_irq_enable();
}

L
Linus Torvalds 已提交
129 130 131 132 133 134
/* bio's attached to a stripe+device for I/O are linked together in bi_sector
 * order without overlap.  There may be several bio's per stripe+device, and
 * a bio could span several devices.
 * When walking this list for a particular stripe+device, we must never proceed
 * beyond a bio that extends past this device, as the next bio might no longer
 * be valid.
135
 * This function is used to determine the 'next' bio in the list, given the sector
L
Linus Torvalds 已提交
136 137
 * of the current stripe+device
 */
138 139
static inline struct bio *r5_next_bio(struct bio *bio, sector_t sector)
{
140
	int sectors = bio_sectors(bio);
141
	if (bio->bi_iter.bi_sector + sectors < sector + STRIPE_SECTORS)
142 143 144 145
		return bio->bi_next;
	else
		return NULL;
}
L
Linus Torvalds 已提交
146

147
/*
148 149
 * We maintain a biased count of active stripes in the bottom 16 bits of
 * bi_phys_segments, and a count of processed stripes in the upper 16 bits
150
 */
151
static inline int raid5_bi_processed_stripes(struct bio *bio)
152
{
153 154
	atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
	return (atomic_read(segments) >> 16) & 0xffff;
155 156
}

157
static inline int raid5_dec_bi_active_stripes(struct bio *bio)
158
{
159 160
	atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
	return atomic_sub_return(1, segments) & 0xffff;
161 162
}

163
static inline void raid5_inc_bi_active_stripes(struct bio *bio)
164
{
165 166
	atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
	atomic_inc(segments);
167 168
}

169 170
static inline void raid5_set_bi_processed_stripes(struct bio *bio,
	unsigned int cnt)
171
{
172 173
	atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
	int old, new;
174

175 176 177 178
	do {
		old = atomic_read(segments);
		new = (old & 0xffff) | (cnt << 16);
	} while (atomic_cmpxchg(segments, old, new) != old);
179 180
}

181
static inline void raid5_set_bi_stripes(struct bio *bio, unsigned int cnt)
182
{
183 184
	atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
	atomic_set(segments, cnt);
185 186
}

187 188 189
/* Find first data disk in a raid6 stripe */
static inline int raid6_d0(struct stripe_head *sh)
{
190 191 192 193
	if (sh->ddf_layout)
		/* ddf always start from first device */
		return 0;
	/* md starts just after Q block */
194 195 196 197 198
	if (sh->qd_idx == sh->disks - 1)
		return 0;
	else
		return sh->qd_idx + 1;
}
199 200 201 202 203
static inline int raid6_next_disk(int disk, int raid_disks)
{
	disk++;
	return (disk < raid_disks) ? disk : 0;
}
204

205 206 207 208 209
/* When walking through the disks in a raid5, starting at raid6_d0,
 * We need to map each disk to a 'slot', where the data disks are slot
 * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk
 * is raid_disks-1.  This help does that mapping.
 */
210 211
static int raid6_idx_to_slot(int idx, struct stripe_head *sh,
			     int *count, int syndrome_disks)
212
{
213
	int slot = *count;
214

215
	if (sh->ddf_layout)
216
		(*count)++;
217
	if (idx == sh->pd_idx)
218
		return syndrome_disks;
219
	if (idx == sh->qd_idx)
220
		return syndrome_disks + 1;
221
	if (!sh->ddf_layout)
222
		(*count)++;
223 224 225
	return slot;
}

226 227 228 229 230 231 232
static void return_io(struct bio *return_bi)
{
	struct bio *bi = return_bi;
	while (bi) {

		return_bi = bi->bi_next;
		bi->bi_next = NULL;
233
		bi->bi_iter.bi_size = 0;
234 235
		trace_block_bio_complete(bdev_get_queue(bi->bi_bdev),
					 bi, 0);
236
		bio_endio(bi, 0);
237 238 239 240
		bi = return_bi;
	}
}

241
static void print_raid5_conf (struct r5conf *conf);
L
Linus Torvalds 已提交
242

243 244 245 246 247 248 249
static int stripe_operations_active(struct stripe_head *sh)
{
	return sh->check_state || sh->reconstruct_state ||
	       test_bit(STRIPE_BIOFILL_RUN, &sh->state) ||
	       test_bit(STRIPE_COMPUTE_RUN, &sh->state);
}

250 251 252 253
static void raid5_wakeup_stripe_thread(struct stripe_head *sh)
{
	struct r5conf *conf = sh->raid_conf;
	struct r5worker_group *group;
254
	int thread_cnt;
255 256 257 258 259 260 261 262 263 264 265
	int i, cpu = sh->cpu;

	if (!cpu_online(cpu)) {
		cpu = cpumask_any(cpu_online_mask);
		sh->cpu = cpu;
	}

	if (list_empty(&sh->lru)) {
		struct r5worker_group *group;
		group = conf->worker_groups + cpu_to_group(cpu);
		list_add_tail(&sh->lru, &group->handle_list);
266 267
		group->stripes_cnt++;
		sh->group = group;
268 269 270 271 272 273 274 275 276
	}

	if (conf->worker_cnt_per_group == 0) {
		md_wakeup_thread(conf->mddev->thread);
		return;
	}

	group = conf->worker_groups + cpu_to_group(sh->cpu);

277 278 279 280 281 282 283 284 285 286 287 288 289 290
	group->workers[0].working = true;
	/* at least one worker should run to avoid race */
	queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work);

	thread_cnt = group->stripes_cnt / MAX_STRIPE_BATCH - 1;
	/* wakeup more workers */
	for (i = 1; i < conf->worker_cnt_per_group && thread_cnt > 0; i++) {
		if (group->workers[i].working == false) {
			group->workers[i].working = true;
			queue_work_on(sh->cpu, raid5_wq,
				      &group->workers[i].work);
			thread_cnt--;
		}
	}
291 292
}

293 294
static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh,
			      struct list_head *temp_inactive_list)
L
Linus Torvalds 已提交
295
{
296 297 298 299
	BUG_ON(!list_empty(&sh->lru));
	BUG_ON(atomic_read(&conf->active_stripes)==0);
	if (test_bit(STRIPE_HANDLE, &sh->state)) {
		if (test_bit(STRIPE_DELAYED, &sh->state) &&
300
		    !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
301
			list_add_tail(&sh->lru, &conf->delayed_list);
302
		else if (test_bit(STRIPE_BIT_DELAY, &sh->state) &&
303 304 305 306 307
			   sh->bm_seq - conf->seq_write > 0)
			list_add_tail(&sh->lru, &conf->bitmap_list);
		else {
			clear_bit(STRIPE_DELAYED, &sh->state);
			clear_bit(STRIPE_BIT_DELAY, &sh->state);
308 309 310 311 312 313
			if (conf->worker_cnt_per_group == 0) {
				list_add_tail(&sh->lru, &conf->handle_list);
			} else {
				raid5_wakeup_stripe_thread(sh);
				return;
			}
314 315 316 317 318 319 320 321 322
		}
		md_wakeup_thread(conf->mddev->thread);
	} else {
		BUG_ON(stripe_operations_active(sh));
		if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
			if (atomic_dec_return(&conf->preread_active_stripes)
			    < IO_THRESHOLD)
				md_wakeup_thread(conf->mddev->thread);
		atomic_dec(&conf->active_stripes);
323 324
		if (!test_bit(STRIPE_EXPANDING, &sh->state))
			list_add_tail(&sh->lru, temp_inactive_list);
L
Linus Torvalds 已提交
325 326
	}
}
327

328 329
static void __release_stripe(struct r5conf *conf, struct stripe_head *sh,
			     struct list_head *temp_inactive_list)
330 331
{
	if (atomic_dec_and_test(&sh->count))
332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363
		do_release_stripe(conf, sh, temp_inactive_list);
}

/*
 * @hash could be NR_STRIPE_HASH_LOCKS, then we have a list of inactive_list
 *
 * Be careful: Only one task can add/delete stripes from temp_inactive_list at
 * given time. Adding stripes only takes device lock, while deleting stripes
 * only takes hash lock.
 */
static void release_inactive_stripe_list(struct r5conf *conf,
					 struct list_head *temp_inactive_list,
					 int hash)
{
	int size;
	bool do_wakeup = false;
	unsigned long flags;

	if (hash == NR_STRIPE_HASH_LOCKS) {
		size = NR_STRIPE_HASH_LOCKS;
		hash = NR_STRIPE_HASH_LOCKS - 1;
	} else
		size = 1;
	while (size) {
		struct list_head *list = &temp_inactive_list[size - 1];

		/*
		 * We don't hold any lock here yet, get_active_stripe() might
		 * remove stripes from the list
		 */
		if (!list_empty_careful(list)) {
			spin_lock_irqsave(conf->hash_locks + hash, flags);
364 365 366
			if (list_empty(conf->inactive_list + hash) &&
			    !list_empty(list))
				atomic_dec(&conf->empty_inactive_list_nr);
367 368 369 370 371 372 373 374 375 376 377 378 379
			list_splice_tail_init(list, conf->inactive_list + hash);
			do_wakeup = true;
			spin_unlock_irqrestore(conf->hash_locks + hash, flags);
		}
		size--;
		hash--;
	}

	if (do_wakeup) {
		wake_up(&conf->wait_for_stripe);
		if (conf->retry_read_aligned)
			md_wakeup_thread(conf->mddev->thread);
	}
380 381
}

S
Shaohua Li 已提交
382
/* should hold conf->device_lock already */
383 384
static int release_stripe_list(struct r5conf *conf,
			       struct list_head *temp_inactive_list)
S
Shaohua Li 已提交
385 386 387 388 389 390
{
	struct stripe_head *sh;
	int count = 0;
	struct llist_node *head;

	head = llist_del_all(&conf->released_stripes);
S
Shaohua Li 已提交
391
	head = llist_reverse_order(head);
S
Shaohua Li 已提交
392
	while (head) {
393 394
		int hash;

S
Shaohua Li 已提交
395 396 397 398 399 400 401 402 403 404
		sh = llist_entry(head, struct stripe_head, release_list);
		head = llist_next(head);
		/* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */
		smp_mb();
		clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state);
		/*
		 * Don't worry the bit is set here, because if the bit is set
		 * again, the count is always > 1. This is true for
		 * STRIPE_ON_UNPLUG_LIST bit too.
		 */
405 406
		hash = sh->hash_lock_index;
		__release_stripe(conf, sh, &temp_inactive_list[hash]);
S
Shaohua Li 已提交
407 408 409 410 411 412
		count++;
	}

	return count;
}

L
Linus Torvalds 已提交
413 414
static void release_stripe(struct stripe_head *sh)
{
415
	struct r5conf *conf = sh->raid_conf;
L
Linus Torvalds 已提交
416
	unsigned long flags;
417 418
	struct list_head list;
	int hash;
S
Shaohua Li 已提交
419
	bool wakeup;
420

421 422 423 424 425
	/* Avoid release_list until the last reference.
	 */
	if (atomic_add_unless(&sh->count, -1, 1))
		return;

426 427
	if (unlikely(!conf->mddev->thread) ||
		test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state))
S
Shaohua Li 已提交
428 429 430 431 432 433
		goto slow_path;
	wakeup = llist_add(&sh->release_list, &conf->released_stripes);
	if (wakeup)
		md_wakeup_thread(conf->mddev->thread);
	return;
slow_path:
434
	local_irq_save(flags);
S
Shaohua Li 已提交
435
	/* we are ok here if STRIPE_ON_RELEASE_LIST is set or not */
436
	if (atomic_dec_and_lock(&sh->count, &conf->device_lock)) {
437 438 439
		INIT_LIST_HEAD(&list);
		hash = sh->hash_lock_index;
		do_release_stripe(conf, sh, &list);
440
		spin_unlock(&conf->device_lock);
441
		release_inactive_stripe_list(conf, &list, hash);
442 443
	}
	local_irq_restore(flags);
L
Linus Torvalds 已提交
444 445
}

446
static inline void remove_hash(struct stripe_head *sh)
L
Linus Torvalds 已提交
447
{
448 449
	pr_debug("remove_hash(), stripe %llu\n",
		(unsigned long long)sh->sector);
L
Linus Torvalds 已提交
450

451
	hlist_del_init(&sh->hash);
L
Linus Torvalds 已提交
452 453
}

454
static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh)
L
Linus Torvalds 已提交
455
{
456
	struct hlist_head *hp = stripe_hash(conf, sh->sector);
L
Linus Torvalds 已提交
457

458 459
	pr_debug("insert_hash(), stripe %llu\n",
		(unsigned long long)sh->sector);
L
Linus Torvalds 已提交
460

461
	hlist_add_head(&sh->hash, hp);
L
Linus Torvalds 已提交
462 463 464
}

/* find an idle stripe, make sure it is unhashed, and return it. */
465
static struct stripe_head *get_free_stripe(struct r5conf *conf, int hash)
L
Linus Torvalds 已提交
466 467 468 469
{
	struct stripe_head *sh = NULL;
	struct list_head *first;

470
	if (list_empty(conf->inactive_list + hash))
L
Linus Torvalds 已提交
471
		goto out;
472
	first = (conf->inactive_list + hash)->next;
L
Linus Torvalds 已提交
473 474 475 476
	sh = list_entry(first, struct stripe_head, lru);
	list_del_init(first);
	remove_hash(sh);
	atomic_inc(&conf->active_stripes);
477
	BUG_ON(hash != sh->hash_lock_index);
478 479
	if (list_empty(conf->inactive_list + hash))
		atomic_inc(&conf->empty_inactive_list_nr);
L
Linus Torvalds 已提交
480 481 482 483
out:
	return sh;
}

484
static void shrink_buffers(struct stripe_head *sh)
L
Linus Torvalds 已提交
485 486 487
{
	struct page *p;
	int i;
488
	int num = sh->raid_conf->pool_size;
L
Linus Torvalds 已提交
489

490
	for (i = 0; i < num ; i++) {
491
		WARN_ON(sh->dev[i].page != sh->dev[i].orig_page);
L
Linus Torvalds 已提交
492 493 494 495
		p = sh->dev[i].page;
		if (!p)
			continue;
		sh->dev[i].page = NULL;
496
		put_page(p);
L
Linus Torvalds 已提交
497 498 499
	}
}

500
static int grow_buffers(struct stripe_head *sh, gfp_t gfp)
L
Linus Torvalds 已提交
501 502
{
	int i;
503
	int num = sh->raid_conf->pool_size;
L
Linus Torvalds 已提交
504

505
	for (i = 0; i < num; i++) {
L
Linus Torvalds 已提交
506 507
		struct page *page;

508
		if (!(page = alloc_page(gfp))) {
L
Linus Torvalds 已提交
509 510 511
			return 1;
		}
		sh->dev[i].page = page;
512
		sh->dev[i].orig_page = page;
L
Linus Torvalds 已提交
513 514 515 516
	}
	return 0;
}

517
static void raid5_build_block(struct stripe_head *sh, int i, int previous);
518
static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,
519
			    struct stripe_head *sh);
L
Linus Torvalds 已提交
520

521
static void init_stripe(struct stripe_head *sh, sector_t sector, int previous)
L
Linus Torvalds 已提交
522
{
523
	struct r5conf *conf = sh->raid_conf;
524
	int i, seq;
L
Linus Torvalds 已提交
525

526 527
	BUG_ON(atomic_read(&sh->count) != 0);
	BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));
528
	BUG_ON(stripe_operations_active(sh));
529
	BUG_ON(sh->batch_head);
530

531
	pr_debug("init_stripe called, stripe %llu\n",
532
		(unsigned long long)sector);
533 534
retry:
	seq = read_seqcount_begin(&conf->gen_lock);
535
	sh->generation = conf->generation - previous;
536
	sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks;
L
Linus Torvalds 已提交
537
	sh->sector = sector;
538
	stripe_set_idx(sector, conf, previous, sh);
L
Linus Torvalds 已提交
539 540
	sh->state = 0;

541
	for (i = sh->disks; i--; ) {
L
Linus Torvalds 已提交
542 543
		struct r5dev *dev = &sh->dev[i];

544
		if (dev->toread || dev->read || dev->towrite || dev->written ||
L
Linus Torvalds 已提交
545
		    test_bit(R5_LOCKED, &dev->flags)) {
546
			printk(KERN_ERR "sector=%llx i=%d %p %p %p %p %d\n",
L
Linus Torvalds 已提交
547
			       (unsigned long long)sh->sector, i, dev->toread,
548
			       dev->read, dev->towrite, dev->written,
L
Linus Torvalds 已提交
549
			       test_bit(R5_LOCKED, &dev->flags));
550
			WARN_ON(1);
L
Linus Torvalds 已提交
551 552
		}
		dev->flags = 0;
553
		raid5_build_block(sh, i, previous);
L
Linus Torvalds 已提交
554
	}
555 556
	if (read_seqcount_retry(&conf->gen_lock, seq))
		goto retry;
557
	sh->overwrite_disks = 0;
L
Linus Torvalds 已提交
558
	insert_hash(conf, sh);
559
	sh->cpu = smp_processor_id();
560
	set_bit(STRIPE_BATCH_READY, &sh->state);
L
Linus Torvalds 已提交
561 562
}

563
static struct stripe_head *__find_stripe(struct r5conf *conf, sector_t sector,
564
					 short generation)
L
Linus Torvalds 已提交
565 566 567
{
	struct stripe_head *sh;

568
	pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector);
569
	hlist_for_each_entry(sh, stripe_hash(conf, sector), hash)
570
		if (sh->sector == sector && sh->generation == generation)
L
Linus Torvalds 已提交
571
			return sh;
572
	pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector);
L
Linus Torvalds 已提交
573 574 575
	return NULL;
}

576 577 578 579 580 581 582 583 584 585 586 587 588
/*
 * Need to check if array has failed when deciding whether to:
 *  - start an array
 *  - remove non-faulty devices
 *  - add a spare
 *  - allow a reshape
 * This determination is simple when no reshape is happening.
 * However if there is a reshape, we need to carefully check
 * both the before and after sections.
 * This is because some failed devices may only affect one
 * of the two sections, and some non-in_sync devices may
 * be insync in the section most affected by failed devices.
 */
589
static int calc_degraded(struct r5conf *conf)
590
{
591
	int degraded, degraded2;
592 593 594 595 596
	int i;

	rcu_read_lock();
	degraded = 0;
	for (i = 0; i < conf->previous_raid_disks; i++) {
597
		struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
598 599
		if (rdev && test_bit(Faulty, &rdev->flags))
			rdev = rcu_dereference(conf->disks[i].replacement);
600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617
		if (!rdev || test_bit(Faulty, &rdev->flags))
			degraded++;
		else if (test_bit(In_sync, &rdev->flags))
			;
		else
			/* not in-sync or faulty.
			 * If the reshape increases the number of devices,
			 * this is being recovered by the reshape, so
			 * this 'previous' section is not in_sync.
			 * If the number of devices is being reduced however,
			 * the device can only be part of the array if
			 * we are reverting a reshape, so this section will
			 * be in-sync.
			 */
			if (conf->raid_disks >= conf->previous_raid_disks)
				degraded++;
	}
	rcu_read_unlock();
618 619
	if (conf->raid_disks == conf->previous_raid_disks)
		return degraded;
620
	rcu_read_lock();
621
	degraded2 = 0;
622
	for (i = 0; i < conf->raid_disks; i++) {
623
		struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
624 625
		if (rdev && test_bit(Faulty, &rdev->flags))
			rdev = rcu_dereference(conf->disks[i].replacement);
626
		if (!rdev || test_bit(Faulty, &rdev->flags))
627
			degraded2++;
628 629 630 631 632 633 634 635 636
		else if (test_bit(In_sync, &rdev->flags))
			;
		else
			/* not in-sync or faulty.
			 * If reshape increases the number of devices, this
			 * section has already been recovered, else it
			 * almost certainly hasn't.
			 */
			if (conf->raid_disks <= conf->previous_raid_disks)
637
				degraded2++;
638 639
	}
	rcu_read_unlock();
640 641 642 643 644 645 646 647 648 649 650 651 652
	if (degraded2 > degraded)
		return degraded2;
	return degraded;
}

static int has_failed(struct r5conf *conf)
{
	int degraded;

	if (conf->mddev->reshape_position == MaxSector)
		return conf->mddev->degraded > conf->max_degraded;

	degraded = calc_degraded(conf);
653 654 655 656 657
	if (degraded > conf->max_degraded)
		return 1;
	return 0;
}

658
static struct stripe_head *
659
get_active_stripe(struct r5conf *conf, sector_t sector,
660
		  int previous, int noblock, int noquiesce)
L
Linus Torvalds 已提交
661 662
{
	struct stripe_head *sh;
663
	int hash = stripe_hash_locks_hash(sector);
L
Linus Torvalds 已提交
664

665
	pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector);
L
Linus Torvalds 已提交
666

667
	spin_lock_irq(conf->hash_locks + hash);
L
Linus Torvalds 已提交
668 669

	do {
670
		wait_event_lock_irq(conf->wait_for_stripe,
671
				    conf->quiesce == 0 || noquiesce,
672
				    *(conf->hash_locks + hash));
673
		sh = __find_stripe(conf, sector, conf->generation - previous);
L
Linus Torvalds 已提交
674
		if (!sh) {
675
			if (!test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state)) {
676
				sh = get_free_stripe(conf, hash);
677 678 679 680 681
				if (!sh && llist_empty(&conf->released_stripes) &&
				    !test_bit(R5_DID_ALLOC, &conf->cache_state))
					set_bit(R5_ALLOC_MORE,
						&conf->cache_state);
			}
L
Linus Torvalds 已提交
682 683 684
			if (noblock && sh == NULL)
				break;
			if (!sh) {
685 686
				set_bit(R5_INACTIVE_BLOCKED,
					&conf->cache_state);
687 688 689 690 691
				wait_event_lock_irq(
					conf->wait_for_stripe,
					!list_empty(conf->inactive_list + hash) &&
					(atomic_read(&conf->active_stripes)
					 < (conf->max_nr_stripes * 3 / 4)
692 693
					 || !test_bit(R5_INACTIVE_BLOCKED,
						      &conf->cache_state)),
694
					*(conf->hash_locks + hash));
695 696
				clear_bit(R5_INACTIVE_BLOCKED,
					  &conf->cache_state);
697
			} else {
698
				init_stripe(sh, sector, previous);
699 700
				atomic_inc(&sh->count);
			}
701
		} else if (!atomic_inc_not_zero(&sh->count)) {
702
			spin_lock(&conf->device_lock);
703
			if (!atomic_read(&sh->count)) {
L
Linus Torvalds 已提交
704 705
				if (!test_bit(STRIPE_HANDLE, &sh->state))
					atomic_inc(&conf->active_stripes);
706 707
				BUG_ON(list_empty(&sh->lru) &&
				       !test_bit(STRIPE_EXPANDING, &sh->state));
708
				list_del_init(&sh->lru);
709 710 711 712
				if (sh->group) {
					sh->group->stripes_cnt--;
					sh->group = NULL;
				}
L
Linus Torvalds 已提交
713
			}
714
			atomic_inc(&sh->count);
715
			spin_unlock(&conf->device_lock);
L
Linus Torvalds 已提交
716 717 718
		}
	} while (sh == NULL);

719
	spin_unlock_irq(conf->hash_locks + hash);
L
Linus Torvalds 已提交
720 721 722
	return sh;
}

723 724 725 726 727 728
static bool is_full_stripe_write(struct stripe_head *sh)
{
	BUG_ON(sh->overwrite_disks > (sh->disks - sh->raid_conf->max_degraded));
	return sh->overwrite_disks == (sh->disks - sh->raid_conf->max_degraded);
}

729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846
static void lock_two_stripes(struct stripe_head *sh1, struct stripe_head *sh2)
{
	local_irq_disable();
	if (sh1 > sh2) {
		spin_lock(&sh2->stripe_lock);
		spin_lock_nested(&sh1->stripe_lock, 1);
	} else {
		spin_lock(&sh1->stripe_lock);
		spin_lock_nested(&sh2->stripe_lock, 1);
	}
}

static void unlock_two_stripes(struct stripe_head *sh1, struct stripe_head *sh2)
{
	spin_unlock(&sh1->stripe_lock);
	spin_unlock(&sh2->stripe_lock);
	local_irq_enable();
}

/* Only freshly new full stripe normal write stripe can be added to a batch list */
static bool stripe_can_batch(struct stripe_head *sh)
{
	return test_bit(STRIPE_BATCH_READY, &sh->state) &&
		is_full_stripe_write(sh);
}

/* we only do back search */
static void stripe_add_to_batch_list(struct r5conf *conf, struct stripe_head *sh)
{
	struct stripe_head *head;
	sector_t head_sector, tmp_sec;
	int hash;
	int dd_idx;

	if (!stripe_can_batch(sh))
		return;
	/* Don't cross chunks, so stripe pd_idx/qd_idx is the same */
	tmp_sec = sh->sector;
	if (!sector_div(tmp_sec, conf->chunk_sectors))
		return;
	head_sector = sh->sector - STRIPE_SECTORS;

	hash = stripe_hash_locks_hash(head_sector);
	spin_lock_irq(conf->hash_locks + hash);
	head = __find_stripe(conf, head_sector, conf->generation);
	if (head && !atomic_inc_not_zero(&head->count)) {
		spin_lock(&conf->device_lock);
		if (!atomic_read(&head->count)) {
			if (!test_bit(STRIPE_HANDLE, &head->state))
				atomic_inc(&conf->active_stripes);
			BUG_ON(list_empty(&head->lru) &&
			       !test_bit(STRIPE_EXPANDING, &head->state));
			list_del_init(&head->lru);
			if (head->group) {
				head->group->stripes_cnt--;
				head->group = NULL;
			}
		}
		atomic_inc(&head->count);
		spin_unlock(&conf->device_lock);
	}
	spin_unlock_irq(conf->hash_locks + hash);

	if (!head)
		return;
	if (!stripe_can_batch(head))
		goto out;

	lock_two_stripes(head, sh);
	/* clear_batch_ready clear the flag */
	if (!stripe_can_batch(head) || !stripe_can_batch(sh))
		goto unlock_out;

	if (sh->batch_head)
		goto unlock_out;

	dd_idx = 0;
	while (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx)
		dd_idx++;
	if (head->dev[dd_idx].towrite->bi_rw != sh->dev[dd_idx].towrite->bi_rw)
		goto unlock_out;

	if (head->batch_head) {
		spin_lock(&head->batch_head->batch_lock);
		/* This batch list is already running */
		if (!stripe_can_batch(head)) {
			spin_unlock(&head->batch_head->batch_lock);
			goto unlock_out;
		}

		/*
		 * at this point, head's BATCH_READY could be cleared, but we
		 * can still add the stripe to batch list
		 */
		list_add(&sh->batch_list, &head->batch_list);
		spin_unlock(&head->batch_head->batch_lock);

		sh->batch_head = head->batch_head;
	} else {
		head->batch_head = head;
		sh->batch_head = head->batch_head;
		spin_lock(&head->batch_lock);
		list_add_tail(&sh->batch_list, &head->batch_list);
		spin_unlock(&head->batch_lock);
	}

	if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
		if (atomic_dec_return(&conf->preread_active_stripes)
		    < IO_THRESHOLD)
			md_wakeup_thread(conf->mddev->thread);

	atomic_inc(&sh->count);
unlock_out:
	unlock_two_stripes(head, sh);
out:
	release_stripe(head);
}

847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867
/* Determine if 'data_offset' or 'new_data_offset' should be used
 * in this stripe_head.
 */
static int use_new_offset(struct r5conf *conf, struct stripe_head *sh)
{
	sector_t progress = conf->reshape_progress;
	/* Need a memory barrier to make sure we see the value
	 * of conf->generation, or ->data_offset that was set before
	 * reshape_progress was updated.
	 */
	smp_rmb();
	if (progress == MaxSector)
		return 0;
	if (sh->generation == conf->generation - 1)
		return 0;
	/* We are in a reshape, and this is a new-generation stripe,
	 * so use new_data_offset.
	 */
	return 1;
}

868 869 870 871
static void
raid5_end_read_request(struct bio *bi, int error);
static void
raid5_end_write_request(struct bio *bi, int error);
872

873
static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)
874
{
875
	struct r5conf *conf = sh->raid_conf;
876
	int i, disks = sh->disks;
877
	struct stripe_head *head_sh = sh;
878 879 880 881 882

	might_sleep();

	for (i = disks; i--; ) {
		int rw;
883
		int replace_only = 0;
884 885
		struct bio *bi, *rbi;
		struct md_rdev *rdev, *rrdev = NULL;
886 887

		sh = head_sh;
T
Tejun Heo 已提交
888 889 890 891 892
		if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) {
			if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags))
				rw = WRITE_FUA;
			else
				rw = WRITE;
893
			if (test_bit(R5_Discard, &sh->dev[i].flags))
S
Shaohua Li 已提交
894
				rw |= REQ_DISCARD;
T
Tejun Heo 已提交
895
		} else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags))
896
			rw = READ;
897 898 899 900 901
		else if (test_and_clear_bit(R5_WantReplace,
					    &sh->dev[i].flags)) {
			rw = WRITE;
			replace_only = 1;
		} else
902
			continue;
S
Shaohua Li 已提交
903 904
		if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags))
			rw |= REQ_SYNC;
905

906
again:
907
		bi = &sh->dev[i].req;
908
		rbi = &sh->dev[i].rreq; /* For writing to replacement */
909 910

		rcu_read_lock();
911
		rrdev = rcu_dereference(conf->disks[i].replacement);
912 913 914 915 916 917
		smp_mb(); /* Ensure that if rrdev is NULL, rdev won't be */
		rdev = rcu_dereference(conf->disks[i].rdev);
		if (!rdev) {
			rdev = rrdev;
			rrdev = NULL;
		}
918 919 920
		if (rw & WRITE) {
			if (replace_only)
				rdev = NULL;
921 922 923
			if (rdev == rrdev)
				/* We raced and saw duplicates */
				rrdev = NULL;
924
		} else {
925
			if (test_bit(R5_ReadRepl, &head_sh->dev[i].flags) && rrdev)
926 927 928
				rdev = rrdev;
			rrdev = NULL;
		}
929

930 931 932 933
		if (rdev && test_bit(Faulty, &rdev->flags))
			rdev = NULL;
		if (rdev)
			atomic_inc(&rdev->nr_pending);
934 935 936 937
		if (rrdev && test_bit(Faulty, &rrdev->flags))
			rrdev = NULL;
		if (rrdev)
			atomic_inc(&rrdev->nr_pending);
938 939
		rcu_read_unlock();

940
		/* We have already checked bad blocks for reads.  Now
941 942
		 * need to check for writes.  We never accept write errors
		 * on the replacement, so we don't to check rrdev.
943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962
		 */
		while ((rw & WRITE) && rdev &&
		       test_bit(WriteErrorSeen, &rdev->flags)) {
			sector_t first_bad;
			int bad_sectors;
			int bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS,
					      &first_bad, &bad_sectors);
			if (!bad)
				break;

			if (bad < 0) {
				set_bit(BlockedBadBlocks, &rdev->flags);
				if (!conf->mddev->external &&
				    conf->mddev->flags) {
					/* It is very unlikely, but we might
					 * still need to write out the
					 * bad block log - better give it
					 * a chance*/
					md_check_recovery(conf->mddev);
				}
963 964 965 966 967 968
				/*
				 * Because md_wait_for_blocked_rdev
				 * will dec nr_pending, we must
				 * increment it first.
				 */
				atomic_inc(&rdev->nr_pending);
969 970 971 972 973 974 975 976
				md_wait_for_blocked_rdev(rdev, conf->mddev);
			} else {
				/* Acknowledged bad block - skip the write */
				rdev_dec_pending(rdev, conf->mddev);
				rdev = NULL;
			}
		}

977
		if (rdev) {
978 979
			if (s->syncing || s->expanding || s->expanded
			    || s->replacing)
980 981
				md_sync_acct(rdev->bdev, STRIPE_SECTORS);

D
Dan Williams 已提交
982 983
			set_bit(STRIPE_IO_STARTED, &sh->state);

K
Kent Overstreet 已提交
984
			bio_reset(bi);
985
			bi->bi_bdev = rdev->bdev;
K
Kent Overstreet 已提交
986 987 988 989 990 991
			bi->bi_rw = rw;
			bi->bi_end_io = (rw & WRITE)
				? raid5_end_write_request
				: raid5_end_read_request;
			bi->bi_private = sh;

992
			pr_debug("%s: for %llu schedule op %ld on disc %d\n",
993
				__func__, (unsigned long long)sh->sector,
994 995
				bi->bi_rw, i);
			atomic_inc(&sh->count);
996 997
			if (sh != head_sh)
				atomic_inc(&head_sh->count);
998
			if (use_new_offset(conf, sh))
999
				bi->bi_iter.bi_sector = (sh->sector
1000 1001
						 + rdev->new_data_offset);
			else
1002
				bi->bi_iter.bi_sector = (sh->sector
1003
						 + rdev->data_offset);
1004
			if (test_bit(R5_ReadNoMerge, &head_sh->dev[i].flags))
1005
				bi->bi_rw |= REQ_NOMERGE;
1006

1007 1008 1009
			if (test_bit(R5_SkipCopy, &sh->dev[i].flags))
				WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
			sh->dev[i].vec.bv_page = sh->dev[i].page;
K
Kent Overstreet 已提交
1010
			bi->bi_vcnt = 1;
1011 1012
			bi->bi_io_vec[0].bv_len = STRIPE_SIZE;
			bi->bi_io_vec[0].bv_offset = 0;
1013
			bi->bi_iter.bi_size = STRIPE_SIZE;
1014 1015 1016 1017 1018 1019
			/*
			 * If this is discard request, set bi_vcnt 0. We don't
			 * want to confuse SCSI because SCSI will replace payload
			 */
			if (rw & REQ_DISCARD)
				bi->bi_vcnt = 0;
1020 1021
			if (rrdev)
				set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags);
1022 1023 1024 1025 1026

			if (conf->mddev->gendisk)
				trace_block_bio_remap(bdev_get_queue(bi->bi_bdev),
						      bi, disk_devt(conf->mddev->gendisk),
						      sh->dev[i].sector);
1027
			generic_make_request(bi);
1028 1029
		}
		if (rrdev) {
1030 1031
			if (s->syncing || s->expanding || s->expanded
			    || s->replacing)
1032 1033 1034 1035
				md_sync_acct(rrdev->bdev, STRIPE_SECTORS);

			set_bit(STRIPE_IO_STARTED, &sh->state);

K
Kent Overstreet 已提交
1036
			bio_reset(rbi);
1037
			rbi->bi_bdev = rrdev->bdev;
K
Kent Overstreet 已提交
1038 1039 1040 1041 1042
			rbi->bi_rw = rw;
			BUG_ON(!(rw & WRITE));
			rbi->bi_end_io = raid5_end_write_request;
			rbi->bi_private = sh;

1043 1044 1045 1046 1047
			pr_debug("%s: for %llu schedule op %ld on "
				 "replacement disc %d\n",
				__func__, (unsigned long long)sh->sector,
				rbi->bi_rw, i);
			atomic_inc(&sh->count);
1048 1049
			if (sh != head_sh)
				atomic_inc(&head_sh->count);
1050
			if (use_new_offset(conf, sh))
1051
				rbi->bi_iter.bi_sector = (sh->sector
1052 1053
						  + rrdev->new_data_offset);
			else
1054
				rbi->bi_iter.bi_sector = (sh->sector
1055
						  + rrdev->data_offset);
1056 1057 1058
			if (test_bit(R5_SkipCopy, &sh->dev[i].flags))
				WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
			sh->dev[i].rvec.bv_page = sh->dev[i].page;
K
Kent Overstreet 已提交
1059
			rbi->bi_vcnt = 1;
1060 1061
			rbi->bi_io_vec[0].bv_len = STRIPE_SIZE;
			rbi->bi_io_vec[0].bv_offset = 0;
1062
			rbi->bi_iter.bi_size = STRIPE_SIZE;
1063 1064 1065 1066 1067 1068
			/*
			 * If this is discard request, set bi_vcnt 0. We don't
			 * want to confuse SCSI because SCSI will replace payload
			 */
			if (rw & REQ_DISCARD)
				rbi->bi_vcnt = 0;
1069 1070 1071 1072
			if (conf->mddev->gendisk)
				trace_block_bio_remap(bdev_get_queue(rbi->bi_bdev),
						      rbi, disk_devt(conf->mddev->gendisk),
						      sh->dev[i].sector);
1073 1074 1075
			generic_make_request(rbi);
		}
		if (!rdev && !rrdev) {
1076
			if (rw & WRITE)
1077 1078 1079 1080 1081 1082
				set_bit(STRIPE_DEGRADED, &sh->state);
			pr_debug("skip op %ld on disc %d for sector %llu\n",
				bi->bi_rw, i, (unsigned long long)sh->sector);
			clear_bit(R5_LOCKED, &sh->dev[i].flags);
			set_bit(STRIPE_HANDLE, &sh->state);
		}
1083 1084 1085 1086 1087 1088 1089

		if (!head_sh->batch_head)
			continue;
		sh = list_first_entry(&sh->batch_list, struct stripe_head,
				      batch_list);
		if (sh != head_sh)
			goto again;
1090 1091 1092 1093
	}
}

static struct dma_async_tx_descriptor *
1094 1095 1096
async_copy_data(int frombio, struct bio *bio, struct page **page,
	sector_t sector, struct dma_async_tx_descriptor *tx,
	struct stripe_head *sh)
1097
{
1098 1099
	struct bio_vec bvl;
	struct bvec_iter iter;
1100 1101
	struct page *bio_page;
	int page_offset;
1102
	struct async_submit_ctl submit;
D
Dan Williams 已提交
1103
	enum async_tx_flags flags = 0;
1104

1105 1106
	if (bio->bi_iter.bi_sector >= sector)
		page_offset = (signed)(bio->bi_iter.bi_sector - sector) * 512;
1107
	else
1108
		page_offset = (signed)(sector - bio->bi_iter.bi_sector) * -512;
1109

D
Dan Williams 已提交
1110 1111 1112 1113
	if (frombio)
		flags |= ASYNC_TX_FENCE;
	init_async_submit(&submit, flags, tx, NULL, NULL, NULL);

1114 1115
	bio_for_each_segment(bvl, bio, iter) {
		int len = bvl.bv_len;
1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130
		int clen;
		int b_offset = 0;

		if (page_offset < 0) {
			b_offset = -page_offset;
			page_offset += b_offset;
			len -= b_offset;
		}

		if (len > 0 && page_offset + len > STRIPE_SIZE)
			clen = STRIPE_SIZE - page_offset;
		else
			clen = len;

		if (clen > 0) {
1131 1132
			b_offset += bvl.bv_offset;
			bio_page = bvl.bv_page;
1133 1134 1135 1136 1137 1138 1139
			if (frombio) {
				if (sh->raid_conf->skip_copy &&
				    b_offset == 0 && page_offset == 0 &&
				    clen == STRIPE_SIZE)
					*page = bio_page;
				else
					tx = async_memcpy(*page, bio_page, page_offset,
1140
						  b_offset, clen, &submit);
1141 1142
			} else
				tx = async_memcpy(bio_page, *page, b_offset,
1143
						  page_offset, clen, &submit);
1144
		}
1145 1146 1147
		/* chain the operations */
		submit.depend_tx = tx;

1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159
		if (clen < len) /* hit end of page */
			break;
		page_offset +=  len;
	}

	return tx;
}

static void ops_complete_biofill(void *stripe_head_ref)
{
	struct stripe_head *sh = stripe_head_ref;
	struct bio *return_bi = NULL;
1160
	int i;
1161

1162
	pr_debug("%s: stripe %llu\n", __func__,
1163 1164 1165 1166 1167 1168 1169
		(unsigned long long)sh->sector);

	/* clear completed biofills */
	for (i = sh->disks; i--; ) {
		struct r5dev *dev = &sh->dev[i];

		/* acknowledge completion of a biofill operation */
1170 1171
		/* and check if we need to reply to a read request,
		 * new R5_Wantfill requests are held off until
1172
		 * !STRIPE_BIOFILL_RUN
1173 1174
		 */
		if (test_and_clear_bit(R5_Wantfill, &dev->flags)) {
1175 1176 1177 1178 1179
			struct bio *rbi, *rbi2;

			BUG_ON(!dev->read);
			rbi = dev->read;
			dev->read = NULL;
1180
			while (rbi && rbi->bi_iter.bi_sector <
1181 1182
				dev->sector + STRIPE_SECTORS) {
				rbi2 = r5_next_bio(rbi, dev->sector);
1183
				if (!raid5_dec_bi_active_stripes(rbi)) {
1184 1185 1186 1187 1188 1189 1190
					rbi->bi_next = return_bi;
					return_bi = rbi;
				}
				rbi = rbi2;
			}
		}
	}
1191
	clear_bit(STRIPE_BIOFILL_RUN, &sh->state);
1192 1193 1194

	return_io(return_bi);

1195
	set_bit(STRIPE_HANDLE, &sh->state);
1196 1197 1198 1199 1200 1201
	release_stripe(sh);
}

static void ops_run_biofill(struct stripe_head *sh)
{
	struct dma_async_tx_descriptor *tx = NULL;
1202
	struct async_submit_ctl submit;
1203 1204
	int i;

1205
	BUG_ON(sh->batch_head);
1206
	pr_debug("%s: stripe %llu\n", __func__,
1207 1208 1209 1210 1211 1212
		(unsigned long long)sh->sector);

	for (i = sh->disks; i--; ) {
		struct r5dev *dev = &sh->dev[i];
		if (test_bit(R5_Wantfill, &dev->flags)) {
			struct bio *rbi;
S
Shaohua Li 已提交
1213
			spin_lock_irq(&sh->stripe_lock);
1214 1215
			dev->read = rbi = dev->toread;
			dev->toread = NULL;
S
Shaohua Li 已提交
1216
			spin_unlock_irq(&sh->stripe_lock);
1217
			while (rbi && rbi->bi_iter.bi_sector <
1218
				dev->sector + STRIPE_SECTORS) {
1219 1220
				tx = async_copy_data(0, rbi, &dev->page,
					dev->sector, tx, sh);
1221 1222 1223 1224 1225 1226
				rbi = r5_next_bio(rbi, dev->sector);
			}
		}
	}

	atomic_inc(&sh->count);
1227 1228
	init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL);
	async_trigger_callback(&submit);
1229 1230
}

1231
static void mark_target_uptodate(struct stripe_head *sh, int target)
1232
{
1233
	struct r5dev *tgt;
1234

1235 1236
	if (target < 0)
		return;
1237

1238
	tgt = &sh->dev[target];
1239 1240 1241
	set_bit(R5_UPTODATE, &tgt->flags);
	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
	clear_bit(R5_Wantcompute, &tgt->flags);
1242 1243
}

1244
static void ops_complete_compute(void *stripe_head_ref)
1245 1246 1247
{
	struct stripe_head *sh = stripe_head_ref;

1248
	pr_debug("%s: stripe %llu\n", __func__,
1249 1250
		(unsigned long long)sh->sector);

1251
	/* mark the computed target(s) as uptodate */
1252
	mark_target_uptodate(sh, sh->ops.target);
1253
	mark_target_uptodate(sh, sh->ops.target2);
1254

1255 1256 1257
	clear_bit(STRIPE_COMPUTE_RUN, &sh->state);
	if (sh->check_state == check_state_compute_run)
		sh->check_state = check_state_compute_result;
1258 1259 1260 1261
	set_bit(STRIPE_HANDLE, &sh->state);
	release_stripe(sh);
}

1262 1263
/* return a pointer to the address conversion region of the scribble buffer */
static addr_conv_t *to_addr_conv(struct stripe_head *sh,
1264
				 struct raid5_percpu *percpu, int i)
1265
{
1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278
	void *addr;

	addr = flex_array_get(percpu->scribble, i);
	return addr + sizeof(struct page *) * (sh->disks + 2);
}

/* return a pointer to the address conversion region of the scribble buffer */
static struct page **to_addr_page(struct raid5_percpu *percpu, int i)
{
	void *addr;

	addr = flex_array_get(percpu->scribble, i);
	return addr;
1279 1280 1281 1282
}

static struct dma_async_tx_descriptor *
ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)
1283 1284
{
	int disks = sh->disks;
1285
	struct page **xor_srcs = to_addr_page(percpu, 0);
1286 1287 1288 1289 1290
	int target = sh->ops.target;
	struct r5dev *tgt = &sh->dev[target];
	struct page *xor_dest = tgt->page;
	int count = 0;
	struct dma_async_tx_descriptor *tx;
1291
	struct async_submit_ctl submit;
1292 1293
	int i;

1294 1295
	BUG_ON(sh->batch_head);

1296
	pr_debug("%s: stripe %llu block: %d\n",
1297
		__func__, (unsigned long long)sh->sector, target);
1298 1299 1300 1301 1302 1303 1304 1305
	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));

	for (i = disks; i--; )
		if (i != target)
			xor_srcs[count++] = sh->dev[i].page;

	atomic_inc(&sh->count);

D
Dan Williams 已提交
1306
	init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL,
1307
			  ops_complete_compute, sh, to_addr_conv(sh, percpu, 0));
1308
	if (unlikely(count == 1))
1309
		tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
1310
	else
1311
		tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
1312 1313 1314 1315

	return tx;
}

1316 1317 1318 1319 1320 1321 1322 1323 1324
/* set_syndrome_sources - populate source buffers for gen_syndrome
 * @srcs - (struct page *) array of size sh->disks
 * @sh - stripe_head to parse
 *
 * Populates srcs in proper layout order for the stripe and returns the
 * 'count' of sources to be used in a call to async_gen_syndrome.  The P
 * destination buffer is recorded in srcs[count] and the Q destination
 * is recorded in srcs[count+1]].
 */
1325 1326 1327
static int set_syndrome_sources(struct page **srcs,
				struct stripe_head *sh,
				int srctype)
1328 1329 1330 1331 1332 1333 1334 1335
{
	int disks = sh->disks;
	int syndrome_disks = sh->ddf_layout ? disks : (disks - 2);
	int d0_idx = raid6_d0(sh);
	int count;
	int i;

	for (i = 0; i < disks; i++)
1336
		srcs[i] = NULL;
1337 1338 1339 1340 1341

	count = 0;
	i = d0_idx;
	do {
		int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
1342
		struct r5dev *dev = &sh->dev[i];
1343

1344 1345 1346 1347 1348 1349 1350
		if (i == sh->qd_idx || i == sh->pd_idx ||
		    (srctype == SYNDROME_SRC_ALL) ||
		    (srctype == SYNDROME_SRC_WANT_DRAIN &&
		     test_bit(R5_Wantdrain, &dev->flags)) ||
		    (srctype == SYNDROME_SRC_WRITTEN &&
		     dev->written))
			srcs[slot] = sh->dev[i].page;
1351 1352 1353
		i = raid6_next_disk(i, disks);
	} while (i != d0_idx);

1354
	return syndrome_disks;
1355 1356 1357 1358 1359 1360
}

static struct dma_async_tx_descriptor *
ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)
{
	int disks = sh->disks;
1361
	struct page **blocks = to_addr_page(percpu, 0);
1362 1363 1364 1365 1366 1367 1368 1369 1370
	int target;
	int qd_idx = sh->qd_idx;
	struct dma_async_tx_descriptor *tx;
	struct async_submit_ctl submit;
	struct r5dev *tgt;
	struct page *dest;
	int i;
	int count;

1371
	BUG_ON(sh->batch_head);
1372 1373 1374 1375
	if (sh->ops.target < 0)
		target = sh->ops.target2;
	else if (sh->ops.target2 < 0)
		target = sh->ops.target;
1376
	else
1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389
		/* we should only have one valid target */
		BUG();
	BUG_ON(target < 0);
	pr_debug("%s: stripe %llu block: %d\n",
		__func__, (unsigned long long)sh->sector, target);

	tgt = &sh->dev[target];
	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
	dest = tgt->page;

	atomic_inc(&sh->count);

	if (target == qd_idx) {
1390
		count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_ALL);
1391 1392
		blocks[count] = NULL; /* regenerating p is not necessary */
		BUG_ON(blocks[count+1] != dest); /* q should already be set */
D
Dan Williams 已提交
1393 1394
		init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
				  ops_complete_compute, sh,
1395
				  to_addr_conv(sh, percpu, 0));
1396 1397 1398 1399 1400 1401 1402 1403 1404 1405
		tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
	} else {
		/* Compute any data- or p-drive using XOR */
		count = 0;
		for (i = disks; i-- ; ) {
			if (i == target || i == qd_idx)
				continue;
			blocks[count++] = sh->dev[i].page;
		}

D
Dan Williams 已提交
1406 1407
		init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
				  NULL, ops_complete_compute, sh,
1408
				  to_addr_conv(sh, percpu, 0));
1409 1410
		tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit);
	}
1411 1412 1413 1414

	return tx;
}

1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426
static struct dma_async_tx_descriptor *
ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
{
	int i, count, disks = sh->disks;
	int syndrome_disks = sh->ddf_layout ? disks : disks-2;
	int d0_idx = raid6_d0(sh);
	int faila = -1, failb = -1;
	int target = sh->ops.target;
	int target2 = sh->ops.target2;
	struct r5dev *tgt = &sh->dev[target];
	struct r5dev *tgt2 = &sh->dev[target2];
	struct dma_async_tx_descriptor *tx;
1427
	struct page **blocks = to_addr_page(percpu, 0);
1428 1429
	struct async_submit_ctl submit;

1430
	BUG_ON(sh->batch_head);
1431 1432 1433 1434 1435 1436
	pr_debug("%s: stripe %llu block1: %d block2: %d\n",
		 __func__, (unsigned long long)sh->sector, target, target2);
	BUG_ON(target < 0 || target2 < 0);
	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
	BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags));

1437
	/* we need to open-code set_syndrome_sources to handle the
1438 1439 1440
	 * slot number conversion for 'faila' and 'failb'
	 */
	for (i = 0; i < disks ; i++)
1441
		blocks[i] = NULL;
1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467
	count = 0;
	i = d0_idx;
	do {
		int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);

		blocks[slot] = sh->dev[i].page;

		if (i == target)
			faila = slot;
		if (i == target2)
			failb = slot;
		i = raid6_next_disk(i, disks);
	} while (i != d0_idx);

	BUG_ON(faila == failb);
	if (failb < faila)
		swap(faila, failb);
	pr_debug("%s: stripe: %llu faila: %d failb: %d\n",
		 __func__, (unsigned long long)sh->sector, faila, failb);

	atomic_inc(&sh->count);

	if (failb == syndrome_disks+1) {
		/* Q disk is one of the missing disks */
		if (faila == syndrome_disks) {
			/* Missing P+Q, just recompute */
D
Dan Williams 已提交
1468 1469
			init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
					  ops_complete_compute, sh,
1470
					  to_addr_conv(sh, percpu, 0));
1471
			return async_gen_syndrome(blocks, 0, syndrome_disks+2,
1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490
						  STRIPE_SIZE, &submit);
		} else {
			struct page *dest;
			int data_target;
			int qd_idx = sh->qd_idx;

			/* Missing D+Q: recompute D from P, then recompute Q */
			if (target == qd_idx)
				data_target = target2;
			else
				data_target = target;

			count = 0;
			for (i = disks; i-- ; ) {
				if (i == data_target || i == qd_idx)
					continue;
				blocks[count++] = sh->dev[i].page;
			}
			dest = sh->dev[data_target].page;
D
Dan Williams 已提交
1491 1492 1493
			init_async_submit(&submit,
					  ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
					  NULL, NULL, NULL,
1494
					  to_addr_conv(sh, percpu, 0));
1495 1496 1497
			tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE,
				       &submit);

1498
			count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_ALL);
D
Dan Williams 已提交
1499 1500
			init_async_submit(&submit, ASYNC_TX_FENCE, tx,
					  ops_complete_compute, sh,
1501
					  to_addr_conv(sh, percpu, 0));
1502 1503 1504 1505
			return async_gen_syndrome(blocks, 0, count+2,
						  STRIPE_SIZE, &submit);
		}
	} else {
1506 1507
		init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
				  ops_complete_compute, sh,
1508
				  to_addr_conv(sh, percpu, 0));
1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519
		if (failb == syndrome_disks) {
			/* We're missing D+P. */
			return async_raid6_datap_recov(syndrome_disks+2,
						       STRIPE_SIZE, faila,
						       blocks, &submit);
		} else {
			/* We're missing D+D. */
			return async_raid6_2data_recov(syndrome_disks+2,
						       STRIPE_SIZE, faila, failb,
						       blocks, &submit);
		}
1520 1521 1522
	}
}

1523 1524 1525 1526
static void ops_complete_prexor(void *stripe_head_ref)
{
	struct stripe_head *sh = stripe_head_ref;

1527
	pr_debug("%s: stripe %llu\n", __func__,
1528 1529 1530 1531
		(unsigned long long)sh->sector);
}

static struct dma_async_tx_descriptor *
1532 1533
ops_run_prexor5(struct stripe_head *sh, struct raid5_percpu *percpu,
		struct dma_async_tx_descriptor *tx)
1534 1535
{
	int disks = sh->disks;
1536
	struct page **xor_srcs = to_addr_page(percpu, 0);
1537
	int count = 0, pd_idx = sh->pd_idx, i;
1538
	struct async_submit_ctl submit;
1539 1540 1541 1542

	/* existing parity data subtracted */
	struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;

1543
	BUG_ON(sh->batch_head);
1544
	pr_debug("%s: stripe %llu\n", __func__,
1545 1546 1547 1548 1549
		(unsigned long long)sh->sector);

	for (i = disks; i--; ) {
		struct r5dev *dev = &sh->dev[i];
		/* Only process blocks that are known to be uptodate */
1550
		if (test_bit(R5_Wantdrain, &dev->flags))
1551 1552 1553
			xor_srcs[count++] = dev->page;
	}

D
Dan Williams 已提交
1554
	init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
1555
			  ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
1556
	tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
1557 1558 1559 1560

	return tx;
}

1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580
static struct dma_async_tx_descriptor *
ops_run_prexor6(struct stripe_head *sh, struct raid5_percpu *percpu,
		struct dma_async_tx_descriptor *tx)
{
	struct page **blocks = to_addr_page(percpu, 0);
	int count;
	struct async_submit_ctl submit;

	pr_debug("%s: stripe %llu\n", __func__,
		(unsigned long long)sh->sector);

	count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_WANT_DRAIN);

	init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_PQ_XOR_DST, tx,
			  ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
	tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE,  &submit);

	return tx;
}

1581
static struct dma_async_tx_descriptor *
1582
ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
1583 1584
{
	int disks = sh->disks;
1585
	int i;
1586
	struct stripe_head *head_sh = sh;
1587

1588
	pr_debug("%s: stripe %llu\n", __func__,
1589 1590 1591
		(unsigned long long)sh->sector);

	for (i = disks; i--; ) {
1592
		struct r5dev *dev;
1593 1594
		struct bio *chosen;

1595 1596
		sh = head_sh;
		if (test_and_clear_bit(R5_Wantdrain, &head_sh->dev[i].flags)) {
1597 1598
			struct bio *wbi;

1599 1600
again:
			dev = &sh->dev[i];
S
Shaohua Li 已提交
1601
			spin_lock_irq(&sh->stripe_lock);
1602 1603
			chosen = dev->towrite;
			dev->towrite = NULL;
1604
			sh->overwrite_disks = 0;
1605 1606
			BUG_ON(dev->written);
			wbi = dev->written = chosen;
S
Shaohua Li 已提交
1607
			spin_unlock_irq(&sh->stripe_lock);
1608
			WARN_ON(dev->page != dev->orig_page);
1609

1610
			while (wbi && wbi->bi_iter.bi_sector <
1611
				dev->sector + STRIPE_SECTORS) {
T
Tejun Heo 已提交
1612 1613
				if (wbi->bi_rw & REQ_FUA)
					set_bit(R5_WantFUA, &dev->flags);
S
Shaohua Li 已提交
1614 1615
				if (wbi->bi_rw & REQ_SYNC)
					set_bit(R5_SyncIO, &dev->flags);
1616
				if (wbi->bi_rw & REQ_DISCARD)
S
Shaohua Li 已提交
1617
					set_bit(R5_Discard, &dev->flags);
1618 1619 1620 1621 1622 1623 1624 1625 1626
				else {
					tx = async_copy_data(1, wbi, &dev->page,
						dev->sector, tx, sh);
					if (dev->page != dev->orig_page) {
						set_bit(R5_SkipCopy, &dev->flags);
						clear_bit(R5_UPTODATE, &dev->flags);
						clear_bit(R5_OVERWRITE, &dev->flags);
					}
				}
1627 1628
				wbi = r5_next_bio(wbi, dev->sector);
			}
1629 1630 1631 1632 1633 1634 1635 1636 1637

			if (head_sh->batch_head) {
				sh = list_first_entry(&sh->batch_list,
						      struct stripe_head,
						      batch_list);
				if (sh == head_sh)
					continue;
				goto again;
			}
1638 1639 1640 1641 1642 1643
		}
	}

	return tx;
}

1644
static void ops_complete_reconstruct(void *stripe_head_ref)
1645 1646
{
	struct stripe_head *sh = stripe_head_ref;
1647 1648 1649 1650
	int disks = sh->disks;
	int pd_idx = sh->pd_idx;
	int qd_idx = sh->qd_idx;
	int i;
1651
	bool fua = false, sync = false, discard = false;
1652

1653
	pr_debug("%s: stripe %llu\n", __func__,
1654 1655
		(unsigned long long)sh->sector);

S
Shaohua Li 已提交
1656
	for (i = disks; i--; ) {
T
Tejun Heo 已提交
1657
		fua |= test_bit(R5_WantFUA, &sh->dev[i].flags);
S
Shaohua Li 已提交
1658
		sync |= test_bit(R5_SyncIO, &sh->dev[i].flags);
1659
		discard |= test_bit(R5_Discard, &sh->dev[i].flags);
S
Shaohua Li 已提交
1660
	}
T
Tejun Heo 已提交
1661

1662 1663
	for (i = disks; i--; ) {
		struct r5dev *dev = &sh->dev[i];
1664

T
Tejun Heo 已提交
1665
		if (dev->written || i == pd_idx || i == qd_idx) {
1666
			if (!discard && !test_bit(R5_SkipCopy, &dev->flags))
1667
				set_bit(R5_UPTODATE, &dev->flags);
T
Tejun Heo 已提交
1668 1669
			if (fua)
				set_bit(R5_WantFUA, &dev->flags);
S
Shaohua Li 已提交
1670 1671
			if (sync)
				set_bit(R5_SyncIO, &dev->flags);
T
Tejun Heo 已提交
1672
		}
1673 1674
	}

1675 1676 1677 1678 1679 1680 1681 1682
	if (sh->reconstruct_state == reconstruct_state_drain_run)
		sh->reconstruct_state = reconstruct_state_drain_result;
	else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run)
		sh->reconstruct_state = reconstruct_state_prexor_drain_result;
	else {
		BUG_ON(sh->reconstruct_state != reconstruct_state_run);
		sh->reconstruct_state = reconstruct_state_result;
	}
1683 1684 1685 1686 1687 1688

	set_bit(STRIPE_HANDLE, &sh->state);
	release_stripe(sh);
}

static void
1689 1690
ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu,
		     struct dma_async_tx_descriptor *tx)
1691 1692
{
	int disks = sh->disks;
1693
	struct page **xor_srcs;
1694
	struct async_submit_ctl submit;
1695
	int count, pd_idx = sh->pd_idx, i;
1696
	struct page *xor_dest;
1697
	int prexor = 0;
1698
	unsigned long flags;
1699 1700 1701
	int j = 0;
	struct stripe_head *head_sh = sh;
	int last_stripe;
1702

1703
	pr_debug("%s: stripe %llu\n", __func__,
1704 1705
		(unsigned long long)sh->sector);

S
Shaohua Li 已提交
1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717
	for (i = 0; i < sh->disks; i++) {
		if (pd_idx == i)
			continue;
		if (!test_bit(R5_Discard, &sh->dev[i].flags))
			break;
	}
	if (i >= sh->disks) {
		atomic_inc(&sh->count);
		set_bit(R5_Discard, &sh->dev[pd_idx].flags);
		ops_complete_reconstruct(sh);
		return;
	}
1718 1719 1720
again:
	count = 0;
	xor_srcs = to_addr_page(percpu, j);
1721 1722 1723
	/* check if prexor is active which means only process blocks
	 * that are part of a read-modify-write (written)
	 */
1724
	if (head_sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
1725
		prexor = 1;
1726 1727 1728
		xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
1729
			if (head_sh->dev[i].written)
1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745
				xor_srcs[count++] = dev->page;
		}
	} else {
		xor_dest = sh->dev[pd_idx].page;
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if (i != pd_idx)
				xor_srcs[count++] = dev->page;
		}
	}

	/* 1/ if we prexor'd then the dest is reused as a source
	 * 2/ if we did not prexor then we are redoing the parity
	 * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST
	 * for the synchronous xor case
	 */
1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760
	last_stripe = !head_sh->batch_head ||
		list_first_entry(&sh->batch_list,
				 struct stripe_head, batch_list) == head_sh;
	if (last_stripe) {
		flags = ASYNC_TX_ACK |
			(prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST);

		atomic_inc(&head_sh->count);
		init_async_submit(&submit, flags, tx, ops_complete_reconstruct, head_sh,
				  to_addr_conv(sh, percpu, j));
	} else {
		flags = prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST;
		init_async_submit(&submit, flags, tx, NULL, NULL,
				  to_addr_conv(sh, percpu, j));
	}
1761

1762 1763 1764 1765
	if (unlikely(count == 1))
		tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
	else
		tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
1766 1767 1768 1769 1770 1771
	if (!last_stripe) {
		j++;
		sh = list_first_entry(&sh->batch_list, struct stripe_head,
				      batch_list);
		goto again;
	}
1772 1773
}

1774 1775 1776 1777 1778
static void
ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,
		     struct dma_async_tx_descriptor *tx)
{
	struct async_submit_ctl submit;
1779 1780 1781 1782
	struct page **blocks;
	int count, i, j = 0;
	struct stripe_head *head_sh = sh;
	int last_stripe;
1783 1784
	int synflags;
	unsigned long txflags;
1785 1786 1787

	pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector);

S
Shaohua Li 已提交
1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801
	for (i = 0; i < sh->disks; i++) {
		if (sh->pd_idx == i || sh->qd_idx == i)
			continue;
		if (!test_bit(R5_Discard, &sh->dev[i].flags))
			break;
	}
	if (i >= sh->disks) {
		atomic_inc(&sh->count);
		set_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);
		set_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);
		ops_complete_reconstruct(sh);
		return;
	}

1802 1803
again:
	blocks = to_addr_page(percpu, j);
1804 1805 1806 1807 1808 1809 1810 1811 1812 1813

	if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
		synflags = SYNDROME_SRC_WRITTEN;
		txflags = ASYNC_TX_ACK | ASYNC_TX_PQ_XOR_DST;
	} else {
		synflags = SYNDROME_SRC_ALL;
		txflags = ASYNC_TX_ACK;
	}

	count = set_syndrome_sources(blocks, sh, synflags);
1814 1815 1816 1817 1818 1819
	last_stripe = !head_sh->batch_head ||
		list_first_entry(&sh->batch_list,
				 struct stripe_head, batch_list) == head_sh;

	if (last_stripe) {
		atomic_inc(&head_sh->count);
1820
		init_async_submit(&submit, txflags, tx, ops_complete_reconstruct,
1821 1822 1823 1824
				  head_sh, to_addr_conv(sh, percpu, j));
	} else
		init_async_submit(&submit, 0, tx, NULL, NULL,
				  to_addr_conv(sh, percpu, j));
1825
	tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE,  &submit);
1826 1827 1828 1829 1830 1831
	if (!last_stripe) {
		j++;
		sh = list_first_entry(&sh->batch_list, struct stripe_head,
				      batch_list);
		goto again;
	}
1832 1833 1834 1835 1836 1837
}

static void ops_complete_check(void *stripe_head_ref)
{
	struct stripe_head *sh = stripe_head_ref;

1838
	pr_debug("%s: stripe %llu\n", __func__,
1839 1840
		(unsigned long long)sh->sector);

1841
	sh->check_state = check_state_check_result;
1842 1843 1844 1845
	set_bit(STRIPE_HANDLE, &sh->state);
	release_stripe(sh);
}

1846
static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu)
1847 1848
{
	int disks = sh->disks;
1849 1850 1851
	int pd_idx = sh->pd_idx;
	int qd_idx = sh->qd_idx;
	struct page *xor_dest;
1852
	struct page **xor_srcs = to_addr_page(percpu, 0);
1853
	struct dma_async_tx_descriptor *tx;
1854
	struct async_submit_ctl submit;
1855 1856
	int count;
	int i;
1857

1858
	pr_debug("%s: stripe %llu\n", __func__,
1859 1860
		(unsigned long long)sh->sector);

1861
	BUG_ON(sh->batch_head);
1862 1863 1864
	count = 0;
	xor_dest = sh->dev[pd_idx].page;
	xor_srcs[count++] = xor_dest;
1865
	for (i = disks; i--; ) {
1866 1867 1868
		if (i == pd_idx || i == qd_idx)
			continue;
		xor_srcs[count++] = sh->dev[i].page;
1869 1870
	}

1871
	init_async_submit(&submit, 0, NULL, NULL, NULL,
1872
			  to_addr_conv(sh, percpu, 0));
D
Dan Williams 已提交
1873
	tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE,
1874
			   &sh->ops.zero_sum_result, &submit);
1875 1876

	atomic_inc(&sh->count);
1877 1878
	init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL);
	tx = async_trigger_callback(&submit);
1879 1880
}

1881 1882
static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp)
{
1883
	struct page **srcs = to_addr_page(percpu, 0);
1884 1885 1886 1887 1888 1889
	struct async_submit_ctl submit;
	int count;

	pr_debug("%s: stripe %llu checkp: %d\n", __func__,
		(unsigned long long)sh->sector, checkp);

1890
	BUG_ON(sh->batch_head);
1891
	count = set_syndrome_sources(srcs, sh, SYNDROME_SRC_ALL);
1892 1893
	if (!checkp)
		srcs[count] = NULL;
1894 1895

	atomic_inc(&sh->count);
1896
	init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check,
1897
			  sh, to_addr_conv(sh, percpu, 0));
1898 1899
	async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE,
			   &sh->ops.zero_sum_result, percpu->spare_page, &submit);
1900 1901
}

N
NeilBrown 已提交
1902
static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
1903 1904 1905
{
	int overlap_clear = 0, i, disks = sh->disks;
	struct dma_async_tx_descriptor *tx = NULL;
1906
	struct r5conf *conf = sh->raid_conf;
1907
	int level = conf->level;
1908 1909
	struct raid5_percpu *percpu;
	unsigned long cpu;
1910

1911 1912
	cpu = get_cpu();
	percpu = per_cpu_ptr(conf->percpu, cpu);
1913
	if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) {
1914 1915 1916 1917
		ops_run_biofill(sh);
		overlap_clear++;
	}

1918
	if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) {
1919 1920 1921 1922 1923 1924 1925 1926 1927 1928
		if (level < 6)
			tx = ops_run_compute5(sh, percpu);
		else {
			if (sh->ops.target2 < 0 || sh->ops.target < 0)
				tx = ops_run_compute6_1(sh, percpu);
			else
				tx = ops_run_compute6_2(sh, percpu);
		}
		/* terminate the chain if reconstruct is not set to be run */
		if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request))
1929 1930
			async_tx_ack(tx);
	}
1931

1932 1933 1934 1935 1936 1937
	if (test_bit(STRIPE_OP_PREXOR, &ops_request)) {
		if (level < 6)
			tx = ops_run_prexor5(sh, percpu, tx);
		else
			tx = ops_run_prexor6(sh, percpu, tx);
	}
1938

1939
	if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) {
1940
		tx = ops_run_biodrain(sh, tx);
1941 1942 1943
		overlap_clear++;
	}

1944 1945 1946 1947 1948 1949
	if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) {
		if (level < 6)
			ops_run_reconstruct5(sh, percpu, tx);
		else
			ops_run_reconstruct6(sh, percpu, tx);
	}
1950

1951 1952 1953 1954 1955 1956 1957 1958 1959 1960
	if (test_bit(STRIPE_OP_CHECK, &ops_request)) {
		if (sh->check_state == check_state_run)
			ops_run_check_p(sh, percpu);
		else if (sh->check_state == check_state_run_q)
			ops_run_check_pq(sh, percpu, 0);
		else if (sh->check_state == check_state_run_pq)
			ops_run_check_pq(sh, percpu, 1);
		else
			BUG();
	}
1961

1962
	if (overlap_clear && !sh->batch_head)
1963 1964 1965 1966 1967
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if (test_and_clear_bit(R5_Overlap, &dev->flags))
				wake_up(&sh->raid_conf->wait_for_overlap);
		}
1968
	put_cpu();
1969 1970
}

1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984
static struct stripe_head *alloc_stripe(struct kmem_cache *sc, gfp_t gfp)
{
	struct stripe_head *sh;

	sh = kmem_cache_zalloc(sc, gfp);
	if (sh) {
		spin_lock_init(&sh->stripe_lock);
		spin_lock_init(&sh->batch_lock);
		INIT_LIST_HEAD(&sh->batch_list);
		INIT_LIST_HEAD(&sh->lru);
		atomic_set(&sh->count, 1);
	}
	return sh;
}
1985
static int grow_one_stripe(struct r5conf *conf, gfp_t gfp)
L
Linus Torvalds 已提交
1986 1987
{
	struct stripe_head *sh;
1988 1989

	sh = alloc_stripe(conf->slab_cache, gfp);
1990 1991
	if (!sh)
		return 0;
N
Namhyung Kim 已提交
1992

1993 1994
	sh->raid_conf = conf;

1995
	if (grow_buffers(sh, gfp)) {
1996
		shrink_buffers(sh);
1997 1998 1999
		kmem_cache_free(conf->slab_cache, sh);
		return 0;
	}
2000 2001
	sh->hash_lock_index =
		conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS;
2002 2003
	/* we just created an active stripe so... */
	atomic_inc(&conf->active_stripes);
2004

2005
	release_stripe(sh);
2006
	conf->max_nr_stripes++;
2007 2008 2009
	return 1;
}

2010
static int grow_stripes(struct r5conf *conf, int num)
2011
{
2012
	struct kmem_cache *sc;
2013
	int devs = max(conf->raid_disks, conf->previous_raid_disks);
L
Linus Torvalds 已提交
2014

2015 2016 2017 2018 2019 2020 2021 2022
	if (conf->mddev->gendisk)
		sprintf(conf->cache_name[0],
			"raid%d-%s", conf->level, mdname(conf->mddev));
	else
		sprintf(conf->cache_name[0],
			"raid%d-%p", conf->level, conf->mddev);
	sprintf(conf->cache_name[1], "%s-alt", conf->cache_name[0]);

2023 2024
	conf->active_name = 0;
	sc = kmem_cache_create(conf->cache_name[conf->active_name],
L
Linus Torvalds 已提交
2025
			       sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev),
2026
			       0, 0, NULL);
L
Linus Torvalds 已提交
2027 2028 2029
	if (!sc)
		return 1;
	conf->slab_cache = sc;
2030
	conf->pool_size = devs;
2031 2032
	while (num--)
		if (!grow_one_stripe(conf, GFP_KERNEL))
L
Linus Torvalds 已提交
2033
			return 1;
2034

L
Linus Torvalds 已提交
2035 2036
	return 0;
}
2037

2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050
/**
 * scribble_len - return the required size of the scribble region
 * @num - total number of disks in the array
 *
 * The size must be enough to contain:
 * 1/ a struct page pointer for each device in the array +2
 * 2/ room to convert each entry in (1) to its corresponding dma
 *    (dma_map_page()) or page (page_address()) address.
 *
 * Note: the +2 is for the destination buffers of the ddf/raid6 case where we
 * calculate over all devices (not just the data blocks), using zeros in place
 * of the P and Q blocks.
 */
2051
static struct flex_array *scribble_alloc(int num, int cnt, gfp_t flags)
2052
{
2053
	struct flex_array *ret;
2054 2055 2056
	size_t len;

	len = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2);
2057 2058 2059 2060 2061 2062 2063 2064 2065
	ret = flex_array_alloc(len, cnt, flags);
	if (!ret)
		return NULL;
	/* always prealloc all elements, so no locking is required */
	if (flex_array_prealloc(ret, 0, cnt, flags)) {
		flex_array_free(ret);
		return NULL;
	}
	return ret;
2066 2067
}

2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096
static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors)
{
	unsigned long cpu;
	int err = 0;

	mddev_suspend(conf->mddev);
	get_online_cpus();
	for_each_present_cpu(cpu) {
		struct raid5_percpu *percpu;
		struct flex_array *scribble;

		percpu = per_cpu_ptr(conf->percpu, cpu);
		scribble = scribble_alloc(new_disks,
					  new_sectors / STRIPE_SECTORS,
					  GFP_NOIO);

		if (scribble) {
			flex_array_free(percpu->scribble);
			percpu->scribble = scribble;
		} else {
			err = -ENOMEM;
			break;
		}
	}
	put_online_cpus();
	mddev_resume(conf->mddev);
	return err;
}

2097
static int resize_stripes(struct r5conf *conf, int newsize)
2098 2099 2100 2101 2102 2103 2104
{
	/* Make all the stripes able to hold 'newsize' devices.
	 * New slots in each stripe get 'page' set to a new page.
	 *
	 * This happens in stages:
	 * 1/ create a new kmem_cache and allocate the required number of
	 *    stripe_heads.
M
Masanari Iida 已提交
2105
	 * 2/ gather all the old stripe_heads and transfer the pages across
2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124
	 *    to the new stripe_heads.  This will have the side effect of
	 *    freezing the array as once all stripe_heads have been collected,
	 *    no IO will be possible.  Old stripe heads are freed once their
	 *    pages have been transferred over, and the old kmem_cache is
	 *    freed when all stripes are done.
	 * 3/ reallocate conf->disks to be suitable bigger.  If this fails,
	 *    we simple return a failre status - no need to clean anything up.
	 * 4/ allocate new pages for the new slots in the new stripe_heads.
	 *    If this fails, we don't bother trying the shrink the
	 *    stripe_heads down again, we just leave them as they are.
	 *    As each stripe_head is processed the new one is released into
	 *    active service.
	 *
	 * Once step2 is started, we cannot afford to wait for a write,
	 * so we use GFP_NOIO allocations.
	 */
	struct stripe_head *osh, *nsh;
	LIST_HEAD(newstripes);
	struct disk_info *ndisks;
2125
	int err;
2126
	struct kmem_cache *sc;
2127
	int i;
2128
	int hash, cnt;
2129 2130 2131 2132

	if (newsize <= conf->pool_size)
		return 0; /* never bother to shrink */

2133 2134 2135
	err = md_allow_write(conf->mddev);
	if (err)
		return err;
2136

2137 2138 2139
	/* Step 1 */
	sc = kmem_cache_create(conf->cache_name[1-conf->active_name],
			       sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev),
2140
			       0, 0, NULL);
2141 2142 2143 2144
	if (!sc)
		return -ENOMEM;

	for (i = conf->max_nr_stripes; i; i--) {
2145
		nsh = alloc_stripe(sc, GFP_KERNEL);
2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165
		if (!nsh)
			break;

		nsh->raid_conf = conf;
		list_add(&nsh->lru, &newstripes);
	}
	if (i) {
		/* didn't get enough, give up */
		while (!list_empty(&newstripes)) {
			nsh = list_entry(newstripes.next, struct stripe_head, lru);
			list_del(&nsh->lru);
			kmem_cache_free(sc, nsh);
		}
		kmem_cache_destroy(sc);
		return -ENOMEM;
	}
	/* Step 2 - Must use GFP_NOIO now.
	 * OK, we have enough stripes, start collecting inactive
	 * stripes and copying them over
	 */
2166 2167
	hash = 0;
	cnt = 0;
2168
	list_for_each_entry(nsh, &newstripes, lru) {
2169 2170 2171 2172 2173 2174 2175
		lock_device_hash_lock(conf, hash);
		wait_event_cmd(conf->wait_for_stripe,
				    !list_empty(conf->inactive_list + hash),
				    unlock_device_hash_lock(conf, hash),
				    lock_device_hash_lock(conf, hash));
		osh = get_free_stripe(conf, hash);
		unlock_device_hash_lock(conf, hash);
2176

2177
		for(i=0; i<conf->pool_size; i++) {
2178
			nsh->dev[i].page = osh->dev[i].page;
2179 2180
			nsh->dev[i].orig_page = osh->dev[i].page;
		}
2181
		nsh->hash_lock_index = hash;
2182
		kmem_cache_free(conf->slab_cache, osh);
2183 2184 2185 2186 2187 2188
		cnt++;
		if (cnt >= conf->max_nr_stripes / NR_STRIPE_HASH_LOCKS +
		    !!((conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS) > hash)) {
			hash++;
			cnt = 0;
		}
2189 2190 2191 2192 2193 2194
	}
	kmem_cache_destroy(conf->slab_cache);

	/* Step 3.
	 * At this point, we are holding all the stripes so the array
	 * is completely stalled, so now is a good time to resize
2195
	 * conf->disks and the scribble region
2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209
	 */
	ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO);
	if (ndisks) {
		for (i=0; i<conf->raid_disks; i++)
			ndisks[i] = conf->disks[i];
		kfree(conf->disks);
		conf->disks = ndisks;
	} else
		err = -ENOMEM;

	/* Step 4, return new stripes to service */
	while(!list_empty(&newstripes)) {
		nsh = list_entry(newstripes.next, struct stripe_head, lru);
		list_del_init(&nsh->lru);
2210

2211 2212 2213 2214
		for (i=conf->raid_disks; i < newsize; i++)
			if (nsh->dev[i].page == NULL) {
				struct page *p = alloc_page(GFP_NOIO);
				nsh->dev[i].page = p;
2215
				nsh->dev[i].orig_page = p;
2216 2217 2218 2219 2220 2221 2222 2223 2224
				if (!p)
					err = -ENOMEM;
			}
		release_stripe(nsh);
	}
	/* critical section pass, GFP_NOIO no longer needed */

	conf->slab_cache = sc;
	conf->active_name = 1-conf->active_name;
2225 2226
	if (!err)
		conf->pool_size = newsize;
2227 2228
	return err;
}
L
Linus Torvalds 已提交
2229

2230
static int drop_one_stripe(struct r5conf *conf)
L
Linus Torvalds 已提交
2231 2232
{
	struct stripe_head *sh;
2233
	int hash = (conf->max_nr_stripes - 1) % NR_STRIPE_HASH_LOCKS;
L
Linus Torvalds 已提交
2234

2235 2236 2237
	spin_lock_irq(conf->hash_locks + hash);
	sh = get_free_stripe(conf, hash);
	spin_unlock_irq(conf->hash_locks + hash);
2238 2239
	if (!sh)
		return 0;
2240
	BUG_ON(atomic_read(&sh->count));
2241
	shrink_buffers(sh);
2242 2243
	kmem_cache_free(conf->slab_cache, sh);
	atomic_dec(&conf->active_stripes);
2244
	conf->max_nr_stripes--;
2245 2246 2247
	return 1;
}

2248
static void shrink_stripes(struct r5conf *conf)
2249
{
2250 2251 2252
	while (conf->max_nr_stripes &&
	       drop_one_stripe(conf))
		;
2253

N
NeilBrown 已提交
2254 2255
	if (conf->slab_cache)
		kmem_cache_destroy(conf->slab_cache);
L
Linus Torvalds 已提交
2256 2257 2258
	conf->slab_cache = NULL;
}

2259
static void raid5_end_read_request(struct bio * bi, int error)
L
Linus Torvalds 已提交
2260
{
2261
	struct stripe_head *sh = bi->bi_private;
2262
	struct r5conf *conf = sh->raid_conf;
2263
	int disks = sh->disks, i;
L
Linus Torvalds 已提交
2264
	int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
2265
	char b[BDEVNAME_SIZE];
2266
	struct md_rdev *rdev = NULL;
2267
	sector_t s;
L
Linus Torvalds 已提交
2268 2269 2270 2271 2272

	for (i=0 ; i<disks; i++)
		if (bi == &sh->dev[i].req)
			break;

2273 2274
	pr_debug("end_read_request %llu/%d, count: %d, uptodate %d.\n",
		(unsigned long long)sh->sector, i, atomic_read(&sh->count),
L
Linus Torvalds 已提交
2275 2276 2277
		uptodate);
	if (i == disks) {
		BUG();
2278
		return;
L
Linus Torvalds 已提交
2279
	}
2280
	if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
2281 2282 2283 2284 2285
		/* If replacement finished while this request was outstanding,
		 * 'replacement' might be NULL already.
		 * In that case it moved down to 'rdev'.
		 * rdev is not removed until all requests are finished.
		 */
2286
		rdev = conf->disks[i].replacement;
2287
	if (!rdev)
2288
		rdev = conf->disks[i].rdev;
L
Linus Torvalds 已提交
2289

2290 2291 2292 2293
	if (use_new_offset(conf, sh))
		s = sh->sector + rdev->new_data_offset;
	else
		s = sh->sector + rdev->data_offset;
L
Linus Torvalds 已提交
2294 2295
	if (uptodate) {
		set_bit(R5_UPTODATE, &sh->dev[i].flags);
2296
		if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
2297 2298 2299 2300
			/* Note that this cannot happen on a
			 * replacement device.  We just fail those on
			 * any error
			 */
2301 2302 2303 2304 2305
			printk_ratelimited(
				KERN_INFO
				"md/raid:%s: read error corrected"
				" (%lu sectors at %llu on %s)\n",
				mdname(conf->mddev), STRIPE_SECTORS,
2306
				(unsigned long long)s,
2307
				bdevname(rdev->bdev, b));
2308
			atomic_add(STRIPE_SECTORS, &rdev->corrected_errors);
2309 2310
			clear_bit(R5_ReadError, &sh->dev[i].flags);
			clear_bit(R5_ReWrite, &sh->dev[i].flags);
2311 2312 2313
		} else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
			clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);

2314 2315
		if (atomic_read(&rdev->read_errors))
			atomic_set(&rdev->read_errors, 0);
L
Linus Torvalds 已提交
2316
	} else {
2317
		const char *bdn = bdevname(rdev->bdev, b);
2318
		int retry = 0;
2319
		int set_bad = 0;
2320

L
Linus Torvalds 已提交
2321
		clear_bit(R5_UPTODATE, &sh->dev[i].flags);
2322
		atomic_inc(&rdev->read_errors);
2323 2324 2325 2326 2327 2328
		if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
			printk_ratelimited(
				KERN_WARNING
				"md/raid:%s: read error on replacement device "
				"(sector %llu on %s).\n",
				mdname(conf->mddev),
2329
				(unsigned long long)s,
2330
				bdn);
2331 2332
		else if (conf->mddev->degraded >= conf->max_degraded) {
			set_bad = 1;
2333 2334 2335 2336 2337
			printk_ratelimited(
				KERN_WARNING
				"md/raid:%s: read error not correctable "
				"(sector %llu on %s).\n",
				mdname(conf->mddev),
2338
				(unsigned long long)s,
2339
				bdn);
2340
		} else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) {
2341
			/* Oh, no!!! */
2342
			set_bad = 1;
2343 2344 2345 2346 2347
			printk_ratelimited(
				KERN_WARNING
				"md/raid:%s: read error NOT corrected!! "
				"(sector %llu on %s).\n",
				mdname(conf->mddev),
2348
				(unsigned long long)s,
2349
				bdn);
2350
		} else if (atomic_read(&rdev->read_errors)
2351
			 > conf->max_nr_stripes)
N
NeilBrown 已提交
2352
			printk(KERN_WARNING
2353
			       "md/raid:%s: Too many read errors, failing device %s.\n",
2354
			       mdname(conf->mddev), bdn);
2355 2356
		else
			retry = 1;
2357 2358 2359
		if (set_bad && test_bit(In_sync, &rdev->flags)
		    && !test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
			retry = 1;
2360
		if (retry)
2361 2362 2363 2364 2365
			if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) {
				set_bit(R5_ReadError, &sh->dev[i].flags);
				clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);
			} else
				set_bit(R5_ReadNoMerge, &sh->dev[i].flags);
2366
		else {
2367 2368
			clear_bit(R5_ReadError, &sh->dev[i].flags);
			clear_bit(R5_ReWrite, &sh->dev[i].flags);
2369 2370 2371 2372 2373
			if (!(set_bad
			      && test_bit(In_sync, &rdev->flags)
			      && rdev_set_badblocks(
				      rdev, sh->sector, STRIPE_SECTORS, 0)))
				md_error(conf->mddev, rdev);
2374
		}
L
Linus Torvalds 已提交
2375
	}
2376
	rdev_dec_pending(rdev, conf->mddev);
L
Linus Torvalds 已提交
2377 2378 2379 2380 2381
	clear_bit(R5_LOCKED, &sh->dev[i].flags);
	set_bit(STRIPE_HANDLE, &sh->state);
	release_stripe(sh);
}

2382
static void raid5_end_write_request(struct bio *bi, int error)
L
Linus Torvalds 已提交
2383
{
2384
	struct stripe_head *sh = bi->bi_private;
2385
	struct r5conf *conf = sh->raid_conf;
2386
	int disks = sh->disks, i;
2387
	struct md_rdev *uninitialized_var(rdev);
L
Linus Torvalds 已提交
2388
	int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
2389 2390
	sector_t first_bad;
	int bad_sectors;
2391
	int replacement = 0;
L
Linus Torvalds 已提交
2392

2393 2394 2395
	for (i = 0 ; i < disks; i++) {
		if (bi == &sh->dev[i].req) {
			rdev = conf->disks[i].rdev;
L
Linus Torvalds 已提交
2396
			break;
2397 2398 2399
		}
		if (bi == &sh->dev[i].rreq) {
			rdev = conf->disks[i].replacement;
2400 2401 2402 2403 2404 2405 2406 2407
			if (rdev)
				replacement = 1;
			else
				/* rdev was removed and 'replacement'
				 * replaced it.  rdev is not removed
				 * until all requests are finished.
				 */
				rdev = conf->disks[i].rdev;
2408 2409 2410
			break;
		}
	}
2411
	pr_debug("end_write_request %llu/%d, count %d, uptodate: %d.\n",
L
Linus Torvalds 已提交
2412 2413 2414 2415
		(unsigned long long)sh->sector, i, atomic_read(&sh->count),
		uptodate);
	if (i == disks) {
		BUG();
2416
		return;
L
Linus Torvalds 已提交
2417 2418
	}

2419 2420 2421 2422 2423 2424 2425 2426 2427
	if (replacement) {
		if (!uptodate)
			md_error(conf->mddev, rdev);
		else if (is_badblock(rdev, sh->sector,
				     STRIPE_SECTORS,
				     &first_bad, &bad_sectors))
			set_bit(R5_MadeGoodRepl, &sh->dev[i].flags);
	} else {
		if (!uptodate) {
2428
			set_bit(STRIPE_DEGRADED, &sh->state);
2429 2430
			set_bit(WriteErrorSeen, &rdev->flags);
			set_bit(R5_WriteError, &sh->dev[i].flags);
2431 2432 2433
			if (!test_and_set_bit(WantReplacement, &rdev->flags))
				set_bit(MD_RECOVERY_NEEDED,
					&rdev->mddev->recovery);
2434 2435
		} else if (is_badblock(rdev, sh->sector,
				       STRIPE_SECTORS,
2436
				       &first_bad, &bad_sectors)) {
2437
			set_bit(R5_MadeGood, &sh->dev[i].flags);
2438 2439 2440 2441 2442 2443 2444
			if (test_bit(R5_ReadError, &sh->dev[i].flags))
				/* That was a successful write so make
				 * sure it looks like we already did
				 * a re-write.
				 */
				set_bit(R5_ReWrite, &sh->dev[i].flags);
		}
2445 2446
	}
	rdev_dec_pending(rdev, conf->mddev);
L
Linus Torvalds 已提交
2447

2448
	if (sh->batch_head && !uptodate && !replacement)
2449 2450
		set_bit(STRIPE_BATCH_ERR, &sh->batch_head->state);

2451 2452
	if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags))
		clear_bit(R5_LOCKED, &sh->dev[i].flags);
L
Linus Torvalds 已提交
2453
	set_bit(STRIPE_HANDLE, &sh->state);
2454
	release_stripe(sh);
2455 2456 2457

	if (sh->batch_head && sh != sh->batch_head)
		release_stripe(sh->batch_head);
L
Linus Torvalds 已提交
2458 2459
}

2460
static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous);
2461

2462
static void raid5_build_block(struct stripe_head *sh, int i, int previous)
L
Linus Torvalds 已提交
2463 2464 2465 2466 2467
{
	struct r5dev *dev = &sh->dev[i];

	bio_init(&dev->req);
	dev->req.bi_io_vec = &dev->vec;
2468
	dev->req.bi_max_vecs = 1;
L
Linus Torvalds 已提交
2469 2470
	dev->req.bi_private = sh;

2471 2472
	bio_init(&dev->rreq);
	dev->rreq.bi_io_vec = &dev->rvec;
2473
	dev->rreq.bi_max_vecs = 1;
2474 2475
	dev->rreq.bi_private = sh;

L
Linus Torvalds 已提交
2476
	dev->flags = 0;
2477
	dev->sector = compute_blocknr(sh, i, previous);
L
Linus Torvalds 已提交
2478 2479
}

2480
static void error(struct mddev *mddev, struct md_rdev *rdev)
L
Linus Torvalds 已提交
2481 2482
{
	char b[BDEVNAME_SIZE];
2483
	struct r5conf *conf = mddev->private;
2484
	unsigned long flags;
2485
	pr_debug("raid456: error called\n");
L
Linus Torvalds 已提交
2486

2487 2488 2489 2490 2491 2492
	spin_lock_irqsave(&conf->device_lock, flags);
	clear_bit(In_sync, &rdev->flags);
	mddev->degraded = calc_degraded(conf);
	spin_unlock_irqrestore(&conf->device_lock, flags);
	set_bit(MD_RECOVERY_INTR, &mddev->recovery);

2493
	set_bit(Blocked, &rdev->flags);
2494 2495 2496 2497 2498 2499 2500 2501 2502
	set_bit(Faulty, &rdev->flags);
	set_bit(MD_CHANGE_DEVS, &mddev->flags);
	printk(KERN_ALERT
	       "md/raid:%s: Disk failure on %s, disabling device.\n"
	       "md/raid:%s: Operation continuing on %d devices.\n",
	       mdname(mddev),
	       bdevname(rdev->bdev, b),
	       mdname(mddev),
	       conf->raid_disks - mddev->degraded);
2503
}
L
Linus Torvalds 已提交
2504 2505 2506 2507 2508

/*
 * Input: a 'big' sector number,
 * Output: index of the data and parity disk, and the sector # in them.
 */
2509
static sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector,
2510 2511
				     int previous, int *dd_idx,
				     struct stripe_head *sh)
L
Linus Torvalds 已提交
2512
{
N
NeilBrown 已提交
2513
	sector_t stripe, stripe2;
2514
	sector_t chunk_number;
L
Linus Torvalds 已提交
2515
	unsigned int chunk_offset;
2516
	int pd_idx, qd_idx;
2517
	int ddf_layout = 0;
L
Linus Torvalds 已提交
2518
	sector_t new_sector;
2519 2520
	int algorithm = previous ? conf->prev_algo
				 : conf->algorithm;
2521 2522
	int sectors_per_chunk = previous ? conf->prev_chunk_sectors
					 : conf->chunk_sectors;
2523 2524 2525
	int raid_disks = previous ? conf->previous_raid_disks
				  : conf->raid_disks;
	int data_disks = raid_disks - conf->max_degraded;
L
Linus Torvalds 已提交
2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537

	/* First compute the information on this sector */

	/*
	 * Compute the chunk number and the sector offset inside the chunk
	 */
	chunk_offset = sector_div(r_sector, sectors_per_chunk);
	chunk_number = r_sector;

	/*
	 * Compute the stripe number
	 */
2538 2539
	stripe = chunk_number;
	*dd_idx = sector_div(stripe, data_disks);
N
NeilBrown 已提交
2540
	stripe2 = stripe;
L
Linus Torvalds 已提交
2541 2542 2543
	/*
	 * Select the parity disk based on the user selected algorithm.
	 */
2544
	pd_idx = qd_idx = -1;
2545 2546
	switch(conf->level) {
	case 4:
2547
		pd_idx = data_disks;
2548 2549
		break;
	case 5:
2550
		switch (algorithm) {
L
Linus Torvalds 已提交
2551
		case ALGORITHM_LEFT_ASYMMETRIC:
N
NeilBrown 已提交
2552
			pd_idx = data_disks - sector_div(stripe2, raid_disks);
2553
			if (*dd_idx >= pd_idx)
L
Linus Torvalds 已提交
2554 2555 2556
				(*dd_idx)++;
			break;
		case ALGORITHM_RIGHT_ASYMMETRIC:
N
NeilBrown 已提交
2557
			pd_idx = sector_div(stripe2, raid_disks);
2558
			if (*dd_idx >= pd_idx)
L
Linus Torvalds 已提交
2559 2560 2561
				(*dd_idx)++;
			break;
		case ALGORITHM_LEFT_SYMMETRIC:
N
NeilBrown 已提交
2562
			pd_idx = data_disks - sector_div(stripe2, raid_disks);
2563
			*dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
L
Linus Torvalds 已提交
2564 2565
			break;
		case ALGORITHM_RIGHT_SYMMETRIC:
N
NeilBrown 已提交
2566
			pd_idx = sector_div(stripe2, raid_disks);
2567
			*dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
L
Linus Torvalds 已提交
2568
			break;
2569 2570 2571 2572 2573 2574 2575
		case ALGORITHM_PARITY_0:
			pd_idx = 0;
			(*dd_idx)++;
			break;
		case ALGORITHM_PARITY_N:
			pd_idx = data_disks;
			break;
L
Linus Torvalds 已提交
2576
		default:
2577
			BUG();
2578 2579 2580 2581
		}
		break;
	case 6:

2582
		switch (algorithm) {
2583
		case ALGORITHM_LEFT_ASYMMETRIC:
N
NeilBrown 已提交
2584
			pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
2585 2586
			qd_idx = pd_idx + 1;
			if (pd_idx == raid_disks-1) {
2587
				(*dd_idx)++;	/* Q D D D P */
2588 2589
				qd_idx = 0;
			} else if (*dd_idx >= pd_idx)
2590 2591 2592
				(*dd_idx) += 2; /* D D P Q D */
			break;
		case ALGORITHM_RIGHT_ASYMMETRIC:
N
NeilBrown 已提交
2593
			pd_idx = sector_div(stripe2, raid_disks);
2594 2595
			qd_idx = pd_idx + 1;
			if (pd_idx == raid_disks-1) {
2596
				(*dd_idx)++;	/* Q D D D P */
2597 2598
				qd_idx = 0;
			} else if (*dd_idx >= pd_idx)
2599 2600 2601
				(*dd_idx) += 2; /* D D P Q D */
			break;
		case ALGORITHM_LEFT_SYMMETRIC:
N
NeilBrown 已提交
2602
			pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
2603 2604
			qd_idx = (pd_idx + 1) % raid_disks;
			*dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
2605 2606
			break;
		case ALGORITHM_RIGHT_SYMMETRIC:
N
NeilBrown 已提交
2607
			pd_idx = sector_div(stripe2, raid_disks);
2608 2609
			qd_idx = (pd_idx + 1) % raid_disks;
			*dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
2610
			break;
2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625

		case ALGORITHM_PARITY_0:
			pd_idx = 0;
			qd_idx = 1;
			(*dd_idx) += 2;
			break;
		case ALGORITHM_PARITY_N:
			pd_idx = data_disks;
			qd_idx = data_disks + 1;
			break;

		case ALGORITHM_ROTATING_ZERO_RESTART:
			/* Exactly the same as RIGHT_ASYMMETRIC, but or
			 * of blocks for computing Q is different.
			 */
N
NeilBrown 已提交
2626
			pd_idx = sector_div(stripe2, raid_disks);
2627 2628 2629 2630 2631 2632
			qd_idx = pd_idx + 1;
			if (pd_idx == raid_disks-1) {
				(*dd_idx)++;	/* Q D D D P */
				qd_idx = 0;
			} else if (*dd_idx >= pd_idx)
				(*dd_idx) += 2; /* D D P Q D */
2633
			ddf_layout = 1;
2634 2635 2636 2637 2638 2639 2640
			break;

		case ALGORITHM_ROTATING_N_RESTART:
			/* Same a left_asymmetric, by first stripe is
			 * D D D P Q  rather than
			 * Q D D D P
			 */
N
NeilBrown 已提交
2641 2642
			stripe2 += 1;
			pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
2643 2644 2645 2646 2647 2648
			qd_idx = pd_idx + 1;
			if (pd_idx == raid_disks-1) {
				(*dd_idx)++;	/* Q D D D P */
				qd_idx = 0;
			} else if (*dd_idx >= pd_idx)
				(*dd_idx) += 2; /* D D P Q D */
2649
			ddf_layout = 1;
2650 2651 2652 2653
			break;

		case ALGORITHM_ROTATING_N_CONTINUE:
			/* Same as left_symmetric but Q is before P */
N
NeilBrown 已提交
2654
			pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
2655 2656
			qd_idx = (pd_idx + raid_disks - 1) % raid_disks;
			*dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
2657
			ddf_layout = 1;
2658 2659 2660 2661
			break;

		case ALGORITHM_LEFT_ASYMMETRIC_6:
			/* RAID5 left_asymmetric, with Q on last device */
N
NeilBrown 已提交
2662
			pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
2663 2664 2665 2666 2667 2668
			if (*dd_idx >= pd_idx)
				(*dd_idx)++;
			qd_idx = raid_disks - 1;
			break;

		case ALGORITHM_RIGHT_ASYMMETRIC_6:
N
NeilBrown 已提交
2669
			pd_idx = sector_div(stripe2, raid_disks-1);
2670 2671 2672 2673 2674 2675
			if (*dd_idx >= pd_idx)
				(*dd_idx)++;
			qd_idx = raid_disks - 1;
			break;

		case ALGORITHM_LEFT_SYMMETRIC_6:
N
NeilBrown 已提交
2676
			pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
2677 2678 2679 2680 2681
			*dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
			qd_idx = raid_disks - 1;
			break;

		case ALGORITHM_RIGHT_SYMMETRIC_6:
N
NeilBrown 已提交
2682
			pd_idx = sector_div(stripe2, raid_disks-1);
2683 2684 2685 2686 2687 2688 2689 2690 2691 2692
			*dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
			qd_idx = raid_disks - 1;
			break;

		case ALGORITHM_PARITY_0_6:
			pd_idx = 0;
			(*dd_idx)++;
			qd_idx = raid_disks - 1;
			break;

2693
		default:
2694
			BUG();
2695 2696
		}
		break;
L
Linus Torvalds 已提交
2697 2698
	}

2699 2700 2701
	if (sh) {
		sh->pd_idx = pd_idx;
		sh->qd_idx = qd_idx;
2702
		sh->ddf_layout = ddf_layout;
2703
	}
L
Linus Torvalds 已提交
2704 2705 2706 2707 2708 2709 2710
	/*
	 * Finally, compute the new sector number
	 */
	new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset;
	return new_sector;
}

2711
static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous)
L
Linus Torvalds 已提交
2712
{
2713
	struct r5conf *conf = sh->raid_conf;
2714 2715
	int raid_disks = sh->disks;
	int data_disks = raid_disks - conf->max_degraded;
L
Linus Torvalds 已提交
2716
	sector_t new_sector = sh->sector, check;
2717 2718
	int sectors_per_chunk = previous ? conf->prev_chunk_sectors
					 : conf->chunk_sectors;
2719 2720
	int algorithm = previous ? conf->prev_algo
				 : conf->algorithm;
L
Linus Torvalds 已提交
2721 2722
	sector_t stripe;
	int chunk_offset;
2723 2724
	sector_t chunk_number;
	int dummy1, dd_idx = i;
L
Linus Torvalds 已提交
2725
	sector_t r_sector;
2726
	struct stripe_head sh2;
L
Linus Torvalds 已提交
2727 2728 2729 2730

	chunk_offset = sector_div(new_sector, sectors_per_chunk);
	stripe = new_sector;

2731 2732 2733 2734 2735
	if (i == sh->pd_idx)
		return 0;
	switch(conf->level) {
	case 4: break;
	case 5:
2736
		switch (algorithm) {
L
Linus Torvalds 已提交
2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747
		case ALGORITHM_LEFT_ASYMMETRIC:
		case ALGORITHM_RIGHT_ASYMMETRIC:
			if (i > sh->pd_idx)
				i--;
			break;
		case ALGORITHM_LEFT_SYMMETRIC:
		case ALGORITHM_RIGHT_SYMMETRIC:
			if (i < sh->pd_idx)
				i += raid_disks;
			i -= (sh->pd_idx + 1);
			break;
2748 2749 2750 2751 2752
		case ALGORITHM_PARITY_0:
			i -= 1;
			break;
		case ALGORITHM_PARITY_N:
			break;
L
Linus Torvalds 已提交
2753
		default:
2754
			BUG();
2755 2756 2757
		}
		break;
	case 6:
2758
		if (i == sh->qd_idx)
2759
			return 0; /* It is the Q disk */
2760
		switch (algorithm) {
2761 2762
		case ALGORITHM_LEFT_ASYMMETRIC:
		case ALGORITHM_RIGHT_ASYMMETRIC:
2763 2764 2765 2766
		case ALGORITHM_ROTATING_ZERO_RESTART:
		case ALGORITHM_ROTATING_N_RESTART:
			if (sh->pd_idx == raid_disks-1)
				i--;	/* Q D D D P */
2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780
			else if (i > sh->pd_idx)
				i -= 2; /* D D P Q D */
			break;
		case ALGORITHM_LEFT_SYMMETRIC:
		case ALGORITHM_RIGHT_SYMMETRIC:
			if (sh->pd_idx == raid_disks-1)
				i--; /* Q D D D P */
			else {
				/* D D P Q D */
				if (i < sh->pd_idx)
					i += raid_disks;
				i -= (sh->pd_idx + 2);
			}
			break;
2781 2782 2783 2784 2785 2786
		case ALGORITHM_PARITY_0:
			i -= 2;
			break;
		case ALGORITHM_PARITY_N:
			break;
		case ALGORITHM_ROTATING_N_CONTINUE:
2787
			/* Like left_symmetric, but P is before Q */
2788 2789
			if (sh->pd_idx == 0)
				i--;	/* P D D D Q */
2790 2791 2792 2793 2794 2795
			else {
				/* D D Q P D */
				if (i < sh->pd_idx)
					i += raid_disks;
				i -= (sh->pd_idx + 1);
			}
2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810
			break;
		case ALGORITHM_LEFT_ASYMMETRIC_6:
		case ALGORITHM_RIGHT_ASYMMETRIC_6:
			if (i > sh->pd_idx)
				i--;
			break;
		case ALGORITHM_LEFT_SYMMETRIC_6:
		case ALGORITHM_RIGHT_SYMMETRIC_6:
			if (i < sh->pd_idx)
				i += data_disks + 1;
			i -= (sh->pd_idx + 1);
			break;
		case ALGORITHM_PARITY_0_6:
			i -= 1;
			break;
2811
		default:
2812
			BUG();
2813 2814
		}
		break;
L
Linus Torvalds 已提交
2815 2816 2817
	}

	chunk_number = stripe * data_disks + i;
2818
	r_sector = chunk_number * sectors_per_chunk + chunk_offset;
L
Linus Torvalds 已提交
2819

2820
	check = raid5_compute_sector(conf, r_sector,
2821
				     previous, &dummy1, &sh2);
2822 2823
	if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx
		|| sh2.qd_idx != sh->qd_idx) {
2824 2825
		printk(KERN_ERR "md/raid:%s: compute_blocknr: map not correct\n",
		       mdname(conf->mddev));
L
Linus Torvalds 已提交
2826 2827 2828 2829 2830
		return 0;
	}
	return r_sector;
}

2831
static void
2832
schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s,
2833
			 int rcw, int expand)
2834
{
2835
	int i, pd_idx = sh->pd_idx, qd_idx = sh->qd_idx, disks = sh->disks;
2836
	struct r5conf *conf = sh->raid_conf;
2837
	int level = conf->level;
2838 2839 2840 2841 2842 2843 2844 2845

	if (rcw) {

		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];

			if (dev->towrite) {
				set_bit(R5_LOCKED, &dev->flags);
2846
				set_bit(R5_Wantdrain, &dev->flags);
2847 2848
				if (!expand)
					clear_bit(R5_UPTODATE, &dev->flags);
2849
				s->locked++;
2850 2851
			}
		}
2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866
		/* if we are not expanding this is a proper write request, and
		 * there will be bios with new data to be drained into the
		 * stripe cache
		 */
		if (!expand) {
			if (!s->locked)
				/* False alarm, nothing to do */
				return;
			sh->reconstruct_state = reconstruct_state_drain_run;
			set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
		} else
			sh->reconstruct_state = reconstruct_state_run;

		set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);

2867
		if (s->locked + conf->max_degraded == disks)
2868
			if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state))
2869
				atomic_inc(&conf->pending_full_writes);
2870 2871 2872
	} else {
		BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) ||
			test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags)));
2873 2874 2875
		BUG_ON(level == 6 &&
			(!(test_bit(R5_UPTODATE, &sh->dev[qd_idx].flags) ||
			   test_bit(R5_Wantcompute, &sh->dev[qd_idx].flags))));
2876 2877 2878

		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
2879
			if (i == pd_idx || i == qd_idx)
2880 2881 2882 2883
				continue;

			if (dev->towrite &&
			    (test_bit(R5_UPTODATE, &dev->flags) ||
2884 2885
			     test_bit(R5_Wantcompute, &dev->flags))) {
				set_bit(R5_Wantdrain, &dev->flags);
2886 2887
				set_bit(R5_LOCKED, &dev->flags);
				clear_bit(R5_UPTODATE, &dev->flags);
2888
				s->locked++;
2889 2890
			}
		}
2891 2892 2893 2894 2895 2896 2897
		if (!s->locked)
			/* False alarm - nothing to do */
			return;
		sh->reconstruct_state = reconstruct_state_prexor_drain_run;
		set_bit(STRIPE_OP_PREXOR, &s->ops_request);
		set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
		set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
2898 2899
	}

2900
	/* keep the parity disk(s) locked while asynchronous operations
2901 2902 2903 2904
	 * are in flight
	 */
	set_bit(R5_LOCKED, &sh->dev[pd_idx].flags);
	clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
2905
	s->locked++;
2906

2907 2908 2909 2910 2911 2912 2913 2914 2915
	if (level == 6) {
		int qd_idx = sh->qd_idx;
		struct r5dev *dev = &sh->dev[qd_idx];

		set_bit(R5_LOCKED, &dev->flags);
		clear_bit(R5_UPTODATE, &dev->flags);
		s->locked++;
	}

2916
	pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n",
2917
		__func__, (unsigned long long)sh->sector,
2918
		s->locked, s->ops_request);
2919
}
2920

L
Linus Torvalds 已提交
2921 2922
/*
 * Each stripe/dev can have one or more bion attached.
2923
 * toread/towrite point to the first in a chain.
L
Linus Torvalds 已提交
2924 2925
 * The bi_next chain must be in order.
 */
2926 2927
static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx,
			  int forwrite, int previous)
L
Linus Torvalds 已提交
2928 2929
{
	struct bio **bip;
2930
	struct r5conf *conf = sh->raid_conf;
2931
	int firstwrite=0;
L
Linus Torvalds 已提交
2932

2933
	pr_debug("adding bi b#%llu to stripe s#%llu\n",
2934
		(unsigned long long)bi->bi_iter.bi_sector,
L
Linus Torvalds 已提交
2935 2936
		(unsigned long long)sh->sector);

S
Shaohua Li 已提交
2937 2938 2939 2940 2941 2942 2943 2944 2945
	/*
	 * If several bio share a stripe. The bio bi_phys_segments acts as a
	 * reference count to avoid race. The reference count should already be
	 * increased before this function is called (for example, in
	 * make_request()), so other bio sharing this stripe will not free the
	 * stripe. If a stripe is owned by one stripe, the stripe lock will
	 * protect it.
	 */
	spin_lock_irq(&sh->stripe_lock);
2946 2947 2948
	/* Don't allow new IO added to stripes in batch list */
	if (sh->batch_head)
		goto overlap;
2949
	if (forwrite) {
L
Linus Torvalds 已提交
2950
		bip = &sh->dev[dd_idx].towrite;
2951
		if (*bip == NULL)
2952 2953
			firstwrite = 1;
	} else
L
Linus Torvalds 已提交
2954
		bip = &sh->dev[dd_idx].toread;
2955 2956
	while (*bip && (*bip)->bi_iter.bi_sector < bi->bi_iter.bi_sector) {
		if (bio_end_sector(*bip) > bi->bi_iter.bi_sector)
L
Linus Torvalds 已提交
2957 2958 2959
			goto overlap;
		bip = & (*bip)->bi_next;
	}
2960
	if (*bip && (*bip)->bi_iter.bi_sector < bio_end_sector(bi))
L
Linus Torvalds 已提交
2961 2962
		goto overlap;

2963 2964 2965
	if (!forwrite || previous)
		clear_bit(STRIPE_BATCH_READY, &sh->state);

2966
	BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next);
L
Linus Torvalds 已提交
2967 2968 2969
	if (*bip)
		bi->bi_next = *bip;
	*bip = bi;
2970
	raid5_inc_bi_active_stripes(bi);
2971

L
Linus Torvalds 已提交
2972 2973 2974 2975 2976
	if (forwrite) {
		/* check if page is covered */
		sector_t sector = sh->dev[dd_idx].sector;
		for (bi=sh->dev[dd_idx].towrite;
		     sector < sh->dev[dd_idx].sector + STRIPE_SECTORS &&
2977
			     bi && bi->bi_iter.bi_sector <= sector;
L
Linus Torvalds 已提交
2978
		     bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) {
K
Kent Overstreet 已提交
2979 2980
			if (bio_end_sector(bi) >= sector)
				sector = bio_end_sector(bi);
L
Linus Torvalds 已提交
2981 2982
		}
		if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS)
2983 2984
			if (!test_and_set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags))
				sh->overwrite_disks++;
L
Linus Torvalds 已提交
2985
	}
2986 2987

	pr_debug("added bi b#%llu to stripe s#%llu, disk %d.\n",
2988
		(unsigned long long)(*bip)->bi_iter.bi_sector,
2989
		(unsigned long long)sh->sector, dd_idx);
2990
	spin_unlock_irq(&sh->stripe_lock);
2991 2992 2993 2994 2995 2996 2997

	if (conf->mddev->bitmap && firstwrite) {
		bitmap_startwrite(conf->mddev->bitmap, sh->sector,
				  STRIPE_SECTORS, 0);
		sh->bm_seq = conf->seq_flush+1;
		set_bit(STRIPE_BIT_DELAY, &sh->state);
	}
2998 2999 3000

	if (stripe_can_batch(sh))
		stripe_add_to_batch_list(conf, sh);
L
Linus Torvalds 已提交
3001 3002 3003 3004
	return 1;

 overlap:
	set_bit(R5_Overlap, &sh->dev[dd_idx].flags);
S
Shaohua Li 已提交
3005
	spin_unlock_irq(&sh->stripe_lock);
L
Linus Torvalds 已提交
3006 3007 3008
	return 0;
}

3009
static void end_reshape(struct r5conf *conf);
3010

3011
static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,
3012
			    struct stripe_head *sh)
3013
{
3014
	int sectors_per_chunk =
3015
		previous ? conf->prev_chunk_sectors : conf->chunk_sectors;
3016
	int dd_idx;
3017
	int chunk_offset = sector_div(stripe, sectors_per_chunk);
3018
	int disks = previous ? conf->previous_raid_disks : conf->raid_disks;
3019

3020 3021
	raid5_compute_sector(conf,
			     stripe * (disks - conf->max_degraded)
3022
			     *sectors_per_chunk + chunk_offset,
3023
			     previous,
3024
			     &dd_idx, sh);
3025 3026
}

3027
static void
3028
handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh,
3029 3030 3031 3032
				struct stripe_head_state *s, int disks,
				struct bio **return_bi)
{
	int i;
3033
	BUG_ON(sh->batch_head);
3034 3035 3036 3037 3038
	for (i = disks; i--; ) {
		struct bio *bi;
		int bitmap_end = 0;

		if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
3039
			struct md_rdev *rdev;
3040 3041 3042
			rcu_read_lock();
			rdev = rcu_dereference(conf->disks[i].rdev);
			if (rdev && test_bit(In_sync, &rdev->flags))
3043 3044 3045
				atomic_inc(&rdev->nr_pending);
			else
				rdev = NULL;
3046
			rcu_read_unlock();
3047 3048 3049 3050 3051 3052 3053 3054
			if (rdev) {
				if (!rdev_set_badblocks(
					    rdev,
					    sh->sector,
					    STRIPE_SECTORS, 0))
					md_error(conf->mddev, rdev);
				rdev_dec_pending(rdev, conf->mddev);
			}
3055
		}
S
Shaohua Li 已提交
3056
		spin_lock_irq(&sh->stripe_lock);
3057 3058 3059
		/* fail all writes first */
		bi = sh->dev[i].towrite;
		sh->dev[i].towrite = NULL;
3060
		sh->overwrite_disks = 0;
S
Shaohua Li 已提交
3061
		spin_unlock_irq(&sh->stripe_lock);
3062
		if (bi)
3063 3064 3065 3066 3067
			bitmap_end = 1;

		if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
			wake_up(&conf->wait_for_overlap);

3068
		while (bi && bi->bi_iter.bi_sector <
3069 3070 3071
			sh->dev[i].sector + STRIPE_SECTORS) {
			struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector);
			clear_bit(BIO_UPTODATE, &bi->bi_flags);
3072
			if (!raid5_dec_bi_active_stripes(bi)) {
3073 3074 3075 3076 3077 3078
				md_write_end(conf->mddev);
				bi->bi_next = *return_bi;
				*return_bi = bi;
			}
			bi = nextbi;
		}
3079 3080 3081 3082
		if (bitmap_end)
			bitmap_endwrite(conf->mddev->bitmap, sh->sector,
				STRIPE_SECTORS, 0, 0);
		bitmap_end = 0;
3083 3084 3085
		/* and fail all 'written' */
		bi = sh->dev[i].written;
		sh->dev[i].written = NULL;
3086 3087 3088 3089 3090
		if (test_and_clear_bit(R5_SkipCopy, &sh->dev[i].flags)) {
			WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
			sh->dev[i].page = sh->dev[i].orig_page;
		}

3091
		if (bi) bitmap_end = 1;
3092
		while (bi && bi->bi_iter.bi_sector <
3093 3094 3095
		       sh->dev[i].sector + STRIPE_SECTORS) {
			struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector);
			clear_bit(BIO_UPTODATE, &bi->bi_flags);
3096
			if (!raid5_dec_bi_active_stripes(bi)) {
3097 3098 3099 3100 3101 3102 3103
				md_write_end(conf->mddev);
				bi->bi_next = *return_bi;
				*return_bi = bi;
			}
			bi = bi2;
		}

3104 3105 3106 3107 3108 3109
		/* fail any reads if this device is non-operational and
		 * the data has not reached the cache yet.
		 */
		if (!test_bit(R5_Wantfill, &sh->dev[i].flags) &&
		    (!test_bit(R5_Insync, &sh->dev[i].flags) ||
		      test_bit(R5_ReadError, &sh->dev[i].flags))) {
3110
			spin_lock_irq(&sh->stripe_lock);
3111 3112
			bi = sh->dev[i].toread;
			sh->dev[i].toread = NULL;
3113
			spin_unlock_irq(&sh->stripe_lock);
3114 3115
			if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
				wake_up(&conf->wait_for_overlap);
3116
			while (bi && bi->bi_iter.bi_sector <
3117 3118 3119 3120
			       sh->dev[i].sector + STRIPE_SECTORS) {
				struct bio *nextbi =
					r5_next_bio(bi, sh->dev[i].sector);
				clear_bit(BIO_UPTODATE, &bi->bi_flags);
3121
				if (!raid5_dec_bi_active_stripes(bi)) {
3122 3123 3124 3125 3126 3127 3128 3129 3130
					bi->bi_next = *return_bi;
					*return_bi = bi;
				}
				bi = nextbi;
			}
		}
		if (bitmap_end)
			bitmap_endwrite(conf->mddev->bitmap, sh->sector,
					STRIPE_SECTORS, 0, 0);
3131 3132 3133 3134
		/* If we were in the middle of a write the parity block might
		 * still be locked - so just clear all R5_LOCKED flags
		 */
		clear_bit(R5_LOCKED, &sh->dev[i].flags);
3135 3136
	}

3137 3138 3139
	if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
		if (atomic_dec_and_test(&conf->pending_full_writes))
			md_wakeup_thread(conf->mddev->thread);
3140 3141
}

3142
static void
3143
handle_failed_sync(struct r5conf *conf, struct stripe_head *sh,
3144 3145 3146 3147 3148
		   struct stripe_head_state *s)
{
	int abort = 0;
	int i;

3149
	BUG_ON(sh->batch_head);
3150
	clear_bit(STRIPE_SYNCING, &sh->state);
3151 3152
	if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))
		wake_up(&conf->wait_for_overlap);
3153
	s->syncing = 0;
3154
	s->replacing = 0;
3155
	/* There is nothing more to do for sync/check/repair.
3156 3157 3158
	 * Don't even need to abort as that is handled elsewhere
	 * if needed, and not always wanted e.g. if there is a known
	 * bad block here.
3159
	 * For recover/replace we need to record a bad block on all
3160 3161
	 * non-sync devices, or abort the recovery
	 */
3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184
	if (test_bit(MD_RECOVERY_RECOVER, &conf->mddev->recovery)) {
		/* During recovery devices cannot be removed, so
		 * locking and refcounting of rdevs is not needed
		 */
		for (i = 0; i < conf->raid_disks; i++) {
			struct md_rdev *rdev = conf->disks[i].rdev;
			if (rdev
			    && !test_bit(Faulty, &rdev->flags)
			    && !test_bit(In_sync, &rdev->flags)
			    && !rdev_set_badblocks(rdev, sh->sector,
						   STRIPE_SECTORS, 0))
				abort = 1;
			rdev = conf->disks[i].replacement;
			if (rdev
			    && !test_bit(Faulty, &rdev->flags)
			    && !test_bit(In_sync, &rdev->flags)
			    && !rdev_set_badblocks(rdev, sh->sector,
						   STRIPE_SECTORS, 0))
				abort = 1;
		}
		if (abort)
			conf->recovery_disabled =
				conf->mddev->recovery_disabled;
3185
	}
3186
	md_done_sync(conf->mddev, STRIPE_SECTORS, !abort);
3187 3188
}

3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204
static int want_replace(struct stripe_head *sh, int disk_idx)
{
	struct md_rdev *rdev;
	int rv = 0;
	/* Doing recovery so rcu locking not required */
	rdev = sh->raid_conf->disks[disk_idx].replacement;
	if (rdev
	    && !test_bit(Faulty, &rdev->flags)
	    && !test_bit(In_sync, &rdev->flags)
	    && (rdev->recovery_offset <= sh->sector
		|| rdev->mddev->recovery_cp <= sh->sector))
		rv = 1;

	return rv;
}

3205
/* fetch_block - checks the given member device to see if its data needs
3206 3207 3208
 * to be read or computed to satisfy a request.
 *
 * Returns 1 when no more member devices need to be checked, otherwise returns
3209
 * 0 to tell the loop in handle_stripe_fill to continue
3210
 */
3211 3212 3213

static int need_this_block(struct stripe_head *sh, struct stripe_head_state *s,
			   int disk_idx, int disks)
3214
{
3215
	struct r5dev *dev = &sh->dev[disk_idx];
3216 3217
	struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]],
				  &sh->dev[s->failed_num[1]] };
3218
	int i;
3219

3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246

	if (test_bit(R5_LOCKED, &dev->flags) ||
	    test_bit(R5_UPTODATE, &dev->flags))
		/* No point reading this as we already have it or have
		 * decided to get it.
		 */
		return 0;

	if (dev->toread ||
	    (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)))
		/* We need this block to directly satisfy a request */
		return 1;

	if (s->syncing || s->expanding ||
	    (s->replacing && want_replace(sh, disk_idx)))
		/* When syncing, or expanding we read everything.
		 * When replacing, we need the replaced block.
		 */
		return 1;

	if ((s->failed >= 1 && fdev[0]->toread) ||
	    (s->failed >= 2 && fdev[1]->toread))
		/* If we want to read from a failed device, then
		 * we need to actually read every other device.
		 */
		return 1;

3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265
	/* Sometimes neither read-modify-write nor reconstruct-write
	 * cycles can work.  In those cases we read every block we
	 * can.  Then the parity-update is certain to have enough to
	 * work with.
	 * This can only be a problem when we need to write something,
	 * and some device has failed.  If either of those tests
	 * fail we need look no further.
	 */
	if (!s->failed || !s->to_write)
		return 0;

	if (test_bit(R5_Insync, &dev->flags) &&
	    !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
		/* Pre-reads at not permitted until after short delay
		 * to gather multiple requests.  However if this
		 * device is no Insync, the block could only be be computed
		 * and there is no need to delay that.
		 */
		return 0;
3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291

	for (i = 0; i < s->failed; i++) {
		if (fdev[i]->towrite &&
		    !test_bit(R5_UPTODATE, &fdev[i]->flags) &&
		    !test_bit(R5_OVERWRITE, &fdev[i]->flags))
			/* If we have a partial write to a failed
			 * device, then we will need to reconstruct
			 * the content of that device, so all other
			 * devices must be read.
			 */
			return 1;
	}

	/* If we are forced to do a reconstruct-write, either because
	 * the current RAID6 implementation only supports that, or
	 * or because parity cannot be trusted and we are currently
	 * recovering it, there is extra need to be careful.
	 * If one of the devices that we would need to read, because
	 * it is not being overwritten (and maybe not written at all)
	 * is missing/faulty, then we need to read everything we can.
	 */
	if (sh->raid_conf->level != 6 &&
	    sh->sector < sh->raid_conf->mddev->recovery_cp)
		/* reconstruct-write isn't being forced */
		return 0;
	for (i = 0; i < s->failed; i++) {
3292 3293 3294
		if (s->failed_num[i] != sh->pd_idx &&
		    s->failed_num[i] != sh->qd_idx &&
		    !test_bit(R5_UPTODATE, &fdev[i]->flags) &&
3295 3296 3297 3298
		    !test_bit(R5_OVERWRITE, &fdev[i]->flags))
			return 1;
	}

3299 3300 3301 3302 3303 3304 3305 3306 3307 3308
	return 0;
}

static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s,
		       int disk_idx, int disks)
{
	struct r5dev *dev = &sh->dev[disk_idx];

	/* is the data in this block needed, and can we get it? */
	if (need_this_block(sh, s, disk_idx, disks)) {
3309 3310 3311 3312 3313
		/* we would like to get this block, possibly by computing it,
		 * otherwise read it if the backing disk is insync
		 */
		BUG_ON(test_bit(R5_Wantcompute, &dev->flags));
		BUG_ON(test_bit(R5_Wantread, &dev->flags));
3314
		BUG_ON(sh->batch_head);
3315
		if ((s->uptodate == disks - 1) &&
3316 3317
		    (s->failed && (disk_idx == s->failed_num[0] ||
				   disk_idx == s->failed_num[1]))) {
3318 3319
			/* have disk failed, and we're requested to fetch it;
			 * do compute it
3320
			 */
3321 3322 3323 3324 3325 3326 3327 3328
			pr_debug("Computing stripe %llu block %d\n",
			       (unsigned long long)sh->sector, disk_idx);
			set_bit(STRIPE_COMPUTE_RUN, &sh->state);
			set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
			set_bit(R5_Wantcompute, &dev->flags);
			sh->ops.target = disk_idx;
			sh->ops.target2 = -1; /* no 2nd target */
			s->req_compute = 1;
3329 3330 3331 3332 3333 3334
			/* Careful: from this point on 'uptodate' is in the eye
			 * of raid_run_ops which services 'compute' operations
			 * before writes. R5_Wantcompute flags a block that will
			 * be R5_UPTODATE by the time it is needed for a
			 * subsequent operation.
			 */
3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347
			s->uptodate++;
			return 1;
		} else if (s->uptodate == disks-2 && s->failed >= 2) {
			/* Computing 2-failure is *very* expensive; only
			 * do it if failed >= 2
			 */
			int other;
			for (other = disks; other--; ) {
				if (other == disk_idx)
					continue;
				if (!test_bit(R5_UPTODATE,
				      &sh->dev[other].flags))
					break;
3348
			}
3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361 3362 3363 3364 3365 3366 3367
			BUG_ON(other < 0);
			pr_debug("Computing stripe %llu blocks %d,%d\n",
			       (unsigned long long)sh->sector,
			       disk_idx, other);
			set_bit(STRIPE_COMPUTE_RUN, &sh->state);
			set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
			set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags);
			set_bit(R5_Wantcompute, &sh->dev[other].flags);
			sh->ops.target = disk_idx;
			sh->ops.target2 = other;
			s->uptodate += 2;
			s->req_compute = 1;
			return 1;
		} else if (test_bit(R5_Insync, &dev->flags)) {
			set_bit(R5_LOCKED, &dev->flags);
			set_bit(R5_Wantread, &dev->flags);
			s->locked++;
			pr_debug("Reading block %d (sync=%d)\n",
				disk_idx, s->syncing);
3368 3369
		}
	}
3370 3371 3372 3373 3374

	return 0;
}

/**
3375
 * handle_stripe_fill - read or compute data to satisfy pending requests.
3376
 */
3377 3378 3379
static void handle_stripe_fill(struct stripe_head *sh,
			       struct stripe_head_state *s,
			       int disks)
3380 3381 3382 3383 3384 3385 3386 3387 3388 3389
{
	int i;

	/* look for blocks to read/compute, skip this if a compute
	 * is already in flight, or if the stripe contents are in the
	 * midst of changing due to a write
	 */
	if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state &&
	    !sh->reconstruct_state)
		for (i = disks; i--; )
3390
			if (fetch_block(sh, s, i, disks))
3391
				break;
3392 3393 3394
	set_bit(STRIPE_HANDLE, &sh->state);
}

3395
/* handle_stripe_clean_event
3396 3397 3398 3399
 * any written block on an uptodate or failed drive can be returned.
 * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but
 * never LOCKED, so we don't need to test 'failed' directly.
 */
3400
static void handle_stripe_clean_event(struct r5conf *conf,
3401 3402 3403 3404
	struct stripe_head *sh, int disks, struct bio **return_bi)
{
	int i;
	struct r5dev *dev;
3405
	int discard_pending = 0;
3406 3407 3408
	struct stripe_head *head_sh = sh;
	bool do_endio = false;
	int wakeup_nr = 0;
3409 3410 3411 3412 3413

	for (i = disks; i--; )
		if (sh->dev[i].written) {
			dev = &sh->dev[i];
			if (!test_bit(R5_LOCKED, &dev->flags) &&
3414
			    (test_bit(R5_UPTODATE, &dev->flags) ||
3415 3416
			     test_bit(R5_Discard, &dev->flags) ||
			     test_bit(R5_SkipCopy, &dev->flags))) {
3417 3418
				/* We can return any write requests */
				struct bio *wbi, *wbi2;
3419
				pr_debug("Return write for disc %d\n", i);
3420 3421
				if (test_and_clear_bit(R5_Discard, &dev->flags))
					clear_bit(R5_UPTODATE, &dev->flags);
3422 3423 3424
				if (test_and_clear_bit(R5_SkipCopy, &dev->flags)) {
					WARN_ON(test_bit(R5_UPTODATE, &dev->flags));
				}
3425 3426 3427 3428
				do_endio = true;

returnbi:
				dev->page = dev->orig_page;
3429 3430
				wbi = dev->written;
				dev->written = NULL;
3431
				while (wbi && wbi->bi_iter.bi_sector <
3432 3433
					dev->sector + STRIPE_SECTORS) {
					wbi2 = r5_next_bio(wbi, dev->sector);
3434
					if (!raid5_dec_bi_active_stripes(wbi)) {
3435 3436 3437 3438 3439 3440
						md_write_end(conf->mddev);
						wbi->bi_next = *return_bi;
						*return_bi = wbi;
					}
					wbi = wbi2;
				}
3441 3442
				bitmap_endwrite(conf->mddev->bitmap, sh->sector,
						STRIPE_SECTORS,
3443
					 !test_bit(STRIPE_DEGRADED, &sh->state),
3444
						0);
3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455
				if (head_sh->batch_head) {
					sh = list_first_entry(&sh->batch_list,
							      struct stripe_head,
							      batch_list);
					if (sh != head_sh) {
						dev = &sh->dev[i];
						goto returnbi;
					}
				}
				sh = head_sh;
				dev = &sh->dev[i];
3456 3457
			} else if (test_bit(R5_Discard, &dev->flags))
				discard_pending = 1;
3458 3459
			WARN_ON(test_bit(R5_SkipCopy, &dev->flags));
			WARN_ON(dev->page != dev->orig_page);
3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470
		}
	if (!discard_pending &&
	    test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)) {
		clear_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);
		clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
		if (sh->qd_idx >= 0) {
			clear_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);
			clear_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags);
		}
		/* now that discard is done we can proceed with any sync */
		clear_bit(STRIPE_DISCARD, &sh->state);
S
Shaohua Li 已提交
3471 3472 3473 3474 3475 3476
		/*
		 * SCSI discard will change some bio fields and the stripe has
		 * no updated data, so remove it from hash list and the stripe
		 * will be reinitialized
		 */
		spin_lock_irq(&conf->device_lock);
3477
unhash:
S
Shaohua Li 已提交
3478
		remove_hash(sh);
3479 3480 3481 3482 3483 3484
		if (head_sh->batch_head) {
			sh = list_first_entry(&sh->batch_list,
					      struct stripe_head, batch_list);
			if (sh != head_sh)
					goto unhash;
		}
S
Shaohua Li 已提交
3485
		spin_unlock_irq(&conf->device_lock);
3486 3487
		sh = head_sh;

3488 3489 3490 3491
		if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state))
			set_bit(STRIPE_HANDLE, &sh->state);

	}
3492 3493 3494 3495

	if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
		if (atomic_dec_and_test(&conf->pending_full_writes))
			md_wakeup_thread(conf->mddev->thread);
3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508

	if (!head_sh->batch_head || !do_endio)
		return;
	for (i = 0; i < head_sh->disks; i++) {
		if (test_and_clear_bit(R5_Overlap, &head_sh->dev[i].flags))
			wakeup_nr++;
	}
	while (!list_empty(&head_sh->batch_list)) {
		int i;
		sh = list_first_entry(&head_sh->batch_list,
				      struct stripe_head, batch_list);
		list_del_init(&sh->batch_list);

3509 3510 3511 3512
		set_mask_bits(&sh->state, ~STRIPE_EXPAND_SYNC_FLAG,
			      head_sh->state & ~((1 << STRIPE_ACTIVE) |
						 (1 << STRIPE_PREREAD_ACTIVE) |
						 STRIPE_EXPAND_SYNC_FLAG));
3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523
		sh->check_state = head_sh->check_state;
		sh->reconstruct_state = head_sh->reconstruct_state;
		for (i = 0; i < sh->disks; i++) {
			if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
				wakeup_nr++;
			sh->dev[i].flags = head_sh->dev[i].flags;
		}

		spin_lock_irq(&sh->stripe_lock);
		sh->batch_head = NULL;
		spin_unlock_irq(&sh->stripe_lock);
3524 3525
		if (sh->state & STRIPE_EXPAND_SYNC_FLAG)
			set_bit(STRIPE_HANDLE, &sh->state);
3526 3527 3528 3529 3530 3531 3532
		release_stripe(sh);
	}

	spin_lock_irq(&head_sh->stripe_lock);
	head_sh->batch_head = NULL;
	spin_unlock_irq(&head_sh->stripe_lock);
	wake_up_nr(&conf->wait_for_overlap, wakeup_nr);
3533 3534
	if (head_sh->state & STRIPE_EXPAND_SYNC_FLAG)
		set_bit(STRIPE_HANDLE, &head_sh->state);
3535 3536
}

3537
static void handle_stripe_dirtying(struct r5conf *conf,
3538 3539 3540
				   struct stripe_head *sh,
				   struct stripe_head_state *s,
				   int disks)
3541 3542
{
	int rmw = 0, rcw = 0, i;
3543 3544
	sector_t recovery_cp = conf->mddev->recovery_cp;

3545
	/* Check whether resync is now happening or should start.
3546 3547 3548 3549 3550 3551
	 * If yes, then the array is dirty (after unclean shutdown or
	 * initial creation), so parity in some stripes might be inconsistent.
	 * In this case, we need to always do reconstruct-write, to ensure
	 * that in case of drive failure or read-error correction, we
	 * generate correct data from the parity.
	 */
3552
	if (conf->rmw_level == PARITY_DISABLE_RMW ||
3553 3554
	    (recovery_cp < MaxSector && sh->sector >= recovery_cp &&
	     s->failed == 0)) {
3555
		/* Calculate the real rcw later - for now make it
3556 3557 3558
		 * look like rcw is cheaper
		 */
		rcw = 1; rmw = 2;
3559 3560
		pr_debug("force RCW rmw_level=%u, recovery_cp=%llu sh->sector=%llu\n",
			 conf->rmw_level, (unsigned long long)recovery_cp,
3561
			 (unsigned long long)sh->sector);
3562
	} else for (i = disks; i--; ) {
3563 3564
		/* would I have to read this buffer for read_modify_write */
		struct r5dev *dev = &sh->dev[i];
3565
		if ((dev->towrite || i == sh->pd_idx || i == sh->qd_idx) &&
3566
		    !test_bit(R5_LOCKED, &dev->flags) &&
3567 3568
		    !(test_bit(R5_UPTODATE, &dev->flags) ||
		      test_bit(R5_Wantcompute, &dev->flags))) {
3569 3570 3571 3572 3573 3574
			if (test_bit(R5_Insync, &dev->flags))
				rmw++;
			else
				rmw += 2*disks;  /* cannot read it */
		}
		/* Would I have to read this buffer for reconstruct_write */
3575 3576
		if (!test_bit(R5_OVERWRITE, &dev->flags) &&
		    i != sh->pd_idx && i != sh->qd_idx &&
3577
		    !test_bit(R5_LOCKED, &dev->flags) &&
3578 3579
		    !(test_bit(R5_UPTODATE, &dev->flags) ||
		    test_bit(R5_Wantcompute, &dev->flags))) {
3580 3581
			if (test_bit(R5_Insync, &dev->flags))
				rcw++;
3582 3583 3584 3585
			else
				rcw += 2*disks;
		}
	}
3586
	pr_debug("for sector %llu, rmw=%d rcw=%d\n",
3587 3588
		(unsigned long long)sh->sector, rmw, rcw);
	set_bit(STRIPE_HANDLE, &sh->state);
3589
	if ((rmw < rcw || (rmw == rcw && conf->rmw_level == PARITY_ENABLE_RMW)) && rmw > 0) {
3590
		/* prefer read-modify-write, but need to get some data */
3591 3592 3593 3594
		if (conf->mddev->queue)
			blk_add_trace_msg(conf->mddev->queue,
					  "raid5 rmw %llu %d",
					  (unsigned long long)sh->sector, rmw);
3595 3596
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
3597
			if ((dev->towrite || i == sh->pd_idx || i == sh->qd_idx) &&
3598
			    !test_bit(R5_LOCKED, &dev->flags) &&
3599 3600
			    !(test_bit(R5_UPTODATE, &dev->flags) ||
			    test_bit(R5_Wantcompute, &dev->flags)) &&
3601
			    test_bit(R5_Insync, &dev->flags)) {
3602 3603 3604 3605
				if (test_bit(STRIPE_PREREAD_ACTIVE,
					     &sh->state)) {
					pr_debug("Read_old block %d for r-m-w\n",
						 i);
3606 3607 3608 3609 3610 3611 3612 3613 3614
					set_bit(R5_LOCKED, &dev->flags);
					set_bit(R5_Wantread, &dev->flags);
					s->locked++;
				} else {
					set_bit(STRIPE_DELAYED, &sh->state);
					set_bit(STRIPE_HANDLE, &sh->state);
				}
			}
		}
N
NeilBrown 已提交
3615
	}
3616
	if ((rcw < rmw || (rcw == rmw && conf->rmw_level != PARITY_ENABLE_RMW)) && rcw > 0) {
3617
		/* want reconstruct write, but need to get some data */
N
NeilBrown 已提交
3618
		int qread =0;
3619
		rcw = 0;
3620 3621 3622
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if (!test_bit(R5_OVERWRITE, &dev->flags) &&
3623
			    i != sh->pd_idx && i != sh->qd_idx &&
3624
			    !test_bit(R5_LOCKED, &dev->flags) &&
3625
			    !(test_bit(R5_UPTODATE, &dev->flags) ||
3626 3627
			      test_bit(R5_Wantcompute, &dev->flags))) {
				rcw++;
3628 3629 3630
				if (test_bit(R5_Insync, &dev->flags) &&
				    test_bit(STRIPE_PREREAD_ACTIVE,
					     &sh->state)) {
3631
					pr_debug("Read_old block "
3632 3633 3634 3635
						"%d for Reconstruct\n", i);
					set_bit(R5_LOCKED, &dev->flags);
					set_bit(R5_Wantread, &dev->flags);
					s->locked++;
N
NeilBrown 已提交
3636
					qread++;
3637 3638 3639 3640 3641 3642
				} else {
					set_bit(STRIPE_DELAYED, &sh->state);
					set_bit(STRIPE_HANDLE, &sh->state);
				}
			}
		}
3643
		if (rcw && conf->mddev->queue)
N
NeilBrown 已提交
3644 3645 3646
			blk_add_trace_msg(conf->mddev->queue, "raid5 rcw %llu %d %d %d",
					  (unsigned long long)sh->sector,
					  rcw, qread, test_bit(STRIPE_DELAYED, &sh->state));
3647
	}
3648 3649 3650 3651 3652

	if (rcw > disks && rmw > disks &&
	    !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
		set_bit(STRIPE_DELAYED, &sh->state);

3653 3654 3655
	/* now if nothing is locked, and if we have enough data,
	 * we can start a write request
	 */
3656 3657
	/* since handle_stripe can be called at any time we need to handle the
	 * case where a compute block operation has been submitted and then a
3658 3659
	 * subsequent call wants to start a write request.  raid_run_ops only
	 * handles the case where compute block and reconstruct are requested
3660 3661 3662
	 * simultaneously.  If this is not the case then new writes need to be
	 * held off until the compute completes.
	 */
3663 3664 3665
	if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) &&
	    (s->locked == 0 && (rcw == 0 || rmw == 0) &&
	    !test_bit(STRIPE_BIT_DELAY, &sh->state)))
3666
		schedule_reconstruction(sh, s, rcw == 0, 0);
3667 3668
}

3669
static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh,
3670 3671
				struct stripe_head_state *s, int disks)
{
3672
	struct r5dev *dev = NULL;
3673

3674
	BUG_ON(sh->batch_head);
3675
	set_bit(STRIPE_HANDLE, &sh->state);
3676

3677 3678 3679
	switch (sh->check_state) {
	case check_state_idle:
		/* start a new check operation if there are no failures */
3680 3681
		if (s->failed == 0) {
			BUG_ON(s->uptodate != disks);
3682 3683
			sh->check_state = check_state_run;
			set_bit(STRIPE_OP_CHECK, &s->ops_request);
3684 3685
			clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
			s->uptodate--;
3686
			break;
3687
		}
3688
		dev = &sh->dev[s->failed_num[0]];
3689 3690 3691 3692 3693 3694 3695 3696 3697
		/* fall through */
	case check_state_compute_result:
		sh->check_state = check_state_idle;
		if (!dev)
			dev = &sh->dev[sh->pd_idx];

		/* check that a write has not made the stripe insync */
		if (test_bit(STRIPE_INSYNC, &sh->state))
			break;
D
Dan Williams 已提交
3698

3699 3700 3701 3702 3703
		/* either failed parity check, or recovery is happening */
		BUG_ON(!test_bit(R5_UPTODATE, &dev->flags));
		BUG_ON(s->uptodate != disks);

		set_bit(R5_LOCKED, &dev->flags);
3704
		s->locked++;
3705
		set_bit(R5_Wantwrite, &dev->flags);
3706

3707 3708
		clear_bit(STRIPE_DEGRADED, &sh->state);
		set_bit(STRIPE_INSYNC, &sh->state);
3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724
		break;
	case check_state_run:
		break; /* we will be called again upon completion */
	case check_state_check_result:
		sh->check_state = check_state_idle;

		/* if a failure occurred during the check operation, leave
		 * STRIPE_INSYNC not set and let the stripe be handled again
		 */
		if (s->failed)
			break;

		/* handle a successful check operation, if parity is correct
		 * we are done.  Otherwise update the mismatch count and repair
		 * parity if !MD_RECOVERY_CHECK
		 */
D
Dan Williams 已提交
3725
		if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0)
3726 3727 3728 3729 3730
			/* parity is correct (on disc,
			 * not in buffer any more)
			 */
			set_bit(STRIPE_INSYNC, &sh->state);
		else {
3731
			atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches);
3732 3733 3734 3735 3736
			if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery))
				/* don't try to repair!! */
				set_bit(STRIPE_INSYNC, &sh->state);
			else {
				sh->check_state = check_state_compute_run;
3737
				set_bit(STRIPE_COMPUTE_RUN, &sh->state);
3738 3739 3740 3741
				set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
				set_bit(R5_Wantcompute,
					&sh->dev[sh->pd_idx].flags);
				sh->ops.target = sh->pd_idx;
3742
				sh->ops.target2 = -1;
3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753
				s->uptodate++;
			}
		}
		break;
	case check_state_compute_run:
		break;
	default:
		printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
		       __func__, sh->check_state,
		       (unsigned long long) sh->sector);
		BUG();
3754 3755 3756
	}
}

3757
static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh,
3758
				  struct stripe_head_state *s,
3759
				  int disks)
3760 3761
{
	int pd_idx = sh->pd_idx;
N
NeilBrown 已提交
3762
	int qd_idx = sh->qd_idx;
3763
	struct r5dev *dev;
3764

3765
	BUG_ON(sh->batch_head);
3766 3767 3768
	set_bit(STRIPE_HANDLE, &sh->state);

	BUG_ON(s->failed > 2);
3769

3770 3771 3772 3773 3774 3775
	/* Want to check and possibly repair P and Q.
	 * However there could be one 'failed' device, in which
	 * case we can only check one of them, possibly using the
	 * other to generate missing data
	 */

3776 3777 3778
	switch (sh->check_state) {
	case check_state_idle:
		/* start a new check operation if there are < 2 failures */
3779
		if (s->failed == s->q_failed) {
3780
			/* The only possible failed device holds Q, so it
3781 3782 3783
			 * makes sense to check P (If anything else were failed,
			 * we would have used P to recreate it).
			 */
3784
			sh->check_state = check_state_run;
3785
		}
3786
		if (!s->q_failed && s->failed < 2) {
3787
			/* Q is not failed, and we didn't use it to generate
3788 3789
			 * anything, so it makes sense to check it
			 */
3790 3791 3792 3793
			if (sh->check_state == check_state_run)
				sh->check_state = check_state_run_pq;
			else
				sh->check_state = check_state_run_q;
3794 3795
		}

3796 3797
		/* discard potentially stale zero_sum_result */
		sh->ops.zero_sum_result = 0;
3798

3799 3800 3801 3802
		if (sh->check_state == check_state_run) {
			/* async_xor_zero_sum destroys the contents of P */
			clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
			s->uptodate--;
3803
		}
3804 3805 3806 3807 3808 3809 3810
		if (sh->check_state >= check_state_run &&
		    sh->check_state <= check_state_run_pq) {
			/* async_syndrome_zero_sum preserves P and Q, so
			 * no need to mark them !uptodate here
			 */
			set_bit(STRIPE_OP_CHECK, &s->ops_request);
			break;
3811 3812
		}

3813 3814 3815 3816 3817
		/* we have 2-disk failure */
		BUG_ON(s->failed != 2);
		/* fall through */
	case check_state_compute_result:
		sh->check_state = check_state_idle;
3818

3819 3820 3821
		/* check that a write has not made the stripe insync */
		if (test_bit(STRIPE_INSYNC, &sh->state))
			break;
3822 3823

		/* now write out any block on a failed drive,
3824
		 * or P or Q if they were recomputed
3825
		 */
3826
		BUG_ON(s->uptodate < disks - 1); /* We don't need Q to recover */
3827
		if (s->failed == 2) {
3828
			dev = &sh->dev[s->failed_num[1]];
3829 3830 3831 3832 3833
			s->locked++;
			set_bit(R5_LOCKED, &dev->flags);
			set_bit(R5_Wantwrite, &dev->flags);
		}
		if (s->failed >= 1) {
3834
			dev = &sh->dev[s->failed_num[0]];
3835 3836 3837 3838
			s->locked++;
			set_bit(R5_LOCKED, &dev->flags);
			set_bit(R5_Wantwrite, &dev->flags);
		}
3839
		if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
3840 3841 3842 3843 3844
			dev = &sh->dev[pd_idx];
			s->locked++;
			set_bit(R5_LOCKED, &dev->flags);
			set_bit(R5_Wantwrite, &dev->flags);
		}
3845
		if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
3846 3847 3848 3849 3850 3851 3852 3853
			dev = &sh->dev[qd_idx];
			s->locked++;
			set_bit(R5_LOCKED, &dev->flags);
			set_bit(R5_Wantwrite, &dev->flags);
		}
		clear_bit(STRIPE_DEGRADED, &sh->state);

		set_bit(STRIPE_INSYNC, &sh->state);
3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882
		break;
	case check_state_run:
	case check_state_run_q:
	case check_state_run_pq:
		break; /* we will be called again upon completion */
	case check_state_check_result:
		sh->check_state = check_state_idle;

		/* handle a successful check operation, if parity is correct
		 * we are done.  Otherwise update the mismatch count and repair
		 * parity if !MD_RECOVERY_CHECK
		 */
		if (sh->ops.zero_sum_result == 0) {
			/* both parities are correct */
			if (!s->failed)
				set_bit(STRIPE_INSYNC, &sh->state);
			else {
				/* in contrast to the raid5 case we can validate
				 * parity, but still have a failure to write
				 * back
				 */
				sh->check_state = check_state_compute_result;
				/* Returning at this point means that we may go
				 * off and bring p and/or q uptodate again so
				 * we make sure to check zero_sum_result again
				 * to verify if p or q need writeback
				 */
			}
		} else {
3883
			atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches);
3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917
			if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery))
				/* don't try to repair!! */
				set_bit(STRIPE_INSYNC, &sh->state);
			else {
				int *target = &sh->ops.target;

				sh->ops.target = -1;
				sh->ops.target2 = -1;
				sh->check_state = check_state_compute_run;
				set_bit(STRIPE_COMPUTE_RUN, &sh->state);
				set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
				if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
					set_bit(R5_Wantcompute,
						&sh->dev[pd_idx].flags);
					*target = pd_idx;
					target = &sh->ops.target2;
					s->uptodate++;
				}
				if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
					set_bit(R5_Wantcompute,
						&sh->dev[qd_idx].flags);
					*target = qd_idx;
					s->uptodate++;
				}
			}
		}
		break;
	case check_state_compute_run:
		break;
	default:
		printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
		       __func__, sh->check_state,
		       (unsigned long long) sh->sector);
		BUG();
3918 3919 3920
	}
}

3921
static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh)
3922 3923 3924 3925 3926 3927
{
	int i;

	/* We have read all the blocks in this stripe and now we need to
	 * copy some of them into a target stripe for expand.
	 */
3928
	struct dma_async_tx_descriptor *tx = NULL;
3929
	BUG_ON(sh->batch_head);
3930 3931
	clear_bit(STRIPE_EXPAND_SOURCE, &sh->state);
	for (i = 0; i < sh->disks; i++)
N
NeilBrown 已提交
3932
		if (i != sh->pd_idx && i != sh->qd_idx) {
3933
			int dd_idx, j;
3934
			struct stripe_head *sh2;
3935
			struct async_submit_ctl submit;
3936

3937
			sector_t bn = compute_blocknr(sh, i, 1);
3938 3939
			sector_t s = raid5_compute_sector(conf, bn, 0,
							  &dd_idx, NULL);
3940
			sh2 = get_active_stripe(conf, s, 0, 1, 1);
3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952
			if (sh2 == NULL)
				/* so far only the early blocks of this stripe
				 * have been requested.  When later blocks
				 * get requested, we will try again
				 */
				continue;
			if (!test_bit(STRIPE_EXPANDING, &sh2->state) ||
			   test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) {
				/* must have already done this block */
				release_stripe(sh2);
				continue;
			}
3953 3954

			/* place all the copies on one channel */
3955
			init_async_submit(&submit, 0, tx, NULL, NULL, NULL);
3956
			tx = async_memcpy(sh2->dev[dd_idx].page,
3957
					  sh->dev[i].page, 0, 0, STRIPE_SIZE,
3958
					  &submit);
3959

3960 3961 3962 3963
			set_bit(R5_Expanded, &sh2->dev[dd_idx].flags);
			set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags);
			for (j = 0; j < conf->raid_disks; j++)
				if (j != sh2->pd_idx &&
3964
				    j != sh2->qd_idx &&
3965 3966 3967 3968 3969 3970 3971
				    !test_bit(R5_Expanded, &sh2->dev[j].flags))
					break;
			if (j == conf->raid_disks) {
				set_bit(STRIPE_EXPAND_READY, &sh2->state);
				set_bit(STRIPE_HANDLE, &sh2->state);
			}
			release_stripe(sh2);
3972

3973
		}
3974
	/* done submitting copies, wait for them to complete */
3975
	async_tx_quiesce(&tx);
3976
}
L
Linus Torvalds 已提交
3977 3978 3979 3980

/*
 * handle_stripe - do things to a stripe.
 *
3981 3982
 * We lock the stripe by setting STRIPE_ACTIVE and then examine the
 * state of various bits to see what needs to be done.
L
Linus Torvalds 已提交
3983
 * Possible results:
3984 3985
 *    return some read requests which now have data
 *    return some write requests which are safely on storage
L
Linus Torvalds 已提交
3986 3987 3988 3989 3990
 *    schedule a read on some buffers
 *    schedule a write of some buffers
 *    return confirmation of parity correctness
 *
 */
3991

3992
static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)
L
Linus Torvalds 已提交
3993
{
3994
	struct r5conf *conf = sh->raid_conf;
3995
	int disks = sh->disks;
3996 3997
	struct r5dev *dev;
	int i;
3998
	int do_recovery = 0;
L
Linus Torvalds 已提交
3999

4000 4001
	memset(s, 0, sizeof(*s));

4002 4003
	s->expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state) && !sh->batch_head;
	s->expanded = test_bit(STRIPE_EXPAND_READY, &sh->state) && !sh->batch_head;
4004 4005
	s->failed_num[0] = -1;
	s->failed_num[1] = -1;
L
Linus Torvalds 已提交
4006

4007
	/* Now to look around and see what can be done */
L
Linus Torvalds 已提交
4008
	rcu_read_lock();
4009
	for (i=disks; i--; ) {
4010
		struct md_rdev *rdev;
4011 4012 4013
		sector_t first_bad;
		int bad_sectors;
		int is_bad = 0;
4014

4015
		dev = &sh->dev[i];
L
Linus Torvalds 已提交
4016

4017
		pr_debug("check %d: state 0x%lx read %p write %p written %p\n",
4018 4019
			 i, dev->flags,
			 dev->toread, dev->towrite, dev->written);
4020 4021 4022 4023 4024 4025 4026 4027
		/* maybe we can reply to a read
		 *
		 * new wantfill requests are only permitted while
		 * ops_complete_biofill is guaranteed to be inactive
		 */
		if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread &&
		    !test_bit(STRIPE_BIOFILL_RUN, &sh->state))
			set_bit(R5_Wantfill, &dev->flags);
L
Linus Torvalds 已提交
4028

4029
		/* now count some things */
4030 4031 4032 4033
		if (test_bit(R5_LOCKED, &dev->flags))
			s->locked++;
		if (test_bit(R5_UPTODATE, &dev->flags))
			s->uptodate++;
4034
		if (test_bit(R5_Wantcompute, &dev->flags)) {
4035 4036
			s->compute++;
			BUG_ON(s->compute > 2);
4037
		}
L
Linus Torvalds 已提交
4038

4039
		if (test_bit(R5_Wantfill, &dev->flags))
4040
			s->to_fill++;
4041
		else if (dev->toread)
4042
			s->to_read++;
4043
		if (dev->towrite) {
4044
			s->to_write++;
4045
			if (!test_bit(R5_OVERWRITE, &dev->flags))
4046
				s->non_overwrite++;
4047
		}
4048
		if (dev->written)
4049
			s->written++;
4050 4051 4052 4053 4054 4055 4056 4057 4058 4059
		/* Prefer to use the replacement for reads, but only
		 * if it is recovered enough and has no bad blocks.
		 */
		rdev = rcu_dereference(conf->disks[i].replacement);
		if (rdev && !test_bit(Faulty, &rdev->flags) &&
		    rdev->recovery_offset >= sh->sector + STRIPE_SECTORS &&
		    !is_badblock(rdev, sh->sector, STRIPE_SECTORS,
				 &first_bad, &bad_sectors))
			set_bit(R5_ReadRepl, &dev->flags);
		else {
4060 4061
			if (rdev)
				set_bit(R5_NeedReplace, &dev->flags);
4062 4063 4064
			rdev = rcu_dereference(conf->disks[i].rdev);
			clear_bit(R5_ReadRepl, &dev->flags);
		}
4065 4066
		if (rdev && test_bit(Faulty, &rdev->flags))
			rdev = NULL;
4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078
		if (rdev) {
			is_bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS,
					     &first_bad, &bad_sectors);
			if (s->blocked_rdev == NULL
			    && (test_bit(Blocked, &rdev->flags)
				|| is_bad < 0)) {
				if (is_bad < 0)
					set_bit(BlockedBadBlocks,
						&rdev->flags);
				s->blocked_rdev = rdev;
				atomic_inc(&rdev->nr_pending);
			}
4079
		}
4080 4081 4082
		clear_bit(R5_Insync, &dev->flags);
		if (!rdev)
			/* Not in-sync */;
4083 4084
		else if (is_bad) {
			/* also not in-sync */
4085 4086
			if (!test_bit(WriteErrorSeen, &rdev->flags) &&
			    test_bit(R5_UPTODATE, &dev->flags)) {
4087 4088 4089 4090 4091 4092 4093
				/* treat as in-sync, but with a read error
				 * which we can now try to correct
				 */
				set_bit(R5_Insync, &dev->flags);
				set_bit(R5_ReadError, &dev->flags);
			}
		} else if (test_bit(In_sync, &rdev->flags))
4094
			set_bit(R5_Insync, &dev->flags);
4095
		else if (sh->sector + STRIPE_SECTORS <= rdev->recovery_offset)
4096
			/* in sync if before recovery_offset */
4097 4098 4099 4100 4101 4102 4103 4104 4105
			set_bit(R5_Insync, &dev->flags);
		else if (test_bit(R5_UPTODATE, &dev->flags) &&
			 test_bit(R5_Expanded, &dev->flags))
			/* If we've reshaped into here, we assume it is Insync.
			 * We will shortly update recovery_offset to make
			 * it official.
			 */
			set_bit(R5_Insync, &dev->flags);

4106
		if (test_bit(R5_WriteError, &dev->flags)) {
4107 4108 4109 4110 4111 4112 4113
			/* This flag does not apply to '.replacement'
			 * only to .rdev, so make sure to check that*/
			struct md_rdev *rdev2 = rcu_dereference(
				conf->disks[i].rdev);
			if (rdev2 == rdev)
				clear_bit(R5_Insync, &dev->flags);
			if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
4114
				s->handle_bad_blocks = 1;
4115
				atomic_inc(&rdev2->nr_pending);
4116 4117 4118
			} else
				clear_bit(R5_WriteError, &dev->flags);
		}
4119
		if (test_bit(R5_MadeGood, &dev->flags)) {
4120 4121 4122 4123 4124
			/* This flag does not apply to '.replacement'
			 * only to .rdev, so make sure to check that*/
			struct md_rdev *rdev2 = rcu_dereference(
				conf->disks[i].rdev);
			if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
4125
				s->handle_bad_blocks = 1;
4126
				atomic_inc(&rdev2->nr_pending);
4127 4128 4129
			} else
				clear_bit(R5_MadeGood, &dev->flags);
		}
4130 4131 4132 4133 4134 4135 4136 4137 4138
		if (test_bit(R5_MadeGoodRepl, &dev->flags)) {
			struct md_rdev *rdev2 = rcu_dereference(
				conf->disks[i].replacement);
			if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
				s->handle_bad_blocks = 1;
				atomic_inc(&rdev2->nr_pending);
			} else
				clear_bit(R5_MadeGoodRepl, &dev->flags);
		}
4139
		if (!test_bit(R5_Insync, &dev->flags)) {
4140 4141 4142
			/* The ReadError flag will just be confusing now */
			clear_bit(R5_ReadError, &dev->flags);
			clear_bit(R5_ReWrite, &dev->flags);
L
Linus Torvalds 已提交
4143
		}
4144 4145 4146
		if (test_bit(R5_ReadError, &dev->flags))
			clear_bit(R5_Insync, &dev->flags);
		if (!test_bit(R5_Insync, &dev->flags)) {
4147 4148 4149
			if (s->failed < 2)
				s->failed_num[s->failed] = i;
			s->failed++;
4150 4151
			if (rdev && !test_bit(Faulty, &rdev->flags))
				do_recovery = 1;
4152
		}
L
Linus Torvalds 已提交
4153
	}
4154 4155 4156 4157
	if (test_bit(STRIPE_SYNCING, &sh->state)) {
		/* If there is a failed device being replaced,
		 *     we must be recovering.
		 * else if we are after recovery_cp, we must be syncing
4158
		 * else if MD_RECOVERY_REQUESTED is set, we also are syncing.
4159 4160 4161 4162 4163
		 * else we can only be replacing
		 * sync and recovery both need to read all devices, and so
		 * use the same flag.
		 */
		if (do_recovery ||
4164 4165
		    sh->sector >= conf->mddev->recovery_cp ||
		    test_bit(MD_RECOVERY_REQUESTED, &(conf->mddev->recovery)))
4166 4167 4168 4169
			s->syncing = 1;
		else
			s->replacing = 1;
	}
L
Linus Torvalds 已提交
4170
	rcu_read_unlock();
4171 4172
}

4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204
static int clear_batch_ready(struct stripe_head *sh)
{
	struct stripe_head *tmp;
	if (!test_and_clear_bit(STRIPE_BATCH_READY, &sh->state))
		return 0;
	spin_lock(&sh->stripe_lock);
	if (!sh->batch_head) {
		spin_unlock(&sh->stripe_lock);
		return 0;
	}

	/*
	 * this stripe could be added to a batch list before we check
	 * BATCH_READY, skips it
	 */
	if (sh->batch_head != sh) {
		spin_unlock(&sh->stripe_lock);
		return 1;
	}
	spin_lock(&sh->batch_lock);
	list_for_each_entry(tmp, &sh->batch_list, batch_list)
		clear_bit(STRIPE_BATCH_READY, &tmp->state);
	spin_unlock(&sh->batch_lock);
	spin_unlock(&sh->stripe_lock);

	/*
	 * BATCH_READY is cleared, no new stripes can be added.
	 * batch_list can be accessed without lock
	 */
	return 0;
}

4205 4206 4207 4208 4209 4210 4211 4212 4213 4214
static void check_break_stripe_batch_list(struct stripe_head *sh)
{
	struct stripe_head *head_sh, *next;
	int i;

	if (!test_and_clear_bit(STRIPE_BATCH_ERR, &sh->state))
		return;

	head_sh = sh;

4215 4216
	list_for_each_entry_safe(sh, next, &head_sh->batch_list, batch_list) {

4217 4218
		list_del_init(&sh->batch_list);

4219 4220 4221 4222 4223
		set_mask_bits(&sh->state, ~STRIPE_EXPAND_SYNC_FLAG,
			      head_sh->state & ~((1 << STRIPE_ACTIVE) |
						 (1 << STRIPE_PREREAD_ACTIVE) |
						 (1 << STRIPE_DEGRADED) |
						 STRIPE_EXPAND_SYNC_FLAG));
4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238
		sh->check_state = head_sh->check_state;
		sh->reconstruct_state = head_sh->reconstruct_state;
		for (i = 0; i < sh->disks; i++)
			sh->dev[i].flags = head_sh->dev[i].flags &
				(~((1 << R5_WriteError) | (1 << R5_Overlap)));

		spin_lock_irq(&sh->stripe_lock);
		sh->batch_head = NULL;
		spin_unlock_irq(&sh->stripe_lock);

		set_bit(STRIPE_HANDLE, &sh->state);
		release_stripe(sh);
	}
}

4239 4240 4241
static void handle_stripe(struct stripe_head *sh)
{
	struct stripe_head_state s;
4242
	struct r5conf *conf = sh->raid_conf;
4243
	int i;
4244 4245
	int prexor;
	int disks = sh->disks;
4246
	struct r5dev *pdev, *qdev;
4247 4248

	clear_bit(STRIPE_HANDLE, &sh->state);
4249
	if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) {
4250 4251 4252 4253 4254 4255
		/* already being handled, ensure it gets handled
		 * again when current action finishes */
		set_bit(STRIPE_HANDLE, &sh->state);
		return;
	}

4256 4257 4258 4259 4260
	if (clear_batch_ready(sh) ) {
		clear_bit_unlock(STRIPE_ACTIVE, &sh->state);
		return;
	}

4261 4262
	check_break_stripe_batch_list(sh);

4263
	if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) && !sh->batch_head) {
4264 4265 4266 4267 4268 4269
		spin_lock(&sh->stripe_lock);
		/* Cannot process 'sync' concurrently with 'discard' */
		if (!test_bit(STRIPE_DISCARD, &sh->state) &&
		    test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) {
			set_bit(STRIPE_SYNCING, &sh->state);
			clear_bit(STRIPE_INSYNC, &sh->state);
4270
			clear_bit(STRIPE_REPLACED, &sh->state);
4271 4272
		}
		spin_unlock(&sh->stripe_lock);
4273 4274 4275 4276 4277 4278 4279 4280
	}
	clear_bit(STRIPE_DELAYED, &sh->state);

	pr_debug("handling stripe %llu, state=%#lx cnt=%d, "
		"pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n",
	       (unsigned long long)sh->sector, sh->state,
	       atomic_read(&sh->count), sh->pd_idx, sh->qd_idx,
	       sh->check_state, sh->reconstruct_state);
4281

4282
	analyse_stripe(sh, &s);
4283

4284 4285 4286 4287 4288
	if (s.handle_bad_blocks) {
		set_bit(STRIPE_HANDLE, &sh->state);
		goto finish;
	}

4289 4290
	if (unlikely(s.blocked_rdev)) {
		if (s.syncing || s.expanding || s.expanded ||
4291
		    s.replacing || s.to_write || s.written) {
4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311
			set_bit(STRIPE_HANDLE, &sh->state);
			goto finish;
		}
		/* There is nothing for the blocked_rdev to block */
		rdev_dec_pending(s.blocked_rdev, conf->mddev);
		s.blocked_rdev = NULL;
	}

	if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) {
		set_bit(STRIPE_OP_BIOFILL, &s.ops_request);
		set_bit(STRIPE_BIOFILL_RUN, &sh->state);
	}

	pr_debug("locked=%d uptodate=%d to_read=%d"
	       " to_write=%d failed=%d failed_num=%d,%d\n",
	       s.locked, s.uptodate, s.to_read, s.to_write, s.failed,
	       s.failed_num[0], s.failed_num[1]);
	/* check if the array has lost more than max_degraded devices and,
	 * if so, some requests might need to be failed.
	 */
4312 4313 4314 4315 4316
	if (s.failed > conf->max_degraded) {
		sh->check_state = 0;
		sh->reconstruct_state = 0;
		if (s.to_read+s.to_write+s.written)
			handle_failed_stripe(conf, sh, &s, disks, &s.return_bi);
4317
		if (s.syncing + s.replacing)
4318 4319
			handle_failed_sync(conf, sh, &s);
	}
4320

4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333
	/* Now we check to see if any write operations have recently
	 * completed
	 */
	prexor = 0;
	if (sh->reconstruct_state == reconstruct_state_prexor_drain_result)
		prexor = 1;
	if (sh->reconstruct_state == reconstruct_state_drain_result ||
	    sh->reconstruct_state == reconstruct_state_prexor_drain_result) {
		sh->reconstruct_state = reconstruct_state_idle;

		/* All the 'written' buffers and the parity block are ready to
		 * be written back to disk
		 */
4334 4335
		BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags) &&
		       !test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags));
4336
		BUG_ON(sh->qd_idx >= 0 &&
4337 4338
		       !test_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags) &&
		       !test_bit(R5_Discard, &sh->dev[sh->qd_idx].flags));
4339 4340 4341 4342 4343 4344 4345 4346 4347
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if (test_bit(R5_LOCKED, &dev->flags) &&
				(i == sh->pd_idx || i == sh->qd_idx ||
				 dev->written)) {
				pr_debug("Writing block %d\n", i);
				set_bit(R5_Wantwrite, &dev->flags);
				if (prexor)
					continue;
4348 4349
				if (s.failed > 1)
					continue;
4350 4351 4352 4353 4354 4355 4356 4357 4358 4359
				if (!test_bit(R5_Insync, &dev->flags) ||
				    ((i == sh->pd_idx || i == sh->qd_idx)  &&
				     s.failed == 0))
					set_bit(STRIPE_INSYNC, &sh->state);
			}
		}
		if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
			s.dec_preread_active = 1;
	}

4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393
	/*
	 * might be able to return some write requests if the parity blocks
	 * are safe, or on a failed drive
	 */
	pdev = &sh->dev[sh->pd_idx];
	s.p_failed = (s.failed >= 1 && s.failed_num[0] == sh->pd_idx)
		|| (s.failed >= 2 && s.failed_num[1] == sh->pd_idx);
	qdev = &sh->dev[sh->qd_idx];
	s.q_failed = (s.failed >= 1 && s.failed_num[0] == sh->qd_idx)
		|| (s.failed >= 2 && s.failed_num[1] == sh->qd_idx)
		|| conf->level < 6;

	if (s.written &&
	    (s.p_failed || ((test_bit(R5_Insync, &pdev->flags)
			     && !test_bit(R5_LOCKED, &pdev->flags)
			     && (test_bit(R5_UPTODATE, &pdev->flags) ||
				 test_bit(R5_Discard, &pdev->flags))))) &&
	    (s.q_failed || ((test_bit(R5_Insync, &qdev->flags)
			     && !test_bit(R5_LOCKED, &qdev->flags)
			     && (test_bit(R5_UPTODATE, &qdev->flags) ||
				 test_bit(R5_Discard, &qdev->flags))))))
		handle_stripe_clean_event(conf, sh, disks, &s.return_bi);

	/* Now we might consider reading some blocks, either to check/generate
	 * parity, or to satisfy requests
	 * or to load a block that is being partially written.
	 */
	if (s.to_read || s.non_overwrite
	    || (conf->level == 6 && s.to_write && s.failed)
	    || (s.syncing && (s.uptodate + s.compute < disks))
	    || s.replacing
	    || s.expanding)
		handle_stripe_fill(sh, &s, disks);

4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416
	/* Now to consider new write requests and what else, if anything
	 * should be read.  We do not handle new writes when:
	 * 1/ A 'write' operation (copy+xor) is already in flight.
	 * 2/ A 'check' operation is in flight, as it may clobber the parity
	 *    block.
	 */
	if (s.to_write && !sh->reconstruct_state && !sh->check_state)
		handle_stripe_dirtying(conf, sh, &s, disks);

	/* maybe we need to check and possibly fix the parity for this stripe
	 * Any reads will already have been scheduled, so we just see if enough
	 * data is available.  The parity check is held off while parity
	 * dependent operations are in flight.
	 */
	if (sh->check_state ||
	    (s.syncing && s.locked == 0 &&
	     !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
	     !test_bit(STRIPE_INSYNC, &sh->state))) {
		if (conf->level == 6)
			handle_parity_checks6(conf, sh, &s, disks);
		else
			handle_parity_checks5(conf, sh, &s, disks);
	}
4417

4418 4419 4420
	if ((s.replacing || s.syncing) && s.locked == 0
	    && !test_bit(STRIPE_COMPUTE_RUN, &sh->state)
	    && !test_bit(STRIPE_REPLACED, &sh->state)) {
4421 4422
		/* Write out to replacement devices where possible */
		for (i = 0; i < conf->raid_disks; i++)
4423 4424
			if (test_bit(R5_NeedReplace, &sh->dev[i].flags)) {
				WARN_ON(!test_bit(R5_UPTODATE, &sh->dev[i].flags));
4425 4426 4427 4428
				set_bit(R5_WantReplace, &sh->dev[i].flags);
				set_bit(R5_LOCKED, &sh->dev[i].flags);
				s.locked++;
			}
4429 4430 4431
		if (s.replacing)
			set_bit(STRIPE_INSYNC, &sh->state);
		set_bit(STRIPE_REPLACED, &sh->state);
4432 4433
	}
	if ((s.syncing || s.replacing) && s.locked == 0 &&
4434
	    !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
4435
	    test_bit(STRIPE_INSYNC, &sh->state)) {
4436 4437
		md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
		clear_bit(STRIPE_SYNCING, &sh->state);
4438 4439
		if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))
			wake_up(&conf->wait_for_overlap);
4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465
	}

	/* If the failed drives are just a ReadError, then we might need
	 * to progress the repair/check process
	 */
	if (s.failed <= conf->max_degraded && !conf->mddev->ro)
		for (i = 0; i < s.failed; i++) {
			struct r5dev *dev = &sh->dev[s.failed_num[i]];
			if (test_bit(R5_ReadError, &dev->flags)
			    && !test_bit(R5_LOCKED, &dev->flags)
			    && test_bit(R5_UPTODATE, &dev->flags)
				) {
				if (!test_bit(R5_ReWrite, &dev->flags)) {
					set_bit(R5_Wantwrite, &dev->flags);
					set_bit(R5_ReWrite, &dev->flags);
					set_bit(R5_LOCKED, &dev->flags);
					s.locked++;
				} else {
					/* let's read it back */
					set_bit(R5_Wantread, &dev->flags);
					set_bit(R5_LOCKED, &dev->flags);
					s.locked++;
				}
			}
		}

4466 4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492
	/* Finish reconstruct operations initiated by the expansion process */
	if (sh->reconstruct_state == reconstruct_state_result) {
		struct stripe_head *sh_src
			= get_active_stripe(conf, sh->sector, 1, 1, 1);
		if (sh_src && test_bit(STRIPE_EXPAND_SOURCE, &sh_src->state)) {
			/* sh cannot be written until sh_src has been read.
			 * so arrange for sh to be delayed a little
			 */
			set_bit(STRIPE_DELAYED, &sh->state);
			set_bit(STRIPE_HANDLE, &sh->state);
			if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE,
					      &sh_src->state))
				atomic_inc(&conf->preread_active_stripes);
			release_stripe(sh_src);
			goto finish;
		}
		if (sh_src)
			release_stripe(sh_src);

		sh->reconstruct_state = reconstruct_state_idle;
		clear_bit(STRIPE_EXPANDING, &sh->state);
		for (i = conf->raid_disks; i--; ) {
			set_bit(R5_Wantwrite, &sh->dev[i].flags);
			set_bit(R5_LOCKED, &sh->dev[i].flags);
			s.locked++;
		}
	}
4493

4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509
	if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) &&
	    !sh->reconstruct_state) {
		/* Need to write out all blocks after computing parity */
		sh->disks = conf->raid_disks;
		stripe_set_idx(sh->sector, conf, 0, sh);
		schedule_reconstruction(sh, &s, 1, 1);
	} else if (s.expanded && !sh->reconstruct_state && s.locked == 0) {
		clear_bit(STRIPE_EXPAND_READY, &sh->state);
		atomic_dec(&conf->reshape_stripes);
		wake_up(&conf->wait_for_overlap);
		md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
	}

	if (s.expanding && s.locked == 0 &&
	    !test_bit(STRIPE_COMPUTE_RUN, &sh->state))
		handle_stripe_expansion(conf, sh);
4510

4511
finish:
4512
	/* wait for this device to become unblocked */
4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524
	if (unlikely(s.blocked_rdev)) {
		if (conf->mddev->external)
			md_wait_for_blocked_rdev(s.blocked_rdev,
						 conf->mddev);
		else
			/* Internal metadata will immediately
			 * be written by raid5d, so we don't
			 * need to wait here.
			 */
			rdev_dec_pending(s.blocked_rdev,
					 conf->mddev);
	}
4525

4526 4527
	if (s.handle_bad_blocks)
		for (i = disks; i--; ) {
4528
			struct md_rdev *rdev;
4529 4530 4531 4532 4533 4534 4535 4536 4537
			struct r5dev *dev = &sh->dev[i];
			if (test_and_clear_bit(R5_WriteError, &dev->flags)) {
				/* We own a safe reference to the rdev */
				rdev = conf->disks[i].rdev;
				if (!rdev_set_badblocks(rdev, sh->sector,
							STRIPE_SECTORS, 0))
					md_error(conf->mddev, rdev);
				rdev_dec_pending(rdev, conf->mddev);
			}
4538 4539 4540
			if (test_and_clear_bit(R5_MadeGood, &dev->flags)) {
				rdev = conf->disks[i].rdev;
				rdev_clear_badblocks(rdev, sh->sector,
4541
						     STRIPE_SECTORS, 0);
4542 4543
				rdev_dec_pending(rdev, conf->mddev);
			}
4544 4545
			if (test_and_clear_bit(R5_MadeGoodRepl, &dev->flags)) {
				rdev = conf->disks[i].replacement;
4546 4547 4548
				if (!rdev)
					/* rdev have been moved down */
					rdev = conf->disks[i].rdev;
4549
				rdev_clear_badblocks(rdev, sh->sector,
4550
						     STRIPE_SECTORS, 0);
4551 4552
				rdev_dec_pending(rdev, conf->mddev);
			}
4553 4554
		}

4555 4556 4557
	if (s.ops_request)
		raid_run_ops(sh, s.ops_request);

D
Dan Williams 已提交
4558
	ops_run_io(sh, &s);
4559

4560
	if (s.dec_preread_active) {
4561
		/* We delay this until after ops_run_io so that if make_request
T
Tejun Heo 已提交
4562
		 * is waiting on a flush, it won't continue until the writes
4563 4564 4565 4566 4567 4568 4569 4570
		 * have actually been submitted.
		 */
		atomic_dec(&conf->preread_active_stripes);
		if (atomic_read(&conf->preread_active_stripes) <
		    IO_THRESHOLD)
			md_wakeup_thread(conf->mddev->thread);
	}

4571
	return_io(s.return_bi);
4572

4573
	clear_bit_unlock(STRIPE_ACTIVE, &sh->state);
4574 4575
}

4576
static void raid5_activate_delayed(struct r5conf *conf)
4577 4578 4579 4580 4581 4582 4583 4584 4585 4586
{
	if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) {
		while (!list_empty(&conf->delayed_list)) {
			struct list_head *l = conf->delayed_list.next;
			struct stripe_head *sh;
			sh = list_entry(l, struct stripe_head, lru);
			list_del_init(l);
			clear_bit(STRIPE_DELAYED, &sh->state);
			if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
				atomic_inc(&conf->preread_active_stripes);
4587
			list_add_tail(&sh->lru, &conf->hold_list);
4588
			raid5_wakeup_stripe_thread(sh);
4589
		}
N
NeilBrown 已提交
4590
	}
4591 4592
}

4593 4594
static void activate_bit_delay(struct r5conf *conf,
	struct list_head *temp_inactive_list)
4595 4596 4597 4598 4599 4600 4601
{
	/* device_lock is held */
	struct list_head head;
	list_add(&head, &conf->bitmap_list);
	list_del_init(&conf->bitmap_list);
	while (!list_empty(&head)) {
		struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru);
4602
		int hash;
4603 4604
		list_del_init(&sh->lru);
		atomic_inc(&sh->count);
4605 4606
		hash = sh->hash_lock_index;
		__release_stripe(conf, sh, &temp_inactive_list[hash]);
4607 4608 4609
	}
}

4610
static int raid5_congested(struct mddev *mddev, int bits)
4611
{
4612
	struct r5conf *conf = mddev->private;
4613 4614 4615 4616

	/* No difference between reads and writes.  Just check
	 * how busy the stripe_cache is
	 */
4617

4618
	if (test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state))
4619 4620 4621
		return 1;
	if (conf->quiesce)
		return 1;
4622
	if (atomic_read(&conf->empty_inactive_list_nr))
4623 4624 4625 4626 4627
		return 1;

	return 0;
}

4628 4629 4630
/* We want read requests to align with chunks where possible,
 * but write requests don't need to.
 */
4631
static int raid5_mergeable_bvec(struct mddev *mddev,
4632 4633
				struct bvec_merge_data *bvm,
				struct bio_vec *biovec)
4634
{
4635
	sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev);
4636
	int max;
4637
	unsigned int chunk_sectors = mddev->chunk_sectors;
4638
	unsigned int bio_sectors = bvm->bi_size >> 9;
4639

4640 4641 4642 4643 4644 4645
	/*
	 * always allow writes to be mergeable, read as well if array
	 * is degraded as we'll go through stripe cache anyway.
	 */
	if ((bvm->bi_rw & 1) == WRITE || mddev->degraded)
		return biovec->bv_len;
4646

4647 4648
	if (mddev->new_chunk_sectors < mddev->chunk_sectors)
		chunk_sectors = mddev->new_chunk_sectors;
4649 4650 4651 4652 4653 4654 4655 4656
	max =  (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9;
	if (max < 0) max = 0;
	if (max <= biovec->bv_len && bio_sectors == 0)
		return biovec->bv_len;
	else
		return max;
}

4657
static int in_chunk_boundary(struct mddev *mddev, struct bio *bio)
4658
{
4659
	sector_t sector = bio->bi_iter.bi_sector + get_start_sect(bio->bi_bdev);
4660
	unsigned int chunk_sectors = mddev->chunk_sectors;
4661
	unsigned int bio_sectors = bio_sectors(bio);
4662

4663 4664
	if (mddev->new_chunk_sectors < mddev->chunk_sectors)
		chunk_sectors = mddev->new_chunk_sectors;
4665 4666 4667 4668
	return  chunk_sectors >=
		((sector & (chunk_sectors - 1)) + bio_sectors);
}

4669 4670 4671 4672
/*
 *  add bio to the retry LIFO  ( in O(1) ... we are in interrupt )
 *  later sampled by raid5d.
 */
4673
static void add_bio_to_retry(struct bio *bi,struct r5conf *conf)
4674 4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685
{
	unsigned long flags;

	spin_lock_irqsave(&conf->device_lock, flags);

	bi->bi_next = conf->retry_read_aligned_list;
	conf->retry_read_aligned_list = bi;

	spin_unlock_irqrestore(&conf->device_lock, flags);
	md_wakeup_thread(conf->mddev->thread);
}

4686
static struct bio *remove_bio_from_retry(struct r5conf *conf)
4687 4688 4689 4690 4691 4692 4693 4694 4695 4696
{
	struct bio *bi;

	bi = conf->retry_read_aligned;
	if (bi) {
		conf->retry_read_aligned = NULL;
		return bi;
	}
	bi = conf->retry_read_aligned_list;
	if(bi) {
4697
		conf->retry_read_aligned_list = bi->bi_next;
4698
		bi->bi_next = NULL;
4699 4700 4701 4702
		/*
		 * this sets the active strip count to 1 and the processed
		 * strip count to zero (upper 8 bits)
		 */
4703
		raid5_set_bi_stripes(bi, 1); /* biased count of active stripes */
4704 4705 4706 4707 4708
	}

	return bi;
}

4709 4710 4711 4712 4713 4714
/*
 *  The "raid5_align_endio" should check if the read succeeded and if it
 *  did, call bio_endio on the original bio (having bio_put the new bio
 *  first).
 *  If the read failed..
 */
4715
static void raid5_align_endio(struct bio *bi, int error)
4716 4717
{
	struct bio* raid_bi  = bi->bi_private;
4718
	struct mddev *mddev;
4719
	struct r5conf *conf;
4720
	int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
4721
	struct md_rdev *rdev;
4722

4723
	bio_put(bi);
4724 4725 4726

	rdev = (void*)raid_bi->bi_next;
	raid_bi->bi_next = NULL;
4727 4728
	mddev = rdev->mddev;
	conf = mddev->private;
4729 4730 4731 4732

	rdev_dec_pending(rdev, conf->mddev);

	if (!error && uptodate) {
4733 4734
		trace_block_bio_complete(bdev_get_queue(raid_bi->bi_bdev),
					 raid_bi, 0);
4735
		bio_endio(raid_bi, 0);
4736 4737
		if (atomic_dec_and_test(&conf->active_aligned_reads))
			wake_up(&conf->wait_for_stripe);
4738
		return;
4739 4740
	}

4741
	pr_debug("raid5_align_endio : io error...handing IO for a retry\n");
4742 4743

	add_bio_to_retry(raid_bi, conf);
4744 4745
}

4746 4747
static int bio_fits_rdev(struct bio *bi)
{
4748
	struct request_queue *q = bdev_get_queue(bi->bi_bdev);
4749

4750
	if (bio_sectors(bi) > queue_max_sectors(q))
4751 4752
		return 0;
	blk_recount_segments(q, bi);
4753
	if (bi->bi_phys_segments > queue_max_segments(q))
4754 4755 4756 4757 4758 4759 4760 4761 4762 4763 4764
		return 0;

	if (q->merge_bvec_fn)
		/* it's too hard to apply the merge_bvec_fn at this stage,
		 * just just give up
		 */
		return 0;

	return 1;
}

4765
static int chunk_aligned_read(struct mddev *mddev, struct bio * raid_bio)
4766
{
4767
	struct r5conf *conf = mddev->private;
N
NeilBrown 已提交
4768
	int dd_idx;
4769
	struct bio* align_bi;
4770
	struct md_rdev *rdev;
4771
	sector_t end_sector;
4772 4773

	if (!in_chunk_boundary(mddev, raid_bio)) {
4774
		pr_debug("chunk_aligned_read : non aligned\n");
4775 4776 4777
		return 0;
	}
	/*
4778
	 * use bio_clone_mddev to make a copy of the bio
4779
	 */
4780
	align_bi = bio_clone_mddev(raid_bio, GFP_NOIO, mddev);
4781 4782 4783 4784 4785 4786 4787 4788 4789 4790 4791
	if (!align_bi)
		return 0;
	/*
	 *   set bi_end_io to a new function, and set bi_private to the
	 *     original bio.
	 */
	align_bi->bi_end_io  = raid5_align_endio;
	align_bi->bi_private = raid_bio;
	/*
	 *	compute position
	 */
4792 4793 4794
	align_bi->bi_iter.bi_sector =
		raid5_compute_sector(conf, raid_bio->bi_iter.bi_sector,
				     0, &dd_idx, NULL);
4795

K
Kent Overstreet 已提交
4796
	end_sector = bio_end_sector(align_bi);
4797
	rcu_read_lock();
4798 4799 4800 4801 4802 4803 4804 4805 4806 4807 4808
	rdev = rcu_dereference(conf->disks[dd_idx].replacement);
	if (!rdev || test_bit(Faulty, &rdev->flags) ||
	    rdev->recovery_offset < end_sector) {
		rdev = rcu_dereference(conf->disks[dd_idx].rdev);
		if (rdev &&
		    (test_bit(Faulty, &rdev->flags) ||
		    !(test_bit(In_sync, &rdev->flags) ||
		      rdev->recovery_offset >= end_sector)))
			rdev = NULL;
	}
	if (rdev) {
4809 4810 4811
		sector_t first_bad;
		int bad_sectors;

4812 4813
		atomic_inc(&rdev->nr_pending);
		rcu_read_unlock();
4814 4815
		raid_bio->bi_next = (void*)rdev;
		align_bi->bi_bdev =  rdev->bdev;
4816
		__clear_bit(BIO_SEG_VALID, &align_bi->bi_flags);
4817

4818
		if (!bio_fits_rdev(align_bi) ||
4819 4820
		    is_badblock(rdev, align_bi->bi_iter.bi_sector,
				bio_sectors(align_bi),
4821 4822
				&first_bad, &bad_sectors)) {
			/* too big in some way, or has a known bad block */
4823 4824 4825 4826 4827
			bio_put(align_bi);
			rdev_dec_pending(rdev, mddev);
			return 0;
		}

4828
		/* No reshape active, so we can trust rdev->data_offset */
4829
		align_bi->bi_iter.bi_sector += rdev->data_offset;
4830

4831 4832 4833
		spin_lock_irq(&conf->device_lock);
		wait_event_lock_irq(conf->wait_for_stripe,
				    conf->quiesce == 0,
4834
				    conf->device_lock);
4835 4836 4837
		atomic_inc(&conf->active_aligned_reads);
		spin_unlock_irq(&conf->device_lock);

4838 4839 4840
		if (mddev->gendisk)
			trace_block_bio_remap(bdev_get_queue(align_bi->bi_bdev),
					      align_bi, disk_devt(mddev->gendisk),
4841
					      raid_bio->bi_iter.bi_sector);
4842 4843 4844 4845
		generic_make_request(align_bi);
		return 1;
	} else {
		rcu_read_unlock();
4846
		bio_put(align_bi);
4847 4848 4849 4850
		return 0;
	}
}

4851 4852 4853 4854 4855 4856 4857 4858 4859 4860
/* __get_priority_stripe - get the next stripe to process
 *
 * Full stripe writes are allowed to pass preread active stripes up until
 * the bypass_threshold is exceeded.  In general the bypass_count
 * increments when the handle_list is handled before the hold_list; however, it
 * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a
 * stripe with in flight i/o.  The bypass_count will be reset when the
 * head of the hold_list has changed, i.e. the head was promoted to the
 * handle_list.
 */
4861
static struct stripe_head *__get_priority_stripe(struct r5conf *conf, int group)
4862
{
4863 4864
	struct stripe_head *sh = NULL, *tmp;
	struct list_head *handle_list = NULL;
4865
	struct r5worker_group *wg = NULL;
4866 4867 4868 4869 4870

	if (conf->worker_cnt_per_group == 0) {
		handle_list = &conf->handle_list;
	} else if (group != ANY_GROUP) {
		handle_list = &conf->worker_groups[group].handle_list;
4871
		wg = &conf->worker_groups[group];
4872 4873 4874 4875
	} else {
		int i;
		for (i = 0; i < conf->group_cnt; i++) {
			handle_list = &conf->worker_groups[i].handle_list;
4876
			wg = &conf->worker_groups[i];
4877 4878 4879 4880
			if (!list_empty(handle_list))
				break;
		}
	}
4881 4882 4883

	pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n",
		  __func__,
4884
		  list_empty(handle_list) ? "empty" : "busy",
4885 4886 4887
		  list_empty(&conf->hold_list) ? "empty" : "busy",
		  atomic_read(&conf->pending_full_writes), conf->bypass_count);

4888 4889
	if (!list_empty(handle_list)) {
		sh = list_entry(handle_list->next, typeof(*sh), lru);
4890 4891 4892 4893 4894 4895 4896 4897 4898 4899 4900 4901 4902 4903 4904 4905 4906

		if (list_empty(&conf->hold_list))
			conf->bypass_count = 0;
		else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) {
			if (conf->hold_list.next == conf->last_hold)
				conf->bypass_count++;
			else {
				conf->last_hold = conf->hold_list.next;
				conf->bypass_count -= conf->bypass_threshold;
				if (conf->bypass_count < 0)
					conf->bypass_count = 0;
			}
		}
	} else if (!list_empty(&conf->hold_list) &&
		   ((conf->bypass_threshold &&
		     conf->bypass_count > conf->bypass_threshold) ||
		    atomic_read(&conf->pending_full_writes) == 0)) {
4907 4908 4909 4910 4911 4912 4913 4914 4915 4916 4917 4918 4919 4920 4921 4922

		list_for_each_entry(tmp, &conf->hold_list,  lru) {
			if (conf->worker_cnt_per_group == 0 ||
			    group == ANY_GROUP ||
			    !cpu_online(tmp->cpu) ||
			    cpu_to_group(tmp->cpu) == group) {
				sh = tmp;
				break;
			}
		}

		if (sh) {
			conf->bypass_count -= conf->bypass_threshold;
			if (conf->bypass_count < 0)
				conf->bypass_count = 0;
		}
4923
		wg = NULL;
4924 4925 4926
	}

	if (!sh)
4927 4928
		return NULL;

4929 4930 4931 4932
	if (wg) {
		wg->stripes_cnt--;
		sh->group = NULL;
	}
4933
	list_del_init(&sh->lru);
4934
	BUG_ON(atomic_inc_return(&sh->count) != 1);
4935 4936
	return sh;
}
4937

4938 4939 4940
struct raid5_plug_cb {
	struct blk_plug_cb	cb;
	struct list_head	list;
4941
	struct list_head	temp_inactive_list[NR_STRIPE_HASH_LOCKS];
4942 4943 4944 4945 4946 4947 4948 4949 4950
};

static void raid5_unplug(struct blk_plug_cb *blk_cb, bool from_schedule)
{
	struct raid5_plug_cb *cb = container_of(
		blk_cb, struct raid5_plug_cb, cb);
	struct stripe_head *sh;
	struct mddev *mddev = cb->cb.data;
	struct r5conf *conf = mddev->private;
N
NeilBrown 已提交
4951
	int cnt = 0;
4952
	int hash;
4953 4954 4955 4956 4957 4958 4959 4960 4961 4962 4963

	if (cb->list.next && !list_empty(&cb->list)) {
		spin_lock_irq(&conf->device_lock);
		while (!list_empty(&cb->list)) {
			sh = list_first_entry(&cb->list, struct stripe_head, lru);
			list_del_init(&sh->lru);
			/*
			 * avoid race release_stripe_plug() sees
			 * STRIPE_ON_UNPLUG_LIST clear but the stripe
			 * is still in our list
			 */
4964
			smp_mb__before_atomic();
4965
			clear_bit(STRIPE_ON_UNPLUG_LIST, &sh->state);
S
Shaohua Li 已提交
4966 4967 4968 4969
			/*
			 * STRIPE_ON_RELEASE_LIST could be set here. In that
			 * case, the count is always > 1 here
			 */
4970 4971
			hash = sh->hash_lock_index;
			__release_stripe(conf, sh, &cb->temp_inactive_list[hash]);
N
NeilBrown 已提交
4972
			cnt++;
4973 4974 4975
		}
		spin_unlock_irq(&conf->device_lock);
	}
4976 4977
	release_inactive_stripe_list(conf, cb->temp_inactive_list,
				     NR_STRIPE_HASH_LOCKS);
4978 4979
	if (mddev->queue)
		trace_block_unplug(mddev->queue, cnt, !from_schedule);
4980 4981 4982 4983 4984 4985 4986 4987 4988 4989 4990 4991 4992 4993 4994 4995 4996 4997
	kfree(cb);
}

static void release_stripe_plug(struct mddev *mddev,
				struct stripe_head *sh)
{
	struct blk_plug_cb *blk_cb = blk_check_plugged(
		raid5_unplug, mddev,
		sizeof(struct raid5_plug_cb));
	struct raid5_plug_cb *cb;

	if (!blk_cb) {
		release_stripe(sh);
		return;
	}

	cb = container_of(blk_cb, struct raid5_plug_cb, cb);

4998 4999
	if (cb->list.next == NULL) {
		int i;
5000
		INIT_LIST_HEAD(&cb->list);
5001 5002 5003
		for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
			INIT_LIST_HEAD(cb->temp_inactive_list + i);
	}
5004 5005 5006 5007 5008 5009 5010

	if (!test_and_set_bit(STRIPE_ON_UNPLUG_LIST, &sh->state))
		list_add_tail(&sh->lru, &cb->list);
	else
		release_stripe(sh);
}

S
Shaohua Li 已提交
5011 5012 5013 5014 5015 5016 5017 5018 5019 5020 5021 5022
static void make_discard_request(struct mddev *mddev, struct bio *bi)
{
	struct r5conf *conf = mddev->private;
	sector_t logical_sector, last_sector;
	struct stripe_head *sh;
	int remaining;
	int stripe_sectors;

	if (mddev->reshape_position != MaxSector)
		/* Skip discard while reshape is happening */
		return;

5023 5024
	logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1);
	last_sector = bi->bi_iter.bi_sector + (bi->bi_iter.bi_size>>9);
S
Shaohua Li 已提交
5025 5026 5027 5028 5029 5030 5031 5032 5033 5034 5035 5036 5037 5038 5039 5040 5041 5042 5043 5044 5045

	bi->bi_next = NULL;
	bi->bi_phys_segments = 1; /* over-loaded to count active stripes */

	stripe_sectors = conf->chunk_sectors *
		(conf->raid_disks - conf->max_degraded);
	logical_sector = DIV_ROUND_UP_SECTOR_T(logical_sector,
					       stripe_sectors);
	sector_div(last_sector, stripe_sectors);

	logical_sector *= conf->chunk_sectors;
	last_sector *= conf->chunk_sectors;

	for (; logical_sector < last_sector;
	     logical_sector += STRIPE_SECTORS) {
		DEFINE_WAIT(w);
		int d;
	again:
		sh = get_active_stripe(conf, logical_sector, 0, 0, 0);
		prepare_to_wait(&conf->wait_for_overlap, &w,
				TASK_UNINTERRUPTIBLE);
5046 5047 5048 5049 5050 5051 5052
		set_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);
		if (test_bit(STRIPE_SYNCING, &sh->state)) {
			release_stripe(sh);
			schedule();
			goto again;
		}
		clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);
S
Shaohua Li 已提交
5053 5054 5055 5056 5057 5058 5059 5060 5061 5062 5063 5064
		spin_lock_irq(&sh->stripe_lock);
		for (d = 0; d < conf->raid_disks; d++) {
			if (d == sh->pd_idx || d == sh->qd_idx)
				continue;
			if (sh->dev[d].towrite || sh->dev[d].toread) {
				set_bit(R5_Overlap, &sh->dev[d].flags);
				spin_unlock_irq(&sh->stripe_lock);
				release_stripe(sh);
				schedule();
				goto again;
			}
		}
5065
		set_bit(STRIPE_DISCARD, &sh->state);
S
Shaohua Li 已提交
5066
		finish_wait(&conf->wait_for_overlap, &w);
5067
		sh->overwrite_disks = 0;
S
Shaohua Li 已提交
5068 5069 5070 5071 5072 5073
		for (d = 0; d < conf->raid_disks; d++) {
			if (d == sh->pd_idx || d == sh->qd_idx)
				continue;
			sh->dev[d].towrite = bi;
			set_bit(R5_OVERWRITE, &sh->dev[d].flags);
			raid5_inc_bi_active_stripes(bi);
5074
			sh->overwrite_disks++;
S
Shaohua Li 已提交
5075 5076 5077 5078 5079 5080 5081 5082 5083 5084 5085 5086 5087 5088 5089 5090 5091 5092 5093 5094 5095 5096 5097 5098 5099 5100 5101 5102
		}
		spin_unlock_irq(&sh->stripe_lock);
		if (conf->mddev->bitmap) {
			for (d = 0;
			     d < conf->raid_disks - conf->max_degraded;
			     d++)
				bitmap_startwrite(mddev->bitmap,
						  sh->sector,
						  STRIPE_SECTORS,
						  0);
			sh->bm_seq = conf->seq_flush + 1;
			set_bit(STRIPE_BIT_DELAY, &sh->state);
		}

		set_bit(STRIPE_HANDLE, &sh->state);
		clear_bit(STRIPE_DELAYED, &sh->state);
		if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
			atomic_inc(&conf->preread_active_stripes);
		release_stripe_plug(mddev, sh);
	}

	remaining = raid5_dec_bi_active_stripes(bi);
	if (remaining == 0) {
		md_write_end(mddev);
		bio_endio(bi, 0);
	}
}

5103
static void make_request(struct mddev *mddev, struct bio * bi)
L
Linus Torvalds 已提交
5104
{
5105
	struct r5conf *conf = mddev->private;
5106
	int dd_idx;
L
Linus Torvalds 已提交
5107 5108 5109
	sector_t new_sector;
	sector_t logical_sector, last_sector;
	struct stripe_head *sh;
5110
	const int rw = bio_data_dir(bi);
5111
	int remaining;
5112 5113
	DEFINE_WAIT(w);
	bool do_prepare;
L
Linus Torvalds 已提交
5114

T
Tejun Heo 已提交
5115 5116
	if (unlikely(bi->bi_rw & REQ_FLUSH)) {
		md_flush_request(mddev, bi);
5117
		return;
5118 5119
	}

5120
	md_write_start(mddev, bi);
5121

5122 5123 5124 5125 5126 5127
	/*
	 * If array is degraded, better not do chunk aligned read because
	 * later we might have to read it again in order to reconstruct
	 * data on failed drives.
	 */
	if (rw == READ && mddev->degraded == 0 &&
5128
	     mddev->reshape_position == MaxSector &&
5129
	     chunk_aligned_read(mddev,bi))
5130
		return;
5131

S
Shaohua Li 已提交
5132 5133 5134 5135 5136
	if (unlikely(bi->bi_rw & REQ_DISCARD)) {
		make_discard_request(mddev, bi);
		return;
	}

5137
	logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1);
K
Kent Overstreet 已提交
5138
	last_sector = bio_end_sector(bi);
L
Linus Torvalds 已提交
5139 5140
	bi->bi_next = NULL;
	bi->bi_phys_segments = 1;	/* over-loaded to count active stripes */
5141

5142
	prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE);
L
Linus Torvalds 已提交
5143
	for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) {
5144
		int previous;
5145
		int seq;
5146

5147
		do_prepare = false;
5148
	retry:
5149
		seq = read_seqcount_begin(&conf->gen_lock);
5150
		previous = 0;
5151 5152 5153
		if (do_prepare)
			prepare_to_wait(&conf->wait_for_overlap, &w,
				TASK_UNINTERRUPTIBLE);
5154
		if (unlikely(conf->reshape_progress != MaxSector)) {
5155
			/* spinlock is needed as reshape_progress may be
5156 5157
			 * 64bit on a 32bit platform, and so it might be
			 * possible to see a half-updated value
5158
			 * Of course reshape_progress could change after
5159 5160 5161 5162
			 * the lock is dropped, so once we get a reference
			 * to the stripe that we think it is, we will have
			 * to check again.
			 */
5163
			spin_lock_irq(&conf->device_lock);
5164
			if (mddev->reshape_backwards
5165 5166
			    ? logical_sector < conf->reshape_progress
			    : logical_sector >= conf->reshape_progress) {
5167 5168
				previous = 1;
			} else {
5169
				if (mddev->reshape_backwards
5170 5171
				    ? logical_sector < conf->reshape_safe
				    : logical_sector >= conf->reshape_safe) {
5172 5173
					spin_unlock_irq(&conf->device_lock);
					schedule();
5174
					do_prepare = true;
5175 5176 5177
					goto retry;
				}
			}
5178 5179
			spin_unlock_irq(&conf->device_lock);
		}
5180

5181 5182
		new_sector = raid5_compute_sector(conf, logical_sector,
						  previous,
5183
						  &dd_idx, NULL);
5184
		pr_debug("raid456: make_request, sector %llu logical %llu\n",
5185
			(unsigned long long)new_sector,
L
Linus Torvalds 已提交
5186 5187
			(unsigned long long)logical_sector);

5188
		sh = get_active_stripe(conf, new_sector, previous,
5189
				       (bi->bi_rw&RWA_MASK), 0);
L
Linus Torvalds 已提交
5190
		if (sh) {
5191
			if (unlikely(previous)) {
5192
				/* expansion might have moved on while waiting for a
5193 5194 5195 5196 5197 5198
				 * stripe, so we must do the range check again.
				 * Expansion could still move past after this
				 * test, but as we are holding a reference to
				 * 'sh', we know that if that happens,
				 *  STRIPE_EXPANDING will get set and the expansion
				 * won't proceed until we finish with the stripe.
5199 5200 5201
				 */
				int must_retry = 0;
				spin_lock_irq(&conf->device_lock);
5202
				if (mddev->reshape_backwards
5203 5204
				    ? logical_sector >= conf->reshape_progress
				    : logical_sector < conf->reshape_progress)
5205 5206 5207 5208 5209
					/* mismatch, need to try again */
					must_retry = 1;
				spin_unlock_irq(&conf->device_lock);
				if (must_retry) {
					release_stripe(sh);
5210
					schedule();
5211
					do_prepare = true;
5212 5213 5214
					goto retry;
				}
			}
5215 5216 5217 5218 5219 5220 5221
			if (read_seqcount_retry(&conf->gen_lock, seq)) {
				/* Might have got the wrong stripe_head
				 * by accident
				 */
				release_stripe(sh);
				goto retry;
			}
5222

5223
			if (rw == WRITE &&
5224
			    logical_sector >= mddev->suspend_lo &&
5225 5226
			    logical_sector < mddev->suspend_hi) {
				release_stripe(sh);
5227 5228 5229 5230 5231 5232 5233 5234
				/* As the suspend_* range is controlled by
				 * userspace, we want an interruptible
				 * wait.
				 */
				flush_signals(current);
				prepare_to_wait(&conf->wait_for_overlap,
						&w, TASK_INTERRUPTIBLE);
				if (logical_sector >= mddev->suspend_lo &&
5235
				    logical_sector < mddev->suspend_hi) {
5236
					schedule();
5237 5238
					do_prepare = true;
				}
5239 5240
				goto retry;
			}
5241 5242

			if (test_bit(STRIPE_EXPANDING, &sh->state) ||
5243
			    !add_stripe_bio(sh, bi, dd_idx, rw, previous)) {
5244 5245
				/* Stripe is busy expanding or
				 * add failed due to overlap.  Flush everything
L
Linus Torvalds 已提交
5246 5247
				 * and wait a while
				 */
N
NeilBrown 已提交
5248
				md_wakeup_thread(mddev->thread);
L
Linus Torvalds 已提交
5249 5250
				release_stripe(sh);
				schedule();
5251
				do_prepare = true;
L
Linus Torvalds 已提交
5252 5253
				goto retry;
			}
5254 5255
			set_bit(STRIPE_HANDLE, &sh->state);
			clear_bit(STRIPE_DELAYED, &sh->state);
5256 5257
			if ((!sh->batch_head || sh == sh->batch_head) &&
			    (bi->bi_rw & REQ_SYNC) &&
5258 5259
			    !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
				atomic_inc(&conf->preread_active_stripes);
5260
			release_stripe_plug(mddev, sh);
L
Linus Torvalds 已提交
5261 5262 5263 5264 5265 5266
		} else {
			/* cannot get stripe for read-ahead, just give-up */
			clear_bit(BIO_UPTODATE, &bi->bi_flags);
			break;
		}
	}
5267
	finish_wait(&conf->wait_for_overlap, &w);
5268

5269
	remaining = raid5_dec_bi_active_stripes(bi);
5270
	if (remaining == 0) {
L
Linus Torvalds 已提交
5271

5272
		if ( rw == WRITE )
L
Linus Torvalds 已提交
5273
			md_write_end(mddev);
5274

5275 5276
		trace_block_bio_complete(bdev_get_queue(bi->bi_bdev),
					 bi, 0);
5277
		bio_endio(bi, 0);
L
Linus Torvalds 已提交
5278 5279 5280
	}
}

5281
static sector_t raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks);
D
Dan Williams 已提交
5282

5283
static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *skipped)
L
Linus Torvalds 已提交
5284
{
5285 5286 5287 5288 5289 5290 5291 5292 5293
	/* reshaping is quite different to recovery/resync so it is
	 * handled quite separately ... here.
	 *
	 * On each call to sync_request, we gather one chunk worth of
	 * destination stripes and flag them as expanding.
	 * Then we find all the source stripes and request reads.
	 * As the reads complete, handle_stripe will copy the data
	 * into the destination stripe and release that stripe.
	 */
5294
	struct r5conf *conf = mddev->private;
L
Linus Torvalds 已提交
5295
	struct stripe_head *sh;
5296
	sector_t first_sector, last_sector;
5297 5298 5299
	int raid_disks = conf->previous_raid_disks;
	int data_disks = raid_disks - conf->max_degraded;
	int new_data_disks = conf->raid_disks - conf->max_degraded;
5300 5301
	int i;
	int dd_idx;
5302
	sector_t writepos, readpos, safepos;
5303
	sector_t stripe_addr;
5304
	int reshape_sectors;
5305
	struct list_head stripes;
5306

5307 5308
	if (sector_nr == 0) {
		/* If restarting in the middle, skip the initial sectors */
5309
		if (mddev->reshape_backwards &&
5310 5311 5312
		    conf->reshape_progress < raid5_size(mddev, 0, 0)) {
			sector_nr = raid5_size(mddev, 0, 0)
				- conf->reshape_progress;
5313
		} else if (!mddev->reshape_backwards &&
5314 5315
			   conf->reshape_progress > 0)
			sector_nr = conf->reshape_progress;
5316
		sector_div(sector_nr, new_data_disks);
5317
		if (sector_nr) {
5318 5319
			mddev->curr_resync_completed = sector_nr;
			sysfs_notify(&mddev->kobj, NULL, "sync_completed");
5320 5321 5322
			*skipped = 1;
			return sector_nr;
		}
5323 5324
	}

5325 5326 5327 5328
	/* We need to process a full chunk at a time.
	 * If old and new chunk sizes differ, we need to process the
	 * largest of these
	 */
5329 5330
	if (mddev->new_chunk_sectors > mddev->chunk_sectors)
		reshape_sectors = mddev->new_chunk_sectors;
5331
	else
5332
		reshape_sectors = mddev->chunk_sectors;
5333

5334 5335 5336 5337 5338
	/* We update the metadata at least every 10 seconds, or when
	 * the data about to be copied would over-write the source of
	 * the data at the front of the range.  i.e. one new_stripe
	 * along from reshape_progress new_maps to after where
	 * reshape_safe old_maps to
5339
	 */
5340
	writepos = conf->reshape_progress;
5341
	sector_div(writepos, new_data_disks);
5342 5343
	readpos = conf->reshape_progress;
	sector_div(readpos, data_disks);
5344
	safepos = conf->reshape_safe;
5345
	sector_div(safepos, data_disks);
5346
	if (mddev->reshape_backwards) {
5347
		writepos -= min_t(sector_t, reshape_sectors, writepos);
5348
		readpos += reshape_sectors;
5349
		safepos += reshape_sectors;
5350
	} else {
5351
		writepos += reshape_sectors;
5352 5353
		readpos -= min_t(sector_t, reshape_sectors, readpos);
		safepos -= min_t(sector_t, reshape_sectors, safepos);
5354
	}
5355

5356 5357 5358 5359 5360 5361 5362 5363 5364 5365 5366 5367 5368 5369 5370
	/* Having calculated the 'writepos' possibly use it
	 * to set 'stripe_addr' which is where we will write to.
	 */
	if (mddev->reshape_backwards) {
		BUG_ON(conf->reshape_progress == 0);
		stripe_addr = writepos;
		BUG_ON((mddev->dev_sectors &
			~((sector_t)reshape_sectors - 1))
		       - reshape_sectors - stripe_addr
		       != sector_nr);
	} else {
		BUG_ON(writepos != sector_nr + reshape_sectors);
		stripe_addr = sector_nr;
	}

5371 5372 5373 5374
	/* 'writepos' is the most advanced device address we might write.
	 * 'readpos' is the least advanced device address we might read.
	 * 'safepos' is the least address recorded in the metadata as having
	 *     been reshaped.
5375 5376 5377 5378
	 * If there is a min_offset_diff, these are adjusted either by
	 * increasing the safepos/readpos if diff is negative, or
	 * increasing writepos if diff is positive.
	 * If 'readpos' is then behind 'writepos', there is no way that we can
5379 5380 5381 5382 5383 5384 5385 5386 5387 5388 5389 5390
	 * ensure safety in the face of a crash - that must be done by userspace
	 * making a backup of the data.  So in that case there is no particular
	 * rush to update metadata.
	 * Otherwise if 'safepos' is behind 'writepos', then we really need to
	 * update the metadata to advance 'safepos' to match 'readpos' so that
	 * we can be safe in the event of a crash.
	 * So we insist on updating metadata if safepos is behind writepos and
	 * readpos is beyond writepos.
	 * In any case, update the metadata every 10 seconds.
	 * Maybe that number should be configurable, but I'm not sure it is
	 * worth it.... maybe it could be a multiple of safemode_delay???
	 */
5391 5392 5393 5394 5395 5396
	if (conf->min_offset_diff < 0) {
		safepos += -conf->min_offset_diff;
		readpos += -conf->min_offset_diff;
	} else
		writepos += conf->min_offset_diff;

5397
	if ((mddev->reshape_backwards
5398 5399 5400
	     ? (safepos > writepos && readpos < writepos)
	     : (safepos < writepos && readpos > writepos)) ||
	    time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
5401 5402
		/* Cannot proceed until we've updated the superblock... */
		wait_event(conf->wait_for_overlap,
5403 5404 5405 5406
			   atomic_read(&conf->reshape_stripes)==0
			   || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
		if (atomic_read(&conf->reshape_stripes) != 0)
			return 0;
5407
		mddev->reshape_position = conf->reshape_progress;
5408
		mddev->curr_resync_completed = sector_nr;
5409
		conf->reshape_checkpoint = jiffies;
5410
		set_bit(MD_CHANGE_DEVS, &mddev->flags);
5411
		md_wakeup_thread(mddev->thread);
5412
		wait_event(mddev->sb_wait, mddev->flags == 0 ||
5413 5414 5415
			   test_bit(MD_RECOVERY_INTR, &mddev->recovery));
		if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
			return 0;
5416
		spin_lock_irq(&conf->device_lock);
5417
		conf->reshape_safe = mddev->reshape_position;
5418 5419
		spin_unlock_irq(&conf->device_lock);
		wake_up(&conf->wait_for_overlap);
5420
		sysfs_notify(&mddev->kobj, NULL, "sync_completed");
5421 5422
	}

5423
	INIT_LIST_HEAD(&stripes);
5424
	for (i = 0; i < reshape_sectors; i += STRIPE_SECTORS) {
5425
		int j;
5426
		int skipped_disk = 0;
5427
		sh = get_active_stripe(conf, stripe_addr+i, 0, 0, 1);
5428 5429 5430 5431 5432 5433 5434 5435 5436
		set_bit(STRIPE_EXPANDING, &sh->state);
		atomic_inc(&conf->reshape_stripes);
		/* If any of this stripe is beyond the end of the old
		 * array, then we need to zero those blocks
		 */
		for (j=sh->disks; j--;) {
			sector_t s;
			if (j == sh->pd_idx)
				continue;
5437
			if (conf->level == 6 &&
5438
			    j == sh->qd_idx)
5439
				continue;
5440
			s = compute_blocknr(sh, j, 0);
D
Dan Williams 已提交
5441
			if (s < raid5_size(mddev, 0, 0)) {
5442
				skipped_disk = 1;
5443 5444 5445 5446 5447 5448
				continue;
			}
			memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE);
			set_bit(R5_Expanded, &sh->dev[j].flags);
			set_bit(R5_UPTODATE, &sh->dev[j].flags);
		}
5449
		if (!skipped_disk) {
5450 5451 5452
			set_bit(STRIPE_EXPAND_READY, &sh->state);
			set_bit(STRIPE_HANDLE, &sh->state);
		}
5453
		list_add(&sh->lru, &stripes);
5454 5455
	}
	spin_lock_irq(&conf->device_lock);
5456
	if (mddev->reshape_backwards)
5457
		conf->reshape_progress -= reshape_sectors * new_data_disks;
5458
	else
5459
		conf->reshape_progress += reshape_sectors * new_data_disks;
5460 5461 5462 5463 5464 5465 5466
	spin_unlock_irq(&conf->device_lock);
	/* Ok, those stripe are ready. We can start scheduling
	 * reads on the source stripes.
	 * The source stripes are determined by mapping the first and last
	 * block on the destination stripes.
	 */
	first_sector =
5467
		raid5_compute_sector(conf, stripe_addr*(new_data_disks),
5468
				     1, &dd_idx, NULL);
5469
	last_sector =
5470
		raid5_compute_sector(conf, ((stripe_addr+reshape_sectors)
5471
					    * new_data_disks - 1),
5472
				     1, &dd_idx, NULL);
A
Andre Noll 已提交
5473 5474
	if (last_sector >= mddev->dev_sectors)
		last_sector = mddev->dev_sectors - 1;
5475
	while (first_sector <= last_sector) {
5476
		sh = get_active_stripe(conf, first_sector, 1, 0, 1);
5477 5478 5479 5480 5481
		set_bit(STRIPE_EXPAND_SOURCE, &sh->state);
		set_bit(STRIPE_HANDLE, &sh->state);
		release_stripe(sh);
		first_sector += STRIPE_SECTORS;
	}
5482 5483 5484 5485 5486 5487 5488 5489
	/* Now that the sources are clearly marked, we can release
	 * the destination stripes
	 */
	while (!list_empty(&stripes)) {
		sh = list_entry(stripes.next, struct stripe_head, lru);
		list_del_init(&sh->lru);
		release_stripe(sh);
	}
5490 5491 5492
	/* If this takes us to the resync_max point where we have to pause,
	 * then we need to write out the superblock.
	 */
5493
	sector_nr += reshape_sectors;
5494 5495
	if ((sector_nr - mddev->curr_resync_completed) * 2
	    >= mddev->resync_max - mddev->curr_resync_completed) {
5496 5497
		/* Cannot proceed until we've updated the superblock... */
		wait_event(conf->wait_for_overlap,
5498 5499 5500 5501
			   atomic_read(&conf->reshape_stripes) == 0
			   || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
		if (atomic_read(&conf->reshape_stripes) != 0)
			goto ret;
5502
		mddev->reshape_position = conf->reshape_progress;
5503
		mddev->curr_resync_completed = sector_nr;
5504
		conf->reshape_checkpoint = jiffies;
5505 5506 5507 5508
		set_bit(MD_CHANGE_DEVS, &mddev->flags);
		md_wakeup_thread(mddev->thread);
		wait_event(mddev->sb_wait,
			   !test_bit(MD_CHANGE_DEVS, &mddev->flags)
5509 5510 5511
			   || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
		if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
			goto ret;
5512
		spin_lock_irq(&conf->device_lock);
5513
		conf->reshape_safe = mddev->reshape_position;
5514 5515
		spin_unlock_irq(&conf->device_lock);
		wake_up(&conf->wait_for_overlap);
5516
		sysfs_notify(&mddev->kobj, NULL, "sync_completed");
5517
	}
5518
ret:
5519
	return reshape_sectors;
5520 5521
}

5522
static inline sector_t sync_request(struct mddev *mddev, sector_t sector_nr, int *skipped)
5523
{
5524
	struct r5conf *conf = mddev->private;
5525
	struct stripe_head *sh;
A
Andre Noll 已提交
5526
	sector_t max_sector = mddev->dev_sectors;
N
NeilBrown 已提交
5527
	sector_t sync_blocks;
5528 5529
	int still_degraded = 0;
	int i;
L
Linus Torvalds 已提交
5530

5531
	if (sector_nr >= max_sector) {
L
Linus Torvalds 已提交
5532
		/* just being told to finish up .. nothing much to do */
5533

5534 5535 5536 5537
		if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {
			end_reshape(conf);
			return 0;
		}
5538 5539 5540 5541

		if (mddev->curr_resync < max_sector) /* aborted */
			bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
					&sync_blocks, 1);
5542
		else /* completed sync */
5543 5544 5545
			conf->fullsync = 0;
		bitmap_close_sync(mddev->bitmap);

L
Linus Torvalds 已提交
5546 5547
		return 0;
	}
5548

5549 5550 5551
	/* Allow raid5_quiesce to complete */
	wait_event(conf->wait_for_overlap, conf->quiesce != 2);

5552 5553
	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
		return reshape_request(mddev, sector_nr, skipped);
5554

5555 5556 5557 5558 5559 5560
	/* No need to check resync_max as we never do more than one
	 * stripe, and as resync_max will always be on a chunk boundary,
	 * if the check in md_do_sync didn't fire, there is no chance
	 * of overstepping resync_max here
	 */

5561
	/* if there is too many failed drives and we are trying
L
Linus Torvalds 已提交
5562 5563 5564
	 * to resync, then assert that we are finished, because there is
	 * nothing we can do.
	 */
5565
	if (mddev->degraded >= conf->max_degraded &&
5566
	    test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
A
Andre Noll 已提交
5567
		sector_t rv = mddev->dev_sectors - sector_nr;
5568
		*skipped = 1;
L
Linus Torvalds 已提交
5569 5570
		return rv;
	}
5571 5572 5573 5574
	if (!test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
	    !conf->fullsync &&
	    !bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) &&
	    sync_blocks >= STRIPE_SECTORS) {
5575 5576 5577 5578 5579
		/* we can skip this block, and probably more */
		sync_blocks /= STRIPE_SECTORS;
		*skipped = 1;
		return sync_blocks * STRIPE_SECTORS; /* keep things rounded to whole stripes */
	}
L
Linus Torvalds 已提交
5580

N
NeilBrown 已提交
5581 5582
	bitmap_cond_end_sync(mddev->bitmap, sector_nr);

5583
	sh = get_active_stripe(conf, sector_nr, 0, 1, 0);
L
Linus Torvalds 已提交
5584
	if (sh == NULL) {
5585
		sh = get_active_stripe(conf, sector_nr, 0, 0, 0);
L
Linus Torvalds 已提交
5586
		/* make sure we don't swamp the stripe cache if someone else
5587
		 * is trying to get access
L
Linus Torvalds 已提交
5588
		 */
5589
		schedule_timeout_uninterruptible(1);
L
Linus Torvalds 已提交
5590
	}
5591
	/* Need to check if array will still be degraded after recovery/resync
5592 5593
	 * Note in case of > 1 drive failures it's possible we're rebuilding
	 * one drive while leaving another faulty drive in array.
5594
	 */
5595 5596 5597 5598 5599
	rcu_read_lock();
	for (i = 0; i < conf->raid_disks; i++) {
		struct md_rdev *rdev = ACCESS_ONCE(conf->disks[i].rdev);

		if (rdev == NULL || test_bit(Faulty, &rdev->flags))
5600
			still_degraded = 1;
5601 5602
	}
	rcu_read_unlock();
5603 5604 5605

	bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, still_degraded);

5606
	set_bit(STRIPE_SYNC_REQUESTED, &sh->state);
5607
	set_bit(STRIPE_HANDLE, &sh->state);
L
Linus Torvalds 已提交
5608 5609 5610 5611 5612 5613

	release_stripe(sh);

	return STRIPE_SECTORS;
}

5614
static int  retry_aligned_read(struct r5conf *conf, struct bio *raid_bio)
5615 5616 5617 5618 5619 5620 5621 5622 5623 5624 5625 5626
{
	/* We may not be able to submit a whole bio at once as there
	 * may not be enough stripe_heads available.
	 * We cannot pre-allocate enough stripe_heads as we may need
	 * more than exist in the cache (if we allow ever large chunks).
	 * So we do one stripe head at a time and record in
	 * ->bi_hw_segments how many have been done.
	 *
	 * We *know* that this entire raid_bio is in one chunk, so
	 * it will be only one 'dd_idx' and only need one call to raid5_compute_sector.
	 */
	struct stripe_head *sh;
5627
	int dd_idx;
5628 5629 5630 5631 5632
	sector_t sector, logical_sector, last_sector;
	int scnt = 0;
	int remaining;
	int handled = 0;

5633 5634
	logical_sector = raid_bio->bi_iter.bi_sector &
		~((sector_t)STRIPE_SECTORS-1);
5635
	sector = raid5_compute_sector(conf, logical_sector,
5636
				      0, &dd_idx, NULL);
K
Kent Overstreet 已提交
5637
	last_sector = bio_end_sector(raid_bio);
5638 5639

	for (; logical_sector < last_sector;
5640 5641 5642
	     logical_sector += STRIPE_SECTORS,
		     sector += STRIPE_SECTORS,
		     scnt++) {
5643

5644
		if (scnt < raid5_bi_processed_stripes(raid_bio))
5645 5646 5647
			/* already done this stripe */
			continue;

5648
		sh = get_active_stripe(conf, sector, 0, 1, 1);
5649 5650 5651

		if (!sh) {
			/* failed to get a stripe - must wait */
5652
			raid5_set_bi_processed_stripes(raid_bio, scnt);
5653 5654 5655 5656
			conf->retry_read_aligned = raid_bio;
			return handled;
		}

5657
		if (!add_stripe_bio(sh, raid_bio, dd_idx, 0, 0)) {
5658
			release_stripe(sh);
5659
			raid5_set_bi_processed_stripes(raid_bio, scnt);
5660 5661 5662 5663
			conf->retry_read_aligned = raid_bio;
			return handled;
		}

5664
		set_bit(R5_ReadNoMerge, &sh->dev[dd_idx].flags);
5665
		handle_stripe(sh);
5666 5667 5668
		release_stripe(sh);
		handled++;
	}
5669
	remaining = raid5_dec_bi_active_stripes(raid_bio);
5670 5671 5672
	if (remaining == 0) {
		trace_block_bio_complete(bdev_get_queue(raid_bio->bi_bdev),
					 raid_bio, 0);
5673
		bio_endio(raid_bio, 0);
5674
	}
5675 5676 5677 5678 5679
	if (atomic_dec_and_test(&conf->active_aligned_reads))
		wake_up(&conf->wait_for_stripe);
	return handled;
}

5680
static int handle_active_stripes(struct r5conf *conf, int group,
5681 5682
				 struct r5worker *worker,
				 struct list_head *temp_inactive_list)
5683 5684
{
	struct stripe_head *batch[MAX_STRIPE_BATCH], *sh;
5685 5686
	int i, batch_size = 0, hash;
	bool release_inactive = false;
5687 5688

	while (batch_size < MAX_STRIPE_BATCH &&
5689
			(sh = __get_priority_stripe(conf, group)) != NULL)
5690 5691
		batch[batch_size++] = sh;

5692 5693 5694 5695 5696 5697 5698 5699
	if (batch_size == 0) {
		for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
			if (!list_empty(temp_inactive_list + i))
				break;
		if (i == NR_STRIPE_HASH_LOCKS)
			return batch_size;
		release_inactive = true;
	}
5700 5701
	spin_unlock_irq(&conf->device_lock);

5702 5703 5704 5705 5706 5707 5708 5709
	release_inactive_stripe_list(conf, temp_inactive_list,
				     NR_STRIPE_HASH_LOCKS);

	if (release_inactive) {
		spin_lock_irq(&conf->device_lock);
		return 0;
	}

5710 5711 5712 5713 5714 5715
	for (i = 0; i < batch_size; i++)
		handle_stripe(batch[i]);

	cond_resched();

	spin_lock_irq(&conf->device_lock);
5716 5717 5718 5719
	for (i = 0; i < batch_size; i++) {
		hash = batch[i]->hash_lock_index;
		__release_stripe(conf, batch[i], &temp_inactive_list[hash]);
	}
5720 5721
	return batch_size;
}
5722

5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 5735 5736 5737 5738 5739
static void raid5_do_work(struct work_struct *work)
{
	struct r5worker *worker = container_of(work, struct r5worker, work);
	struct r5worker_group *group = worker->group;
	struct r5conf *conf = group->conf;
	int group_id = group - conf->worker_groups;
	int handled;
	struct blk_plug plug;

	pr_debug("+++ raid5worker active\n");

	blk_start_plug(&plug);
	handled = 0;
	spin_lock_irq(&conf->device_lock);
	while (1) {
		int batch_size, released;

5740
		released = release_stripe_list(conf, worker->temp_inactive_list);
5741

5742 5743
		batch_size = handle_active_stripes(conf, group_id, worker,
						   worker->temp_inactive_list);
5744
		worker->working = false;
5745 5746 5747 5748 5749 5750 5751 5752 5753 5754 5755 5756
		if (!batch_size && !released)
			break;
		handled += batch_size;
	}
	pr_debug("%d stripes handled\n", handled);

	spin_unlock_irq(&conf->device_lock);
	blk_finish_plug(&plug);

	pr_debug("--- raid5worker inactive\n");
}

L
Linus Torvalds 已提交
5757 5758 5759 5760 5761 5762 5763
/*
 * This is our raid5 kernel thread.
 *
 * We scan the hash table for stripes which can be handled now.
 * During the scan, completed stripes are saved for us by the interrupt
 * handler, so that they will not have to wait for our next wakeup.
 */
S
Shaohua Li 已提交
5764
static void raid5d(struct md_thread *thread)
L
Linus Torvalds 已提交
5765
{
S
Shaohua Li 已提交
5766
	struct mddev *mddev = thread->mddev;
5767
	struct r5conf *conf = mddev->private;
L
Linus Torvalds 已提交
5768
	int handled;
5769
	struct blk_plug plug;
L
Linus Torvalds 已提交
5770

5771
	pr_debug("+++ raid5d active\n");
L
Linus Torvalds 已提交
5772 5773 5774

	md_check_recovery(mddev);

5775
	blk_start_plug(&plug);
L
Linus Torvalds 已提交
5776 5777 5778
	handled = 0;
	spin_lock_irq(&conf->device_lock);
	while (1) {
5779
		struct bio *bio;
S
Shaohua Li 已提交
5780 5781
		int batch_size, released;

5782
		released = release_stripe_list(conf, conf->temp_inactive_list);
5783 5784
		if (released)
			clear_bit(R5_DID_ALLOC, &conf->cache_state);
L
Linus Torvalds 已提交
5785

5786
		if (
5787 5788 5789
		    !list_empty(&conf->bitmap_list)) {
			/* Now is a good time to flush some bitmap updates */
			conf->seq_flush++;
5790
			spin_unlock_irq(&conf->device_lock);
5791
			bitmap_unplug(mddev->bitmap);
5792
			spin_lock_irq(&conf->device_lock);
5793
			conf->seq_write = conf->seq_flush;
5794
			activate_bit_delay(conf, conf->temp_inactive_list);
5795
		}
5796
		raid5_activate_delayed(conf);
5797

5798 5799 5800 5801 5802 5803 5804 5805 5806 5807
		while ((bio = remove_bio_from_retry(conf))) {
			int ok;
			spin_unlock_irq(&conf->device_lock);
			ok = retry_aligned_read(conf, bio);
			spin_lock_irq(&conf->device_lock);
			if (!ok)
				break;
			handled++;
		}

5808 5809
		batch_size = handle_active_stripes(conf, ANY_GROUP, NULL,
						   conf->temp_inactive_list);
S
Shaohua Li 已提交
5810
		if (!batch_size && !released)
L
Linus Torvalds 已提交
5811
			break;
5812
		handled += batch_size;
L
Linus Torvalds 已提交
5813

5814 5815
		if (mddev->flags & ~(1<<MD_CHANGE_PENDING)) {
			spin_unlock_irq(&conf->device_lock);
5816
			md_check_recovery(mddev);
5817 5818
			spin_lock_irq(&conf->device_lock);
		}
L
Linus Torvalds 已提交
5819
	}
5820
	pr_debug("%d stripes handled\n", handled);
L
Linus Torvalds 已提交
5821 5822

	spin_unlock_irq(&conf->device_lock);
5823 5824 5825 5826 5827 5828 5829
	if (test_and_clear_bit(R5_ALLOC_MORE, &conf->cache_state)) {
		grow_one_stripe(conf, __GFP_NOWARN);
		/* Set flag even if allocation failed.  This helps
		 * slow down allocation requests when mem is short
		 */
		set_bit(R5_DID_ALLOC, &conf->cache_state);
	}
L
Linus Torvalds 已提交
5830

5831
	async_tx_issue_pending_all();
5832
	blk_finish_plug(&plug);
L
Linus Torvalds 已提交
5833

5834
	pr_debug("--- raid5d inactive\n");
L
Linus Torvalds 已提交
5835 5836
}

5837
static ssize_t
5838
raid5_show_stripe_cache_size(struct mddev *mddev, char *page)
5839
{
5840 5841 5842 5843
	struct r5conf *conf;
	int ret = 0;
	spin_lock(&mddev->lock);
	conf = mddev->private;
5844
	if (conf)
5845
		ret = sprintf(page, "%d\n", conf->min_nr_stripes);
5846 5847
	spin_unlock(&mddev->lock);
	return ret;
5848 5849
}

5850
int
5851
raid5_set_cache_size(struct mddev *mddev, int size)
5852
{
5853
	struct r5conf *conf = mddev->private;
5854 5855
	int err;

5856
	if (size <= 16 || size > 32768)
5857
		return -EINVAL;
5858

5859
	conf->min_nr_stripes = size;
5860 5861 5862 5863
	while (size < conf->max_nr_stripes &&
	       drop_one_stripe(conf))
		;

5864

5865 5866 5867
	err = md_allow_write(mddev);
	if (err)
		return err;
5868 5869 5870 5871 5872

	while (size > conf->max_nr_stripes)
		if (!grow_one_stripe(conf, GFP_KERNEL))
			break;

5873 5874 5875 5876 5877
	return 0;
}
EXPORT_SYMBOL(raid5_set_cache_size);

static ssize_t
5878
raid5_store_stripe_cache_size(struct mddev *mddev, const char *page, size_t len)
5879
{
5880
	struct r5conf *conf;
5881 5882 5883 5884 5885
	unsigned long new;
	int err;

	if (len >= PAGE_SIZE)
		return -EINVAL;
5886
	if (kstrtoul(page, 10, &new))
5887
		return -EINVAL;
5888
	err = mddev_lock(mddev);
5889 5890
	if (err)
		return err;
5891 5892 5893 5894 5895 5896 5897 5898
	conf = mddev->private;
	if (!conf)
		err = -ENODEV;
	else
		err = raid5_set_cache_size(mddev, new);
	mddev_unlock(mddev);

	return err ?: len;
5899
}
5900

5901 5902 5903 5904
static struct md_sysfs_entry
raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR,
				raid5_show_stripe_cache_size,
				raid5_store_stripe_cache_size);
5905

5906 5907 5908 5909 5910 5911 5912 5913 5914 5915 5916 5917 5918 5919 5920 5921 5922 5923 5924 5925 5926 5927 5928 5929 5930 5931 5932 5933 5934 5935 5936 5937 5938 5939 5940 5941 5942 5943 5944 5945 5946 5947 5948
static ssize_t
raid5_show_rmw_level(struct mddev  *mddev, char *page)
{
	struct r5conf *conf = mddev->private;
	if (conf)
		return sprintf(page, "%d\n", conf->rmw_level);
	else
		return 0;
}

static ssize_t
raid5_store_rmw_level(struct mddev  *mddev, const char *page, size_t len)
{
	struct r5conf *conf = mddev->private;
	unsigned long new;

	if (!conf)
		return -ENODEV;

	if (len >= PAGE_SIZE)
		return -EINVAL;

	if (kstrtoul(page, 10, &new))
		return -EINVAL;

	if (new != PARITY_DISABLE_RMW && !raid6_call.xor_syndrome)
		return -EINVAL;

	if (new != PARITY_DISABLE_RMW &&
	    new != PARITY_ENABLE_RMW &&
	    new != PARITY_PREFER_RMW)
		return -EINVAL;

	conf->rmw_level = new;
	return len;
}

static struct md_sysfs_entry
raid5_rmw_level = __ATTR(rmw_level, S_IRUGO | S_IWUSR,
			 raid5_show_rmw_level,
			 raid5_store_rmw_level);


5949
static ssize_t
5950
raid5_show_preread_threshold(struct mddev *mddev, char *page)
5951
{
5952 5953 5954 5955
	struct r5conf *conf;
	int ret = 0;
	spin_lock(&mddev->lock);
	conf = mddev->private;
5956
	if (conf)
5957 5958 5959
		ret = sprintf(page, "%d\n", conf->bypass_threshold);
	spin_unlock(&mddev->lock);
	return ret;
5960 5961 5962
}

static ssize_t
5963
raid5_store_preread_threshold(struct mddev *mddev, const char *page, size_t len)
5964
{
5965
	struct r5conf *conf;
5966
	unsigned long new;
5967 5968
	int err;

5969 5970
	if (len >= PAGE_SIZE)
		return -EINVAL;
5971
	if (kstrtoul(page, 10, &new))
5972
		return -EINVAL;
5973 5974 5975 5976 5977 5978 5979

	err = mddev_lock(mddev);
	if (err)
		return err;
	conf = mddev->private;
	if (!conf)
		err = -ENODEV;
5980
	else if (new > conf->min_nr_stripes)
5981 5982 5983 5984 5985
		err = -EINVAL;
	else
		conf->bypass_threshold = new;
	mddev_unlock(mddev);
	return err ?: len;
5986 5987 5988 5989 5990 5991 5992 5993
}

static struct md_sysfs_entry
raid5_preread_bypass_threshold = __ATTR(preread_bypass_threshold,
					S_IRUGO | S_IWUSR,
					raid5_show_preread_threshold,
					raid5_store_preread_threshold);

5994 5995 5996
static ssize_t
raid5_show_skip_copy(struct mddev *mddev, char *page)
{
5997 5998 5999 6000
	struct r5conf *conf;
	int ret = 0;
	spin_lock(&mddev->lock);
	conf = mddev->private;
6001
	if (conf)
6002 6003 6004
		ret = sprintf(page, "%d\n", conf->skip_copy);
	spin_unlock(&mddev->lock);
	return ret;
6005 6006 6007 6008 6009
}

static ssize_t
raid5_store_skip_copy(struct mddev *mddev, const char *page, size_t len)
{
6010
	struct r5conf *conf;
6011
	unsigned long new;
6012 6013
	int err;

6014 6015 6016 6017 6018
	if (len >= PAGE_SIZE)
		return -EINVAL;
	if (kstrtoul(page, 10, &new))
		return -EINVAL;
	new = !!new;
6019 6020 6021 6022 6023 6024 6025 6026 6027 6028 6029 6030 6031 6032 6033 6034 6035 6036 6037 6038

	err = mddev_lock(mddev);
	if (err)
		return err;
	conf = mddev->private;
	if (!conf)
		err = -ENODEV;
	else if (new != conf->skip_copy) {
		mddev_suspend(mddev);
		conf->skip_copy = new;
		if (new)
			mddev->queue->backing_dev_info.capabilities |=
				BDI_CAP_STABLE_WRITES;
		else
			mddev->queue->backing_dev_info.capabilities &=
				~BDI_CAP_STABLE_WRITES;
		mddev_resume(mddev);
	}
	mddev_unlock(mddev);
	return err ?: len;
6039 6040 6041 6042 6043 6044 6045
}

static struct md_sysfs_entry
raid5_skip_copy = __ATTR(skip_copy, S_IRUGO | S_IWUSR,
					raid5_show_skip_copy,
					raid5_store_skip_copy);

6046
static ssize_t
6047
stripe_cache_active_show(struct mddev *mddev, char *page)
6048
{
6049
	struct r5conf *conf = mddev->private;
6050 6051 6052 6053
	if (conf)
		return sprintf(page, "%d\n", atomic_read(&conf->active_stripes));
	else
		return 0;
6054 6055
}

6056 6057
static struct md_sysfs_entry
raid5_stripecache_active = __ATTR_RO(stripe_cache_active);
6058

6059 6060 6061
static ssize_t
raid5_show_group_thread_cnt(struct mddev *mddev, char *page)
{
6062 6063 6064 6065
	struct r5conf *conf;
	int ret = 0;
	spin_lock(&mddev->lock);
	conf = mddev->private;
6066
	if (conf)
6067 6068 6069
		ret = sprintf(page, "%d\n", conf->worker_cnt_per_group);
	spin_unlock(&mddev->lock);
	return ret;
6070 6071
}

6072 6073 6074 6075
static int alloc_thread_groups(struct r5conf *conf, int cnt,
			       int *group_cnt,
			       int *worker_cnt_per_group,
			       struct r5worker_group **worker_groups);
6076 6077 6078
static ssize_t
raid5_store_group_thread_cnt(struct mddev *mddev, const char *page, size_t len)
{
6079
	struct r5conf *conf;
6080 6081
	unsigned long new;
	int err;
6082 6083
	struct r5worker_group *new_groups, *old_groups;
	int group_cnt, worker_cnt_per_group;
6084 6085 6086 6087 6088 6089

	if (len >= PAGE_SIZE)
		return -EINVAL;
	if (kstrtoul(page, 10, &new))
		return -EINVAL;

6090 6091 6092 6093 6094 6095 6096 6097
	err = mddev_lock(mddev);
	if (err)
		return err;
	conf = mddev->private;
	if (!conf)
		err = -ENODEV;
	else if (new != conf->worker_cnt_per_group) {
		mddev_suspend(mddev);
6098

6099 6100 6101
		old_groups = conf->worker_groups;
		if (old_groups)
			flush_workqueue(raid5_wq);
6102

6103 6104 6105 6106 6107 6108 6109 6110 6111
		err = alloc_thread_groups(conf, new,
					  &group_cnt, &worker_cnt_per_group,
					  &new_groups);
		if (!err) {
			spin_lock_irq(&conf->device_lock);
			conf->group_cnt = group_cnt;
			conf->worker_cnt_per_group = worker_cnt_per_group;
			conf->worker_groups = new_groups;
			spin_unlock_irq(&conf->device_lock);
6112

6113 6114 6115 6116 6117
			if (old_groups)
				kfree(old_groups[0].workers);
			kfree(old_groups);
		}
		mddev_resume(mddev);
6118
	}
6119
	mddev_unlock(mddev);
6120

6121
	return err ?: len;
6122 6123 6124 6125 6126 6127 6128
}

static struct md_sysfs_entry
raid5_group_thread_cnt = __ATTR(group_thread_cnt, S_IRUGO | S_IWUSR,
				raid5_show_group_thread_cnt,
				raid5_store_group_thread_cnt);

6129
static struct attribute *raid5_attrs[] =  {
6130 6131
	&raid5_stripecache_size.attr,
	&raid5_stripecache_active.attr,
6132
	&raid5_preread_bypass_threshold.attr,
6133
	&raid5_group_thread_cnt.attr,
6134
	&raid5_skip_copy.attr,
6135
	&raid5_rmw_level.attr,
6136 6137
	NULL,
};
6138 6139 6140
static struct attribute_group raid5_attrs_group = {
	.name = NULL,
	.attrs = raid5_attrs,
6141 6142
};

6143 6144 6145 6146
static int alloc_thread_groups(struct r5conf *conf, int cnt,
			       int *group_cnt,
			       int *worker_cnt_per_group,
			       struct r5worker_group **worker_groups)
6147
{
6148
	int i, j, k;
6149 6150 6151
	ssize_t size;
	struct r5worker *workers;

6152
	*worker_cnt_per_group = cnt;
6153
	if (cnt == 0) {
6154 6155
		*group_cnt = 0;
		*worker_groups = NULL;
6156 6157
		return 0;
	}
6158
	*group_cnt = num_possible_nodes();
6159
	size = sizeof(struct r5worker) * cnt;
6160 6161 6162 6163
	workers = kzalloc(size * *group_cnt, GFP_NOIO);
	*worker_groups = kzalloc(sizeof(struct r5worker_group) *
				*group_cnt, GFP_NOIO);
	if (!*worker_groups || !workers) {
6164
		kfree(workers);
6165
		kfree(*worker_groups);
6166 6167 6168
		return -ENOMEM;
	}

6169
	for (i = 0; i < *group_cnt; i++) {
6170 6171
		struct r5worker_group *group;

6172
		group = &(*worker_groups)[i];
6173 6174 6175 6176 6177
		INIT_LIST_HEAD(&group->handle_list);
		group->conf = conf;
		group->workers = workers + i * cnt;

		for (j = 0; j < cnt; j++) {
6178 6179 6180 6181 6182 6183
			struct r5worker *worker = group->workers + j;
			worker->group = group;
			INIT_WORK(&worker->work, raid5_do_work);

			for (k = 0; k < NR_STRIPE_HASH_LOCKS; k++)
				INIT_LIST_HEAD(worker->temp_inactive_list + k);
6184 6185 6186 6187 6188 6189 6190 6191 6192 6193 6194 6195 6196 6197
		}
	}

	return 0;
}

static void free_thread_groups(struct r5conf *conf)
{
	if (conf->worker_groups)
		kfree(conf->worker_groups[0].workers);
	kfree(conf->worker_groups);
	conf->worker_groups = NULL;
}

6198
static sector_t
6199
raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks)
6200
{
6201
	struct r5conf *conf = mddev->private;
6202 6203 6204

	if (!sectors)
		sectors = mddev->dev_sectors;
6205
	if (!raid_disks)
6206
		/* size is defined by the smallest of previous and new size */
6207
		raid_disks = min(conf->raid_disks, conf->previous_raid_disks);
6208

6209
	sectors &= ~((sector_t)mddev->chunk_sectors - 1);
6210
	sectors &= ~((sector_t)mddev->new_chunk_sectors - 1);
6211 6212 6213
	return sectors * (raid_disks - conf->max_degraded);
}

6214 6215 6216
static void free_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu)
{
	safe_put_page(percpu->spare_page);
6217 6218
	if (percpu->scribble)
		flex_array_free(percpu->scribble);
6219 6220 6221 6222 6223 6224 6225 6226 6227
	percpu->spare_page = NULL;
	percpu->scribble = NULL;
}

static int alloc_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu)
{
	if (conf->level == 6 && !percpu->spare_page)
		percpu->spare_page = alloc_page(GFP_KERNEL);
	if (!percpu->scribble)
6228
		percpu->scribble = scribble_alloc(max(conf->raid_disks,
6229 6230 6231 6232 6233
						      conf->previous_raid_disks),
						  max(conf->chunk_sectors,
						      conf->prev_chunk_sectors)
						   / STRIPE_SECTORS,
						  GFP_KERNEL);
6234 6235 6236 6237 6238 6239 6240 6241 6242

	if (!percpu->scribble || (conf->level == 6 && !percpu->spare_page)) {
		free_scratch_buffer(conf, percpu);
		return -ENOMEM;
	}

	return 0;
}

6243
static void raid5_free_percpu(struct r5conf *conf)
6244 6245 6246 6247 6248 6249 6250 6251 6252
{
	unsigned long cpu;

	if (!conf->percpu)
		return;

#ifdef CONFIG_HOTPLUG_CPU
	unregister_cpu_notifier(&conf->cpu_notify);
#endif
6253 6254 6255 6256

	get_online_cpus();
	for_each_possible_cpu(cpu)
		free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));
6257 6258 6259 6260 6261
	put_online_cpus();

	free_percpu(conf->percpu);
}

6262
static void free_conf(struct r5conf *conf)
6263
{
6264 6265
	if (conf->shrinker.seeks)
		unregister_shrinker(&conf->shrinker);
6266
	free_thread_groups(conf);
6267
	shrink_stripes(conf);
6268
	raid5_free_percpu(conf);
6269 6270 6271 6272 6273
	kfree(conf->disks);
	kfree(conf->stripe_hashtbl);
	kfree(conf);
}

6274 6275 6276 6277
#ifdef CONFIG_HOTPLUG_CPU
static int raid456_cpu_notify(struct notifier_block *nfb, unsigned long action,
			      void *hcpu)
{
6278
	struct r5conf *conf = container_of(nfb, struct r5conf, cpu_notify);
6279 6280 6281 6282 6283 6284
	long cpu = (long)hcpu;
	struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu);

	switch (action) {
	case CPU_UP_PREPARE:
	case CPU_UP_PREPARE_FROZEN:
6285
		if (alloc_scratch_buffer(conf, percpu)) {
6286 6287
			pr_err("%s: failed memory allocation for cpu%ld\n",
			       __func__, cpu);
6288
			return notifier_from_errno(-ENOMEM);
6289 6290 6291 6292
		}
		break;
	case CPU_DEAD:
	case CPU_DEAD_FROZEN:
6293
		free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));
6294 6295 6296 6297 6298 6299 6300 6301
		break;
	default:
		break;
	}
	return NOTIFY_OK;
}
#endif

6302
static int raid5_alloc_percpu(struct r5conf *conf)
6303 6304
{
	unsigned long cpu;
6305
	int err = 0;
6306

6307 6308
	conf->percpu = alloc_percpu(struct raid5_percpu);
	if (!conf->percpu)
6309
		return -ENOMEM;
6310 6311 6312 6313 6314 6315 6316 6317

#ifdef CONFIG_HOTPLUG_CPU
	conf->cpu_notify.notifier_call = raid456_cpu_notify;
	conf->cpu_notify.priority = 0;
	err = register_cpu_notifier(&conf->cpu_notify);
	if (err)
		return err;
#endif
6318 6319 6320

	get_online_cpus();
	for_each_present_cpu(cpu) {
6321 6322 6323 6324
		err = alloc_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));
		if (err) {
			pr_err("%s: failed memory allocation for cpu%ld\n",
			       __func__, cpu);
6325 6326 6327 6328 6329 6330 6331 6332
			break;
		}
	}
	put_online_cpus();

	return err;
}

6333 6334 6335 6336 6337 6338 6339 6340 6341 6342 6343 6344 6345 6346 6347 6348 6349 6350 6351 6352 6353 6354 6355 6356
static unsigned long raid5_cache_scan(struct shrinker *shrink,
				      struct shrink_control *sc)
{
	struct r5conf *conf = container_of(shrink, struct r5conf, shrinker);
	int ret = 0;
	while (ret < sc->nr_to_scan) {
		if (drop_one_stripe(conf) == 0)
			return SHRINK_STOP;
		ret++;
	}
	return ret;
}

static unsigned long raid5_cache_count(struct shrinker *shrink,
				       struct shrink_control *sc)
{
	struct r5conf *conf = container_of(shrink, struct r5conf, shrinker);

	if (conf->max_nr_stripes < conf->min_nr_stripes)
		/* unlikely, but not impossible */
		return 0;
	return conf->max_nr_stripes - conf->min_nr_stripes;
}

6357
static struct r5conf *setup_conf(struct mddev *mddev)
L
Linus Torvalds 已提交
6358
{
6359
	struct r5conf *conf;
6360
	int raid_disk, memory, max_disks;
6361
	struct md_rdev *rdev;
L
Linus Torvalds 已提交
6362
	struct disk_info *disk;
6363
	char pers_name[6];
6364
	int i;
6365 6366
	int group_cnt, worker_cnt_per_group;
	struct r5worker_group *new_group;
L
Linus Torvalds 已提交
6367

N
NeilBrown 已提交
6368 6369 6370
	if (mddev->new_level != 5
	    && mddev->new_level != 4
	    && mddev->new_level != 6) {
6371
		printk(KERN_ERR "md/raid:%s: raid level not set to 4/5/6 (%d)\n",
N
NeilBrown 已提交
6372 6373
		       mdname(mddev), mddev->new_level);
		return ERR_PTR(-EIO);
L
Linus Torvalds 已提交
6374
	}
N
NeilBrown 已提交
6375 6376 6377 6378
	if ((mddev->new_level == 5
	     && !algorithm_valid_raid5(mddev->new_layout)) ||
	    (mddev->new_level == 6
	     && !algorithm_valid_raid6(mddev->new_layout))) {
6379
		printk(KERN_ERR "md/raid:%s: layout %d not supported\n",
N
NeilBrown 已提交
6380 6381
		       mdname(mddev), mddev->new_layout);
		return ERR_PTR(-EIO);
6382
	}
N
NeilBrown 已提交
6383
	if (mddev->new_level == 6 && mddev->raid_disks < 4) {
6384
		printk(KERN_ERR "md/raid:%s: not enough configured devices (%d, minimum 4)\n",
N
NeilBrown 已提交
6385 6386
		       mdname(mddev), mddev->raid_disks);
		return ERR_PTR(-EINVAL);
6387 6388
	}

6389 6390 6391
	if (!mddev->new_chunk_sectors ||
	    (mddev->new_chunk_sectors << 9) % PAGE_SIZE ||
	    !is_power_of_2(mddev->new_chunk_sectors)) {
6392 6393
		printk(KERN_ERR "md/raid:%s: invalid chunk size %d\n",
		       mdname(mddev), mddev->new_chunk_sectors << 9);
N
NeilBrown 已提交
6394
		return ERR_PTR(-EINVAL);
6395 6396
	}

6397
	conf = kzalloc(sizeof(struct r5conf), GFP_KERNEL);
N
NeilBrown 已提交
6398
	if (conf == NULL)
L
Linus Torvalds 已提交
6399
		goto abort;
6400
	/* Don't enable multi-threading by default*/
6401 6402 6403 6404 6405 6406
	if (!alloc_thread_groups(conf, 0, &group_cnt, &worker_cnt_per_group,
				 &new_group)) {
		conf->group_cnt = group_cnt;
		conf->worker_cnt_per_group = worker_cnt_per_group;
		conf->worker_groups = new_group;
	} else
6407
		goto abort;
6408
	spin_lock_init(&conf->device_lock);
6409
	seqcount_init(&conf->gen_lock);
6410 6411 6412 6413 6414 6415
	init_waitqueue_head(&conf->wait_for_stripe);
	init_waitqueue_head(&conf->wait_for_overlap);
	INIT_LIST_HEAD(&conf->handle_list);
	INIT_LIST_HEAD(&conf->hold_list);
	INIT_LIST_HEAD(&conf->delayed_list);
	INIT_LIST_HEAD(&conf->bitmap_list);
S
Shaohua Li 已提交
6416
	init_llist_head(&conf->released_stripes);
6417 6418 6419 6420
	atomic_set(&conf->active_stripes, 0);
	atomic_set(&conf->preread_active_stripes, 0);
	atomic_set(&conf->active_aligned_reads, 0);
	conf->bypass_threshold = BYPASS_THRESHOLD;
6421
	conf->recovery_disabled = mddev->recovery_disabled - 1;
N
NeilBrown 已提交
6422 6423 6424 6425 6426

	conf->raid_disks = mddev->raid_disks;
	if (mddev->reshape_position == MaxSector)
		conf->previous_raid_disks = mddev->raid_disks;
	else
6427
		conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks;
6428
	max_disks = max(conf->raid_disks, conf->previous_raid_disks);
6429

6430
	conf->disks = kzalloc(max_disks * sizeof(struct disk_info),
6431 6432 6433
			      GFP_KERNEL);
	if (!conf->disks)
		goto abort;
6434

L
Linus Torvalds 已提交
6435 6436
	conf->mddev = mddev;

6437
	if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL)
L
Linus Torvalds 已提交
6438 6439
		goto abort;

6440 6441 6442 6443 6444 6445 6446 6447 6448 6449 6450 6451 6452 6453 6454
	/* We init hash_locks[0] separately to that it can be used
	 * as the reference lock in the spin_lock_nest_lock() call
	 * in lock_all_device_hash_locks_irq in order to convince
	 * lockdep that we know what we are doing.
	 */
	spin_lock_init(conf->hash_locks);
	for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++)
		spin_lock_init(conf->hash_locks + i);

	for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
		INIT_LIST_HEAD(conf->inactive_list + i);

	for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
		INIT_LIST_HEAD(conf->temp_inactive_list + i);

6455
	conf->level = mddev->new_level;
6456
	conf->chunk_sectors = mddev->new_chunk_sectors;
6457 6458 6459
	if (raid5_alloc_percpu(conf) != 0)
		goto abort;

6460
	pr_debug("raid456: run(%s) called.\n", mdname(mddev));
L
Linus Torvalds 已提交
6461

N
NeilBrown 已提交
6462
	rdev_for_each(rdev, mddev) {
L
Linus Torvalds 已提交
6463
		raid_disk = rdev->raid_disk;
6464
		if (raid_disk >= max_disks
L
Linus Torvalds 已提交
6465 6466 6467 6468
		    || raid_disk < 0)
			continue;
		disk = conf->disks + raid_disk;

6469 6470 6471 6472 6473 6474 6475 6476 6477
		if (test_bit(Replacement, &rdev->flags)) {
			if (disk->replacement)
				goto abort;
			disk->replacement = rdev;
		} else {
			if (disk->rdev)
				goto abort;
			disk->rdev = rdev;
		}
L
Linus Torvalds 已提交
6478

6479
		if (test_bit(In_sync, &rdev->flags)) {
L
Linus Torvalds 已提交
6480
			char b[BDEVNAME_SIZE];
6481 6482 6483
			printk(KERN_INFO "md/raid:%s: device %s operational as raid"
			       " disk %d\n",
			       mdname(mddev), bdevname(rdev->bdev, b), raid_disk);
J
Jonathan Brassow 已提交
6484
		} else if (rdev->saved_raid_disk != raid_disk)
6485 6486
			/* Cannot rely on bitmap to complete recovery */
			conf->fullsync = 1;
L
Linus Torvalds 已提交
6487 6488
	}

N
NeilBrown 已提交
6489
	conf->level = mddev->new_level;
6490
	if (conf->level == 6) {
6491
		conf->max_degraded = 2;
6492 6493 6494 6495 6496
		if (raid6_call.xor_syndrome)
			conf->rmw_level = PARITY_ENABLE_RMW;
		else
			conf->rmw_level = PARITY_DISABLE_RMW;
	} else {
6497
		conf->max_degraded = 1;
6498 6499
		conf->rmw_level = PARITY_ENABLE_RMW;
	}
N
NeilBrown 已提交
6500
	conf->algorithm = mddev->new_layout;
6501
	conf->reshape_progress = mddev->reshape_position;
6502
	if (conf->reshape_progress != MaxSector) {
6503
		conf->prev_chunk_sectors = mddev->chunk_sectors;
6504 6505
		conf->prev_algo = mddev->layout;
	}
L
Linus Torvalds 已提交
6506

6507 6508
	conf->min_nr_stripes = NR_STRIPES;
	memory = conf->min_nr_stripes * (sizeof(struct stripe_head) +
6509
		 max_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024;
6510
	atomic_set(&conf->empty_inactive_list_nr, NR_STRIPE_HASH_LOCKS);
6511
	if (grow_stripes(conf, conf->min_nr_stripes)) {
N
NeilBrown 已提交
6512
		printk(KERN_ERR
6513 6514
		       "md/raid:%s: couldn't allocate %dkB for buffers\n",
		       mdname(mddev), memory);
N
NeilBrown 已提交
6515 6516
		goto abort;
	} else
6517 6518
		printk(KERN_INFO "md/raid:%s: allocated %dkB\n",
		       mdname(mddev), memory);
6519 6520 6521 6522 6523 6524 6525 6526 6527 6528 6529
	/*
	 * Losing a stripe head costs more than the time to refill it,
	 * it reduces the queue depth and so can hurt throughput.
	 * So set it rather large, scaled by number of devices.
	 */
	conf->shrinker.seeks = DEFAULT_SEEKS * conf->raid_disks * 4;
	conf->shrinker.scan_objects = raid5_cache_scan;
	conf->shrinker.count_objects = raid5_cache_count;
	conf->shrinker.batch = 128;
	conf->shrinker.flags = 0;
	register_shrinker(&conf->shrinker);
L
Linus Torvalds 已提交
6530

6531 6532
	sprintf(pers_name, "raid%d", mddev->new_level);
	conf->thread = md_register_thread(raid5d, mddev, pers_name);
N
NeilBrown 已提交
6533 6534
	if (!conf->thread) {
		printk(KERN_ERR
6535
		       "md/raid:%s: couldn't allocate thread.\n",
N
NeilBrown 已提交
6536
		       mdname(mddev));
6537 6538
		goto abort;
	}
N
NeilBrown 已提交
6539 6540 6541 6542 6543

	return conf;

 abort:
	if (conf) {
6544
		free_conf(conf);
N
NeilBrown 已提交
6545 6546 6547 6548 6549
		return ERR_PTR(-EIO);
	} else
		return ERR_PTR(-ENOMEM);
}

6550 6551 6552 6553 6554 6555 6556 6557 6558 6559 6560 6561
static int only_parity(int raid_disk, int algo, int raid_disks, int max_degraded)
{
	switch (algo) {
	case ALGORITHM_PARITY_0:
		if (raid_disk < max_degraded)
			return 1;
		break;
	case ALGORITHM_PARITY_N:
		if (raid_disk >= raid_disks - max_degraded)
			return 1;
		break;
	case ALGORITHM_PARITY_0_6:
6562
		if (raid_disk == 0 ||
6563 6564 6565 6566 6567 6568 6569 6570 6571 6572 6573 6574 6575
		    raid_disk == raid_disks - 1)
			return 1;
		break;
	case ALGORITHM_LEFT_ASYMMETRIC_6:
	case ALGORITHM_RIGHT_ASYMMETRIC_6:
	case ALGORITHM_LEFT_SYMMETRIC_6:
	case ALGORITHM_RIGHT_SYMMETRIC_6:
		if (raid_disk == raid_disks - 1)
			return 1;
	}
	return 0;
}

6576
static int run(struct mddev *mddev)
N
NeilBrown 已提交
6577
{
6578
	struct r5conf *conf;
6579
	int working_disks = 0;
6580
	int dirty_parity_disks = 0;
6581
	struct md_rdev *rdev;
6582
	sector_t reshape_offset = 0;
6583
	int i;
6584 6585
	long long min_offset_diff = 0;
	int first = 1;
N
NeilBrown 已提交
6586

6587
	if (mddev->recovery_cp != MaxSector)
6588
		printk(KERN_NOTICE "md/raid:%s: not clean"
6589 6590
		       " -- starting background reconstruction\n",
		       mdname(mddev));
6591 6592 6593 6594 6595 6596 6597 6598 6599 6600 6601 6602 6603 6604 6605 6606 6607

	rdev_for_each(rdev, mddev) {
		long long diff;
		if (rdev->raid_disk < 0)
			continue;
		diff = (rdev->new_data_offset - rdev->data_offset);
		if (first) {
			min_offset_diff = diff;
			first = 0;
		} else if (mddev->reshape_backwards &&
			 diff < min_offset_diff)
			min_offset_diff = diff;
		else if (!mddev->reshape_backwards &&
			 diff > min_offset_diff)
			min_offset_diff = diff;
	}

N
NeilBrown 已提交
6608 6609
	if (mddev->reshape_position != MaxSector) {
		/* Check that we can continue the reshape.
6610 6611 6612 6613 6614 6615 6616 6617 6618 6619
		 * Difficulties arise if the stripe we would write to
		 * next is at or after the stripe we would read from next.
		 * For a reshape that changes the number of devices, this
		 * is only possible for a very short time, and mdadm makes
		 * sure that time appears to have past before assembling
		 * the array.  So we fail if that time hasn't passed.
		 * For a reshape that keeps the number of devices the same
		 * mdadm must be monitoring the reshape can keeping the
		 * critical areas read-only and backed up.  It will start
		 * the array in read-only mode, so we check for that.
N
NeilBrown 已提交
6620 6621 6622
		 */
		sector_t here_new, here_old;
		int old_disks;
6623
		int max_degraded = (mddev->level == 6 ? 2 : 1);
N
NeilBrown 已提交
6624

6625
		if (mddev->new_level != mddev->level) {
6626
			printk(KERN_ERR "md/raid:%s: unsupported reshape "
N
NeilBrown 已提交
6627 6628 6629 6630 6631 6632 6633 6634 6635 6636
			       "required - aborting.\n",
			       mdname(mddev));
			return -EINVAL;
		}
		old_disks = mddev->raid_disks - mddev->delta_disks;
		/* reshape_position must be on a new-stripe boundary, and one
		 * further up in new geometry must map after here in old
		 * geometry.
		 */
		here_new = mddev->reshape_position;
6637
		if (sector_div(here_new, mddev->new_chunk_sectors *
N
NeilBrown 已提交
6638
			       (mddev->raid_disks - max_degraded))) {
6639 6640
			printk(KERN_ERR "md/raid:%s: reshape_position not "
			       "on a stripe boundary\n", mdname(mddev));
N
NeilBrown 已提交
6641 6642
			return -EINVAL;
		}
6643
		reshape_offset = here_new * mddev->new_chunk_sectors;
N
NeilBrown 已提交
6644 6645
		/* here_new is the stripe we will write to */
		here_old = mddev->reshape_position;
6646
		sector_div(here_old, mddev->chunk_sectors *
N
NeilBrown 已提交
6647 6648 6649
			   (old_disks-max_degraded));
		/* here_old is the first stripe that we might need to read
		 * from */
6650
		if (mddev->delta_disks == 0) {
6651 6652 6653 6654 6655 6656
			if ((here_new * mddev->new_chunk_sectors !=
			     here_old * mddev->chunk_sectors)) {
				printk(KERN_ERR "md/raid:%s: reshape position is"
				       " confused - aborting\n", mdname(mddev));
				return -EINVAL;
			}
6657
			/* We cannot be sure it is safe to start an in-place
6658
			 * reshape.  It is only safe if user-space is monitoring
6659 6660 6661 6662 6663
			 * and taking constant backups.
			 * mdadm always starts a situation like this in
			 * readonly mode so it can take control before
			 * allowing any writes.  So just check for that.
			 */
6664 6665 6666 6667 6668 6669 6670
			if (abs(min_offset_diff) >= mddev->chunk_sectors &&
			    abs(min_offset_diff) >= mddev->new_chunk_sectors)
				/* not really in-place - so OK */;
			else if (mddev->ro == 0) {
				printk(KERN_ERR "md/raid:%s: in-place reshape "
				       "must be started in read-only mode "
				       "- aborting\n",
6671
				       mdname(mddev));
6672 6673
				return -EINVAL;
			}
6674
		} else if (mddev->reshape_backwards
6675
		    ? (here_new * mddev->new_chunk_sectors + min_offset_diff <=
6676 6677
		       here_old * mddev->chunk_sectors)
		    : (here_new * mddev->new_chunk_sectors >=
6678
		       here_old * mddev->chunk_sectors + (-min_offset_diff))) {
N
NeilBrown 已提交
6679
			/* Reading from the same stripe as writing to - bad */
6680 6681 6682
			printk(KERN_ERR "md/raid:%s: reshape_position too early for "
			       "auto-recovery - aborting.\n",
			       mdname(mddev));
N
NeilBrown 已提交
6683 6684
			return -EINVAL;
		}
6685 6686
		printk(KERN_INFO "md/raid:%s: reshape will continue\n",
		       mdname(mddev));
N
NeilBrown 已提交
6687 6688 6689 6690
		/* OK, we should be able to continue; */
	} else {
		BUG_ON(mddev->level != mddev->new_level);
		BUG_ON(mddev->layout != mddev->new_layout);
6691
		BUG_ON(mddev->chunk_sectors != mddev->new_chunk_sectors);
N
NeilBrown 已提交
6692
		BUG_ON(mddev->delta_disks != 0);
L
Linus Torvalds 已提交
6693
	}
N
NeilBrown 已提交
6694

6695 6696 6697 6698 6699
	if (mddev->private == NULL)
		conf = setup_conf(mddev);
	else
		conf = mddev->private;

N
NeilBrown 已提交
6700 6701 6702
	if (IS_ERR(conf))
		return PTR_ERR(conf);

6703
	conf->min_offset_diff = min_offset_diff;
N
NeilBrown 已提交
6704 6705 6706 6707
	mddev->thread = conf->thread;
	conf->thread = NULL;
	mddev->private = conf;

6708 6709 6710 6711 6712 6713 6714 6715 6716 6717 6718
	for (i = 0; i < conf->raid_disks && conf->previous_raid_disks;
	     i++) {
		rdev = conf->disks[i].rdev;
		if (!rdev && conf->disks[i].replacement) {
			/* The replacement is all we have yet */
			rdev = conf->disks[i].replacement;
			conf->disks[i].replacement = NULL;
			clear_bit(Replacement, &rdev->flags);
			conf->disks[i].rdev = rdev;
		}
		if (!rdev)
6719
			continue;
6720 6721 6722 6723 6724 6725 6726
		if (conf->disks[i].replacement &&
		    conf->reshape_progress != MaxSector) {
			/* replacements and reshape simply do not mix. */
			printk(KERN_ERR "md: cannot handle concurrent "
			       "replacement and reshape.\n");
			goto abort;
		}
6727
		if (test_bit(In_sync, &rdev->flags)) {
N
NeilBrown 已提交
6728
			working_disks++;
6729 6730
			continue;
		}
6731 6732 6733 6734 6735 6736 6737 6738 6739 6740 6741 6742
		/* This disc is not fully in-sync.  However if it
		 * just stored parity (beyond the recovery_offset),
		 * when we don't need to be concerned about the
		 * array being dirty.
		 * When reshape goes 'backwards', we never have
		 * partially completed devices, so we only need
		 * to worry about reshape going forwards.
		 */
		/* Hack because v0.91 doesn't store recovery_offset properly. */
		if (mddev->major_version == 0 &&
		    mddev->minor_version > 90)
			rdev->recovery_offset = reshape_offset;
6743

6744 6745 6746 6747 6748 6749 6750 6751 6752 6753 6754 6755 6756 6757 6758
		if (rdev->recovery_offset < reshape_offset) {
			/* We need to check old and new layout */
			if (!only_parity(rdev->raid_disk,
					 conf->algorithm,
					 conf->raid_disks,
					 conf->max_degraded))
				continue;
		}
		if (!only_parity(rdev->raid_disk,
				 conf->prev_algo,
				 conf->previous_raid_disks,
				 conf->max_degraded))
			continue;
		dirty_parity_disks++;
	}
N
NeilBrown 已提交
6759

6760 6761 6762
	/*
	 * 0 for a fully functional array, 1 or 2 for a degraded array.
	 */
6763
	mddev->degraded = calc_degraded(conf);
N
NeilBrown 已提交
6764

6765
	if (has_failed(conf)) {
6766
		printk(KERN_ERR "md/raid:%s: not enough operational devices"
L
Linus Torvalds 已提交
6767
			" (%d/%d failed)\n",
6768
			mdname(mddev), mddev->degraded, conf->raid_disks);
L
Linus Torvalds 已提交
6769 6770 6771
		goto abort;
	}

N
NeilBrown 已提交
6772
	/* device size must be a multiple of chunk size */
6773
	mddev->dev_sectors &= ~(mddev->chunk_sectors - 1);
N
NeilBrown 已提交
6774 6775
	mddev->resync_max_sectors = mddev->dev_sectors;

6776
	if (mddev->degraded > dirty_parity_disks &&
L
Linus Torvalds 已提交
6777
	    mddev->recovery_cp != MaxSector) {
6778 6779
		if (mddev->ok_start_degraded)
			printk(KERN_WARNING
6780 6781
			       "md/raid:%s: starting dirty degraded array"
			       " - data corruption possible.\n",
6782 6783 6784
			       mdname(mddev));
		else {
			printk(KERN_ERR
6785
			       "md/raid:%s: cannot start dirty degraded array.\n",
6786 6787 6788
			       mdname(mddev));
			goto abort;
		}
L
Linus Torvalds 已提交
6789 6790 6791
	}

	if (mddev->degraded == 0)
6792 6793
		printk(KERN_INFO "md/raid:%s: raid level %d active with %d out of %d"
		       " devices, algorithm %d\n", mdname(mddev), conf->level,
6794 6795
		       mddev->raid_disks-mddev->degraded, mddev->raid_disks,
		       mddev->new_layout);
L
Linus Torvalds 已提交
6796
	else
6797 6798 6799 6800 6801
		printk(KERN_ALERT "md/raid:%s: raid level %d active with %d"
		       " out of %d devices, algorithm %d\n",
		       mdname(mddev), conf->level,
		       mddev->raid_disks - mddev->degraded,
		       mddev->raid_disks, mddev->new_layout);
L
Linus Torvalds 已提交
6802 6803 6804

	print_raid5_conf(conf);

6805 6806
	if (conf->reshape_progress != MaxSector) {
		conf->reshape_safe = conf->reshape_progress;
6807 6808 6809 6810 6811 6812
		atomic_set(&conf->reshape_stripes, 0);
		clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
		clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
		set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
		set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
		mddev->sync_thread = md_register_thread(md_do_sync, mddev,
6813
							"reshape");
6814 6815
	}

L
Linus Torvalds 已提交
6816
	/* Ok, everything is just fine now */
6817 6818
	if (mddev->to_remove == &raid5_attrs_group)
		mddev->to_remove = NULL;
N
NeilBrown 已提交
6819 6820
	else if (mddev->kobj.sd &&
	    sysfs_create_group(&mddev->kobj, &raid5_attrs_group))
6821
		printk(KERN_WARNING
6822
		       "raid5: failed to create sysfs attributes for %s\n",
6823
		       mdname(mddev));
6824
	md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
6825

6826
	if (mddev->queue) {
6827
		int chunk_size;
S
Shaohua Li 已提交
6828
		bool discard_supported = true;
6829 6830 6831 6832 6833 6834 6835 6836 6837
		/* read-ahead size must cover two whole stripes, which
		 * is 2 * (datadisks) * chunksize where 'n' is the
		 * number of raid devices
		 */
		int data_disks = conf->previous_raid_disks - conf->max_degraded;
		int stripe = data_disks *
			((mddev->chunk_sectors << 9) / PAGE_SIZE);
		if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
			mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
N
NeilBrown 已提交
6838

6839 6840 6841 6842
		chunk_size = mddev->chunk_sectors << 9;
		blk_queue_io_min(mddev->queue, chunk_size);
		blk_queue_io_opt(mddev->queue, chunk_size *
				 (conf->raid_disks - conf->max_degraded));
6843
		mddev->queue->limits.raid_partial_stripes_expensive = 1;
S
Shaohua Li 已提交
6844 6845 6846 6847 6848
		/*
		 * We can only discard a whole stripe. It doesn't make sense to
		 * discard data disk but write parity disk
		 */
		stripe = stripe * PAGE_SIZE;
6849 6850 6851 6852
		/* Round up to power of 2, as discard handling
		 * currently assumes that */
		while ((stripe-1) & stripe)
			stripe = (stripe | (stripe-1)) + 1;
S
Shaohua Li 已提交
6853 6854 6855 6856
		mddev->queue->limits.discard_alignment = stripe;
		mddev->queue->limits.discard_granularity = stripe;
		/*
		 * unaligned part of discard request will be ignored, so can't
6857
		 * guarantee discard_zeroes_data
S
Shaohua Li 已提交
6858 6859
		 */
		mddev->queue->limits.discard_zeroes_data = 0;
6860

6861 6862
		blk_queue_max_write_same_sectors(mddev->queue, 0);

6863
		rdev_for_each(rdev, mddev) {
6864 6865
			disk_stack_limits(mddev->gendisk, rdev->bdev,
					  rdev->data_offset << 9);
6866 6867
			disk_stack_limits(mddev->gendisk, rdev->bdev,
					  rdev->new_data_offset << 9);
S
Shaohua Li 已提交
6868 6869 6870 6871 6872 6873 6874 6875 6876 6877 6878 6879 6880 6881
			/*
			 * discard_zeroes_data is required, otherwise data
			 * could be lost. Consider a scenario: discard a stripe
			 * (the stripe could be inconsistent if
			 * discard_zeroes_data is 0); write one disk of the
			 * stripe (the stripe could be inconsistent again
			 * depending on which disks are used to calculate
			 * parity); the disk is broken; The stripe data of this
			 * disk is lost.
			 */
			if (!blk_queue_discard(bdev_get_queue(rdev->bdev)) ||
			    !bdev_get_queue(rdev->bdev)->
						limits.discard_zeroes_data)
				discard_supported = false;
6882 6883 6884 6885 6886 6887 6888 6889 6890 6891 6892 6893
			/* Unfortunately, discard_zeroes_data is not currently
			 * a guarantee - just a hint.  So we only allow DISCARD
			 * if the sysadmin has confirmed that only safe devices
			 * are in use by setting a module parameter.
			 */
			if (!devices_handle_discard_safely) {
				if (discard_supported) {
					pr_info("md/raid456: discard support disabled due to uncertainty.\n");
					pr_info("Set raid456.devices_handle_discard_safely=Y to override.\n");
				}
				discard_supported = false;
			}
6894
		}
S
Shaohua Li 已提交
6895 6896 6897 6898 6899 6900 6901 6902 6903

		if (discard_supported &&
		   mddev->queue->limits.max_discard_sectors >= stripe &&
		   mddev->queue->limits.discard_granularity >= stripe)
			queue_flag_set_unlocked(QUEUE_FLAG_DISCARD,
						mddev->queue);
		else
			queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD,
						mddev->queue);
6904
	}
6905

L
Linus Torvalds 已提交
6906 6907
	return 0;
abort:
6908
	md_unregister_thread(&mddev->thread);
N
NeilBrown 已提交
6909 6910
	print_raid5_conf(conf);
	free_conf(conf);
L
Linus Torvalds 已提交
6911
	mddev->private = NULL;
6912
	printk(KERN_ALERT "md/raid:%s: failed to run raid set.\n", mdname(mddev));
L
Linus Torvalds 已提交
6913 6914 6915
	return -EIO;
}

N
NeilBrown 已提交
6916
static void raid5_free(struct mddev *mddev, void *priv)
L
Linus Torvalds 已提交
6917
{
N
NeilBrown 已提交
6918
	struct r5conf *conf = priv;
L
Linus Torvalds 已提交
6919

6920
	free_conf(conf);
6921
	mddev->to_remove = &raid5_attrs_group;
L
Linus Torvalds 已提交
6922 6923
}

6924
static void status(struct seq_file *seq, struct mddev *mddev)
L
Linus Torvalds 已提交
6925
{
6926
	struct r5conf *conf = mddev->private;
L
Linus Torvalds 已提交
6927 6928
	int i;

6929 6930
	seq_printf(seq, " level %d, %dk chunk, algorithm %d", mddev->level,
		mddev->chunk_sectors / 2, mddev->layout);
6931
	seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded);
L
Linus Torvalds 已提交
6932 6933 6934
	for (i = 0; i < conf->raid_disks; i++)
		seq_printf (seq, "%s",
			       conf->disks[i].rdev &&
6935
			       test_bit(In_sync, &conf->disks[i].rdev->flags) ? "U" : "_");
L
Linus Torvalds 已提交
6936 6937 6938
	seq_printf (seq, "]");
}

6939
static void print_raid5_conf (struct r5conf *conf)
L
Linus Torvalds 已提交
6940 6941 6942 6943
{
	int i;
	struct disk_info *tmp;

6944
	printk(KERN_DEBUG "RAID conf printout:\n");
L
Linus Torvalds 已提交
6945 6946 6947 6948
	if (!conf) {
		printk("(conf==NULL)\n");
		return;
	}
6949 6950 6951
	printk(KERN_DEBUG " --- level:%d rd:%d wd:%d\n", conf->level,
	       conf->raid_disks,
	       conf->raid_disks - conf->mddev->degraded);
L
Linus Torvalds 已提交
6952 6953 6954 6955 6956

	for (i = 0; i < conf->raid_disks; i++) {
		char b[BDEVNAME_SIZE];
		tmp = conf->disks + i;
		if (tmp->rdev)
6957 6958 6959
			printk(KERN_DEBUG " disk %d, o:%d, dev:%s\n",
			       i, !test_bit(Faulty, &tmp->rdev->flags),
			       bdevname(tmp->rdev->bdev, b));
L
Linus Torvalds 已提交
6960 6961 6962
	}
}

6963
static int raid5_spare_active(struct mddev *mddev)
L
Linus Torvalds 已提交
6964 6965
{
	int i;
6966
	struct r5conf *conf = mddev->private;
L
Linus Torvalds 已提交
6967
	struct disk_info *tmp;
6968 6969
	int count = 0;
	unsigned long flags;
L
Linus Torvalds 已提交
6970 6971 6972

	for (i = 0; i < conf->raid_disks; i++) {
		tmp = conf->disks + i;
6973 6974 6975 6976 6977 6978 6979 6980 6981 6982 6983 6984 6985 6986 6987 6988 6989 6990 6991
		if (tmp->replacement
		    && tmp->replacement->recovery_offset == MaxSector
		    && !test_bit(Faulty, &tmp->replacement->flags)
		    && !test_and_set_bit(In_sync, &tmp->replacement->flags)) {
			/* Replacement has just become active. */
			if (!tmp->rdev
			    || !test_and_clear_bit(In_sync, &tmp->rdev->flags))
				count++;
			if (tmp->rdev) {
				/* Replaced device not technically faulty,
				 * but we need to be sure it gets removed
				 * and never re-added.
				 */
				set_bit(Faulty, &tmp->rdev->flags);
				sysfs_notify_dirent_safe(
					tmp->rdev->sysfs_state);
			}
			sysfs_notify_dirent_safe(tmp->replacement->sysfs_state);
		} else if (tmp->rdev
6992
		    && tmp->rdev->recovery_offset == MaxSector
6993
		    && !test_bit(Faulty, &tmp->rdev->flags)
6994
		    && !test_and_set_bit(In_sync, &tmp->rdev->flags)) {
6995
			count++;
6996
			sysfs_notify_dirent_safe(tmp->rdev->sysfs_state);
L
Linus Torvalds 已提交
6997 6998
		}
	}
6999
	spin_lock_irqsave(&conf->device_lock, flags);
7000
	mddev->degraded = calc_degraded(conf);
7001
	spin_unlock_irqrestore(&conf->device_lock, flags);
L
Linus Torvalds 已提交
7002
	print_raid5_conf(conf);
7003
	return count;
L
Linus Torvalds 已提交
7004 7005
}

7006
static int raid5_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
L
Linus Torvalds 已提交
7007
{
7008
	struct r5conf *conf = mddev->private;
L
Linus Torvalds 已提交
7009
	int err = 0;
7010
	int number = rdev->raid_disk;
7011
	struct md_rdev **rdevp;
L
Linus Torvalds 已提交
7012 7013 7014
	struct disk_info *p = conf->disks + number;

	print_raid5_conf(conf);
7015 7016 7017 7018 7019 7020 7021 7022 7023 7024 7025 7026 7027 7028 7029 7030 7031 7032 7033 7034 7035 7036
	if (rdev == p->rdev)
		rdevp = &p->rdev;
	else if (rdev == p->replacement)
		rdevp = &p->replacement;
	else
		return 0;

	if (number >= conf->raid_disks &&
	    conf->reshape_progress == MaxSector)
		clear_bit(In_sync, &rdev->flags);

	if (test_bit(In_sync, &rdev->flags) ||
	    atomic_read(&rdev->nr_pending)) {
		err = -EBUSY;
		goto abort;
	}
	/* Only remove non-faulty devices if recovery
	 * isn't possible.
	 */
	if (!test_bit(Faulty, &rdev->flags) &&
	    mddev->recovery_disabled != conf->recovery_disabled &&
	    !has_failed(conf) &&
7037
	    (!p->replacement || p->replacement == rdev) &&
7038 7039 7040 7041 7042 7043 7044 7045 7046 7047
	    number < conf->raid_disks) {
		err = -EBUSY;
		goto abort;
	}
	*rdevp = NULL;
	synchronize_rcu();
	if (atomic_read(&rdev->nr_pending)) {
		/* lost the race, try later */
		err = -EBUSY;
		*rdevp = rdev;
7048 7049 7050 7051 7052 7053 7054 7055 7056 7057 7058 7059 7060 7061
	} else if (p->replacement) {
		/* We must have just cleared 'rdev' */
		p->rdev = p->replacement;
		clear_bit(Replacement, &p->replacement->flags);
		smp_mb(); /* Make sure other CPUs may see both as identical
			   * but will never see neither - if they are careful
			   */
		p->replacement = NULL;
		clear_bit(WantReplacement, &rdev->flags);
	} else
		/* We might have just removed the Replacement as faulty-
		 * clear the bit just in case
		 */
		clear_bit(WantReplacement, &rdev->flags);
L
Linus Torvalds 已提交
7062 7063 7064 7065 7066 7067
abort:

	print_raid5_conf(conf);
	return err;
}

7068
static int raid5_add_disk(struct mddev *mddev, struct md_rdev *rdev)
L
Linus Torvalds 已提交
7069
{
7070
	struct r5conf *conf = mddev->private;
7071
	int err = -EEXIST;
L
Linus Torvalds 已提交
7072 7073
	int disk;
	struct disk_info *p;
7074 7075
	int first = 0;
	int last = conf->raid_disks - 1;
L
Linus Torvalds 已提交
7076

7077 7078 7079
	if (mddev->recovery_disabled == conf->recovery_disabled)
		return -EBUSY;

N
NeilBrown 已提交
7080
	if (rdev->saved_raid_disk < 0 && has_failed(conf))
L
Linus Torvalds 已提交
7081
		/* no point adding a device */
7082
		return -EINVAL;
L
Linus Torvalds 已提交
7083

7084 7085
	if (rdev->raid_disk >= 0)
		first = last = rdev->raid_disk;
L
Linus Torvalds 已提交
7086 7087

	/*
7088 7089
	 * find the disk ... but prefer rdev->saved_raid_disk
	 * if possible.
L
Linus Torvalds 已提交
7090
	 */
7091
	if (rdev->saved_raid_disk >= 0 &&
7092
	    rdev->saved_raid_disk >= first &&
7093
	    conf->disks[rdev->saved_raid_disk].rdev == NULL)
7094 7095 7096
		first = rdev->saved_raid_disk;

	for (disk = first; disk <= last; disk++) {
7097 7098
		p = conf->disks + disk;
		if (p->rdev == NULL) {
7099
			clear_bit(In_sync, &rdev->flags);
L
Linus Torvalds 已提交
7100
			rdev->raid_disk = disk;
7101
			err = 0;
7102 7103
			if (rdev->saved_raid_disk != disk)
				conf->fullsync = 1;
7104
			rcu_assign_pointer(p->rdev, rdev);
7105
			goto out;
L
Linus Torvalds 已提交
7106
		}
7107 7108 7109
	}
	for (disk = first; disk <= last; disk++) {
		p = conf->disks + disk;
7110 7111 7112 7113 7114 7115 7116 7117 7118 7119 7120
		if (test_bit(WantReplacement, &p->rdev->flags) &&
		    p->replacement == NULL) {
			clear_bit(In_sync, &rdev->flags);
			set_bit(Replacement, &rdev->flags);
			rdev->raid_disk = disk;
			err = 0;
			conf->fullsync = 1;
			rcu_assign_pointer(p->replacement, rdev);
			break;
		}
	}
7121
out:
L
Linus Torvalds 已提交
7122
	print_raid5_conf(conf);
7123
	return err;
L
Linus Torvalds 已提交
7124 7125
}

7126
static int raid5_resize(struct mddev *mddev, sector_t sectors)
L
Linus Torvalds 已提交
7127 7128 7129 7130 7131 7132 7133 7134
{
	/* no resync is happening, and there is enough space
	 * on all devices, so we can resize.
	 * We need to make sure resync covers any new space.
	 * If the array is shrinking we should possibly wait until
	 * any io in the removed space completes, but it hardly seems
	 * worth it.
	 */
7135
	sector_t newsize;
7136
	sectors &= ~((sector_t)mddev->chunk_sectors - 1);
7137 7138 7139
	newsize = raid5_size(mddev, sectors, mddev->raid_disks);
	if (mddev->external_size &&
	    mddev->array_sectors > newsize)
D
Dan Williams 已提交
7140
		return -EINVAL;
7141 7142 7143 7144 7145 7146
	if (mddev->bitmap) {
		int ret = bitmap_resize(mddev->bitmap, sectors, 0, 0);
		if (ret)
			return ret;
	}
	md_set_array_sectors(mddev, newsize);
7147
	set_capacity(mddev->gendisk, mddev->array_sectors);
7148
	revalidate_disk(mddev->gendisk);
7149 7150
	if (sectors > mddev->dev_sectors &&
	    mddev->recovery_cp > mddev->dev_sectors) {
A
Andre Noll 已提交
7151
		mddev->recovery_cp = mddev->dev_sectors;
L
Linus Torvalds 已提交
7152 7153
		set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
	}
A
Andre Noll 已提交
7154
	mddev->dev_sectors = sectors;
7155
	mddev->resync_max_sectors = sectors;
L
Linus Torvalds 已提交
7156 7157 7158
	return 0;
}

7159
static int check_stripe_cache(struct mddev *mddev)
7160 7161 7162 7163 7164 7165 7166 7167 7168
{
	/* Can only proceed if there are plenty of stripe_heads.
	 * We need a minimum of one full stripe,, and for sensible progress
	 * it is best to have about 4 times that.
	 * If we require 4 times, then the default 256 4K stripe_heads will
	 * allow for chunk sizes up to 256K, which is probably OK.
	 * If the chunk size is greater, user-space should request more
	 * stripe_heads first.
	 */
7169
	struct r5conf *conf = mddev->private;
7170
	if (((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4
7171
	    > conf->min_nr_stripes ||
7172
	    ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4
7173
	    > conf->min_nr_stripes) {
7174 7175
		printk(KERN_WARNING "md/raid:%s: reshape: not enough stripes.  Needed %lu\n",
		       mdname(mddev),
7176 7177 7178 7179 7180 7181 7182
		       ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9)
			/ STRIPE_SIZE)*4);
		return 0;
	}
	return 1;
}

7183
static int check_reshape(struct mddev *mddev)
7184
{
7185
	struct r5conf *conf = mddev->private;
7186

7187 7188
	if (mddev->delta_disks == 0 &&
	    mddev->new_layout == mddev->layout &&
7189
	    mddev->new_chunk_sectors == mddev->chunk_sectors)
7190
		return 0; /* nothing to do */
7191
	if (has_failed(conf))
7192
		return -EINVAL;
7193
	if (mddev->delta_disks < 0 && mddev->reshape_position == MaxSector) {
7194 7195 7196 7197 7198 7199 7200 7201 7202 7203 7204
		/* We might be able to shrink, but the devices must
		 * be made bigger first.
		 * For raid6, 4 is the minimum size.
		 * Otherwise 2 is the minimum
		 */
		int min = 2;
		if (mddev->level == 6)
			min = 4;
		if (mddev->raid_disks + mddev->delta_disks < min)
			return -EINVAL;
	}
7205

7206
	if (!check_stripe_cache(mddev))
7207 7208
		return -ENOSPC;

7209 7210 7211 7212 7213 7214 7215 7216 7217
	if (mddev->new_chunk_sectors > mddev->chunk_sectors ||
	    mddev->delta_disks > 0)
		if (resize_chunks(conf,
				  conf->previous_raid_disks
				  + max(0, mddev->delta_disks),
				  max(mddev->new_chunk_sectors,
				      mddev->chunk_sectors)
			    ) < 0)
			return -ENOMEM;
7218 7219
	return resize_stripes(conf, (conf->previous_raid_disks
				     + mddev->delta_disks));
7220 7221
}

7222
static int raid5_start_reshape(struct mddev *mddev)
7223
{
7224
	struct r5conf *conf = mddev->private;
7225
	struct md_rdev *rdev;
7226
	int spares = 0;
7227
	unsigned long flags;
7228

7229
	if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
7230 7231
		return -EBUSY;

7232 7233 7234
	if (!check_stripe_cache(mddev))
		return -ENOSPC;

7235 7236 7237
	if (has_failed(conf))
		return -EINVAL;

7238
	rdev_for_each(rdev, mddev) {
7239 7240
		if (!test_bit(In_sync, &rdev->flags)
		    && !test_bit(Faulty, &rdev->flags))
7241
			spares++;
7242
	}
7243

7244
	if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded)
7245 7246 7247 7248 7249
		/* Not enough devices even to make a degraded array
		 * of that size
		 */
		return -EINVAL;

7250 7251 7252 7253 7254 7255
	/* Refuse to reduce size of the array.  Any reductions in
	 * array size must be through explicit setting of array_size
	 * attribute.
	 */
	if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks)
	    < mddev->array_sectors) {
7256
		printk(KERN_ERR "md/raid:%s: array size must be reduced "
7257 7258 7259 7260
		       "before number of disks\n", mdname(mddev));
		return -EINVAL;
	}

7261
	atomic_set(&conf->reshape_stripes, 0);
7262
	spin_lock_irq(&conf->device_lock);
7263
	write_seqcount_begin(&conf->gen_lock);
7264
	conf->previous_raid_disks = conf->raid_disks;
7265
	conf->raid_disks += mddev->delta_disks;
7266 7267
	conf->prev_chunk_sectors = conf->chunk_sectors;
	conf->chunk_sectors = mddev->new_chunk_sectors;
7268 7269
	conf->prev_algo = conf->algorithm;
	conf->algorithm = mddev->new_layout;
7270 7271 7272 7273 7274
	conf->generation++;
	/* Code that selects data_offset needs to see the generation update
	 * if reshape_progress has been set - so a memory barrier needed.
	 */
	smp_mb();
7275
	if (mddev->reshape_backwards)
7276 7277 7278 7279
		conf->reshape_progress = raid5_size(mddev, 0, 0);
	else
		conf->reshape_progress = 0;
	conf->reshape_safe = conf->reshape_progress;
7280
	write_seqcount_end(&conf->gen_lock);
7281 7282
	spin_unlock_irq(&conf->device_lock);

7283 7284 7285 7286 7287 7288 7289
	/* Now make sure any requests that proceeded on the assumption
	 * the reshape wasn't running - like Discard or Read - have
	 * completed.
	 */
	mddev_suspend(mddev);
	mddev_resume(mddev);

7290 7291
	/* Add some new drives, as many as will fit.
	 * We know there are enough to make the newly sized array work.
7292 7293 7294 7295
	 * Don't add devices if we are reducing the number of
	 * devices in the array.  This is because it is not possible
	 * to correctly record the "partially reconstructed" state of
	 * such devices during the reshape and confusion could result.
7296
	 */
7297
	if (mddev->delta_disks >= 0) {
N
NeilBrown 已提交
7298
		rdev_for_each(rdev, mddev)
7299 7300 7301 7302
			if (rdev->raid_disk < 0 &&
			    !test_bit(Faulty, &rdev->flags)) {
				if (raid5_add_disk(mddev, rdev) == 0) {
					if (rdev->raid_disk
7303
					    >= conf->previous_raid_disks)
7304
						set_bit(In_sync, &rdev->flags);
7305
					else
7306
						rdev->recovery_offset = 0;
7307 7308

					if (sysfs_link_rdev(mddev, rdev))
7309
						/* Failure here is OK */;
7310
				}
7311 7312 7313 7314 7315
			} else if (rdev->raid_disk >= conf->previous_raid_disks
				   && !test_bit(Faulty, &rdev->flags)) {
				/* This is a spare that was manually added */
				set_bit(In_sync, &rdev->flags);
			}
7316

7317 7318 7319 7320
		/* When a reshape changes the number of devices,
		 * ->degraded is measured against the larger of the
		 * pre and post number of devices.
		 */
7321
		spin_lock_irqsave(&conf->device_lock, flags);
7322
		mddev->degraded = calc_degraded(conf);
7323 7324
		spin_unlock_irqrestore(&conf->device_lock, flags);
	}
7325
	mddev->raid_disks = conf->raid_disks;
7326
	mddev->reshape_position = conf->reshape_progress;
7327
	set_bit(MD_CHANGE_DEVS, &mddev->flags);
7328

7329 7330 7331 7332 7333
	clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
	clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
	set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
	set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
	mddev->sync_thread = md_register_thread(md_do_sync, mddev,
7334
						"reshape");
7335 7336 7337
	if (!mddev->sync_thread) {
		mddev->recovery = 0;
		spin_lock_irq(&conf->device_lock);
7338
		write_seqcount_begin(&conf->gen_lock);
7339
		mddev->raid_disks = conf->raid_disks = conf->previous_raid_disks;
7340 7341 7342
		mddev->new_chunk_sectors =
			conf->chunk_sectors = conf->prev_chunk_sectors;
		mddev->new_layout = conf->algorithm = conf->prev_algo;
7343 7344 7345
		rdev_for_each(rdev, mddev)
			rdev->new_data_offset = rdev->data_offset;
		smp_wmb();
7346
		conf->generation --;
7347
		conf->reshape_progress = MaxSector;
7348
		mddev->reshape_position = MaxSector;
7349
		write_seqcount_end(&conf->gen_lock);
7350 7351 7352
		spin_unlock_irq(&conf->device_lock);
		return -EAGAIN;
	}
7353
	conf->reshape_checkpoint = jiffies;
7354 7355 7356 7357 7358
	md_wakeup_thread(mddev->sync_thread);
	md_new_event(mddev);
	return 0;
}

7359 7360 7361
/* This is called from the reshape thread and should make any
 * changes needed in 'conf'
 */
7362
static void end_reshape(struct r5conf *conf)
7363 7364
{

7365
	if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) {
7366
		struct md_rdev *rdev;
7367 7368

		spin_lock_irq(&conf->device_lock);
7369
		conf->previous_raid_disks = conf->raid_disks;
7370 7371 7372
		rdev_for_each(rdev, conf->mddev)
			rdev->data_offset = rdev->new_data_offset;
		smp_wmb();
7373
		conf->reshape_progress = MaxSector;
7374
		spin_unlock_irq(&conf->device_lock);
7375
		wake_up(&conf->wait_for_overlap);
7376 7377 7378 7379

		/* read-ahead size must cover two whole stripes, which is
		 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices
		 */
7380
		if (conf->mddev->queue) {
7381
			int data_disks = conf->raid_disks - conf->max_degraded;
7382
			int stripe = data_disks * ((conf->chunk_sectors << 9)
7383
						   / PAGE_SIZE);
7384 7385 7386
			if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
				conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
		}
7387 7388 7389
	}
}

7390 7391 7392
/* This is called from the raid5d thread with mddev_lock held.
 * It makes config changes to the device.
 */
7393
static void raid5_finish_reshape(struct mddev *mddev)
7394
{
7395
	struct r5conf *conf = mddev->private;
7396 7397 7398

	if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {

7399 7400 7401
		if (mddev->delta_disks > 0) {
			md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
			set_capacity(mddev->gendisk, mddev->array_sectors);
7402
			revalidate_disk(mddev->gendisk);
7403 7404
		} else {
			int d;
7405 7406 7407
			spin_lock_irq(&conf->device_lock);
			mddev->degraded = calc_degraded(conf);
			spin_unlock_irq(&conf->device_lock);
7408 7409
			for (d = conf->raid_disks ;
			     d < conf->raid_disks - mddev->delta_disks;
7410
			     d++) {
7411
				struct md_rdev *rdev = conf->disks[d].rdev;
7412 7413 7414 7415 7416
				if (rdev)
					clear_bit(In_sync, &rdev->flags);
				rdev = conf->disks[d].replacement;
				if (rdev)
					clear_bit(In_sync, &rdev->flags);
7417
			}
7418
		}
7419
		mddev->layout = conf->algorithm;
7420
		mddev->chunk_sectors = conf->chunk_sectors;
7421 7422
		mddev->reshape_position = MaxSector;
		mddev->delta_disks = 0;
7423
		mddev->reshape_backwards = 0;
7424 7425 7426
	}
}

7427
static void raid5_quiesce(struct mddev *mddev, int state)
7428
{
7429
	struct r5conf *conf = mddev->private;
7430 7431

	switch(state) {
7432 7433 7434 7435
	case 2: /* resume for a suspend */
		wake_up(&conf->wait_for_overlap);
		break;

7436
	case 1: /* stop all writes */
7437
		lock_all_device_hash_locks_irq(conf);
7438 7439 7440 7441
		/* '2' tells resync/reshape to pause so that all
		 * active stripes can drain
		 */
		conf->quiesce = 2;
7442
		wait_event_cmd(conf->wait_for_stripe,
7443 7444
				    atomic_read(&conf->active_stripes) == 0 &&
				    atomic_read(&conf->active_aligned_reads) == 0,
7445 7446
				    unlock_all_device_hash_locks_irq(conf),
				    lock_all_device_hash_locks_irq(conf));
7447
		conf->quiesce = 1;
7448
		unlock_all_device_hash_locks_irq(conf);
7449 7450
		/* allow reshape to continue */
		wake_up(&conf->wait_for_overlap);
7451 7452 7453
		break;

	case 0: /* re-enable writes */
7454
		lock_all_device_hash_locks_irq(conf);
7455 7456
		conf->quiesce = 0;
		wake_up(&conf->wait_for_stripe);
7457
		wake_up(&conf->wait_for_overlap);
7458
		unlock_all_device_hash_locks_irq(conf);
7459 7460 7461
		break;
	}
}
7462

7463
static void *raid45_takeover_raid0(struct mddev *mddev, int level)
7464
{
7465
	struct r0conf *raid0_conf = mddev->private;
7466
	sector_t sectors;
7467

D
Dan Williams 已提交
7468
	/* for raid0 takeover only one zone is supported */
7469
	if (raid0_conf->nr_strip_zones > 1) {
7470 7471
		printk(KERN_ERR "md/raid:%s: cannot takeover raid0 with more than one zone.\n",
		       mdname(mddev));
D
Dan Williams 已提交
7472 7473 7474
		return ERR_PTR(-EINVAL);
	}

7475 7476
	sectors = raid0_conf->strip_zone[0].zone_end;
	sector_div(sectors, raid0_conf->strip_zone[0].nb_dev);
7477
	mddev->dev_sectors = sectors;
D
Dan Williams 已提交
7478
	mddev->new_level = level;
7479 7480 7481 7482 7483 7484 7485 7486 7487 7488
	mddev->new_layout = ALGORITHM_PARITY_N;
	mddev->new_chunk_sectors = mddev->chunk_sectors;
	mddev->raid_disks += 1;
	mddev->delta_disks = 1;
	/* make sure it will be not marked as dirty */
	mddev->recovery_cp = MaxSector;

	return setup_conf(mddev);
}

7489
static void *raid5_takeover_raid1(struct mddev *mddev)
7490 7491 7492 7493 7494 7495 7496 7497 7498 7499 7500 7501 7502 7503 7504 7505 7506 7507 7508 7509 7510
{
	int chunksect;

	if (mddev->raid_disks != 2 ||
	    mddev->degraded > 1)
		return ERR_PTR(-EINVAL);

	/* Should check if there are write-behind devices? */

	chunksect = 64*2; /* 64K by default */

	/* The array must be an exact multiple of chunksize */
	while (chunksect && (mddev->array_sectors & (chunksect-1)))
		chunksect >>= 1;

	if ((chunksect<<9) < STRIPE_SIZE)
		/* array size does not allow a suitable chunk size */
		return ERR_PTR(-EINVAL);

	mddev->new_level = 5;
	mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC;
7511
	mddev->new_chunk_sectors = chunksect;
7512 7513 7514 7515

	return setup_conf(mddev);
}

7516
static void *raid5_takeover_raid6(struct mddev *mddev)
7517 7518 7519 7520 7521 7522 7523 7524 7525 7526 7527 7528 7529 7530 7531 7532 7533 7534 7535 7536 7537 7538 7539 7540 7541 7542 7543 7544 7545 7546 7547 7548
{
	int new_layout;

	switch (mddev->layout) {
	case ALGORITHM_LEFT_ASYMMETRIC_6:
		new_layout = ALGORITHM_LEFT_ASYMMETRIC;
		break;
	case ALGORITHM_RIGHT_ASYMMETRIC_6:
		new_layout = ALGORITHM_RIGHT_ASYMMETRIC;
		break;
	case ALGORITHM_LEFT_SYMMETRIC_6:
		new_layout = ALGORITHM_LEFT_SYMMETRIC;
		break;
	case ALGORITHM_RIGHT_SYMMETRIC_6:
		new_layout = ALGORITHM_RIGHT_SYMMETRIC;
		break;
	case ALGORITHM_PARITY_0_6:
		new_layout = ALGORITHM_PARITY_0;
		break;
	case ALGORITHM_PARITY_N:
		new_layout = ALGORITHM_PARITY_N;
		break;
	default:
		return ERR_PTR(-EINVAL);
	}
	mddev->new_level = 5;
	mddev->new_layout = new_layout;
	mddev->delta_disks = -1;
	mddev->raid_disks -= 1;
	return setup_conf(mddev);
}

7549
static int raid5_check_reshape(struct mddev *mddev)
7550
{
7551 7552 7553 7554
	/* For a 2-drive array, the layout and chunk size can be changed
	 * immediately as not restriping is needed.
	 * For larger arrays we record the new value - after validation
	 * to be used by a reshape pass.
7555
	 */
7556
	struct r5conf *conf = mddev->private;
7557
	int new_chunk = mddev->new_chunk_sectors;
7558

7559
	if (mddev->new_layout >= 0 && !algorithm_valid_raid5(mddev->new_layout))
7560 7561
		return -EINVAL;
	if (new_chunk > 0) {
7562
		if (!is_power_of_2(new_chunk))
7563
			return -EINVAL;
7564
		if (new_chunk < (PAGE_SIZE>>9))
7565
			return -EINVAL;
7566
		if (mddev->array_sectors & (new_chunk-1))
7567 7568 7569 7570 7571 7572
			/* not factor of array size */
			return -EINVAL;
	}

	/* They look valid */

7573
	if (mddev->raid_disks == 2) {
7574 7575 7576 7577
		/* can make the change immediately */
		if (mddev->new_layout >= 0) {
			conf->algorithm = mddev->new_layout;
			mddev->layout = mddev->new_layout;
7578 7579
		}
		if (new_chunk > 0) {
7580 7581
			conf->chunk_sectors = new_chunk ;
			mddev->chunk_sectors = new_chunk;
7582 7583 7584
		}
		set_bit(MD_CHANGE_DEVS, &mddev->flags);
		md_wakeup_thread(mddev->thread);
7585
	}
7586
	return check_reshape(mddev);
7587 7588
}

7589
static int raid6_check_reshape(struct mddev *mddev)
7590
{
7591
	int new_chunk = mddev->new_chunk_sectors;
7592

7593
	if (mddev->new_layout >= 0 && !algorithm_valid_raid6(mddev->new_layout))
7594
		return -EINVAL;
7595
	if (new_chunk > 0) {
7596
		if (!is_power_of_2(new_chunk))
7597
			return -EINVAL;
7598
		if (new_chunk < (PAGE_SIZE >> 9))
7599
			return -EINVAL;
7600
		if (mddev->array_sectors & (new_chunk-1))
7601 7602
			/* not factor of array size */
			return -EINVAL;
7603
	}
7604 7605

	/* They look valid */
7606
	return check_reshape(mddev);
7607 7608
}

7609
static void *raid5_takeover(struct mddev *mddev)
7610 7611
{
	/* raid5 can take over:
D
Dan Williams 已提交
7612
	 *  raid0 - if there is only one strip zone - make it a raid4 layout
7613 7614 7615 7616
	 *  raid1 - if there are two drives.  We need to know the chunk size
	 *  raid4 - trivial - just use a raid4 layout.
	 *  raid6 - Providing it is a *_6 layout
	 */
D
Dan Williams 已提交
7617 7618
	if (mddev->level == 0)
		return raid45_takeover_raid0(mddev, 5);
7619 7620
	if (mddev->level == 1)
		return raid5_takeover_raid1(mddev);
7621 7622 7623 7624 7625
	if (mddev->level == 4) {
		mddev->new_layout = ALGORITHM_PARITY_N;
		mddev->new_level = 5;
		return setup_conf(mddev);
	}
7626 7627
	if (mddev->level == 6)
		return raid5_takeover_raid6(mddev);
7628 7629 7630 7631

	return ERR_PTR(-EINVAL);
}

7632
static void *raid4_takeover(struct mddev *mddev)
7633
{
D
Dan Williams 已提交
7634 7635 7636
	/* raid4 can take over:
	 *  raid0 - if there is only one strip zone
	 *  raid5 - if layout is right
7637
	 */
D
Dan Williams 已提交
7638 7639
	if (mddev->level == 0)
		return raid45_takeover_raid0(mddev, 4);
7640 7641 7642 7643 7644 7645 7646 7647
	if (mddev->level == 5 &&
	    mddev->layout == ALGORITHM_PARITY_N) {
		mddev->new_layout = 0;
		mddev->new_level = 4;
		return setup_conf(mddev);
	}
	return ERR_PTR(-EINVAL);
}
7648

7649
static struct md_personality raid5_personality;
7650

7651
static void *raid6_takeover(struct mddev *mddev)
7652 7653 7654 7655 7656 7657 7658 7659 7660 7661 7662 7663 7664 7665 7666 7667 7668 7669 7670 7671 7672 7673 7674 7675 7676 7677 7678 7679 7680 7681 7682 7683 7684 7685 7686 7687 7688 7689 7690 7691 7692 7693 7694 7695 7696
{
	/* Currently can only take over a raid5.  We map the
	 * personality to an equivalent raid6 personality
	 * with the Q block at the end.
	 */
	int new_layout;

	if (mddev->pers != &raid5_personality)
		return ERR_PTR(-EINVAL);
	if (mddev->degraded > 1)
		return ERR_PTR(-EINVAL);
	if (mddev->raid_disks > 253)
		return ERR_PTR(-EINVAL);
	if (mddev->raid_disks < 3)
		return ERR_PTR(-EINVAL);

	switch (mddev->layout) {
	case ALGORITHM_LEFT_ASYMMETRIC:
		new_layout = ALGORITHM_LEFT_ASYMMETRIC_6;
		break;
	case ALGORITHM_RIGHT_ASYMMETRIC:
		new_layout = ALGORITHM_RIGHT_ASYMMETRIC_6;
		break;
	case ALGORITHM_LEFT_SYMMETRIC:
		new_layout = ALGORITHM_LEFT_SYMMETRIC_6;
		break;
	case ALGORITHM_RIGHT_SYMMETRIC:
		new_layout = ALGORITHM_RIGHT_SYMMETRIC_6;
		break;
	case ALGORITHM_PARITY_0:
		new_layout = ALGORITHM_PARITY_0_6;
		break;
	case ALGORITHM_PARITY_N:
		new_layout = ALGORITHM_PARITY_N;
		break;
	default:
		return ERR_PTR(-EINVAL);
	}
	mddev->new_level = 6;
	mddev->new_layout = new_layout;
	mddev->delta_disks = 1;
	mddev->raid_disks += 1;
	return setup_conf(mddev);
}

7697
static struct md_personality raid6_personality =
7698 7699 7700 7701 7702 7703
{
	.name		= "raid6",
	.level		= 6,
	.owner		= THIS_MODULE,
	.make_request	= make_request,
	.run		= run,
N
NeilBrown 已提交
7704
	.free		= raid5_free,
7705 7706 7707 7708 7709 7710 7711
	.status		= status,
	.error_handler	= error,
	.hot_add_disk	= raid5_add_disk,
	.hot_remove_disk= raid5_remove_disk,
	.spare_active	= raid5_spare_active,
	.sync_request	= sync_request,
	.resize		= raid5_resize,
7712
	.size		= raid5_size,
7713
	.check_reshape	= raid6_check_reshape,
7714
	.start_reshape  = raid5_start_reshape,
7715
	.finish_reshape = raid5_finish_reshape,
7716
	.quiesce	= raid5_quiesce,
7717
	.takeover	= raid6_takeover,
7718
	.congested	= raid5_congested,
7719
	.mergeable_bvec	= raid5_mergeable_bvec,
7720
};
7721
static struct md_personality raid5_personality =
L
Linus Torvalds 已提交
7722 7723
{
	.name		= "raid5",
7724
	.level		= 5,
L
Linus Torvalds 已提交
7725 7726 7727
	.owner		= THIS_MODULE,
	.make_request	= make_request,
	.run		= run,
N
NeilBrown 已提交
7728
	.free		= raid5_free,
L
Linus Torvalds 已提交
7729 7730 7731 7732 7733 7734 7735
	.status		= status,
	.error_handler	= error,
	.hot_add_disk	= raid5_add_disk,
	.hot_remove_disk= raid5_remove_disk,
	.spare_active	= raid5_spare_active,
	.sync_request	= sync_request,
	.resize		= raid5_resize,
7736
	.size		= raid5_size,
7737 7738
	.check_reshape	= raid5_check_reshape,
	.start_reshape  = raid5_start_reshape,
7739
	.finish_reshape = raid5_finish_reshape,
7740
	.quiesce	= raid5_quiesce,
7741
	.takeover	= raid5_takeover,
7742
	.congested	= raid5_congested,
7743
	.mergeable_bvec	= raid5_mergeable_bvec,
L
Linus Torvalds 已提交
7744 7745
};

7746
static struct md_personality raid4_personality =
L
Linus Torvalds 已提交
7747
{
7748 7749 7750 7751 7752
	.name		= "raid4",
	.level		= 4,
	.owner		= THIS_MODULE,
	.make_request	= make_request,
	.run		= run,
N
NeilBrown 已提交
7753
	.free		= raid5_free,
7754 7755 7756 7757 7758 7759 7760
	.status		= status,
	.error_handler	= error,
	.hot_add_disk	= raid5_add_disk,
	.hot_remove_disk= raid5_remove_disk,
	.spare_active	= raid5_spare_active,
	.sync_request	= sync_request,
	.resize		= raid5_resize,
7761
	.size		= raid5_size,
7762 7763
	.check_reshape	= raid5_check_reshape,
	.start_reshape  = raid5_start_reshape,
7764
	.finish_reshape = raid5_finish_reshape,
7765
	.quiesce	= raid5_quiesce,
7766
	.takeover	= raid4_takeover,
7767
	.congested	= raid5_congested,
7768
	.mergeable_bvec	= raid5_mergeable_bvec,
7769 7770 7771 7772
};

static int __init raid5_init(void)
{
7773 7774 7775 7776
	raid5_wq = alloc_workqueue("raid5wq",
		WQ_UNBOUND|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE|WQ_SYSFS, 0);
	if (!raid5_wq)
		return -ENOMEM;
7777
	register_md_personality(&raid6_personality);
7778 7779 7780
	register_md_personality(&raid5_personality);
	register_md_personality(&raid4_personality);
	return 0;
L
Linus Torvalds 已提交
7781 7782
}

7783
static void raid5_exit(void)
L
Linus Torvalds 已提交
7784
{
7785
	unregister_md_personality(&raid6_personality);
7786 7787
	unregister_md_personality(&raid5_personality);
	unregister_md_personality(&raid4_personality);
7788
	destroy_workqueue(raid5_wq);
L
Linus Torvalds 已提交
7789 7790 7791 7792 7793
}

module_init(raid5_init);
module_exit(raid5_exit);
MODULE_LICENSE("GPL");
7794
MODULE_DESCRIPTION("RAID4/5/6 (striping with parity) personality for MD");
L
Linus Torvalds 已提交
7795
MODULE_ALIAS("md-personality-4"); /* RAID5 */
7796 7797
MODULE_ALIAS("md-raid5");
MODULE_ALIAS("md-raid4");
7798 7799
MODULE_ALIAS("md-level-5");
MODULE_ALIAS("md-level-4");
7800 7801 7802 7803 7804 7805 7806
MODULE_ALIAS("md-personality-8"); /* RAID6 */
MODULE_ALIAS("md-raid6");
MODULE_ALIAS("md-level-6");

/* This used to be two separate modules, they were: */
MODULE_ALIAS("raid5");
MODULE_ALIAS("raid6");