raid5.c 200.4 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4
/*
 * raid5.c : Multiple Devices driver for Linux
 *	   Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman
 *	   Copyright (C) 1999, 2000 Ingo Molnar
5
 *	   Copyright (C) 2002, 2003 H. Peter Anvin
L
Linus Torvalds 已提交
6
 *
7 8 9
 * RAID-4/5/6 management functions.
 * Thanks to Penguin Computing for making the RAID-6 development possible
 * by donating a test server!
L
Linus Torvalds 已提交
10 11 12 13 14 15 16 17 18 19 20
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2, or (at your option)
 * any later version.
 *
 * You should have received a copy of the GNU General Public License
 * (for example /usr/src/linux/COPYING); if not, write to the Free
 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

21 22 23 24 25 26 27 28 29
/*
 * BITMAP UNPLUGGING:
 *
 * The sequencing for updating the bitmap reliably is a little
 * subtle (and I got it wrong the first time) so it deserves some
 * explanation.
 *
 * We group bitmap updates into batches.  Each batch has a number.
 * We may write out several batches at once, but that isn't very important.
30 31
 * conf->seq_write is the number of the last batch successfully written.
 * conf->seq_flush is the number of the last batch that was closed to
32 33 34
 *    new additions.
 * When we discover that we will need to write to any block in a stripe
 * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq
35
 * the number of the batch it will be in. This is seq_flush+1.
36 37 38 39 40 41 42 43 44
 * When we are ready to do a write, if that batch hasn't been written yet,
 *   we plug the array and queue the stripe for later.
 * When an unplug happens, we increment bm_flush, thus closing the current
 *   batch.
 * When we notice that bm_flush > bm_write, we write out all pending updates
 * to the bitmap, and advance bm_write to where bm_flush was.
 * This may occasionally write a bit out twice, but is sure never to
 * miss any bits.
 */
L
Linus Torvalds 已提交
45

46
#include <linux/blkdev.h>
47
#include <linux/kthread.h>
48
#include <linux/raid/pq.h>
49
#include <linux/async_tx.h>
50
#include <linux/module.h>
51
#include <linux/async.h>
52
#include <linux/seq_file.h>
53
#include <linux/cpu.h>
54
#include <linux/slab.h>
55
#include <linux/ratelimit.h>
56
#include <linux/nodemask.h>
N
NeilBrown 已提交
57 58
#include <trace/events/block.h>

59
#include "md.h"
60
#include "raid5.h"
61
#include "raid0.h"
62
#include "bitmap.h"
63

64 65 66
#define cpu_to_group(cpu) cpu_to_node(cpu)
#define ANY_GROUP NUMA_NO_NODE

67 68 69 70
static bool devices_handle_discard_safely = false;
module_param(devices_handle_discard_safely, bool, 0644);
MODULE_PARM_DESC(devices_handle_discard_safely,
		 "Set to Y if all devices in each array reliably return zeroes on reads from discarded regions");
71
static struct workqueue_struct *raid5_wq;
L
Linus Torvalds 已提交
72 73 74 75 76 77 78 79 80
/*
 * Stripe cache
 */

#define NR_STRIPES		256
#define STRIPE_SIZE		PAGE_SIZE
#define STRIPE_SHIFT		(PAGE_SHIFT - 9)
#define STRIPE_SECTORS		(STRIPE_SIZE>>9)
#define	IO_THRESHOLD		1
81
#define BYPASS_THRESHOLD	1
82
#define NR_HASH			(PAGE_SIZE / sizeof(struct hlist_head))
L
Linus Torvalds 已提交
83
#define HASH_MASK		(NR_HASH - 1)
84
#define MAX_STRIPE_BATCH	8
L
Linus Torvalds 已提交
85

86
static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect)
87 88 89 90
{
	int hash = (sect >> STRIPE_SHIFT) & HASH_MASK;
	return &conf->stripe_hashtbl[hash];
}
L
Linus Torvalds 已提交
91

92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
static inline int stripe_hash_locks_hash(sector_t sect)
{
	return (sect >> STRIPE_SHIFT) & STRIPE_HASH_LOCKS_MASK;
}

static inline void lock_device_hash_lock(struct r5conf *conf, int hash)
{
	spin_lock_irq(conf->hash_locks + hash);
	spin_lock(&conf->device_lock);
}

static inline void unlock_device_hash_lock(struct r5conf *conf, int hash)
{
	spin_unlock(&conf->device_lock);
	spin_unlock_irq(conf->hash_locks + hash);
}

static inline void lock_all_device_hash_locks_irq(struct r5conf *conf)
{
	int i;
	local_irq_disable();
	spin_lock(conf->hash_locks);
	for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++)
		spin_lock_nest_lock(conf->hash_locks + i, conf->hash_locks);
	spin_lock(&conf->device_lock);
}

static inline void unlock_all_device_hash_locks_irq(struct r5conf *conf)
{
	int i;
	spin_unlock(&conf->device_lock);
	for (i = NR_STRIPE_HASH_LOCKS; i; i--)
		spin_unlock(conf->hash_locks + i - 1);
	local_irq_enable();
}

L
Linus Torvalds 已提交
128 129 130 131 132 133
/* bio's attached to a stripe+device for I/O are linked together in bi_sector
 * order without overlap.  There may be several bio's per stripe+device, and
 * a bio could span several devices.
 * When walking this list for a particular stripe+device, we must never proceed
 * beyond a bio that extends past this device, as the next bio might no longer
 * be valid.
134
 * This function is used to determine the 'next' bio in the list, given the sector
L
Linus Torvalds 已提交
135 136
 * of the current stripe+device
 */
137 138
static inline struct bio *r5_next_bio(struct bio *bio, sector_t sector)
{
139
	int sectors = bio_sectors(bio);
140
	if (bio->bi_iter.bi_sector + sectors < sector + STRIPE_SECTORS)
141 142 143 144
		return bio->bi_next;
	else
		return NULL;
}
L
Linus Torvalds 已提交
145

146
/*
147 148
 * We maintain a biased count of active stripes in the bottom 16 bits of
 * bi_phys_segments, and a count of processed stripes in the upper 16 bits
149
 */
150
static inline int raid5_bi_processed_stripes(struct bio *bio)
151
{
152 153
	atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
	return (atomic_read(segments) >> 16) & 0xffff;
154 155
}

156
static inline int raid5_dec_bi_active_stripes(struct bio *bio)
157
{
158 159
	atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
	return atomic_sub_return(1, segments) & 0xffff;
160 161
}

162
static inline void raid5_inc_bi_active_stripes(struct bio *bio)
163
{
164 165
	atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
	atomic_inc(segments);
166 167
}

168 169
static inline void raid5_set_bi_processed_stripes(struct bio *bio,
	unsigned int cnt)
170
{
171 172
	atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
	int old, new;
173

174 175 176 177
	do {
		old = atomic_read(segments);
		new = (old & 0xffff) | (cnt << 16);
	} while (atomic_cmpxchg(segments, old, new) != old);
178 179
}

180
static inline void raid5_set_bi_stripes(struct bio *bio, unsigned int cnt)
181
{
182 183
	atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
	atomic_set(segments, cnt);
184 185
}

186 187 188
/* Find first data disk in a raid6 stripe */
static inline int raid6_d0(struct stripe_head *sh)
{
189 190 191 192
	if (sh->ddf_layout)
		/* ddf always start from first device */
		return 0;
	/* md starts just after Q block */
193 194 195 196 197
	if (sh->qd_idx == sh->disks - 1)
		return 0;
	else
		return sh->qd_idx + 1;
}
198 199 200 201 202
static inline int raid6_next_disk(int disk, int raid_disks)
{
	disk++;
	return (disk < raid_disks) ? disk : 0;
}
203

204 205 206 207 208
/* When walking through the disks in a raid5, starting at raid6_d0,
 * We need to map each disk to a 'slot', where the data disks are slot
 * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk
 * is raid_disks-1.  This help does that mapping.
 */
209 210
static int raid6_idx_to_slot(int idx, struct stripe_head *sh,
			     int *count, int syndrome_disks)
211
{
212
	int slot = *count;
213

214
	if (sh->ddf_layout)
215
		(*count)++;
216
	if (idx == sh->pd_idx)
217
		return syndrome_disks;
218
	if (idx == sh->qd_idx)
219
		return syndrome_disks + 1;
220
	if (!sh->ddf_layout)
221
		(*count)++;
222 223 224
	return slot;
}

225 226 227 228 229 230 231
static void return_io(struct bio *return_bi)
{
	struct bio *bi = return_bi;
	while (bi) {

		return_bi = bi->bi_next;
		bi->bi_next = NULL;
232
		bi->bi_iter.bi_size = 0;
233 234
		trace_block_bio_complete(bdev_get_queue(bi->bi_bdev),
					 bi, 0);
235
		bio_endio(bi, 0);
236 237 238 239
		bi = return_bi;
	}
}

240
static void print_raid5_conf (struct r5conf *conf);
L
Linus Torvalds 已提交
241

242 243 244 245 246 247 248
static int stripe_operations_active(struct stripe_head *sh)
{
	return sh->check_state || sh->reconstruct_state ||
	       test_bit(STRIPE_BIOFILL_RUN, &sh->state) ||
	       test_bit(STRIPE_COMPUTE_RUN, &sh->state);
}

249 250 251 252
static void raid5_wakeup_stripe_thread(struct stripe_head *sh)
{
	struct r5conf *conf = sh->raid_conf;
	struct r5worker_group *group;
253
	int thread_cnt;
254 255 256 257 258 259 260 261 262 263 264
	int i, cpu = sh->cpu;

	if (!cpu_online(cpu)) {
		cpu = cpumask_any(cpu_online_mask);
		sh->cpu = cpu;
	}

	if (list_empty(&sh->lru)) {
		struct r5worker_group *group;
		group = conf->worker_groups + cpu_to_group(cpu);
		list_add_tail(&sh->lru, &group->handle_list);
265 266
		group->stripes_cnt++;
		sh->group = group;
267 268 269 270 271 272 273 274 275
	}

	if (conf->worker_cnt_per_group == 0) {
		md_wakeup_thread(conf->mddev->thread);
		return;
	}

	group = conf->worker_groups + cpu_to_group(sh->cpu);

276 277 278 279 280 281 282 283 284 285 286 287 288 289
	group->workers[0].working = true;
	/* at least one worker should run to avoid race */
	queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work);

	thread_cnt = group->stripes_cnt / MAX_STRIPE_BATCH - 1;
	/* wakeup more workers */
	for (i = 1; i < conf->worker_cnt_per_group && thread_cnt > 0; i++) {
		if (group->workers[i].working == false) {
			group->workers[i].working = true;
			queue_work_on(sh->cpu, raid5_wq,
				      &group->workers[i].work);
			thread_cnt--;
		}
	}
290 291
}

292 293
static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh,
			      struct list_head *temp_inactive_list)
L
Linus Torvalds 已提交
294
{
295 296 297 298
	BUG_ON(!list_empty(&sh->lru));
	BUG_ON(atomic_read(&conf->active_stripes)==0);
	if (test_bit(STRIPE_HANDLE, &sh->state)) {
		if (test_bit(STRIPE_DELAYED, &sh->state) &&
299
		    !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) {
300
			list_add_tail(&sh->lru, &conf->delayed_list);
301 302 303 304
			if (atomic_read(&conf->preread_active_stripes)
			    < IO_THRESHOLD)
				md_wakeup_thread(conf->mddev->thread);
		} else if (test_bit(STRIPE_BIT_DELAY, &sh->state) &&
305 306 307 308 309
			   sh->bm_seq - conf->seq_write > 0)
			list_add_tail(&sh->lru, &conf->bitmap_list);
		else {
			clear_bit(STRIPE_DELAYED, &sh->state);
			clear_bit(STRIPE_BIT_DELAY, &sh->state);
310 311 312 313 314 315
			if (conf->worker_cnt_per_group == 0) {
				list_add_tail(&sh->lru, &conf->handle_list);
			} else {
				raid5_wakeup_stripe_thread(sh);
				return;
			}
316 317 318 319 320 321 322 323 324
		}
		md_wakeup_thread(conf->mddev->thread);
	} else {
		BUG_ON(stripe_operations_active(sh));
		if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
			if (atomic_dec_return(&conf->preread_active_stripes)
			    < IO_THRESHOLD)
				md_wakeup_thread(conf->mddev->thread);
		atomic_dec(&conf->active_stripes);
325 326
		if (!test_bit(STRIPE_EXPANDING, &sh->state))
			list_add_tail(&sh->lru, temp_inactive_list);
L
Linus Torvalds 已提交
327 328
	}
}
329

330 331
static void __release_stripe(struct r5conf *conf, struct stripe_head *sh,
			     struct list_head *temp_inactive_list)
332 333
{
	if (atomic_dec_and_test(&sh->count))
334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365
		do_release_stripe(conf, sh, temp_inactive_list);
}

/*
 * @hash could be NR_STRIPE_HASH_LOCKS, then we have a list of inactive_list
 *
 * Be careful: Only one task can add/delete stripes from temp_inactive_list at
 * given time. Adding stripes only takes device lock, while deleting stripes
 * only takes hash lock.
 */
static void release_inactive_stripe_list(struct r5conf *conf,
					 struct list_head *temp_inactive_list,
					 int hash)
{
	int size;
	bool do_wakeup = false;
	unsigned long flags;

	if (hash == NR_STRIPE_HASH_LOCKS) {
		size = NR_STRIPE_HASH_LOCKS;
		hash = NR_STRIPE_HASH_LOCKS - 1;
	} else
		size = 1;
	while (size) {
		struct list_head *list = &temp_inactive_list[size - 1];

		/*
		 * We don't hold any lock here yet, get_active_stripe() might
		 * remove stripes from the list
		 */
		if (!list_empty_careful(list)) {
			spin_lock_irqsave(conf->hash_locks + hash, flags);
366 367 368
			if (list_empty(conf->inactive_list + hash) &&
			    !list_empty(list))
				atomic_dec(&conf->empty_inactive_list_nr);
369 370 371 372 373 374 375 376 377 378 379 380 381
			list_splice_tail_init(list, conf->inactive_list + hash);
			do_wakeup = true;
			spin_unlock_irqrestore(conf->hash_locks + hash, flags);
		}
		size--;
		hash--;
	}

	if (do_wakeup) {
		wake_up(&conf->wait_for_stripe);
		if (conf->retry_read_aligned)
			md_wakeup_thread(conf->mddev->thread);
	}
382 383
}

S
Shaohua Li 已提交
384
/* should hold conf->device_lock already */
385 386
static int release_stripe_list(struct r5conf *conf,
			       struct list_head *temp_inactive_list)
S
Shaohua Li 已提交
387 388 389 390 391 392
{
	struct stripe_head *sh;
	int count = 0;
	struct llist_node *head;

	head = llist_del_all(&conf->released_stripes);
S
Shaohua Li 已提交
393
	head = llist_reverse_order(head);
S
Shaohua Li 已提交
394
	while (head) {
395 396
		int hash;

S
Shaohua Li 已提交
397 398 399 400 401 402 403 404 405 406
		sh = llist_entry(head, struct stripe_head, release_list);
		head = llist_next(head);
		/* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */
		smp_mb();
		clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state);
		/*
		 * Don't worry the bit is set here, because if the bit is set
		 * again, the count is always > 1. This is true for
		 * STRIPE_ON_UNPLUG_LIST bit too.
		 */
407 408
		hash = sh->hash_lock_index;
		__release_stripe(conf, sh, &temp_inactive_list[hash]);
S
Shaohua Li 已提交
409 410 411 412 413 414
		count++;
	}

	return count;
}

L
Linus Torvalds 已提交
415 416
static void release_stripe(struct stripe_head *sh)
{
417
	struct r5conf *conf = sh->raid_conf;
L
Linus Torvalds 已提交
418
	unsigned long flags;
419 420
	struct list_head list;
	int hash;
S
Shaohua Li 已提交
421
	bool wakeup;
422

423 424 425 426 427
	/* Avoid release_list until the last reference.
	 */
	if (atomic_add_unless(&sh->count, -1, 1))
		return;

428 429
	if (unlikely(!conf->mddev->thread) ||
		test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state))
S
Shaohua Li 已提交
430 431 432 433 434 435
		goto slow_path;
	wakeup = llist_add(&sh->release_list, &conf->released_stripes);
	if (wakeup)
		md_wakeup_thread(conf->mddev->thread);
	return;
slow_path:
436
	local_irq_save(flags);
S
Shaohua Li 已提交
437
	/* we are ok here if STRIPE_ON_RELEASE_LIST is set or not */
438
	if (atomic_dec_and_lock(&sh->count, &conf->device_lock)) {
439 440 441
		INIT_LIST_HEAD(&list);
		hash = sh->hash_lock_index;
		do_release_stripe(conf, sh, &list);
442
		spin_unlock(&conf->device_lock);
443
		release_inactive_stripe_list(conf, &list, hash);
444 445
	}
	local_irq_restore(flags);
L
Linus Torvalds 已提交
446 447
}

448
static inline void remove_hash(struct stripe_head *sh)
L
Linus Torvalds 已提交
449
{
450 451
	pr_debug("remove_hash(), stripe %llu\n",
		(unsigned long long)sh->sector);
L
Linus Torvalds 已提交
452

453
	hlist_del_init(&sh->hash);
L
Linus Torvalds 已提交
454 455
}

456
static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh)
L
Linus Torvalds 已提交
457
{
458
	struct hlist_head *hp = stripe_hash(conf, sh->sector);
L
Linus Torvalds 已提交
459

460 461
	pr_debug("insert_hash(), stripe %llu\n",
		(unsigned long long)sh->sector);
L
Linus Torvalds 已提交
462

463
	hlist_add_head(&sh->hash, hp);
L
Linus Torvalds 已提交
464 465 466 467
}


/* find an idle stripe, make sure it is unhashed, and return it. */
468
static struct stripe_head *get_free_stripe(struct r5conf *conf, int hash)
L
Linus Torvalds 已提交
469 470 471 472
{
	struct stripe_head *sh = NULL;
	struct list_head *first;

473
	if (list_empty(conf->inactive_list + hash))
L
Linus Torvalds 已提交
474
		goto out;
475
	first = (conf->inactive_list + hash)->next;
L
Linus Torvalds 已提交
476 477 478 479
	sh = list_entry(first, struct stripe_head, lru);
	list_del_init(first);
	remove_hash(sh);
	atomic_inc(&conf->active_stripes);
480
	BUG_ON(hash != sh->hash_lock_index);
481 482
	if (list_empty(conf->inactive_list + hash))
		atomic_inc(&conf->empty_inactive_list_nr);
L
Linus Torvalds 已提交
483 484 485 486
out:
	return sh;
}

487
static void shrink_buffers(struct stripe_head *sh)
L
Linus Torvalds 已提交
488 489 490
{
	struct page *p;
	int i;
491
	int num = sh->raid_conf->pool_size;
L
Linus Torvalds 已提交
492

493
	for (i = 0; i < num ; i++) {
494
		WARN_ON(sh->dev[i].page != sh->dev[i].orig_page);
L
Linus Torvalds 已提交
495 496 497 498
		p = sh->dev[i].page;
		if (!p)
			continue;
		sh->dev[i].page = NULL;
499
		put_page(p);
L
Linus Torvalds 已提交
500 501 502
	}
}

503
static int grow_buffers(struct stripe_head *sh)
L
Linus Torvalds 已提交
504 505
{
	int i;
506
	int num = sh->raid_conf->pool_size;
L
Linus Torvalds 已提交
507

508
	for (i = 0; i < num; i++) {
L
Linus Torvalds 已提交
509 510 511 512 513 514
		struct page *page;

		if (!(page = alloc_page(GFP_KERNEL))) {
			return 1;
		}
		sh->dev[i].page = page;
515
		sh->dev[i].orig_page = page;
L
Linus Torvalds 已提交
516 517 518 519
	}
	return 0;
}

520
static void raid5_build_block(struct stripe_head *sh, int i, int previous);
521
static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,
522
			    struct stripe_head *sh);
L
Linus Torvalds 已提交
523

524
static void init_stripe(struct stripe_head *sh, sector_t sector, int previous)
L
Linus Torvalds 已提交
525
{
526
	struct r5conf *conf = sh->raid_conf;
527
	int i, seq;
L
Linus Torvalds 已提交
528

529 530
	BUG_ON(atomic_read(&sh->count) != 0);
	BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));
531
	BUG_ON(stripe_operations_active(sh));
532

533
	pr_debug("init_stripe called, stripe %llu\n",
L
Linus Torvalds 已提交
534 535 536
		(unsigned long long)sh->sector);

	remove_hash(sh);
537 538
retry:
	seq = read_seqcount_begin(&conf->gen_lock);
539
	sh->generation = conf->generation - previous;
540
	sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks;
L
Linus Torvalds 已提交
541
	sh->sector = sector;
542
	stripe_set_idx(sector, conf, previous, sh);
L
Linus Torvalds 已提交
543 544
	sh->state = 0;

545 546

	for (i = sh->disks; i--; ) {
L
Linus Torvalds 已提交
547 548
		struct r5dev *dev = &sh->dev[i];

549
		if (dev->toread || dev->read || dev->towrite || dev->written ||
L
Linus Torvalds 已提交
550
		    test_bit(R5_LOCKED, &dev->flags)) {
551
			printk(KERN_ERR "sector=%llx i=%d %p %p %p %p %d\n",
L
Linus Torvalds 已提交
552
			       (unsigned long long)sh->sector, i, dev->toread,
553
			       dev->read, dev->towrite, dev->written,
L
Linus Torvalds 已提交
554
			       test_bit(R5_LOCKED, &dev->flags));
555
			WARN_ON(1);
L
Linus Torvalds 已提交
556 557
		}
		dev->flags = 0;
558
		raid5_build_block(sh, i, previous);
L
Linus Torvalds 已提交
559
	}
560 561
	if (read_seqcount_retry(&conf->gen_lock, seq))
		goto retry;
L
Linus Torvalds 已提交
562
	insert_hash(conf, sh);
563
	sh->cpu = smp_processor_id();
L
Linus Torvalds 已提交
564 565
}

566
static struct stripe_head *__find_stripe(struct r5conf *conf, sector_t sector,
567
					 short generation)
L
Linus Torvalds 已提交
568 569 570
{
	struct stripe_head *sh;

571
	pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector);
572
	hlist_for_each_entry(sh, stripe_hash(conf, sector), hash)
573
		if (sh->sector == sector && sh->generation == generation)
L
Linus Torvalds 已提交
574
			return sh;
575
	pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector);
L
Linus Torvalds 已提交
576 577 578
	return NULL;
}

579 580 581 582 583 584 585 586 587 588 589 590 591
/*
 * Need to check if array has failed when deciding whether to:
 *  - start an array
 *  - remove non-faulty devices
 *  - add a spare
 *  - allow a reshape
 * This determination is simple when no reshape is happening.
 * However if there is a reshape, we need to carefully check
 * both the before and after sections.
 * This is because some failed devices may only affect one
 * of the two sections, and some non-in_sync devices may
 * be insync in the section most affected by failed devices.
 */
592
static int calc_degraded(struct r5conf *conf)
593
{
594
	int degraded, degraded2;
595 596 597 598 599
	int i;

	rcu_read_lock();
	degraded = 0;
	for (i = 0; i < conf->previous_raid_disks; i++) {
600
		struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
601 602
		if (rdev && test_bit(Faulty, &rdev->flags))
			rdev = rcu_dereference(conf->disks[i].replacement);
603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620
		if (!rdev || test_bit(Faulty, &rdev->flags))
			degraded++;
		else if (test_bit(In_sync, &rdev->flags))
			;
		else
			/* not in-sync or faulty.
			 * If the reshape increases the number of devices,
			 * this is being recovered by the reshape, so
			 * this 'previous' section is not in_sync.
			 * If the number of devices is being reduced however,
			 * the device can only be part of the array if
			 * we are reverting a reshape, so this section will
			 * be in-sync.
			 */
			if (conf->raid_disks >= conf->previous_raid_disks)
				degraded++;
	}
	rcu_read_unlock();
621 622
	if (conf->raid_disks == conf->previous_raid_disks)
		return degraded;
623
	rcu_read_lock();
624
	degraded2 = 0;
625
	for (i = 0; i < conf->raid_disks; i++) {
626
		struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
627 628
		if (rdev && test_bit(Faulty, &rdev->flags))
			rdev = rcu_dereference(conf->disks[i].replacement);
629
		if (!rdev || test_bit(Faulty, &rdev->flags))
630
			degraded2++;
631 632 633 634 635 636 637 638 639
		else if (test_bit(In_sync, &rdev->flags))
			;
		else
			/* not in-sync or faulty.
			 * If reshape increases the number of devices, this
			 * section has already been recovered, else it
			 * almost certainly hasn't.
			 */
			if (conf->raid_disks <= conf->previous_raid_disks)
640
				degraded2++;
641 642
	}
	rcu_read_unlock();
643 644 645 646 647 648 649 650 651 652 653 654 655
	if (degraded2 > degraded)
		return degraded2;
	return degraded;
}

static int has_failed(struct r5conf *conf)
{
	int degraded;

	if (conf->mddev->reshape_position == MaxSector)
		return conf->mddev->degraded > conf->max_degraded;

	degraded = calc_degraded(conf);
656 657 658 659 660
	if (degraded > conf->max_degraded)
		return 1;
	return 0;
}

661
static struct stripe_head *
662
get_active_stripe(struct r5conf *conf, sector_t sector,
663
		  int previous, int noblock, int noquiesce)
L
Linus Torvalds 已提交
664 665
{
	struct stripe_head *sh;
666
	int hash = stripe_hash_locks_hash(sector);
L
Linus Torvalds 已提交
667

668
	pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector);
L
Linus Torvalds 已提交
669

670
	spin_lock_irq(conf->hash_locks + hash);
L
Linus Torvalds 已提交
671 672

	do {
673
		wait_event_lock_irq(conf->wait_for_stripe,
674
				    conf->quiesce == 0 || noquiesce,
675
				    *(conf->hash_locks + hash));
676
		sh = __find_stripe(conf, sector, conf->generation - previous);
L
Linus Torvalds 已提交
677 678
		if (!sh) {
			if (!conf->inactive_blocked)
679
				sh = get_free_stripe(conf, hash);
L
Linus Torvalds 已提交
680 681 682 683
			if (noblock && sh == NULL)
				break;
			if (!sh) {
				conf->inactive_blocked = 1;
684 685 686 687 688 689 690
				wait_event_lock_irq(
					conf->wait_for_stripe,
					!list_empty(conf->inactive_list + hash) &&
					(atomic_read(&conf->active_stripes)
					 < (conf->max_nr_stripes * 3 / 4)
					 || !conf->inactive_blocked),
					*(conf->hash_locks + hash));
L
Linus Torvalds 已提交
691
				conf->inactive_blocked = 0;
692
			} else {
693
				init_stripe(sh, sector, previous);
694 695
				atomic_inc(&sh->count);
			}
696
		} else if (!atomic_inc_not_zero(&sh->count)) {
697
			spin_lock(&conf->device_lock);
698
			if (!atomic_read(&sh->count)) {
L
Linus Torvalds 已提交
699 700
				if (!test_bit(STRIPE_HANDLE, &sh->state))
					atomic_inc(&conf->active_stripes);
701 702
				BUG_ON(list_empty(&sh->lru) &&
				       !test_bit(STRIPE_EXPANDING, &sh->state));
703
				list_del_init(&sh->lru);
704 705 706 707
				if (sh->group) {
					sh->group->stripes_cnt--;
					sh->group = NULL;
				}
L
Linus Torvalds 已提交
708
			}
709
			atomic_inc(&sh->count);
710
			spin_unlock(&conf->device_lock);
L
Linus Torvalds 已提交
711 712 713
		}
	} while (sh == NULL);

714
	spin_unlock_irq(conf->hash_locks + hash);
L
Linus Torvalds 已提交
715 716 717
	return sh;
}

718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738
/* Determine if 'data_offset' or 'new_data_offset' should be used
 * in this stripe_head.
 */
static int use_new_offset(struct r5conf *conf, struct stripe_head *sh)
{
	sector_t progress = conf->reshape_progress;
	/* Need a memory barrier to make sure we see the value
	 * of conf->generation, or ->data_offset that was set before
	 * reshape_progress was updated.
	 */
	smp_rmb();
	if (progress == MaxSector)
		return 0;
	if (sh->generation == conf->generation - 1)
		return 0;
	/* We are in a reshape, and this is a new-generation stripe,
	 * so use new_data_offset.
	 */
	return 1;
}

739 740 741 742
static void
raid5_end_read_request(struct bio *bi, int error);
static void
raid5_end_write_request(struct bio *bi, int error);
743

744
static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)
745
{
746
	struct r5conf *conf = sh->raid_conf;
747 748 749 750 751 752
	int i, disks = sh->disks;

	might_sleep();

	for (i = disks; i--; ) {
		int rw;
753
		int replace_only = 0;
754 755
		struct bio *bi, *rbi;
		struct md_rdev *rdev, *rrdev = NULL;
T
Tejun Heo 已提交
756 757 758 759 760
		if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) {
			if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags))
				rw = WRITE_FUA;
			else
				rw = WRITE;
761
			if (test_bit(R5_Discard, &sh->dev[i].flags))
S
Shaohua Li 已提交
762
				rw |= REQ_DISCARD;
T
Tejun Heo 已提交
763
		} else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags))
764
			rw = READ;
765 766 767 768 769
		else if (test_and_clear_bit(R5_WantReplace,
					    &sh->dev[i].flags)) {
			rw = WRITE;
			replace_only = 1;
		} else
770
			continue;
S
Shaohua Li 已提交
771 772
		if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags))
			rw |= REQ_SYNC;
773 774

		bi = &sh->dev[i].req;
775
		rbi = &sh->dev[i].rreq; /* For writing to replacement */
776 777

		rcu_read_lock();
778
		rrdev = rcu_dereference(conf->disks[i].replacement);
779 780 781 782 783 784
		smp_mb(); /* Ensure that if rrdev is NULL, rdev won't be */
		rdev = rcu_dereference(conf->disks[i].rdev);
		if (!rdev) {
			rdev = rrdev;
			rrdev = NULL;
		}
785 786 787
		if (rw & WRITE) {
			if (replace_only)
				rdev = NULL;
788 789 790
			if (rdev == rrdev)
				/* We raced and saw duplicates */
				rrdev = NULL;
791
		} else {
792
			if (test_bit(R5_ReadRepl, &sh->dev[i].flags) && rrdev)
793 794 795
				rdev = rrdev;
			rrdev = NULL;
		}
796

797 798 799 800
		if (rdev && test_bit(Faulty, &rdev->flags))
			rdev = NULL;
		if (rdev)
			atomic_inc(&rdev->nr_pending);
801 802 803 804
		if (rrdev && test_bit(Faulty, &rrdev->flags))
			rrdev = NULL;
		if (rrdev)
			atomic_inc(&rrdev->nr_pending);
805 806
		rcu_read_unlock();

807
		/* We have already checked bad blocks for reads.  Now
808 809
		 * need to check for writes.  We never accept write errors
		 * on the replacement, so we don't to check rrdev.
810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829
		 */
		while ((rw & WRITE) && rdev &&
		       test_bit(WriteErrorSeen, &rdev->flags)) {
			sector_t first_bad;
			int bad_sectors;
			int bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS,
					      &first_bad, &bad_sectors);
			if (!bad)
				break;

			if (bad < 0) {
				set_bit(BlockedBadBlocks, &rdev->flags);
				if (!conf->mddev->external &&
				    conf->mddev->flags) {
					/* It is very unlikely, but we might
					 * still need to write out the
					 * bad block log - better give it
					 * a chance*/
					md_check_recovery(conf->mddev);
				}
830 831 832 833 834 835
				/*
				 * Because md_wait_for_blocked_rdev
				 * will dec nr_pending, we must
				 * increment it first.
				 */
				atomic_inc(&rdev->nr_pending);
836 837 838 839 840 841 842 843
				md_wait_for_blocked_rdev(rdev, conf->mddev);
			} else {
				/* Acknowledged bad block - skip the write */
				rdev_dec_pending(rdev, conf->mddev);
				rdev = NULL;
			}
		}

844
		if (rdev) {
845 846
			if (s->syncing || s->expanding || s->expanded
			    || s->replacing)
847 848
				md_sync_acct(rdev->bdev, STRIPE_SECTORS);

D
Dan Williams 已提交
849 850
			set_bit(STRIPE_IO_STARTED, &sh->state);

K
Kent Overstreet 已提交
851
			bio_reset(bi);
852
			bi->bi_bdev = rdev->bdev;
K
Kent Overstreet 已提交
853 854 855 856 857 858
			bi->bi_rw = rw;
			bi->bi_end_io = (rw & WRITE)
				? raid5_end_write_request
				: raid5_end_read_request;
			bi->bi_private = sh;

859
			pr_debug("%s: for %llu schedule op %ld on disc %d\n",
860
				__func__, (unsigned long long)sh->sector,
861 862
				bi->bi_rw, i);
			atomic_inc(&sh->count);
863
			if (use_new_offset(conf, sh))
864
				bi->bi_iter.bi_sector = (sh->sector
865 866
						 + rdev->new_data_offset);
			else
867
				bi->bi_iter.bi_sector = (sh->sector
868
						 + rdev->data_offset);
869
			if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
870
				bi->bi_rw |= REQ_NOMERGE;
871

872 873 874
			if (test_bit(R5_SkipCopy, &sh->dev[i].flags))
				WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
			sh->dev[i].vec.bv_page = sh->dev[i].page;
K
Kent Overstreet 已提交
875
			bi->bi_vcnt = 1;
876 877
			bi->bi_io_vec[0].bv_len = STRIPE_SIZE;
			bi->bi_io_vec[0].bv_offset = 0;
878
			bi->bi_iter.bi_size = STRIPE_SIZE;
879 880 881 882 883 884
			/*
			 * If this is discard request, set bi_vcnt 0. We don't
			 * want to confuse SCSI because SCSI will replace payload
			 */
			if (rw & REQ_DISCARD)
				bi->bi_vcnt = 0;
885 886
			if (rrdev)
				set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags);
887 888 889 890 891

			if (conf->mddev->gendisk)
				trace_block_bio_remap(bdev_get_queue(bi->bi_bdev),
						      bi, disk_devt(conf->mddev->gendisk),
						      sh->dev[i].sector);
892
			generic_make_request(bi);
893 894
		}
		if (rrdev) {
895 896
			if (s->syncing || s->expanding || s->expanded
			    || s->replacing)
897 898 899 900
				md_sync_acct(rrdev->bdev, STRIPE_SECTORS);

			set_bit(STRIPE_IO_STARTED, &sh->state);

K
Kent Overstreet 已提交
901
			bio_reset(rbi);
902
			rbi->bi_bdev = rrdev->bdev;
K
Kent Overstreet 已提交
903 904 905 906 907
			rbi->bi_rw = rw;
			BUG_ON(!(rw & WRITE));
			rbi->bi_end_io = raid5_end_write_request;
			rbi->bi_private = sh;

908 909 910 911 912
			pr_debug("%s: for %llu schedule op %ld on "
				 "replacement disc %d\n",
				__func__, (unsigned long long)sh->sector,
				rbi->bi_rw, i);
			atomic_inc(&sh->count);
913
			if (use_new_offset(conf, sh))
914
				rbi->bi_iter.bi_sector = (sh->sector
915 916
						  + rrdev->new_data_offset);
			else
917
				rbi->bi_iter.bi_sector = (sh->sector
918
						  + rrdev->data_offset);
919 920 921
			if (test_bit(R5_SkipCopy, &sh->dev[i].flags))
				WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
			sh->dev[i].rvec.bv_page = sh->dev[i].page;
K
Kent Overstreet 已提交
922
			rbi->bi_vcnt = 1;
923 924
			rbi->bi_io_vec[0].bv_len = STRIPE_SIZE;
			rbi->bi_io_vec[0].bv_offset = 0;
925
			rbi->bi_iter.bi_size = STRIPE_SIZE;
926 927 928 929 930 931
			/*
			 * If this is discard request, set bi_vcnt 0. We don't
			 * want to confuse SCSI because SCSI will replace payload
			 */
			if (rw & REQ_DISCARD)
				rbi->bi_vcnt = 0;
932 933 934 935
			if (conf->mddev->gendisk)
				trace_block_bio_remap(bdev_get_queue(rbi->bi_bdev),
						      rbi, disk_devt(conf->mddev->gendisk),
						      sh->dev[i].sector);
936 937 938
			generic_make_request(rbi);
		}
		if (!rdev && !rrdev) {
939
			if (rw & WRITE)
940 941 942 943 944 945 946 947 948 949
				set_bit(STRIPE_DEGRADED, &sh->state);
			pr_debug("skip op %ld on disc %d for sector %llu\n",
				bi->bi_rw, i, (unsigned long long)sh->sector);
			clear_bit(R5_LOCKED, &sh->dev[i].flags);
			set_bit(STRIPE_HANDLE, &sh->state);
		}
	}
}

static struct dma_async_tx_descriptor *
950 951 952
async_copy_data(int frombio, struct bio *bio, struct page **page,
	sector_t sector, struct dma_async_tx_descriptor *tx,
	struct stripe_head *sh)
953
{
954 955
	struct bio_vec bvl;
	struct bvec_iter iter;
956 957
	struct page *bio_page;
	int page_offset;
958
	struct async_submit_ctl submit;
D
Dan Williams 已提交
959
	enum async_tx_flags flags = 0;
960

961 962
	if (bio->bi_iter.bi_sector >= sector)
		page_offset = (signed)(bio->bi_iter.bi_sector - sector) * 512;
963
	else
964
		page_offset = (signed)(sector - bio->bi_iter.bi_sector) * -512;
965

D
Dan Williams 已提交
966 967 968 969
	if (frombio)
		flags |= ASYNC_TX_FENCE;
	init_async_submit(&submit, flags, tx, NULL, NULL, NULL);

970 971
	bio_for_each_segment(bvl, bio, iter) {
		int len = bvl.bv_len;
972 973 974 975 976 977 978 979 980 981 982 983 984 985 986
		int clen;
		int b_offset = 0;

		if (page_offset < 0) {
			b_offset = -page_offset;
			page_offset += b_offset;
			len -= b_offset;
		}

		if (len > 0 && page_offset + len > STRIPE_SIZE)
			clen = STRIPE_SIZE - page_offset;
		else
			clen = len;

		if (clen > 0) {
987 988
			b_offset += bvl.bv_offset;
			bio_page = bvl.bv_page;
989 990 991 992 993 994 995
			if (frombio) {
				if (sh->raid_conf->skip_copy &&
				    b_offset == 0 && page_offset == 0 &&
				    clen == STRIPE_SIZE)
					*page = bio_page;
				else
					tx = async_memcpy(*page, bio_page, page_offset,
996
						  b_offset, clen, &submit);
997 998
			} else
				tx = async_memcpy(bio_page, *page, b_offset,
999
						  page_offset, clen, &submit);
1000
		}
1001 1002 1003
		/* chain the operations */
		submit.depend_tx = tx;

1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015
		if (clen < len) /* hit end of page */
			break;
		page_offset +=  len;
	}

	return tx;
}

static void ops_complete_biofill(void *stripe_head_ref)
{
	struct stripe_head *sh = stripe_head_ref;
	struct bio *return_bi = NULL;
1016
	int i;
1017

1018
	pr_debug("%s: stripe %llu\n", __func__,
1019 1020 1021 1022 1023 1024 1025
		(unsigned long long)sh->sector);

	/* clear completed biofills */
	for (i = sh->disks; i--; ) {
		struct r5dev *dev = &sh->dev[i];

		/* acknowledge completion of a biofill operation */
1026 1027
		/* and check if we need to reply to a read request,
		 * new R5_Wantfill requests are held off until
1028
		 * !STRIPE_BIOFILL_RUN
1029 1030
		 */
		if (test_and_clear_bit(R5_Wantfill, &dev->flags)) {
1031 1032 1033 1034 1035
			struct bio *rbi, *rbi2;

			BUG_ON(!dev->read);
			rbi = dev->read;
			dev->read = NULL;
1036
			while (rbi && rbi->bi_iter.bi_sector <
1037 1038
				dev->sector + STRIPE_SECTORS) {
				rbi2 = r5_next_bio(rbi, dev->sector);
1039
				if (!raid5_dec_bi_active_stripes(rbi)) {
1040 1041 1042 1043 1044 1045 1046
					rbi->bi_next = return_bi;
					return_bi = rbi;
				}
				rbi = rbi2;
			}
		}
	}
1047
	clear_bit(STRIPE_BIOFILL_RUN, &sh->state);
1048 1049 1050

	return_io(return_bi);

1051
	set_bit(STRIPE_HANDLE, &sh->state);
1052 1053 1054 1055 1056 1057
	release_stripe(sh);
}

static void ops_run_biofill(struct stripe_head *sh)
{
	struct dma_async_tx_descriptor *tx = NULL;
1058
	struct async_submit_ctl submit;
1059 1060
	int i;

1061
	pr_debug("%s: stripe %llu\n", __func__,
1062 1063 1064 1065 1066 1067
		(unsigned long long)sh->sector);

	for (i = sh->disks; i--; ) {
		struct r5dev *dev = &sh->dev[i];
		if (test_bit(R5_Wantfill, &dev->flags)) {
			struct bio *rbi;
S
Shaohua Li 已提交
1068
			spin_lock_irq(&sh->stripe_lock);
1069 1070
			dev->read = rbi = dev->toread;
			dev->toread = NULL;
S
Shaohua Li 已提交
1071
			spin_unlock_irq(&sh->stripe_lock);
1072
			while (rbi && rbi->bi_iter.bi_sector <
1073
				dev->sector + STRIPE_SECTORS) {
1074 1075
				tx = async_copy_data(0, rbi, &dev->page,
					dev->sector, tx, sh);
1076 1077 1078 1079 1080 1081
				rbi = r5_next_bio(rbi, dev->sector);
			}
		}
	}

	atomic_inc(&sh->count);
1082 1083
	init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL);
	async_trigger_callback(&submit);
1084 1085
}

1086
static void mark_target_uptodate(struct stripe_head *sh, int target)
1087
{
1088
	struct r5dev *tgt;
1089

1090 1091
	if (target < 0)
		return;
1092

1093
	tgt = &sh->dev[target];
1094 1095 1096
	set_bit(R5_UPTODATE, &tgt->flags);
	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
	clear_bit(R5_Wantcompute, &tgt->flags);
1097 1098
}

1099
static void ops_complete_compute(void *stripe_head_ref)
1100 1101 1102
{
	struct stripe_head *sh = stripe_head_ref;

1103
	pr_debug("%s: stripe %llu\n", __func__,
1104 1105
		(unsigned long long)sh->sector);

1106
	/* mark the computed target(s) as uptodate */
1107
	mark_target_uptodate(sh, sh->ops.target);
1108
	mark_target_uptodate(sh, sh->ops.target2);
1109

1110 1111 1112
	clear_bit(STRIPE_COMPUTE_RUN, &sh->state);
	if (sh->check_state == check_state_compute_run)
		sh->check_state = check_state_compute_result;
1113 1114 1115 1116
	set_bit(STRIPE_HANDLE, &sh->state);
	release_stripe(sh);
}

1117 1118 1119 1120 1121 1122 1123 1124 1125
/* return a pointer to the address conversion region of the scribble buffer */
static addr_conv_t *to_addr_conv(struct stripe_head *sh,
				 struct raid5_percpu *percpu)
{
	return percpu->scribble + sizeof(struct page *) * (sh->disks + 2);
}

static struct dma_async_tx_descriptor *
ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)
1126 1127
{
	int disks = sh->disks;
1128
	struct page **xor_srcs = percpu->scribble;
1129 1130 1131 1132 1133
	int target = sh->ops.target;
	struct r5dev *tgt = &sh->dev[target];
	struct page *xor_dest = tgt->page;
	int count = 0;
	struct dma_async_tx_descriptor *tx;
1134
	struct async_submit_ctl submit;
1135 1136 1137
	int i;

	pr_debug("%s: stripe %llu block: %d\n",
1138
		__func__, (unsigned long long)sh->sector, target);
1139 1140 1141 1142 1143 1144 1145 1146
	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));

	for (i = disks; i--; )
		if (i != target)
			xor_srcs[count++] = sh->dev[i].page;

	atomic_inc(&sh->count);

D
Dan Williams 已提交
1147
	init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL,
1148
			  ops_complete_compute, sh, to_addr_conv(sh, percpu));
1149
	if (unlikely(count == 1))
1150
		tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
1151
	else
1152
		tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
1153 1154 1155 1156

	return tx;
}

1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174
/* set_syndrome_sources - populate source buffers for gen_syndrome
 * @srcs - (struct page *) array of size sh->disks
 * @sh - stripe_head to parse
 *
 * Populates srcs in proper layout order for the stripe and returns the
 * 'count' of sources to be used in a call to async_gen_syndrome.  The P
 * destination buffer is recorded in srcs[count] and the Q destination
 * is recorded in srcs[count+1]].
 */
static int set_syndrome_sources(struct page **srcs, struct stripe_head *sh)
{
	int disks = sh->disks;
	int syndrome_disks = sh->ddf_layout ? disks : (disks - 2);
	int d0_idx = raid6_d0(sh);
	int count;
	int i;

	for (i = 0; i < disks; i++)
1175
		srcs[i] = NULL;
1176 1177 1178 1179 1180 1181 1182 1183 1184 1185

	count = 0;
	i = d0_idx;
	do {
		int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);

		srcs[slot] = sh->dev[i].page;
		i = raid6_next_disk(i, disks);
	} while (i != d0_idx);

1186
	return syndrome_disks;
1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206
}

static struct dma_async_tx_descriptor *
ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)
{
	int disks = sh->disks;
	struct page **blocks = percpu->scribble;
	int target;
	int qd_idx = sh->qd_idx;
	struct dma_async_tx_descriptor *tx;
	struct async_submit_ctl submit;
	struct r5dev *tgt;
	struct page *dest;
	int i;
	int count;

	if (sh->ops.target < 0)
		target = sh->ops.target2;
	else if (sh->ops.target2 < 0)
		target = sh->ops.target;
1207
	else
1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223
		/* we should only have one valid target */
		BUG();
	BUG_ON(target < 0);
	pr_debug("%s: stripe %llu block: %d\n",
		__func__, (unsigned long long)sh->sector, target);

	tgt = &sh->dev[target];
	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
	dest = tgt->page;

	atomic_inc(&sh->count);

	if (target == qd_idx) {
		count = set_syndrome_sources(blocks, sh);
		blocks[count] = NULL; /* regenerating p is not necessary */
		BUG_ON(blocks[count+1] != dest); /* q should already be set */
D
Dan Williams 已提交
1224 1225
		init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
				  ops_complete_compute, sh,
1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236
				  to_addr_conv(sh, percpu));
		tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
	} else {
		/* Compute any data- or p-drive using XOR */
		count = 0;
		for (i = disks; i-- ; ) {
			if (i == target || i == qd_idx)
				continue;
			blocks[count++] = sh->dev[i].page;
		}

D
Dan Williams 已提交
1237 1238
		init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
				  NULL, ops_complete_compute, sh,
1239 1240 1241
				  to_addr_conv(sh, percpu));
		tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit);
	}
1242 1243 1244 1245

	return tx;
}

1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266
static struct dma_async_tx_descriptor *
ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
{
	int i, count, disks = sh->disks;
	int syndrome_disks = sh->ddf_layout ? disks : disks-2;
	int d0_idx = raid6_d0(sh);
	int faila = -1, failb = -1;
	int target = sh->ops.target;
	int target2 = sh->ops.target2;
	struct r5dev *tgt = &sh->dev[target];
	struct r5dev *tgt2 = &sh->dev[target2];
	struct dma_async_tx_descriptor *tx;
	struct page **blocks = percpu->scribble;
	struct async_submit_ctl submit;

	pr_debug("%s: stripe %llu block1: %d block2: %d\n",
		 __func__, (unsigned long long)sh->sector, target, target2);
	BUG_ON(target < 0 || target2 < 0);
	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
	BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags));

1267
	/* we need to open-code set_syndrome_sources to handle the
1268 1269 1270
	 * slot number conversion for 'faila' and 'failb'
	 */
	for (i = 0; i < disks ; i++)
1271
		blocks[i] = NULL;
1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297
	count = 0;
	i = d0_idx;
	do {
		int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);

		blocks[slot] = sh->dev[i].page;

		if (i == target)
			faila = slot;
		if (i == target2)
			failb = slot;
		i = raid6_next_disk(i, disks);
	} while (i != d0_idx);

	BUG_ON(faila == failb);
	if (failb < faila)
		swap(faila, failb);
	pr_debug("%s: stripe: %llu faila: %d failb: %d\n",
		 __func__, (unsigned long long)sh->sector, faila, failb);

	atomic_inc(&sh->count);

	if (failb == syndrome_disks+1) {
		/* Q disk is one of the missing disks */
		if (faila == syndrome_disks) {
			/* Missing P+Q, just recompute */
D
Dan Williams 已提交
1298 1299 1300
			init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
					  ops_complete_compute, sh,
					  to_addr_conv(sh, percpu));
1301
			return async_gen_syndrome(blocks, 0, syndrome_disks+2,
1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320
						  STRIPE_SIZE, &submit);
		} else {
			struct page *dest;
			int data_target;
			int qd_idx = sh->qd_idx;

			/* Missing D+Q: recompute D from P, then recompute Q */
			if (target == qd_idx)
				data_target = target2;
			else
				data_target = target;

			count = 0;
			for (i = disks; i-- ; ) {
				if (i == data_target || i == qd_idx)
					continue;
				blocks[count++] = sh->dev[i].page;
			}
			dest = sh->dev[data_target].page;
D
Dan Williams 已提交
1321 1322 1323 1324
			init_async_submit(&submit,
					  ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
					  NULL, NULL, NULL,
					  to_addr_conv(sh, percpu));
1325 1326 1327 1328
			tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE,
				       &submit);

			count = set_syndrome_sources(blocks, sh);
D
Dan Williams 已提交
1329 1330 1331
			init_async_submit(&submit, ASYNC_TX_FENCE, tx,
					  ops_complete_compute, sh,
					  to_addr_conv(sh, percpu));
1332 1333 1334 1335
			return async_gen_syndrome(blocks, 0, count+2,
						  STRIPE_SIZE, &submit);
		}
	} else {
1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349
		init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
				  ops_complete_compute, sh,
				  to_addr_conv(sh, percpu));
		if (failb == syndrome_disks) {
			/* We're missing D+P. */
			return async_raid6_datap_recov(syndrome_disks+2,
						       STRIPE_SIZE, faila,
						       blocks, &submit);
		} else {
			/* We're missing D+D. */
			return async_raid6_2data_recov(syndrome_disks+2,
						       STRIPE_SIZE, faila, failb,
						       blocks, &submit);
		}
1350 1351 1352 1353
	}
}


1354 1355 1356 1357
static void ops_complete_prexor(void *stripe_head_ref)
{
	struct stripe_head *sh = stripe_head_ref;

1358
	pr_debug("%s: stripe %llu\n", __func__,
1359 1360 1361 1362
		(unsigned long long)sh->sector);
}

static struct dma_async_tx_descriptor *
1363 1364
ops_run_prexor(struct stripe_head *sh, struct raid5_percpu *percpu,
	       struct dma_async_tx_descriptor *tx)
1365 1366
{
	int disks = sh->disks;
1367
	struct page **xor_srcs = percpu->scribble;
1368
	int count = 0, pd_idx = sh->pd_idx, i;
1369
	struct async_submit_ctl submit;
1370 1371 1372 1373

	/* existing parity data subtracted */
	struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;

1374
	pr_debug("%s: stripe %llu\n", __func__,
1375 1376 1377 1378 1379
		(unsigned long long)sh->sector);

	for (i = disks; i--; ) {
		struct r5dev *dev = &sh->dev[i];
		/* Only process blocks that are known to be uptodate */
1380
		if (test_bit(R5_Wantdrain, &dev->flags))
1381 1382 1383
			xor_srcs[count++] = dev->page;
	}

D
Dan Williams 已提交
1384
	init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
1385
			  ops_complete_prexor, sh, to_addr_conv(sh, percpu));
1386
	tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
1387 1388 1389 1390 1391

	return tx;
}

static struct dma_async_tx_descriptor *
1392
ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
1393 1394
{
	int disks = sh->disks;
1395
	int i;
1396

1397
	pr_debug("%s: stripe %llu\n", __func__,
1398 1399 1400 1401 1402 1403
		(unsigned long long)sh->sector);

	for (i = disks; i--; ) {
		struct r5dev *dev = &sh->dev[i];
		struct bio *chosen;

1404
		if (test_and_clear_bit(R5_Wantdrain, &dev->flags)) {
1405 1406
			struct bio *wbi;

S
Shaohua Li 已提交
1407
			spin_lock_irq(&sh->stripe_lock);
1408 1409 1410 1411
			chosen = dev->towrite;
			dev->towrite = NULL;
			BUG_ON(dev->written);
			wbi = dev->written = chosen;
S
Shaohua Li 已提交
1412
			spin_unlock_irq(&sh->stripe_lock);
1413
			WARN_ON(dev->page != dev->orig_page);
1414

1415
			while (wbi && wbi->bi_iter.bi_sector <
1416
				dev->sector + STRIPE_SECTORS) {
T
Tejun Heo 已提交
1417 1418
				if (wbi->bi_rw & REQ_FUA)
					set_bit(R5_WantFUA, &dev->flags);
S
Shaohua Li 已提交
1419 1420
				if (wbi->bi_rw & REQ_SYNC)
					set_bit(R5_SyncIO, &dev->flags);
1421
				if (wbi->bi_rw & REQ_DISCARD)
S
Shaohua Li 已提交
1422
					set_bit(R5_Discard, &dev->flags);
1423 1424 1425 1426 1427 1428 1429 1430 1431
				else {
					tx = async_copy_data(1, wbi, &dev->page,
						dev->sector, tx, sh);
					if (dev->page != dev->orig_page) {
						set_bit(R5_SkipCopy, &dev->flags);
						clear_bit(R5_UPTODATE, &dev->flags);
						clear_bit(R5_OVERWRITE, &dev->flags);
					}
				}
1432 1433 1434 1435 1436 1437 1438 1439
				wbi = r5_next_bio(wbi, dev->sector);
			}
		}
	}

	return tx;
}

1440
static void ops_complete_reconstruct(void *stripe_head_ref)
1441 1442
{
	struct stripe_head *sh = stripe_head_ref;
1443 1444 1445 1446
	int disks = sh->disks;
	int pd_idx = sh->pd_idx;
	int qd_idx = sh->qd_idx;
	int i;
1447
	bool fua = false, sync = false, discard = false;
1448

1449
	pr_debug("%s: stripe %llu\n", __func__,
1450 1451
		(unsigned long long)sh->sector);

S
Shaohua Li 已提交
1452
	for (i = disks; i--; ) {
T
Tejun Heo 已提交
1453
		fua |= test_bit(R5_WantFUA, &sh->dev[i].flags);
S
Shaohua Li 已提交
1454
		sync |= test_bit(R5_SyncIO, &sh->dev[i].flags);
1455
		discard |= test_bit(R5_Discard, &sh->dev[i].flags);
S
Shaohua Li 已提交
1456
	}
T
Tejun Heo 已提交
1457

1458 1459
	for (i = disks; i--; ) {
		struct r5dev *dev = &sh->dev[i];
1460

T
Tejun Heo 已提交
1461
		if (dev->written || i == pd_idx || i == qd_idx) {
1462
			if (!discard && !test_bit(R5_SkipCopy, &dev->flags))
1463
				set_bit(R5_UPTODATE, &dev->flags);
T
Tejun Heo 已提交
1464 1465
			if (fua)
				set_bit(R5_WantFUA, &dev->flags);
S
Shaohua Li 已提交
1466 1467
			if (sync)
				set_bit(R5_SyncIO, &dev->flags);
T
Tejun Heo 已提交
1468
		}
1469 1470
	}

1471 1472 1473 1474 1475 1476 1477 1478
	if (sh->reconstruct_state == reconstruct_state_drain_run)
		sh->reconstruct_state = reconstruct_state_drain_result;
	else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run)
		sh->reconstruct_state = reconstruct_state_prexor_drain_result;
	else {
		BUG_ON(sh->reconstruct_state != reconstruct_state_run);
		sh->reconstruct_state = reconstruct_state_result;
	}
1479 1480 1481 1482 1483 1484

	set_bit(STRIPE_HANDLE, &sh->state);
	release_stripe(sh);
}

static void
1485 1486
ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu,
		     struct dma_async_tx_descriptor *tx)
1487 1488
{
	int disks = sh->disks;
1489
	struct page **xor_srcs = percpu->scribble;
1490
	struct async_submit_ctl submit;
1491 1492
	int count = 0, pd_idx = sh->pd_idx, i;
	struct page *xor_dest;
1493
	int prexor = 0;
1494 1495
	unsigned long flags;

1496
	pr_debug("%s: stripe %llu\n", __func__,
1497 1498
		(unsigned long long)sh->sector);

S
Shaohua Li 已提交
1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510
	for (i = 0; i < sh->disks; i++) {
		if (pd_idx == i)
			continue;
		if (!test_bit(R5_Discard, &sh->dev[i].flags))
			break;
	}
	if (i >= sh->disks) {
		atomic_inc(&sh->count);
		set_bit(R5_Discard, &sh->dev[pd_idx].flags);
		ops_complete_reconstruct(sh);
		return;
	}
1511 1512 1513
	/* check if prexor is active which means only process blocks
	 * that are part of a read-modify-write (written)
	 */
1514 1515
	if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
		prexor = 1;
1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535
		xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if (dev->written)
				xor_srcs[count++] = dev->page;
		}
	} else {
		xor_dest = sh->dev[pd_idx].page;
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if (i != pd_idx)
				xor_srcs[count++] = dev->page;
		}
	}

	/* 1/ if we prexor'd then the dest is reused as a source
	 * 2/ if we did not prexor then we are redoing the parity
	 * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST
	 * for the synchronous xor case
	 */
1536
	flags = ASYNC_TX_ACK |
1537 1538 1539 1540
		(prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST);

	atomic_inc(&sh->count);

1541
	init_async_submit(&submit, flags, tx, ops_complete_reconstruct, sh,
1542
			  to_addr_conv(sh, percpu));
1543 1544 1545 1546
	if (unlikely(count == 1))
		tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
	else
		tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
1547 1548
}

1549 1550 1551 1552 1553 1554
static void
ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,
		     struct dma_async_tx_descriptor *tx)
{
	struct async_submit_ctl submit;
	struct page **blocks = percpu->scribble;
S
Shaohua Li 已提交
1555
	int count, i;
1556 1557 1558

	pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector);

S
Shaohua Li 已提交
1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572
	for (i = 0; i < sh->disks; i++) {
		if (sh->pd_idx == i || sh->qd_idx == i)
			continue;
		if (!test_bit(R5_Discard, &sh->dev[i].flags))
			break;
	}
	if (i >= sh->disks) {
		atomic_inc(&sh->count);
		set_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);
		set_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);
		ops_complete_reconstruct(sh);
		return;
	}

1573 1574 1575 1576 1577 1578 1579
	count = set_syndrome_sources(blocks, sh);

	atomic_inc(&sh->count);

	init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_reconstruct,
			  sh, to_addr_conv(sh, percpu));
	async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE,  &submit);
1580 1581 1582 1583 1584 1585
}

static void ops_complete_check(void *stripe_head_ref)
{
	struct stripe_head *sh = stripe_head_ref;

1586
	pr_debug("%s: stripe %llu\n", __func__,
1587 1588
		(unsigned long long)sh->sector);

1589
	sh->check_state = check_state_check_result;
1590 1591 1592 1593
	set_bit(STRIPE_HANDLE, &sh->state);
	release_stripe(sh);
}

1594
static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu)
1595 1596
{
	int disks = sh->disks;
1597 1598 1599
	int pd_idx = sh->pd_idx;
	int qd_idx = sh->qd_idx;
	struct page *xor_dest;
1600
	struct page **xor_srcs = percpu->scribble;
1601
	struct dma_async_tx_descriptor *tx;
1602
	struct async_submit_ctl submit;
1603 1604
	int count;
	int i;
1605

1606
	pr_debug("%s: stripe %llu\n", __func__,
1607 1608
		(unsigned long long)sh->sector);

1609 1610 1611
	count = 0;
	xor_dest = sh->dev[pd_idx].page;
	xor_srcs[count++] = xor_dest;
1612
	for (i = disks; i--; ) {
1613 1614 1615
		if (i == pd_idx || i == qd_idx)
			continue;
		xor_srcs[count++] = sh->dev[i].page;
1616 1617
	}

1618 1619
	init_async_submit(&submit, 0, NULL, NULL, NULL,
			  to_addr_conv(sh, percpu));
D
Dan Williams 已提交
1620
	tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE,
1621
			   &sh->ops.zero_sum_result, &submit);
1622 1623

	atomic_inc(&sh->count);
1624 1625
	init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL);
	tx = async_trigger_callback(&submit);
1626 1627
}

1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639
static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp)
{
	struct page **srcs = percpu->scribble;
	struct async_submit_ctl submit;
	int count;

	pr_debug("%s: stripe %llu checkp: %d\n", __func__,
		(unsigned long long)sh->sector, checkp);

	count = set_syndrome_sources(srcs, sh);
	if (!checkp)
		srcs[count] = NULL;
1640 1641

	atomic_inc(&sh->count);
1642 1643 1644 1645
	init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check,
			  sh, to_addr_conv(sh, percpu));
	async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE,
			   &sh->ops.zero_sum_result, percpu->spare_page, &submit);
1646 1647
}

N
NeilBrown 已提交
1648
static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
1649 1650 1651
{
	int overlap_clear = 0, i, disks = sh->disks;
	struct dma_async_tx_descriptor *tx = NULL;
1652
	struct r5conf *conf = sh->raid_conf;
1653
	int level = conf->level;
1654 1655
	struct raid5_percpu *percpu;
	unsigned long cpu;
1656

1657 1658
	cpu = get_cpu();
	percpu = per_cpu_ptr(conf->percpu, cpu);
1659
	if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) {
1660 1661 1662 1663
		ops_run_biofill(sh);
		overlap_clear++;
	}

1664
	if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) {
1665 1666 1667 1668 1669 1670 1671 1672 1673 1674
		if (level < 6)
			tx = ops_run_compute5(sh, percpu);
		else {
			if (sh->ops.target2 < 0 || sh->ops.target < 0)
				tx = ops_run_compute6_1(sh, percpu);
			else
				tx = ops_run_compute6_2(sh, percpu);
		}
		/* terminate the chain if reconstruct is not set to be run */
		if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request))
1675 1676
			async_tx_ack(tx);
	}
1677

1678
	if (test_bit(STRIPE_OP_PREXOR, &ops_request))
1679
		tx = ops_run_prexor(sh, percpu, tx);
1680

1681
	if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) {
1682
		tx = ops_run_biodrain(sh, tx);
1683 1684 1685
		overlap_clear++;
	}

1686 1687 1688 1689 1690 1691
	if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) {
		if (level < 6)
			ops_run_reconstruct5(sh, percpu, tx);
		else
			ops_run_reconstruct6(sh, percpu, tx);
	}
1692

1693 1694 1695 1696 1697 1698 1699 1700 1701 1702
	if (test_bit(STRIPE_OP_CHECK, &ops_request)) {
		if (sh->check_state == check_state_run)
			ops_run_check_p(sh, percpu);
		else if (sh->check_state == check_state_run_q)
			ops_run_check_pq(sh, percpu, 0);
		else if (sh->check_state == check_state_run_pq)
			ops_run_check_pq(sh, percpu, 1);
		else
			BUG();
	}
1703 1704 1705 1706 1707 1708 1709

	if (overlap_clear)
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if (test_and_clear_bit(R5_Overlap, &dev->flags))
				wake_up(&sh->raid_conf->wait_for_overlap);
		}
1710
	put_cpu();
1711 1712
}

1713
static int grow_one_stripe(struct r5conf *conf, int hash)
L
Linus Torvalds 已提交
1714 1715
{
	struct stripe_head *sh;
N
Namhyung Kim 已提交
1716
	sh = kmem_cache_zalloc(conf->slab_cache, GFP_KERNEL);
1717 1718
	if (!sh)
		return 0;
N
Namhyung Kim 已提交
1719

1720 1721
	sh->raid_conf = conf;

S
Shaohua Li 已提交
1722 1723
	spin_lock_init(&sh->stripe_lock);

1724 1725
	if (grow_buffers(sh)) {
		shrink_buffers(sh);
1726 1727 1728
		kmem_cache_free(conf->slab_cache, sh);
		return 0;
	}
1729
	sh->hash_lock_index = hash;
1730 1731 1732 1733 1734 1735 1736 1737
	/* we just created an active stripe so... */
	atomic_set(&sh->count, 1);
	atomic_inc(&conf->active_stripes);
	INIT_LIST_HEAD(&sh->lru);
	release_stripe(sh);
	return 1;
}

1738
static int grow_stripes(struct r5conf *conf, int num)
1739
{
1740
	struct kmem_cache *sc;
1741
	int devs = max(conf->raid_disks, conf->previous_raid_disks);
1742
	int hash;
L
Linus Torvalds 已提交
1743

1744 1745 1746 1747 1748 1749 1750 1751
	if (conf->mddev->gendisk)
		sprintf(conf->cache_name[0],
			"raid%d-%s", conf->level, mdname(conf->mddev));
	else
		sprintf(conf->cache_name[0],
			"raid%d-%p", conf->level, conf->mddev);
	sprintf(conf->cache_name[1], "%s-alt", conf->cache_name[0]);

1752 1753
	conf->active_name = 0;
	sc = kmem_cache_create(conf->cache_name[conf->active_name],
L
Linus Torvalds 已提交
1754
			       sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev),
1755
			       0, 0, NULL);
L
Linus Torvalds 已提交
1756 1757 1758
	if (!sc)
		return 1;
	conf->slab_cache = sc;
1759
	conf->pool_size = devs;
1760 1761 1762
	hash = conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS;
	while (num--) {
		if (!grow_one_stripe(conf, hash))
L
Linus Torvalds 已提交
1763
			return 1;
1764 1765 1766
		conf->max_nr_stripes++;
		hash = (hash + 1) % NR_STRIPE_HASH_LOCKS;
	}
L
Linus Torvalds 已提交
1767 1768
	return 0;
}
1769

1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791
/**
 * scribble_len - return the required size of the scribble region
 * @num - total number of disks in the array
 *
 * The size must be enough to contain:
 * 1/ a struct page pointer for each device in the array +2
 * 2/ room to convert each entry in (1) to its corresponding dma
 *    (dma_map_page()) or page (page_address()) address.
 *
 * Note: the +2 is for the destination buffers of the ddf/raid6 case where we
 * calculate over all devices (not just the data blocks), using zeros in place
 * of the P and Q blocks.
 */
static size_t scribble_len(int num)
{
	size_t len;

	len = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2);

	return len;
}

1792
static int resize_stripes(struct r5conf *conf, int newsize)
1793 1794 1795 1796 1797 1798 1799
{
	/* Make all the stripes able to hold 'newsize' devices.
	 * New slots in each stripe get 'page' set to a new page.
	 *
	 * This happens in stages:
	 * 1/ create a new kmem_cache and allocate the required number of
	 *    stripe_heads.
M
Masanari Iida 已提交
1800
	 * 2/ gather all the old stripe_heads and transfer the pages across
1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819
	 *    to the new stripe_heads.  This will have the side effect of
	 *    freezing the array as once all stripe_heads have been collected,
	 *    no IO will be possible.  Old stripe heads are freed once their
	 *    pages have been transferred over, and the old kmem_cache is
	 *    freed when all stripes are done.
	 * 3/ reallocate conf->disks to be suitable bigger.  If this fails,
	 *    we simple return a failre status - no need to clean anything up.
	 * 4/ allocate new pages for the new slots in the new stripe_heads.
	 *    If this fails, we don't bother trying the shrink the
	 *    stripe_heads down again, we just leave them as they are.
	 *    As each stripe_head is processed the new one is released into
	 *    active service.
	 *
	 * Once step2 is started, we cannot afford to wait for a write,
	 * so we use GFP_NOIO allocations.
	 */
	struct stripe_head *osh, *nsh;
	LIST_HEAD(newstripes);
	struct disk_info *ndisks;
1820
	unsigned long cpu;
1821
	int err;
1822
	struct kmem_cache *sc;
1823
	int i;
1824
	int hash, cnt;
1825 1826 1827 1828

	if (newsize <= conf->pool_size)
		return 0; /* never bother to shrink */

1829 1830 1831
	err = md_allow_write(conf->mddev);
	if (err)
		return err;
1832

1833 1834 1835
	/* Step 1 */
	sc = kmem_cache_create(conf->cache_name[1-conf->active_name],
			       sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev),
1836
			       0, 0, NULL);
1837 1838 1839 1840
	if (!sc)
		return -ENOMEM;

	for (i = conf->max_nr_stripes; i; i--) {
N
Namhyung Kim 已提交
1841
		nsh = kmem_cache_zalloc(sc, GFP_KERNEL);
1842 1843 1844 1845
		if (!nsh)
			break;

		nsh->raid_conf = conf;
1846
		spin_lock_init(&nsh->stripe_lock);
1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863

		list_add(&nsh->lru, &newstripes);
	}
	if (i) {
		/* didn't get enough, give up */
		while (!list_empty(&newstripes)) {
			nsh = list_entry(newstripes.next, struct stripe_head, lru);
			list_del(&nsh->lru);
			kmem_cache_free(sc, nsh);
		}
		kmem_cache_destroy(sc);
		return -ENOMEM;
	}
	/* Step 2 - Must use GFP_NOIO now.
	 * OK, we have enough stripes, start collecting inactive
	 * stripes and copying them over
	 */
1864 1865
	hash = 0;
	cnt = 0;
1866
	list_for_each_entry(nsh, &newstripes, lru) {
1867 1868 1869 1870 1871 1872 1873
		lock_device_hash_lock(conf, hash);
		wait_event_cmd(conf->wait_for_stripe,
				    !list_empty(conf->inactive_list + hash),
				    unlock_device_hash_lock(conf, hash),
				    lock_device_hash_lock(conf, hash));
		osh = get_free_stripe(conf, hash);
		unlock_device_hash_lock(conf, hash);
1874
		atomic_set(&nsh->count, 1);
1875
		for(i=0; i<conf->pool_size; i++) {
1876
			nsh->dev[i].page = osh->dev[i].page;
1877 1878
			nsh->dev[i].orig_page = osh->dev[i].page;
		}
1879 1880
		for( ; i<newsize; i++)
			nsh->dev[i].page = NULL;
1881
		nsh->hash_lock_index = hash;
1882
		kmem_cache_free(conf->slab_cache, osh);
1883 1884 1885 1886 1887 1888
		cnt++;
		if (cnt >= conf->max_nr_stripes / NR_STRIPE_HASH_LOCKS +
		    !!((conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS) > hash)) {
			hash++;
			cnt = 0;
		}
1889 1890 1891 1892 1893 1894
	}
	kmem_cache_destroy(conf->slab_cache);

	/* Step 3.
	 * At this point, we are holding all the stripes so the array
	 * is completely stalled, so now is a good time to resize
1895
	 * conf->disks and the scribble region
1896 1897 1898 1899 1900 1901 1902 1903 1904 1905
	 */
	ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO);
	if (ndisks) {
		for (i=0; i<conf->raid_disks; i++)
			ndisks[i] = conf->disks[i];
		kfree(conf->disks);
		conf->disks = ndisks;
	} else
		err = -ENOMEM;

1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924
	get_online_cpus();
	conf->scribble_len = scribble_len(newsize);
	for_each_present_cpu(cpu) {
		struct raid5_percpu *percpu;
		void *scribble;

		percpu = per_cpu_ptr(conf->percpu, cpu);
		scribble = kmalloc(conf->scribble_len, GFP_NOIO);

		if (scribble) {
			kfree(percpu->scribble);
			percpu->scribble = scribble;
		} else {
			err = -ENOMEM;
			break;
		}
	}
	put_online_cpus();

1925 1926 1927 1928
	/* Step 4, return new stripes to service */
	while(!list_empty(&newstripes)) {
		nsh = list_entry(newstripes.next, struct stripe_head, lru);
		list_del_init(&nsh->lru);
1929

1930 1931 1932 1933
		for (i=conf->raid_disks; i < newsize; i++)
			if (nsh->dev[i].page == NULL) {
				struct page *p = alloc_page(GFP_NOIO);
				nsh->dev[i].page = p;
1934
				nsh->dev[i].orig_page = p;
1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946
				if (!p)
					err = -ENOMEM;
			}
		release_stripe(nsh);
	}
	/* critical section pass, GFP_NOIO no longer needed */

	conf->slab_cache = sc;
	conf->active_name = 1-conf->active_name;
	conf->pool_size = newsize;
	return err;
}
L
Linus Torvalds 已提交
1947

1948
static int drop_one_stripe(struct r5conf *conf, int hash)
L
Linus Torvalds 已提交
1949 1950 1951
{
	struct stripe_head *sh;

1952 1953 1954
	spin_lock_irq(conf->hash_locks + hash);
	sh = get_free_stripe(conf, hash);
	spin_unlock_irq(conf->hash_locks + hash);
1955 1956
	if (!sh)
		return 0;
1957
	BUG_ON(atomic_read(&sh->count));
1958
	shrink_buffers(sh);
1959 1960 1961 1962 1963
	kmem_cache_free(conf->slab_cache, sh);
	atomic_dec(&conf->active_stripes);
	return 1;
}

1964
static void shrink_stripes(struct r5conf *conf)
1965
{
1966 1967 1968 1969
	int hash;
	for (hash = 0; hash < NR_STRIPE_HASH_LOCKS; hash++)
		while (drop_one_stripe(conf, hash))
			;
1970

N
NeilBrown 已提交
1971 1972
	if (conf->slab_cache)
		kmem_cache_destroy(conf->slab_cache);
L
Linus Torvalds 已提交
1973 1974 1975
	conf->slab_cache = NULL;
}

1976
static void raid5_end_read_request(struct bio * bi, int error)
L
Linus Torvalds 已提交
1977
{
1978
	struct stripe_head *sh = bi->bi_private;
1979
	struct r5conf *conf = sh->raid_conf;
1980
	int disks = sh->disks, i;
L
Linus Torvalds 已提交
1981
	int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
1982
	char b[BDEVNAME_SIZE];
1983
	struct md_rdev *rdev = NULL;
1984
	sector_t s;
L
Linus Torvalds 已提交
1985 1986 1987 1988 1989

	for (i=0 ; i<disks; i++)
		if (bi == &sh->dev[i].req)
			break;

1990 1991
	pr_debug("end_read_request %llu/%d, count: %d, uptodate %d.\n",
		(unsigned long long)sh->sector, i, atomic_read(&sh->count),
L
Linus Torvalds 已提交
1992 1993 1994
		uptodate);
	if (i == disks) {
		BUG();
1995
		return;
L
Linus Torvalds 已提交
1996
	}
1997
	if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
1998 1999 2000 2001 2002
		/* If replacement finished while this request was outstanding,
		 * 'replacement' might be NULL already.
		 * In that case it moved down to 'rdev'.
		 * rdev is not removed until all requests are finished.
		 */
2003
		rdev = conf->disks[i].replacement;
2004
	if (!rdev)
2005
		rdev = conf->disks[i].rdev;
L
Linus Torvalds 已提交
2006

2007 2008 2009 2010
	if (use_new_offset(conf, sh))
		s = sh->sector + rdev->new_data_offset;
	else
		s = sh->sector + rdev->data_offset;
L
Linus Torvalds 已提交
2011 2012
	if (uptodate) {
		set_bit(R5_UPTODATE, &sh->dev[i].flags);
2013
		if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
2014 2015 2016 2017
			/* Note that this cannot happen on a
			 * replacement device.  We just fail those on
			 * any error
			 */
2018 2019 2020 2021 2022
			printk_ratelimited(
				KERN_INFO
				"md/raid:%s: read error corrected"
				" (%lu sectors at %llu on %s)\n",
				mdname(conf->mddev), STRIPE_SECTORS,
2023
				(unsigned long long)s,
2024
				bdevname(rdev->bdev, b));
2025
			atomic_add(STRIPE_SECTORS, &rdev->corrected_errors);
2026 2027
			clear_bit(R5_ReadError, &sh->dev[i].flags);
			clear_bit(R5_ReWrite, &sh->dev[i].flags);
2028 2029 2030
		} else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
			clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);

2031 2032
		if (atomic_read(&rdev->read_errors))
			atomic_set(&rdev->read_errors, 0);
L
Linus Torvalds 已提交
2033
	} else {
2034
		const char *bdn = bdevname(rdev->bdev, b);
2035
		int retry = 0;
2036
		int set_bad = 0;
2037

L
Linus Torvalds 已提交
2038
		clear_bit(R5_UPTODATE, &sh->dev[i].flags);
2039
		atomic_inc(&rdev->read_errors);
2040 2041 2042 2043 2044 2045
		if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
			printk_ratelimited(
				KERN_WARNING
				"md/raid:%s: read error on replacement device "
				"(sector %llu on %s).\n",
				mdname(conf->mddev),
2046
				(unsigned long long)s,
2047
				bdn);
2048 2049
		else if (conf->mddev->degraded >= conf->max_degraded) {
			set_bad = 1;
2050 2051 2052 2053 2054
			printk_ratelimited(
				KERN_WARNING
				"md/raid:%s: read error not correctable "
				"(sector %llu on %s).\n",
				mdname(conf->mddev),
2055
				(unsigned long long)s,
2056
				bdn);
2057
		} else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) {
2058
			/* Oh, no!!! */
2059
			set_bad = 1;
2060 2061 2062 2063 2064
			printk_ratelimited(
				KERN_WARNING
				"md/raid:%s: read error NOT corrected!! "
				"(sector %llu on %s).\n",
				mdname(conf->mddev),
2065
				(unsigned long long)s,
2066
				bdn);
2067
		} else if (atomic_read(&rdev->read_errors)
2068
			 > conf->max_nr_stripes)
N
NeilBrown 已提交
2069
			printk(KERN_WARNING
2070
			       "md/raid:%s: Too many read errors, failing device %s.\n",
2071
			       mdname(conf->mddev), bdn);
2072 2073
		else
			retry = 1;
2074 2075 2076
		if (set_bad && test_bit(In_sync, &rdev->flags)
		    && !test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
			retry = 1;
2077
		if (retry)
2078 2079 2080 2081 2082
			if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) {
				set_bit(R5_ReadError, &sh->dev[i].flags);
				clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);
			} else
				set_bit(R5_ReadNoMerge, &sh->dev[i].flags);
2083
		else {
2084 2085
			clear_bit(R5_ReadError, &sh->dev[i].flags);
			clear_bit(R5_ReWrite, &sh->dev[i].flags);
2086 2087 2088 2089 2090
			if (!(set_bad
			      && test_bit(In_sync, &rdev->flags)
			      && rdev_set_badblocks(
				      rdev, sh->sector, STRIPE_SECTORS, 0)))
				md_error(conf->mddev, rdev);
2091
		}
L
Linus Torvalds 已提交
2092
	}
2093
	rdev_dec_pending(rdev, conf->mddev);
L
Linus Torvalds 已提交
2094 2095 2096 2097 2098
	clear_bit(R5_LOCKED, &sh->dev[i].flags);
	set_bit(STRIPE_HANDLE, &sh->state);
	release_stripe(sh);
}

2099
static void raid5_end_write_request(struct bio *bi, int error)
L
Linus Torvalds 已提交
2100
{
2101
	struct stripe_head *sh = bi->bi_private;
2102
	struct r5conf *conf = sh->raid_conf;
2103
	int disks = sh->disks, i;
2104
	struct md_rdev *uninitialized_var(rdev);
L
Linus Torvalds 已提交
2105
	int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
2106 2107
	sector_t first_bad;
	int bad_sectors;
2108
	int replacement = 0;
L
Linus Torvalds 已提交
2109

2110 2111 2112
	for (i = 0 ; i < disks; i++) {
		if (bi == &sh->dev[i].req) {
			rdev = conf->disks[i].rdev;
L
Linus Torvalds 已提交
2113
			break;
2114 2115 2116
		}
		if (bi == &sh->dev[i].rreq) {
			rdev = conf->disks[i].replacement;
2117 2118 2119 2120 2121 2122 2123 2124
			if (rdev)
				replacement = 1;
			else
				/* rdev was removed and 'replacement'
				 * replaced it.  rdev is not removed
				 * until all requests are finished.
				 */
				rdev = conf->disks[i].rdev;
2125 2126 2127
			break;
		}
	}
2128
	pr_debug("end_write_request %llu/%d, count %d, uptodate: %d.\n",
L
Linus Torvalds 已提交
2129 2130 2131 2132
		(unsigned long long)sh->sector, i, atomic_read(&sh->count),
		uptodate);
	if (i == disks) {
		BUG();
2133
		return;
L
Linus Torvalds 已提交
2134 2135
	}

2136 2137 2138 2139 2140 2141 2142 2143 2144
	if (replacement) {
		if (!uptodate)
			md_error(conf->mddev, rdev);
		else if (is_badblock(rdev, sh->sector,
				     STRIPE_SECTORS,
				     &first_bad, &bad_sectors))
			set_bit(R5_MadeGoodRepl, &sh->dev[i].flags);
	} else {
		if (!uptodate) {
2145
			set_bit(STRIPE_DEGRADED, &sh->state);
2146 2147
			set_bit(WriteErrorSeen, &rdev->flags);
			set_bit(R5_WriteError, &sh->dev[i].flags);
2148 2149 2150
			if (!test_and_set_bit(WantReplacement, &rdev->flags))
				set_bit(MD_RECOVERY_NEEDED,
					&rdev->mddev->recovery);
2151 2152
		} else if (is_badblock(rdev, sh->sector,
				       STRIPE_SECTORS,
2153
				       &first_bad, &bad_sectors)) {
2154
			set_bit(R5_MadeGood, &sh->dev[i].flags);
2155 2156 2157 2158 2159 2160 2161
			if (test_bit(R5_ReadError, &sh->dev[i].flags))
				/* That was a successful write so make
				 * sure it looks like we already did
				 * a re-write.
				 */
				set_bit(R5_ReWrite, &sh->dev[i].flags);
		}
2162 2163
	}
	rdev_dec_pending(rdev, conf->mddev);
L
Linus Torvalds 已提交
2164

2165 2166
	if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags))
		clear_bit(R5_LOCKED, &sh->dev[i].flags);
L
Linus Torvalds 已提交
2167
	set_bit(STRIPE_HANDLE, &sh->state);
2168
	release_stripe(sh);
L
Linus Torvalds 已提交
2169 2170
}

2171
static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous);
2172

2173
static void raid5_build_block(struct stripe_head *sh, int i, int previous)
L
Linus Torvalds 已提交
2174 2175 2176 2177 2178
{
	struct r5dev *dev = &sh->dev[i];

	bio_init(&dev->req);
	dev->req.bi_io_vec = &dev->vec;
2179
	dev->req.bi_max_vecs = 1;
L
Linus Torvalds 已提交
2180 2181
	dev->req.bi_private = sh;

2182 2183
	bio_init(&dev->rreq);
	dev->rreq.bi_io_vec = &dev->rvec;
2184
	dev->rreq.bi_max_vecs = 1;
2185 2186
	dev->rreq.bi_private = sh;

L
Linus Torvalds 已提交
2187
	dev->flags = 0;
2188
	dev->sector = compute_blocknr(sh, i, previous);
L
Linus Torvalds 已提交
2189 2190
}

2191
static void error(struct mddev *mddev, struct md_rdev *rdev)
L
Linus Torvalds 已提交
2192 2193
{
	char b[BDEVNAME_SIZE];
2194
	struct r5conf *conf = mddev->private;
2195
	unsigned long flags;
2196
	pr_debug("raid456: error called\n");
L
Linus Torvalds 已提交
2197

2198 2199 2200 2201 2202 2203
	spin_lock_irqsave(&conf->device_lock, flags);
	clear_bit(In_sync, &rdev->flags);
	mddev->degraded = calc_degraded(conf);
	spin_unlock_irqrestore(&conf->device_lock, flags);
	set_bit(MD_RECOVERY_INTR, &mddev->recovery);

2204
	set_bit(Blocked, &rdev->flags);
2205 2206 2207 2208 2209 2210 2211 2212 2213
	set_bit(Faulty, &rdev->flags);
	set_bit(MD_CHANGE_DEVS, &mddev->flags);
	printk(KERN_ALERT
	       "md/raid:%s: Disk failure on %s, disabling device.\n"
	       "md/raid:%s: Operation continuing on %d devices.\n",
	       mdname(mddev),
	       bdevname(rdev->bdev, b),
	       mdname(mddev),
	       conf->raid_disks - mddev->degraded);
2214
}
L
Linus Torvalds 已提交
2215 2216 2217 2218 2219

/*
 * Input: a 'big' sector number,
 * Output: index of the data and parity disk, and the sector # in them.
 */
2220
static sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector,
2221 2222
				     int previous, int *dd_idx,
				     struct stripe_head *sh)
L
Linus Torvalds 已提交
2223
{
N
NeilBrown 已提交
2224
	sector_t stripe, stripe2;
2225
	sector_t chunk_number;
L
Linus Torvalds 已提交
2226
	unsigned int chunk_offset;
2227
	int pd_idx, qd_idx;
2228
	int ddf_layout = 0;
L
Linus Torvalds 已提交
2229
	sector_t new_sector;
2230 2231
	int algorithm = previous ? conf->prev_algo
				 : conf->algorithm;
2232 2233
	int sectors_per_chunk = previous ? conf->prev_chunk_sectors
					 : conf->chunk_sectors;
2234 2235 2236
	int raid_disks = previous ? conf->previous_raid_disks
				  : conf->raid_disks;
	int data_disks = raid_disks - conf->max_degraded;
L
Linus Torvalds 已提交
2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248

	/* First compute the information on this sector */

	/*
	 * Compute the chunk number and the sector offset inside the chunk
	 */
	chunk_offset = sector_div(r_sector, sectors_per_chunk);
	chunk_number = r_sector;

	/*
	 * Compute the stripe number
	 */
2249 2250
	stripe = chunk_number;
	*dd_idx = sector_div(stripe, data_disks);
N
NeilBrown 已提交
2251
	stripe2 = stripe;
L
Linus Torvalds 已提交
2252 2253 2254
	/*
	 * Select the parity disk based on the user selected algorithm.
	 */
2255
	pd_idx = qd_idx = -1;
2256 2257
	switch(conf->level) {
	case 4:
2258
		pd_idx = data_disks;
2259 2260
		break;
	case 5:
2261
		switch (algorithm) {
L
Linus Torvalds 已提交
2262
		case ALGORITHM_LEFT_ASYMMETRIC:
N
NeilBrown 已提交
2263
			pd_idx = data_disks - sector_div(stripe2, raid_disks);
2264
			if (*dd_idx >= pd_idx)
L
Linus Torvalds 已提交
2265 2266 2267
				(*dd_idx)++;
			break;
		case ALGORITHM_RIGHT_ASYMMETRIC:
N
NeilBrown 已提交
2268
			pd_idx = sector_div(stripe2, raid_disks);
2269
			if (*dd_idx >= pd_idx)
L
Linus Torvalds 已提交
2270 2271 2272
				(*dd_idx)++;
			break;
		case ALGORITHM_LEFT_SYMMETRIC:
N
NeilBrown 已提交
2273
			pd_idx = data_disks - sector_div(stripe2, raid_disks);
2274
			*dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
L
Linus Torvalds 已提交
2275 2276
			break;
		case ALGORITHM_RIGHT_SYMMETRIC:
N
NeilBrown 已提交
2277
			pd_idx = sector_div(stripe2, raid_disks);
2278
			*dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
L
Linus Torvalds 已提交
2279
			break;
2280 2281 2282 2283 2284 2285 2286
		case ALGORITHM_PARITY_0:
			pd_idx = 0;
			(*dd_idx)++;
			break;
		case ALGORITHM_PARITY_N:
			pd_idx = data_disks;
			break;
L
Linus Torvalds 已提交
2287
		default:
2288
			BUG();
2289 2290 2291 2292
		}
		break;
	case 6:

2293
		switch (algorithm) {
2294
		case ALGORITHM_LEFT_ASYMMETRIC:
N
NeilBrown 已提交
2295
			pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
2296 2297
			qd_idx = pd_idx + 1;
			if (pd_idx == raid_disks-1) {
2298
				(*dd_idx)++;	/* Q D D D P */
2299 2300
				qd_idx = 0;
			} else if (*dd_idx >= pd_idx)
2301 2302 2303
				(*dd_idx) += 2; /* D D P Q D */
			break;
		case ALGORITHM_RIGHT_ASYMMETRIC:
N
NeilBrown 已提交
2304
			pd_idx = sector_div(stripe2, raid_disks);
2305 2306
			qd_idx = pd_idx + 1;
			if (pd_idx == raid_disks-1) {
2307
				(*dd_idx)++;	/* Q D D D P */
2308 2309
				qd_idx = 0;
			} else if (*dd_idx >= pd_idx)
2310 2311 2312
				(*dd_idx) += 2; /* D D P Q D */
			break;
		case ALGORITHM_LEFT_SYMMETRIC:
N
NeilBrown 已提交
2313
			pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
2314 2315
			qd_idx = (pd_idx + 1) % raid_disks;
			*dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
2316 2317
			break;
		case ALGORITHM_RIGHT_SYMMETRIC:
N
NeilBrown 已提交
2318
			pd_idx = sector_div(stripe2, raid_disks);
2319 2320
			qd_idx = (pd_idx + 1) % raid_disks;
			*dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
2321
			break;
2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333 2334 2335 2336

		case ALGORITHM_PARITY_0:
			pd_idx = 0;
			qd_idx = 1;
			(*dd_idx) += 2;
			break;
		case ALGORITHM_PARITY_N:
			pd_idx = data_disks;
			qd_idx = data_disks + 1;
			break;

		case ALGORITHM_ROTATING_ZERO_RESTART:
			/* Exactly the same as RIGHT_ASYMMETRIC, but or
			 * of blocks for computing Q is different.
			 */
N
NeilBrown 已提交
2337
			pd_idx = sector_div(stripe2, raid_disks);
2338 2339 2340 2341 2342 2343
			qd_idx = pd_idx + 1;
			if (pd_idx == raid_disks-1) {
				(*dd_idx)++;	/* Q D D D P */
				qd_idx = 0;
			} else if (*dd_idx >= pd_idx)
				(*dd_idx) += 2; /* D D P Q D */
2344
			ddf_layout = 1;
2345 2346 2347 2348 2349 2350 2351
			break;

		case ALGORITHM_ROTATING_N_RESTART:
			/* Same a left_asymmetric, by first stripe is
			 * D D D P Q  rather than
			 * Q D D D P
			 */
N
NeilBrown 已提交
2352 2353
			stripe2 += 1;
			pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
2354 2355 2356 2357 2358 2359
			qd_idx = pd_idx + 1;
			if (pd_idx == raid_disks-1) {
				(*dd_idx)++;	/* Q D D D P */
				qd_idx = 0;
			} else if (*dd_idx >= pd_idx)
				(*dd_idx) += 2; /* D D P Q D */
2360
			ddf_layout = 1;
2361 2362 2363 2364
			break;

		case ALGORITHM_ROTATING_N_CONTINUE:
			/* Same as left_symmetric but Q is before P */
N
NeilBrown 已提交
2365
			pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
2366 2367
			qd_idx = (pd_idx + raid_disks - 1) % raid_disks;
			*dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
2368
			ddf_layout = 1;
2369 2370 2371 2372
			break;

		case ALGORITHM_LEFT_ASYMMETRIC_6:
			/* RAID5 left_asymmetric, with Q on last device */
N
NeilBrown 已提交
2373
			pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
2374 2375 2376 2377 2378 2379
			if (*dd_idx >= pd_idx)
				(*dd_idx)++;
			qd_idx = raid_disks - 1;
			break;

		case ALGORITHM_RIGHT_ASYMMETRIC_6:
N
NeilBrown 已提交
2380
			pd_idx = sector_div(stripe2, raid_disks-1);
2381 2382 2383 2384 2385 2386
			if (*dd_idx >= pd_idx)
				(*dd_idx)++;
			qd_idx = raid_disks - 1;
			break;

		case ALGORITHM_LEFT_SYMMETRIC_6:
N
NeilBrown 已提交
2387
			pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
2388 2389 2390 2391 2392
			*dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
			qd_idx = raid_disks - 1;
			break;

		case ALGORITHM_RIGHT_SYMMETRIC_6:
N
NeilBrown 已提交
2393
			pd_idx = sector_div(stripe2, raid_disks-1);
2394 2395 2396 2397 2398 2399 2400 2401 2402 2403
			*dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
			qd_idx = raid_disks - 1;
			break;

		case ALGORITHM_PARITY_0_6:
			pd_idx = 0;
			(*dd_idx)++;
			qd_idx = raid_disks - 1;
			break;

2404
		default:
2405
			BUG();
2406 2407
		}
		break;
L
Linus Torvalds 已提交
2408 2409
	}

2410 2411 2412
	if (sh) {
		sh->pd_idx = pd_idx;
		sh->qd_idx = qd_idx;
2413
		sh->ddf_layout = ddf_layout;
2414
	}
L
Linus Torvalds 已提交
2415 2416 2417 2418 2419 2420 2421 2422
	/*
	 * Finally, compute the new sector number
	 */
	new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset;
	return new_sector;
}


2423
static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous)
L
Linus Torvalds 已提交
2424
{
2425
	struct r5conf *conf = sh->raid_conf;
2426 2427
	int raid_disks = sh->disks;
	int data_disks = raid_disks - conf->max_degraded;
L
Linus Torvalds 已提交
2428
	sector_t new_sector = sh->sector, check;
2429 2430
	int sectors_per_chunk = previous ? conf->prev_chunk_sectors
					 : conf->chunk_sectors;
2431 2432
	int algorithm = previous ? conf->prev_algo
				 : conf->algorithm;
L
Linus Torvalds 已提交
2433 2434
	sector_t stripe;
	int chunk_offset;
2435 2436
	sector_t chunk_number;
	int dummy1, dd_idx = i;
L
Linus Torvalds 已提交
2437
	sector_t r_sector;
2438
	struct stripe_head sh2;
L
Linus Torvalds 已提交
2439

2440

L
Linus Torvalds 已提交
2441 2442 2443
	chunk_offset = sector_div(new_sector, sectors_per_chunk);
	stripe = new_sector;

2444 2445 2446 2447 2448
	if (i == sh->pd_idx)
		return 0;
	switch(conf->level) {
	case 4: break;
	case 5:
2449
		switch (algorithm) {
L
Linus Torvalds 已提交
2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460
		case ALGORITHM_LEFT_ASYMMETRIC:
		case ALGORITHM_RIGHT_ASYMMETRIC:
			if (i > sh->pd_idx)
				i--;
			break;
		case ALGORITHM_LEFT_SYMMETRIC:
		case ALGORITHM_RIGHT_SYMMETRIC:
			if (i < sh->pd_idx)
				i += raid_disks;
			i -= (sh->pd_idx + 1);
			break;
2461 2462 2463 2464 2465
		case ALGORITHM_PARITY_0:
			i -= 1;
			break;
		case ALGORITHM_PARITY_N:
			break;
L
Linus Torvalds 已提交
2466
		default:
2467
			BUG();
2468 2469 2470
		}
		break;
	case 6:
2471
		if (i == sh->qd_idx)
2472
			return 0; /* It is the Q disk */
2473
		switch (algorithm) {
2474 2475
		case ALGORITHM_LEFT_ASYMMETRIC:
		case ALGORITHM_RIGHT_ASYMMETRIC:
2476 2477 2478 2479
		case ALGORITHM_ROTATING_ZERO_RESTART:
		case ALGORITHM_ROTATING_N_RESTART:
			if (sh->pd_idx == raid_disks-1)
				i--;	/* Q D D D P */
2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493
			else if (i > sh->pd_idx)
				i -= 2; /* D D P Q D */
			break;
		case ALGORITHM_LEFT_SYMMETRIC:
		case ALGORITHM_RIGHT_SYMMETRIC:
			if (sh->pd_idx == raid_disks-1)
				i--; /* Q D D D P */
			else {
				/* D D P Q D */
				if (i < sh->pd_idx)
					i += raid_disks;
				i -= (sh->pd_idx + 2);
			}
			break;
2494 2495 2496 2497 2498 2499
		case ALGORITHM_PARITY_0:
			i -= 2;
			break;
		case ALGORITHM_PARITY_N:
			break;
		case ALGORITHM_ROTATING_N_CONTINUE:
2500
			/* Like left_symmetric, but P is before Q */
2501 2502
			if (sh->pd_idx == 0)
				i--;	/* P D D D Q */
2503 2504 2505 2506 2507 2508
			else {
				/* D D Q P D */
				if (i < sh->pd_idx)
					i += raid_disks;
				i -= (sh->pd_idx + 1);
			}
2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523
			break;
		case ALGORITHM_LEFT_ASYMMETRIC_6:
		case ALGORITHM_RIGHT_ASYMMETRIC_6:
			if (i > sh->pd_idx)
				i--;
			break;
		case ALGORITHM_LEFT_SYMMETRIC_6:
		case ALGORITHM_RIGHT_SYMMETRIC_6:
			if (i < sh->pd_idx)
				i += data_disks + 1;
			i -= (sh->pd_idx + 1);
			break;
		case ALGORITHM_PARITY_0_6:
			i -= 1;
			break;
2524
		default:
2525
			BUG();
2526 2527
		}
		break;
L
Linus Torvalds 已提交
2528 2529 2530
	}

	chunk_number = stripe * data_disks + i;
2531
	r_sector = chunk_number * sectors_per_chunk + chunk_offset;
L
Linus Torvalds 已提交
2532

2533
	check = raid5_compute_sector(conf, r_sector,
2534
				     previous, &dummy1, &sh2);
2535 2536
	if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx
		|| sh2.qd_idx != sh->qd_idx) {
2537 2538
		printk(KERN_ERR "md/raid:%s: compute_blocknr: map not correct\n",
		       mdname(conf->mddev));
L
Linus Torvalds 已提交
2539 2540 2541 2542 2543 2544
		return 0;
	}
	return r_sector;
}


2545
static void
2546
schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s,
2547
			 int rcw, int expand)
2548 2549
{
	int i, pd_idx = sh->pd_idx, disks = sh->disks;
2550
	struct r5conf *conf = sh->raid_conf;
2551
	int level = conf->level;
2552 2553 2554 2555 2556 2557 2558 2559

	if (rcw) {

		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];

			if (dev->towrite) {
				set_bit(R5_LOCKED, &dev->flags);
2560
				set_bit(R5_Wantdrain, &dev->flags);
2561 2562
				if (!expand)
					clear_bit(R5_UPTODATE, &dev->flags);
2563
				s->locked++;
2564 2565
			}
		}
2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580
		/* if we are not expanding this is a proper write request, and
		 * there will be bios with new data to be drained into the
		 * stripe cache
		 */
		if (!expand) {
			if (!s->locked)
				/* False alarm, nothing to do */
				return;
			sh->reconstruct_state = reconstruct_state_drain_run;
			set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
		} else
			sh->reconstruct_state = reconstruct_state_run;

		set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);

2581
		if (s->locked + conf->max_degraded == disks)
2582
			if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state))
2583
				atomic_inc(&conf->pending_full_writes);
2584
	} else {
2585
		BUG_ON(level == 6);
2586 2587 2588 2589 2590 2591 2592 2593 2594 2595
		BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) ||
			test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags)));

		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if (i == pd_idx)
				continue;

			if (dev->towrite &&
			    (test_bit(R5_UPTODATE, &dev->flags) ||
2596 2597
			     test_bit(R5_Wantcompute, &dev->flags))) {
				set_bit(R5_Wantdrain, &dev->flags);
2598 2599
				set_bit(R5_LOCKED, &dev->flags);
				clear_bit(R5_UPTODATE, &dev->flags);
2600
				s->locked++;
2601 2602
			}
		}
2603 2604 2605 2606 2607 2608 2609
		if (!s->locked)
			/* False alarm - nothing to do */
			return;
		sh->reconstruct_state = reconstruct_state_prexor_drain_run;
		set_bit(STRIPE_OP_PREXOR, &s->ops_request);
		set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
		set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
2610 2611
	}

2612
	/* keep the parity disk(s) locked while asynchronous operations
2613 2614 2615 2616
	 * are in flight
	 */
	set_bit(R5_LOCKED, &sh->dev[pd_idx].flags);
	clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
2617
	s->locked++;
2618

2619 2620 2621 2622 2623 2624 2625 2626 2627
	if (level == 6) {
		int qd_idx = sh->qd_idx;
		struct r5dev *dev = &sh->dev[qd_idx];

		set_bit(R5_LOCKED, &dev->flags);
		clear_bit(R5_UPTODATE, &dev->flags);
		s->locked++;
	}

2628
	pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n",
2629
		__func__, (unsigned long long)sh->sector,
2630
		s->locked, s->ops_request);
2631
}
2632

L
Linus Torvalds 已提交
2633 2634
/*
 * Each stripe/dev can have one or more bion attached.
2635
 * toread/towrite point to the first in a chain.
L
Linus Torvalds 已提交
2636 2637 2638 2639 2640
 * The bi_next chain must be in order.
 */
static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx, int forwrite)
{
	struct bio **bip;
2641
	struct r5conf *conf = sh->raid_conf;
2642
	int firstwrite=0;
L
Linus Torvalds 已提交
2643

2644
	pr_debug("adding bi b#%llu to stripe s#%llu\n",
2645
		(unsigned long long)bi->bi_iter.bi_sector,
L
Linus Torvalds 已提交
2646 2647
		(unsigned long long)sh->sector);

S
Shaohua Li 已提交
2648 2649 2650 2651 2652 2653 2654 2655 2656
	/*
	 * If several bio share a stripe. The bio bi_phys_segments acts as a
	 * reference count to avoid race. The reference count should already be
	 * increased before this function is called (for example, in
	 * make_request()), so other bio sharing this stripe will not free the
	 * stripe. If a stripe is owned by one stripe, the stripe lock will
	 * protect it.
	 */
	spin_lock_irq(&sh->stripe_lock);
2657
	if (forwrite) {
L
Linus Torvalds 已提交
2658
		bip = &sh->dev[dd_idx].towrite;
2659
		if (*bip == NULL)
2660 2661
			firstwrite = 1;
	} else
L
Linus Torvalds 已提交
2662
		bip = &sh->dev[dd_idx].toread;
2663 2664
	while (*bip && (*bip)->bi_iter.bi_sector < bi->bi_iter.bi_sector) {
		if (bio_end_sector(*bip) > bi->bi_iter.bi_sector)
L
Linus Torvalds 已提交
2665 2666 2667
			goto overlap;
		bip = & (*bip)->bi_next;
	}
2668
	if (*bip && (*bip)->bi_iter.bi_sector < bio_end_sector(bi))
L
Linus Torvalds 已提交
2669 2670
		goto overlap;

2671
	BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next);
L
Linus Torvalds 已提交
2672 2673 2674
	if (*bip)
		bi->bi_next = *bip;
	*bip = bi;
2675
	raid5_inc_bi_active_stripes(bi);
2676

L
Linus Torvalds 已提交
2677 2678 2679 2680 2681
	if (forwrite) {
		/* check if page is covered */
		sector_t sector = sh->dev[dd_idx].sector;
		for (bi=sh->dev[dd_idx].towrite;
		     sector < sh->dev[dd_idx].sector + STRIPE_SECTORS &&
2682
			     bi && bi->bi_iter.bi_sector <= sector;
L
Linus Torvalds 已提交
2683
		     bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) {
K
Kent Overstreet 已提交
2684 2685
			if (bio_end_sector(bi) >= sector)
				sector = bio_end_sector(bi);
L
Linus Torvalds 已提交
2686 2687 2688 2689
		}
		if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS)
			set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags);
	}
2690 2691

	pr_debug("added bi b#%llu to stripe s#%llu, disk %d.\n",
2692
		(unsigned long long)(*bip)->bi_iter.bi_sector,
2693
		(unsigned long long)sh->sector, dd_idx);
2694
	spin_unlock_irq(&sh->stripe_lock);
2695 2696 2697 2698 2699 2700 2701

	if (conf->mddev->bitmap && firstwrite) {
		bitmap_startwrite(conf->mddev->bitmap, sh->sector,
				  STRIPE_SECTORS, 0);
		sh->bm_seq = conf->seq_flush+1;
		set_bit(STRIPE_BIT_DELAY, &sh->state);
	}
L
Linus Torvalds 已提交
2702 2703 2704 2705
	return 1;

 overlap:
	set_bit(R5_Overlap, &sh->dev[dd_idx].flags);
S
Shaohua Li 已提交
2706
	spin_unlock_irq(&sh->stripe_lock);
L
Linus Torvalds 已提交
2707 2708 2709
	return 0;
}

2710
static void end_reshape(struct r5conf *conf);
2711

2712
static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,
2713
			    struct stripe_head *sh)
2714
{
2715
	int sectors_per_chunk =
2716
		previous ? conf->prev_chunk_sectors : conf->chunk_sectors;
2717
	int dd_idx;
2718
	int chunk_offset = sector_div(stripe, sectors_per_chunk);
2719
	int disks = previous ? conf->previous_raid_disks : conf->raid_disks;
2720

2721 2722
	raid5_compute_sector(conf,
			     stripe * (disks - conf->max_degraded)
2723
			     *sectors_per_chunk + chunk_offset,
2724
			     previous,
2725
			     &dd_idx, sh);
2726 2727
}

2728
static void
2729
handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh,
2730 2731 2732 2733 2734 2735 2736 2737 2738
				struct stripe_head_state *s, int disks,
				struct bio **return_bi)
{
	int i;
	for (i = disks; i--; ) {
		struct bio *bi;
		int bitmap_end = 0;

		if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
2739
			struct md_rdev *rdev;
2740 2741 2742
			rcu_read_lock();
			rdev = rcu_dereference(conf->disks[i].rdev);
			if (rdev && test_bit(In_sync, &rdev->flags))
2743 2744 2745
				atomic_inc(&rdev->nr_pending);
			else
				rdev = NULL;
2746
			rcu_read_unlock();
2747 2748 2749 2750 2751 2752 2753 2754
			if (rdev) {
				if (!rdev_set_badblocks(
					    rdev,
					    sh->sector,
					    STRIPE_SECTORS, 0))
					md_error(conf->mddev, rdev);
				rdev_dec_pending(rdev, conf->mddev);
			}
2755
		}
S
Shaohua Li 已提交
2756
		spin_lock_irq(&sh->stripe_lock);
2757 2758 2759
		/* fail all writes first */
		bi = sh->dev[i].towrite;
		sh->dev[i].towrite = NULL;
S
Shaohua Li 已提交
2760
		spin_unlock_irq(&sh->stripe_lock);
2761
		if (bi)
2762 2763 2764 2765 2766
			bitmap_end = 1;

		if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
			wake_up(&conf->wait_for_overlap);

2767
		while (bi && bi->bi_iter.bi_sector <
2768 2769 2770
			sh->dev[i].sector + STRIPE_SECTORS) {
			struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector);
			clear_bit(BIO_UPTODATE, &bi->bi_flags);
2771
			if (!raid5_dec_bi_active_stripes(bi)) {
2772 2773 2774 2775 2776 2777
				md_write_end(conf->mddev);
				bi->bi_next = *return_bi;
				*return_bi = bi;
			}
			bi = nextbi;
		}
2778 2779 2780 2781
		if (bitmap_end)
			bitmap_endwrite(conf->mddev->bitmap, sh->sector,
				STRIPE_SECTORS, 0, 0);
		bitmap_end = 0;
2782 2783 2784
		/* and fail all 'written' */
		bi = sh->dev[i].written;
		sh->dev[i].written = NULL;
2785 2786 2787 2788 2789
		if (test_and_clear_bit(R5_SkipCopy, &sh->dev[i].flags)) {
			WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
			sh->dev[i].page = sh->dev[i].orig_page;
		}

2790
		if (bi) bitmap_end = 1;
2791
		while (bi && bi->bi_iter.bi_sector <
2792 2793 2794
		       sh->dev[i].sector + STRIPE_SECTORS) {
			struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector);
			clear_bit(BIO_UPTODATE, &bi->bi_flags);
2795
			if (!raid5_dec_bi_active_stripes(bi)) {
2796 2797 2798 2799 2800 2801 2802
				md_write_end(conf->mddev);
				bi->bi_next = *return_bi;
				*return_bi = bi;
			}
			bi = bi2;
		}

2803 2804 2805 2806 2807 2808
		/* fail any reads if this device is non-operational and
		 * the data has not reached the cache yet.
		 */
		if (!test_bit(R5_Wantfill, &sh->dev[i].flags) &&
		    (!test_bit(R5_Insync, &sh->dev[i].flags) ||
		      test_bit(R5_ReadError, &sh->dev[i].flags))) {
2809
			spin_lock_irq(&sh->stripe_lock);
2810 2811
			bi = sh->dev[i].toread;
			sh->dev[i].toread = NULL;
2812
			spin_unlock_irq(&sh->stripe_lock);
2813 2814
			if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
				wake_up(&conf->wait_for_overlap);
2815
			while (bi && bi->bi_iter.bi_sector <
2816 2817 2818 2819
			       sh->dev[i].sector + STRIPE_SECTORS) {
				struct bio *nextbi =
					r5_next_bio(bi, sh->dev[i].sector);
				clear_bit(BIO_UPTODATE, &bi->bi_flags);
2820
				if (!raid5_dec_bi_active_stripes(bi)) {
2821 2822 2823 2824 2825 2826 2827 2828 2829
					bi->bi_next = *return_bi;
					*return_bi = bi;
				}
				bi = nextbi;
			}
		}
		if (bitmap_end)
			bitmap_endwrite(conf->mddev->bitmap, sh->sector,
					STRIPE_SECTORS, 0, 0);
2830 2831 2832 2833
		/* If we were in the middle of a write the parity block might
		 * still be locked - so just clear all R5_LOCKED flags
		 */
		clear_bit(R5_LOCKED, &sh->dev[i].flags);
2834 2835
	}

2836 2837 2838
	if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
		if (atomic_dec_and_test(&conf->pending_full_writes))
			md_wakeup_thread(conf->mddev->thread);
2839 2840
}

2841
static void
2842
handle_failed_sync(struct r5conf *conf, struct stripe_head *sh,
2843 2844 2845 2846 2847 2848
		   struct stripe_head_state *s)
{
	int abort = 0;
	int i;

	clear_bit(STRIPE_SYNCING, &sh->state);
2849 2850
	if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))
		wake_up(&conf->wait_for_overlap);
2851
	s->syncing = 0;
2852
	s->replacing = 0;
2853
	/* There is nothing more to do for sync/check/repair.
2854 2855 2856
	 * Don't even need to abort as that is handled elsewhere
	 * if needed, and not always wanted e.g. if there is a known
	 * bad block here.
2857
	 * For recover/replace we need to record a bad block on all
2858 2859
	 * non-sync devices, or abort the recovery
	 */
2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882
	if (test_bit(MD_RECOVERY_RECOVER, &conf->mddev->recovery)) {
		/* During recovery devices cannot be removed, so
		 * locking and refcounting of rdevs is not needed
		 */
		for (i = 0; i < conf->raid_disks; i++) {
			struct md_rdev *rdev = conf->disks[i].rdev;
			if (rdev
			    && !test_bit(Faulty, &rdev->flags)
			    && !test_bit(In_sync, &rdev->flags)
			    && !rdev_set_badblocks(rdev, sh->sector,
						   STRIPE_SECTORS, 0))
				abort = 1;
			rdev = conf->disks[i].replacement;
			if (rdev
			    && !test_bit(Faulty, &rdev->flags)
			    && !test_bit(In_sync, &rdev->flags)
			    && !rdev_set_badblocks(rdev, sh->sector,
						   STRIPE_SECTORS, 0))
				abort = 1;
		}
		if (abort)
			conf->recovery_disabled =
				conf->mddev->recovery_disabled;
2883
	}
2884
	md_done_sync(conf->mddev, STRIPE_SECTORS, !abort);
2885 2886
}

2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902
static int want_replace(struct stripe_head *sh, int disk_idx)
{
	struct md_rdev *rdev;
	int rv = 0;
	/* Doing recovery so rcu locking not required */
	rdev = sh->raid_conf->disks[disk_idx].replacement;
	if (rdev
	    && !test_bit(Faulty, &rdev->flags)
	    && !test_bit(In_sync, &rdev->flags)
	    && (rdev->recovery_offset <= sh->sector
		|| rdev->mddev->recovery_cp <= sh->sector))
		rv = 1;

	return rv;
}

2903
/* fetch_block - checks the given member device to see if its data needs
2904 2905 2906
 * to be read or computed to satisfy a request.
 *
 * Returns 1 when no more member devices need to be checked, otherwise returns
2907
 * 0 to tell the loop in handle_stripe_fill to continue
2908
 */
2909 2910
static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s,
		       int disk_idx, int disks)
2911
{
2912
	struct r5dev *dev = &sh->dev[disk_idx];
2913 2914
	struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]],
				  &sh->dev[s->failed_num[1]] };
2915

2916
	/* is the data in this block needed, and can we get it? */
2917 2918 2919 2920 2921
	if (!test_bit(R5_LOCKED, &dev->flags) &&
	    !test_bit(R5_UPTODATE, &dev->flags) &&
	    (dev->toread ||
	     (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)) ||
	     s->syncing || s->expanding ||
2922
	     (s->replacing && want_replace(sh, disk_idx)) ||
2923 2924
	     (s->failed >= 1 && fdev[0]->toread) ||
	     (s->failed >= 2 && fdev[1]->toread) ||
2925
	     (sh->raid_conf->level <= 5 && s->failed && fdev[0]->towrite &&
2926
	      (!test_bit(R5_Insync, &dev->flags) || test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) &&
2927
	      !test_bit(R5_OVERWRITE, &fdev[0]->flags)) ||
2928
	     (sh->raid_conf->level == 6 && s->failed && s->to_write &&
2929
	      s->to_write - s->non_overwrite < sh->raid_conf->raid_disks - 2 &&
2930
	      (!test_bit(R5_Insync, &dev->flags) || test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))))) {
2931 2932 2933 2934 2935 2936
		/* we would like to get this block, possibly by computing it,
		 * otherwise read it if the backing disk is insync
		 */
		BUG_ON(test_bit(R5_Wantcompute, &dev->flags));
		BUG_ON(test_bit(R5_Wantread, &dev->flags));
		if ((s->uptodate == disks - 1) &&
2937 2938
		    (s->failed && (disk_idx == s->failed_num[0] ||
				   disk_idx == s->failed_num[1]))) {
2939 2940
			/* have disk failed, and we're requested to fetch it;
			 * do compute it
2941
			 */
2942 2943 2944 2945 2946 2947 2948 2949
			pr_debug("Computing stripe %llu block %d\n",
			       (unsigned long long)sh->sector, disk_idx);
			set_bit(STRIPE_COMPUTE_RUN, &sh->state);
			set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
			set_bit(R5_Wantcompute, &dev->flags);
			sh->ops.target = disk_idx;
			sh->ops.target2 = -1; /* no 2nd target */
			s->req_compute = 1;
2950 2951 2952 2953 2954 2955
			/* Careful: from this point on 'uptodate' is in the eye
			 * of raid_run_ops which services 'compute' operations
			 * before writes. R5_Wantcompute flags a block that will
			 * be R5_UPTODATE by the time it is needed for a
			 * subsequent operation.
			 */
2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968
			s->uptodate++;
			return 1;
		} else if (s->uptodate == disks-2 && s->failed >= 2) {
			/* Computing 2-failure is *very* expensive; only
			 * do it if failed >= 2
			 */
			int other;
			for (other = disks; other--; ) {
				if (other == disk_idx)
					continue;
				if (!test_bit(R5_UPTODATE,
				      &sh->dev[other].flags))
					break;
2969
			}
2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985 2986 2987 2988
			BUG_ON(other < 0);
			pr_debug("Computing stripe %llu blocks %d,%d\n",
			       (unsigned long long)sh->sector,
			       disk_idx, other);
			set_bit(STRIPE_COMPUTE_RUN, &sh->state);
			set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
			set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags);
			set_bit(R5_Wantcompute, &sh->dev[other].flags);
			sh->ops.target = disk_idx;
			sh->ops.target2 = other;
			s->uptodate += 2;
			s->req_compute = 1;
			return 1;
		} else if (test_bit(R5_Insync, &dev->flags)) {
			set_bit(R5_LOCKED, &dev->flags);
			set_bit(R5_Wantread, &dev->flags);
			s->locked++;
			pr_debug("Reading block %d (sync=%d)\n",
				disk_idx, s->syncing);
2989 2990
		}
	}
2991 2992 2993 2994 2995

	return 0;
}

/**
2996
 * handle_stripe_fill - read or compute data to satisfy pending requests.
2997
 */
2998 2999 3000
static void handle_stripe_fill(struct stripe_head *sh,
			       struct stripe_head_state *s,
			       int disks)
3001 3002 3003 3004 3005 3006 3007 3008 3009 3010
{
	int i;

	/* look for blocks to read/compute, skip this if a compute
	 * is already in flight, or if the stripe contents are in the
	 * midst of changing due to a write
	 */
	if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state &&
	    !sh->reconstruct_state)
		for (i = disks; i--; )
3011
			if (fetch_block(sh, s, i, disks))
3012
				break;
3013 3014 3015 3016
	set_bit(STRIPE_HANDLE, &sh->state);
}


3017
/* handle_stripe_clean_event
3018 3019 3020 3021
 * any written block on an uptodate or failed drive can be returned.
 * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but
 * never LOCKED, so we don't need to test 'failed' directly.
 */
3022
static void handle_stripe_clean_event(struct r5conf *conf,
3023 3024 3025 3026
	struct stripe_head *sh, int disks, struct bio **return_bi)
{
	int i;
	struct r5dev *dev;
3027
	int discard_pending = 0;
3028 3029 3030 3031 3032

	for (i = disks; i--; )
		if (sh->dev[i].written) {
			dev = &sh->dev[i];
			if (!test_bit(R5_LOCKED, &dev->flags) &&
3033
			    (test_bit(R5_UPTODATE, &dev->flags) ||
3034 3035
			     test_bit(R5_Discard, &dev->flags) ||
			     test_bit(R5_SkipCopy, &dev->flags))) {
3036 3037
				/* We can return any write requests */
				struct bio *wbi, *wbi2;
3038
				pr_debug("Return write for disc %d\n", i);
3039 3040
				if (test_and_clear_bit(R5_Discard, &dev->flags))
					clear_bit(R5_UPTODATE, &dev->flags);
3041 3042 3043 3044
				if (test_and_clear_bit(R5_SkipCopy, &dev->flags)) {
					WARN_ON(test_bit(R5_UPTODATE, &dev->flags));
					dev->page = dev->orig_page;
				}
3045 3046
				wbi = dev->written;
				dev->written = NULL;
3047
				while (wbi && wbi->bi_iter.bi_sector <
3048 3049
					dev->sector + STRIPE_SECTORS) {
					wbi2 = r5_next_bio(wbi, dev->sector);
3050
					if (!raid5_dec_bi_active_stripes(wbi)) {
3051 3052 3053 3054 3055 3056
						md_write_end(conf->mddev);
						wbi->bi_next = *return_bi;
						*return_bi = wbi;
					}
					wbi = wbi2;
				}
3057 3058
				bitmap_endwrite(conf->mddev->bitmap, sh->sector,
						STRIPE_SECTORS,
3059
					 !test_bit(STRIPE_DEGRADED, &sh->state),
3060
						0);
3061 3062
			} else if (test_bit(R5_Discard, &dev->flags))
				discard_pending = 1;
3063 3064
			WARN_ON(test_bit(R5_SkipCopy, &dev->flags));
			WARN_ON(dev->page != dev->orig_page);
3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075
		}
	if (!discard_pending &&
	    test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)) {
		clear_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);
		clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
		if (sh->qd_idx >= 0) {
			clear_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);
			clear_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags);
		}
		/* now that discard is done we can proceed with any sync */
		clear_bit(STRIPE_DISCARD, &sh->state);
S
Shaohua Li 已提交
3076 3077 3078 3079 3080 3081 3082 3083
		/*
		 * SCSI discard will change some bio fields and the stripe has
		 * no updated data, so remove it from hash list and the stripe
		 * will be reinitialized
		 */
		spin_lock_irq(&conf->device_lock);
		remove_hash(sh);
		spin_unlock_irq(&conf->device_lock);
3084 3085 3086 3087
		if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state))
			set_bit(STRIPE_HANDLE, &sh->state);

	}
3088 3089 3090 3091

	if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
		if (atomic_dec_and_test(&conf->pending_full_writes))
			md_wakeup_thread(conf->mddev->thread);
3092 3093
}

3094
static void handle_stripe_dirtying(struct r5conf *conf,
3095 3096 3097
				   struct stripe_head *sh,
				   struct stripe_head_state *s,
				   int disks)
3098 3099
{
	int rmw = 0, rcw = 0, i;
3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112
	sector_t recovery_cp = conf->mddev->recovery_cp;

	/* RAID6 requires 'rcw' in current implementation.
	 * Otherwise, check whether resync is now happening or should start.
	 * If yes, then the array is dirty (after unclean shutdown or
	 * initial creation), so parity in some stripes might be inconsistent.
	 * In this case, we need to always do reconstruct-write, to ensure
	 * that in case of drive failure or read-error correction, we
	 * generate correct data from the parity.
	 */
	if (conf->max_degraded == 2 ||
	    (recovery_cp < MaxSector && sh->sector >= recovery_cp)) {
		/* Calculate the real rcw later - for now make it
3113 3114 3115
		 * look like rcw is cheaper
		 */
		rcw = 1; rmw = 2;
3116 3117 3118
		pr_debug("force RCW max_degraded=%u, recovery_cp=%llu sh->sector=%llu\n",
			 conf->max_degraded, (unsigned long long)recovery_cp,
			 (unsigned long long)sh->sector);
3119
	} else for (i = disks; i--; ) {
3120 3121 3122 3123
		/* would I have to read this buffer for read_modify_write */
		struct r5dev *dev = &sh->dev[i];
		if ((dev->towrite || i == sh->pd_idx) &&
		    !test_bit(R5_LOCKED, &dev->flags) &&
3124 3125
		    !(test_bit(R5_UPTODATE, &dev->flags) ||
		      test_bit(R5_Wantcompute, &dev->flags))) {
3126 3127 3128 3129 3130 3131 3132 3133
			if (test_bit(R5_Insync, &dev->flags))
				rmw++;
			else
				rmw += 2*disks;  /* cannot read it */
		}
		/* Would I have to read this buffer for reconstruct_write */
		if (!test_bit(R5_OVERWRITE, &dev->flags) && i != sh->pd_idx &&
		    !test_bit(R5_LOCKED, &dev->flags) &&
3134 3135
		    !(test_bit(R5_UPTODATE, &dev->flags) ||
		    test_bit(R5_Wantcompute, &dev->flags))) {
3136 3137
			if (test_bit(R5_Insync, &dev->flags))
				rcw++;
3138 3139 3140 3141
			else
				rcw += 2*disks;
		}
	}
3142
	pr_debug("for sector %llu, rmw=%d rcw=%d\n",
3143 3144
		(unsigned long long)sh->sector, rmw, rcw);
	set_bit(STRIPE_HANDLE, &sh->state);
N
NeilBrown 已提交
3145
	if (rmw < rcw && rmw > 0) {
3146
		/* prefer read-modify-write, but need to get some data */
3147 3148 3149 3150
		if (conf->mddev->queue)
			blk_add_trace_msg(conf->mddev->queue,
					  "raid5 rmw %llu %d",
					  (unsigned long long)sh->sector, rmw);
3151 3152 3153 3154
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if ((dev->towrite || i == sh->pd_idx) &&
			    !test_bit(R5_LOCKED, &dev->flags) &&
3155 3156
			    !(test_bit(R5_UPTODATE, &dev->flags) ||
			    test_bit(R5_Wantcompute, &dev->flags)) &&
3157
			    test_bit(R5_Insync, &dev->flags)) {
3158 3159 3160 3161
				if (test_bit(STRIPE_PREREAD_ACTIVE,
					     &sh->state)) {
					pr_debug("Read_old block %d for r-m-w\n",
						 i);
3162 3163 3164 3165 3166 3167 3168 3169 3170
					set_bit(R5_LOCKED, &dev->flags);
					set_bit(R5_Wantread, &dev->flags);
					s->locked++;
				} else {
					set_bit(STRIPE_DELAYED, &sh->state);
					set_bit(STRIPE_HANDLE, &sh->state);
				}
			}
		}
N
NeilBrown 已提交
3171
	}
3172
	if (rcw <= rmw && rcw > 0) {
3173
		/* want reconstruct write, but need to get some data */
N
NeilBrown 已提交
3174
		int qread =0;
3175
		rcw = 0;
3176 3177 3178
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if (!test_bit(R5_OVERWRITE, &dev->flags) &&
3179
			    i != sh->pd_idx && i != sh->qd_idx &&
3180
			    !test_bit(R5_LOCKED, &dev->flags) &&
3181
			    !(test_bit(R5_UPTODATE, &dev->flags) ||
3182 3183
			      test_bit(R5_Wantcompute, &dev->flags))) {
				rcw++;
3184 3185 3186
				if (test_bit(R5_Insync, &dev->flags) &&
				    test_bit(STRIPE_PREREAD_ACTIVE,
					     &sh->state)) {
3187
					pr_debug("Read_old block "
3188 3189 3190 3191
						"%d for Reconstruct\n", i);
					set_bit(R5_LOCKED, &dev->flags);
					set_bit(R5_Wantread, &dev->flags);
					s->locked++;
N
NeilBrown 已提交
3192
					qread++;
3193 3194 3195 3196 3197 3198
				} else {
					set_bit(STRIPE_DELAYED, &sh->state);
					set_bit(STRIPE_HANDLE, &sh->state);
				}
			}
		}
3199
		if (rcw && conf->mddev->queue)
N
NeilBrown 已提交
3200 3201 3202
			blk_add_trace_msg(conf->mddev->queue, "raid5 rcw %llu %d %d %d",
					  (unsigned long long)sh->sector,
					  rcw, qread, test_bit(STRIPE_DELAYED, &sh->state));
3203
	}
3204 3205 3206
	/* now if nothing is locked, and if we have enough data,
	 * we can start a write request
	 */
3207 3208
	/* since handle_stripe can be called at any time we need to handle the
	 * case where a compute block operation has been submitted and then a
3209 3210
	 * subsequent call wants to start a write request.  raid_run_ops only
	 * handles the case where compute block and reconstruct are requested
3211 3212 3213
	 * simultaneously.  If this is not the case then new writes need to be
	 * held off until the compute completes.
	 */
3214 3215 3216
	if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) &&
	    (s->locked == 0 && (rcw == 0 || rmw == 0) &&
	    !test_bit(STRIPE_BIT_DELAY, &sh->state)))
3217
		schedule_reconstruction(sh, s, rcw == 0, 0);
3218 3219
}

3220
static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh,
3221 3222
				struct stripe_head_state *s, int disks)
{
3223
	struct r5dev *dev = NULL;
3224

3225
	set_bit(STRIPE_HANDLE, &sh->state);
3226

3227 3228 3229
	switch (sh->check_state) {
	case check_state_idle:
		/* start a new check operation if there are no failures */
3230 3231
		if (s->failed == 0) {
			BUG_ON(s->uptodate != disks);
3232 3233
			sh->check_state = check_state_run;
			set_bit(STRIPE_OP_CHECK, &s->ops_request);
3234 3235
			clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
			s->uptodate--;
3236
			break;
3237
		}
3238
		dev = &sh->dev[s->failed_num[0]];
3239 3240 3241 3242 3243 3244 3245 3246 3247
		/* fall through */
	case check_state_compute_result:
		sh->check_state = check_state_idle;
		if (!dev)
			dev = &sh->dev[sh->pd_idx];

		/* check that a write has not made the stripe insync */
		if (test_bit(STRIPE_INSYNC, &sh->state))
			break;
D
Dan Williams 已提交
3248

3249 3250 3251 3252 3253
		/* either failed parity check, or recovery is happening */
		BUG_ON(!test_bit(R5_UPTODATE, &dev->flags));
		BUG_ON(s->uptodate != disks);

		set_bit(R5_LOCKED, &dev->flags);
3254
		s->locked++;
3255
		set_bit(R5_Wantwrite, &dev->flags);
3256

3257 3258
		clear_bit(STRIPE_DEGRADED, &sh->state);
		set_bit(STRIPE_INSYNC, &sh->state);
3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274
		break;
	case check_state_run:
		break; /* we will be called again upon completion */
	case check_state_check_result:
		sh->check_state = check_state_idle;

		/* if a failure occurred during the check operation, leave
		 * STRIPE_INSYNC not set and let the stripe be handled again
		 */
		if (s->failed)
			break;

		/* handle a successful check operation, if parity is correct
		 * we are done.  Otherwise update the mismatch count and repair
		 * parity if !MD_RECOVERY_CHECK
		 */
D
Dan Williams 已提交
3275
		if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0)
3276 3277 3278 3279 3280
			/* parity is correct (on disc,
			 * not in buffer any more)
			 */
			set_bit(STRIPE_INSYNC, &sh->state);
		else {
3281
			atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches);
3282 3283 3284 3285 3286
			if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery))
				/* don't try to repair!! */
				set_bit(STRIPE_INSYNC, &sh->state);
			else {
				sh->check_state = check_state_compute_run;
3287
				set_bit(STRIPE_COMPUTE_RUN, &sh->state);
3288 3289 3290 3291
				set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
				set_bit(R5_Wantcompute,
					&sh->dev[sh->pd_idx].flags);
				sh->ops.target = sh->pd_idx;
3292
				sh->ops.target2 = -1;
3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303
				s->uptodate++;
			}
		}
		break;
	case check_state_compute_run:
		break;
	default:
		printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
		       __func__, sh->check_state,
		       (unsigned long long) sh->sector);
		BUG();
3304 3305 3306 3307
	}
}


3308
static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh,
3309
				  struct stripe_head_state *s,
3310
				  int disks)
3311 3312
{
	int pd_idx = sh->pd_idx;
N
NeilBrown 已提交
3313
	int qd_idx = sh->qd_idx;
3314
	struct r5dev *dev;
3315 3316 3317 3318

	set_bit(STRIPE_HANDLE, &sh->state);

	BUG_ON(s->failed > 2);
3319

3320 3321 3322 3323 3324 3325
	/* Want to check and possibly repair P and Q.
	 * However there could be one 'failed' device, in which
	 * case we can only check one of them, possibly using the
	 * other to generate missing data
	 */

3326 3327 3328
	switch (sh->check_state) {
	case check_state_idle:
		/* start a new check operation if there are < 2 failures */
3329
		if (s->failed == s->q_failed) {
3330
			/* The only possible failed device holds Q, so it
3331 3332 3333
			 * makes sense to check P (If anything else were failed,
			 * we would have used P to recreate it).
			 */
3334
			sh->check_state = check_state_run;
3335
		}
3336
		if (!s->q_failed && s->failed < 2) {
3337
			/* Q is not failed, and we didn't use it to generate
3338 3339
			 * anything, so it makes sense to check it
			 */
3340 3341 3342 3343
			if (sh->check_state == check_state_run)
				sh->check_state = check_state_run_pq;
			else
				sh->check_state = check_state_run_q;
3344 3345
		}

3346 3347
		/* discard potentially stale zero_sum_result */
		sh->ops.zero_sum_result = 0;
3348

3349 3350 3351 3352
		if (sh->check_state == check_state_run) {
			/* async_xor_zero_sum destroys the contents of P */
			clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
			s->uptodate--;
3353
		}
3354 3355 3356 3357 3358 3359 3360
		if (sh->check_state >= check_state_run &&
		    sh->check_state <= check_state_run_pq) {
			/* async_syndrome_zero_sum preserves P and Q, so
			 * no need to mark them !uptodate here
			 */
			set_bit(STRIPE_OP_CHECK, &s->ops_request);
			break;
3361 3362
		}

3363 3364 3365 3366 3367
		/* we have 2-disk failure */
		BUG_ON(s->failed != 2);
		/* fall through */
	case check_state_compute_result:
		sh->check_state = check_state_idle;
3368

3369 3370 3371
		/* check that a write has not made the stripe insync */
		if (test_bit(STRIPE_INSYNC, &sh->state))
			break;
3372 3373

		/* now write out any block on a failed drive,
3374
		 * or P or Q if they were recomputed
3375
		 */
3376
		BUG_ON(s->uptodate < disks - 1); /* We don't need Q to recover */
3377
		if (s->failed == 2) {
3378
			dev = &sh->dev[s->failed_num[1]];
3379 3380 3381 3382 3383
			s->locked++;
			set_bit(R5_LOCKED, &dev->flags);
			set_bit(R5_Wantwrite, &dev->flags);
		}
		if (s->failed >= 1) {
3384
			dev = &sh->dev[s->failed_num[0]];
3385 3386 3387 3388
			s->locked++;
			set_bit(R5_LOCKED, &dev->flags);
			set_bit(R5_Wantwrite, &dev->flags);
		}
3389
		if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
3390 3391 3392 3393 3394
			dev = &sh->dev[pd_idx];
			s->locked++;
			set_bit(R5_LOCKED, &dev->flags);
			set_bit(R5_Wantwrite, &dev->flags);
		}
3395
		if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
3396 3397 3398 3399 3400 3401 3402 3403
			dev = &sh->dev[qd_idx];
			s->locked++;
			set_bit(R5_LOCKED, &dev->flags);
			set_bit(R5_Wantwrite, &dev->flags);
		}
		clear_bit(STRIPE_DEGRADED, &sh->state);

		set_bit(STRIPE_INSYNC, &sh->state);
3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432
		break;
	case check_state_run:
	case check_state_run_q:
	case check_state_run_pq:
		break; /* we will be called again upon completion */
	case check_state_check_result:
		sh->check_state = check_state_idle;

		/* handle a successful check operation, if parity is correct
		 * we are done.  Otherwise update the mismatch count and repair
		 * parity if !MD_RECOVERY_CHECK
		 */
		if (sh->ops.zero_sum_result == 0) {
			/* both parities are correct */
			if (!s->failed)
				set_bit(STRIPE_INSYNC, &sh->state);
			else {
				/* in contrast to the raid5 case we can validate
				 * parity, but still have a failure to write
				 * back
				 */
				sh->check_state = check_state_compute_result;
				/* Returning at this point means that we may go
				 * off and bring p and/or q uptodate again so
				 * we make sure to check zero_sum_result again
				 * to verify if p or q need writeback
				 */
			}
		} else {
3433
			atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches);
3434 3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467
			if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery))
				/* don't try to repair!! */
				set_bit(STRIPE_INSYNC, &sh->state);
			else {
				int *target = &sh->ops.target;

				sh->ops.target = -1;
				sh->ops.target2 = -1;
				sh->check_state = check_state_compute_run;
				set_bit(STRIPE_COMPUTE_RUN, &sh->state);
				set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
				if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
					set_bit(R5_Wantcompute,
						&sh->dev[pd_idx].flags);
					*target = pd_idx;
					target = &sh->ops.target2;
					s->uptodate++;
				}
				if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
					set_bit(R5_Wantcompute,
						&sh->dev[qd_idx].flags);
					*target = qd_idx;
					s->uptodate++;
				}
			}
		}
		break;
	case check_state_compute_run:
		break;
	default:
		printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
		       __func__, sh->check_state,
		       (unsigned long long) sh->sector);
		BUG();
3468 3469 3470
	}
}

3471
static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh)
3472 3473 3474 3475 3476 3477
{
	int i;

	/* We have read all the blocks in this stripe and now we need to
	 * copy some of them into a target stripe for expand.
	 */
3478
	struct dma_async_tx_descriptor *tx = NULL;
3479 3480
	clear_bit(STRIPE_EXPAND_SOURCE, &sh->state);
	for (i = 0; i < sh->disks; i++)
N
NeilBrown 已提交
3481
		if (i != sh->pd_idx && i != sh->qd_idx) {
3482
			int dd_idx, j;
3483
			struct stripe_head *sh2;
3484
			struct async_submit_ctl submit;
3485

3486
			sector_t bn = compute_blocknr(sh, i, 1);
3487 3488
			sector_t s = raid5_compute_sector(conf, bn, 0,
							  &dd_idx, NULL);
3489
			sh2 = get_active_stripe(conf, s, 0, 1, 1);
3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501
			if (sh2 == NULL)
				/* so far only the early blocks of this stripe
				 * have been requested.  When later blocks
				 * get requested, we will try again
				 */
				continue;
			if (!test_bit(STRIPE_EXPANDING, &sh2->state) ||
			   test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) {
				/* must have already done this block */
				release_stripe(sh2);
				continue;
			}
3502 3503

			/* place all the copies on one channel */
3504
			init_async_submit(&submit, 0, tx, NULL, NULL, NULL);
3505
			tx = async_memcpy(sh2->dev[dd_idx].page,
3506
					  sh->dev[i].page, 0, 0, STRIPE_SIZE,
3507
					  &submit);
3508

3509 3510 3511 3512
			set_bit(R5_Expanded, &sh2->dev[dd_idx].flags);
			set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags);
			for (j = 0; j < conf->raid_disks; j++)
				if (j != sh2->pd_idx &&
3513
				    j != sh2->qd_idx &&
3514 3515 3516 3517 3518 3519 3520
				    !test_bit(R5_Expanded, &sh2->dev[j].flags))
					break;
			if (j == conf->raid_disks) {
				set_bit(STRIPE_EXPAND_READY, &sh2->state);
				set_bit(STRIPE_HANDLE, &sh2->state);
			}
			release_stripe(sh2);
3521

3522
		}
3523
	/* done submitting copies, wait for them to complete */
3524
	async_tx_quiesce(&tx);
3525
}
L
Linus Torvalds 已提交
3526 3527 3528 3529

/*
 * handle_stripe - do things to a stripe.
 *
3530 3531
 * We lock the stripe by setting STRIPE_ACTIVE and then examine the
 * state of various bits to see what needs to be done.
L
Linus Torvalds 已提交
3532
 * Possible results:
3533 3534
 *    return some read requests which now have data
 *    return some write requests which are safely on storage
L
Linus Torvalds 已提交
3535 3536 3537 3538 3539
 *    schedule a read on some buffers
 *    schedule a write of some buffers
 *    return confirmation of parity correctness
 *
 */
3540

3541
static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)
L
Linus Torvalds 已提交
3542
{
3543
	struct r5conf *conf = sh->raid_conf;
3544
	int disks = sh->disks;
3545 3546
	struct r5dev *dev;
	int i;
3547
	int do_recovery = 0;
L
Linus Torvalds 已提交
3548

3549 3550 3551 3552 3553 3554
	memset(s, 0, sizeof(*s));

	s->expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state);
	s->expanded = test_bit(STRIPE_EXPAND_READY, &sh->state);
	s->failed_num[0] = -1;
	s->failed_num[1] = -1;
L
Linus Torvalds 已提交
3555

3556
	/* Now to look around and see what can be done */
L
Linus Torvalds 已提交
3557
	rcu_read_lock();
3558
	for (i=disks; i--; ) {
3559
		struct md_rdev *rdev;
3560 3561 3562
		sector_t first_bad;
		int bad_sectors;
		int is_bad = 0;
3563

3564
		dev = &sh->dev[i];
L
Linus Torvalds 已提交
3565

3566
		pr_debug("check %d: state 0x%lx read %p write %p written %p\n",
3567 3568
			 i, dev->flags,
			 dev->toread, dev->towrite, dev->written);
3569 3570 3571 3572 3573 3574 3575 3576
		/* maybe we can reply to a read
		 *
		 * new wantfill requests are only permitted while
		 * ops_complete_biofill is guaranteed to be inactive
		 */
		if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread &&
		    !test_bit(STRIPE_BIOFILL_RUN, &sh->state))
			set_bit(R5_Wantfill, &dev->flags);
L
Linus Torvalds 已提交
3577

3578
		/* now count some things */
3579 3580 3581 3582
		if (test_bit(R5_LOCKED, &dev->flags))
			s->locked++;
		if (test_bit(R5_UPTODATE, &dev->flags))
			s->uptodate++;
3583
		if (test_bit(R5_Wantcompute, &dev->flags)) {
3584 3585
			s->compute++;
			BUG_ON(s->compute > 2);
3586
		}
L
Linus Torvalds 已提交
3587

3588
		if (test_bit(R5_Wantfill, &dev->flags))
3589
			s->to_fill++;
3590
		else if (dev->toread)
3591
			s->to_read++;
3592
		if (dev->towrite) {
3593
			s->to_write++;
3594
			if (!test_bit(R5_OVERWRITE, &dev->flags))
3595
				s->non_overwrite++;
3596
		}
3597
		if (dev->written)
3598
			s->written++;
3599 3600 3601 3602 3603 3604 3605 3606 3607 3608
		/* Prefer to use the replacement for reads, but only
		 * if it is recovered enough and has no bad blocks.
		 */
		rdev = rcu_dereference(conf->disks[i].replacement);
		if (rdev && !test_bit(Faulty, &rdev->flags) &&
		    rdev->recovery_offset >= sh->sector + STRIPE_SECTORS &&
		    !is_badblock(rdev, sh->sector, STRIPE_SECTORS,
				 &first_bad, &bad_sectors))
			set_bit(R5_ReadRepl, &dev->flags);
		else {
3609 3610
			if (rdev)
				set_bit(R5_NeedReplace, &dev->flags);
3611 3612 3613
			rdev = rcu_dereference(conf->disks[i].rdev);
			clear_bit(R5_ReadRepl, &dev->flags);
		}
3614 3615
		if (rdev && test_bit(Faulty, &rdev->flags))
			rdev = NULL;
3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627
		if (rdev) {
			is_bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS,
					     &first_bad, &bad_sectors);
			if (s->blocked_rdev == NULL
			    && (test_bit(Blocked, &rdev->flags)
				|| is_bad < 0)) {
				if (is_bad < 0)
					set_bit(BlockedBadBlocks,
						&rdev->flags);
				s->blocked_rdev = rdev;
				atomic_inc(&rdev->nr_pending);
			}
3628
		}
3629 3630 3631
		clear_bit(R5_Insync, &dev->flags);
		if (!rdev)
			/* Not in-sync */;
3632 3633
		else if (is_bad) {
			/* also not in-sync */
3634 3635
			if (!test_bit(WriteErrorSeen, &rdev->flags) &&
			    test_bit(R5_UPTODATE, &dev->flags)) {
3636 3637 3638 3639 3640 3641 3642
				/* treat as in-sync, but with a read error
				 * which we can now try to correct
				 */
				set_bit(R5_Insync, &dev->flags);
				set_bit(R5_ReadError, &dev->flags);
			}
		} else if (test_bit(In_sync, &rdev->flags))
3643
			set_bit(R5_Insync, &dev->flags);
3644
		else if (sh->sector + STRIPE_SECTORS <= rdev->recovery_offset)
3645
			/* in sync if before recovery_offset */
3646 3647 3648 3649 3650 3651 3652 3653 3654
			set_bit(R5_Insync, &dev->flags);
		else if (test_bit(R5_UPTODATE, &dev->flags) &&
			 test_bit(R5_Expanded, &dev->flags))
			/* If we've reshaped into here, we assume it is Insync.
			 * We will shortly update recovery_offset to make
			 * it official.
			 */
			set_bit(R5_Insync, &dev->flags);

3655
		if (test_bit(R5_WriteError, &dev->flags)) {
3656 3657 3658 3659 3660 3661 3662
			/* This flag does not apply to '.replacement'
			 * only to .rdev, so make sure to check that*/
			struct md_rdev *rdev2 = rcu_dereference(
				conf->disks[i].rdev);
			if (rdev2 == rdev)
				clear_bit(R5_Insync, &dev->flags);
			if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
3663
				s->handle_bad_blocks = 1;
3664
				atomic_inc(&rdev2->nr_pending);
3665 3666 3667
			} else
				clear_bit(R5_WriteError, &dev->flags);
		}
3668
		if (test_bit(R5_MadeGood, &dev->flags)) {
3669 3670 3671 3672 3673
			/* This flag does not apply to '.replacement'
			 * only to .rdev, so make sure to check that*/
			struct md_rdev *rdev2 = rcu_dereference(
				conf->disks[i].rdev);
			if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
3674
				s->handle_bad_blocks = 1;
3675
				atomic_inc(&rdev2->nr_pending);
3676 3677 3678
			} else
				clear_bit(R5_MadeGood, &dev->flags);
		}
3679 3680 3681 3682 3683 3684 3685 3686 3687
		if (test_bit(R5_MadeGoodRepl, &dev->flags)) {
			struct md_rdev *rdev2 = rcu_dereference(
				conf->disks[i].replacement);
			if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
				s->handle_bad_blocks = 1;
				atomic_inc(&rdev2->nr_pending);
			} else
				clear_bit(R5_MadeGoodRepl, &dev->flags);
		}
3688
		if (!test_bit(R5_Insync, &dev->flags)) {
3689 3690 3691
			/* The ReadError flag will just be confusing now */
			clear_bit(R5_ReadError, &dev->flags);
			clear_bit(R5_ReWrite, &dev->flags);
L
Linus Torvalds 已提交
3692
		}
3693 3694 3695
		if (test_bit(R5_ReadError, &dev->flags))
			clear_bit(R5_Insync, &dev->flags);
		if (!test_bit(R5_Insync, &dev->flags)) {
3696 3697 3698
			if (s->failed < 2)
				s->failed_num[s->failed] = i;
			s->failed++;
3699 3700
			if (rdev && !test_bit(Faulty, &rdev->flags))
				do_recovery = 1;
3701
		}
L
Linus Torvalds 已提交
3702
	}
3703 3704 3705 3706
	if (test_bit(STRIPE_SYNCING, &sh->state)) {
		/* If there is a failed device being replaced,
		 *     we must be recovering.
		 * else if we are after recovery_cp, we must be syncing
3707
		 * else if MD_RECOVERY_REQUESTED is set, we also are syncing.
3708 3709 3710 3711 3712
		 * else we can only be replacing
		 * sync and recovery both need to read all devices, and so
		 * use the same flag.
		 */
		if (do_recovery ||
3713 3714
		    sh->sector >= conf->mddev->recovery_cp ||
		    test_bit(MD_RECOVERY_REQUESTED, &(conf->mddev->recovery)))
3715 3716 3717 3718
			s->syncing = 1;
		else
			s->replacing = 1;
	}
L
Linus Torvalds 已提交
3719
	rcu_read_unlock();
3720 3721 3722 3723 3724
}

static void handle_stripe(struct stripe_head *sh)
{
	struct stripe_head_state s;
3725
	struct r5conf *conf = sh->raid_conf;
3726
	int i;
3727 3728
	int prexor;
	int disks = sh->disks;
3729
	struct r5dev *pdev, *qdev;
3730 3731

	clear_bit(STRIPE_HANDLE, &sh->state);
3732
	if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) {
3733 3734 3735 3736 3737 3738
		/* already being handled, ensure it gets handled
		 * again when current action finishes */
		set_bit(STRIPE_HANDLE, &sh->state);
		return;
	}

3739 3740 3741 3742 3743 3744 3745
	if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state)) {
		spin_lock(&sh->stripe_lock);
		/* Cannot process 'sync' concurrently with 'discard' */
		if (!test_bit(STRIPE_DISCARD, &sh->state) &&
		    test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) {
			set_bit(STRIPE_SYNCING, &sh->state);
			clear_bit(STRIPE_INSYNC, &sh->state);
3746
			clear_bit(STRIPE_REPLACED, &sh->state);
3747 3748
		}
		spin_unlock(&sh->stripe_lock);
3749 3750 3751 3752 3753 3754 3755 3756
	}
	clear_bit(STRIPE_DELAYED, &sh->state);

	pr_debug("handling stripe %llu, state=%#lx cnt=%d, "
		"pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n",
	       (unsigned long long)sh->sector, sh->state,
	       atomic_read(&sh->count), sh->pd_idx, sh->qd_idx,
	       sh->check_state, sh->reconstruct_state);
3757

3758
	analyse_stripe(sh, &s);
3759

3760 3761 3762 3763 3764
	if (s.handle_bad_blocks) {
		set_bit(STRIPE_HANDLE, &sh->state);
		goto finish;
	}

3765 3766
	if (unlikely(s.blocked_rdev)) {
		if (s.syncing || s.expanding || s.expanded ||
3767
		    s.replacing || s.to_write || s.written) {
3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787
			set_bit(STRIPE_HANDLE, &sh->state);
			goto finish;
		}
		/* There is nothing for the blocked_rdev to block */
		rdev_dec_pending(s.blocked_rdev, conf->mddev);
		s.blocked_rdev = NULL;
	}

	if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) {
		set_bit(STRIPE_OP_BIOFILL, &s.ops_request);
		set_bit(STRIPE_BIOFILL_RUN, &sh->state);
	}

	pr_debug("locked=%d uptodate=%d to_read=%d"
	       " to_write=%d failed=%d failed_num=%d,%d\n",
	       s.locked, s.uptodate, s.to_read, s.to_write, s.failed,
	       s.failed_num[0], s.failed_num[1]);
	/* check if the array has lost more than max_degraded devices and,
	 * if so, some requests might need to be failed.
	 */
3788 3789 3790 3791 3792
	if (s.failed > conf->max_degraded) {
		sh->check_state = 0;
		sh->reconstruct_state = 0;
		if (s.to_read+s.to_write+s.written)
			handle_failed_stripe(conf, sh, &s, disks, &s.return_bi);
3793
		if (s.syncing + s.replacing)
3794 3795
			handle_failed_sync(conf, sh, &s);
	}
3796

3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809
	/* Now we check to see if any write operations have recently
	 * completed
	 */
	prexor = 0;
	if (sh->reconstruct_state == reconstruct_state_prexor_drain_result)
		prexor = 1;
	if (sh->reconstruct_state == reconstruct_state_drain_result ||
	    sh->reconstruct_state == reconstruct_state_prexor_drain_result) {
		sh->reconstruct_state = reconstruct_state_idle;

		/* All the 'written' buffers and the parity block are ready to
		 * be written back to disk
		 */
3810 3811
		BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags) &&
		       !test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags));
3812
		BUG_ON(sh->qd_idx >= 0 &&
3813 3814
		       !test_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags) &&
		       !test_bit(R5_Discard, &sh->dev[sh->qd_idx].flags));
3815 3816 3817 3818 3819 3820 3821 3822 3823
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if (test_bit(R5_LOCKED, &dev->flags) &&
				(i == sh->pd_idx || i == sh->qd_idx ||
				 dev->written)) {
				pr_debug("Writing block %d\n", i);
				set_bit(R5_Wantwrite, &dev->flags);
				if (prexor)
					continue;
3824 3825
				if (s.failed > 1)
					continue;
3826 3827 3828 3829 3830 3831 3832 3833 3834 3835
				if (!test_bit(R5_Insync, &dev->flags) ||
				    ((i == sh->pd_idx || i == sh->qd_idx)  &&
				     s.failed == 0))
					set_bit(STRIPE_INSYNC, &sh->state);
			}
		}
		if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
			s.dec_preread_active = 1;
	}

3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869
	/*
	 * might be able to return some write requests if the parity blocks
	 * are safe, or on a failed drive
	 */
	pdev = &sh->dev[sh->pd_idx];
	s.p_failed = (s.failed >= 1 && s.failed_num[0] == sh->pd_idx)
		|| (s.failed >= 2 && s.failed_num[1] == sh->pd_idx);
	qdev = &sh->dev[sh->qd_idx];
	s.q_failed = (s.failed >= 1 && s.failed_num[0] == sh->qd_idx)
		|| (s.failed >= 2 && s.failed_num[1] == sh->qd_idx)
		|| conf->level < 6;

	if (s.written &&
	    (s.p_failed || ((test_bit(R5_Insync, &pdev->flags)
			     && !test_bit(R5_LOCKED, &pdev->flags)
			     && (test_bit(R5_UPTODATE, &pdev->flags) ||
				 test_bit(R5_Discard, &pdev->flags))))) &&
	    (s.q_failed || ((test_bit(R5_Insync, &qdev->flags)
			     && !test_bit(R5_LOCKED, &qdev->flags)
			     && (test_bit(R5_UPTODATE, &qdev->flags) ||
				 test_bit(R5_Discard, &qdev->flags))))))
		handle_stripe_clean_event(conf, sh, disks, &s.return_bi);

	/* Now we might consider reading some blocks, either to check/generate
	 * parity, or to satisfy requests
	 * or to load a block that is being partially written.
	 */
	if (s.to_read || s.non_overwrite
	    || (conf->level == 6 && s.to_write && s.failed)
	    || (s.syncing && (s.uptodate + s.compute < disks))
	    || s.replacing
	    || s.expanding)
		handle_stripe_fill(sh, &s, disks);

3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892
	/* Now to consider new write requests and what else, if anything
	 * should be read.  We do not handle new writes when:
	 * 1/ A 'write' operation (copy+xor) is already in flight.
	 * 2/ A 'check' operation is in flight, as it may clobber the parity
	 *    block.
	 */
	if (s.to_write && !sh->reconstruct_state && !sh->check_state)
		handle_stripe_dirtying(conf, sh, &s, disks);

	/* maybe we need to check and possibly fix the parity for this stripe
	 * Any reads will already have been scheduled, so we just see if enough
	 * data is available.  The parity check is held off while parity
	 * dependent operations are in flight.
	 */
	if (sh->check_state ||
	    (s.syncing && s.locked == 0 &&
	     !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
	     !test_bit(STRIPE_INSYNC, &sh->state))) {
		if (conf->level == 6)
			handle_parity_checks6(conf, sh, &s, disks);
		else
			handle_parity_checks5(conf, sh, &s, disks);
	}
3893

3894 3895 3896
	if ((s.replacing || s.syncing) && s.locked == 0
	    && !test_bit(STRIPE_COMPUTE_RUN, &sh->state)
	    && !test_bit(STRIPE_REPLACED, &sh->state)) {
3897 3898
		/* Write out to replacement devices where possible */
		for (i = 0; i < conf->raid_disks; i++)
3899 3900
			if (test_bit(R5_NeedReplace, &sh->dev[i].flags)) {
				WARN_ON(!test_bit(R5_UPTODATE, &sh->dev[i].flags));
3901 3902 3903 3904
				set_bit(R5_WantReplace, &sh->dev[i].flags);
				set_bit(R5_LOCKED, &sh->dev[i].flags);
				s.locked++;
			}
3905 3906 3907
		if (s.replacing)
			set_bit(STRIPE_INSYNC, &sh->state);
		set_bit(STRIPE_REPLACED, &sh->state);
3908 3909
	}
	if ((s.syncing || s.replacing) && s.locked == 0 &&
3910
	    !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
3911
	    test_bit(STRIPE_INSYNC, &sh->state)) {
3912 3913
		md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
		clear_bit(STRIPE_SYNCING, &sh->state);
3914 3915
		if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))
			wake_up(&conf->wait_for_overlap);
3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942
	}

	/* If the failed drives are just a ReadError, then we might need
	 * to progress the repair/check process
	 */
	if (s.failed <= conf->max_degraded && !conf->mddev->ro)
		for (i = 0; i < s.failed; i++) {
			struct r5dev *dev = &sh->dev[s.failed_num[i]];
			if (test_bit(R5_ReadError, &dev->flags)
			    && !test_bit(R5_LOCKED, &dev->flags)
			    && test_bit(R5_UPTODATE, &dev->flags)
				) {
				if (!test_bit(R5_ReWrite, &dev->flags)) {
					set_bit(R5_Wantwrite, &dev->flags);
					set_bit(R5_ReWrite, &dev->flags);
					set_bit(R5_LOCKED, &dev->flags);
					s.locked++;
				} else {
					/* let's read it back */
					set_bit(R5_Wantread, &dev->flags);
					set_bit(R5_LOCKED, &dev->flags);
					s.locked++;
				}
			}
		}


3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969
	/* Finish reconstruct operations initiated by the expansion process */
	if (sh->reconstruct_state == reconstruct_state_result) {
		struct stripe_head *sh_src
			= get_active_stripe(conf, sh->sector, 1, 1, 1);
		if (sh_src && test_bit(STRIPE_EXPAND_SOURCE, &sh_src->state)) {
			/* sh cannot be written until sh_src has been read.
			 * so arrange for sh to be delayed a little
			 */
			set_bit(STRIPE_DELAYED, &sh->state);
			set_bit(STRIPE_HANDLE, &sh->state);
			if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE,
					      &sh_src->state))
				atomic_inc(&conf->preread_active_stripes);
			release_stripe(sh_src);
			goto finish;
		}
		if (sh_src)
			release_stripe(sh_src);

		sh->reconstruct_state = reconstruct_state_idle;
		clear_bit(STRIPE_EXPANDING, &sh->state);
		for (i = conf->raid_disks; i--; ) {
			set_bit(R5_Wantwrite, &sh->dev[i].flags);
			set_bit(R5_LOCKED, &sh->dev[i].flags);
			s.locked++;
		}
	}
3970

3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986
	if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) &&
	    !sh->reconstruct_state) {
		/* Need to write out all blocks after computing parity */
		sh->disks = conf->raid_disks;
		stripe_set_idx(sh->sector, conf, 0, sh);
		schedule_reconstruction(sh, &s, 1, 1);
	} else if (s.expanded && !sh->reconstruct_state && s.locked == 0) {
		clear_bit(STRIPE_EXPAND_READY, &sh->state);
		atomic_dec(&conf->reshape_stripes);
		wake_up(&conf->wait_for_overlap);
		md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
	}

	if (s.expanding && s.locked == 0 &&
	    !test_bit(STRIPE_COMPUTE_RUN, &sh->state))
		handle_stripe_expansion(conf, sh);
3987

3988
finish:
3989
	/* wait for this device to become unblocked */
3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001
	if (unlikely(s.blocked_rdev)) {
		if (conf->mddev->external)
			md_wait_for_blocked_rdev(s.blocked_rdev,
						 conf->mddev);
		else
			/* Internal metadata will immediately
			 * be written by raid5d, so we don't
			 * need to wait here.
			 */
			rdev_dec_pending(s.blocked_rdev,
					 conf->mddev);
	}
4002

4003 4004
	if (s.handle_bad_blocks)
		for (i = disks; i--; ) {
4005
			struct md_rdev *rdev;
4006 4007 4008 4009 4010 4011 4012 4013 4014
			struct r5dev *dev = &sh->dev[i];
			if (test_and_clear_bit(R5_WriteError, &dev->flags)) {
				/* We own a safe reference to the rdev */
				rdev = conf->disks[i].rdev;
				if (!rdev_set_badblocks(rdev, sh->sector,
							STRIPE_SECTORS, 0))
					md_error(conf->mddev, rdev);
				rdev_dec_pending(rdev, conf->mddev);
			}
4015 4016 4017
			if (test_and_clear_bit(R5_MadeGood, &dev->flags)) {
				rdev = conf->disks[i].rdev;
				rdev_clear_badblocks(rdev, sh->sector,
4018
						     STRIPE_SECTORS, 0);
4019 4020
				rdev_dec_pending(rdev, conf->mddev);
			}
4021 4022
			if (test_and_clear_bit(R5_MadeGoodRepl, &dev->flags)) {
				rdev = conf->disks[i].replacement;
4023 4024 4025
				if (!rdev)
					/* rdev have been moved down */
					rdev = conf->disks[i].rdev;
4026
				rdev_clear_badblocks(rdev, sh->sector,
4027
						     STRIPE_SECTORS, 0);
4028 4029
				rdev_dec_pending(rdev, conf->mddev);
			}
4030 4031
		}

4032 4033 4034
	if (s.ops_request)
		raid_run_ops(sh, s.ops_request);

D
Dan Williams 已提交
4035
	ops_run_io(sh, &s);
4036

4037
	if (s.dec_preread_active) {
4038
		/* We delay this until after ops_run_io so that if make_request
T
Tejun Heo 已提交
4039
		 * is waiting on a flush, it won't continue until the writes
4040 4041 4042 4043 4044 4045 4046 4047
		 * have actually been submitted.
		 */
		atomic_dec(&conf->preread_active_stripes);
		if (atomic_read(&conf->preread_active_stripes) <
		    IO_THRESHOLD)
			md_wakeup_thread(conf->mddev->thread);
	}

4048
	return_io(s.return_bi);
4049

4050
	clear_bit_unlock(STRIPE_ACTIVE, &sh->state);
4051 4052
}

4053
static void raid5_activate_delayed(struct r5conf *conf)
4054 4055 4056 4057 4058 4059 4060 4061 4062 4063
{
	if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) {
		while (!list_empty(&conf->delayed_list)) {
			struct list_head *l = conf->delayed_list.next;
			struct stripe_head *sh;
			sh = list_entry(l, struct stripe_head, lru);
			list_del_init(l);
			clear_bit(STRIPE_DELAYED, &sh->state);
			if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
				atomic_inc(&conf->preread_active_stripes);
4064
			list_add_tail(&sh->lru, &conf->hold_list);
4065
			raid5_wakeup_stripe_thread(sh);
4066
		}
N
NeilBrown 已提交
4067
	}
4068 4069
}

4070 4071
static void activate_bit_delay(struct r5conf *conf,
	struct list_head *temp_inactive_list)
4072 4073 4074 4075 4076 4077 4078
{
	/* device_lock is held */
	struct list_head head;
	list_add(&head, &conf->bitmap_list);
	list_del_init(&conf->bitmap_list);
	while (!list_empty(&head)) {
		struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru);
4079
		int hash;
4080 4081
		list_del_init(&sh->lru);
		atomic_inc(&sh->count);
4082 4083
		hash = sh->hash_lock_index;
		__release_stripe(conf, sh, &temp_inactive_list[hash]);
4084 4085 4086
	}
}

4087
int md_raid5_congested(struct mddev *mddev, int bits)
4088
{
4089
	struct r5conf *conf = mddev->private;
4090 4091 4092 4093

	/* No difference between reads and writes.  Just check
	 * how busy the stripe_cache is
	 */
4094

4095 4096 4097 4098
	if (conf->inactive_blocked)
		return 1;
	if (conf->quiesce)
		return 1;
4099
	if (atomic_read(&conf->empty_inactive_list_nr))
4100 4101 4102 4103
		return 1;

	return 0;
}
N
NeilBrown 已提交
4104 4105 4106 4107
EXPORT_SYMBOL_GPL(md_raid5_congested);

static int raid5_congested(void *data, int bits)
{
4108
	struct mddev *mddev = data;
N
NeilBrown 已提交
4109 4110 4111 4112

	return mddev_congested(mddev, bits) ||
		md_raid5_congested(mddev, bits);
}
4113

4114 4115 4116
/* We want read requests to align with chunks where possible,
 * but write requests don't need to.
 */
4117 4118 4119
static int raid5_mergeable_bvec(struct request_queue *q,
				struct bvec_merge_data *bvm,
				struct bio_vec *biovec)
4120
{
4121
	struct mddev *mddev = q->queuedata;
4122
	sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev);
4123
	int max;
4124
	unsigned int chunk_sectors = mddev->chunk_sectors;
4125
	unsigned int bio_sectors = bvm->bi_size >> 9;
4126

4127
	if ((bvm->bi_rw & 1) == WRITE)
4128 4129
		return biovec->bv_len; /* always allow writes to be mergeable */

4130 4131
	if (mddev->new_chunk_sectors < mddev->chunk_sectors)
		chunk_sectors = mddev->new_chunk_sectors;
4132 4133 4134 4135 4136 4137 4138 4139
	max =  (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9;
	if (max < 0) max = 0;
	if (max <= biovec->bv_len && bio_sectors == 0)
		return biovec->bv_len;
	else
		return max;
}

4140

4141
static int in_chunk_boundary(struct mddev *mddev, struct bio *bio)
4142
{
4143
	sector_t sector = bio->bi_iter.bi_sector + get_start_sect(bio->bi_bdev);
4144
	unsigned int chunk_sectors = mddev->chunk_sectors;
4145
	unsigned int bio_sectors = bio_sectors(bio);
4146

4147 4148
	if (mddev->new_chunk_sectors < mddev->chunk_sectors)
		chunk_sectors = mddev->new_chunk_sectors;
4149 4150 4151 4152
	return  chunk_sectors >=
		((sector & (chunk_sectors - 1)) + bio_sectors);
}

4153 4154 4155 4156
/*
 *  add bio to the retry LIFO  ( in O(1) ... we are in interrupt )
 *  later sampled by raid5d.
 */
4157
static void add_bio_to_retry(struct bio *bi,struct r5conf *conf)
4158 4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170
{
	unsigned long flags;

	spin_lock_irqsave(&conf->device_lock, flags);

	bi->bi_next = conf->retry_read_aligned_list;
	conf->retry_read_aligned_list = bi;

	spin_unlock_irqrestore(&conf->device_lock, flags);
	md_wakeup_thread(conf->mddev->thread);
}


4171
static struct bio *remove_bio_from_retry(struct r5conf *conf)
4172 4173 4174 4175 4176 4177 4178 4179 4180 4181
{
	struct bio *bi;

	bi = conf->retry_read_aligned;
	if (bi) {
		conf->retry_read_aligned = NULL;
		return bi;
	}
	bi = conf->retry_read_aligned_list;
	if(bi) {
4182
		conf->retry_read_aligned_list = bi->bi_next;
4183
		bi->bi_next = NULL;
4184 4185 4186 4187
		/*
		 * this sets the active strip count to 1 and the processed
		 * strip count to zero (upper 8 bits)
		 */
4188
		raid5_set_bi_stripes(bi, 1); /* biased count of active stripes */
4189 4190 4191 4192 4193 4194
	}

	return bi;
}


4195 4196 4197 4198 4199 4200
/*
 *  The "raid5_align_endio" should check if the read succeeded and if it
 *  did, call bio_endio on the original bio (having bio_put the new bio
 *  first).
 *  If the read failed..
 */
4201
static void raid5_align_endio(struct bio *bi, int error)
4202 4203
{
	struct bio* raid_bi  = bi->bi_private;
4204
	struct mddev *mddev;
4205
	struct r5conf *conf;
4206
	int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
4207
	struct md_rdev *rdev;
4208

4209
	bio_put(bi);
4210 4211 4212

	rdev = (void*)raid_bi->bi_next;
	raid_bi->bi_next = NULL;
4213 4214
	mddev = rdev->mddev;
	conf = mddev->private;
4215 4216 4217 4218

	rdev_dec_pending(rdev, conf->mddev);

	if (!error && uptodate) {
4219 4220
		trace_block_bio_complete(bdev_get_queue(raid_bi->bi_bdev),
					 raid_bi, 0);
4221
		bio_endio(raid_bi, 0);
4222 4223
		if (atomic_dec_and_test(&conf->active_aligned_reads))
			wake_up(&conf->wait_for_stripe);
4224
		return;
4225 4226 4227
	}


4228
	pr_debug("raid5_align_endio : io error...handing IO for a retry\n");
4229 4230

	add_bio_to_retry(raid_bi, conf);
4231 4232
}

4233 4234
static int bio_fits_rdev(struct bio *bi)
{
4235
	struct request_queue *q = bdev_get_queue(bi->bi_bdev);
4236

4237
	if (bio_sectors(bi) > queue_max_sectors(q))
4238 4239
		return 0;
	blk_recount_segments(q, bi);
4240
	if (bi->bi_phys_segments > queue_max_segments(q))
4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252
		return 0;

	if (q->merge_bvec_fn)
		/* it's too hard to apply the merge_bvec_fn at this stage,
		 * just just give up
		 */
		return 0;

	return 1;
}


4253
static int chunk_aligned_read(struct mddev *mddev, struct bio * raid_bio)
4254
{
4255
	struct r5conf *conf = mddev->private;
N
NeilBrown 已提交
4256
	int dd_idx;
4257
	struct bio* align_bi;
4258
	struct md_rdev *rdev;
4259
	sector_t end_sector;
4260 4261

	if (!in_chunk_boundary(mddev, raid_bio)) {
4262
		pr_debug("chunk_aligned_read : non aligned\n");
4263 4264 4265
		return 0;
	}
	/*
4266
	 * use bio_clone_mddev to make a copy of the bio
4267
	 */
4268
	align_bi = bio_clone_mddev(raid_bio, GFP_NOIO, mddev);
4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279
	if (!align_bi)
		return 0;
	/*
	 *   set bi_end_io to a new function, and set bi_private to the
	 *     original bio.
	 */
	align_bi->bi_end_io  = raid5_align_endio;
	align_bi->bi_private = raid_bio;
	/*
	 *	compute position
	 */
4280 4281 4282
	align_bi->bi_iter.bi_sector =
		raid5_compute_sector(conf, raid_bio->bi_iter.bi_sector,
				     0, &dd_idx, NULL);
4283

K
Kent Overstreet 已提交
4284
	end_sector = bio_end_sector(align_bi);
4285
	rcu_read_lock();
4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296
	rdev = rcu_dereference(conf->disks[dd_idx].replacement);
	if (!rdev || test_bit(Faulty, &rdev->flags) ||
	    rdev->recovery_offset < end_sector) {
		rdev = rcu_dereference(conf->disks[dd_idx].rdev);
		if (rdev &&
		    (test_bit(Faulty, &rdev->flags) ||
		    !(test_bit(In_sync, &rdev->flags) ||
		      rdev->recovery_offset >= end_sector)))
			rdev = NULL;
	}
	if (rdev) {
4297 4298 4299
		sector_t first_bad;
		int bad_sectors;

4300 4301
		atomic_inc(&rdev->nr_pending);
		rcu_read_unlock();
4302 4303
		raid_bio->bi_next = (void*)rdev;
		align_bi->bi_bdev =  rdev->bdev;
4304
		__clear_bit(BIO_SEG_VALID, &align_bi->bi_flags);
4305

4306
		if (!bio_fits_rdev(align_bi) ||
4307 4308
		    is_badblock(rdev, align_bi->bi_iter.bi_sector,
				bio_sectors(align_bi),
4309 4310
				&first_bad, &bad_sectors)) {
			/* too big in some way, or has a known bad block */
4311 4312 4313 4314 4315
			bio_put(align_bi);
			rdev_dec_pending(rdev, mddev);
			return 0;
		}

4316
		/* No reshape active, so we can trust rdev->data_offset */
4317
		align_bi->bi_iter.bi_sector += rdev->data_offset;
4318

4319 4320 4321
		spin_lock_irq(&conf->device_lock);
		wait_event_lock_irq(conf->wait_for_stripe,
				    conf->quiesce == 0,
4322
				    conf->device_lock);
4323 4324 4325
		atomic_inc(&conf->active_aligned_reads);
		spin_unlock_irq(&conf->device_lock);

4326 4327 4328
		if (mddev->gendisk)
			trace_block_bio_remap(bdev_get_queue(align_bi->bi_bdev),
					      align_bi, disk_devt(mddev->gendisk),
4329
					      raid_bio->bi_iter.bi_sector);
4330 4331 4332 4333
		generic_make_request(align_bi);
		return 1;
	} else {
		rcu_read_unlock();
4334
		bio_put(align_bi);
4335 4336 4337 4338
		return 0;
	}
}

4339 4340 4341 4342 4343 4344 4345 4346 4347 4348
/* __get_priority_stripe - get the next stripe to process
 *
 * Full stripe writes are allowed to pass preread active stripes up until
 * the bypass_threshold is exceeded.  In general the bypass_count
 * increments when the handle_list is handled before the hold_list; however, it
 * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a
 * stripe with in flight i/o.  The bypass_count will be reset when the
 * head of the hold_list has changed, i.e. the head was promoted to the
 * handle_list.
 */
4349
static struct stripe_head *__get_priority_stripe(struct r5conf *conf, int group)
4350
{
4351 4352
	struct stripe_head *sh = NULL, *tmp;
	struct list_head *handle_list = NULL;
4353
	struct r5worker_group *wg = NULL;
4354 4355 4356 4357 4358

	if (conf->worker_cnt_per_group == 0) {
		handle_list = &conf->handle_list;
	} else if (group != ANY_GROUP) {
		handle_list = &conf->worker_groups[group].handle_list;
4359
		wg = &conf->worker_groups[group];
4360 4361 4362 4363
	} else {
		int i;
		for (i = 0; i < conf->group_cnt; i++) {
			handle_list = &conf->worker_groups[i].handle_list;
4364
			wg = &conf->worker_groups[i];
4365 4366 4367 4368
			if (!list_empty(handle_list))
				break;
		}
	}
4369 4370 4371

	pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n",
		  __func__,
4372
		  list_empty(handle_list) ? "empty" : "busy",
4373 4374 4375
		  list_empty(&conf->hold_list) ? "empty" : "busy",
		  atomic_read(&conf->pending_full_writes), conf->bypass_count);

4376 4377
	if (!list_empty(handle_list)) {
		sh = list_entry(handle_list->next, typeof(*sh), lru);
4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394

		if (list_empty(&conf->hold_list))
			conf->bypass_count = 0;
		else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) {
			if (conf->hold_list.next == conf->last_hold)
				conf->bypass_count++;
			else {
				conf->last_hold = conf->hold_list.next;
				conf->bypass_count -= conf->bypass_threshold;
				if (conf->bypass_count < 0)
					conf->bypass_count = 0;
			}
		}
	} else if (!list_empty(&conf->hold_list) &&
		   ((conf->bypass_threshold &&
		     conf->bypass_count > conf->bypass_threshold) ||
		    atomic_read(&conf->pending_full_writes) == 0)) {
4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410

		list_for_each_entry(tmp, &conf->hold_list,  lru) {
			if (conf->worker_cnt_per_group == 0 ||
			    group == ANY_GROUP ||
			    !cpu_online(tmp->cpu) ||
			    cpu_to_group(tmp->cpu) == group) {
				sh = tmp;
				break;
			}
		}

		if (sh) {
			conf->bypass_count -= conf->bypass_threshold;
			if (conf->bypass_count < 0)
				conf->bypass_count = 0;
		}
4411
		wg = NULL;
4412 4413 4414
	}

	if (!sh)
4415 4416
		return NULL;

4417 4418 4419 4420
	if (wg) {
		wg->stripes_cnt--;
		sh->group = NULL;
	}
4421
	list_del_init(&sh->lru);
4422
	BUG_ON(atomic_inc_return(&sh->count) != 1);
4423 4424
	return sh;
}
4425

4426 4427 4428
struct raid5_plug_cb {
	struct blk_plug_cb	cb;
	struct list_head	list;
4429
	struct list_head	temp_inactive_list[NR_STRIPE_HASH_LOCKS];
4430 4431 4432 4433 4434 4435 4436 4437 4438
};

static void raid5_unplug(struct blk_plug_cb *blk_cb, bool from_schedule)
{
	struct raid5_plug_cb *cb = container_of(
		blk_cb, struct raid5_plug_cb, cb);
	struct stripe_head *sh;
	struct mddev *mddev = cb->cb.data;
	struct r5conf *conf = mddev->private;
N
NeilBrown 已提交
4439
	int cnt = 0;
4440
	int hash;
4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451

	if (cb->list.next && !list_empty(&cb->list)) {
		spin_lock_irq(&conf->device_lock);
		while (!list_empty(&cb->list)) {
			sh = list_first_entry(&cb->list, struct stripe_head, lru);
			list_del_init(&sh->lru);
			/*
			 * avoid race release_stripe_plug() sees
			 * STRIPE_ON_UNPLUG_LIST clear but the stripe
			 * is still in our list
			 */
4452
			smp_mb__before_atomic();
4453
			clear_bit(STRIPE_ON_UNPLUG_LIST, &sh->state);
S
Shaohua Li 已提交
4454 4455 4456 4457
			/*
			 * STRIPE_ON_RELEASE_LIST could be set here. In that
			 * case, the count is always > 1 here
			 */
4458 4459
			hash = sh->hash_lock_index;
			__release_stripe(conf, sh, &cb->temp_inactive_list[hash]);
N
NeilBrown 已提交
4460
			cnt++;
4461 4462 4463
		}
		spin_unlock_irq(&conf->device_lock);
	}
4464 4465
	release_inactive_stripe_list(conf, cb->temp_inactive_list,
				     NR_STRIPE_HASH_LOCKS);
4466 4467
	if (mddev->queue)
		trace_block_unplug(mddev->queue, cnt, !from_schedule);
4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485
	kfree(cb);
}

static void release_stripe_plug(struct mddev *mddev,
				struct stripe_head *sh)
{
	struct blk_plug_cb *blk_cb = blk_check_plugged(
		raid5_unplug, mddev,
		sizeof(struct raid5_plug_cb));
	struct raid5_plug_cb *cb;

	if (!blk_cb) {
		release_stripe(sh);
		return;
	}

	cb = container_of(blk_cb, struct raid5_plug_cb, cb);

4486 4487
	if (cb->list.next == NULL) {
		int i;
4488
		INIT_LIST_HEAD(&cb->list);
4489 4490 4491
		for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
			INIT_LIST_HEAD(cb->temp_inactive_list + i);
	}
4492 4493 4494 4495 4496 4497 4498

	if (!test_and_set_bit(STRIPE_ON_UNPLUG_LIST, &sh->state))
		list_add_tail(&sh->lru, &cb->list);
	else
		release_stripe(sh);
}

S
Shaohua Li 已提交
4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510
static void make_discard_request(struct mddev *mddev, struct bio *bi)
{
	struct r5conf *conf = mddev->private;
	sector_t logical_sector, last_sector;
	struct stripe_head *sh;
	int remaining;
	int stripe_sectors;

	if (mddev->reshape_position != MaxSector)
		/* Skip discard while reshape is happening */
		return;

4511 4512
	logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1);
	last_sector = bi->bi_iter.bi_sector + (bi->bi_iter.bi_size>>9);
S
Shaohua Li 已提交
4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533

	bi->bi_next = NULL;
	bi->bi_phys_segments = 1; /* over-loaded to count active stripes */

	stripe_sectors = conf->chunk_sectors *
		(conf->raid_disks - conf->max_degraded);
	logical_sector = DIV_ROUND_UP_SECTOR_T(logical_sector,
					       stripe_sectors);
	sector_div(last_sector, stripe_sectors);

	logical_sector *= conf->chunk_sectors;
	last_sector *= conf->chunk_sectors;

	for (; logical_sector < last_sector;
	     logical_sector += STRIPE_SECTORS) {
		DEFINE_WAIT(w);
		int d;
	again:
		sh = get_active_stripe(conf, logical_sector, 0, 0, 0);
		prepare_to_wait(&conf->wait_for_overlap, &w,
				TASK_UNINTERRUPTIBLE);
4534 4535 4536 4537 4538 4539 4540
		set_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);
		if (test_bit(STRIPE_SYNCING, &sh->state)) {
			release_stripe(sh);
			schedule();
			goto again;
		}
		clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);
S
Shaohua Li 已提交
4541 4542 4543 4544 4545 4546 4547 4548 4549 4550 4551 4552
		spin_lock_irq(&sh->stripe_lock);
		for (d = 0; d < conf->raid_disks; d++) {
			if (d == sh->pd_idx || d == sh->qd_idx)
				continue;
			if (sh->dev[d].towrite || sh->dev[d].toread) {
				set_bit(R5_Overlap, &sh->dev[d].flags);
				spin_unlock_irq(&sh->stripe_lock);
				release_stripe(sh);
				schedule();
				goto again;
			}
		}
4553
		set_bit(STRIPE_DISCARD, &sh->state);
S
Shaohua Li 已提交
4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588
		finish_wait(&conf->wait_for_overlap, &w);
		for (d = 0; d < conf->raid_disks; d++) {
			if (d == sh->pd_idx || d == sh->qd_idx)
				continue;
			sh->dev[d].towrite = bi;
			set_bit(R5_OVERWRITE, &sh->dev[d].flags);
			raid5_inc_bi_active_stripes(bi);
		}
		spin_unlock_irq(&sh->stripe_lock);
		if (conf->mddev->bitmap) {
			for (d = 0;
			     d < conf->raid_disks - conf->max_degraded;
			     d++)
				bitmap_startwrite(mddev->bitmap,
						  sh->sector,
						  STRIPE_SECTORS,
						  0);
			sh->bm_seq = conf->seq_flush + 1;
			set_bit(STRIPE_BIT_DELAY, &sh->state);
		}

		set_bit(STRIPE_HANDLE, &sh->state);
		clear_bit(STRIPE_DELAYED, &sh->state);
		if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
			atomic_inc(&conf->preread_active_stripes);
		release_stripe_plug(mddev, sh);
	}

	remaining = raid5_dec_bi_active_stripes(bi);
	if (remaining == 0) {
		md_write_end(mddev);
		bio_endio(bi, 0);
	}
}

4589
static void make_request(struct mddev *mddev, struct bio * bi)
L
Linus Torvalds 已提交
4590
{
4591
	struct r5conf *conf = mddev->private;
4592
	int dd_idx;
L
Linus Torvalds 已提交
4593 4594 4595
	sector_t new_sector;
	sector_t logical_sector, last_sector;
	struct stripe_head *sh;
4596
	const int rw = bio_data_dir(bi);
4597
	int remaining;
4598 4599
	DEFINE_WAIT(w);
	bool do_prepare;
L
Linus Torvalds 已提交
4600

T
Tejun Heo 已提交
4601 4602
	if (unlikely(bi->bi_rw & REQ_FLUSH)) {
		md_flush_request(mddev, bi);
4603
		return;
4604 4605
	}

4606
	md_write_start(mddev, bi);
4607

4608
	if (rw == READ &&
4609
	     mddev->reshape_position == MaxSector &&
4610
	     chunk_aligned_read(mddev,bi))
4611
		return;
4612

S
Shaohua Li 已提交
4613 4614 4615 4616 4617
	if (unlikely(bi->bi_rw & REQ_DISCARD)) {
		make_discard_request(mddev, bi);
		return;
	}

4618
	logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1);
K
Kent Overstreet 已提交
4619
	last_sector = bio_end_sector(bi);
L
Linus Torvalds 已提交
4620 4621
	bi->bi_next = NULL;
	bi->bi_phys_segments = 1;	/* over-loaded to count active stripes */
4622

4623
	prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE);
L
Linus Torvalds 已提交
4624
	for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) {
4625
		int previous;
4626
		int seq;
4627

4628
		do_prepare = false;
4629
	retry:
4630
		seq = read_seqcount_begin(&conf->gen_lock);
4631
		previous = 0;
4632 4633 4634
		if (do_prepare)
			prepare_to_wait(&conf->wait_for_overlap, &w,
				TASK_UNINTERRUPTIBLE);
4635
		if (unlikely(conf->reshape_progress != MaxSector)) {
4636
			/* spinlock is needed as reshape_progress may be
4637 4638
			 * 64bit on a 32bit platform, and so it might be
			 * possible to see a half-updated value
4639
			 * Of course reshape_progress could change after
4640 4641 4642 4643
			 * the lock is dropped, so once we get a reference
			 * to the stripe that we think it is, we will have
			 * to check again.
			 */
4644
			spin_lock_irq(&conf->device_lock);
4645
			if (mddev->reshape_backwards
4646 4647
			    ? logical_sector < conf->reshape_progress
			    : logical_sector >= conf->reshape_progress) {
4648 4649
				previous = 1;
			} else {
4650
				if (mddev->reshape_backwards
4651 4652
				    ? logical_sector < conf->reshape_safe
				    : logical_sector >= conf->reshape_safe) {
4653 4654
					spin_unlock_irq(&conf->device_lock);
					schedule();
4655
					do_prepare = true;
4656 4657 4658
					goto retry;
				}
			}
4659 4660
			spin_unlock_irq(&conf->device_lock);
		}
4661

4662 4663
		new_sector = raid5_compute_sector(conf, logical_sector,
						  previous,
4664
						  &dd_idx, NULL);
4665
		pr_debug("raid456: make_request, sector %llu logical %llu\n",
4666
			(unsigned long long)new_sector,
L
Linus Torvalds 已提交
4667 4668
			(unsigned long long)logical_sector);

4669
		sh = get_active_stripe(conf, new_sector, previous,
4670
				       (bi->bi_rw&RWA_MASK), 0);
L
Linus Torvalds 已提交
4671
		if (sh) {
4672
			if (unlikely(previous)) {
4673
				/* expansion might have moved on while waiting for a
4674 4675 4676 4677 4678 4679
				 * stripe, so we must do the range check again.
				 * Expansion could still move past after this
				 * test, but as we are holding a reference to
				 * 'sh', we know that if that happens,
				 *  STRIPE_EXPANDING will get set and the expansion
				 * won't proceed until we finish with the stripe.
4680 4681 4682
				 */
				int must_retry = 0;
				spin_lock_irq(&conf->device_lock);
4683
				if (mddev->reshape_backwards
4684 4685
				    ? logical_sector >= conf->reshape_progress
				    : logical_sector < conf->reshape_progress)
4686 4687 4688 4689 4690
					/* mismatch, need to try again */
					must_retry = 1;
				spin_unlock_irq(&conf->device_lock);
				if (must_retry) {
					release_stripe(sh);
4691
					schedule();
4692
					do_prepare = true;
4693 4694 4695
					goto retry;
				}
			}
4696 4697 4698 4699 4700 4701 4702
			if (read_seqcount_retry(&conf->gen_lock, seq)) {
				/* Might have got the wrong stripe_head
				 * by accident
				 */
				release_stripe(sh);
				goto retry;
			}
4703

4704
			if (rw == WRITE &&
4705
			    logical_sector >= mddev->suspend_lo &&
4706 4707
			    logical_sector < mddev->suspend_hi) {
				release_stripe(sh);
4708 4709 4710 4711 4712 4713 4714 4715
				/* As the suspend_* range is controlled by
				 * userspace, we want an interruptible
				 * wait.
				 */
				flush_signals(current);
				prepare_to_wait(&conf->wait_for_overlap,
						&w, TASK_INTERRUPTIBLE);
				if (logical_sector >= mddev->suspend_lo &&
4716
				    logical_sector < mddev->suspend_hi) {
4717
					schedule();
4718 4719
					do_prepare = true;
				}
4720 4721
				goto retry;
			}
4722 4723

			if (test_bit(STRIPE_EXPANDING, &sh->state) ||
4724
			    !add_stripe_bio(sh, bi, dd_idx, rw)) {
4725 4726
				/* Stripe is busy expanding or
				 * add failed due to overlap.  Flush everything
L
Linus Torvalds 已提交
4727 4728
				 * and wait a while
				 */
N
NeilBrown 已提交
4729
				md_wakeup_thread(mddev->thread);
L
Linus Torvalds 已提交
4730 4731
				release_stripe(sh);
				schedule();
4732
				do_prepare = true;
L
Linus Torvalds 已提交
4733 4734
				goto retry;
			}
4735 4736
			set_bit(STRIPE_HANDLE, &sh->state);
			clear_bit(STRIPE_DELAYED, &sh->state);
4737
			if ((bi->bi_rw & REQ_SYNC) &&
4738 4739
			    !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
				atomic_inc(&conf->preread_active_stripes);
4740
			release_stripe_plug(mddev, sh);
L
Linus Torvalds 已提交
4741 4742 4743 4744 4745 4746
		} else {
			/* cannot get stripe for read-ahead, just give-up */
			clear_bit(BIO_UPTODATE, &bi->bi_flags);
			break;
		}
	}
4747
	finish_wait(&conf->wait_for_overlap, &w);
4748

4749
	remaining = raid5_dec_bi_active_stripes(bi);
4750
	if (remaining == 0) {
L
Linus Torvalds 已提交
4751

4752
		if ( rw == WRITE )
L
Linus Torvalds 已提交
4753
			md_write_end(mddev);
4754

4755 4756
		trace_block_bio_complete(bdev_get_queue(bi->bi_bdev),
					 bi, 0);
4757
		bio_endio(bi, 0);
L
Linus Torvalds 已提交
4758 4759 4760
	}
}

4761
static sector_t raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks);
D
Dan Williams 已提交
4762

4763
static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *skipped)
L
Linus Torvalds 已提交
4764
{
4765 4766 4767 4768 4769 4770 4771 4772 4773
	/* reshaping is quite different to recovery/resync so it is
	 * handled quite separately ... here.
	 *
	 * On each call to sync_request, we gather one chunk worth of
	 * destination stripes and flag them as expanding.
	 * Then we find all the source stripes and request reads.
	 * As the reads complete, handle_stripe will copy the data
	 * into the destination stripe and release that stripe.
	 */
4774
	struct r5conf *conf = mddev->private;
L
Linus Torvalds 已提交
4775
	struct stripe_head *sh;
4776
	sector_t first_sector, last_sector;
4777 4778 4779
	int raid_disks = conf->previous_raid_disks;
	int data_disks = raid_disks - conf->max_degraded;
	int new_data_disks = conf->raid_disks - conf->max_degraded;
4780 4781
	int i;
	int dd_idx;
4782
	sector_t writepos, readpos, safepos;
4783
	sector_t stripe_addr;
4784
	int reshape_sectors;
4785
	struct list_head stripes;
4786

4787 4788
	if (sector_nr == 0) {
		/* If restarting in the middle, skip the initial sectors */
4789
		if (mddev->reshape_backwards &&
4790 4791 4792
		    conf->reshape_progress < raid5_size(mddev, 0, 0)) {
			sector_nr = raid5_size(mddev, 0, 0)
				- conf->reshape_progress;
4793
		} else if (!mddev->reshape_backwards &&
4794 4795
			   conf->reshape_progress > 0)
			sector_nr = conf->reshape_progress;
4796
		sector_div(sector_nr, new_data_disks);
4797
		if (sector_nr) {
4798 4799
			mddev->curr_resync_completed = sector_nr;
			sysfs_notify(&mddev->kobj, NULL, "sync_completed");
4800 4801 4802
			*skipped = 1;
			return sector_nr;
		}
4803 4804
	}

4805 4806 4807 4808
	/* We need to process a full chunk at a time.
	 * If old and new chunk sizes differ, we need to process the
	 * largest of these
	 */
4809 4810
	if (mddev->new_chunk_sectors > mddev->chunk_sectors)
		reshape_sectors = mddev->new_chunk_sectors;
4811
	else
4812
		reshape_sectors = mddev->chunk_sectors;
4813

4814 4815 4816 4817 4818
	/* We update the metadata at least every 10 seconds, or when
	 * the data about to be copied would over-write the source of
	 * the data at the front of the range.  i.e. one new_stripe
	 * along from reshape_progress new_maps to after where
	 * reshape_safe old_maps to
4819
	 */
4820
	writepos = conf->reshape_progress;
4821
	sector_div(writepos, new_data_disks);
4822 4823
	readpos = conf->reshape_progress;
	sector_div(readpos, data_disks);
4824
	safepos = conf->reshape_safe;
4825
	sector_div(safepos, data_disks);
4826
	if (mddev->reshape_backwards) {
4827
		writepos -= min_t(sector_t, reshape_sectors, writepos);
4828
		readpos += reshape_sectors;
4829
		safepos += reshape_sectors;
4830
	} else {
4831
		writepos += reshape_sectors;
4832 4833
		readpos -= min_t(sector_t, reshape_sectors, readpos);
		safepos -= min_t(sector_t, reshape_sectors, safepos);
4834
	}
4835

4836 4837 4838 4839 4840 4841 4842 4843 4844 4845 4846 4847 4848 4849 4850
	/* Having calculated the 'writepos' possibly use it
	 * to set 'stripe_addr' which is where we will write to.
	 */
	if (mddev->reshape_backwards) {
		BUG_ON(conf->reshape_progress == 0);
		stripe_addr = writepos;
		BUG_ON((mddev->dev_sectors &
			~((sector_t)reshape_sectors - 1))
		       - reshape_sectors - stripe_addr
		       != sector_nr);
	} else {
		BUG_ON(writepos != sector_nr + reshape_sectors);
		stripe_addr = sector_nr;
	}

4851 4852 4853 4854
	/* 'writepos' is the most advanced device address we might write.
	 * 'readpos' is the least advanced device address we might read.
	 * 'safepos' is the least address recorded in the metadata as having
	 *     been reshaped.
4855 4856 4857 4858
	 * If there is a min_offset_diff, these are adjusted either by
	 * increasing the safepos/readpos if diff is negative, or
	 * increasing writepos if diff is positive.
	 * If 'readpos' is then behind 'writepos', there is no way that we can
4859 4860 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870
	 * ensure safety in the face of a crash - that must be done by userspace
	 * making a backup of the data.  So in that case there is no particular
	 * rush to update metadata.
	 * Otherwise if 'safepos' is behind 'writepos', then we really need to
	 * update the metadata to advance 'safepos' to match 'readpos' so that
	 * we can be safe in the event of a crash.
	 * So we insist on updating metadata if safepos is behind writepos and
	 * readpos is beyond writepos.
	 * In any case, update the metadata every 10 seconds.
	 * Maybe that number should be configurable, but I'm not sure it is
	 * worth it.... maybe it could be a multiple of safemode_delay???
	 */
4871 4872 4873 4874 4875 4876
	if (conf->min_offset_diff < 0) {
		safepos += -conf->min_offset_diff;
		readpos += -conf->min_offset_diff;
	} else
		writepos += conf->min_offset_diff;

4877
	if ((mddev->reshape_backwards
4878 4879 4880
	     ? (safepos > writepos && readpos < writepos)
	     : (safepos < writepos && readpos > writepos)) ||
	    time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
4881 4882
		/* Cannot proceed until we've updated the superblock... */
		wait_event(conf->wait_for_overlap,
4883 4884 4885 4886
			   atomic_read(&conf->reshape_stripes)==0
			   || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
		if (atomic_read(&conf->reshape_stripes) != 0)
			return 0;
4887
		mddev->reshape_position = conf->reshape_progress;
4888
		mddev->curr_resync_completed = sector_nr;
4889
		conf->reshape_checkpoint = jiffies;
4890
		set_bit(MD_CHANGE_DEVS, &mddev->flags);
4891
		md_wakeup_thread(mddev->thread);
4892
		wait_event(mddev->sb_wait, mddev->flags == 0 ||
4893 4894 4895
			   test_bit(MD_RECOVERY_INTR, &mddev->recovery));
		if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
			return 0;
4896
		spin_lock_irq(&conf->device_lock);
4897
		conf->reshape_safe = mddev->reshape_position;
4898 4899
		spin_unlock_irq(&conf->device_lock);
		wake_up(&conf->wait_for_overlap);
4900
		sysfs_notify(&mddev->kobj, NULL, "sync_completed");
4901 4902
	}

4903
	INIT_LIST_HEAD(&stripes);
4904
	for (i = 0; i < reshape_sectors; i += STRIPE_SECTORS) {
4905
		int j;
4906
		int skipped_disk = 0;
4907
		sh = get_active_stripe(conf, stripe_addr+i, 0, 0, 1);
4908 4909 4910 4911 4912 4913 4914 4915 4916
		set_bit(STRIPE_EXPANDING, &sh->state);
		atomic_inc(&conf->reshape_stripes);
		/* If any of this stripe is beyond the end of the old
		 * array, then we need to zero those blocks
		 */
		for (j=sh->disks; j--;) {
			sector_t s;
			if (j == sh->pd_idx)
				continue;
4917
			if (conf->level == 6 &&
4918
			    j == sh->qd_idx)
4919
				continue;
4920
			s = compute_blocknr(sh, j, 0);
D
Dan Williams 已提交
4921
			if (s < raid5_size(mddev, 0, 0)) {
4922
				skipped_disk = 1;
4923 4924 4925 4926 4927 4928
				continue;
			}
			memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE);
			set_bit(R5_Expanded, &sh->dev[j].flags);
			set_bit(R5_UPTODATE, &sh->dev[j].flags);
		}
4929
		if (!skipped_disk) {
4930 4931 4932
			set_bit(STRIPE_EXPAND_READY, &sh->state);
			set_bit(STRIPE_HANDLE, &sh->state);
		}
4933
		list_add(&sh->lru, &stripes);
4934 4935
	}
	spin_lock_irq(&conf->device_lock);
4936
	if (mddev->reshape_backwards)
4937
		conf->reshape_progress -= reshape_sectors * new_data_disks;
4938
	else
4939
		conf->reshape_progress += reshape_sectors * new_data_disks;
4940 4941 4942 4943 4944 4945 4946
	spin_unlock_irq(&conf->device_lock);
	/* Ok, those stripe are ready. We can start scheduling
	 * reads on the source stripes.
	 * The source stripes are determined by mapping the first and last
	 * block on the destination stripes.
	 */
	first_sector =
4947
		raid5_compute_sector(conf, stripe_addr*(new_data_disks),
4948
				     1, &dd_idx, NULL);
4949
	last_sector =
4950
		raid5_compute_sector(conf, ((stripe_addr+reshape_sectors)
4951
					    * new_data_disks - 1),
4952
				     1, &dd_idx, NULL);
A
Andre Noll 已提交
4953 4954
	if (last_sector >= mddev->dev_sectors)
		last_sector = mddev->dev_sectors - 1;
4955
	while (first_sector <= last_sector) {
4956
		sh = get_active_stripe(conf, first_sector, 1, 0, 1);
4957 4958 4959 4960 4961
		set_bit(STRIPE_EXPAND_SOURCE, &sh->state);
		set_bit(STRIPE_HANDLE, &sh->state);
		release_stripe(sh);
		first_sector += STRIPE_SECTORS;
	}
4962 4963 4964 4965 4966 4967 4968 4969
	/* Now that the sources are clearly marked, we can release
	 * the destination stripes
	 */
	while (!list_empty(&stripes)) {
		sh = list_entry(stripes.next, struct stripe_head, lru);
		list_del_init(&sh->lru);
		release_stripe(sh);
	}
4970 4971 4972
	/* If this takes us to the resync_max point where we have to pause,
	 * then we need to write out the superblock.
	 */
4973
	sector_nr += reshape_sectors;
4974 4975
	if ((sector_nr - mddev->curr_resync_completed) * 2
	    >= mddev->resync_max - mddev->curr_resync_completed) {
4976 4977
		/* Cannot proceed until we've updated the superblock... */
		wait_event(conf->wait_for_overlap,
4978 4979 4980 4981
			   atomic_read(&conf->reshape_stripes) == 0
			   || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
		if (atomic_read(&conf->reshape_stripes) != 0)
			goto ret;
4982
		mddev->reshape_position = conf->reshape_progress;
4983
		mddev->curr_resync_completed = sector_nr;
4984
		conf->reshape_checkpoint = jiffies;
4985 4986 4987 4988
		set_bit(MD_CHANGE_DEVS, &mddev->flags);
		md_wakeup_thread(mddev->thread);
		wait_event(mddev->sb_wait,
			   !test_bit(MD_CHANGE_DEVS, &mddev->flags)
4989 4990 4991
			   || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
		if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
			goto ret;
4992
		spin_lock_irq(&conf->device_lock);
4993
		conf->reshape_safe = mddev->reshape_position;
4994 4995
		spin_unlock_irq(&conf->device_lock);
		wake_up(&conf->wait_for_overlap);
4996
		sysfs_notify(&mddev->kobj, NULL, "sync_completed");
4997
	}
4998
ret:
4999
	return reshape_sectors;
5000 5001 5002
}

/* FIXME go_faster isn't used */
5003
static inline sector_t sync_request(struct mddev *mddev, sector_t sector_nr, int *skipped, int go_faster)
5004
{
5005
	struct r5conf *conf = mddev->private;
5006
	struct stripe_head *sh;
A
Andre Noll 已提交
5007
	sector_t max_sector = mddev->dev_sectors;
N
NeilBrown 已提交
5008
	sector_t sync_blocks;
5009 5010
	int still_degraded = 0;
	int i;
L
Linus Torvalds 已提交
5011

5012
	if (sector_nr >= max_sector) {
L
Linus Torvalds 已提交
5013
		/* just being told to finish up .. nothing much to do */
5014

5015 5016 5017 5018
		if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {
			end_reshape(conf);
			return 0;
		}
5019 5020 5021 5022

		if (mddev->curr_resync < max_sector) /* aborted */
			bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
					&sync_blocks, 1);
5023
		else /* completed sync */
5024 5025 5026
			conf->fullsync = 0;
		bitmap_close_sync(mddev->bitmap);

L
Linus Torvalds 已提交
5027 5028
		return 0;
	}
5029

5030 5031 5032
	/* Allow raid5_quiesce to complete */
	wait_event(conf->wait_for_overlap, conf->quiesce != 2);

5033 5034
	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
		return reshape_request(mddev, sector_nr, skipped);
5035

5036 5037 5038 5039 5040 5041
	/* No need to check resync_max as we never do more than one
	 * stripe, and as resync_max will always be on a chunk boundary,
	 * if the check in md_do_sync didn't fire, there is no chance
	 * of overstepping resync_max here
	 */

5042
	/* if there is too many failed drives and we are trying
L
Linus Torvalds 已提交
5043 5044 5045
	 * to resync, then assert that we are finished, because there is
	 * nothing we can do.
	 */
5046
	if (mddev->degraded >= conf->max_degraded &&
5047
	    test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
A
Andre Noll 已提交
5048
		sector_t rv = mddev->dev_sectors - sector_nr;
5049
		*skipped = 1;
L
Linus Torvalds 已提交
5050 5051
		return rv;
	}
5052 5053 5054 5055
	if (!test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
	    !conf->fullsync &&
	    !bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) &&
	    sync_blocks >= STRIPE_SECTORS) {
5056 5057 5058 5059 5060
		/* we can skip this block, and probably more */
		sync_blocks /= STRIPE_SECTORS;
		*skipped = 1;
		return sync_blocks * STRIPE_SECTORS; /* keep things rounded to whole stripes */
	}
L
Linus Torvalds 已提交
5061

N
NeilBrown 已提交
5062 5063
	bitmap_cond_end_sync(mddev->bitmap, sector_nr);

5064
	sh = get_active_stripe(conf, sector_nr, 0, 1, 0);
L
Linus Torvalds 已提交
5065
	if (sh == NULL) {
5066
		sh = get_active_stripe(conf, sector_nr, 0, 0, 0);
L
Linus Torvalds 已提交
5067
		/* make sure we don't swamp the stripe cache if someone else
5068
		 * is trying to get access
L
Linus Torvalds 已提交
5069
		 */
5070
		schedule_timeout_uninterruptible(1);
L
Linus Torvalds 已提交
5071
	}
5072 5073 5074 5075
	/* Need to check if array will still be degraded after recovery/resync
	 * We don't need to check the 'failed' flag as when that gets set,
	 * recovery aborts.
	 */
5076
	for (i = 0; i < conf->raid_disks; i++)
5077 5078 5079 5080 5081
		if (conf->disks[i].rdev == NULL)
			still_degraded = 1;

	bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, still_degraded);

5082
	set_bit(STRIPE_SYNC_REQUESTED, &sh->state);
5083
	set_bit(STRIPE_HANDLE, &sh->state);
L
Linus Torvalds 已提交
5084 5085 5086 5087 5088 5089

	release_stripe(sh);

	return STRIPE_SECTORS;
}

5090
static int  retry_aligned_read(struct r5conf *conf, struct bio *raid_bio)
5091 5092 5093 5094 5095 5096 5097 5098 5099 5100 5101 5102
{
	/* We may not be able to submit a whole bio at once as there
	 * may not be enough stripe_heads available.
	 * We cannot pre-allocate enough stripe_heads as we may need
	 * more than exist in the cache (if we allow ever large chunks).
	 * So we do one stripe head at a time and record in
	 * ->bi_hw_segments how many have been done.
	 *
	 * We *know* that this entire raid_bio is in one chunk, so
	 * it will be only one 'dd_idx' and only need one call to raid5_compute_sector.
	 */
	struct stripe_head *sh;
5103
	int dd_idx;
5104 5105 5106 5107 5108
	sector_t sector, logical_sector, last_sector;
	int scnt = 0;
	int remaining;
	int handled = 0;

5109 5110
	logical_sector = raid_bio->bi_iter.bi_sector &
		~((sector_t)STRIPE_SECTORS-1);
5111
	sector = raid5_compute_sector(conf, logical_sector,
5112
				      0, &dd_idx, NULL);
K
Kent Overstreet 已提交
5113
	last_sector = bio_end_sector(raid_bio);
5114 5115

	for (; logical_sector < last_sector;
5116 5117 5118
	     logical_sector += STRIPE_SECTORS,
		     sector += STRIPE_SECTORS,
		     scnt++) {
5119

5120
		if (scnt < raid5_bi_processed_stripes(raid_bio))
5121 5122 5123
			/* already done this stripe */
			continue;

5124
		sh = get_active_stripe(conf, sector, 0, 1, 1);
5125 5126 5127

		if (!sh) {
			/* failed to get a stripe - must wait */
5128
			raid5_set_bi_processed_stripes(raid_bio, scnt);
5129 5130 5131 5132
			conf->retry_read_aligned = raid_bio;
			return handled;
		}

5133 5134
		if (!add_stripe_bio(sh, raid_bio, dd_idx, 0)) {
			release_stripe(sh);
5135
			raid5_set_bi_processed_stripes(raid_bio, scnt);
5136 5137 5138 5139
			conf->retry_read_aligned = raid_bio;
			return handled;
		}

5140
		set_bit(R5_ReadNoMerge, &sh->dev[dd_idx].flags);
5141
		handle_stripe(sh);
5142 5143 5144
		release_stripe(sh);
		handled++;
	}
5145
	remaining = raid5_dec_bi_active_stripes(raid_bio);
5146 5147 5148
	if (remaining == 0) {
		trace_block_bio_complete(bdev_get_queue(raid_bio->bi_bdev),
					 raid_bio, 0);
5149
		bio_endio(raid_bio, 0);
5150
	}
5151 5152 5153 5154 5155
	if (atomic_dec_and_test(&conf->active_aligned_reads))
		wake_up(&conf->wait_for_stripe);
	return handled;
}

5156
static int handle_active_stripes(struct r5conf *conf, int group,
5157 5158
				 struct r5worker *worker,
				 struct list_head *temp_inactive_list)
5159 5160
{
	struct stripe_head *batch[MAX_STRIPE_BATCH], *sh;
5161 5162
	int i, batch_size = 0, hash;
	bool release_inactive = false;
5163 5164

	while (batch_size < MAX_STRIPE_BATCH &&
5165
			(sh = __get_priority_stripe(conf, group)) != NULL)
5166 5167
		batch[batch_size++] = sh;

5168 5169 5170 5171 5172 5173 5174 5175
	if (batch_size == 0) {
		for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
			if (!list_empty(temp_inactive_list + i))
				break;
		if (i == NR_STRIPE_HASH_LOCKS)
			return batch_size;
		release_inactive = true;
	}
5176 5177
	spin_unlock_irq(&conf->device_lock);

5178 5179 5180 5181 5182 5183 5184 5185
	release_inactive_stripe_list(conf, temp_inactive_list,
				     NR_STRIPE_HASH_LOCKS);

	if (release_inactive) {
		spin_lock_irq(&conf->device_lock);
		return 0;
	}

5186 5187 5188 5189 5190 5191
	for (i = 0; i < batch_size; i++)
		handle_stripe(batch[i]);

	cond_resched();

	spin_lock_irq(&conf->device_lock);
5192 5193 5194 5195
	for (i = 0; i < batch_size; i++) {
		hash = batch[i]->hash_lock_index;
		__release_stripe(conf, batch[i], &temp_inactive_list[hash]);
	}
5196 5197
	return batch_size;
}
5198

5199 5200 5201 5202 5203 5204 5205 5206 5207 5208 5209 5210 5211 5212 5213 5214 5215
static void raid5_do_work(struct work_struct *work)
{
	struct r5worker *worker = container_of(work, struct r5worker, work);
	struct r5worker_group *group = worker->group;
	struct r5conf *conf = group->conf;
	int group_id = group - conf->worker_groups;
	int handled;
	struct blk_plug plug;

	pr_debug("+++ raid5worker active\n");

	blk_start_plug(&plug);
	handled = 0;
	spin_lock_irq(&conf->device_lock);
	while (1) {
		int batch_size, released;

5216
		released = release_stripe_list(conf, worker->temp_inactive_list);
5217

5218 5219
		batch_size = handle_active_stripes(conf, group_id, worker,
						   worker->temp_inactive_list);
5220
		worker->working = false;
5221 5222 5223 5224 5225 5226 5227 5228 5229 5230 5231 5232
		if (!batch_size && !released)
			break;
		handled += batch_size;
	}
	pr_debug("%d stripes handled\n", handled);

	spin_unlock_irq(&conf->device_lock);
	blk_finish_plug(&plug);

	pr_debug("--- raid5worker inactive\n");
}

L
Linus Torvalds 已提交
5233 5234 5235 5236 5237 5238 5239
/*
 * This is our raid5 kernel thread.
 *
 * We scan the hash table for stripes which can be handled now.
 * During the scan, completed stripes are saved for us by the interrupt
 * handler, so that they will not have to wait for our next wakeup.
 */
S
Shaohua Li 已提交
5240
static void raid5d(struct md_thread *thread)
L
Linus Torvalds 已提交
5241
{
S
Shaohua Li 已提交
5242
	struct mddev *mddev = thread->mddev;
5243
	struct r5conf *conf = mddev->private;
L
Linus Torvalds 已提交
5244
	int handled;
5245
	struct blk_plug plug;
L
Linus Torvalds 已提交
5246

5247
	pr_debug("+++ raid5d active\n");
L
Linus Torvalds 已提交
5248 5249 5250

	md_check_recovery(mddev);

5251
	blk_start_plug(&plug);
L
Linus Torvalds 已提交
5252 5253 5254
	handled = 0;
	spin_lock_irq(&conf->device_lock);
	while (1) {
5255
		struct bio *bio;
S
Shaohua Li 已提交
5256 5257
		int batch_size, released;

5258
		released = release_stripe_list(conf, conf->temp_inactive_list);
L
Linus Torvalds 已提交
5259

5260
		if (
5261 5262 5263
		    !list_empty(&conf->bitmap_list)) {
			/* Now is a good time to flush some bitmap updates */
			conf->seq_flush++;
5264
			spin_unlock_irq(&conf->device_lock);
5265
			bitmap_unplug(mddev->bitmap);
5266
			spin_lock_irq(&conf->device_lock);
5267
			conf->seq_write = conf->seq_flush;
5268
			activate_bit_delay(conf, conf->temp_inactive_list);
5269
		}
5270
		raid5_activate_delayed(conf);
5271

5272 5273 5274 5275 5276 5277 5278 5279 5280 5281
		while ((bio = remove_bio_from_retry(conf))) {
			int ok;
			spin_unlock_irq(&conf->device_lock);
			ok = retry_aligned_read(conf, bio);
			spin_lock_irq(&conf->device_lock);
			if (!ok)
				break;
			handled++;
		}

5282 5283
		batch_size = handle_active_stripes(conf, ANY_GROUP, NULL,
						   conf->temp_inactive_list);
S
Shaohua Li 已提交
5284
		if (!batch_size && !released)
L
Linus Torvalds 已提交
5285
			break;
5286
		handled += batch_size;
L
Linus Torvalds 已提交
5287

5288 5289
		if (mddev->flags & ~(1<<MD_CHANGE_PENDING)) {
			spin_unlock_irq(&conf->device_lock);
5290
			md_check_recovery(mddev);
5291 5292
			spin_lock_irq(&conf->device_lock);
		}
L
Linus Torvalds 已提交
5293
	}
5294
	pr_debug("%d stripes handled\n", handled);
L
Linus Torvalds 已提交
5295 5296 5297

	spin_unlock_irq(&conf->device_lock);

5298
	async_tx_issue_pending_all();
5299
	blk_finish_plug(&plug);
L
Linus Torvalds 已提交
5300

5301
	pr_debug("--- raid5d inactive\n");
L
Linus Torvalds 已提交
5302 5303
}

5304
static ssize_t
5305
raid5_show_stripe_cache_size(struct mddev *mddev, char *page)
5306
{
5307
	struct r5conf *conf = mddev->private;
5308 5309 5310 5311
	if (conf)
		return sprintf(page, "%d\n", conf->max_nr_stripes);
	else
		return 0;
5312 5313
}

5314
int
5315
raid5_set_cache_size(struct mddev *mddev, int size)
5316
{
5317
	struct r5conf *conf = mddev->private;
5318
	int err;
5319
	int hash;
5320

5321
	if (size <= 16 || size > 32768)
5322
		return -EINVAL;
5323
	hash = (conf->max_nr_stripes - 1) % NR_STRIPE_HASH_LOCKS;
5324
	while (size < conf->max_nr_stripes) {
5325
		if (drop_one_stripe(conf, hash))
5326 5327 5328
			conf->max_nr_stripes--;
		else
			break;
5329 5330 5331
		hash--;
		if (hash < 0)
			hash = NR_STRIPE_HASH_LOCKS - 1;
5332
	}
5333 5334 5335
	err = md_allow_write(mddev);
	if (err)
		return err;
5336
	hash = conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS;
5337
	while (size > conf->max_nr_stripes) {
5338
		if (grow_one_stripe(conf, hash))
5339 5340
			conf->max_nr_stripes++;
		else break;
5341
		hash = (hash + 1) % NR_STRIPE_HASH_LOCKS;
5342
	}
5343 5344 5345 5346 5347
	return 0;
}
EXPORT_SYMBOL(raid5_set_cache_size);

static ssize_t
5348
raid5_store_stripe_cache_size(struct mddev *mddev, const char *page, size_t len)
5349
{
5350
	struct r5conf *conf = mddev->private;
5351 5352 5353 5354 5355 5356 5357 5358
	unsigned long new;
	int err;

	if (len >= PAGE_SIZE)
		return -EINVAL;
	if (!conf)
		return -ENODEV;

5359
	if (kstrtoul(page, 10, &new))
5360 5361 5362 5363
		return -EINVAL;
	err = raid5_set_cache_size(mddev, new);
	if (err)
		return err;
5364 5365
	return len;
}
5366

5367 5368 5369 5370
static struct md_sysfs_entry
raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR,
				raid5_show_stripe_cache_size,
				raid5_store_stripe_cache_size);
5371

5372
static ssize_t
5373
raid5_show_preread_threshold(struct mddev *mddev, char *page)
5374
{
5375
	struct r5conf *conf = mddev->private;
5376 5377 5378 5379 5380 5381 5382
	if (conf)
		return sprintf(page, "%d\n", conf->bypass_threshold);
	else
		return 0;
}

static ssize_t
5383
raid5_store_preread_threshold(struct mddev *mddev, const char *page, size_t len)
5384
{
5385
	struct r5conf *conf = mddev->private;
5386
	unsigned long new;
5387 5388 5389 5390 5391
	if (len >= PAGE_SIZE)
		return -EINVAL;
	if (!conf)
		return -ENODEV;

5392
	if (kstrtoul(page, 10, &new))
5393
		return -EINVAL;
5394
	if (new > conf->max_nr_stripes)
5395 5396 5397 5398 5399 5400 5401 5402 5403 5404 5405
		return -EINVAL;
	conf->bypass_threshold = new;
	return len;
}

static struct md_sysfs_entry
raid5_preread_bypass_threshold = __ATTR(preread_bypass_threshold,
					S_IRUGO | S_IWUSR,
					raid5_show_preread_threshold,
					raid5_store_preread_threshold);

5406 5407 5408 5409 5410 5411 5412 5413 5414 5415 5416 5417 5418 5419 5420 5421 5422 5423 5424 5425 5426 5427 5428 5429 5430 5431 5432 5433 5434 5435 5436 5437 5438 5439 5440 5441 5442 5443 5444 5445 5446 5447 5448 5449
static ssize_t
raid5_show_skip_copy(struct mddev *mddev, char *page)
{
	struct r5conf *conf = mddev->private;
	if (conf)
		return sprintf(page, "%d\n", conf->skip_copy);
	else
		return 0;
}

static ssize_t
raid5_store_skip_copy(struct mddev *mddev, const char *page, size_t len)
{
	struct r5conf *conf = mddev->private;
	unsigned long new;
	if (len >= PAGE_SIZE)
		return -EINVAL;
	if (!conf)
		return -ENODEV;

	if (kstrtoul(page, 10, &new))
		return -EINVAL;
	new = !!new;
	if (new == conf->skip_copy)
		return len;

	mddev_suspend(mddev);
	conf->skip_copy = new;
	if (new)
		mddev->queue->backing_dev_info.capabilities |=
						BDI_CAP_STABLE_WRITES;
	else
		mddev->queue->backing_dev_info.capabilities &=
						~BDI_CAP_STABLE_WRITES;
	mddev_resume(mddev);
	return len;
}

static struct md_sysfs_entry
raid5_skip_copy = __ATTR(skip_copy, S_IRUGO | S_IWUSR,
					raid5_show_skip_copy,
					raid5_store_skip_copy);


5450
static ssize_t
5451
stripe_cache_active_show(struct mddev *mddev, char *page)
5452
{
5453
	struct r5conf *conf = mddev->private;
5454 5455 5456 5457
	if (conf)
		return sprintf(page, "%d\n", atomic_read(&conf->active_stripes));
	else
		return 0;
5458 5459
}

5460 5461
static struct md_sysfs_entry
raid5_stripecache_active = __ATTR_RO(stripe_cache_active);
5462

5463 5464 5465 5466 5467 5468 5469 5470 5471 5472
static ssize_t
raid5_show_group_thread_cnt(struct mddev *mddev, char *page)
{
	struct r5conf *conf = mddev->private;
	if (conf)
		return sprintf(page, "%d\n", conf->worker_cnt_per_group);
	else
		return 0;
}

5473 5474 5475 5476
static int alloc_thread_groups(struct r5conf *conf, int cnt,
			       int *group_cnt,
			       int *worker_cnt_per_group,
			       struct r5worker_group **worker_groups);
5477 5478 5479 5480 5481 5482
static ssize_t
raid5_store_group_thread_cnt(struct mddev *mddev, const char *page, size_t len)
{
	struct r5conf *conf = mddev->private;
	unsigned long new;
	int err;
5483 5484
	struct r5worker_group *new_groups, *old_groups;
	int group_cnt, worker_cnt_per_group;
5485 5486 5487 5488 5489 5490 5491 5492 5493 5494 5495 5496 5497 5498 5499

	if (len >= PAGE_SIZE)
		return -EINVAL;
	if (!conf)
		return -ENODEV;

	if (kstrtoul(page, 10, &new))
		return -EINVAL;

	if (new == conf->worker_cnt_per_group)
		return len;

	mddev_suspend(mddev);

	old_groups = conf->worker_groups;
5500 5501 5502
	if (old_groups)
		flush_workqueue(raid5_wq);

5503 5504 5505 5506 5507 5508 5509 5510 5511
	err = alloc_thread_groups(conf, new,
				  &group_cnt, &worker_cnt_per_group,
				  &new_groups);
	if (!err) {
		spin_lock_irq(&conf->device_lock);
		conf->group_cnt = group_cnt;
		conf->worker_cnt_per_group = worker_cnt_per_group;
		conf->worker_groups = new_groups;
		spin_unlock_irq(&conf->device_lock);
5512 5513 5514 5515 5516 5517 5518 5519 5520 5521 5522 5523 5524 5525 5526 5527 5528 5529

		if (old_groups)
			kfree(old_groups[0].workers);
		kfree(old_groups);
	}

	mddev_resume(mddev);

	if (err)
		return err;
	return len;
}

static struct md_sysfs_entry
raid5_group_thread_cnt = __ATTR(group_thread_cnt, S_IRUGO | S_IWUSR,
				raid5_show_group_thread_cnt,
				raid5_store_group_thread_cnt);

5530
static struct attribute *raid5_attrs[] =  {
5531 5532
	&raid5_stripecache_size.attr,
	&raid5_stripecache_active.attr,
5533
	&raid5_preread_bypass_threshold.attr,
5534
	&raid5_group_thread_cnt.attr,
5535
	&raid5_skip_copy.attr,
5536 5537
	NULL,
};
5538 5539 5540
static struct attribute_group raid5_attrs_group = {
	.name = NULL,
	.attrs = raid5_attrs,
5541 5542
};

5543 5544 5545 5546
static int alloc_thread_groups(struct r5conf *conf, int cnt,
			       int *group_cnt,
			       int *worker_cnt_per_group,
			       struct r5worker_group **worker_groups)
5547
{
5548
	int i, j, k;
5549 5550 5551
	ssize_t size;
	struct r5worker *workers;

5552
	*worker_cnt_per_group = cnt;
5553
	if (cnt == 0) {
5554 5555
		*group_cnt = 0;
		*worker_groups = NULL;
5556 5557
		return 0;
	}
5558
	*group_cnt = num_possible_nodes();
5559
	size = sizeof(struct r5worker) * cnt;
5560 5561 5562 5563
	workers = kzalloc(size * *group_cnt, GFP_NOIO);
	*worker_groups = kzalloc(sizeof(struct r5worker_group) *
				*group_cnt, GFP_NOIO);
	if (!*worker_groups || !workers) {
5564
		kfree(workers);
5565
		kfree(*worker_groups);
5566 5567 5568
		return -ENOMEM;
	}

5569
	for (i = 0; i < *group_cnt; i++) {
5570 5571
		struct r5worker_group *group;

5572
		group = &(*worker_groups)[i];
5573 5574 5575 5576 5577
		INIT_LIST_HEAD(&group->handle_list);
		group->conf = conf;
		group->workers = workers + i * cnt;

		for (j = 0; j < cnt; j++) {
5578 5579 5580 5581 5582 5583
			struct r5worker *worker = group->workers + j;
			worker->group = group;
			INIT_WORK(&worker->work, raid5_do_work);

			for (k = 0; k < NR_STRIPE_HASH_LOCKS; k++)
				INIT_LIST_HEAD(worker->temp_inactive_list + k);
5584 5585 5586 5587 5588 5589 5590 5591 5592 5593 5594 5595 5596 5597
		}
	}

	return 0;
}

static void free_thread_groups(struct r5conf *conf)
{
	if (conf->worker_groups)
		kfree(conf->worker_groups[0].workers);
	kfree(conf->worker_groups);
	conf->worker_groups = NULL;
}

5598
static sector_t
5599
raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks)
5600
{
5601
	struct r5conf *conf = mddev->private;
5602 5603 5604

	if (!sectors)
		sectors = mddev->dev_sectors;
5605
	if (!raid_disks)
5606
		/* size is defined by the smallest of previous and new size */
5607
		raid_disks = min(conf->raid_disks, conf->previous_raid_disks);
5608

5609
	sectors &= ~((sector_t)mddev->chunk_sectors - 1);
5610
	sectors &= ~((sector_t)mddev->new_chunk_sectors - 1);
5611 5612 5613
	return sectors * (raid_disks - conf->max_degraded);
}

5614 5615 5616 5617 5618 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628 5629 5630 5631 5632 5633 5634 5635 5636
static void free_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu)
{
	safe_put_page(percpu->spare_page);
	kfree(percpu->scribble);
	percpu->spare_page = NULL;
	percpu->scribble = NULL;
}

static int alloc_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu)
{
	if (conf->level == 6 && !percpu->spare_page)
		percpu->spare_page = alloc_page(GFP_KERNEL);
	if (!percpu->scribble)
		percpu->scribble = kmalloc(conf->scribble_len, GFP_KERNEL);

	if (!percpu->scribble || (conf->level == 6 && !percpu->spare_page)) {
		free_scratch_buffer(conf, percpu);
		return -ENOMEM;
	}

	return 0;
}

5637
static void raid5_free_percpu(struct r5conf *conf)
5638 5639 5640 5641 5642 5643 5644 5645 5646
{
	unsigned long cpu;

	if (!conf->percpu)
		return;

#ifdef CONFIG_HOTPLUG_CPU
	unregister_cpu_notifier(&conf->cpu_notify);
#endif
5647 5648 5649 5650

	get_online_cpus();
	for_each_possible_cpu(cpu)
		free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));
5651 5652 5653 5654 5655
	put_online_cpus();

	free_percpu(conf->percpu);
}

5656
static void free_conf(struct r5conf *conf)
5657
{
5658
	free_thread_groups(conf);
5659
	shrink_stripes(conf);
5660
	raid5_free_percpu(conf);
5661 5662 5663 5664 5665
	kfree(conf->disks);
	kfree(conf->stripe_hashtbl);
	kfree(conf);
}

5666 5667 5668 5669
#ifdef CONFIG_HOTPLUG_CPU
static int raid456_cpu_notify(struct notifier_block *nfb, unsigned long action,
			      void *hcpu)
{
5670
	struct r5conf *conf = container_of(nfb, struct r5conf, cpu_notify);
5671 5672 5673 5674 5675 5676
	long cpu = (long)hcpu;
	struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu);

	switch (action) {
	case CPU_UP_PREPARE:
	case CPU_UP_PREPARE_FROZEN:
5677
		if (alloc_scratch_buffer(conf, percpu)) {
5678 5679
			pr_err("%s: failed memory allocation for cpu%ld\n",
			       __func__, cpu);
5680
			return notifier_from_errno(-ENOMEM);
5681 5682 5683 5684
		}
		break;
	case CPU_DEAD:
	case CPU_DEAD_FROZEN:
5685
		free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));
5686 5687 5688 5689 5690 5691 5692 5693
		break;
	default:
		break;
	}
	return NOTIFY_OK;
}
#endif

5694
static int raid5_alloc_percpu(struct r5conf *conf)
5695 5696
{
	unsigned long cpu;
5697
	int err = 0;
5698

5699 5700
	conf->percpu = alloc_percpu(struct raid5_percpu);
	if (!conf->percpu)
5701
		return -ENOMEM;
5702 5703 5704 5705 5706 5707 5708 5709

#ifdef CONFIG_HOTPLUG_CPU
	conf->cpu_notify.notifier_call = raid456_cpu_notify;
	conf->cpu_notify.priority = 0;
	err = register_cpu_notifier(&conf->cpu_notify);
	if (err)
		return err;
#endif
5710 5711 5712

	get_online_cpus();
	for_each_present_cpu(cpu) {
5713 5714 5715 5716
		err = alloc_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));
		if (err) {
			pr_err("%s: failed memory allocation for cpu%ld\n",
			       __func__, cpu);
5717 5718 5719 5720 5721 5722 5723 5724
			break;
		}
	}
	put_online_cpus();

	return err;
}

5725
static struct r5conf *setup_conf(struct mddev *mddev)
L
Linus Torvalds 已提交
5726
{
5727
	struct r5conf *conf;
5728
	int raid_disk, memory, max_disks;
5729
	struct md_rdev *rdev;
L
Linus Torvalds 已提交
5730
	struct disk_info *disk;
5731
	char pers_name[6];
5732
	int i;
5733 5734
	int group_cnt, worker_cnt_per_group;
	struct r5worker_group *new_group;
L
Linus Torvalds 已提交
5735

N
NeilBrown 已提交
5736 5737 5738
	if (mddev->new_level != 5
	    && mddev->new_level != 4
	    && mddev->new_level != 6) {
5739
		printk(KERN_ERR "md/raid:%s: raid level not set to 4/5/6 (%d)\n",
N
NeilBrown 已提交
5740 5741
		       mdname(mddev), mddev->new_level);
		return ERR_PTR(-EIO);
L
Linus Torvalds 已提交
5742
	}
N
NeilBrown 已提交
5743 5744 5745 5746
	if ((mddev->new_level == 5
	     && !algorithm_valid_raid5(mddev->new_layout)) ||
	    (mddev->new_level == 6
	     && !algorithm_valid_raid6(mddev->new_layout))) {
5747
		printk(KERN_ERR "md/raid:%s: layout %d not supported\n",
N
NeilBrown 已提交
5748 5749
		       mdname(mddev), mddev->new_layout);
		return ERR_PTR(-EIO);
5750
	}
N
NeilBrown 已提交
5751
	if (mddev->new_level == 6 && mddev->raid_disks < 4) {
5752
		printk(KERN_ERR "md/raid:%s: not enough configured devices (%d, minimum 4)\n",
N
NeilBrown 已提交
5753 5754
		       mdname(mddev), mddev->raid_disks);
		return ERR_PTR(-EINVAL);
5755 5756
	}

5757 5758 5759
	if (!mddev->new_chunk_sectors ||
	    (mddev->new_chunk_sectors << 9) % PAGE_SIZE ||
	    !is_power_of_2(mddev->new_chunk_sectors)) {
5760 5761
		printk(KERN_ERR "md/raid:%s: invalid chunk size %d\n",
		       mdname(mddev), mddev->new_chunk_sectors << 9);
N
NeilBrown 已提交
5762
		return ERR_PTR(-EINVAL);
5763 5764
	}

5765
	conf = kzalloc(sizeof(struct r5conf), GFP_KERNEL);
N
NeilBrown 已提交
5766
	if (conf == NULL)
L
Linus Torvalds 已提交
5767
		goto abort;
5768
	/* Don't enable multi-threading by default*/
5769 5770 5771 5772 5773 5774
	if (!alloc_thread_groups(conf, 0, &group_cnt, &worker_cnt_per_group,
				 &new_group)) {
		conf->group_cnt = group_cnt;
		conf->worker_cnt_per_group = worker_cnt_per_group;
		conf->worker_groups = new_group;
	} else
5775
		goto abort;
5776
	spin_lock_init(&conf->device_lock);
5777
	seqcount_init(&conf->gen_lock);
5778 5779 5780 5781 5782 5783
	init_waitqueue_head(&conf->wait_for_stripe);
	init_waitqueue_head(&conf->wait_for_overlap);
	INIT_LIST_HEAD(&conf->handle_list);
	INIT_LIST_HEAD(&conf->hold_list);
	INIT_LIST_HEAD(&conf->delayed_list);
	INIT_LIST_HEAD(&conf->bitmap_list);
S
Shaohua Li 已提交
5784
	init_llist_head(&conf->released_stripes);
5785 5786 5787 5788
	atomic_set(&conf->active_stripes, 0);
	atomic_set(&conf->preread_active_stripes, 0);
	atomic_set(&conf->active_aligned_reads, 0);
	conf->bypass_threshold = BYPASS_THRESHOLD;
5789
	conf->recovery_disabled = mddev->recovery_disabled - 1;
N
NeilBrown 已提交
5790 5791 5792 5793 5794

	conf->raid_disks = mddev->raid_disks;
	if (mddev->reshape_position == MaxSector)
		conf->previous_raid_disks = mddev->raid_disks;
	else
5795
		conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks;
5796 5797
	max_disks = max(conf->raid_disks, conf->previous_raid_disks);
	conf->scribble_len = scribble_len(max_disks);
5798

5799
	conf->disks = kzalloc(max_disks * sizeof(struct disk_info),
5800 5801 5802
			      GFP_KERNEL);
	if (!conf->disks)
		goto abort;
5803

L
Linus Torvalds 已提交
5804 5805
	conf->mddev = mddev;

5806
	if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL)
L
Linus Torvalds 已提交
5807 5808
		goto abort;

5809 5810 5811 5812 5813 5814 5815 5816 5817 5818 5819 5820 5821 5822 5823
	/* We init hash_locks[0] separately to that it can be used
	 * as the reference lock in the spin_lock_nest_lock() call
	 * in lock_all_device_hash_locks_irq in order to convince
	 * lockdep that we know what we are doing.
	 */
	spin_lock_init(conf->hash_locks);
	for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++)
		spin_lock_init(conf->hash_locks + i);

	for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
		INIT_LIST_HEAD(conf->inactive_list + i);

	for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
		INIT_LIST_HEAD(conf->temp_inactive_list + i);

5824 5825 5826 5827
	conf->level = mddev->new_level;
	if (raid5_alloc_percpu(conf) != 0)
		goto abort;

5828
	pr_debug("raid456: run(%s) called.\n", mdname(mddev));
L
Linus Torvalds 已提交
5829

N
NeilBrown 已提交
5830
	rdev_for_each(rdev, mddev) {
L
Linus Torvalds 已提交
5831
		raid_disk = rdev->raid_disk;
5832
		if (raid_disk >= max_disks
L
Linus Torvalds 已提交
5833 5834 5835 5836
		    || raid_disk < 0)
			continue;
		disk = conf->disks + raid_disk;

5837 5838 5839 5840 5841 5842 5843 5844 5845
		if (test_bit(Replacement, &rdev->flags)) {
			if (disk->replacement)
				goto abort;
			disk->replacement = rdev;
		} else {
			if (disk->rdev)
				goto abort;
			disk->rdev = rdev;
		}
L
Linus Torvalds 已提交
5846

5847
		if (test_bit(In_sync, &rdev->flags)) {
L
Linus Torvalds 已提交
5848
			char b[BDEVNAME_SIZE];
5849 5850 5851
			printk(KERN_INFO "md/raid:%s: device %s operational as raid"
			       " disk %d\n",
			       mdname(mddev), bdevname(rdev->bdev, b), raid_disk);
J
Jonathan Brassow 已提交
5852
		} else if (rdev->saved_raid_disk != raid_disk)
5853 5854
			/* Cannot rely on bitmap to complete recovery */
			conf->fullsync = 1;
L
Linus Torvalds 已提交
5855 5856
	}

5857
	conf->chunk_sectors = mddev->new_chunk_sectors;
N
NeilBrown 已提交
5858
	conf->level = mddev->new_level;
5859 5860 5861 5862
	if (conf->level == 6)
		conf->max_degraded = 2;
	else
		conf->max_degraded = 1;
N
NeilBrown 已提交
5863
	conf->algorithm = mddev->new_layout;
5864
	conf->reshape_progress = mddev->reshape_position;
5865
	if (conf->reshape_progress != MaxSector) {
5866
		conf->prev_chunk_sectors = mddev->chunk_sectors;
5867 5868
		conf->prev_algo = mddev->layout;
	}
L
Linus Torvalds 已提交
5869

N
NeilBrown 已提交
5870
	memory = conf->max_nr_stripes * (sizeof(struct stripe_head) +
5871
		 max_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024;
5872
	atomic_set(&conf->empty_inactive_list_nr, NR_STRIPE_HASH_LOCKS);
5873
	if (grow_stripes(conf, NR_STRIPES)) {
N
NeilBrown 已提交
5874
		printk(KERN_ERR
5875 5876
		       "md/raid:%s: couldn't allocate %dkB for buffers\n",
		       mdname(mddev), memory);
N
NeilBrown 已提交
5877 5878
		goto abort;
	} else
5879 5880
		printk(KERN_INFO "md/raid:%s: allocated %dkB\n",
		       mdname(mddev), memory);
L
Linus Torvalds 已提交
5881

5882 5883
	sprintf(pers_name, "raid%d", mddev->new_level);
	conf->thread = md_register_thread(raid5d, mddev, pers_name);
N
NeilBrown 已提交
5884 5885
	if (!conf->thread) {
		printk(KERN_ERR
5886
		       "md/raid:%s: couldn't allocate thread.\n",
N
NeilBrown 已提交
5887
		       mdname(mddev));
5888 5889
		goto abort;
	}
N
NeilBrown 已提交
5890 5891 5892 5893 5894

	return conf;

 abort:
	if (conf) {
5895
		free_conf(conf);
N
NeilBrown 已提交
5896 5897 5898 5899 5900
		return ERR_PTR(-EIO);
	} else
		return ERR_PTR(-ENOMEM);
}

5901 5902 5903 5904 5905 5906 5907 5908 5909 5910 5911 5912 5913 5914 5915 5916 5917 5918 5919 5920 5921 5922 5923 5924 5925 5926 5927

static int only_parity(int raid_disk, int algo, int raid_disks, int max_degraded)
{
	switch (algo) {
	case ALGORITHM_PARITY_0:
		if (raid_disk < max_degraded)
			return 1;
		break;
	case ALGORITHM_PARITY_N:
		if (raid_disk >= raid_disks - max_degraded)
			return 1;
		break;
	case ALGORITHM_PARITY_0_6:
		if (raid_disk == 0 || 
		    raid_disk == raid_disks - 1)
			return 1;
		break;
	case ALGORITHM_LEFT_ASYMMETRIC_6:
	case ALGORITHM_RIGHT_ASYMMETRIC_6:
	case ALGORITHM_LEFT_SYMMETRIC_6:
	case ALGORITHM_RIGHT_SYMMETRIC_6:
		if (raid_disk == raid_disks - 1)
			return 1;
	}
	return 0;
}

5928
static int run(struct mddev *mddev)
N
NeilBrown 已提交
5929
{
5930
	struct r5conf *conf;
5931
	int working_disks = 0;
5932
	int dirty_parity_disks = 0;
5933
	struct md_rdev *rdev;
5934
	sector_t reshape_offset = 0;
5935
	int i;
5936 5937
	long long min_offset_diff = 0;
	int first = 1;
N
NeilBrown 已提交
5938

5939
	if (mddev->recovery_cp != MaxSector)
5940
		printk(KERN_NOTICE "md/raid:%s: not clean"
5941 5942
		       " -- starting background reconstruction\n",
		       mdname(mddev));
5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953 5954 5955 5956 5957 5958 5959

	rdev_for_each(rdev, mddev) {
		long long diff;
		if (rdev->raid_disk < 0)
			continue;
		diff = (rdev->new_data_offset - rdev->data_offset);
		if (first) {
			min_offset_diff = diff;
			first = 0;
		} else if (mddev->reshape_backwards &&
			 diff < min_offset_diff)
			min_offset_diff = diff;
		else if (!mddev->reshape_backwards &&
			 diff > min_offset_diff)
			min_offset_diff = diff;
	}

N
NeilBrown 已提交
5960 5961
	if (mddev->reshape_position != MaxSector) {
		/* Check that we can continue the reshape.
5962 5963 5964 5965 5966 5967 5968 5969 5970 5971
		 * Difficulties arise if the stripe we would write to
		 * next is at or after the stripe we would read from next.
		 * For a reshape that changes the number of devices, this
		 * is only possible for a very short time, and mdadm makes
		 * sure that time appears to have past before assembling
		 * the array.  So we fail if that time hasn't passed.
		 * For a reshape that keeps the number of devices the same
		 * mdadm must be monitoring the reshape can keeping the
		 * critical areas read-only and backed up.  It will start
		 * the array in read-only mode, so we check for that.
N
NeilBrown 已提交
5972 5973 5974
		 */
		sector_t here_new, here_old;
		int old_disks;
5975
		int max_degraded = (mddev->level == 6 ? 2 : 1);
N
NeilBrown 已提交
5976

5977
		if (mddev->new_level != mddev->level) {
5978
			printk(KERN_ERR "md/raid:%s: unsupported reshape "
N
NeilBrown 已提交
5979 5980 5981 5982 5983 5984 5985 5986 5987 5988
			       "required - aborting.\n",
			       mdname(mddev));
			return -EINVAL;
		}
		old_disks = mddev->raid_disks - mddev->delta_disks;
		/* reshape_position must be on a new-stripe boundary, and one
		 * further up in new geometry must map after here in old
		 * geometry.
		 */
		here_new = mddev->reshape_position;
5989
		if (sector_div(here_new, mddev->new_chunk_sectors *
N
NeilBrown 已提交
5990
			       (mddev->raid_disks - max_degraded))) {
5991 5992
			printk(KERN_ERR "md/raid:%s: reshape_position not "
			       "on a stripe boundary\n", mdname(mddev));
N
NeilBrown 已提交
5993 5994
			return -EINVAL;
		}
5995
		reshape_offset = here_new * mddev->new_chunk_sectors;
N
NeilBrown 已提交
5996 5997
		/* here_new is the stripe we will write to */
		here_old = mddev->reshape_position;
5998
		sector_div(here_old, mddev->chunk_sectors *
N
NeilBrown 已提交
5999 6000 6001
			   (old_disks-max_degraded));
		/* here_old is the first stripe that we might need to read
		 * from */
6002
		if (mddev->delta_disks == 0) {
6003 6004 6005 6006 6007 6008
			if ((here_new * mddev->new_chunk_sectors !=
			     here_old * mddev->chunk_sectors)) {
				printk(KERN_ERR "md/raid:%s: reshape position is"
				       " confused - aborting\n", mdname(mddev));
				return -EINVAL;
			}
6009
			/* We cannot be sure it is safe to start an in-place
6010
			 * reshape.  It is only safe if user-space is monitoring
6011 6012 6013 6014 6015
			 * and taking constant backups.
			 * mdadm always starts a situation like this in
			 * readonly mode so it can take control before
			 * allowing any writes.  So just check for that.
			 */
6016 6017 6018 6019 6020 6021 6022
			if (abs(min_offset_diff) >= mddev->chunk_sectors &&
			    abs(min_offset_diff) >= mddev->new_chunk_sectors)
				/* not really in-place - so OK */;
			else if (mddev->ro == 0) {
				printk(KERN_ERR "md/raid:%s: in-place reshape "
				       "must be started in read-only mode "
				       "- aborting\n",
6023
				       mdname(mddev));
6024 6025
				return -EINVAL;
			}
6026
		} else if (mddev->reshape_backwards
6027
		    ? (here_new * mddev->new_chunk_sectors + min_offset_diff <=
6028 6029
		       here_old * mddev->chunk_sectors)
		    : (here_new * mddev->new_chunk_sectors >=
6030
		       here_old * mddev->chunk_sectors + (-min_offset_diff))) {
N
NeilBrown 已提交
6031
			/* Reading from the same stripe as writing to - bad */
6032 6033 6034
			printk(KERN_ERR "md/raid:%s: reshape_position too early for "
			       "auto-recovery - aborting.\n",
			       mdname(mddev));
N
NeilBrown 已提交
6035 6036
			return -EINVAL;
		}
6037 6038
		printk(KERN_INFO "md/raid:%s: reshape will continue\n",
		       mdname(mddev));
N
NeilBrown 已提交
6039 6040 6041 6042
		/* OK, we should be able to continue; */
	} else {
		BUG_ON(mddev->level != mddev->new_level);
		BUG_ON(mddev->layout != mddev->new_layout);
6043
		BUG_ON(mddev->chunk_sectors != mddev->new_chunk_sectors);
N
NeilBrown 已提交
6044
		BUG_ON(mddev->delta_disks != 0);
L
Linus Torvalds 已提交
6045
	}
N
NeilBrown 已提交
6046

6047 6048 6049 6050 6051
	if (mddev->private == NULL)
		conf = setup_conf(mddev);
	else
		conf = mddev->private;

N
NeilBrown 已提交
6052 6053 6054
	if (IS_ERR(conf))
		return PTR_ERR(conf);

6055
	conf->min_offset_diff = min_offset_diff;
N
NeilBrown 已提交
6056 6057 6058 6059
	mddev->thread = conf->thread;
	conf->thread = NULL;
	mddev->private = conf;

6060 6061 6062 6063 6064 6065 6066 6067 6068 6069 6070
	for (i = 0; i < conf->raid_disks && conf->previous_raid_disks;
	     i++) {
		rdev = conf->disks[i].rdev;
		if (!rdev && conf->disks[i].replacement) {
			/* The replacement is all we have yet */
			rdev = conf->disks[i].replacement;
			conf->disks[i].replacement = NULL;
			clear_bit(Replacement, &rdev->flags);
			conf->disks[i].rdev = rdev;
		}
		if (!rdev)
6071
			continue;
6072 6073 6074 6075 6076 6077 6078
		if (conf->disks[i].replacement &&
		    conf->reshape_progress != MaxSector) {
			/* replacements and reshape simply do not mix. */
			printk(KERN_ERR "md: cannot handle concurrent "
			       "replacement and reshape.\n");
			goto abort;
		}
6079
		if (test_bit(In_sync, &rdev->flags)) {
N
NeilBrown 已提交
6080
			working_disks++;
6081 6082
			continue;
		}
6083 6084 6085 6086 6087 6088 6089 6090 6091 6092 6093 6094
		/* This disc is not fully in-sync.  However if it
		 * just stored parity (beyond the recovery_offset),
		 * when we don't need to be concerned about the
		 * array being dirty.
		 * When reshape goes 'backwards', we never have
		 * partially completed devices, so we only need
		 * to worry about reshape going forwards.
		 */
		/* Hack because v0.91 doesn't store recovery_offset properly. */
		if (mddev->major_version == 0 &&
		    mddev->minor_version > 90)
			rdev->recovery_offset = reshape_offset;
6095

6096 6097 6098 6099 6100 6101 6102 6103 6104 6105 6106 6107 6108 6109 6110
		if (rdev->recovery_offset < reshape_offset) {
			/* We need to check old and new layout */
			if (!only_parity(rdev->raid_disk,
					 conf->algorithm,
					 conf->raid_disks,
					 conf->max_degraded))
				continue;
		}
		if (!only_parity(rdev->raid_disk,
				 conf->prev_algo,
				 conf->previous_raid_disks,
				 conf->max_degraded))
			continue;
		dirty_parity_disks++;
	}
N
NeilBrown 已提交
6111

6112 6113 6114
	/*
	 * 0 for a fully functional array, 1 or 2 for a degraded array.
	 */
6115
	mddev->degraded = calc_degraded(conf);
N
NeilBrown 已提交
6116

6117
	if (has_failed(conf)) {
6118
		printk(KERN_ERR "md/raid:%s: not enough operational devices"
L
Linus Torvalds 已提交
6119
			" (%d/%d failed)\n",
6120
			mdname(mddev), mddev->degraded, conf->raid_disks);
L
Linus Torvalds 已提交
6121 6122 6123
		goto abort;
	}

N
NeilBrown 已提交
6124
	/* device size must be a multiple of chunk size */
6125
	mddev->dev_sectors &= ~(mddev->chunk_sectors - 1);
N
NeilBrown 已提交
6126 6127
	mddev->resync_max_sectors = mddev->dev_sectors;

6128
	if (mddev->degraded > dirty_parity_disks &&
L
Linus Torvalds 已提交
6129
	    mddev->recovery_cp != MaxSector) {
6130 6131
		if (mddev->ok_start_degraded)
			printk(KERN_WARNING
6132 6133
			       "md/raid:%s: starting dirty degraded array"
			       " - data corruption possible.\n",
6134 6135 6136
			       mdname(mddev));
		else {
			printk(KERN_ERR
6137
			       "md/raid:%s: cannot start dirty degraded array.\n",
6138 6139 6140
			       mdname(mddev));
			goto abort;
		}
L
Linus Torvalds 已提交
6141 6142 6143
	}

	if (mddev->degraded == 0)
6144 6145
		printk(KERN_INFO "md/raid:%s: raid level %d active with %d out of %d"
		       " devices, algorithm %d\n", mdname(mddev), conf->level,
6146 6147
		       mddev->raid_disks-mddev->degraded, mddev->raid_disks,
		       mddev->new_layout);
L
Linus Torvalds 已提交
6148
	else
6149 6150 6151 6152 6153
		printk(KERN_ALERT "md/raid:%s: raid level %d active with %d"
		       " out of %d devices, algorithm %d\n",
		       mdname(mddev), conf->level,
		       mddev->raid_disks - mddev->degraded,
		       mddev->raid_disks, mddev->new_layout);
L
Linus Torvalds 已提交
6154 6155 6156

	print_raid5_conf(conf);

6157 6158
	if (conf->reshape_progress != MaxSector) {
		conf->reshape_safe = conf->reshape_progress;
6159 6160 6161 6162 6163 6164
		atomic_set(&conf->reshape_stripes, 0);
		clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
		clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
		set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
		set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
		mddev->sync_thread = md_register_thread(md_do_sync, mddev,
6165
							"reshape");
6166 6167
	}

L
Linus Torvalds 已提交
6168 6169

	/* Ok, everything is just fine now */
6170 6171
	if (mddev->to_remove == &raid5_attrs_group)
		mddev->to_remove = NULL;
N
NeilBrown 已提交
6172 6173
	else if (mddev->kobj.sd &&
	    sysfs_create_group(&mddev->kobj, &raid5_attrs_group))
6174
		printk(KERN_WARNING
6175
		       "raid5: failed to create sysfs attributes for %s\n",
6176
		       mdname(mddev));
6177
	md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
6178

6179
	if (mddev->queue) {
6180
		int chunk_size;
S
Shaohua Li 已提交
6181
		bool discard_supported = true;
6182 6183 6184 6185 6186 6187 6188 6189 6190
		/* read-ahead size must cover two whole stripes, which
		 * is 2 * (datadisks) * chunksize where 'n' is the
		 * number of raid devices
		 */
		int data_disks = conf->previous_raid_disks - conf->max_degraded;
		int stripe = data_disks *
			((mddev->chunk_sectors << 9) / PAGE_SIZE);
		if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
			mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
N
NeilBrown 已提交
6191

6192
		blk_queue_merge_bvec(mddev->queue, raid5_mergeable_bvec);
6193

N
NeilBrown 已提交
6194 6195
		mddev->queue->backing_dev_info.congested_data = mddev;
		mddev->queue->backing_dev_info.congested_fn = raid5_congested;
6196

6197 6198 6199 6200
		chunk_size = mddev->chunk_sectors << 9;
		blk_queue_io_min(mddev->queue, chunk_size);
		blk_queue_io_opt(mddev->queue, chunk_size *
				 (conf->raid_disks - conf->max_degraded));
6201
		mddev->queue->limits.raid_partial_stripes_expensive = 1;
S
Shaohua Li 已提交
6202 6203 6204 6205 6206
		/*
		 * We can only discard a whole stripe. It doesn't make sense to
		 * discard data disk but write parity disk
		 */
		stripe = stripe * PAGE_SIZE;
6207 6208 6209 6210
		/* Round up to power of 2, as discard handling
		 * currently assumes that */
		while ((stripe-1) & stripe)
			stripe = (stripe | (stripe-1)) + 1;
S
Shaohua Li 已提交
6211 6212 6213 6214
		mddev->queue->limits.discard_alignment = stripe;
		mddev->queue->limits.discard_granularity = stripe;
		/*
		 * unaligned part of discard request will be ignored, so can't
6215
		 * guarantee discard_zeroes_data
S
Shaohua Li 已提交
6216 6217
		 */
		mddev->queue->limits.discard_zeroes_data = 0;
6218

6219 6220
		blk_queue_max_write_same_sectors(mddev->queue, 0);

6221
		rdev_for_each(rdev, mddev) {
6222 6223
			disk_stack_limits(mddev->gendisk, rdev->bdev,
					  rdev->data_offset << 9);
6224 6225
			disk_stack_limits(mddev->gendisk, rdev->bdev,
					  rdev->new_data_offset << 9);
S
Shaohua Li 已提交
6226 6227 6228 6229 6230 6231 6232 6233 6234 6235 6236 6237 6238 6239
			/*
			 * discard_zeroes_data is required, otherwise data
			 * could be lost. Consider a scenario: discard a stripe
			 * (the stripe could be inconsistent if
			 * discard_zeroes_data is 0); write one disk of the
			 * stripe (the stripe could be inconsistent again
			 * depending on which disks are used to calculate
			 * parity); the disk is broken; The stripe data of this
			 * disk is lost.
			 */
			if (!blk_queue_discard(bdev_get_queue(rdev->bdev)) ||
			    !bdev_get_queue(rdev->bdev)->
						limits.discard_zeroes_data)
				discard_supported = false;
6240 6241 6242 6243 6244 6245 6246 6247 6248 6249 6250 6251
			/* Unfortunately, discard_zeroes_data is not currently
			 * a guarantee - just a hint.  So we only allow DISCARD
			 * if the sysadmin has confirmed that only safe devices
			 * are in use by setting a module parameter.
			 */
			if (!devices_handle_discard_safely) {
				if (discard_supported) {
					pr_info("md/raid456: discard support disabled due to uncertainty.\n");
					pr_info("Set raid456.devices_handle_discard_safely=Y to override.\n");
				}
				discard_supported = false;
			}
6252
		}
S
Shaohua Li 已提交
6253 6254 6255 6256 6257 6258 6259 6260 6261

		if (discard_supported &&
		   mddev->queue->limits.max_discard_sectors >= stripe &&
		   mddev->queue->limits.discard_granularity >= stripe)
			queue_flag_set_unlocked(QUEUE_FLAG_DISCARD,
						mddev->queue);
		else
			queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD,
						mddev->queue);
6262
	}
6263

L
Linus Torvalds 已提交
6264 6265
	return 0;
abort:
6266
	md_unregister_thread(&mddev->thread);
N
NeilBrown 已提交
6267 6268
	print_raid5_conf(conf);
	free_conf(conf);
L
Linus Torvalds 已提交
6269
	mddev->private = NULL;
6270
	printk(KERN_ALERT "md/raid:%s: failed to run raid set.\n", mdname(mddev));
L
Linus Torvalds 已提交
6271 6272 6273
	return -EIO;
}

6274
static int stop(struct mddev *mddev)
L
Linus Torvalds 已提交
6275
{
6276
	struct r5conf *conf = mddev->private;
L
Linus Torvalds 已提交
6277

6278
	md_unregister_thread(&mddev->thread);
N
NeilBrown 已提交
6279 6280
	if (mddev->queue)
		mddev->queue->backing_dev_info.congested_fn = NULL;
6281
	free_conf(conf);
6282 6283
	mddev->private = NULL;
	mddev->to_remove = &raid5_attrs_group;
L
Linus Torvalds 已提交
6284 6285 6286
	return 0;
}

6287
static void status(struct seq_file *seq, struct mddev *mddev)
L
Linus Torvalds 已提交
6288
{
6289
	struct r5conf *conf = mddev->private;
L
Linus Torvalds 已提交
6290 6291
	int i;

6292 6293
	seq_printf(seq, " level %d, %dk chunk, algorithm %d", mddev->level,
		mddev->chunk_sectors / 2, mddev->layout);
6294
	seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded);
L
Linus Torvalds 已提交
6295 6296 6297
	for (i = 0; i < conf->raid_disks; i++)
		seq_printf (seq, "%s",
			       conf->disks[i].rdev &&
6298
			       test_bit(In_sync, &conf->disks[i].rdev->flags) ? "U" : "_");
L
Linus Torvalds 已提交
6299 6300 6301
	seq_printf (seq, "]");
}

6302
static void print_raid5_conf (struct r5conf *conf)
L
Linus Torvalds 已提交
6303 6304 6305 6306
{
	int i;
	struct disk_info *tmp;

6307
	printk(KERN_DEBUG "RAID conf printout:\n");
L
Linus Torvalds 已提交
6308 6309 6310 6311
	if (!conf) {
		printk("(conf==NULL)\n");
		return;
	}
6312 6313 6314
	printk(KERN_DEBUG " --- level:%d rd:%d wd:%d\n", conf->level,
	       conf->raid_disks,
	       conf->raid_disks - conf->mddev->degraded);
L
Linus Torvalds 已提交
6315 6316 6317 6318 6319

	for (i = 0; i < conf->raid_disks; i++) {
		char b[BDEVNAME_SIZE];
		tmp = conf->disks + i;
		if (tmp->rdev)
6320 6321 6322
			printk(KERN_DEBUG " disk %d, o:%d, dev:%s\n",
			       i, !test_bit(Faulty, &tmp->rdev->flags),
			       bdevname(tmp->rdev->bdev, b));
L
Linus Torvalds 已提交
6323 6324 6325
	}
}

6326
static int raid5_spare_active(struct mddev *mddev)
L
Linus Torvalds 已提交
6327 6328
{
	int i;
6329
	struct r5conf *conf = mddev->private;
L
Linus Torvalds 已提交
6330
	struct disk_info *tmp;
6331 6332
	int count = 0;
	unsigned long flags;
L
Linus Torvalds 已提交
6333 6334 6335

	for (i = 0; i < conf->raid_disks; i++) {
		tmp = conf->disks + i;
6336 6337 6338 6339 6340 6341 6342 6343 6344 6345 6346 6347 6348 6349 6350 6351 6352 6353 6354
		if (tmp->replacement
		    && tmp->replacement->recovery_offset == MaxSector
		    && !test_bit(Faulty, &tmp->replacement->flags)
		    && !test_and_set_bit(In_sync, &tmp->replacement->flags)) {
			/* Replacement has just become active. */
			if (!tmp->rdev
			    || !test_and_clear_bit(In_sync, &tmp->rdev->flags))
				count++;
			if (tmp->rdev) {
				/* Replaced device not technically faulty,
				 * but we need to be sure it gets removed
				 * and never re-added.
				 */
				set_bit(Faulty, &tmp->rdev->flags);
				sysfs_notify_dirent_safe(
					tmp->rdev->sysfs_state);
			}
			sysfs_notify_dirent_safe(tmp->replacement->sysfs_state);
		} else if (tmp->rdev
6355
		    && tmp->rdev->recovery_offset == MaxSector
6356
		    && !test_bit(Faulty, &tmp->rdev->flags)
6357
		    && !test_and_set_bit(In_sync, &tmp->rdev->flags)) {
6358
			count++;
6359
			sysfs_notify_dirent_safe(tmp->rdev->sysfs_state);
L
Linus Torvalds 已提交
6360 6361
		}
	}
6362
	spin_lock_irqsave(&conf->device_lock, flags);
6363
	mddev->degraded = calc_degraded(conf);
6364
	spin_unlock_irqrestore(&conf->device_lock, flags);
L
Linus Torvalds 已提交
6365
	print_raid5_conf(conf);
6366
	return count;
L
Linus Torvalds 已提交
6367 6368
}

6369
static int raid5_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
L
Linus Torvalds 已提交
6370
{
6371
	struct r5conf *conf = mddev->private;
L
Linus Torvalds 已提交
6372
	int err = 0;
6373
	int number = rdev->raid_disk;
6374
	struct md_rdev **rdevp;
L
Linus Torvalds 已提交
6375 6376 6377
	struct disk_info *p = conf->disks + number;

	print_raid5_conf(conf);
6378 6379 6380 6381 6382 6383 6384 6385 6386 6387 6388 6389 6390 6391 6392 6393 6394 6395 6396 6397 6398 6399
	if (rdev == p->rdev)
		rdevp = &p->rdev;
	else if (rdev == p->replacement)
		rdevp = &p->replacement;
	else
		return 0;

	if (number >= conf->raid_disks &&
	    conf->reshape_progress == MaxSector)
		clear_bit(In_sync, &rdev->flags);

	if (test_bit(In_sync, &rdev->flags) ||
	    atomic_read(&rdev->nr_pending)) {
		err = -EBUSY;
		goto abort;
	}
	/* Only remove non-faulty devices if recovery
	 * isn't possible.
	 */
	if (!test_bit(Faulty, &rdev->flags) &&
	    mddev->recovery_disabled != conf->recovery_disabled &&
	    !has_failed(conf) &&
6400
	    (!p->replacement || p->replacement == rdev) &&
6401 6402 6403 6404 6405 6406 6407 6408 6409 6410
	    number < conf->raid_disks) {
		err = -EBUSY;
		goto abort;
	}
	*rdevp = NULL;
	synchronize_rcu();
	if (atomic_read(&rdev->nr_pending)) {
		/* lost the race, try later */
		err = -EBUSY;
		*rdevp = rdev;
6411 6412 6413 6414 6415 6416 6417 6418 6419 6420 6421 6422 6423 6424
	} else if (p->replacement) {
		/* We must have just cleared 'rdev' */
		p->rdev = p->replacement;
		clear_bit(Replacement, &p->replacement->flags);
		smp_mb(); /* Make sure other CPUs may see both as identical
			   * but will never see neither - if they are careful
			   */
		p->replacement = NULL;
		clear_bit(WantReplacement, &rdev->flags);
	} else
		/* We might have just removed the Replacement as faulty-
		 * clear the bit just in case
		 */
		clear_bit(WantReplacement, &rdev->flags);
L
Linus Torvalds 已提交
6425 6426 6427 6428 6429 6430
abort:

	print_raid5_conf(conf);
	return err;
}

6431
static int raid5_add_disk(struct mddev *mddev, struct md_rdev *rdev)
L
Linus Torvalds 已提交
6432
{
6433
	struct r5conf *conf = mddev->private;
6434
	int err = -EEXIST;
L
Linus Torvalds 已提交
6435 6436
	int disk;
	struct disk_info *p;
6437 6438
	int first = 0;
	int last = conf->raid_disks - 1;
L
Linus Torvalds 已提交
6439

6440 6441 6442
	if (mddev->recovery_disabled == conf->recovery_disabled)
		return -EBUSY;

N
NeilBrown 已提交
6443
	if (rdev->saved_raid_disk < 0 && has_failed(conf))
L
Linus Torvalds 已提交
6444
		/* no point adding a device */
6445
		return -EINVAL;
L
Linus Torvalds 已提交
6446

6447 6448
	if (rdev->raid_disk >= 0)
		first = last = rdev->raid_disk;
L
Linus Torvalds 已提交
6449 6450

	/*
6451 6452
	 * find the disk ... but prefer rdev->saved_raid_disk
	 * if possible.
L
Linus Torvalds 已提交
6453
	 */
6454
	if (rdev->saved_raid_disk >= 0 &&
6455
	    rdev->saved_raid_disk >= first &&
6456
	    conf->disks[rdev->saved_raid_disk].rdev == NULL)
6457 6458 6459
		first = rdev->saved_raid_disk;

	for (disk = first; disk <= last; disk++) {
6460 6461
		p = conf->disks + disk;
		if (p->rdev == NULL) {
6462
			clear_bit(In_sync, &rdev->flags);
L
Linus Torvalds 已提交
6463
			rdev->raid_disk = disk;
6464
			err = 0;
6465 6466
			if (rdev->saved_raid_disk != disk)
				conf->fullsync = 1;
6467
			rcu_assign_pointer(p->rdev, rdev);
6468
			goto out;
L
Linus Torvalds 已提交
6469
		}
6470 6471 6472
	}
	for (disk = first; disk <= last; disk++) {
		p = conf->disks + disk;
6473 6474 6475 6476 6477 6478 6479 6480 6481 6482 6483
		if (test_bit(WantReplacement, &p->rdev->flags) &&
		    p->replacement == NULL) {
			clear_bit(In_sync, &rdev->flags);
			set_bit(Replacement, &rdev->flags);
			rdev->raid_disk = disk;
			err = 0;
			conf->fullsync = 1;
			rcu_assign_pointer(p->replacement, rdev);
			break;
		}
	}
6484
out:
L
Linus Torvalds 已提交
6485
	print_raid5_conf(conf);
6486
	return err;
L
Linus Torvalds 已提交
6487 6488
}

6489
static int raid5_resize(struct mddev *mddev, sector_t sectors)
L
Linus Torvalds 已提交
6490 6491 6492 6493 6494 6495 6496 6497
{
	/* no resync is happening, and there is enough space
	 * on all devices, so we can resize.
	 * We need to make sure resync covers any new space.
	 * If the array is shrinking we should possibly wait until
	 * any io in the removed space completes, but it hardly seems
	 * worth it.
	 */
6498
	sector_t newsize;
6499
	sectors &= ~((sector_t)mddev->chunk_sectors - 1);
6500 6501 6502
	newsize = raid5_size(mddev, sectors, mddev->raid_disks);
	if (mddev->external_size &&
	    mddev->array_sectors > newsize)
D
Dan Williams 已提交
6503
		return -EINVAL;
6504 6505 6506 6507 6508 6509
	if (mddev->bitmap) {
		int ret = bitmap_resize(mddev->bitmap, sectors, 0, 0);
		if (ret)
			return ret;
	}
	md_set_array_sectors(mddev, newsize);
6510
	set_capacity(mddev->gendisk, mddev->array_sectors);
6511
	revalidate_disk(mddev->gendisk);
6512 6513
	if (sectors > mddev->dev_sectors &&
	    mddev->recovery_cp > mddev->dev_sectors) {
A
Andre Noll 已提交
6514
		mddev->recovery_cp = mddev->dev_sectors;
L
Linus Torvalds 已提交
6515 6516
		set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
	}
A
Andre Noll 已提交
6517
	mddev->dev_sectors = sectors;
6518
	mddev->resync_max_sectors = sectors;
L
Linus Torvalds 已提交
6519 6520 6521
	return 0;
}

6522
static int check_stripe_cache(struct mddev *mddev)
6523 6524 6525 6526 6527 6528 6529 6530 6531
{
	/* Can only proceed if there are plenty of stripe_heads.
	 * We need a minimum of one full stripe,, and for sensible progress
	 * it is best to have about 4 times that.
	 * If we require 4 times, then the default 256 4K stripe_heads will
	 * allow for chunk sizes up to 256K, which is probably OK.
	 * If the chunk size is greater, user-space should request more
	 * stripe_heads first.
	 */
6532
	struct r5conf *conf = mddev->private;
6533 6534 6535 6536
	if (((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4
	    > conf->max_nr_stripes ||
	    ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4
	    > conf->max_nr_stripes) {
6537 6538
		printk(KERN_WARNING "md/raid:%s: reshape: not enough stripes.  Needed %lu\n",
		       mdname(mddev),
6539 6540 6541 6542 6543 6544 6545
		       ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9)
			/ STRIPE_SIZE)*4);
		return 0;
	}
	return 1;
}

6546
static int check_reshape(struct mddev *mddev)
6547
{
6548
	struct r5conf *conf = mddev->private;
6549

6550 6551
	if (mddev->delta_disks == 0 &&
	    mddev->new_layout == mddev->layout &&
6552
	    mddev->new_chunk_sectors == mddev->chunk_sectors)
6553
		return 0; /* nothing to do */
6554
	if (has_failed(conf))
6555
		return -EINVAL;
6556
	if (mddev->delta_disks < 0 && mddev->reshape_position == MaxSector) {
6557 6558 6559 6560 6561 6562 6563 6564 6565 6566 6567
		/* We might be able to shrink, but the devices must
		 * be made bigger first.
		 * For raid6, 4 is the minimum size.
		 * Otherwise 2 is the minimum
		 */
		int min = 2;
		if (mddev->level == 6)
			min = 4;
		if (mddev->raid_disks + mddev->delta_disks < min)
			return -EINVAL;
	}
6568

6569
	if (!check_stripe_cache(mddev))
6570 6571
		return -ENOSPC;

6572 6573
	return resize_stripes(conf, (conf->previous_raid_disks
				     + mddev->delta_disks));
6574 6575
}

6576
static int raid5_start_reshape(struct mddev *mddev)
6577
{
6578
	struct r5conf *conf = mddev->private;
6579
	struct md_rdev *rdev;
6580
	int spares = 0;
6581
	unsigned long flags;
6582

6583
	if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
6584 6585
		return -EBUSY;

6586 6587 6588
	if (!check_stripe_cache(mddev))
		return -ENOSPC;

6589 6590 6591
	if (has_failed(conf))
		return -EINVAL;

6592
	rdev_for_each(rdev, mddev) {
6593 6594
		if (!test_bit(In_sync, &rdev->flags)
		    && !test_bit(Faulty, &rdev->flags))
6595
			spares++;
6596
	}
6597

6598
	if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded)
6599 6600 6601 6602 6603
		/* Not enough devices even to make a degraded array
		 * of that size
		 */
		return -EINVAL;

6604 6605 6606 6607 6608 6609
	/* Refuse to reduce size of the array.  Any reductions in
	 * array size must be through explicit setting of array_size
	 * attribute.
	 */
	if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks)
	    < mddev->array_sectors) {
6610
		printk(KERN_ERR "md/raid:%s: array size must be reduced "
6611 6612 6613 6614
		       "before number of disks\n", mdname(mddev));
		return -EINVAL;
	}

6615
	atomic_set(&conf->reshape_stripes, 0);
6616
	spin_lock_irq(&conf->device_lock);
6617
	write_seqcount_begin(&conf->gen_lock);
6618
	conf->previous_raid_disks = conf->raid_disks;
6619
	conf->raid_disks += mddev->delta_disks;
6620 6621
	conf->prev_chunk_sectors = conf->chunk_sectors;
	conf->chunk_sectors = mddev->new_chunk_sectors;
6622 6623
	conf->prev_algo = conf->algorithm;
	conf->algorithm = mddev->new_layout;
6624 6625 6626 6627 6628
	conf->generation++;
	/* Code that selects data_offset needs to see the generation update
	 * if reshape_progress has been set - so a memory barrier needed.
	 */
	smp_mb();
6629
	if (mddev->reshape_backwards)
6630 6631 6632 6633
		conf->reshape_progress = raid5_size(mddev, 0, 0);
	else
		conf->reshape_progress = 0;
	conf->reshape_safe = conf->reshape_progress;
6634
	write_seqcount_end(&conf->gen_lock);
6635 6636
	spin_unlock_irq(&conf->device_lock);

6637 6638 6639 6640 6641 6642 6643
	/* Now make sure any requests that proceeded on the assumption
	 * the reshape wasn't running - like Discard or Read - have
	 * completed.
	 */
	mddev_suspend(mddev);
	mddev_resume(mddev);

6644 6645
	/* Add some new drives, as many as will fit.
	 * We know there are enough to make the newly sized array work.
6646 6647 6648 6649
	 * Don't add devices if we are reducing the number of
	 * devices in the array.  This is because it is not possible
	 * to correctly record the "partially reconstructed" state of
	 * such devices during the reshape and confusion could result.
6650
	 */
6651
	if (mddev->delta_disks >= 0) {
N
NeilBrown 已提交
6652
		rdev_for_each(rdev, mddev)
6653 6654 6655 6656
			if (rdev->raid_disk < 0 &&
			    !test_bit(Faulty, &rdev->flags)) {
				if (raid5_add_disk(mddev, rdev) == 0) {
					if (rdev->raid_disk
6657
					    >= conf->previous_raid_disks)
6658
						set_bit(In_sync, &rdev->flags);
6659
					else
6660
						rdev->recovery_offset = 0;
6661 6662

					if (sysfs_link_rdev(mddev, rdev))
6663
						/* Failure here is OK */;
6664
				}
6665 6666 6667 6668 6669
			} else if (rdev->raid_disk >= conf->previous_raid_disks
				   && !test_bit(Faulty, &rdev->flags)) {
				/* This is a spare that was manually added */
				set_bit(In_sync, &rdev->flags);
			}
6670

6671 6672 6673 6674
		/* When a reshape changes the number of devices,
		 * ->degraded is measured against the larger of the
		 * pre and post number of devices.
		 */
6675
		spin_lock_irqsave(&conf->device_lock, flags);
6676
		mddev->degraded = calc_degraded(conf);
6677 6678
		spin_unlock_irqrestore(&conf->device_lock, flags);
	}
6679
	mddev->raid_disks = conf->raid_disks;
6680
	mddev->reshape_position = conf->reshape_progress;
6681
	set_bit(MD_CHANGE_DEVS, &mddev->flags);
6682

6683 6684 6685 6686 6687
	clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
	clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
	set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
	set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
	mddev->sync_thread = md_register_thread(md_do_sync, mddev,
6688
						"reshape");
6689 6690 6691
	if (!mddev->sync_thread) {
		mddev->recovery = 0;
		spin_lock_irq(&conf->device_lock);
6692
		write_seqcount_begin(&conf->gen_lock);
6693
		mddev->raid_disks = conf->raid_disks = conf->previous_raid_disks;
6694 6695 6696
		mddev->new_chunk_sectors =
			conf->chunk_sectors = conf->prev_chunk_sectors;
		mddev->new_layout = conf->algorithm = conf->prev_algo;
6697 6698 6699
		rdev_for_each(rdev, mddev)
			rdev->new_data_offset = rdev->data_offset;
		smp_wmb();
6700
		conf->generation --;
6701
		conf->reshape_progress = MaxSector;
6702
		mddev->reshape_position = MaxSector;
6703
		write_seqcount_end(&conf->gen_lock);
6704 6705 6706
		spin_unlock_irq(&conf->device_lock);
		return -EAGAIN;
	}
6707
	conf->reshape_checkpoint = jiffies;
6708 6709 6710 6711 6712
	md_wakeup_thread(mddev->sync_thread);
	md_new_event(mddev);
	return 0;
}

6713 6714 6715
/* This is called from the reshape thread and should make any
 * changes needed in 'conf'
 */
6716
static void end_reshape(struct r5conf *conf)
6717 6718
{

6719
	if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) {
6720
		struct md_rdev *rdev;
6721 6722

		spin_lock_irq(&conf->device_lock);
6723
		conf->previous_raid_disks = conf->raid_disks;
6724 6725 6726
		rdev_for_each(rdev, conf->mddev)
			rdev->data_offset = rdev->new_data_offset;
		smp_wmb();
6727
		conf->reshape_progress = MaxSector;
6728
		spin_unlock_irq(&conf->device_lock);
6729
		wake_up(&conf->wait_for_overlap);
6730 6731 6732 6733

		/* read-ahead size must cover two whole stripes, which is
		 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices
		 */
6734
		if (conf->mddev->queue) {
6735
			int data_disks = conf->raid_disks - conf->max_degraded;
6736
			int stripe = data_disks * ((conf->chunk_sectors << 9)
6737
						   / PAGE_SIZE);
6738 6739 6740
			if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
				conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
		}
6741 6742 6743
	}
}

6744 6745 6746
/* This is called from the raid5d thread with mddev_lock held.
 * It makes config changes to the device.
 */
6747
static void raid5_finish_reshape(struct mddev *mddev)
6748
{
6749
	struct r5conf *conf = mddev->private;
6750 6751 6752

	if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {

6753 6754 6755
		if (mddev->delta_disks > 0) {
			md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
			set_capacity(mddev->gendisk, mddev->array_sectors);
6756
			revalidate_disk(mddev->gendisk);
6757 6758
		} else {
			int d;
6759 6760 6761
			spin_lock_irq(&conf->device_lock);
			mddev->degraded = calc_degraded(conf);
			spin_unlock_irq(&conf->device_lock);
6762 6763
			for (d = conf->raid_disks ;
			     d < conf->raid_disks - mddev->delta_disks;
6764
			     d++) {
6765
				struct md_rdev *rdev = conf->disks[d].rdev;
6766 6767 6768 6769 6770
				if (rdev)
					clear_bit(In_sync, &rdev->flags);
				rdev = conf->disks[d].replacement;
				if (rdev)
					clear_bit(In_sync, &rdev->flags);
6771
			}
6772
		}
6773
		mddev->layout = conf->algorithm;
6774
		mddev->chunk_sectors = conf->chunk_sectors;
6775 6776
		mddev->reshape_position = MaxSector;
		mddev->delta_disks = 0;
6777
		mddev->reshape_backwards = 0;
6778 6779 6780
	}
}

6781
static void raid5_quiesce(struct mddev *mddev, int state)
6782
{
6783
	struct r5conf *conf = mddev->private;
6784 6785

	switch(state) {
6786 6787 6788 6789
	case 2: /* resume for a suspend */
		wake_up(&conf->wait_for_overlap);
		break;

6790
	case 1: /* stop all writes */
6791
		lock_all_device_hash_locks_irq(conf);
6792 6793 6794 6795
		/* '2' tells resync/reshape to pause so that all
		 * active stripes can drain
		 */
		conf->quiesce = 2;
6796
		wait_event_cmd(conf->wait_for_stripe,
6797 6798
				    atomic_read(&conf->active_stripes) == 0 &&
				    atomic_read(&conf->active_aligned_reads) == 0,
6799 6800
				    unlock_all_device_hash_locks_irq(conf),
				    lock_all_device_hash_locks_irq(conf));
6801
		conf->quiesce = 1;
6802
		unlock_all_device_hash_locks_irq(conf);
6803 6804
		/* allow reshape to continue */
		wake_up(&conf->wait_for_overlap);
6805 6806 6807
		break;

	case 0: /* re-enable writes */
6808
		lock_all_device_hash_locks_irq(conf);
6809 6810
		conf->quiesce = 0;
		wake_up(&conf->wait_for_stripe);
6811
		wake_up(&conf->wait_for_overlap);
6812
		unlock_all_device_hash_locks_irq(conf);
6813 6814 6815
		break;
	}
}
6816

6817

6818
static void *raid45_takeover_raid0(struct mddev *mddev, int level)
6819
{
6820
	struct r0conf *raid0_conf = mddev->private;
6821
	sector_t sectors;
6822

D
Dan Williams 已提交
6823
	/* for raid0 takeover only one zone is supported */
6824
	if (raid0_conf->nr_strip_zones > 1) {
6825 6826
		printk(KERN_ERR "md/raid:%s: cannot takeover raid0 with more than one zone.\n",
		       mdname(mddev));
D
Dan Williams 已提交
6827 6828 6829
		return ERR_PTR(-EINVAL);
	}

6830 6831
	sectors = raid0_conf->strip_zone[0].zone_end;
	sector_div(sectors, raid0_conf->strip_zone[0].nb_dev);
6832
	mddev->dev_sectors = sectors;
D
Dan Williams 已提交
6833
	mddev->new_level = level;
6834 6835 6836 6837 6838 6839 6840 6841 6842 6843 6844
	mddev->new_layout = ALGORITHM_PARITY_N;
	mddev->new_chunk_sectors = mddev->chunk_sectors;
	mddev->raid_disks += 1;
	mddev->delta_disks = 1;
	/* make sure it will be not marked as dirty */
	mddev->recovery_cp = MaxSector;

	return setup_conf(mddev);
}


6845
static void *raid5_takeover_raid1(struct mddev *mddev)
6846 6847 6848 6849 6850 6851 6852 6853 6854 6855 6856 6857 6858 6859 6860 6861 6862 6863 6864 6865 6866
{
	int chunksect;

	if (mddev->raid_disks != 2 ||
	    mddev->degraded > 1)
		return ERR_PTR(-EINVAL);

	/* Should check if there are write-behind devices? */

	chunksect = 64*2; /* 64K by default */

	/* The array must be an exact multiple of chunksize */
	while (chunksect && (mddev->array_sectors & (chunksect-1)))
		chunksect >>= 1;

	if ((chunksect<<9) < STRIPE_SIZE)
		/* array size does not allow a suitable chunk size */
		return ERR_PTR(-EINVAL);

	mddev->new_level = 5;
	mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC;
6867
	mddev->new_chunk_sectors = chunksect;
6868 6869 6870 6871

	return setup_conf(mddev);
}

6872
static void *raid5_takeover_raid6(struct mddev *mddev)
6873 6874 6875 6876 6877 6878 6879 6880 6881 6882 6883 6884 6885 6886 6887 6888 6889 6890 6891 6892 6893 6894 6895 6896 6897 6898 6899 6900 6901 6902 6903 6904
{
	int new_layout;

	switch (mddev->layout) {
	case ALGORITHM_LEFT_ASYMMETRIC_6:
		new_layout = ALGORITHM_LEFT_ASYMMETRIC;
		break;
	case ALGORITHM_RIGHT_ASYMMETRIC_6:
		new_layout = ALGORITHM_RIGHT_ASYMMETRIC;
		break;
	case ALGORITHM_LEFT_SYMMETRIC_6:
		new_layout = ALGORITHM_LEFT_SYMMETRIC;
		break;
	case ALGORITHM_RIGHT_SYMMETRIC_6:
		new_layout = ALGORITHM_RIGHT_SYMMETRIC;
		break;
	case ALGORITHM_PARITY_0_6:
		new_layout = ALGORITHM_PARITY_0;
		break;
	case ALGORITHM_PARITY_N:
		new_layout = ALGORITHM_PARITY_N;
		break;
	default:
		return ERR_PTR(-EINVAL);
	}
	mddev->new_level = 5;
	mddev->new_layout = new_layout;
	mddev->delta_disks = -1;
	mddev->raid_disks -= 1;
	return setup_conf(mddev);
}

6905

6906
static int raid5_check_reshape(struct mddev *mddev)
6907
{
6908 6909 6910 6911
	/* For a 2-drive array, the layout and chunk size can be changed
	 * immediately as not restriping is needed.
	 * For larger arrays we record the new value - after validation
	 * to be used by a reshape pass.
6912
	 */
6913
	struct r5conf *conf = mddev->private;
6914
	int new_chunk = mddev->new_chunk_sectors;
6915

6916
	if (mddev->new_layout >= 0 && !algorithm_valid_raid5(mddev->new_layout))
6917 6918
		return -EINVAL;
	if (new_chunk > 0) {
6919
		if (!is_power_of_2(new_chunk))
6920
			return -EINVAL;
6921
		if (new_chunk < (PAGE_SIZE>>9))
6922
			return -EINVAL;
6923
		if (mddev->array_sectors & (new_chunk-1))
6924 6925 6926 6927 6928 6929
			/* not factor of array size */
			return -EINVAL;
	}

	/* They look valid */

6930
	if (mddev->raid_disks == 2) {
6931 6932 6933 6934
		/* can make the change immediately */
		if (mddev->new_layout >= 0) {
			conf->algorithm = mddev->new_layout;
			mddev->layout = mddev->new_layout;
6935 6936
		}
		if (new_chunk > 0) {
6937 6938
			conf->chunk_sectors = new_chunk ;
			mddev->chunk_sectors = new_chunk;
6939 6940 6941
		}
		set_bit(MD_CHANGE_DEVS, &mddev->flags);
		md_wakeup_thread(mddev->thread);
6942
	}
6943
	return check_reshape(mddev);
6944 6945
}

6946
static int raid6_check_reshape(struct mddev *mddev)
6947
{
6948
	int new_chunk = mddev->new_chunk_sectors;
6949

6950
	if (mddev->new_layout >= 0 && !algorithm_valid_raid6(mddev->new_layout))
6951
		return -EINVAL;
6952
	if (new_chunk > 0) {
6953
		if (!is_power_of_2(new_chunk))
6954
			return -EINVAL;
6955
		if (new_chunk < (PAGE_SIZE >> 9))
6956
			return -EINVAL;
6957
		if (mddev->array_sectors & (new_chunk-1))
6958 6959
			/* not factor of array size */
			return -EINVAL;
6960
	}
6961 6962

	/* They look valid */
6963
	return check_reshape(mddev);
6964 6965
}

6966
static void *raid5_takeover(struct mddev *mddev)
6967 6968
{
	/* raid5 can take over:
D
Dan Williams 已提交
6969
	 *  raid0 - if there is only one strip zone - make it a raid4 layout
6970 6971 6972 6973
	 *  raid1 - if there are two drives.  We need to know the chunk size
	 *  raid4 - trivial - just use a raid4 layout.
	 *  raid6 - Providing it is a *_6 layout
	 */
D
Dan Williams 已提交
6974 6975
	if (mddev->level == 0)
		return raid45_takeover_raid0(mddev, 5);
6976 6977
	if (mddev->level == 1)
		return raid5_takeover_raid1(mddev);
6978 6979 6980 6981 6982
	if (mddev->level == 4) {
		mddev->new_layout = ALGORITHM_PARITY_N;
		mddev->new_level = 5;
		return setup_conf(mddev);
	}
6983 6984
	if (mddev->level == 6)
		return raid5_takeover_raid6(mddev);
6985 6986 6987 6988

	return ERR_PTR(-EINVAL);
}

6989
static void *raid4_takeover(struct mddev *mddev)
6990
{
D
Dan Williams 已提交
6991 6992 6993
	/* raid4 can take over:
	 *  raid0 - if there is only one strip zone
	 *  raid5 - if layout is right
6994
	 */
D
Dan Williams 已提交
6995 6996
	if (mddev->level == 0)
		return raid45_takeover_raid0(mddev, 4);
6997 6998 6999 7000 7001 7002 7003 7004
	if (mddev->level == 5 &&
	    mddev->layout == ALGORITHM_PARITY_N) {
		mddev->new_layout = 0;
		mddev->new_level = 4;
		return setup_conf(mddev);
	}
	return ERR_PTR(-EINVAL);
}
7005

7006
static struct md_personality raid5_personality;
7007

7008
static void *raid6_takeover(struct mddev *mddev)
7009 7010 7011 7012 7013 7014 7015 7016 7017 7018 7019 7020 7021 7022 7023 7024 7025 7026 7027 7028 7029 7030 7031 7032 7033 7034 7035 7036 7037 7038 7039 7040 7041 7042 7043 7044 7045 7046 7047 7048 7049 7050 7051 7052 7053 7054
{
	/* Currently can only take over a raid5.  We map the
	 * personality to an equivalent raid6 personality
	 * with the Q block at the end.
	 */
	int new_layout;

	if (mddev->pers != &raid5_personality)
		return ERR_PTR(-EINVAL);
	if (mddev->degraded > 1)
		return ERR_PTR(-EINVAL);
	if (mddev->raid_disks > 253)
		return ERR_PTR(-EINVAL);
	if (mddev->raid_disks < 3)
		return ERR_PTR(-EINVAL);

	switch (mddev->layout) {
	case ALGORITHM_LEFT_ASYMMETRIC:
		new_layout = ALGORITHM_LEFT_ASYMMETRIC_6;
		break;
	case ALGORITHM_RIGHT_ASYMMETRIC:
		new_layout = ALGORITHM_RIGHT_ASYMMETRIC_6;
		break;
	case ALGORITHM_LEFT_SYMMETRIC:
		new_layout = ALGORITHM_LEFT_SYMMETRIC_6;
		break;
	case ALGORITHM_RIGHT_SYMMETRIC:
		new_layout = ALGORITHM_RIGHT_SYMMETRIC_6;
		break;
	case ALGORITHM_PARITY_0:
		new_layout = ALGORITHM_PARITY_0_6;
		break;
	case ALGORITHM_PARITY_N:
		new_layout = ALGORITHM_PARITY_N;
		break;
	default:
		return ERR_PTR(-EINVAL);
	}
	mddev->new_level = 6;
	mddev->new_layout = new_layout;
	mddev->delta_disks = 1;
	mddev->raid_disks += 1;
	return setup_conf(mddev);
}


7055
static struct md_personality raid6_personality =
7056 7057 7058 7059 7060 7061 7062 7063 7064 7065 7066 7067 7068 7069
{
	.name		= "raid6",
	.level		= 6,
	.owner		= THIS_MODULE,
	.make_request	= make_request,
	.run		= run,
	.stop		= stop,
	.status		= status,
	.error_handler	= error,
	.hot_add_disk	= raid5_add_disk,
	.hot_remove_disk= raid5_remove_disk,
	.spare_active	= raid5_spare_active,
	.sync_request	= sync_request,
	.resize		= raid5_resize,
7070
	.size		= raid5_size,
7071
	.check_reshape	= raid6_check_reshape,
7072
	.start_reshape  = raid5_start_reshape,
7073
	.finish_reshape = raid5_finish_reshape,
7074
	.quiesce	= raid5_quiesce,
7075
	.takeover	= raid6_takeover,
7076
};
7077
static struct md_personality raid5_personality =
L
Linus Torvalds 已提交
7078 7079
{
	.name		= "raid5",
7080
	.level		= 5,
L
Linus Torvalds 已提交
7081 7082 7083 7084 7085 7086 7087 7088 7089 7090 7091
	.owner		= THIS_MODULE,
	.make_request	= make_request,
	.run		= run,
	.stop		= stop,
	.status		= status,
	.error_handler	= error,
	.hot_add_disk	= raid5_add_disk,
	.hot_remove_disk= raid5_remove_disk,
	.spare_active	= raid5_spare_active,
	.sync_request	= sync_request,
	.resize		= raid5_resize,
7092
	.size		= raid5_size,
7093 7094
	.check_reshape	= raid5_check_reshape,
	.start_reshape  = raid5_start_reshape,
7095
	.finish_reshape = raid5_finish_reshape,
7096
	.quiesce	= raid5_quiesce,
7097
	.takeover	= raid5_takeover,
L
Linus Torvalds 已提交
7098 7099
};

7100
static struct md_personality raid4_personality =
L
Linus Torvalds 已提交
7101
{
7102 7103 7104 7105 7106 7107 7108 7109 7110 7111 7112 7113 7114
	.name		= "raid4",
	.level		= 4,
	.owner		= THIS_MODULE,
	.make_request	= make_request,
	.run		= run,
	.stop		= stop,
	.status		= status,
	.error_handler	= error,
	.hot_add_disk	= raid5_add_disk,
	.hot_remove_disk= raid5_remove_disk,
	.spare_active	= raid5_spare_active,
	.sync_request	= sync_request,
	.resize		= raid5_resize,
7115
	.size		= raid5_size,
7116 7117
	.check_reshape	= raid5_check_reshape,
	.start_reshape  = raid5_start_reshape,
7118
	.finish_reshape = raid5_finish_reshape,
7119
	.quiesce	= raid5_quiesce,
7120
	.takeover	= raid4_takeover,
7121 7122 7123 7124
};

static int __init raid5_init(void)
{
7125 7126 7127 7128
	raid5_wq = alloc_workqueue("raid5wq",
		WQ_UNBOUND|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE|WQ_SYSFS, 0);
	if (!raid5_wq)
		return -ENOMEM;
7129
	register_md_personality(&raid6_personality);
7130 7131 7132
	register_md_personality(&raid5_personality);
	register_md_personality(&raid4_personality);
	return 0;
L
Linus Torvalds 已提交
7133 7134
}

7135
static void raid5_exit(void)
L
Linus Torvalds 已提交
7136
{
7137
	unregister_md_personality(&raid6_personality);
7138 7139
	unregister_md_personality(&raid5_personality);
	unregister_md_personality(&raid4_personality);
7140
	destroy_workqueue(raid5_wq);
L
Linus Torvalds 已提交
7141 7142 7143 7144 7145
}

module_init(raid5_init);
module_exit(raid5_exit);
MODULE_LICENSE("GPL");
7146
MODULE_DESCRIPTION("RAID4/5/6 (striping with parity) personality for MD");
L
Linus Torvalds 已提交
7147
MODULE_ALIAS("md-personality-4"); /* RAID5 */
7148 7149
MODULE_ALIAS("md-raid5");
MODULE_ALIAS("md-raid4");
7150 7151
MODULE_ALIAS("md-level-5");
MODULE_ALIAS("md-level-4");
7152 7153 7154 7155 7156 7157 7158
MODULE_ALIAS("md-personality-8"); /* RAID6 */
MODULE_ALIAS("md-raid6");
MODULE_ALIAS("md-level-6");

/* This used to be two separate modules, they were: */
MODULE_ALIAS("raid5");
MODULE_ALIAS("raid6");