raid5.c 215.8 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4
/*
 * raid5.c : Multiple Devices driver for Linux
 *	   Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman
 *	   Copyright (C) 1999, 2000 Ingo Molnar
5
 *	   Copyright (C) 2002, 2003 H. Peter Anvin
L
Linus Torvalds 已提交
6
 *
7 8 9
 * RAID-4/5/6 management functions.
 * Thanks to Penguin Computing for making the RAID-6 development possible
 * by donating a test server!
L
Linus Torvalds 已提交
10 11 12 13 14 15 16 17 18 19 20
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2, or (at your option)
 * any later version.
 *
 * You should have received a copy of the GNU General Public License
 * (for example /usr/src/linux/COPYING); if not, write to the Free
 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

21 22 23 24 25 26 27 28 29
/*
 * BITMAP UNPLUGGING:
 *
 * The sequencing for updating the bitmap reliably is a little
 * subtle (and I got it wrong the first time) so it deserves some
 * explanation.
 *
 * We group bitmap updates into batches.  Each batch has a number.
 * We may write out several batches at once, but that isn't very important.
30 31
 * conf->seq_write is the number of the last batch successfully written.
 * conf->seq_flush is the number of the last batch that was closed to
32 33 34
 *    new additions.
 * When we discover that we will need to write to any block in a stripe
 * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq
35
 * the number of the batch it will be in. This is seq_flush+1.
36 37 38 39 40 41 42 43 44
 * When we are ready to do a write, if that batch hasn't been written yet,
 *   we plug the array and queue the stripe for later.
 * When an unplug happens, we increment bm_flush, thus closing the current
 *   batch.
 * When we notice that bm_flush > bm_write, we write out all pending updates
 * to the bitmap, and advance bm_write to where bm_flush was.
 * This may occasionally write a bit out twice, but is sure never to
 * miss any bits.
 */
L
Linus Torvalds 已提交
45

46
#include <linux/blkdev.h>
47
#include <linux/kthread.h>
48
#include <linux/raid/pq.h>
49
#include <linux/async_tx.h>
50
#include <linux/module.h>
51
#include <linux/async.h>
52
#include <linux/seq_file.h>
53
#include <linux/cpu.h>
54
#include <linux/slab.h>
55
#include <linux/ratelimit.h>
56
#include <linux/nodemask.h>
57
#include <linux/flex_array.h>
N
NeilBrown 已提交
58 59
#include <trace/events/block.h>

60
#include "md.h"
61
#include "raid5.h"
62
#include "raid0.h"
63
#include "bitmap.h"
64

65 66 67
#define cpu_to_group(cpu) cpu_to_node(cpu)
#define ANY_GROUP NUMA_NO_NODE

68 69 70 71
static bool devices_handle_discard_safely = false;
module_param(devices_handle_discard_safely, bool, 0644);
MODULE_PARM_DESC(devices_handle_discard_safely,
		 "Set to Y if all devices in each array reliably return zeroes on reads from discarded regions");
72
static struct workqueue_struct *raid5_wq;
L
Linus Torvalds 已提交
73 74 75 76 77 78 79 80 81
/*
 * Stripe cache
 */

#define NR_STRIPES		256
#define STRIPE_SIZE		PAGE_SIZE
#define STRIPE_SHIFT		(PAGE_SHIFT - 9)
#define STRIPE_SECTORS		(STRIPE_SIZE>>9)
#define	IO_THRESHOLD		1
82
#define BYPASS_THRESHOLD	1
83
#define NR_HASH			(PAGE_SIZE / sizeof(struct hlist_head))
L
Linus Torvalds 已提交
84
#define HASH_MASK		(NR_HASH - 1)
85
#define MAX_STRIPE_BATCH	8
L
Linus Torvalds 已提交
86

87
static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect)
88 89 90 91
{
	int hash = (sect >> STRIPE_SHIFT) & HASH_MASK;
	return &conf->stripe_hashtbl[hash];
}
L
Linus Torvalds 已提交
92

93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128
static inline int stripe_hash_locks_hash(sector_t sect)
{
	return (sect >> STRIPE_SHIFT) & STRIPE_HASH_LOCKS_MASK;
}

static inline void lock_device_hash_lock(struct r5conf *conf, int hash)
{
	spin_lock_irq(conf->hash_locks + hash);
	spin_lock(&conf->device_lock);
}

static inline void unlock_device_hash_lock(struct r5conf *conf, int hash)
{
	spin_unlock(&conf->device_lock);
	spin_unlock_irq(conf->hash_locks + hash);
}

static inline void lock_all_device_hash_locks_irq(struct r5conf *conf)
{
	int i;
	local_irq_disable();
	spin_lock(conf->hash_locks);
	for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++)
		spin_lock_nest_lock(conf->hash_locks + i, conf->hash_locks);
	spin_lock(&conf->device_lock);
}

static inline void unlock_all_device_hash_locks_irq(struct r5conf *conf)
{
	int i;
	spin_unlock(&conf->device_lock);
	for (i = NR_STRIPE_HASH_LOCKS; i; i--)
		spin_unlock(conf->hash_locks + i - 1);
	local_irq_enable();
}

L
Linus Torvalds 已提交
129 130 131 132 133 134
/* bio's attached to a stripe+device for I/O are linked together in bi_sector
 * order without overlap.  There may be several bio's per stripe+device, and
 * a bio could span several devices.
 * When walking this list for a particular stripe+device, we must never proceed
 * beyond a bio that extends past this device, as the next bio might no longer
 * be valid.
135
 * This function is used to determine the 'next' bio in the list, given the sector
L
Linus Torvalds 已提交
136 137
 * of the current stripe+device
 */
138 139
static inline struct bio *r5_next_bio(struct bio *bio, sector_t sector)
{
140
	int sectors = bio_sectors(bio);
141
	if (bio->bi_iter.bi_sector + sectors < sector + STRIPE_SECTORS)
142 143 144 145
		return bio->bi_next;
	else
		return NULL;
}
L
Linus Torvalds 已提交
146

147
/*
148 149
 * We maintain a biased count of active stripes in the bottom 16 bits of
 * bi_phys_segments, and a count of processed stripes in the upper 16 bits
150
 */
151
static inline int raid5_bi_processed_stripes(struct bio *bio)
152
{
153 154
	atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
	return (atomic_read(segments) >> 16) & 0xffff;
155 156
}

157
static inline int raid5_dec_bi_active_stripes(struct bio *bio)
158
{
159 160
	atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
	return atomic_sub_return(1, segments) & 0xffff;
161 162
}

163
static inline void raid5_inc_bi_active_stripes(struct bio *bio)
164
{
165 166
	atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
	atomic_inc(segments);
167 168
}

169 170
static inline void raid5_set_bi_processed_stripes(struct bio *bio,
	unsigned int cnt)
171
{
172 173
	atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
	int old, new;
174

175 176 177 178
	do {
		old = atomic_read(segments);
		new = (old & 0xffff) | (cnt << 16);
	} while (atomic_cmpxchg(segments, old, new) != old);
179 180
}

181
static inline void raid5_set_bi_stripes(struct bio *bio, unsigned int cnt)
182
{
183 184
	atomic_t *segments = (atomic_t *)&bio->bi_phys_segments;
	atomic_set(segments, cnt);
185 186
}

187 188 189
/* Find first data disk in a raid6 stripe */
static inline int raid6_d0(struct stripe_head *sh)
{
190 191 192 193
	if (sh->ddf_layout)
		/* ddf always start from first device */
		return 0;
	/* md starts just after Q block */
194 195 196 197 198
	if (sh->qd_idx == sh->disks - 1)
		return 0;
	else
		return sh->qd_idx + 1;
}
199 200 201 202 203
static inline int raid6_next_disk(int disk, int raid_disks)
{
	disk++;
	return (disk < raid_disks) ? disk : 0;
}
204

205 206 207 208 209
/* When walking through the disks in a raid5, starting at raid6_d0,
 * We need to map each disk to a 'slot', where the data disks are slot
 * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk
 * is raid_disks-1.  This help does that mapping.
 */
210 211
static int raid6_idx_to_slot(int idx, struct stripe_head *sh,
			     int *count, int syndrome_disks)
212
{
213
	int slot = *count;
214

215
	if (sh->ddf_layout)
216
		(*count)++;
217
	if (idx == sh->pd_idx)
218
		return syndrome_disks;
219
	if (idx == sh->qd_idx)
220
		return syndrome_disks + 1;
221
	if (!sh->ddf_layout)
222
		(*count)++;
223 224 225
	return slot;
}

226 227 228 229 230 231 232
static void return_io(struct bio *return_bi)
{
	struct bio *bi = return_bi;
	while (bi) {

		return_bi = bi->bi_next;
		bi->bi_next = NULL;
233
		bi->bi_iter.bi_size = 0;
234 235
		trace_block_bio_complete(bdev_get_queue(bi->bi_bdev),
					 bi, 0);
236
		bio_endio(bi, 0);
237 238 239 240
		bi = return_bi;
	}
}

241
static void print_raid5_conf (struct r5conf *conf);
L
Linus Torvalds 已提交
242

243 244 245 246 247 248 249
static int stripe_operations_active(struct stripe_head *sh)
{
	return sh->check_state || sh->reconstruct_state ||
	       test_bit(STRIPE_BIOFILL_RUN, &sh->state) ||
	       test_bit(STRIPE_COMPUTE_RUN, &sh->state);
}

250 251 252 253
static void raid5_wakeup_stripe_thread(struct stripe_head *sh)
{
	struct r5conf *conf = sh->raid_conf;
	struct r5worker_group *group;
254
	int thread_cnt;
255 256 257 258 259 260 261 262 263 264 265
	int i, cpu = sh->cpu;

	if (!cpu_online(cpu)) {
		cpu = cpumask_any(cpu_online_mask);
		sh->cpu = cpu;
	}

	if (list_empty(&sh->lru)) {
		struct r5worker_group *group;
		group = conf->worker_groups + cpu_to_group(cpu);
		list_add_tail(&sh->lru, &group->handle_list);
266 267
		group->stripes_cnt++;
		sh->group = group;
268 269 270 271 272 273 274 275 276
	}

	if (conf->worker_cnt_per_group == 0) {
		md_wakeup_thread(conf->mddev->thread);
		return;
	}

	group = conf->worker_groups + cpu_to_group(sh->cpu);

277 278 279 280 281 282 283 284 285 286 287 288 289 290
	group->workers[0].working = true;
	/* at least one worker should run to avoid race */
	queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work);

	thread_cnt = group->stripes_cnt / MAX_STRIPE_BATCH - 1;
	/* wakeup more workers */
	for (i = 1; i < conf->worker_cnt_per_group && thread_cnt > 0; i++) {
		if (group->workers[i].working == false) {
			group->workers[i].working = true;
			queue_work_on(sh->cpu, raid5_wq,
				      &group->workers[i].work);
			thread_cnt--;
		}
	}
291 292
}

293 294
static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh,
			      struct list_head *temp_inactive_list)
L
Linus Torvalds 已提交
295
{
296 297 298 299
	BUG_ON(!list_empty(&sh->lru));
	BUG_ON(atomic_read(&conf->active_stripes)==0);
	if (test_bit(STRIPE_HANDLE, &sh->state)) {
		if (test_bit(STRIPE_DELAYED, &sh->state) &&
300
		    !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
301
			list_add_tail(&sh->lru, &conf->delayed_list);
302
		else if (test_bit(STRIPE_BIT_DELAY, &sh->state) &&
303 304 305 306 307
			   sh->bm_seq - conf->seq_write > 0)
			list_add_tail(&sh->lru, &conf->bitmap_list);
		else {
			clear_bit(STRIPE_DELAYED, &sh->state);
			clear_bit(STRIPE_BIT_DELAY, &sh->state);
308 309 310 311 312 313
			if (conf->worker_cnt_per_group == 0) {
				list_add_tail(&sh->lru, &conf->handle_list);
			} else {
				raid5_wakeup_stripe_thread(sh);
				return;
			}
314 315 316 317 318 319 320 321 322
		}
		md_wakeup_thread(conf->mddev->thread);
	} else {
		BUG_ON(stripe_operations_active(sh));
		if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
			if (atomic_dec_return(&conf->preread_active_stripes)
			    < IO_THRESHOLD)
				md_wakeup_thread(conf->mddev->thread);
		atomic_dec(&conf->active_stripes);
323 324
		if (!test_bit(STRIPE_EXPANDING, &sh->state))
			list_add_tail(&sh->lru, temp_inactive_list);
L
Linus Torvalds 已提交
325 326
	}
}
327

328 329
static void __release_stripe(struct r5conf *conf, struct stripe_head *sh,
			     struct list_head *temp_inactive_list)
330 331
{
	if (atomic_dec_and_test(&sh->count))
332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363
		do_release_stripe(conf, sh, temp_inactive_list);
}

/*
 * @hash could be NR_STRIPE_HASH_LOCKS, then we have a list of inactive_list
 *
 * Be careful: Only one task can add/delete stripes from temp_inactive_list at
 * given time. Adding stripes only takes device lock, while deleting stripes
 * only takes hash lock.
 */
static void release_inactive_stripe_list(struct r5conf *conf,
					 struct list_head *temp_inactive_list,
					 int hash)
{
	int size;
	bool do_wakeup = false;
	unsigned long flags;

	if (hash == NR_STRIPE_HASH_LOCKS) {
		size = NR_STRIPE_HASH_LOCKS;
		hash = NR_STRIPE_HASH_LOCKS - 1;
	} else
		size = 1;
	while (size) {
		struct list_head *list = &temp_inactive_list[size - 1];

		/*
		 * We don't hold any lock here yet, get_active_stripe() might
		 * remove stripes from the list
		 */
		if (!list_empty_careful(list)) {
			spin_lock_irqsave(conf->hash_locks + hash, flags);
364 365 366
			if (list_empty(conf->inactive_list + hash) &&
			    !list_empty(list))
				atomic_dec(&conf->empty_inactive_list_nr);
367 368 369 370 371 372 373 374 375 376 377 378 379
			list_splice_tail_init(list, conf->inactive_list + hash);
			do_wakeup = true;
			spin_unlock_irqrestore(conf->hash_locks + hash, flags);
		}
		size--;
		hash--;
	}

	if (do_wakeup) {
		wake_up(&conf->wait_for_stripe);
		if (conf->retry_read_aligned)
			md_wakeup_thread(conf->mddev->thread);
	}
380 381
}

S
Shaohua Li 已提交
382
/* should hold conf->device_lock already */
383 384
static int release_stripe_list(struct r5conf *conf,
			       struct list_head *temp_inactive_list)
S
Shaohua Li 已提交
385 386 387 388 389 390
{
	struct stripe_head *sh;
	int count = 0;
	struct llist_node *head;

	head = llist_del_all(&conf->released_stripes);
S
Shaohua Li 已提交
391
	head = llist_reverse_order(head);
S
Shaohua Li 已提交
392
	while (head) {
393 394
		int hash;

S
Shaohua Li 已提交
395 396 397 398 399 400 401 402 403 404
		sh = llist_entry(head, struct stripe_head, release_list);
		head = llist_next(head);
		/* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */
		smp_mb();
		clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state);
		/*
		 * Don't worry the bit is set here, because if the bit is set
		 * again, the count is always > 1. This is true for
		 * STRIPE_ON_UNPLUG_LIST bit too.
		 */
405 406
		hash = sh->hash_lock_index;
		__release_stripe(conf, sh, &temp_inactive_list[hash]);
S
Shaohua Li 已提交
407 408 409 410 411 412
		count++;
	}

	return count;
}

L
Linus Torvalds 已提交
413 414
static void release_stripe(struct stripe_head *sh)
{
415
	struct r5conf *conf = sh->raid_conf;
L
Linus Torvalds 已提交
416
	unsigned long flags;
417 418
	struct list_head list;
	int hash;
S
Shaohua Li 已提交
419
	bool wakeup;
420

421 422 423 424 425
	/* Avoid release_list until the last reference.
	 */
	if (atomic_add_unless(&sh->count, -1, 1))
		return;

426 427
	if (unlikely(!conf->mddev->thread) ||
		test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state))
S
Shaohua Li 已提交
428 429 430 431 432 433
		goto slow_path;
	wakeup = llist_add(&sh->release_list, &conf->released_stripes);
	if (wakeup)
		md_wakeup_thread(conf->mddev->thread);
	return;
slow_path:
434
	local_irq_save(flags);
S
Shaohua Li 已提交
435
	/* we are ok here if STRIPE_ON_RELEASE_LIST is set or not */
436
	if (atomic_dec_and_lock(&sh->count, &conf->device_lock)) {
437 438 439
		INIT_LIST_HEAD(&list);
		hash = sh->hash_lock_index;
		do_release_stripe(conf, sh, &list);
440
		spin_unlock(&conf->device_lock);
441
		release_inactive_stripe_list(conf, &list, hash);
442 443
	}
	local_irq_restore(flags);
L
Linus Torvalds 已提交
444 445
}

446
static inline void remove_hash(struct stripe_head *sh)
L
Linus Torvalds 已提交
447
{
448 449
	pr_debug("remove_hash(), stripe %llu\n",
		(unsigned long long)sh->sector);
L
Linus Torvalds 已提交
450

451
	hlist_del_init(&sh->hash);
L
Linus Torvalds 已提交
452 453
}

454
static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh)
L
Linus Torvalds 已提交
455
{
456
	struct hlist_head *hp = stripe_hash(conf, sh->sector);
L
Linus Torvalds 已提交
457

458 459
	pr_debug("insert_hash(), stripe %llu\n",
		(unsigned long long)sh->sector);
L
Linus Torvalds 已提交
460

461
	hlist_add_head(&sh->hash, hp);
L
Linus Torvalds 已提交
462 463 464
}

/* find an idle stripe, make sure it is unhashed, and return it. */
465
static struct stripe_head *get_free_stripe(struct r5conf *conf, int hash)
L
Linus Torvalds 已提交
466 467 468 469
{
	struct stripe_head *sh = NULL;
	struct list_head *first;

470
	if (list_empty(conf->inactive_list + hash))
L
Linus Torvalds 已提交
471
		goto out;
472
	first = (conf->inactive_list + hash)->next;
L
Linus Torvalds 已提交
473 474 475 476
	sh = list_entry(first, struct stripe_head, lru);
	list_del_init(first);
	remove_hash(sh);
	atomic_inc(&conf->active_stripes);
477
	BUG_ON(hash != sh->hash_lock_index);
478 479
	if (list_empty(conf->inactive_list + hash))
		atomic_inc(&conf->empty_inactive_list_nr);
L
Linus Torvalds 已提交
480 481 482 483
out:
	return sh;
}

484
static void shrink_buffers(struct stripe_head *sh)
L
Linus Torvalds 已提交
485 486 487
{
	struct page *p;
	int i;
488
	int num = sh->raid_conf->pool_size;
L
Linus Torvalds 已提交
489

490
	for (i = 0; i < num ; i++) {
491
		WARN_ON(sh->dev[i].page != sh->dev[i].orig_page);
L
Linus Torvalds 已提交
492 493 494 495
		p = sh->dev[i].page;
		if (!p)
			continue;
		sh->dev[i].page = NULL;
496
		put_page(p);
L
Linus Torvalds 已提交
497 498 499
	}
}

500
static int grow_buffers(struct stripe_head *sh, gfp_t gfp)
L
Linus Torvalds 已提交
501 502
{
	int i;
503
	int num = sh->raid_conf->pool_size;
L
Linus Torvalds 已提交
504

505
	for (i = 0; i < num; i++) {
L
Linus Torvalds 已提交
506 507
		struct page *page;

508
		if (!(page = alloc_page(gfp))) {
L
Linus Torvalds 已提交
509 510 511
			return 1;
		}
		sh->dev[i].page = page;
512
		sh->dev[i].orig_page = page;
L
Linus Torvalds 已提交
513 514 515 516
	}
	return 0;
}

517
static void raid5_build_block(struct stripe_head *sh, int i, int previous);
518
static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,
519
			    struct stripe_head *sh);
L
Linus Torvalds 已提交
520

521
static void init_stripe(struct stripe_head *sh, sector_t sector, int previous)
L
Linus Torvalds 已提交
522
{
523
	struct r5conf *conf = sh->raid_conf;
524
	int i, seq;
L
Linus Torvalds 已提交
525

526 527
	BUG_ON(atomic_read(&sh->count) != 0);
	BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));
528
	BUG_ON(stripe_operations_active(sh));
529
	BUG_ON(sh->batch_head);
530

531
	pr_debug("init_stripe called, stripe %llu\n",
532
		(unsigned long long)sector);
533 534
retry:
	seq = read_seqcount_begin(&conf->gen_lock);
535
	sh->generation = conf->generation - previous;
536
	sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks;
L
Linus Torvalds 已提交
537
	sh->sector = sector;
538
	stripe_set_idx(sector, conf, previous, sh);
L
Linus Torvalds 已提交
539 540
	sh->state = 0;

541
	for (i = sh->disks; i--; ) {
L
Linus Torvalds 已提交
542 543
		struct r5dev *dev = &sh->dev[i];

544
		if (dev->toread || dev->read || dev->towrite || dev->written ||
L
Linus Torvalds 已提交
545
		    test_bit(R5_LOCKED, &dev->flags)) {
546
			printk(KERN_ERR "sector=%llx i=%d %p %p %p %p %d\n",
L
Linus Torvalds 已提交
547
			       (unsigned long long)sh->sector, i, dev->toread,
548
			       dev->read, dev->towrite, dev->written,
L
Linus Torvalds 已提交
549
			       test_bit(R5_LOCKED, &dev->flags));
550
			WARN_ON(1);
L
Linus Torvalds 已提交
551 552
		}
		dev->flags = 0;
553
		raid5_build_block(sh, i, previous);
L
Linus Torvalds 已提交
554
	}
555 556
	if (read_seqcount_retry(&conf->gen_lock, seq))
		goto retry;
557
	sh->overwrite_disks = 0;
L
Linus Torvalds 已提交
558
	insert_hash(conf, sh);
559
	sh->cpu = smp_processor_id();
560
	set_bit(STRIPE_BATCH_READY, &sh->state);
L
Linus Torvalds 已提交
561 562
}

563
static struct stripe_head *__find_stripe(struct r5conf *conf, sector_t sector,
564
					 short generation)
L
Linus Torvalds 已提交
565 566 567
{
	struct stripe_head *sh;

568
	pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector);
569
	hlist_for_each_entry(sh, stripe_hash(conf, sector), hash)
570
		if (sh->sector == sector && sh->generation == generation)
L
Linus Torvalds 已提交
571
			return sh;
572
	pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector);
L
Linus Torvalds 已提交
573 574 575
	return NULL;
}

576 577 578 579 580 581 582 583 584 585 586 587 588
/*
 * Need to check if array has failed when deciding whether to:
 *  - start an array
 *  - remove non-faulty devices
 *  - add a spare
 *  - allow a reshape
 * This determination is simple when no reshape is happening.
 * However if there is a reshape, we need to carefully check
 * both the before and after sections.
 * This is because some failed devices may only affect one
 * of the two sections, and some non-in_sync devices may
 * be insync in the section most affected by failed devices.
 */
589
static int calc_degraded(struct r5conf *conf)
590
{
591
	int degraded, degraded2;
592 593 594 595 596
	int i;

	rcu_read_lock();
	degraded = 0;
	for (i = 0; i < conf->previous_raid_disks; i++) {
597
		struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
598 599
		if (rdev && test_bit(Faulty, &rdev->flags))
			rdev = rcu_dereference(conf->disks[i].replacement);
600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617
		if (!rdev || test_bit(Faulty, &rdev->flags))
			degraded++;
		else if (test_bit(In_sync, &rdev->flags))
			;
		else
			/* not in-sync or faulty.
			 * If the reshape increases the number of devices,
			 * this is being recovered by the reshape, so
			 * this 'previous' section is not in_sync.
			 * If the number of devices is being reduced however,
			 * the device can only be part of the array if
			 * we are reverting a reshape, so this section will
			 * be in-sync.
			 */
			if (conf->raid_disks >= conf->previous_raid_disks)
				degraded++;
	}
	rcu_read_unlock();
618 619
	if (conf->raid_disks == conf->previous_raid_disks)
		return degraded;
620
	rcu_read_lock();
621
	degraded2 = 0;
622
	for (i = 0; i < conf->raid_disks; i++) {
623
		struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
624 625
		if (rdev && test_bit(Faulty, &rdev->flags))
			rdev = rcu_dereference(conf->disks[i].replacement);
626
		if (!rdev || test_bit(Faulty, &rdev->flags))
627
			degraded2++;
628 629 630 631 632 633 634 635 636
		else if (test_bit(In_sync, &rdev->flags))
			;
		else
			/* not in-sync or faulty.
			 * If reshape increases the number of devices, this
			 * section has already been recovered, else it
			 * almost certainly hasn't.
			 */
			if (conf->raid_disks <= conf->previous_raid_disks)
637
				degraded2++;
638 639
	}
	rcu_read_unlock();
640 641 642 643 644 645 646 647 648 649 650 651 652
	if (degraded2 > degraded)
		return degraded2;
	return degraded;
}

static int has_failed(struct r5conf *conf)
{
	int degraded;

	if (conf->mddev->reshape_position == MaxSector)
		return conf->mddev->degraded > conf->max_degraded;

	degraded = calc_degraded(conf);
653 654 655 656 657
	if (degraded > conf->max_degraded)
		return 1;
	return 0;
}

658
static struct stripe_head *
659
get_active_stripe(struct r5conf *conf, sector_t sector,
660
		  int previous, int noblock, int noquiesce)
L
Linus Torvalds 已提交
661 662
{
	struct stripe_head *sh;
663
	int hash = stripe_hash_locks_hash(sector);
L
Linus Torvalds 已提交
664

665
	pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector);
L
Linus Torvalds 已提交
666

667
	spin_lock_irq(conf->hash_locks + hash);
L
Linus Torvalds 已提交
668 669

	do {
670
		wait_event_lock_irq(conf->wait_for_stripe,
671
				    conf->quiesce == 0 || noquiesce,
672
				    *(conf->hash_locks + hash));
673
		sh = __find_stripe(conf, sector, conf->generation - previous);
L
Linus Torvalds 已提交
674 675
		if (!sh) {
			if (!conf->inactive_blocked)
676
				sh = get_free_stripe(conf, hash);
L
Linus Torvalds 已提交
677 678 679 680
			if (noblock && sh == NULL)
				break;
			if (!sh) {
				conf->inactive_blocked = 1;
681 682 683 684 685 686 687
				wait_event_lock_irq(
					conf->wait_for_stripe,
					!list_empty(conf->inactive_list + hash) &&
					(atomic_read(&conf->active_stripes)
					 < (conf->max_nr_stripes * 3 / 4)
					 || !conf->inactive_blocked),
					*(conf->hash_locks + hash));
L
Linus Torvalds 已提交
688
				conf->inactive_blocked = 0;
689
			} else {
690
				init_stripe(sh, sector, previous);
691 692
				atomic_inc(&sh->count);
			}
693
		} else if (!atomic_inc_not_zero(&sh->count)) {
694
			spin_lock(&conf->device_lock);
695
			if (!atomic_read(&sh->count)) {
L
Linus Torvalds 已提交
696 697
				if (!test_bit(STRIPE_HANDLE, &sh->state))
					atomic_inc(&conf->active_stripes);
698 699
				BUG_ON(list_empty(&sh->lru) &&
				       !test_bit(STRIPE_EXPANDING, &sh->state));
700
				list_del_init(&sh->lru);
701 702 703 704
				if (sh->group) {
					sh->group->stripes_cnt--;
					sh->group = NULL;
				}
L
Linus Torvalds 已提交
705
			}
706
			atomic_inc(&sh->count);
707
			spin_unlock(&conf->device_lock);
L
Linus Torvalds 已提交
708 709 710
		}
	} while (sh == NULL);

711
	spin_unlock_irq(conf->hash_locks + hash);
L
Linus Torvalds 已提交
712 713 714
	return sh;
}

715 716 717 718 719 720
static bool is_full_stripe_write(struct stripe_head *sh)
{
	BUG_ON(sh->overwrite_disks > (sh->disks - sh->raid_conf->max_degraded));
	return sh->overwrite_disks == (sh->disks - sh->raid_conf->max_degraded);
}

721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838
static void lock_two_stripes(struct stripe_head *sh1, struct stripe_head *sh2)
{
	local_irq_disable();
	if (sh1 > sh2) {
		spin_lock(&sh2->stripe_lock);
		spin_lock_nested(&sh1->stripe_lock, 1);
	} else {
		spin_lock(&sh1->stripe_lock);
		spin_lock_nested(&sh2->stripe_lock, 1);
	}
}

static void unlock_two_stripes(struct stripe_head *sh1, struct stripe_head *sh2)
{
	spin_unlock(&sh1->stripe_lock);
	spin_unlock(&sh2->stripe_lock);
	local_irq_enable();
}

/* Only freshly new full stripe normal write stripe can be added to a batch list */
static bool stripe_can_batch(struct stripe_head *sh)
{
	return test_bit(STRIPE_BATCH_READY, &sh->state) &&
		is_full_stripe_write(sh);
}

/* we only do back search */
static void stripe_add_to_batch_list(struct r5conf *conf, struct stripe_head *sh)
{
	struct stripe_head *head;
	sector_t head_sector, tmp_sec;
	int hash;
	int dd_idx;

	if (!stripe_can_batch(sh))
		return;
	/* Don't cross chunks, so stripe pd_idx/qd_idx is the same */
	tmp_sec = sh->sector;
	if (!sector_div(tmp_sec, conf->chunk_sectors))
		return;
	head_sector = sh->sector - STRIPE_SECTORS;

	hash = stripe_hash_locks_hash(head_sector);
	spin_lock_irq(conf->hash_locks + hash);
	head = __find_stripe(conf, head_sector, conf->generation);
	if (head && !atomic_inc_not_zero(&head->count)) {
		spin_lock(&conf->device_lock);
		if (!atomic_read(&head->count)) {
			if (!test_bit(STRIPE_HANDLE, &head->state))
				atomic_inc(&conf->active_stripes);
			BUG_ON(list_empty(&head->lru) &&
			       !test_bit(STRIPE_EXPANDING, &head->state));
			list_del_init(&head->lru);
			if (head->group) {
				head->group->stripes_cnt--;
				head->group = NULL;
			}
		}
		atomic_inc(&head->count);
		spin_unlock(&conf->device_lock);
	}
	spin_unlock_irq(conf->hash_locks + hash);

	if (!head)
		return;
	if (!stripe_can_batch(head))
		goto out;

	lock_two_stripes(head, sh);
	/* clear_batch_ready clear the flag */
	if (!stripe_can_batch(head) || !stripe_can_batch(sh))
		goto unlock_out;

	if (sh->batch_head)
		goto unlock_out;

	dd_idx = 0;
	while (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx)
		dd_idx++;
	if (head->dev[dd_idx].towrite->bi_rw != sh->dev[dd_idx].towrite->bi_rw)
		goto unlock_out;

	if (head->batch_head) {
		spin_lock(&head->batch_head->batch_lock);
		/* This batch list is already running */
		if (!stripe_can_batch(head)) {
			spin_unlock(&head->batch_head->batch_lock);
			goto unlock_out;
		}

		/*
		 * at this point, head's BATCH_READY could be cleared, but we
		 * can still add the stripe to batch list
		 */
		list_add(&sh->batch_list, &head->batch_list);
		spin_unlock(&head->batch_head->batch_lock);

		sh->batch_head = head->batch_head;
	} else {
		head->batch_head = head;
		sh->batch_head = head->batch_head;
		spin_lock(&head->batch_lock);
		list_add_tail(&sh->batch_list, &head->batch_list);
		spin_unlock(&head->batch_lock);
	}

	if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
		if (atomic_dec_return(&conf->preread_active_stripes)
		    < IO_THRESHOLD)
			md_wakeup_thread(conf->mddev->thread);

	atomic_inc(&sh->count);
unlock_out:
	unlock_two_stripes(head, sh);
out:
	release_stripe(head);
}

839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859
/* Determine if 'data_offset' or 'new_data_offset' should be used
 * in this stripe_head.
 */
static int use_new_offset(struct r5conf *conf, struct stripe_head *sh)
{
	sector_t progress = conf->reshape_progress;
	/* Need a memory barrier to make sure we see the value
	 * of conf->generation, or ->data_offset that was set before
	 * reshape_progress was updated.
	 */
	smp_rmb();
	if (progress == MaxSector)
		return 0;
	if (sh->generation == conf->generation - 1)
		return 0;
	/* We are in a reshape, and this is a new-generation stripe,
	 * so use new_data_offset.
	 */
	return 1;
}

860 861 862 863
static void
raid5_end_read_request(struct bio *bi, int error);
static void
raid5_end_write_request(struct bio *bi, int error);
864

865
static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)
866
{
867
	struct r5conf *conf = sh->raid_conf;
868
	int i, disks = sh->disks;
869
	struct stripe_head *head_sh = sh;
870 871 872 873 874

	might_sleep();

	for (i = disks; i--; ) {
		int rw;
875
		int replace_only = 0;
876 877
		struct bio *bi, *rbi;
		struct md_rdev *rdev, *rrdev = NULL;
878 879

		sh = head_sh;
T
Tejun Heo 已提交
880 881 882 883 884
		if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) {
			if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags))
				rw = WRITE_FUA;
			else
				rw = WRITE;
885
			if (test_bit(R5_Discard, &sh->dev[i].flags))
S
Shaohua Li 已提交
886
				rw |= REQ_DISCARD;
T
Tejun Heo 已提交
887
		} else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags))
888
			rw = READ;
889 890 891 892 893
		else if (test_and_clear_bit(R5_WantReplace,
					    &sh->dev[i].flags)) {
			rw = WRITE;
			replace_only = 1;
		} else
894
			continue;
S
Shaohua Li 已提交
895 896
		if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags))
			rw |= REQ_SYNC;
897

898
again:
899
		bi = &sh->dev[i].req;
900
		rbi = &sh->dev[i].rreq; /* For writing to replacement */
901 902

		rcu_read_lock();
903
		rrdev = rcu_dereference(conf->disks[i].replacement);
904 905 906 907 908 909
		smp_mb(); /* Ensure that if rrdev is NULL, rdev won't be */
		rdev = rcu_dereference(conf->disks[i].rdev);
		if (!rdev) {
			rdev = rrdev;
			rrdev = NULL;
		}
910 911 912
		if (rw & WRITE) {
			if (replace_only)
				rdev = NULL;
913 914 915
			if (rdev == rrdev)
				/* We raced and saw duplicates */
				rrdev = NULL;
916
		} else {
917
			if (test_bit(R5_ReadRepl, &head_sh->dev[i].flags) && rrdev)
918 919 920
				rdev = rrdev;
			rrdev = NULL;
		}
921

922 923 924 925
		if (rdev && test_bit(Faulty, &rdev->flags))
			rdev = NULL;
		if (rdev)
			atomic_inc(&rdev->nr_pending);
926 927 928 929
		if (rrdev && test_bit(Faulty, &rrdev->flags))
			rrdev = NULL;
		if (rrdev)
			atomic_inc(&rrdev->nr_pending);
930 931
		rcu_read_unlock();

932
		/* We have already checked bad blocks for reads.  Now
933 934
		 * need to check for writes.  We never accept write errors
		 * on the replacement, so we don't to check rrdev.
935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954
		 */
		while ((rw & WRITE) && rdev &&
		       test_bit(WriteErrorSeen, &rdev->flags)) {
			sector_t first_bad;
			int bad_sectors;
			int bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS,
					      &first_bad, &bad_sectors);
			if (!bad)
				break;

			if (bad < 0) {
				set_bit(BlockedBadBlocks, &rdev->flags);
				if (!conf->mddev->external &&
				    conf->mddev->flags) {
					/* It is very unlikely, but we might
					 * still need to write out the
					 * bad block log - better give it
					 * a chance*/
					md_check_recovery(conf->mddev);
				}
955 956 957 958 959 960
				/*
				 * Because md_wait_for_blocked_rdev
				 * will dec nr_pending, we must
				 * increment it first.
				 */
				atomic_inc(&rdev->nr_pending);
961 962 963 964 965 966 967 968
				md_wait_for_blocked_rdev(rdev, conf->mddev);
			} else {
				/* Acknowledged bad block - skip the write */
				rdev_dec_pending(rdev, conf->mddev);
				rdev = NULL;
			}
		}

969
		if (rdev) {
970 971
			if (s->syncing || s->expanding || s->expanded
			    || s->replacing)
972 973
				md_sync_acct(rdev->bdev, STRIPE_SECTORS);

D
Dan Williams 已提交
974 975
			set_bit(STRIPE_IO_STARTED, &sh->state);

K
Kent Overstreet 已提交
976
			bio_reset(bi);
977
			bi->bi_bdev = rdev->bdev;
K
Kent Overstreet 已提交
978 979 980 981 982 983
			bi->bi_rw = rw;
			bi->bi_end_io = (rw & WRITE)
				? raid5_end_write_request
				: raid5_end_read_request;
			bi->bi_private = sh;

984
			pr_debug("%s: for %llu schedule op %ld on disc %d\n",
985
				__func__, (unsigned long long)sh->sector,
986 987
				bi->bi_rw, i);
			atomic_inc(&sh->count);
988 989
			if (sh != head_sh)
				atomic_inc(&head_sh->count);
990
			if (use_new_offset(conf, sh))
991
				bi->bi_iter.bi_sector = (sh->sector
992 993
						 + rdev->new_data_offset);
			else
994
				bi->bi_iter.bi_sector = (sh->sector
995
						 + rdev->data_offset);
996
			if (test_bit(R5_ReadNoMerge, &head_sh->dev[i].flags))
997
				bi->bi_rw |= REQ_NOMERGE;
998

999 1000 1001
			if (test_bit(R5_SkipCopy, &sh->dev[i].flags))
				WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
			sh->dev[i].vec.bv_page = sh->dev[i].page;
K
Kent Overstreet 已提交
1002
			bi->bi_vcnt = 1;
1003 1004
			bi->bi_io_vec[0].bv_len = STRIPE_SIZE;
			bi->bi_io_vec[0].bv_offset = 0;
1005
			bi->bi_iter.bi_size = STRIPE_SIZE;
1006 1007 1008 1009 1010 1011
			/*
			 * If this is discard request, set bi_vcnt 0. We don't
			 * want to confuse SCSI because SCSI will replace payload
			 */
			if (rw & REQ_DISCARD)
				bi->bi_vcnt = 0;
1012 1013
			if (rrdev)
				set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags);
1014 1015 1016 1017 1018

			if (conf->mddev->gendisk)
				trace_block_bio_remap(bdev_get_queue(bi->bi_bdev),
						      bi, disk_devt(conf->mddev->gendisk),
						      sh->dev[i].sector);
1019
			generic_make_request(bi);
1020 1021
		}
		if (rrdev) {
1022 1023
			if (s->syncing || s->expanding || s->expanded
			    || s->replacing)
1024 1025 1026 1027
				md_sync_acct(rrdev->bdev, STRIPE_SECTORS);

			set_bit(STRIPE_IO_STARTED, &sh->state);

K
Kent Overstreet 已提交
1028
			bio_reset(rbi);
1029
			rbi->bi_bdev = rrdev->bdev;
K
Kent Overstreet 已提交
1030 1031 1032 1033 1034
			rbi->bi_rw = rw;
			BUG_ON(!(rw & WRITE));
			rbi->bi_end_io = raid5_end_write_request;
			rbi->bi_private = sh;

1035 1036 1037 1038 1039
			pr_debug("%s: for %llu schedule op %ld on "
				 "replacement disc %d\n",
				__func__, (unsigned long long)sh->sector,
				rbi->bi_rw, i);
			atomic_inc(&sh->count);
1040 1041
			if (sh != head_sh)
				atomic_inc(&head_sh->count);
1042
			if (use_new_offset(conf, sh))
1043
				rbi->bi_iter.bi_sector = (sh->sector
1044 1045
						  + rrdev->new_data_offset);
			else
1046
				rbi->bi_iter.bi_sector = (sh->sector
1047
						  + rrdev->data_offset);
1048 1049 1050
			if (test_bit(R5_SkipCopy, &sh->dev[i].flags))
				WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
			sh->dev[i].rvec.bv_page = sh->dev[i].page;
K
Kent Overstreet 已提交
1051
			rbi->bi_vcnt = 1;
1052 1053
			rbi->bi_io_vec[0].bv_len = STRIPE_SIZE;
			rbi->bi_io_vec[0].bv_offset = 0;
1054
			rbi->bi_iter.bi_size = STRIPE_SIZE;
1055 1056 1057 1058 1059 1060
			/*
			 * If this is discard request, set bi_vcnt 0. We don't
			 * want to confuse SCSI because SCSI will replace payload
			 */
			if (rw & REQ_DISCARD)
				rbi->bi_vcnt = 0;
1061 1062 1063 1064
			if (conf->mddev->gendisk)
				trace_block_bio_remap(bdev_get_queue(rbi->bi_bdev),
						      rbi, disk_devt(conf->mddev->gendisk),
						      sh->dev[i].sector);
1065 1066 1067
			generic_make_request(rbi);
		}
		if (!rdev && !rrdev) {
1068
			if (rw & WRITE)
1069 1070 1071 1072
				set_bit(STRIPE_DEGRADED, &sh->state);
			pr_debug("skip op %ld on disc %d for sector %llu\n",
				bi->bi_rw, i, (unsigned long long)sh->sector);
			clear_bit(R5_LOCKED, &sh->dev[i].flags);
1073 1074 1075
			if (sh->batch_head)
				set_bit(STRIPE_BATCH_ERR,
					&sh->batch_head->state);
1076 1077
			set_bit(STRIPE_HANDLE, &sh->state);
		}
1078 1079 1080 1081 1082 1083 1084

		if (!head_sh->batch_head)
			continue;
		sh = list_first_entry(&sh->batch_list, struct stripe_head,
				      batch_list);
		if (sh != head_sh)
			goto again;
1085 1086 1087 1088
	}
}

static struct dma_async_tx_descriptor *
1089 1090 1091
async_copy_data(int frombio, struct bio *bio, struct page **page,
	sector_t sector, struct dma_async_tx_descriptor *tx,
	struct stripe_head *sh)
1092
{
1093 1094
	struct bio_vec bvl;
	struct bvec_iter iter;
1095 1096
	struct page *bio_page;
	int page_offset;
1097
	struct async_submit_ctl submit;
D
Dan Williams 已提交
1098
	enum async_tx_flags flags = 0;
1099

1100 1101
	if (bio->bi_iter.bi_sector >= sector)
		page_offset = (signed)(bio->bi_iter.bi_sector - sector) * 512;
1102
	else
1103
		page_offset = (signed)(sector - bio->bi_iter.bi_sector) * -512;
1104

D
Dan Williams 已提交
1105 1106 1107 1108
	if (frombio)
		flags |= ASYNC_TX_FENCE;
	init_async_submit(&submit, flags, tx, NULL, NULL, NULL);

1109 1110
	bio_for_each_segment(bvl, bio, iter) {
		int len = bvl.bv_len;
1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125
		int clen;
		int b_offset = 0;

		if (page_offset < 0) {
			b_offset = -page_offset;
			page_offset += b_offset;
			len -= b_offset;
		}

		if (len > 0 && page_offset + len > STRIPE_SIZE)
			clen = STRIPE_SIZE - page_offset;
		else
			clen = len;

		if (clen > 0) {
1126 1127
			b_offset += bvl.bv_offset;
			bio_page = bvl.bv_page;
1128 1129 1130 1131 1132 1133 1134
			if (frombio) {
				if (sh->raid_conf->skip_copy &&
				    b_offset == 0 && page_offset == 0 &&
				    clen == STRIPE_SIZE)
					*page = bio_page;
				else
					tx = async_memcpy(*page, bio_page, page_offset,
1135
						  b_offset, clen, &submit);
1136 1137
			} else
				tx = async_memcpy(bio_page, *page, b_offset,
1138
						  page_offset, clen, &submit);
1139
		}
1140 1141 1142
		/* chain the operations */
		submit.depend_tx = tx;

1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154
		if (clen < len) /* hit end of page */
			break;
		page_offset +=  len;
	}

	return tx;
}

static void ops_complete_biofill(void *stripe_head_ref)
{
	struct stripe_head *sh = stripe_head_ref;
	struct bio *return_bi = NULL;
1155
	int i;
1156

1157
	pr_debug("%s: stripe %llu\n", __func__,
1158 1159 1160 1161 1162 1163 1164
		(unsigned long long)sh->sector);

	/* clear completed biofills */
	for (i = sh->disks; i--; ) {
		struct r5dev *dev = &sh->dev[i];

		/* acknowledge completion of a biofill operation */
1165 1166
		/* and check if we need to reply to a read request,
		 * new R5_Wantfill requests are held off until
1167
		 * !STRIPE_BIOFILL_RUN
1168 1169
		 */
		if (test_and_clear_bit(R5_Wantfill, &dev->flags)) {
1170 1171 1172 1173 1174
			struct bio *rbi, *rbi2;

			BUG_ON(!dev->read);
			rbi = dev->read;
			dev->read = NULL;
1175
			while (rbi && rbi->bi_iter.bi_sector <
1176 1177
				dev->sector + STRIPE_SECTORS) {
				rbi2 = r5_next_bio(rbi, dev->sector);
1178
				if (!raid5_dec_bi_active_stripes(rbi)) {
1179 1180 1181 1182 1183 1184 1185
					rbi->bi_next = return_bi;
					return_bi = rbi;
				}
				rbi = rbi2;
			}
		}
	}
1186
	clear_bit(STRIPE_BIOFILL_RUN, &sh->state);
1187 1188 1189

	return_io(return_bi);

1190
	set_bit(STRIPE_HANDLE, &sh->state);
1191 1192 1193 1194 1195 1196
	release_stripe(sh);
}

static void ops_run_biofill(struct stripe_head *sh)
{
	struct dma_async_tx_descriptor *tx = NULL;
1197
	struct async_submit_ctl submit;
1198 1199
	int i;

1200
	BUG_ON(sh->batch_head);
1201
	pr_debug("%s: stripe %llu\n", __func__,
1202 1203 1204 1205 1206 1207
		(unsigned long long)sh->sector);

	for (i = sh->disks; i--; ) {
		struct r5dev *dev = &sh->dev[i];
		if (test_bit(R5_Wantfill, &dev->flags)) {
			struct bio *rbi;
S
Shaohua Li 已提交
1208
			spin_lock_irq(&sh->stripe_lock);
1209 1210
			dev->read = rbi = dev->toread;
			dev->toread = NULL;
S
Shaohua Li 已提交
1211
			spin_unlock_irq(&sh->stripe_lock);
1212
			while (rbi && rbi->bi_iter.bi_sector <
1213
				dev->sector + STRIPE_SECTORS) {
1214 1215
				tx = async_copy_data(0, rbi, &dev->page,
					dev->sector, tx, sh);
1216 1217 1218 1219 1220 1221
				rbi = r5_next_bio(rbi, dev->sector);
			}
		}
	}

	atomic_inc(&sh->count);
1222 1223
	init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL);
	async_trigger_callback(&submit);
1224 1225
}

1226
static void mark_target_uptodate(struct stripe_head *sh, int target)
1227
{
1228
	struct r5dev *tgt;
1229

1230 1231
	if (target < 0)
		return;
1232

1233
	tgt = &sh->dev[target];
1234 1235 1236
	set_bit(R5_UPTODATE, &tgt->flags);
	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
	clear_bit(R5_Wantcompute, &tgt->flags);
1237 1238
}

1239
static void ops_complete_compute(void *stripe_head_ref)
1240 1241 1242
{
	struct stripe_head *sh = stripe_head_ref;

1243
	pr_debug("%s: stripe %llu\n", __func__,
1244 1245
		(unsigned long long)sh->sector);

1246
	/* mark the computed target(s) as uptodate */
1247
	mark_target_uptodate(sh, sh->ops.target);
1248
	mark_target_uptodate(sh, sh->ops.target2);
1249

1250 1251 1252
	clear_bit(STRIPE_COMPUTE_RUN, &sh->state);
	if (sh->check_state == check_state_compute_run)
		sh->check_state = check_state_compute_result;
1253 1254 1255 1256
	set_bit(STRIPE_HANDLE, &sh->state);
	release_stripe(sh);
}

1257 1258
/* return a pointer to the address conversion region of the scribble buffer */
static addr_conv_t *to_addr_conv(struct stripe_head *sh,
1259
				 struct raid5_percpu *percpu, int i)
1260
{
1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273
	void *addr;

	addr = flex_array_get(percpu->scribble, i);
	return addr + sizeof(struct page *) * (sh->disks + 2);
}

/* return a pointer to the address conversion region of the scribble buffer */
static struct page **to_addr_page(struct raid5_percpu *percpu, int i)
{
	void *addr;

	addr = flex_array_get(percpu->scribble, i);
	return addr;
1274 1275 1276 1277
}

static struct dma_async_tx_descriptor *
ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)
1278 1279
{
	int disks = sh->disks;
1280
	struct page **xor_srcs = to_addr_page(percpu, 0);
1281 1282 1283 1284 1285
	int target = sh->ops.target;
	struct r5dev *tgt = &sh->dev[target];
	struct page *xor_dest = tgt->page;
	int count = 0;
	struct dma_async_tx_descriptor *tx;
1286
	struct async_submit_ctl submit;
1287 1288
	int i;

1289 1290
	BUG_ON(sh->batch_head);

1291
	pr_debug("%s: stripe %llu block: %d\n",
1292
		__func__, (unsigned long long)sh->sector, target);
1293 1294 1295 1296 1297 1298 1299 1300
	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));

	for (i = disks; i--; )
		if (i != target)
			xor_srcs[count++] = sh->dev[i].page;

	atomic_inc(&sh->count);

D
Dan Williams 已提交
1301
	init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL,
1302
			  ops_complete_compute, sh, to_addr_conv(sh, percpu, 0));
1303
	if (unlikely(count == 1))
1304
		tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
1305
	else
1306
		tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
1307 1308 1309 1310

	return tx;
}

1311 1312 1313 1314 1315 1316 1317 1318 1319
/* set_syndrome_sources - populate source buffers for gen_syndrome
 * @srcs - (struct page *) array of size sh->disks
 * @sh - stripe_head to parse
 *
 * Populates srcs in proper layout order for the stripe and returns the
 * 'count' of sources to be used in a call to async_gen_syndrome.  The P
 * destination buffer is recorded in srcs[count] and the Q destination
 * is recorded in srcs[count+1]].
 */
1320 1321 1322
static int set_syndrome_sources(struct page **srcs,
				struct stripe_head *sh,
				int srctype)
1323 1324 1325 1326 1327 1328 1329 1330
{
	int disks = sh->disks;
	int syndrome_disks = sh->ddf_layout ? disks : (disks - 2);
	int d0_idx = raid6_d0(sh);
	int count;
	int i;

	for (i = 0; i < disks; i++)
1331
		srcs[i] = NULL;
1332 1333 1334 1335 1336

	count = 0;
	i = d0_idx;
	do {
		int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
1337
		struct r5dev *dev = &sh->dev[i];
1338

1339 1340 1341 1342 1343 1344 1345
		if (i == sh->qd_idx || i == sh->pd_idx ||
		    (srctype == SYNDROME_SRC_ALL) ||
		    (srctype == SYNDROME_SRC_WANT_DRAIN &&
		     test_bit(R5_Wantdrain, &dev->flags)) ||
		    (srctype == SYNDROME_SRC_WRITTEN &&
		     dev->written))
			srcs[slot] = sh->dev[i].page;
1346 1347 1348
		i = raid6_next_disk(i, disks);
	} while (i != d0_idx);

1349
	return syndrome_disks;
1350 1351 1352 1353 1354 1355
}

static struct dma_async_tx_descriptor *
ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)
{
	int disks = sh->disks;
1356
	struct page **blocks = to_addr_page(percpu, 0);
1357 1358 1359 1360 1361 1362 1363 1364 1365
	int target;
	int qd_idx = sh->qd_idx;
	struct dma_async_tx_descriptor *tx;
	struct async_submit_ctl submit;
	struct r5dev *tgt;
	struct page *dest;
	int i;
	int count;

1366
	BUG_ON(sh->batch_head);
1367 1368 1369 1370
	if (sh->ops.target < 0)
		target = sh->ops.target2;
	else if (sh->ops.target2 < 0)
		target = sh->ops.target;
1371
	else
1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384
		/* we should only have one valid target */
		BUG();
	BUG_ON(target < 0);
	pr_debug("%s: stripe %llu block: %d\n",
		__func__, (unsigned long long)sh->sector, target);

	tgt = &sh->dev[target];
	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
	dest = tgt->page;

	atomic_inc(&sh->count);

	if (target == qd_idx) {
1385
		count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_ALL);
1386 1387
		blocks[count] = NULL; /* regenerating p is not necessary */
		BUG_ON(blocks[count+1] != dest); /* q should already be set */
D
Dan Williams 已提交
1388 1389
		init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
				  ops_complete_compute, sh,
1390
				  to_addr_conv(sh, percpu, 0));
1391 1392 1393 1394 1395 1396 1397 1398 1399 1400
		tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit);
	} else {
		/* Compute any data- or p-drive using XOR */
		count = 0;
		for (i = disks; i-- ; ) {
			if (i == target || i == qd_idx)
				continue;
			blocks[count++] = sh->dev[i].page;
		}

D
Dan Williams 已提交
1401 1402
		init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
				  NULL, ops_complete_compute, sh,
1403
				  to_addr_conv(sh, percpu, 0));
1404 1405
		tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit);
	}
1406 1407 1408 1409

	return tx;
}

1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421
static struct dma_async_tx_descriptor *
ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
{
	int i, count, disks = sh->disks;
	int syndrome_disks = sh->ddf_layout ? disks : disks-2;
	int d0_idx = raid6_d0(sh);
	int faila = -1, failb = -1;
	int target = sh->ops.target;
	int target2 = sh->ops.target2;
	struct r5dev *tgt = &sh->dev[target];
	struct r5dev *tgt2 = &sh->dev[target2];
	struct dma_async_tx_descriptor *tx;
1422
	struct page **blocks = to_addr_page(percpu, 0);
1423 1424
	struct async_submit_ctl submit;

1425
	BUG_ON(sh->batch_head);
1426 1427 1428 1429 1430 1431
	pr_debug("%s: stripe %llu block1: %d block2: %d\n",
		 __func__, (unsigned long long)sh->sector, target, target2);
	BUG_ON(target < 0 || target2 < 0);
	BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
	BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags));

1432
	/* we need to open-code set_syndrome_sources to handle the
1433 1434 1435
	 * slot number conversion for 'faila' and 'failb'
	 */
	for (i = 0; i < disks ; i++)
1436
		blocks[i] = NULL;
1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462
	count = 0;
	i = d0_idx;
	do {
		int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);

		blocks[slot] = sh->dev[i].page;

		if (i == target)
			faila = slot;
		if (i == target2)
			failb = slot;
		i = raid6_next_disk(i, disks);
	} while (i != d0_idx);

	BUG_ON(faila == failb);
	if (failb < faila)
		swap(faila, failb);
	pr_debug("%s: stripe: %llu faila: %d failb: %d\n",
		 __func__, (unsigned long long)sh->sector, faila, failb);

	atomic_inc(&sh->count);

	if (failb == syndrome_disks+1) {
		/* Q disk is one of the missing disks */
		if (faila == syndrome_disks) {
			/* Missing P+Q, just recompute */
D
Dan Williams 已提交
1463 1464
			init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
					  ops_complete_compute, sh,
1465
					  to_addr_conv(sh, percpu, 0));
1466
			return async_gen_syndrome(blocks, 0, syndrome_disks+2,
1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485
						  STRIPE_SIZE, &submit);
		} else {
			struct page *dest;
			int data_target;
			int qd_idx = sh->qd_idx;

			/* Missing D+Q: recompute D from P, then recompute Q */
			if (target == qd_idx)
				data_target = target2;
			else
				data_target = target;

			count = 0;
			for (i = disks; i-- ; ) {
				if (i == data_target || i == qd_idx)
					continue;
				blocks[count++] = sh->dev[i].page;
			}
			dest = sh->dev[data_target].page;
D
Dan Williams 已提交
1486 1487 1488
			init_async_submit(&submit,
					  ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
					  NULL, NULL, NULL,
1489
					  to_addr_conv(sh, percpu, 0));
1490 1491 1492
			tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE,
				       &submit);

1493
			count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_ALL);
D
Dan Williams 已提交
1494 1495
			init_async_submit(&submit, ASYNC_TX_FENCE, tx,
					  ops_complete_compute, sh,
1496
					  to_addr_conv(sh, percpu, 0));
1497 1498 1499 1500
			return async_gen_syndrome(blocks, 0, count+2,
						  STRIPE_SIZE, &submit);
		}
	} else {
1501 1502
		init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
				  ops_complete_compute, sh,
1503
				  to_addr_conv(sh, percpu, 0));
1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514
		if (failb == syndrome_disks) {
			/* We're missing D+P. */
			return async_raid6_datap_recov(syndrome_disks+2,
						       STRIPE_SIZE, faila,
						       blocks, &submit);
		} else {
			/* We're missing D+D. */
			return async_raid6_2data_recov(syndrome_disks+2,
						       STRIPE_SIZE, faila, failb,
						       blocks, &submit);
		}
1515 1516 1517
	}
}

1518 1519 1520 1521
static void ops_complete_prexor(void *stripe_head_ref)
{
	struct stripe_head *sh = stripe_head_ref;

1522
	pr_debug("%s: stripe %llu\n", __func__,
1523 1524 1525 1526
		(unsigned long long)sh->sector);
}

static struct dma_async_tx_descriptor *
1527 1528
ops_run_prexor5(struct stripe_head *sh, struct raid5_percpu *percpu,
		struct dma_async_tx_descriptor *tx)
1529 1530
{
	int disks = sh->disks;
1531
	struct page **xor_srcs = to_addr_page(percpu, 0);
1532
	int count = 0, pd_idx = sh->pd_idx, i;
1533
	struct async_submit_ctl submit;
1534 1535 1536 1537

	/* existing parity data subtracted */
	struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;

1538
	BUG_ON(sh->batch_head);
1539
	pr_debug("%s: stripe %llu\n", __func__,
1540 1541 1542 1543 1544
		(unsigned long long)sh->sector);

	for (i = disks; i--; ) {
		struct r5dev *dev = &sh->dev[i];
		/* Only process blocks that are known to be uptodate */
1545
		if (test_bit(R5_Wantdrain, &dev->flags))
1546 1547 1548
			xor_srcs[count++] = dev->page;
	}

D
Dan Williams 已提交
1549
	init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
1550
			  ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
1551
	tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
1552 1553 1554 1555

	return tx;
}

1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575
static struct dma_async_tx_descriptor *
ops_run_prexor6(struct stripe_head *sh, struct raid5_percpu *percpu,
		struct dma_async_tx_descriptor *tx)
{
	struct page **blocks = to_addr_page(percpu, 0);
	int count;
	struct async_submit_ctl submit;

	pr_debug("%s: stripe %llu\n", __func__,
		(unsigned long long)sh->sector);

	count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_WANT_DRAIN);

	init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_PQ_XOR_DST, tx,
			  ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
	tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE,  &submit);

	return tx;
}

1576
static struct dma_async_tx_descriptor *
1577
ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
1578 1579
{
	int disks = sh->disks;
1580
	int i;
1581
	struct stripe_head *head_sh = sh;
1582

1583
	pr_debug("%s: stripe %llu\n", __func__,
1584 1585 1586
		(unsigned long long)sh->sector);

	for (i = disks; i--; ) {
1587
		struct r5dev *dev;
1588 1589
		struct bio *chosen;

1590 1591
		sh = head_sh;
		if (test_and_clear_bit(R5_Wantdrain, &head_sh->dev[i].flags)) {
1592 1593
			struct bio *wbi;

1594 1595
again:
			dev = &sh->dev[i];
S
Shaohua Li 已提交
1596
			spin_lock_irq(&sh->stripe_lock);
1597 1598
			chosen = dev->towrite;
			dev->towrite = NULL;
1599
			sh->overwrite_disks = 0;
1600 1601
			BUG_ON(dev->written);
			wbi = dev->written = chosen;
S
Shaohua Li 已提交
1602
			spin_unlock_irq(&sh->stripe_lock);
1603
			WARN_ON(dev->page != dev->orig_page);
1604

1605
			while (wbi && wbi->bi_iter.bi_sector <
1606
				dev->sector + STRIPE_SECTORS) {
T
Tejun Heo 已提交
1607 1608
				if (wbi->bi_rw & REQ_FUA)
					set_bit(R5_WantFUA, &dev->flags);
S
Shaohua Li 已提交
1609 1610
				if (wbi->bi_rw & REQ_SYNC)
					set_bit(R5_SyncIO, &dev->flags);
1611
				if (wbi->bi_rw & REQ_DISCARD)
S
Shaohua Li 已提交
1612
					set_bit(R5_Discard, &dev->flags);
1613 1614 1615 1616 1617 1618 1619 1620 1621
				else {
					tx = async_copy_data(1, wbi, &dev->page,
						dev->sector, tx, sh);
					if (dev->page != dev->orig_page) {
						set_bit(R5_SkipCopy, &dev->flags);
						clear_bit(R5_UPTODATE, &dev->flags);
						clear_bit(R5_OVERWRITE, &dev->flags);
					}
				}
1622 1623
				wbi = r5_next_bio(wbi, dev->sector);
			}
1624 1625 1626 1627 1628 1629 1630 1631 1632

			if (head_sh->batch_head) {
				sh = list_first_entry(&sh->batch_list,
						      struct stripe_head,
						      batch_list);
				if (sh == head_sh)
					continue;
				goto again;
			}
1633 1634 1635 1636 1637 1638
		}
	}

	return tx;
}

1639
static void ops_complete_reconstruct(void *stripe_head_ref)
1640 1641
{
	struct stripe_head *sh = stripe_head_ref;
1642 1643 1644 1645
	int disks = sh->disks;
	int pd_idx = sh->pd_idx;
	int qd_idx = sh->qd_idx;
	int i;
1646
	bool fua = false, sync = false, discard = false;
1647

1648
	pr_debug("%s: stripe %llu\n", __func__,
1649 1650
		(unsigned long long)sh->sector);

S
Shaohua Li 已提交
1651
	for (i = disks; i--; ) {
T
Tejun Heo 已提交
1652
		fua |= test_bit(R5_WantFUA, &sh->dev[i].flags);
S
Shaohua Li 已提交
1653
		sync |= test_bit(R5_SyncIO, &sh->dev[i].flags);
1654
		discard |= test_bit(R5_Discard, &sh->dev[i].flags);
S
Shaohua Li 已提交
1655
	}
T
Tejun Heo 已提交
1656

1657 1658
	for (i = disks; i--; ) {
		struct r5dev *dev = &sh->dev[i];
1659

T
Tejun Heo 已提交
1660
		if (dev->written || i == pd_idx || i == qd_idx) {
1661
			if (!discard && !test_bit(R5_SkipCopy, &dev->flags))
1662
				set_bit(R5_UPTODATE, &dev->flags);
T
Tejun Heo 已提交
1663 1664
			if (fua)
				set_bit(R5_WantFUA, &dev->flags);
S
Shaohua Li 已提交
1665 1666
			if (sync)
				set_bit(R5_SyncIO, &dev->flags);
T
Tejun Heo 已提交
1667
		}
1668 1669
	}

1670 1671 1672 1673 1674 1675 1676 1677
	if (sh->reconstruct_state == reconstruct_state_drain_run)
		sh->reconstruct_state = reconstruct_state_drain_result;
	else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run)
		sh->reconstruct_state = reconstruct_state_prexor_drain_result;
	else {
		BUG_ON(sh->reconstruct_state != reconstruct_state_run);
		sh->reconstruct_state = reconstruct_state_result;
	}
1678 1679 1680 1681 1682 1683

	set_bit(STRIPE_HANDLE, &sh->state);
	release_stripe(sh);
}

static void
1684 1685
ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu,
		     struct dma_async_tx_descriptor *tx)
1686 1687
{
	int disks = sh->disks;
1688
	struct page **xor_srcs;
1689
	struct async_submit_ctl submit;
1690
	int count, pd_idx = sh->pd_idx, i;
1691
	struct page *xor_dest;
1692
	int prexor = 0;
1693
	unsigned long flags;
1694 1695 1696
	int j = 0;
	struct stripe_head *head_sh = sh;
	int last_stripe;
1697

1698
	pr_debug("%s: stripe %llu\n", __func__,
1699 1700
		(unsigned long long)sh->sector);

S
Shaohua Li 已提交
1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712
	for (i = 0; i < sh->disks; i++) {
		if (pd_idx == i)
			continue;
		if (!test_bit(R5_Discard, &sh->dev[i].flags))
			break;
	}
	if (i >= sh->disks) {
		atomic_inc(&sh->count);
		set_bit(R5_Discard, &sh->dev[pd_idx].flags);
		ops_complete_reconstruct(sh);
		return;
	}
1713 1714 1715
again:
	count = 0;
	xor_srcs = to_addr_page(percpu, j);
1716 1717 1718
	/* check if prexor is active which means only process blocks
	 * that are part of a read-modify-write (written)
	 */
1719
	if (head_sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
1720
		prexor = 1;
1721 1722 1723
		xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
1724
			if (head_sh->dev[i].written)
1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740
				xor_srcs[count++] = dev->page;
		}
	} else {
		xor_dest = sh->dev[pd_idx].page;
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if (i != pd_idx)
				xor_srcs[count++] = dev->page;
		}
	}

	/* 1/ if we prexor'd then the dest is reused as a source
	 * 2/ if we did not prexor then we are redoing the parity
	 * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST
	 * for the synchronous xor case
	 */
1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755
	last_stripe = !head_sh->batch_head ||
		list_first_entry(&sh->batch_list,
				 struct stripe_head, batch_list) == head_sh;
	if (last_stripe) {
		flags = ASYNC_TX_ACK |
			(prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST);

		atomic_inc(&head_sh->count);
		init_async_submit(&submit, flags, tx, ops_complete_reconstruct, head_sh,
				  to_addr_conv(sh, percpu, j));
	} else {
		flags = prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST;
		init_async_submit(&submit, flags, tx, NULL, NULL,
				  to_addr_conv(sh, percpu, j));
	}
1756

1757 1758 1759 1760
	if (unlikely(count == 1))
		tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit);
	else
		tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit);
1761 1762 1763 1764 1765 1766
	if (!last_stripe) {
		j++;
		sh = list_first_entry(&sh->batch_list, struct stripe_head,
				      batch_list);
		goto again;
	}
1767 1768
}

1769 1770 1771 1772 1773
static void
ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,
		     struct dma_async_tx_descriptor *tx)
{
	struct async_submit_ctl submit;
1774 1775 1776 1777
	struct page **blocks;
	int count, i, j = 0;
	struct stripe_head *head_sh = sh;
	int last_stripe;
1778 1779
	int synflags;
	unsigned long txflags;
1780 1781 1782

	pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector);

S
Shaohua Li 已提交
1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796
	for (i = 0; i < sh->disks; i++) {
		if (sh->pd_idx == i || sh->qd_idx == i)
			continue;
		if (!test_bit(R5_Discard, &sh->dev[i].flags))
			break;
	}
	if (i >= sh->disks) {
		atomic_inc(&sh->count);
		set_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);
		set_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);
		ops_complete_reconstruct(sh);
		return;
	}

1797 1798
again:
	blocks = to_addr_page(percpu, j);
1799 1800 1801 1802 1803 1804 1805 1806 1807 1808

	if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
		synflags = SYNDROME_SRC_WRITTEN;
		txflags = ASYNC_TX_ACK | ASYNC_TX_PQ_XOR_DST;
	} else {
		synflags = SYNDROME_SRC_ALL;
		txflags = ASYNC_TX_ACK;
	}

	count = set_syndrome_sources(blocks, sh, synflags);
1809 1810 1811 1812 1813 1814
	last_stripe = !head_sh->batch_head ||
		list_first_entry(&sh->batch_list,
				 struct stripe_head, batch_list) == head_sh;

	if (last_stripe) {
		atomic_inc(&head_sh->count);
1815
		init_async_submit(&submit, txflags, tx, ops_complete_reconstruct,
1816 1817 1818 1819
				  head_sh, to_addr_conv(sh, percpu, j));
	} else
		init_async_submit(&submit, 0, tx, NULL, NULL,
				  to_addr_conv(sh, percpu, j));
1820
	async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE,  &submit);
1821 1822 1823 1824 1825 1826
	if (!last_stripe) {
		j++;
		sh = list_first_entry(&sh->batch_list, struct stripe_head,
				      batch_list);
		goto again;
	}
1827 1828 1829 1830 1831 1832
}

static void ops_complete_check(void *stripe_head_ref)
{
	struct stripe_head *sh = stripe_head_ref;

1833
	pr_debug("%s: stripe %llu\n", __func__,
1834 1835
		(unsigned long long)sh->sector);

1836
	sh->check_state = check_state_check_result;
1837 1838 1839 1840
	set_bit(STRIPE_HANDLE, &sh->state);
	release_stripe(sh);
}

1841
static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu)
1842 1843
{
	int disks = sh->disks;
1844 1845 1846
	int pd_idx = sh->pd_idx;
	int qd_idx = sh->qd_idx;
	struct page *xor_dest;
1847
	struct page **xor_srcs = to_addr_page(percpu, 0);
1848
	struct dma_async_tx_descriptor *tx;
1849
	struct async_submit_ctl submit;
1850 1851
	int count;
	int i;
1852

1853
	pr_debug("%s: stripe %llu\n", __func__,
1854 1855
		(unsigned long long)sh->sector);

1856
	BUG_ON(sh->batch_head);
1857 1858 1859
	count = 0;
	xor_dest = sh->dev[pd_idx].page;
	xor_srcs[count++] = xor_dest;
1860
	for (i = disks; i--; ) {
1861 1862 1863
		if (i == pd_idx || i == qd_idx)
			continue;
		xor_srcs[count++] = sh->dev[i].page;
1864 1865
	}

1866
	init_async_submit(&submit, 0, NULL, NULL, NULL,
1867
			  to_addr_conv(sh, percpu, 0));
D
Dan Williams 已提交
1868
	tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE,
1869
			   &sh->ops.zero_sum_result, &submit);
1870 1871

	atomic_inc(&sh->count);
1872 1873
	init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL);
	tx = async_trigger_callback(&submit);
1874 1875
}

1876 1877
static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp)
{
1878
	struct page **srcs = to_addr_page(percpu, 0);
1879 1880 1881 1882 1883 1884
	struct async_submit_ctl submit;
	int count;

	pr_debug("%s: stripe %llu checkp: %d\n", __func__,
		(unsigned long long)sh->sector, checkp);

1885
	BUG_ON(sh->batch_head);
1886
	count = set_syndrome_sources(srcs, sh, SYNDROME_SRC_ALL);
1887 1888
	if (!checkp)
		srcs[count] = NULL;
1889 1890

	atomic_inc(&sh->count);
1891
	init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check,
1892
			  sh, to_addr_conv(sh, percpu, 0));
1893 1894
	async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE,
			   &sh->ops.zero_sum_result, percpu->spare_page, &submit);
1895 1896
}

N
NeilBrown 已提交
1897
static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
1898 1899 1900
{
	int overlap_clear = 0, i, disks = sh->disks;
	struct dma_async_tx_descriptor *tx = NULL;
1901
	struct r5conf *conf = sh->raid_conf;
1902
	int level = conf->level;
1903 1904
	struct raid5_percpu *percpu;
	unsigned long cpu;
1905

1906 1907
	cpu = get_cpu();
	percpu = per_cpu_ptr(conf->percpu, cpu);
1908
	if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) {
1909 1910 1911 1912
		ops_run_biofill(sh);
		overlap_clear++;
	}

1913
	if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) {
1914 1915 1916 1917 1918 1919 1920 1921 1922 1923
		if (level < 6)
			tx = ops_run_compute5(sh, percpu);
		else {
			if (sh->ops.target2 < 0 || sh->ops.target < 0)
				tx = ops_run_compute6_1(sh, percpu);
			else
				tx = ops_run_compute6_2(sh, percpu);
		}
		/* terminate the chain if reconstruct is not set to be run */
		if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request))
1924 1925
			async_tx_ack(tx);
	}
1926

1927 1928 1929 1930 1931 1932
	if (test_bit(STRIPE_OP_PREXOR, &ops_request)) {
		if (level < 6)
			tx = ops_run_prexor5(sh, percpu, tx);
		else
			tx = ops_run_prexor6(sh, percpu, tx);
	}
1933

1934
	if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) {
1935
		tx = ops_run_biodrain(sh, tx);
1936 1937 1938
		overlap_clear++;
	}

1939 1940 1941 1942 1943 1944
	if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) {
		if (level < 6)
			ops_run_reconstruct5(sh, percpu, tx);
		else
			ops_run_reconstruct6(sh, percpu, tx);
	}
1945

1946 1947 1948 1949 1950 1951 1952 1953 1954 1955
	if (test_bit(STRIPE_OP_CHECK, &ops_request)) {
		if (sh->check_state == check_state_run)
			ops_run_check_p(sh, percpu);
		else if (sh->check_state == check_state_run_q)
			ops_run_check_pq(sh, percpu, 0);
		else if (sh->check_state == check_state_run_pq)
			ops_run_check_pq(sh, percpu, 1);
		else
			BUG();
	}
1956

1957
	if (overlap_clear && !sh->batch_head)
1958 1959 1960 1961 1962
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if (test_and_clear_bit(R5_Overlap, &dev->flags))
				wake_up(&sh->raid_conf->wait_for_overlap);
		}
1963
	put_cpu();
1964 1965
}

1966
static int grow_one_stripe(struct r5conf *conf, int hash, gfp_t gfp)
L
Linus Torvalds 已提交
1967 1968
{
	struct stripe_head *sh;
1969
	sh = kmem_cache_zalloc(conf->slab_cache, gfp);
1970 1971
	if (!sh)
		return 0;
N
Namhyung Kim 已提交
1972

1973 1974
	sh->raid_conf = conf;

S
Shaohua Li 已提交
1975 1976
	spin_lock_init(&sh->stripe_lock);

1977
	if (grow_buffers(sh, gfp)) {
1978
		shrink_buffers(sh);
1979 1980 1981
		kmem_cache_free(conf->slab_cache, sh);
		return 0;
	}
1982
	sh->hash_lock_index = hash;
1983 1984 1985 1986
	/* we just created an active stripe so... */
	atomic_set(&sh->count, 1);
	atomic_inc(&conf->active_stripes);
	INIT_LIST_HEAD(&sh->lru);
1987 1988 1989 1990

	spin_lock_init(&sh->batch_lock);
	INIT_LIST_HEAD(&sh->batch_list);
	sh->batch_head = NULL;
1991 1992 1993 1994
	release_stripe(sh);
	return 1;
}

1995
static int grow_stripes(struct r5conf *conf, int num)
1996
{
1997
	struct kmem_cache *sc;
1998
	int devs = max(conf->raid_disks, conf->previous_raid_disks);
1999
	int hash;
L
Linus Torvalds 已提交
2000

2001 2002 2003 2004 2005 2006 2007 2008
	if (conf->mddev->gendisk)
		sprintf(conf->cache_name[0],
			"raid%d-%s", conf->level, mdname(conf->mddev));
	else
		sprintf(conf->cache_name[0],
			"raid%d-%p", conf->level, conf->mddev);
	sprintf(conf->cache_name[1], "%s-alt", conf->cache_name[0]);

2009 2010
	conf->active_name = 0;
	sc = kmem_cache_create(conf->cache_name[conf->active_name],
L
Linus Torvalds 已提交
2011
			       sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev),
2012
			       0, 0, NULL);
L
Linus Torvalds 已提交
2013 2014 2015
	if (!sc)
		return 1;
	conf->slab_cache = sc;
2016
	conf->pool_size = devs;
2017 2018
	hash = conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS;
	while (num--) {
2019
		if (!grow_one_stripe(conf, hash, GFP_KERNEL))
L
Linus Torvalds 已提交
2020
			return 1;
2021 2022 2023
		conf->max_nr_stripes++;
		hash = (hash + 1) % NR_STRIPE_HASH_LOCKS;
	}
L
Linus Torvalds 已提交
2024 2025
	return 0;
}
2026

2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039
/**
 * scribble_len - return the required size of the scribble region
 * @num - total number of disks in the array
 *
 * The size must be enough to contain:
 * 1/ a struct page pointer for each device in the array +2
 * 2/ room to convert each entry in (1) to its corresponding dma
 *    (dma_map_page()) or page (page_address()) address.
 *
 * Note: the +2 is for the destination buffers of the ddf/raid6 case where we
 * calculate over all devices (not just the data blocks), using zeros in place
 * of the P and Q blocks.
 */
2040
static struct flex_array *scribble_alloc(int num, int cnt, gfp_t flags)
2041
{
2042
	struct flex_array *ret;
2043 2044 2045
	size_t len;

	len = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2);
2046 2047 2048 2049 2050 2051 2052 2053 2054
	ret = flex_array_alloc(len, cnt, flags);
	if (!ret)
		return NULL;
	/* always prealloc all elements, so no locking is required */
	if (flex_array_prealloc(ret, 0, cnt, flags)) {
		flex_array_free(ret);
		return NULL;
	}
	return ret;
2055 2056
}

2057
static int resize_stripes(struct r5conf *conf, int newsize)
2058 2059 2060 2061 2062 2063 2064
{
	/* Make all the stripes able to hold 'newsize' devices.
	 * New slots in each stripe get 'page' set to a new page.
	 *
	 * This happens in stages:
	 * 1/ create a new kmem_cache and allocate the required number of
	 *    stripe_heads.
M
Masanari Iida 已提交
2065
	 * 2/ gather all the old stripe_heads and transfer the pages across
2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084
	 *    to the new stripe_heads.  This will have the side effect of
	 *    freezing the array as once all stripe_heads have been collected,
	 *    no IO will be possible.  Old stripe heads are freed once their
	 *    pages have been transferred over, and the old kmem_cache is
	 *    freed when all stripes are done.
	 * 3/ reallocate conf->disks to be suitable bigger.  If this fails,
	 *    we simple return a failre status - no need to clean anything up.
	 * 4/ allocate new pages for the new slots in the new stripe_heads.
	 *    If this fails, we don't bother trying the shrink the
	 *    stripe_heads down again, we just leave them as they are.
	 *    As each stripe_head is processed the new one is released into
	 *    active service.
	 *
	 * Once step2 is started, we cannot afford to wait for a write,
	 * so we use GFP_NOIO allocations.
	 */
	struct stripe_head *osh, *nsh;
	LIST_HEAD(newstripes);
	struct disk_info *ndisks;
2085
	unsigned long cpu;
2086
	int err;
2087
	struct kmem_cache *sc;
2088
	int i;
2089
	int hash, cnt;
2090 2091 2092 2093

	if (newsize <= conf->pool_size)
		return 0; /* never bother to shrink */

2094 2095 2096
	err = md_allow_write(conf->mddev);
	if (err)
		return err;
2097

2098 2099 2100
	/* Step 1 */
	sc = kmem_cache_create(conf->cache_name[1-conf->active_name],
			       sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev),
2101
			       0, 0, NULL);
2102 2103 2104 2105
	if (!sc)
		return -ENOMEM;

	for (i = conf->max_nr_stripes; i; i--) {
N
Namhyung Kim 已提交
2106
		nsh = kmem_cache_zalloc(sc, GFP_KERNEL);
2107 2108 2109 2110
		if (!nsh)
			break;

		nsh->raid_conf = conf;
2111
		spin_lock_init(&nsh->stripe_lock);
2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128

		list_add(&nsh->lru, &newstripes);
	}
	if (i) {
		/* didn't get enough, give up */
		while (!list_empty(&newstripes)) {
			nsh = list_entry(newstripes.next, struct stripe_head, lru);
			list_del(&nsh->lru);
			kmem_cache_free(sc, nsh);
		}
		kmem_cache_destroy(sc);
		return -ENOMEM;
	}
	/* Step 2 - Must use GFP_NOIO now.
	 * OK, we have enough stripes, start collecting inactive
	 * stripes and copying them over
	 */
2129 2130
	hash = 0;
	cnt = 0;
2131
	list_for_each_entry(nsh, &newstripes, lru) {
2132 2133 2134 2135 2136 2137 2138
		lock_device_hash_lock(conf, hash);
		wait_event_cmd(conf->wait_for_stripe,
				    !list_empty(conf->inactive_list + hash),
				    unlock_device_hash_lock(conf, hash),
				    lock_device_hash_lock(conf, hash));
		osh = get_free_stripe(conf, hash);
		unlock_device_hash_lock(conf, hash);
2139
		atomic_set(&nsh->count, 1);
2140
		for(i=0; i<conf->pool_size; i++) {
2141
			nsh->dev[i].page = osh->dev[i].page;
2142 2143
			nsh->dev[i].orig_page = osh->dev[i].page;
		}
2144 2145
		for( ; i<newsize; i++)
			nsh->dev[i].page = NULL;
2146
		nsh->hash_lock_index = hash;
2147
		kmem_cache_free(conf->slab_cache, osh);
2148 2149 2150 2151 2152 2153
		cnt++;
		if (cnt >= conf->max_nr_stripes / NR_STRIPE_HASH_LOCKS +
		    !!((conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS) > hash)) {
			hash++;
			cnt = 0;
		}
2154 2155 2156 2157 2158 2159
	}
	kmem_cache_destroy(conf->slab_cache);

	/* Step 3.
	 * At this point, we are holding all the stripes so the array
	 * is completely stalled, so now is a good time to resize
2160
	 * conf->disks and the scribble region
2161 2162 2163 2164 2165 2166 2167 2168 2169 2170
	 */
	ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO);
	if (ndisks) {
		for (i=0; i<conf->raid_disks; i++)
			ndisks[i] = conf->disks[i];
		kfree(conf->disks);
		conf->disks = ndisks;
	} else
		err = -ENOMEM;

2171 2172 2173
	get_online_cpus();
	for_each_present_cpu(cpu) {
		struct raid5_percpu *percpu;
2174
		struct flex_array *scribble;
2175 2176

		percpu = per_cpu_ptr(conf->percpu, cpu);
2177 2178
		scribble = scribble_alloc(newsize, conf->chunk_sectors /
			STRIPE_SECTORS, GFP_NOIO);
2179 2180

		if (scribble) {
2181
			flex_array_free(percpu->scribble);
2182 2183 2184 2185 2186 2187 2188 2189
			percpu->scribble = scribble;
		} else {
			err = -ENOMEM;
			break;
		}
	}
	put_online_cpus();

2190 2191 2192 2193
	/* Step 4, return new stripes to service */
	while(!list_empty(&newstripes)) {
		nsh = list_entry(newstripes.next, struct stripe_head, lru);
		list_del_init(&nsh->lru);
2194

2195 2196 2197 2198
		for (i=conf->raid_disks; i < newsize; i++)
			if (nsh->dev[i].page == NULL) {
				struct page *p = alloc_page(GFP_NOIO);
				nsh->dev[i].page = p;
2199
				nsh->dev[i].orig_page = p;
2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211
				if (!p)
					err = -ENOMEM;
			}
		release_stripe(nsh);
	}
	/* critical section pass, GFP_NOIO no longer needed */

	conf->slab_cache = sc;
	conf->active_name = 1-conf->active_name;
	conf->pool_size = newsize;
	return err;
}
L
Linus Torvalds 已提交
2212

2213
static int drop_one_stripe(struct r5conf *conf, int hash)
L
Linus Torvalds 已提交
2214 2215 2216
{
	struct stripe_head *sh;

2217 2218 2219
	spin_lock_irq(conf->hash_locks + hash);
	sh = get_free_stripe(conf, hash);
	spin_unlock_irq(conf->hash_locks + hash);
2220 2221
	if (!sh)
		return 0;
2222
	BUG_ON(atomic_read(&sh->count));
2223
	shrink_buffers(sh);
2224 2225 2226 2227 2228
	kmem_cache_free(conf->slab_cache, sh);
	atomic_dec(&conf->active_stripes);
	return 1;
}

2229
static void shrink_stripes(struct r5conf *conf)
2230
{
2231 2232 2233 2234
	int hash;
	for (hash = 0; hash < NR_STRIPE_HASH_LOCKS; hash++)
		while (drop_one_stripe(conf, hash))
			;
2235

N
NeilBrown 已提交
2236 2237
	if (conf->slab_cache)
		kmem_cache_destroy(conf->slab_cache);
L
Linus Torvalds 已提交
2238 2239 2240
	conf->slab_cache = NULL;
}

2241
static void raid5_end_read_request(struct bio * bi, int error)
L
Linus Torvalds 已提交
2242
{
2243
	struct stripe_head *sh = bi->bi_private;
2244
	struct r5conf *conf = sh->raid_conf;
2245
	int disks = sh->disks, i;
L
Linus Torvalds 已提交
2246
	int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
2247
	char b[BDEVNAME_SIZE];
2248
	struct md_rdev *rdev = NULL;
2249
	sector_t s;
L
Linus Torvalds 已提交
2250 2251 2252 2253 2254

	for (i=0 ; i<disks; i++)
		if (bi == &sh->dev[i].req)
			break;

2255 2256
	pr_debug("end_read_request %llu/%d, count: %d, uptodate %d.\n",
		(unsigned long long)sh->sector, i, atomic_read(&sh->count),
L
Linus Torvalds 已提交
2257 2258 2259
		uptodate);
	if (i == disks) {
		BUG();
2260
		return;
L
Linus Torvalds 已提交
2261
	}
2262
	if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
2263 2264 2265 2266 2267
		/* If replacement finished while this request was outstanding,
		 * 'replacement' might be NULL already.
		 * In that case it moved down to 'rdev'.
		 * rdev is not removed until all requests are finished.
		 */
2268
		rdev = conf->disks[i].replacement;
2269
	if (!rdev)
2270
		rdev = conf->disks[i].rdev;
L
Linus Torvalds 已提交
2271

2272 2273 2274 2275
	if (use_new_offset(conf, sh))
		s = sh->sector + rdev->new_data_offset;
	else
		s = sh->sector + rdev->data_offset;
L
Linus Torvalds 已提交
2276 2277
	if (uptodate) {
		set_bit(R5_UPTODATE, &sh->dev[i].flags);
2278
		if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
2279 2280 2281 2282
			/* Note that this cannot happen on a
			 * replacement device.  We just fail those on
			 * any error
			 */
2283 2284 2285 2286 2287
			printk_ratelimited(
				KERN_INFO
				"md/raid:%s: read error corrected"
				" (%lu sectors at %llu on %s)\n",
				mdname(conf->mddev), STRIPE_SECTORS,
2288
				(unsigned long long)s,
2289
				bdevname(rdev->bdev, b));
2290
			atomic_add(STRIPE_SECTORS, &rdev->corrected_errors);
2291 2292
			clear_bit(R5_ReadError, &sh->dev[i].flags);
			clear_bit(R5_ReWrite, &sh->dev[i].flags);
2293 2294 2295
		} else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
			clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);

2296 2297
		if (atomic_read(&rdev->read_errors))
			atomic_set(&rdev->read_errors, 0);
L
Linus Torvalds 已提交
2298
	} else {
2299
		const char *bdn = bdevname(rdev->bdev, b);
2300
		int retry = 0;
2301
		int set_bad = 0;
2302

L
Linus Torvalds 已提交
2303
		clear_bit(R5_UPTODATE, &sh->dev[i].flags);
2304
		atomic_inc(&rdev->read_errors);
2305 2306 2307 2308 2309 2310
		if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
			printk_ratelimited(
				KERN_WARNING
				"md/raid:%s: read error on replacement device "
				"(sector %llu on %s).\n",
				mdname(conf->mddev),
2311
				(unsigned long long)s,
2312
				bdn);
2313 2314
		else if (conf->mddev->degraded >= conf->max_degraded) {
			set_bad = 1;
2315 2316 2317 2318 2319
			printk_ratelimited(
				KERN_WARNING
				"md/raid:%s: read error not correctable "
				"(sector %llu on %s).\n",
				mdname(conf->mddev),
2320
				(unsigned long long)s,
2321
				bdn);
2322
		} else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) {
2323
			/* Oh, no!!! */
2324
			set_bad = 1;
2325 2326 2327 2328 2329
			printk_ratelimited(
				KERN_WARNING
				"md/raid:%s: read error NOT corrected!! "
				"(sector %llu on %s).\n",
				mdname(conf->mddev),
2330
				(unsigned long long)s,
2331
				bdn);
2332
		} else if (atomic_read(&rdev->read_errors)
2333
			 > conf->max_nr_stripes)
N
NeilBrown 已提交
2334
			printk(KERN_WARNING
2335
			       "md/raid:%s: Too many read errors, failing device %s.\n",
2336
			       mdname(conf->mddev), bdn);
2337 2338
		else
			retry = 1;
2339 2340 2341
		if (set_bad && test_bit(In_sync, &rdev->flags)
		    && !test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
			retry = 1;
2342
		if (retry)
2343 2344 2345 2346 2347
			if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) {
				set_bit(R5_ReadError, &sh->dev[i].flags);
				clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);
			} else
				set_bit(R5_ReadNoMerge, &sh->dev[i].flags);
2348
		else {
2349 2350
			clear_bit(R5_ReadError, &sh->dev[i].flags);
			clear_bit(R5_ReWrite, &sh->dev[i].flags);
2351 2352 2353 2354 2355
			if (!(set_bad
			      && test_bit(In_sync, &rdev->flags)
			      && rdev_set_badblocks(
				      rdev, sh->sector, STRIPE_SECTORS, 0)))
				md_error(conf->mddev, rdev);
2356
		}
L
Linus Torvalds 已提交
2357
	}
2358
	rdev_dec_pending(rdev, conf->mddev);
L
Linus Torvalds 已提交
2359 2360 2361 2362 2363
	clear_bit(R5_LOCKED, &sh->dev[i].flags);
	set_bit(STRIPE_HANDLE, &sh->state);
	release_stripe(sh);
}

2364
static void raid5_end_write_request(struct bio *bi, int error)
L
Linus Torvalds 已提交
2365
{
2366
	struct stripe_head *sh = bi->bi_private;
2367
	struct r5conf *conf = sh->raid_conf;
2368
	int disks = sh->disks, i;
2369
	struct md_rdev *uninitialized_var(rdev);
L
Linus Torvalds 已提交
2370
	int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
2371 2372
	sector_t first_bad;
	int bad_sectors;
2373
	int replacement = 0;
L
Linus Torvalds 已提交
2374

2375 2376 2377
	for (i = 0 ; i < disks; i++) {
		if (bi == &sh->dev[i].req) {
			rdev = conf->disks[i].rdev;
L
Linus Torvalds 已提交
2378
			break;
2379 2380 2381
		}
		if (bi == &sh->dev[i].rreq) {
			rdev = conf->disks[i].replacement;
2382 2383 2384 2385 2386 2387 2388 2389
			if (rdev)
				replacement = 1;
			else
				/* rdev was removed and 'replacement'
				 * replaced it.  rdev is not removed
				 * until all requests are finished.
				 */
				rdev = conf->disks[i].rdev;
2390 2391 2392
			break;
		}
	}
2393
	pr_debug("end_write_request %llu/%d, count %d, uptodate: %d.\n",
L
Linus Torvalds 已提交
2394 2395 2396 2397
		(unsigned long long)sh->sector, i, atomic_read(&sh->count),
		uptodate);
	if (i == disks) {
		BUG();
2398
		return;
L
Linus Torvalds 已提交
2399 2400
	}

2401 2402 2403 2404 2405 2406 2407 2408 2409
	if (replacement) {
		if (!uptodate)
			md_error(conf->mddev, rdev);
		else if (is_badblock(rdev, sh->sector,
				     STRIPE_SECTORS,
				     &first_bad, &bad_sectors))
			set_bit(R5_MadeGoodRepl, &sh->dev[i].flags);
	} else {
		if (!uptodate) {
2410
			set_bit(STRIPE_DEGRADED, &sh->state);
2411 2412
			set_bit(WriteErrorSeen, &rdev->flags);
			set_bit(R5_WriteError, &sh->dev[i].flags);
2413 2414 2415
			if (!test_and_set_bit(WantReplacement, &rdev->flags))
				set_bit(MD_RECOVERY_NEEDED,
					&rdev->mddev->recovery);
2416 2417
		} else if (is_badblock(rdev, sh->sector,
				       STRIPE_SECTORS,
2418
				       &first_bad, &bad_sectors)) {
2419
			set_bit(R5_MadeGood, &sh->dev[i].flags);
2420 2421 2422 2423 2424 2425 2426
			if (test_bit(R5_ReadError, &sh->dev[i].flags))
				/* That was a successful write so make
				 * sure it looks like we already did
				 * a re-write.
				 */
				set_bit(R5_ReWrite, &sh->dev[i].flags);
		}
2427 2428
	}
	rdev_dec_pending(rdev, conf->mddev);
L
Linus Torvalds 已提交
2429

2430 2431 2432
	if (sh->batch_head && !uptodate)
		set_bit(STRIPE_BATCH_ERR, &sh->batch_head->state);

2433 2434
	if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags))
		clear_bit(R5_LOCKED, &sh->dev[i].flags);
L
Linus Torvalds 已提交
2435
	set_bit(STRIPE_HANDLE, &sh->state);
2436
	release_stripe(sh);
2437 2438 2439

	if (sh->batch_head && sh != sh->batch_head)
		release_stripe(sh->batch_head);
L
Linus Torvalds 已提交
2440 2441
}

2442
static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous);
2443

2444
static void raid5_build_block(struct stripe_head *sh, int i, int previous)
L
Linus Torvalds 已提交
2445 2446 2447 2448 2449
{
	struct r5dev *dev = &sh->dev[i];

	bio_init(&dev->req);
	dev->req.bi_io_vec = &dev->vec;
2450
	dev->req.bi_max_vecs = 1;
L
Linus Torvalds 已提交
2451 2452
	dev->req.bi_private = sh;

2453 2454
	bio_init(&dev->rreq);
	dev->rreq.bi_io_vec = &dev->rvec;
2455
	dev->rreq.bi_max_vecs = 1;
2456 2457
	dev->rreq.bi_private = sh;

L
Linus Torvalds 已提交
2458
	dev->flags = 0;
2459
	dev->sector = compute_blocknr(sh, i, previous);
L
Linus Torvalds 已提交
2460 2461
}

2462
static void error(struct mddev *mddev, struct md_rdev *rdev)
L
Linus Torvalds 已提交
2463 2464
{
	char b[BDEVNAME_SIZE];
2465
	struct r5conf *conf = mddev->private;
2466
	unsigned long flags;
2467
	pr_debug("raid456: error called\n");
L
Linus Torvalds 已提交
2468

2469 2470 2471 2472 2473 2474
	spin_lock_irqsave(&conf->device_lock, flags);
	clear_bit(In_sync, &rdev->flags);
	mddev->degraded = calc_degraded(conf);
	spin_unlock_irqrestore(&conf->device_lock, flags);
	set_bit(MD_RECOVERY_INTR, &mddev->recovery);

2475
	set_bit(Blocked, &rdev->flags);
2476 2477 2478 2479 2480 2481 2482 2483 2484
	set_bit(Faulty, &rdev->flags);
	set_bit(MD_CHANGE_DEVS, &mddev->flags);
	printk(KERN_ALERT
	       "md/raid:%s: Disk failure on %s, disabling device.\n"
	       "md/raid:%s: Operation continuing on %d devices.\n",
	       mdname(mddev),
	       bdevname(rdev->bdev, b),
	       mdname(mddev),
	       conf->raid_disks - mddev->degraded);
2485
}
L
Linus Torvalds 已提交
2486 2487 2488 2489 2490

/*
 * Input: a 'big' sector number,
 * Output: index of the data and parity disk, and the sector # in them.
 */
2491
static sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector,
2492 2493
				     int previous, int *dd_idx,
				     struct stripe_head *sh)
L
Linus Torvalds 已提交
2494
{
N
NeilBrown 已提交
2495
	sector_t stripe, stripe2;
2496
	sector_t chunk_number;
L
Linus Torvalds 已提交
2497
	unsigned int chunk_offset;
2498
	int pd_idx, qd_idx;
2499
	int ddf_layout = 0;
L
Linus Torvalds 已提交
2500
	sector_t new_sector;
2501 2502
	int algorithm = previous ? conf->prev_algo
				 : conf->algorithm;
2503 2504
	int sectors_per_chunk = previous ? conf->prev_chunk_sectors
					 : conf->chunk_sectors;
2505 2506 2507
	int raid_disks = previous ? conf->previous_raid_disks
				  : conf->raid_disks;
	int data_disks = raid_disks - conf->max_degraded;
L
Linus Torvalds 已提交
2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519

	/* First compute the information on this sector */

	/*
	 * Compute the chunk number and the sector offset inside the chunk
	 */
	chunk_offset = sector_div(r_sector, sectors_per_chunk);
	chunk_number = r_sector;

	/*
	 * Compute the stripe number
	 */
2520 2521
	stripe = chunk_number;
	*dd_idx = sector_div(stripe, data_disks);
N
NeilBrown 已提交
2522
	stripe2 = stripe;
L
Linus Torvalds 已提交
2523 2524 2525
	/*
	 * Select the parity disk based on the user selected algorithm.
	 */
2526
	pd_idx = qd_idx = -1;
2527 2528
	switch(conf->level) {
	case 4:
2529
		pd_idx = data_disks;
2530 2531
		break;
	case 5:
2532
		switch (algorithm) {
L
Linus Torvalds 已提交
2533
		case ALGORITHM_LEFT_ASYMMETRIC:
N
NeilBrown 已提交
2534
			pd_idx = data_disks - sector_div(stripe2, raid_disks);
2535
			if (*dd_idx >= pd_idx)
L
Linus Torvalds 已提交
2536 2537 2538
				(*dd_idx)++;
			break;
		case ALGORITHM_RIGHT_ASYMMETRIC:
N
NeilBrown 已提交
2539
			pd_idx = sector_div(stripe2, raid_disks);
2540
			if (*dd_idx >= pd_idx)
L
Linus Torvalds 已提交
2541 2542 2543
				(*dd_idx)++;
			break;
		case ALGORITHM_LEFT_SYMMETRIC:
N
NeilBrown 已提交
2544
			pd_idx = data_disks - sector_div(stripe2, raid_disks);
2545
			*dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
L
Linus Torvalds 已提交
2546 2547
			break;
		case ALGORITHM_RIGHT_SYMMETRIC:
N
NeilBrown 已提交
2548
			pd_idx = sector_div(stripe2, raid_disks);
2549
			*dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
L
Linus Torvalds 已提交
2550
			break;
2551 2552 2553 2554 2555 2556 2557
		case ALGORITHM_PARITY_0:
			pd_idx = 0;
			(*dd_idx)++;
			break;
		case ALGORITHM_PARITY_N:
			pd_idx = data_disks;
			break;
L
Linus Torvalds 已提交
2558
		default:
2559
			BUG();
2560 2561 2562 2563
		}
		break;
	case 6:

2564
		switch (algorithm) {
2565
		case ALGORITHM_LEFT_ASYMMETRIC:
N
NeilBrown 已提交
2566
			pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
2567 2568
			qd_idx = pd_idx + 1;
			if (pd_idx == raid_disks-1) {
2569
				(*dd_idx)++;	/* Q D D D P */
2570 2571
				qd_idx = 0;
			} else if (*dd_idx >= pd_idx)
2572 2573 2574
				(*dd_idx) += 2; /* D D P Q D */
			break;
		case ALGORITHM_RIGHT_ASYMMETRIC:
N
NeilBrown 已提交
2575
			pd_idx = sector_div(stripe2, raid_disks);
2576 2577
			qd_idx = pd_idx + 1;
			if (pd_idx == raid_disks-1) {
2578
				(*dd_idx)++;	/* Q D D D P */
2579 2580
				qd_idx = 0;
			} else if (*dd_idx >= pd_idx)
2581 2582 2583
				(*dd_idx) += 2; /* D D P Q D */
			break;
		case ALGORITHM_LEFT_SYMMETRIC:
N
NeilBrown 已提交
2584
			pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
2585 2586
			qd_idx = (pd_idx + 1) % raid_disks;
			*dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
2587 2588
			break;
		case ALGORITHM_RIGHT_SYMMETRIC:
N
NeilBrown 已提交
2589
			pd_idx = sector_div(stripe2, raid_disks);
2590 2591
			qd_idx = (pd_idx + 1) % raid_disks;
			*dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
2592
			break;
2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607

		case ALGORITHM_PARITY_0:
			pd_idx = 0;
			qd_idx = 1;
			(*dd_idx) += 2;
			break;
		case ALGORITHM_PARITY_N:
			pd_idx = data_disks;
			qd_idx = data_disks + 1;
			break;

		case ALGORITHM_ROTATING_ZERO_RESTART:
			/* Exactly the same as RIGHT_ASYMMETRIC, but or
			 * of blocks for computing Q is different.
			 */
N
NeilBrown 已提交
2608
			pd_idx = sector_div(stripe2, raid_disks);
2609 2610 2611 2612 2613 2614
			qd_idx = pd_idx + 1;
			if (pd_idx == raid_disks-1) {
				(*dd_idx)++;	/* Q D D D P */
				qd_idx = 0;
			} else if (*dd_idx >= pd_idx)
				(*dd_idx) += 2; /* D D P Q D */
2615
			ddf_layout = 1;
2616 2617 2618 2619 2620 2621 2622
			break;

		case ALGORITHM_ROTATING_N_RESTART:
			/* Same a left_asymmetric, by first stripe is
			 * D D D P Q  rather than
			 * Q D D D P
			 */
N
NeilBrown 已提交
2623 2624
			stripe2 += 1;
			pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
2625 2626 2627 2628 2629 2630
			qd_idx = pd_idx + 1;
			if (pd_idx == raid_disks-1) {
				(*dd_idx)++;	/* Q D D D P */
				qd_idx = 0;
			} else if (*dd_idx >= pd_idx)
				(*dd_idx) += 2; /* D D P Q D */
2631
			ddf_layout = 1;
2632 2633 2634 2635
			break;

		case ALGORITHM_ROTATING_N_CONTINUE:
			/* Same as left_symmetric but Q is before P */
N
NeilBrown 已提交
2636
			pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
2637 2638
			qd_idx = (pd_idx + raid_disks - 1) % raid_disks;
			*dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
2639
			ddf_layout = 1;
2640 2641 2642 2643
			break;

		case ALGORITHM_LEFT_ASYMMETRIC_6:
			/* RAID5 left_asymmetric, with Q on last device */
N
NeilBrown 已提交
2644
			pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
2645 2646 2647 2648 2649 2650
			if (*dd_idx >= pd_idx)
				(*dd_idx)++;
			qd_idx = raid_disks - 1;
			break;

		case ALGORITHM_RIGHT_ASYMMETRIC_6:
N
NeilBrown 已提交
2651
			pd_idx = sector_div(stripe2, raid_disks-1);
2652 2653 2654 2655 2656 2657
			if (*dd_idx >= pd_idx)
				(*dd_idx)++;
			qd_idx = raid_disks - 1;
			break;

		case ALGORITHM_LEFT_SYMMETRIC_6:
N
NeilBrown 已提交
2658
			pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
2659 2660 2661 2662 2663
			*dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
			qd_idx = raid_disks - 1;
			break;

		case ALGORITHM_RIGHT_SYMMETRIC_6:
N
NeilBrown 已提交
2664
			pd_idx = sector_div(stripe2, raid_disks-1);
2665 2666 2667 2668 2669 2670 2671 2672 2673 2674
			*dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
			qd_idx = raid_disks - 1;
			break;

		case ALGORITHM_PARITY_0_6:
			pd_idx = 0;
			(*dd_idx)++;
			qd_idx = raid_disks - 1;
			break;

2675
		default:
2676
			BUG();
2677 2678
		}
		break;
L
Linus Torvalds 已提交
2679 2680
	}

2681 2682 2683
	if (sh) {
		sh->pd_idx = pd_idx;
		sh->qd_idx = qd_idx;
2684
		sh->ddf_layout = ddf_layout;
2685
	}
L
Linus Torvalds 已提交
2686 2687 2688 2689 2690 2691 2692
	/*
	 * Finally, compute the new sector number
	 */
	new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset;
	return new_sector;
}

2693
static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous)
L
Linus Torvalds 已提交
2694
{
2695
	struct r5conf *conf = sh->raid_conf;
2696 2697
	int raid_disks = sh->disks;
	int data_disks = raid_disks - conf->max_degraded;
L
Linus Torvalds 已提交
2698
	sector_t new_sector = sh->sector, check;
2699 2700
	int sectors_per_chunk = previous ? conf->prev_chunk_sectors
					 : conf->chunk_sectors;
2701 2702
	int algorithm = previous ? conf->prev_algo
				 : conf->algorithm;
L
Linus Torvalds 已提交
2703 2704
	sector_t stripe;
	int chunk_offset;
2705 2706
	sector_t chunk_number;
	int dummy1, dd_idx = i;
L
Linus Torvalds 已提交
2707
	sector_t r_sector;
2708
	struct stripe_head sh2;
L
Linus Torvalds 已提交
2709 2710 2711 2712

	chunk_offset = sector_div(new_sector, sectors_per_chunk);
	stripe = new_sector;

2713 2714 2715 2716 2717
	if (i == sh->pd_idx)
		return 0;
	switch(conf->level) {
	case 4: break;
	case 5:
2718
		switch (algorithm) {
L
Linus Torvalds 已提交
2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729
		case ALGORITHM_LEFT_ASYMMETRIC:
		case ALGORITHM_RIGHT_ASYMMETRIC:
			if (i > sh->pd_idx)
				i--;
			break;
		case ALGORITHM_LEFT_SYMMETRIC:
		case ALGORITHM_RIGHT_SYMMETRIC:
			if (i < sh->pd_idx)
				i += raid_disks;
			i -= (sh->pd_idx + 1);
			break;
2730 2731 2732 2733 2734
		case ALGORITHM_PARITY_0:
			i -= 1;
			break;
		case ALGORITHM_PARITY_N:
			break;
L
Linus Torvalds 已提交
2735
		default:
2736
			BUG();
2737 2738 2739
		}
		break;
	case 6:
2740
		if (i == sh->qd_idx)
2741
			return 0; /* It is the Q disk */
2742
		switch (algorithm) {
2743 2744
		case ALGORITHM_LEFT_ASYMMETRIC:
		case ALGORITHM_RIGHT_ASYMMETRIC:
2745 2746 2747 2748
		case ALGORITHM_ROTATING_ZERO_RESTART:
		case ALGORITHM_ROTATING_N_RESTART:
			if (sh->pd_idx == raid_disks-1)
				i--;	/* Q D D D P */
2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762
			else if (i > sh->pd_idx)
				i -= 2; /* D D P Q D */
			break;
		case ALGORITHM_LEFT_SYMMETRIC:
		case ALGORITHM_RIGHT_SYMMETRIC:
			if (sh->pd_idx == raid_disks-1)
				i--; /* Q D D D P */
			else {
				/* D D P Q D */
				if (i < sh->pd_idx)
					i += raid_disks;
				i -= (sh->pd_idx + 2);
			}
			break;
2763 2764 2765 2766 2767 2768
		case ALGORITHM_PARITY_0:
			i -= 2;
			break;
		case ALGORITHM_PARITY_N:
			break;
		case ALGORITHM_ROTATING_N_CONTINUE:
2769
			/* Like left_symmetric, but P is before Q */
2770 2771
			if (sh->pd_idx == 0)
				i--;	/* P D D D Q */
2772 2773 2774 2775 2776 2777
			else {
				/* D D Q P D */
				if (i < sh->pd_idx)
					i += raid_disks;
				i -= (sh->pd_idx + 1);
			}
2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792
			break;
		case ALGORITHM_LEFT_ASYMMETRIC_6:
		case ALGORITHM_RIGHT_ASYMMETRIC_6:
			if (i > sh->pd_idx)
				i--;
			break;
		case ALGORITHM_LEFT_SYMMETRIC_6:
		case ALGORITHM_RIGHT_SYMMETRIC_6:
			if (i < sh->pd_idx)
				i += data_disks + 1;
			i -= (sh->pd_idx + 1);
			break;
		case ALGORITHM_PARITY_0_6:
			i -= 1;
			break;
2793
		default:
2794
			BUG();
2795 2796
		}
		break;
L
Linus Torvalds 已提交
2797 2798 2799
	}

	chunk_number = stripe * data_disks + i;
2800
	r_sector = chunk_number * sectors_per_chunk + chunk_offset;
L
Linus Torvalds 已提交
2801

2802
	check = raid5_compute_sector(conf, r_sector,
2803
				     previous, &dummy1, &sh2);
2804 2805
	if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx
		|| sh2.qd_idx != sh->qd_idx) {
2806 2807
		printk(KERN_ERR "md/raid:%s: compute_blocknr: map not correct\n",
		       mdname(conf->mddev));
L
Linus Torvalds 已提交
2808 2809 2810 2811 2812
		return 0;
	}
	return r_sector;
}

2813
static void
2814
schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s,
2815
			 int rcw, int expand)
2816
{
2817
	int i, pd_idx = sh->pd_idx, qd_idx = sh->qd_idx, disks = sh->disks;
2818
	struct r5conf *conf = sh->raid_conf;
2819
	int level = conf->level;
2820 2821 2822 2823 2824 2825 2826 2827

	if (rcw) {

		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];

			if (dev->towrite) {
				set_bit(R5_LOCKED, &dev->flags);
2828
				set_bit(R5_Wantdrain, &dev->flags);
2829 2830
				if (!expand)
					clear_bit(R5_UPTODATE, &dev->flags);
2831
				s->locked++;
2832 2833
			}
		}
2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848
		/* if we are not expanding this is a proper write request, and
		 * there will be bios with new data to be drained into the
		 * stripe cache
		 */
		if (!expand) {
			if (!s->locked)
				/* False alarm, nothing to do */
				return;
			sh->reconstruct_state = reconstruct_state_drain_run;
			set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
		} else
			sh->reconstruct_state = reconstruct_state_run;

		set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);

2849
		if (s->locked + conf->max_degraded == disks)
2850
			if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state))
2851
				atomic_inc(&conf->pending_full_writes);
2852 2853 2854
	} else {
		BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) ||
			test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags)));
2855 2856 2857
		BUG_ON(level == 6 &&
			(!(test_bit(R5_UPTODATE, &sh->dev[qd_idx].flags) ||
			   test_bit(R5_Wantcompute, &sh->dev[qd_idx].flags))));
2858 2859 2860

		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
2861
			if (i == pd_idx || i == qd_idx)
2862 2863 2864 2865
				continue;

			if (dev->towrite &&
			    (test_bit(R5_UPTODATE, &dev->flags) ||
2866 2867
			     test_bit(R5_Wantcompute, &dev->flags))) {
				set_bit(R5_Wantdrain, &dev->flags);
2868 2869
				set_bit(R5_LOCKED, &dev->flags);
				clear_bit(R5_UPTODATE, &dev->flags);
2870
				s->locked++;
2871 2872
			}
		}
2873 2874 2875 2876 2877 2878 2879
		if (!s->locked)
			/* False alarm - nothing to do */
			return;
		sh->reconstruct_state = reconstruct_state_prexor_drain_run;
		set_bit(STRIPE_OP_PREXOR, &s->ops_request);
		set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
		set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
2880 2881
	}

2882
	/* keep the parity disk(s) locked while asynchronous operations
2883 2884 2885 2886
	 * are in flight
	 */
	set_bit(R5_LOCKED, &sh->dev[pd_idx].flags);
	clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
2887
	s->locked++;
2888

2889 2890 2891 2892 2893 2894 2895 2896 2897
	if (level == 6) {
		int qd_idx = sh->qd_idx;
		struct r5dev *dev = &sh->dev[qd_idx];

		set_bit(R5_LOCKED, &dev->flags);
		clear_bit(R5_UPTODATE, &dev->flags);
		s->locked++;
	}

2898
	pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n",
2899
		__func__, (unsigned long long)sh->sector,
2900
		s->locked, s->ops_request);
2901
}
2902

L
Linus Torvalds 已提交
2903 2904
/*
 * Each stripe/dev can have one or more bion attached.
2905
 * toread/towrite point to the first in a chain.
L
Linus Torvalds 已提交
2906 2907
 * The bi_next chain must be in order.
 */
2908 2909
static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx,
			  int forwrite, int previous)
L
Linus Torvalds 已提交
2910 2911
{
	struct bio **bip;
2912
	struct r5conf *conf = sh->raid_conf;
2913
	int firstwrite=0;
L
Linus Torvalds 已提交
2914

2915
	pr_debug("adding bi b#%llu to stripe s#%llu\n",
2916
		(unsigned long long)bi->bi_iter.bi_sector,
L
Linus Torvalds 已提交
2917 2918
		(unsigned long long)sh->sector);

S
Shaohua Li 已提交
2919 2920 2921 2922 2923 2924 2925 2926 2927
	/*
	 * If several bio share a stripe. The bio bi_phys_segments acts as a
	 * reference count to avoid race. The reference count should already be
	 * increased before this function is called (for example, in
	 * make_request()), so other bio sharing this stripe will not free the
	 * stripe. If a stripe is owned by one stripe, the stripe lock will
	 * protect it.
	 */
	spin_lock_irq(&sh->stripe_lock);
2928 2929 2930
	/* Don't allow new IO added to stripes in batch list */
	if (sh->batch_head)
		goto overlap;
2931
	if (forwrite) {
L
Linus Torvalds 已提交
2932
		bip = &sh->dev[dd_idx].towrite;
2933
		if (*bip == NULL)
2934 2935
			firstwrite = 1;
	} else
L
Linus Torvalds 已提交
2936
		bip = &sh->dev[dd_idx].toread;
2937 2938
	while (*bip && (*bip)->bi_iter.bi_sector < bi->bi_iter.bi_sector) {
		if (bio_end_sector(*bip) > bi->bi_iter.bi_sector)
L
Linus Torvalds 已提交
2939 2940 2941
			goto overlap;
		bip = & (*bip)->bi_next;
	}
2942
	if (*bip && (*bip)->bi_iter.bi_sector < bio_end_sector(bi))
L
Linus Torvalds 已提交
2943 2944
		goto overlap;

2945 2946 2947
	if (!forwrite || previous)
		clear_bit(STRIPE_BATCH_READY, &sh->state);

2948
	BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next);
L
Linus Torvalds 已提交
2949 2950 2951
	if (*bip)
		bi->bi_next = *bip;
	*bip = bi;
2952
	raid5_inc_bi_active_stripes(bi);
2953

L
Linus Torvalds 已提交
2954 2955 2956 2957 2958
	if (forwrite) {
		/* check if page is covered */
		sector_t sector = sh->dev[dd_idx].sector;
		for (bi=sh->dev[dd_idx].towrite;
		     sector < sh->dev[dd_idx].sector + STRIPE_SECTORS &&
2959
			     bi && bi->bi_iter.bi_sector <= sector;
L
Linus Torvalds 已提交
2960
		     bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) {
K
Kent Overstreet 已提交
2961 2962
			if (bio_end_sector(bi) >= sector)
				sector = bio_end_sector(bi);
L
Linus Torvalds 已提交
2963 2964
		}
		if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS)
2965 2966
			if (!test_and_set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags))
				sh->overwrite_disks++;
L
Linus Torvalds 已提交
2967
	}
2968 2969

	pr_debug("added bi b#%llu to stripe s#%llu, disk %d.\n",
2970
		(unsigned long long)(*bip)->bi_iter.bi_sector,
2971
		(unsigned long long)sh->sector, dd_idx);
2972
	spin_unlock_irq(&sh->stripe_lock);
2973 2974 2975 2976 2977 2978 2979

	if (conf->mddev->bitmap && firstwrite) {
		bitmap_startwrite(conf->mddev->bitmap, sh->sector,
				  STRIPE_SECTORS, 0);
		sh->bm_seq = conf->seq_flush+1;
		set_bit(STRIPE_BIT_DELAY, &sh->state);
	}
2980 2981 2982

	if (stripe_can_batch(sh))
		stripe_add_to_batch_list(conf, sh);
L
Linus Torvalds 已提交
2983 2984 2985 2986
	return 1;

 overlap:
	set_bit(R5_Overlap, &sh->dev[dd_idx].flags);
S
Shaohua Li 已提交
2987
	spin_unlock_irq(&sh->stripe_lock);
L
Linus Torvalds 已提交
2988 2989 2990
	return 0;
}

2991
static void end_reshape(struct r5conf *conf);
2992

2993
static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,
2994
			    struct stripe_head *sh)
2995
{
2996
	int sectors_per_chunk =
2997
		previous ? conf->prev_chunk_sectors : conf->chunk_sectors;
2998
	int dd_idx;
2999
	int chunk_offset = sector_div(stripe, sectors_per_chunk);
3000
	int disks = previous ? conf->previous_raid_disks : conf->raid_disks;
3001

3002 3003
	raid5_compute_sector(conf,
			     stripe * (disks - conf->max_degraded)
3004
			     *sectors_per_chunk + chunk_offset,
3005
			     previous,
3006
			     &dd_idx, sh);
3007 3008
}

3009
static void
3010
handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh,
3011 3012 3013 3014
				struct stripe_head_state *s, int disks,
				struct bio **return_bi)
{
	int i;
3015
	BUG_ON(sh->batch_head);
3016 3017 3018 3019 3020
	for (i = disks; i--; ) {
		struct bio *bi;
		int bitmap_end = 0;

		if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
3021
			struct md_rdev *rdev;
3022 3023 3024
			rcu_read_lock();
			rdev = rcu_dereference(conf->disks[i].rdev);
			if (rdev && test_bit(In_sync, &rdev->flags))
3025 3026 3027
				atomic_inc(&rdev->nr_pending);
			else
				rdev = NULL;
3028
			rcu_read_unlock();
3029 3030 3031 3032 3033 3034 3035 3036
			if (rdev) {
				if (!rdev_set_badblocks(
					    rdev,
					    sh->sector,
					    STRIPE_SECTORS, 0))
					md_error(conf->mddev, rdev);
				rdev_dec_pending(rdev, conf->mddev);
			}
3037
		}
S
Shaohua Li 已提交
3038
		spin_lock_irq(&sh->stripe_lock);
3039 3040 3041
		/* fail all writes first */
		bi = sh->dev[i].towrite;
		sh->dev[i].towrite = NULL;
3042
		sh->overwrite_disks = 0;
S
Shaohua Li 已提交
3043
		spin_unlock_irq(&sh->stripe_lock);
3044
		if (bi)
3045 3046 3047 3048 3049
			bitmap_end = 1;

		if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
			wake_up(&conf->wait_for_overlap);

3050
		while (bi && bi->bi_iter.bi_sector <
3051 3052 3053
			sh->dev[i].sector + STRIPE_SECTORS) {
			struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector);
			clear_bit(BIO_UPTODATE, &bi->bi_flags);
3054
			if (!raid5_dec_bi_active_stripes(bi)) {
3055 3056 3057 3058 3059 3060
				md_write_end(conf->mddev);
				bi->bi_next = *return_bi;
				*return_bi = bi;
			}
			bi = nextbi;
		}
3061 3062 3063 3064
		if (bitmap_end)
			bitmap_endwrite(conf->mddev->bitmap, sh->sector,
				STRIPE_SECTORS, 0, 0);
		bitmap_end = 0;
3065 3066 3067
		/* and fail all 'written' */
		bi = sh->dev[i].written;
		sh->dev[i].written = NULL;
3068 3069 3070 3071 3072
		if (test_and_clear_bit(R5_SkipCopy, &sh->dev[i].flags)) {
			WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
			sh->dev[i].page = sh->dev[i].orig_page;
		}

3073
		if (bi) bitmap_end = 1;
3074
		while (bi && bi->bi_iter.bi_sector <
3075 3076 3077
		       sh->dev[i].sector + STRIPE_SECTORS) {
			struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector);
			clear_bit(BIO_UPTODATE, &bi->bi_flags);
3078
			if (!raid5_dec_bi_active_stripes(bi)) {
3079 3080 3081 3082 3083 3084 3085
				md_write_end(conf->mddev);
				bi->bi_next = *return_bi;
				*return_bi = bi;
			}
			bi = bi2;
		}

3086 3087 3088 3089 3090 3091
		/* fail any reads if this device is non-operational and
		 * the data has not reached the cache yet.
		 */
		if (!test_bit(R5_Wantfill, &sh->dev[i].flags) &&
		    (!test_bit(R5_Insync, &sh->dev[i].flags) ||
		      test_bit(R5_ReadError, &sh->dev[i].flags))) {
3092
			spin_lock_irq(&sh->stripe_lock);
3093 3094
			bi = sh->dev[i].toread;
			sh->dev[i].toread = NULL;
3095
			spin_unlock_irq(&sh->stripe_lock);
3096 3097
			if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
				wake_up(&conf->wait_for_overlap);
3098
			while (bi && bi->bi_iter.bi_sector <
3099 3100 3101 3102
			       sh->dev[i].sector + STRIPE_SECTORS) {
				struct bio *nextbi =
					r5_next_bio(bi, sh->dev[i].sector);
				clear_bit(BIO_UPTODATE, &bi->bi_flags);
3103
				if (!raid5_dec_bi_active_stripes(bi)) {
3104 3105 3106 3107 3108 3109 3110 3111 3112
					bi->bi_next = *return_bi;
					*return_bi = bi;
				}
				bi = nextbi;
			}
		}
		if (bitmap_end)
			bitmap_endwrite(conf->mddev->bitmap, sh->sector,
					STRIPE_SECTORS, 0, 0);
3113 3114 3115 3116
		/* If we were in the middle of a write the parity block might
		 * still be locked - so just clear all R5_LOCKED flags
		 */
		clear_bit(R5_LOCKED, &sh->dev[i].flags);
3117 3118
	}

3119 3120 3121
	if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
		if (atomic_dec_and_test(&conf->pending_full_writes))
			md_wakeup_thread(conf->mddev->thread);
3122 3123
}

3124
static void
3125
handle_failed_sync(struct r5conf *conf, struct stripe_head *sh,
3126 3127 3128 3129 3130
		   struct stripe_head_state *s)
{
	int abort = 0;
	int i;

3131
	BUG_ON(sh->batch_head);
3132
	clear_bit(STRIPE_SYNCING, &sh->state);
3133 3134
	if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))
		wake_up(&conf->wait_for_overlap);
3135
	s->syncing = 0;
3136
	s->replacing = 0;
3137
	/* There is nothing more to do for sync/check/repair.
3138 3139 3140
	 * Don't even need to abort as that is handled elsewhere
	 * if needed, and not always wanted e.g. if there is a known
	 * bad block here.
3141
	 * For recover/replace we need to record a bad block on all
3142 3143
	 * non-sync devices, or abort the recovery
	 */
3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166
	if (test_bit(MD_RECOVERY_RECOVER, &conf->mddev->recovery)) {
		/* During recovery devices cannot be removed, so
		 * locking and refcounting of rdevs is not needed
		 */
		for (i = 0; i < conf->raid_disks; i++) {
			struct md_rdev *rdev = conf->disks[i].rdev;
			if (rdev
			    && !test_bit(Faulty, &rdev->flags)
			    && !test_bit(In_sync, &rdev->flags)
			    && !rdev_set_badblocks(rdev, sh->sector,
						   STRIPE_SECTORS, 0))
				abort = 1;
			rdev = conf->disks[i].replacement;
			if (rdev
			    && !test_bit(Faulty, &rdev->flags)
			    && !test_bit(In_sync, &rdev->flags)
			    && !rdev_set_badblocks(rdev, sh->sector,
						   STRIPE_SECTORS, 0))
				abort = 1;
		}
		if (abort)
			conf->recovery_disabled =
				conf->mddev->recovery_disabled;
3167
	}
3168
	md_done_sync(conf->mddev, STRIPE_SECTORS, !abort);
3169 3170
}

3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186
static int want_replace(struct stripe_head *sh, int disk_idx)
{
	struct md_rdev *rdev;
	int rv = 0;
	/* Doing recovery so rcu locking not required */
	rdev = sh->raid_conf->disks[disk_idx].replacement;
	if (rdev
	    && !test_bit(Faulty, &rdev->flags)
	    && !test_bit(In_sync, &rdev->flags)
	    && (rdev->recovery_offset <= sh->sector
		|| rdev->mddev->recovery_cp <= sh->sector))
		rv = 1;

	return rv;
}

3187
/* fetch_block - checks the given member device to see if its data needs
3188 3189 3190
 * to be read or computed to satisfy a request.
 *
 * Returns 1 when no more member devices need to be checked, otherwise returns
3191
 * 0 to tell the loop in handle_stripe_fill to continue
3192
 */
3193 3194 3195

static int need_this_block(struct stripe_head *sh, struct stripe_head_state *s,
			   int disk_idx, int disks)
3196
{
3197
	struct r5dev *dev = &sh->dev[disk_idx];
3198 3199
	struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]],
				  &sh->dev[s->failed_num[1]] };
3200
	int i;
3201

3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228

	if (test_bit(R5_LOCKED, &dev->flags) ||
	    test_bit(R5_UPTODATE, &dev->flags))
		/* No point reading this as we already have it or have
		 * decided to get it.
		 */
		return 0;

	if (dev->toread ||
	    (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)))
		/* We need this block to directly satisfy a request */
		return 1;

	if (s->syncing || s->expanding ||
	    (s->replacing && want_replace(sh, disk_idx)))
		/* When syncing, or expanding we read everything.
		 * When replacing, we need the replaced block.
		 */
		return 1;

	if ((s->failed >= 1 && fdev[0]->toread) ||
	    (s->failed >= 2 && fdev[1]->toread))
		/* If we want to read from a failed device, then
		 * we need to actually read every other device.
		 */
		return 1;

3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247
	/* Sometimes neither read-modify-write nor reconstruct-write
	 * cycles can work.  In those cases we read every block we
	 * can.  Then the parity-update is certain to have enough to
	 * work with.
	 * This can only be a problem when we need to write something,
	 * and some device has failed.  If either of those tests
	 * fail we need look no further.
	 */
	if (!s->failed || !s->to_write)
		return 0;

	if (test_bit(R5_Insync, &dev->flags) &&
	    !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
		/* Pre-reads at not permitted until after short delay
		 * to gather multiple requests.  However if this
		 * device is no Insync, the block could only be be computed
		 * and there is no need to delay that.
		 */
		return 0;
3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278

	for (i = 0; i < s->failed; i++) {
		if (fdev[i]->towrite &&
		    !test_bit(R5_UPTODATE, &fdev[i]->flags) &&
		    !test_bit(R5_OVERWRITE, &fdev[i]->flags))
			/* If we have a partial write to a failed
			 * device, then we will need to reconstruct
			 * the content of that device, so all other
			 * devices must be read.
			 */
			return 1;
	}

	/* If we are forced to do a reconstruct-write, either because
	 * the current RAID6 implementation only supports that, or
	 * or because parity cannot be trusted and we are currently
	 * recovering it, there is extra need to be careful.
	 * If one of the devices that we would need to read, because
	 * it is not being overwritten (and maybe not written at all)
	 * is missing/faulty, then we need to read everything we can.
	 */
	if (sh->raid_conf->level != 6 &&
	    sh->sector < sh->raid_conf->mddev->recovery_cp)
		/* reconstruct-write isn't being forced */
		return 0;
	for (i = 0; i < s->failed; i++) {
		if (!test_bit(R5_UPTODATE, &fdev[i]->flags) &&
		    !test_bit(R5_OVERWRITE, &fdev[i]->flags))
			return 1;
	}

3279 3280 3281 3282 3283 3284 3285 3286 3287 3288
	return 0;
}

static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s,
		       int disk_idx, int disks)
{
	struct r5dev *dev = &sh->dev[disk_idx];

	/* is the data in this block needed, and can we get it? */
	if (need_this_block(sh, s, disk_idx, disks)) {
3289 3290 3291 3292 3293 3294
		/* we would like to get this block, possibly by computing it,
		 * otherwise read it if the backing disk is insync
		 */
		BUG_ON(test_bit(R5_Wantcompute, &dev->flags));
		BUG_ON(test_bit(R5_Wantread, &dev->flags));
		if ((s->uptodate == disks - 1) &&
3295 3296
		    (s->failed && (disk_idx == s->failed_num[0] ||
				   disk_idx == s->failed_num[1]))) {
3297 3298
			/* have disk failed, and we're requested to fetch it;
			 * do compute it
3299
			 */
3300 3301 3302 3303 3304 3305 3306 3307
			pr_debug("Computing stripe %llu block %d\n",
			       (unsigned long long)sh->sector, disk_idx);
			set_bit(STRIPE_COMPUTE_RUN, &sh->state);
			set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
			set_bit(R5_Wantcompute, &dev->flags);
			sh->ops.target = disk_idx;
			sh->ops.target2 = -1; /* no 2nd target */
			s->req_compute = 1;
3308 3309 3310 3311 3312 3313
			/* Careful: from this point on 'uptodate' is in the eye
			 * of raid_run_ops which services 'compute' operations
			 * before writes. R5_Wantcompute flags a block that will
			 * be R5_UPTODATE by the time it is needed for a
			 * subsequent operation.
			 */
3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326
			s->uptodate++;
			return 1;
		} else if (s->uptodate == disks-2 && s->failed >= 2) {
			/* Computing 2-failure is *very* expensive; only
			 * do it if failed >= 2
			 */
			int other;
			for (other = disks; other--; ) {
				if (other == disk_idx)
					continue;
				if (!test_bit(R5_UPTODATE,
				      &sh->dev[other].flags))
					break;
3327
			}
3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346
			BUG_ON(other < 0);
			pr_debug("Computing stripe %llu blocks %d,%d\n",
			       (unsigned long long)sh->sector,
			       disk_idx, other);
			set_bit(STRIPE_COMPUTE_RUN, &sh->state);
			set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
			set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags);
			set_bit(R5_Wantcompute, &sh->dev[other].flags);
			sh->ops.target = disk_idx;
			sh->ops.target2 = other;
			s->uptodate += 2;
			s->req_compute = 1;
			return 1;
		} else if (test_bit(R5_Insync, &dev->flags)) {
			set_bit(R5_LOCKED, &dev->flags);
			set_bit(R5_Wantread, &dev->flags);
			s->locked++;
			pr_debug("Reading block %d (sync=%d)\n",
				disk_idx, s->syncing);
3347 3348
		}
	}
3349 3350 3351 3352 3353

	return 0;
}

/**
3354
 * handle_stripe_fill - read or compute data to satisfy pending requests.
3355
 */
3356 3357 3358
static void handle_stripe_fill(struct stripe_head *sh,
			       struct stripe_head_state *s,
			       int disks)
3359 3360 3361
{
	int i;

3362
	BUG_ON(sh->batch_head);
3363 3364 3365 3366 3367 3368 3369
	/* look for blocks to read/compute, skip this if a compute
	 * is already in flight, or if the stripe contents are in the
	 * midst of changing due to a write
	 */
	if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state &&
	    !sh->reconstruct_state)
		for (i = disks; i--; )
3370
			if (fetch_block(sh, s, i, disks))
3371
				break;
3372 3373 3374
	set_bit(STRIPE_HANDLE, &sh->state);
}

3375
/* handle_stripe_clean_event
3376 3377 3378 3379
 * any written block on an uptodate or failed drive can be returned.
 * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but
 * never LOCKED, so we don't need to test 'failed' directly.
 */
3380
static void handle_stripe_clean_event(struct r5conf *conf,
3381 3382 3383 3384
	struct stripe_head *sh, int disks, struct bio **return_bi)
{
	int i;
	struct r5dev *dev;
3385
	int discard_pending = 0;
3386 3387 3388
	struct stripe_head *head_sh = sh;
	bool do_endio = false;
	int wakeup_nr = 0;
3389 3390 3391 3392 3393

	for (i = disks; i--; )
		if (sh->dev[i].written) {
			dev = &sh->dev[i];
			if (!test_bit(R5_LOCKED, &dev->flags) &&
3394
			    (test_bit(R5_UPTODATE, &dev->flags) ||
3395 3396
			     test_bit(R5_Discard, &dev->flags) ||
			     test_bit(R5_SkipCopy, &dev->flags))) {
3397 3398
				/* We can return any write requests */
				struct bio *wbi, *wbi2;
3399
				pr_debug("Return write for disc %d\n", i);
3400 3401
				if (test_and_clear_bit(R5_Discard, &dev->flags))
					clear_bit(R5_UPTODATE, &dev->flags);
3402 3403 3404
				if (test_and_clear_bit(R5_SkipCopy, &dev->flags)) {
					WARN_ON(test_bit(R5_UPTODATE, &dev->flags));
				}
3405 3406 3407 3408
				do_endio = true;

returnbi:
				dev->page = dev->orig_page;
3409 3410
				wbi = dev->written;
				dev->written = NULL;
3411
				while (wbi && wbi->bi_iter.bi_sector <
3412 3413
					dev->sector + STRIPE_SECTORS) {
					wbi2 = r5_next_bio(wbi, dev->sector);
3414
					if (!raid5_dec_bi_active_stripes(wbi)) {
3415 3416 3417 3418 3419 3420
						md_write_end(conf->mddev);
						wbi->bi_next = *return_bi;
						*return_bi = wbi;
					}
					wbi = wbi2;
				}
3421 3422
				bitmap_endwrite(conf->mddev->bitmap, sh->sector,
						STRIPE_SECTORS,
3423
					 !test_bit(STRIPE_DEGRADED, &sh->state),
3424
						0);
3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435
				if (head_sh->batch_head) {
					sh = list_first_entry(&sh->batch_list,
							      struct stripe_head,
							      batch_list);
					if (sh != head_sh) {
						dev = &sh->dev[i];
						goto returnbi;
					}
				}
				sh = head_sh;
				dev = &sh->dev[i];
3436 3437
			} else if (test_bit(R5_Discard, &dev->flags))
				discard_pending = 1;
3438 3439
			WARN_ON(test_bit(R5_SkipCopy, &dev->flags));
			WARN_ON(dev->page != dev->orig_page);
3440 3441 3442 3443 3444 3445 3446 3447 3448 3449 3450
		}
	if (!discard_pending &&
	    test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)) {
		clear_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);
		clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
		if (sh->qd_idx >= 0) {
			clear_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);
			clear_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags);
		}
		/* now that discard is done we can proceed with any sync */
		clear_bit(STRIPE_DISCARD, &sh->state);
S
Shaohua Li 已提交
3451 3452 3453 3454 3455 3456
		/*
		 * SCSI discard will change some bio fields and the stripe has
		 * no updated data, so remove it from hash list and the stripe
		 * will be reinitialized
		 */
		spin_lock_irq(&conf->device_lock);
3457
unhash:
S
Shaohua Li 已提交
3458
		remove_hash(sh);
3459 3460 3461 3462 3463 3464
		if (head_sh->batch_head) {
			sh = list_first_entry(&sh->batch_list,
					      struct stripe_head, batch_list);
			if (sh != head_sh)
					goto unhash;
		}
S
Shaohua Li 已提交
3465
		spin_unlock_irq(&conf->device_lock);
3466 3467
		sh = head_sh;

3468 3469 3470 3471
		if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state))
			set_bit(STRIPE_HANDLE, &sh->state);

	}
3472 3473 3474 3475

	if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
		if (atomic_dec_and_test(&conf->pending_full_writes))
			md_wakeup_thread(conf->mddev->thread);
3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487 3488

	if (!head_sh->batch_head || !do_endio)
		return;
	for (i = 0; i < head_sh->disks; i++) {
		if (test_and_clear_bit(R5_Overlap, &head_sh->dev[i].flags))
			wakeup_nr++;
	}
	while (!list_empty(&head_sh->batch_list)) {
		int i;
		sh = list_first_entry(&head_sh->batch_list,
				      struct stripe_head, batch_list);
		list_del_init(&sh->batch_list);

3489 3490 3491 3492
		set_mask_bits(&sh->state, ~STRIPE_EXPAND_SYNC_FLAG,
			      head_sh->state & ~((1 << STRIPE_ACTIVE) |
						 (1 << STRIPE_PREREAD_ACTIVE) |
						 STRIPE_EXPAND_SYNC_FLAG));
3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503
		sh->check_state = head_sh->check_state;
		sh->reconstruct_state = head_sh->reconstruct_state;
		for (i = 0; i < sh->disks; i++) {
			if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
				wakeup_nr++;
			sh->dev[i].flags = head_sh->dev[i].flags;
		}

		spin_lock_irq(&sh->stripe_lock);
		sh->batch_head = NULL;
		spin_unlock_irq(&sh->stripe_lock);
3504 3505
		if (sh->state & STRIPE_EXPAND_SYNC_FLAG)
			set_bit(STRIPE_HANDLE, &sh->state);
3506 3507 3508 3509 3510 3511 3512
		release_stripe(sh);
	}

	spin_lock_irq(&head_sh->stripe_lock);
	head_sh->batch_head = NULL;
	spin_unlock_irq(&head_sh->stripe_lock);
	wake_up_nr(&conf->wait_for_overlap, wakeup_nr);
3513 3514
	if (head_sh->state & STRIPE_EXPAND_SYNC_FLAG)
		set_bit(STRIPE_HANDLE, &head_sh->state);
3515 3516
}

3517
static void handle_stripe_dirtying(struct r5conf *conf,
3518 3519 3520
				   struct stripe_head *sh,
				   struct stripe_head_state *s,
				   int disks)
3521 3522
{
	int rmw = 0, rcw = 0, i;
3523 3524
	sector_t recovery_cp = conf->mddev->recovery_cp;

3525
	/* Check whether resync is now happening or should start.
3526 3527 3528 3529 3530 3531
	 * If yes, then the array is dirty (after unclean shutdown or
	 * initial creation), so parity in some stripes might be inconsistent.
	 * In this case, we need to always do reconstruct-write, to ensure
	 * that in case of drive failure or read-error correction, we
	 * generate correct data from the parity.
	 */
3532
	if (conf->rmw_level == PARITY_DISABLE_RMW ||
3533 3534
	    (recovery_cp < MaxSector && sh->sector >= recovery_cp &&
	     s->failed == 0)) {
3535
		/* Calculate the real rcw later - for now make it
3536 3537 3538
		 * look like rcw is cheaper
		 */
		rcw = 1; rmw = 2;
3539 3540
		pr_debug("force RCW rmw_level=%u, recovery_cp=%llu sh->sector=%llu\n",
			 conf->rmw_level, (unsigned long long)recovery_cp,
3541
			 (unsigned long long)sh->sector);
3542
	} else for (i = disks; i--; ) {
3543 3544
		/* would I have to read this buffer for read_modify_write */
		struct r5dev *dev = &sh->dev[i];
3545
		if ((dev->towrite || i == sh->pd_idx || i == sh->qd_idx) &&
3546
		    !test_bit(R5_LOCKED, &dev->flags) &&
3547 3548
		    !(test_bit(R5_UPTODATE, &dev->flags) ||
		      test_bit(R5_Wantcompute, &dev->flags))) {
3549 3550 3551 3552 3553 3554
			if (test_bit(R5_Insync, &dev->flags))
				rmw++;
			else
				rmw += 2*disks;  /* cannot read it */
		}
		/* Would I have to read this buffer for reconstruct_write */
3555 3556
		if (!test_bit(R5_OVERWRITE, &dev->flags) &&
		    i != sh->pd_idx && i != sh->qd_idx &&
3557
		    !test_bit(R5_LOCKED, &dev->flags) &&
3558 3559
		    !(test_bit(R5_UPTODATE, &dev->flags) ||
		    test_bit(R5_Wantcompute, &dev->flags))) {
3560 3561
			if (test_bit(R5_Insync, &dev->flags))
				rcw++;
3562 3563 3564 3565
			else
				rcw += 2*disks;
		}
	}
3566
	pr_debug("for sector %llu, rmw=%d rcw=%d\n",
3567 3568
		(unsigned long long)sh->sector, rmw, rcw);
	set_bit(STRIPE_HANDLE, &sh->state);
3569
	if ((rmw < rcw || (rmw == rcw && conf->rmw_level == PARITY_ENABLE_RMW)) && rmw > 0) {
3570
		/* prefer read-modify-write, but need to get some data */
3571 3572 3573 3574
		if (conf->mddev->queue)
			blk_add_trace_msg(conf->mddev->queue,
					  "raid5 rmw %llu %d",
					  (unsigned long long)sh->sector, rmw);
3575 3576
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
3577
			if ((dev->towrite || i == sh->pd_idx || i == sh->qd_idx) &&
3578
			    !test_bit(R5_LOCKED, &dev->flags) &&
3579 3580
			    !(test_bit(R5_UPTODATE, &dev->flags) ||
			    test_bit(R5_Wantcompute, &dev->flags)) &&
3581
			    test_bit(R5_Insync, &dev->flags)) {
3582 3583 3584 3585
				if (test_bit(STRIPE_PREREAD_ACTIVE,
					     &sh->state)) {
					pr_debug("Read_old block %d for r-m-w\n",
						 i);
3586 3587 3588 3589 3590 3591 3592 3593 3594
					set_bit(R5_LOCKED, &dev->flags);
					set_bit(R5_Wantread, &dev->flags);
					s->locked++;
				} else {
					set_bit(STRIPE_DELAYED, &sh->state);
					set_bit(STRIPE_HANDLE, &sh->state);
				}
			}
		}
N
NeilBrown 已提交
3595
	}
3596
	if ((rcw < rmw || (rcw == rmw && conf->rmw_level != PARITY_ENABLE_RMW)) && rcw > 0) {
3597
		/* want reconstruct write, but need to get some data */
N
NeilBrown 已提交
3598
		int qread =0;
3599
		rcw = 0;
3600 3601 3602
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if (!test_bit(R5_OVERWRITE, &dev->flags) &&
3603
			    i != sh->pd_idx && i != sh->qd_idx &&
3604
			    !test_bit(R5_LOCKED, &dev->flags) &&
3605
			    !(test_bit(R5_UPTODATE, &dev->flags) ||
3606 3607
			      test_bit(R5_Wantcompute, &dev->flags))) {
				rcw++;
3608 3609 3610
				if (test_bit(R5_Insync, &dev->flags) &&
				    test_bit(STRIPE_PREREAD_ACTIVE,
					     &sh->state)) {
3611
					pr_debug("Read_old block "
3612 3613 3614 3615
						"%d for Reconstruct\n", i);
					set_bit(R5_LOCKED, &dev->flags);
					set_bit(R5_Wantread, &dev->flags);
					s->locked++;
N
NeilBrown 已提交
3616
					qread++;
3617 3618 3619 3620 3621 3622
				} else {
					set_bit(STRIPE_DELAYED, &sh->state);
					set_bit(STRIPE_HANDLE, &sh->state);
				}
			}
		}
3623
		if (rcw && conf->mddev->queue)
N
NeilBrown 已提交
3624 3625 3626
			blk_add_trace_msg(conf->mddev->queue, "raid5 rcw %llu %d %d %d",
					  (unsigned long long)sh->sector,
					  rcw, qread, test_bit(STRIPE_DELAYED, &sh->state));
3627
	}
3628 3629 3630 3631 3632

	if (rcw > disks && rmw > disks &&
	    !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
		set_bit(STRIPE_DELAYED, &sh->state);

3633 3634 3635
	/* now if nothing is locked, and if we have enough data,
	 * we can start a write request
	 */
3636 3637
	/* since handle_stripe can be called at any time we need to handle the
	 * case where a compute block operation has been submitted and then a
3638 3639
	 * subsequent call wants to start a write request.  raid_run_ops only
	 * handles the case where compute block and reconstruct are requested
3640 3641 3642
	 * simultaneously.  If this is not the case then new writes need to be
	 * held off until the compute completes.
	 */
3643 3644 3645
	if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) &&
	    (s->locked == 0 && (rcw == 0 || rmw == 0) &&
	    !test_bit(STRIPE_BIT_DELAY, &sh->state)))
3646
		schedule_reconstruction(sh, s, rcw == 0, 0);
3647 3648
}

3649
static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh,
3650 3651
				struct stripe_head_state *s, int disks)
{
3652
	struct r5dev *dev = NULL;
3653

3654
	BUG_ON(sh->batch_head);
3655
	set_bit(STRIPE_HANDLE, &sh->state);
3656

3657 3658 3659
	switch (sh->check_state) {
	case check_state_idle:
		/* start a new check operation if there are no failures */
3660 3661
		if (s->failed == 0) {
			BUG_ON(s->uptodate != disks);
3662 3663
			sh->check_state = check_state_run;
			set_bit(STRIPE_OP_CHECK, &s->ops_request);
3664 3665
			clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
			s->uptodate--;
3666
			break;
3667
		}
3668
		dev = &sh->dev[s->failed_num[0]];
3669 3670 3671 3672 3673 3674 3675 3676 3677
		/* fall through */
	case check_state_compute_result:
		sh->check_state = check_state_idle;
		if (!dev)
			dev = &sh->dev[sh->pd_idx];

		/* check that a write has not made the stripe insync */
		if (test_bit(STRIPE_INSYNC, &sh->state))
			break;
D
Dan Williams 已提交
3678

3679 3680 3681 3682 3683
		/* either failed parity check, or recovery is happening */
		BUG_ON(!test_bit(R5_UPTODATE, &dev->flags));
		BUG_ON(s->uptodate != disks);

		set_bit(R5_LOCKED, &dev->flags);
3684
		s->locked++;
3685
		set_bit(R5_Wantwrite, &dev->flags);
3686

3687 3688
		clear_bit(STRIPE_DEGRADED, &sh->state);
		set_bit(STRIPE_INSYNC, &sh->state);
3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704
		break;
	case check_state_run:
		break; /* we will be called again upon completion */
	case check_state_check_result:
		sh->check_state = check_state_idle;

		/* if a failure occurred during the check operation, leave
		 * STRIPE_INSYNC not set and let the stripe be handled again
		 */
		if (s->failed)
			break;

		/* handle a successful check operation, if parity is correct
		 * we are done.  Otherwise update the mismatch count and repair
		 * parity if !MD_RECOVERY_CHECK
		 */
D
Dan Williams 已提交
3705
		if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0)
3706 3707 3708 3709 3710
			/* parity is correct (on disc,
			 * not in buffer any more)
			 */
			set_bit(STRIPE_INSYNC, &sh->state);
		else {
3711
			atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches);
3712 3713 3714 3715 3716
			if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery))
				/* don't try to repair!! */
				set_bit(STRIPE_INSYNC, &sh->state);
			else {
				sh->check_state = check_state_compute_run;
3717
				set_bit(STRIPE_COMPUTE_RUN, &sh->state);
3718 3719 3720 3721
				set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
				set_bit(R5_Wantcompute,
					&sh->dev[sh->pd_idx].flags);
				sh->ops.target = sh->pd_idx;
3722
				sh->ops.target2 = -1;
3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733
				s->uptodate++;
			}
		}
		break;
	case check_state_compute_run:
		break;
	default:
		printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
		       __func__, sh->check_state,
		       (unsigned long long) sh->sector);
		BUG();
3734 3735 3736
	}
}

3737
static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh,
3738
				  struct stripe_head_state *s,
3739
				  int disks)
3740 3741
{
	int pd_idx = sh->pd_idx;
N
NeilBrown 已提交
3742
	int qd_idx = sh->qd_idx;
3743
	struct r5dev *dev;
3744

3745
	BUG_ON(sh->batch_head);
3746 3747 3748
	set_bit(STRIPE_HANDLE, &sh->state);

	BUG_ON(s->failed > 2);
3749

3750 3751 3752 3753 3754 3755
	/* Want to check and possibly repair P and Q.
	 * However there could be one 'failed' device, in which
	 * case we can only check one of them, possibly using the
	 * other to generate missing data
	 */

3756 3757 3758
	switch (sh->check_state) {
	case check_state_idle:
		/* start a new check operation if there are < 2 failures */
3759
		if (s->failed == s->q_failed) {
3760
			/* The only possible failed device holds Q, so it
3761 3762 3763
			 * makes sense to check P (If anything else were failed,
			 * we would have used P to recreate it).
			 */
3764
			sh->check_state = check_state_run;
3765
		}
3766
		if (!s->q_failed && s->failed < 2) {
3767
			/* Q is not failed, and we didn't use it to generate
3768 3769
			 * anything, so it makes sense to check it
			 */
3770 3771 3772 3773
			if (sh->check_state == check_state_run)
				sh->check_state = check_state_run_pq;
			else
				sh->check_state = check_state_run_q;
3774 3775
		}

3776 3777
		/* discard potentially stale zero_sum_result */
		sh->ops.zero_sum_result = 0;
3778

3779 3780 3781 3782
		if (sh->check_state == check_state_run) {
			/* async_xor_zero_sum destroys the contents of P */
			clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
			s->uptodate--;
3783
		}
3784 3785 3786 3787 3788 3789 3790
		if (sh->check_state >= check_state_run &&
		    sh->check_state <= check_state_run_pq) {
			/* async_syndrome_zero_sum preserves P and Q, so
			 * no need to mark them !uptodate here
			 */
			set_bit(STRIPE_OP_CHECK, &s->ops_request);
			break;
3791 3792
		}

3793 3794 3795 3796 3797
		/* we have 2-disk failure */
		BUG_ON(s->failed != 2);
		/* fall through */
	case check_state_compute_result:
		sh->check_state = check_state_idle;
3798

3799 3800 3801
		/* check that a write has not made the stripe insync */
		if (test_bit(STRIPE_INSYNC, &sh->state))
			break;
3802 3803

		/* now write out any block on a failed drive,
3804
		 * or P or Q if they were recomputed
3805
		 */
3806
		BUG_ON(s->uptodate < disks - 1); /* We don't need Q to recover */
3807
		if (s->failed == 2) {
3808
			dev = &sh->dev[s->failed_num[1]];
3809 3810 3811 3812 3813
			s->locked++;
			set_bit(R5_LOCKED, &dev->flags);
			set_bit(R5_Wantwrite, &dev->flags);
		}
		if (s->failed >= 1) {
3814
			dev = &sh->dev[s->failed_num[0]];
3815 3816 3817 3818
			s->locked++;
			set_bit(R5_LOCKED, &dev->flags);
			set_bit(R5_Wantwrite, &dev->flags);
		}
3819
		if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
3820 3821 3822 3823 3824
			dev = &sh->dev[pd_idx];
			s->locked++;
			set_bit(R5_LOCKED, &dev->flags);
			set_bit(R5_Wantwrite, &dev->flags);
		}
3825
		if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
3826 3827 3828 3829 3830 3831 3832 3833
			dev = &sh->dev[qd_idx];
			s->locked++;
			set_bit(R5_LOCKED, &dev->flags);
			set_bit(R5_Wantwrite, &dev->flags);
		}
		clear_bit(STRIPE_DEGRADED, &sh->state);

		set_bit(STRIPE_INSYNC, &sh->state);
3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862
		break;
	case check_state_run:
	case check_state_run_q:
	case check_state_run_pq:
		break; /* we will be called again upon completion */
	case check_state_check_result:
		sh->check_state = check_state_idle;

		/* handle a successful check operation, if parity is correct
		 * we are done.  Otherwise update the mismatch count and repair
		 * parity if !MD_RECOVERY_CHECK
		 */
		if (sh->ops.zero_sum_result == 0) {
			/* both parities are correct */
			if (!s->failed)
				set_bit(STRIPE_INSYNC, &sh->state);
			else {
				/* in contrast to the raid5 case we can validate
				 * parity, but still have a failure to write
				 * back
				 */
				sh->check_state = check_state_compute_result;
				/* Returning at this point means that we may go
				 * off and bring p and/or q uptodate again so
				 * we make sure to check zero_sum_result again
				 * to verify if p or q need writeback
				 */
			}
		} else {
3863
			atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches);
3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897
			if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery))
				/* don't try to repair!! */
				set_bit(STRIPE_INSYNC, &sh->state);
			else {
				int *target = &sh->ops.target;

				sh->ops.target = -1;
				sh->ops.target2 = -1;
				sh->check_state = check_state_compute_run;
				set_bit(STRIPE_COMPUTE_RUN, &sh->state);
				set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
				if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
					set_bit(R5_Wantcompute,
						&sh->dev[pd_idx].flags);
					*target = pd_idx;
					target = &sh->ops.target2;
					s->uptodate++;
				}
				if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
					set_bit(R5_Wantcompute,
						&sh->dev[qd_idx].flags);
					*target = qd_idx;
					s->uptodate++;
				}
			}
		}
		break;
	case check_state_compute_run:
		break;
	default:
		printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n",
		       __func__, sh->check_state,
		       (unsigned long long) sh->sector);
		BUG();
3898 3899 3900
	}
}

3901
static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh)
3902 3903 3904 3905 3906 3907
{
	int i;

	/* We have read all the blocks in this stripe and now we need to
	 * copy some of them into a target stripe for expand.
	 */
3908
	struct dma_async_tx_descriptor *tx = NULL;
3909
	BUG_ON(sh->batch_head);
3910 3911
	clear_bit(STRIPE_EXPAND_SOURCE, &sh->state);
	for (i = 0; i < sh->disks; i++)
N
NeilBrown 已提交
3912
		if (i != sh->pd_idx && i != sh->qd_idx) {
3913
			int dd_idx, j;
3914
			struct stripe_head *sh2;
3915
			struct async_submit_ctl submit;
3916

3917
			sector_t bn = compute_blocknr(sh, i, 1);
3918 3919
			sector_t s = raid5_compute_sector(conf, bn, 0,
							  &dd_idx, NULL);
3920
			sh2 = get_active_stripe(conf, s, 0, 1, 1);
3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932
			if (sh2 == NULL)
				/* so far only the early blocks of this stripe
				 * have been requested.  When later blocks
				 * get requested, we will try again
				 */
				continue;
			if (!test_bit(STRIPE_EXPANDING, &sh2->state) ||
			   test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) {
				/* must have already done this block */
				release_stripe(sh2);
				continue;
			}
3933 3934

			/* place all the copies on one channel */
3935
			init_async_submit(&submit, 0, tx, NULL, NULL, NULL);
3936
			tx = async_memcpy(sh2->dev[dd_idx].page,
3937
					  sh->dev[i].page, 0, 0, STRIPE_SIZE,
3938
					  &submit);
3939

3940 3941 3942 3943
			set_bit(R5_Expanded, &sh2->dev[dd_idx].flags);
			set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags);
			for (j = 0; j < conf->raid_disks; j++)
				if (j != sh2->pd_idx &&
3944
				    j != sh2->qd_idx &&
3945 3946 3947 3948 3949 3950 3951
				    !test_bit(R5_Expanded, &sh2->dev[j].flags))
					break;
			if (j == conf->raid_disks) {
				set_bit(STRIPE_EXPAND_READY, &sh2->state);
				set_bit(STRIPE_HANDLE, &sh2->state);
			}
			release_stripe(sh2);
3952

3953
		}
3954
	/* done submitting copies, wait for them to complete */
3955
	async_tx_quiesce(&tx);
3956
}
L
Linus Torvalds 已提交
3957 3958 3959 3960

/*
 * handle_stripe - do things to a stripe.
 *
3961 3962
 * We lock the stripe by setting STRIPE_ACTIVE and then examine the
 * state of various bits to see what needs to be done.
L
Linus Torvalds 已提交
3963
 * Possible results:
3964 3965
 *    return some read requests which now have data
 *    return some write requests which are safely on storage
L
Linus Torvalds 已提交
3966 3967 3968 3969 3970
 *    schedule a read on some buffers
 *    schedule a write of some buffers
 *    return confirmation of parity correctness
 *
 */
3971

3972
static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)
L
Linus Torvalds 已提交
3973
{
3974
	struct r5conf *conf = sh->raid_conf;
3975
	int disks = sh->disks;
3976 3977
	struct r5dev *dev;
	int i;
3978
	int do_recovery = 0;
L
Linus Torvalds 已提交
3979

3980 3981
	memset(s, 0, sizeof(*s));

3982 3983
	s->expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state) && !sh->batch_head;
	s->expanded = test_bit(STRIPE_EXPAND_READY, &sh->state) && !sh->batch_head;
3984 3985
	s->failed_num[0] = -1;
	s->failed_num[1] = -1;
L
Linus Torvalds 已提交
3986

3987
	/* Now to look around and see what can be done */
L
Linus Torvalds 已提交
3988
	rcu_read_lock();
3989
	for (i=disks; i--; ) {
3990
		struct md_rdev *rdev;
3991 3992 3993
		sector_t first_bad;
		int bad_sectors;
		int is_bad = 0;
3994

3995
		dev = &sh->dev[i];
L
Linus Torvalds 已提交
3996

3997
		pr_debug("check %d: state 0x%lx read %p write %p written %p\n",
3998 3999
			 i, dev->flags,
			 dev->toread, dev->towrite, dev->written);
4000 4001 4002 4003 4004 4005 4006 4007
		/* maybe we can reply to a read
		 *
		 * new wantfill requests are only permitted while
		 * ops_complete_biofill is guaranteed to be inactive
		 */
		if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread &&
		    !test_bit(STRIPE_BIOFILL_RUN, &sh->state))
			set_bit(R5_Wantfill, &dev->flags);
L
Linus Torvalds 已提交
4008

4009
		/* now count some things */
4010 4011 4012 4013
		if (test_bit(R5_LOCKED, &dev->flags))
			s->locked++;
		if (test_bit(R5_UPTODATE, &dev->flags))
			s->uptodate++;
4014
		if (test_bit(R5_Wantcompute, &dev->flags)) {
4015 4016
			s->compute++;
			BUG_ON(s->compute > 2);
4017
		}
L
Linus Torvalds 已提交
4018

4019
		if (test_bit(R5_Wantfill, &dev->flags))
4020
			s->to_fill++;
4021
		else if (dev->toread)
4022
			s->to_read++;
4023
		if (dev->towrite) {
4024
			s->to_write++;
4025
			if (!test_bit(R5_OVERWRITE, &dev->flags))
4026
				s->non_overwrite++;
4027
		}
4028
		if (dev->written)
4029
			s->written++;
4030 4031 4032 4033 4034 4035 4036 4037 4038 4039
		/* Prefer to use the replacement for reads, but only
		 * if it is recovered enough and has no bad blocks.
		 */
		rdev = rcu_dereference(conf->disks[i].replacement);
		if (rdev && !test_bit(Faulty, &rdev->flags) &&
		    rdev->recovery_offset >= sh->sector + STRIPE_SECTORS &&
		    !is_badblock(rdev, sh->sector, STRIPE_SECTORS,
				 &first_bad, &bad_sectors))
			set_bit(R5_ReadRepl, &dev->flags);
		else {
4040 4041
			if (rdev)
				set_bit(R5_NeedReplace, &dev->flags);
4042 4043 4044
			rdev = rcu_dereference(conf->disks[i].rdev);
			clear_bit(R5_ReadRepl, &dev->flags);
		}
4045 4046
		if (rdev && test_bit(Faulty, &rdev->flags))
			rdev = NULL;
4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058
		if (rdev) {
			is_bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS,
					     &first_bad, &bad_sectors);
			if (s->blocked_rdev == NULL
			    && (test_bit(Blocked, &rdev->flags)
				|| is_bad < 0)) {
				if (is_bad < 0)
					set_bit(BlockedBadBlocks,
						&rdev->flags);
				s->blocked_rdev = rdev;
				atomic_inc(&rdev->nr_pending);
			}
4059
		}
4060 4061 4062
		clear_bit(R5_Insync, &dev->flags);
		if (!rdev)
			/* Not in-sync */;
4063 4064
		else if (is_bad) {
			/* also not in-sync */
4065 4066
			if (!test_bit(WriteErrorSeen, &rdev->flags) &&
			    test_bit(R5_UPTODATE, &dev->flags)) {
4067 4068 4069 4070 4071 4072 4073
				/* treat as in-sync, but with a read error
				 * which we can now try to correct
				 */
				set_bit(R5_Insync, &dev->flags);
				set_bit(R5_ReadError, &dev->flags);
			}
		} else if (test_bit(In_sync, &rdev->flags))
4074
			set_bit(R5_Insync, &dev->flags);
4075
		else if (sh->sector + STRIPE_SECTORS <= rdev->recovery_offset)
4076
			/* in sync if before recovery_offset */
4077 4078 4079 4080 4081 4082 4083 4084 4085
			set_bit(R5_Insync, &dev->flags);
		else if (test_bit(R5_UPTODATE, &dev->flags) &&
			 test_bit(R5_Expanded, &dev->flags))
			/* If we've reshaped into here, we assume it is Insync.
			 * We will shortly update recovery_offset to make
			 * it official.
			 */
			set_bit(R5_Insync, &dev->flags);

4086
		if (test_bit(R5_WriteError, &dev->flags)) {
4087 4088 4089 4090 4091 4092 4093
			/* This flag does not apply to '.replacement'
			 * only to .rdev, so make sure to check that*/
			struct md_rdev *rdev2 = rcu_dereference(
				conf->disks[i].rdev);
			if (rdev2 == rdev)
				clear_bit(R5_Insync, &dev->flags);
			if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
4094
				s->handle_bad_blocks = 1;
4095
				atomic_inc(&rdev2->nr_pending);
4096 4097 4098
			} else
				clear_bit(R5_WriteError, &dev->flags);
		}
4099
		if (test_bit(R5_MadeGood, &dev->flags)) {
4100 4101 4102 4103 4104
			/* This flag does not apply to '.replacement'
			 * only to .rdev, so make sure to check that*/
			struct md_rdev *rdev2 = rcu_dereference(
				conf->disks[i].rdev);
			if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
4105
				s->handle_bad_blocks = 1;
4106
				atomic_inc(&rdev2->nr_pending);
4107 4108 4109
			} else
				clear_bit(R5_MadeGood, &dev->flags);
		}
4110 4111 4112 4113 4114 4115 4116 4117 4118
		if (test_bit(R5_MadeGoodRepl, &dev->flags)) {
			struct md_rdev *rdev2 = rcu_dereference(
				conf->disks[i].replacement);
			if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
				s->handle_bad_blocks = 1;
				atomic_inc(&rdev2->nr_pending);
			} else
				clear_bit(R5_MadeGoodRepl, &dev->flags);
		}
4119
		if (!test_bit(R5_Insync, &dev->flags)) {
4120 4121 4122
			/* The ReadError flag will just be confusing now */
			clear_bit(R5_ReadError, &dev->flags);
			clear_bit(R5_ReWrite, &dev->flags);
L
Linus Torvalds 已提交
4123
		}
4124 4125 4126
		if (test_bit(R5_ReadError, &dev->flags))
			clear_bit(R5_Insync, &dev->flags);
		if (!test_bit(R5_Insync, &dev->flags)) {
4127 4128 4129
			if (s->failed < 2)
				s->failed_num[s->failed] = i;
			s->failed++;
4130 4131
			if (rdev && !test_bit(Faulty, &rdev->flags))
				do_recovery = 1;
4132
		}
L
Linus Torvalds 已提交
4133
	}
4134 4135 4136 4137
	if (test_bit(STRIPE_SYNCING, &sh->state)) {
		/* If there is a failed device being replaced,
		 *     we must be recovering.
		 * else if we are after recovery_cp, we must be syncing
4138
		 * else if MD_RECOVERY_REQUESTED is set, we also are syncing.
4139 4140 4141 4142 4143
		 * else we can only be replacing
		 * sync and recovery both need to read all devices, and so
		 * use the same flag.
		 */
		if (do_recovery ||
4144 4145
		    sh->sector >= conf->mddev->recovery_cp ||
		    test_bit(MD_RECOVERY_REQUESTED, &(conf->mddev->recovery)))
4146 4147 4148 4149
			s->syncing = 1;
		else
			s->replacing = 1;
	}
L
Linus Torvalds 已提交
4150
	rcu_read_unlock();
4151 4152
}

4153 4154 4155 4156 4157 4158 4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184
static int clear_batch_ready(struct stripe_head *sh)
{
	struct stripe_head *tmp;
	if (!test_and_clear_bit(STRIPE_BATCH_READY, &sh->state))
		return 0;
	spin_lock(&sh->stripe_lock);
	if (!sh->batch_head) {
		spin_unlock(&sh->stripe_lock);
		return 0;
	}

	/*
	 * this stripe could be added to a batch list before we check
	 * BATCH_READY, skips it
	 */
	if (sh->batch_head != sh) {
		spin_unlock(&sh->stripe_lock);
		return 1;
	}
	spin_lock(&sh->batch_lock);
	list_for_each_entry(tmp, &sh->batch_list, batch_list)
		clear_bit(STRIPE_BATCH_READY, &tmp->state);
	spin_unlock(&sh->batch_lock);
	spin_unlock(&sh->stripe_lock);

	/*
	 * BATCH_READY is cleared, no new stripes can be added.
	 * batch_list can be accessed without lock
	 */
	return 0;
}

4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204
static void check_break_stripe_batch_list(struct stripe_head *sh)
{
	struct stripe_head *head_sh, *next;
	int i;

	if (!test_and_clear_bit(STRIPE_BATCH_ERR, &sh->state))
		return;

	head_sh = sh;
	do {
		sh = list_first_entry(&sh->batch_list,
				      struct stripe_head, batch_list);
		BUG_ON(sh == head_sh);
	} while (!test_bit(STRIPE_DEGRADED, &sh->state));

	while (sh != head_sh) {
		next = list_first_entry(&sh->batch_list,
					struct stripe_head, batch_list);
		list_del_init(&sh->batch_list);

4205 4206 4207 4208 4209
		set_mask_bits(&sh->state, ~STRIPE_EXPAND_SYNC_FLAG,
			      head_sh->state & ~((1 << STRIPE_ACTIVE) |
						 (1 << STRIPE_PREREAD_ACTIVE) |
						 (1 << STRIPE_DEGRADED) |
						 STRIPE_EXPAND_SYNC_FLAG));
4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226
		sh->check_state = head_sh->check_state;
		sh->reconstruct_state = head_sh->reconstruct_state;
		for (i = 0; i < sh->disks; i++)
			sh->dev[i].flags = head_sh->dev[i].flags &
				(~((1 << R5_WriteError) | (1 << R5_Overlap)));

		spin_lock_irq(&sh->stripe_lock);
		sh->batch_head = NULL;
		spin_unlock_irq(&sh->stripe_lock);

		set_bit(STRIPE_HANDLE, &sh->state);
		release_stripe(sh);

		sh = next;
	}
}

4227 4228 4229
static void handle_stripe(struct stripe_head *sh)
{
	struct stripe_head_state s;
4230
	struct r5conf *conf = sh->raid_conf;
4231
	int i;
4232 4233
	int prexor;
	int disks = sh->disks;
4234
	struct r5dev *pdev, *qdev;
4235 4236

	clear_bit(STRIPE_HANDLE, &sh->state);
4237
	if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) {
4238 4239 4240 4241 4242 4243
		/* already being handled, ensure it gets handled
		 * again when current action finishes */
		set_bit(STRIPE_HANDLE, &sh->state);
		return;
	}

4244 4245 4246 4247 4248
	if (clear_batch_ready(sh) ) {
		clear_bit_unlock(STRIPE_ACTIVE, &sh->state);
		return;
	}

4249 4250
	check_break_stripe_batch_list(sh);

4251
	if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) && !sh->batch_head) {
4252 4253 4254 4255 4256 4257
		spin_lock(&sh->stripe_lock);
		/* Cannot process 'sync' concurrently with 'discard' */
		if (!test_bit(STRIPE_DISCARD, &sh->state) &&
		    test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) {
			set_bit(STRIPE_SYNCING, &sh->state);
			clear_bit(STRIPE_INSYNC, &sh->state);
4258
			clear_bit(STRIPE_REPLACED, &sh->state);
4259 4260
		}
		spin_unlock(&sh->stripe_lock);
4261 4262 4263 4264 4265 4266 4267 4268
	}
	clear_bit(STRIPE_DELAYED, &sh->state);

	pr_debug("handling stripe %llu, state=%#lx cnt=%d, "
		"pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n",
	       (unsigned long long)sh->sector, sh->state,
	       atomic_read(&sh->count), sh->pd_idx, sh->qd_idx,
	       sh->check_state, sh->reconstruct_state);
4269

4270
	analyse_stripe(sh, &s);
4271

4272 4273 4274 4275 4276
	if (s.handle_bad_blocks) {
		set_bit(STRIPE_HANDLE, &sh->state);
		goto finish;
	}

4277 4278
	if (unlikely(s.blocked_rdev)) {
		if (s.syncing || s.expanding || s.expanded ||
4279
		    s.replacing || s.to_write || s.written) {
4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299
			set_bit(STRIPE_HANDLE, &sh->state);
			goto finish;
		}
		/* There is nothing for the blocked_rdev to block */
		rdev_dec_pending(s.blocked_rdev, conf->mddev);
		s.blocked_rdev = NULL;
	}

	if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) {
		set_bit(STRIPE_OP_BIOFILL, &s.ops_request);
		set_bit(STRIPE_BIOFILL_RUN, &sh->state);
	}

	pr_debug("locked=%d uptodate=%d to_read=%d"
	       " to_write=%d failed=%d failed_num=%d,%d\n",
	       s.locked, s.uptodate, s.to_read, s.to_write, s.failed,
	       s.failed_num[0], s.failed_num[1]);
	/* check if the array has lost more than max_degraded devices and,
	 * if so, some requests might need to be failed.
	 */
4300 4301 4302 4303 4304
	if (s.failed > conf->max_degraded) {
		sh->check_state = 0;
		sh->reconstruct_state = 0;
		if (s.to_read+s.to_write+s.written)
			handle_failed_stripe(conf, sh, &s, disks, &s.return_bi);
4305
		if (s.syncing + s.replacing)
4306 4307
			handle_failed_sync(conf, sh, &s);
	}
4308

4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321
	/* Now we check to see if any write operations have recently
	 * completed
	 */
	prexor = 0;
	if (sh->reconstruct_state == reconstruct_state_prexor_drain_result)
		prexor = 1;
	if (sh->reconstruct_state == reconstruct_state_drain_result ||
	    sh->reconstruct_state == reconstruct_state_prexor_drain_result) {
		sh->reconstruct_state = reconstruct_state_idle;

		/* All the 'written' buffers and the parity block are ready to
		 * be written back to disk
		 */
4322 4323
		BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags) &&
		       !test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags));
4324
		BUG_ON(sh->qd_idx >= 0 &&
4325 4326
		       !test_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags) &&
		       !test_bit(R5_Discard, &sh->dev[sh->qd_idx].flags));
4327 4328 4329 4330 4331 4332 4333 4334 4335
		for (i = disks; i--; ) {
			struct r5dev *dev = &sh->dev[i];
			if (test_bit(R5_LOCKED, &dev->flags) &&
				(i == sh->pd_idx || i == sh->qd_idx ||
				 dev->written)) {
				pr_debug("Writing block %d\n", i);
				set_bit(R5_Wantwrite, &dev->flags);
				if (prexor)
					continue;
4336 4337
				if (s.failed > 1)
					continue;
4338 4339 4340 4341 4342 4343 4344 4345 4346 4347
				if (!test_bit(R5_Insync, &dev->flags) ||
				    ((i == sh->pd_idx || i == sh->qd_idx)  &&
				     s.failed == 0))
					set_bit(STRIPE_INSYNC, &sh->state);
			}
		}
		if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
			s.dec_preread_active = 1;
	}

4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381
	/*
	 * might be able to return some write requests if the parity blocks
	 * are safe, or on a failed drive
	 */
	pdev = &sh->dev[sh->pd_idx];
	s.p_failed = (s.failed >= 1 && s.failed_num[0] == sh->pd_idx)
		|| (s.failed >= 2 && s.failed_num[1] == sh->pd_idx);
	qdev = &sh->dev[sh->qd_idx];
	s.q_failed = (s.failed >= 1 && s.failed_num[0] == sh->qd_idx)
		|| (s.failed >= 2 && s.failed_num[1] == sh->qd_idx)
		|| conf->level < 6;

	if (s.written &&
	    (s.p_failed || ((test_bit(R5_Insync, &pdev->flags)
			     && !test_bit(R5_LOCKED, &pdev->flags)
			     && (test_bit(R5_UPTODATE, &pdev->flags) ||
				 test_bit(R5_Discard, &pdev->flags))))) &&
	    (s.q_failed || ((test_bit(R5_Insync, &qdev->flags)
			     && !test_bit(R5_LOCKED, &qdev->flags)
			     && (test_bit(R5_UPTODATE, &qdev->flags) ||
				 test_bit(R5_Discard, &qdev->flags))))))
		handle_stripe_clean_event(conf, sh, disks, &s.return_bi);

	/* Now we might consider reading some blocks, either to check/generate
	 * parity, or to satisfy requests
	 * or to load a block that is being partially written.
	 */
	if (s.to_read || s.non_overwrite
	    || (conf->level == 6 && s.to_write && s.failed)
	    || (s.syncing && (s.uptodate + s.compute < disks))
	    || s.replacing
	    || s.expanding)
		handle_stripe_fill(sh, &s, disks);

4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404
	/* Now to consider new write requests and what else, if anything
	 * should be read.  We do not handle new writes when:
	 * 1/ A 'write' operation (copy+xor) is already in flight.
	 * 2/ A 'check' operation is in flight, as it may clobber the parity
	 *    block.
	 */
	if (s.to_write && !sh->reconstruct_state && !sh->check_state)
		handle_stripe_dirtying(conf, sh, &s, disks);

	/* maybe we need to check and possibly fix the parity for this stripe
	 * Any reads will already have been scheduled, so we just see if enough
	 * data is available.  The parity check is held off while parity
	 * dependent operations are in flight.
	 */
	if (sh->check_state ||
	    (s.syncing && s.locked == 0 &&
	     !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
	     !test_bit(STRIPE_INSYNC, &sh->state))) {
		if (conf->level == 6)
			handle_parity_checks6(conf, sh, &s, disks);
		else
			handle_parity_checks5(conf, sh, &s, disks);
	}
4405

4406 4407 4408
	if ((s.replacing || s.syncing) && s.locked == 0
	    && !test_bit(STRIPE_COMPUTE_RUN, &sh->state)
	    && !test_bit(STRIPE_REPLACED, &sh->state)) {
4409 4410
		/* Write out to replacement devices where possible */
		for (i = 0; i < conf->raid_disks; i++)
4411 4412
			if (test_bit(R5_NeedReplace, &sh->dev[i].flags)) {
				WARN_ON(!test_bit(R5_UPTODATE, &sh->dev[i].flags));
4413 4414 4415 4416
				set_bit(R5_WantReplace, &sh->dev[i].flags);
				set_bit(R5_LOCKED, &sh->dev[i].flags);
				s.locked++;
			}
4417 4418 4419
		if (s.replacing)
			set_bit(STRIPE_INSYNC, &sh->state);
		set_bit(STRIPE_REPLACED, &sh->state);
4420 4421
	}
	if ((s.syncing || s.replacing) && s.locked == 0 &&
4422
	    !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
4423
	    test_bit(STRIPE_INSYNC, &sh->state)) {
4424 4425
		md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
		clear_bit(STRIPE_SYNCING, &sh->state);
4426 4427
		if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))
			wake_up(&conf->wait_for_overlap);
4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453
	}

	/* If the failed drives are just a ReadError, then we might need
	 * to progress the repair/check process
	 */
	if (s.failed <= conf->max_degraded && !conf->mddev->ro)
		for (i = 0; i < s.failed; i++) {
			struct r5dev *dev = &sh->dev[s.failed_num[i]];
			if (test_bit(R5_ReadError, &dev->flags)
			    && !test_bit(R5_LOCKED, &dev->flags)
			    && test_bit(R5_UPTODATE, &dev->flags)
				) {
				if (!test_bit(R5_ReWrite, &dev->flags)) {
					set_bit(R5_Wantwrite, &dev->flags);
					set_bit(R5_ReWrite, &dev->flags);
					set_bit(R5_LOCKED, &dev->flags);
					s.locked++;
				} else {
					/* let's read it back */
					set_bit(R5_Wantread, &dev->flags);
					set_bit(R5_LOCKED, &dev->flags);
					s.locked++;
				}
			}
		}

4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480
	/* Finish reconstruct operations initiated by the expansion process */
	if (sh->reconstruct_state == reconstruct_state_result) {
		struct stripe_head *sh_src
			= get_active_stripe(conf, sh->sector, 1, 1, 1);
		if (sh_src && test_bit(STRIPE_EXPAND_SOURCE, &sh_src->state)) {
			/* sh cannot be written until sh_src has been read.
			 * so arrange for sh to be delayed a little
			 */
			set_bit(STRIPE_DELAYED, &sh->state);
			set_bit(STRIPE_HANDLE, &sh->state);
			if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE,
					      &sh_src->state))
				atomic_inc(&conf->preread_active_stripes);
			release_stripe(sh_src);
			goto finish;
		}
		if (sh_src)
			release_stripe(sh_src);

		sh->reconstruct_state = reconstruct_state_idle;
		clear_bit(STRIPE_EXPANDING, &sh->state);
		for (i = conf->raid_disks; i--; ) {
			set_bit(R5_Wantwrite, &sh->dev[i].flags);
			set_bit(R5_LOCKED, &sh->dev[i].flags);
			s.locked++;
		}
	}
4481

4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497
	if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) &&
	    !sh->reconstruct_state) {
		/* Need to write out all blocks after computing parity */
		sh->disks = conf->raid_disks;
		stripe_set_idx(sh->sector, conf, 0, sh);
		schedule_reconstruction(sh, &s, 1, 1);
	} else if (s.expanded && !sh->reconstruct_state && s.locked == 0) {
		clear_bit(STRIPE_EXPAND_READY, &sh->state);
		atomic_dec(&conf->reshape_stripes);
		wake_up(&conf->wait_for_overlap);
		md_done_sync(conf->mddev, STRIPE_SECTORS, 1);
	}

	if (s.expanding && s.locked == 0 &&
	    !test_bit(STRIPE_COMPUTE_RUN, &sh->state))
		handle_stripe_expansion(conf, sh);
4498

4499
finish:
4500
	/* wait for this device to become unblocked */
4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512
	if (unlikely(s.blocked_rdev)) {
		if (conf->mddev->external)
			md_wait_for_blocked_rdev(s.blocked_rdev,
						 conf->mddev);
		else
			/* Internal metadata will immediately
			 * be written by raid5d, so we don't
			 * need to wait here.
			 */
			rdev_dec_pending(s.blocked_rdev,
					 conf->mddev);
	}
4513

4514 4515
	if (s.handle_bad_blocks)
		for (i = disks; i--; ) {
4516
			struct md_rdev *rdev;
4517 4518 4519 4520 4521 4522 4523 4524 4525
			struct r5dev *dev = &sh->dev[i];
			if (test_and_clear_bit(R5_WriteError, &dev->flags)) {
				/* We own a safe reference to the rdev */
				rdev = conf->disks[i].rdev;
				if (!rdev_set_badblocks(rdev, sh->sector,
							STRIPE_SECTORS, 0))
					md_error(conf->mddev, rdev);
				rdev_dec_pending(rdev, conf->mddev);
			}
4526 4527 4528
			if (test_and_clear_bit(R5_MadeGood, &dev->flags)) {
				rdev = conf->disks[i].rdev;
				rdev_clear_badblocks(rdev, sh->sector,
4529
						     STRIPE_SECTORS, 0);
4530 4531
				rdev_dec_pending(rdev, conf->mddev);
			}
4532 4533
			if (test_and_clear_bit(R5_MadeGoodRepl, &dev->flags)) {
				rdev = conf->disks[i].replacement;
4534 4535 4536
				if (!rdev)
					/* rdev have been moved down */
					rdev = conf->disks[i].rdev;
4537
				rdev_clear_badblocks(rdev, sh->sector,
4538
						     STRIPE_SECTORS, 0);
4539 4540
				rdev_dec_pending(rdev, conf->mddev);
			}
4541 4542
		}

4543 4544 4545
	if (s.ops_request)
		raid_run_ops(sh, s.ops_request);

D
Dan Williams 已提交
4546
	ops_run_io(sh, &s);
4547

4548
	if (s.dec_preread_active) {
4549
		/* We delay this until after ops_run_io so that if make_request
T
Tejun Heo 已提交
4550
		 * is waiting on a flush, it won't continue until the writes
4551 4552 4553 4554 4555 4556 4557 4558
		 * have actually been submitted.
		 */
		atomic_dec(&conf->preread_active_stripes);
		if (atomic_read(&conf->preread_active_stripes) <
		    IO_THRESHOLD)
			md_wakeup_thread(conf->mddev->thread);
	}

4559
	return_io(s.return_bi);
4560

4561
	clear_bit_unlock(STRIPE_ACTIVE, &sh->state);
4562 4563
}

4564
static void raid5_activate_delayed(struct r5conf *conf)
4565 4566 4567 4568 4569 4570 4571 4572 4573 4574
{
	if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) {
		while (!list_empty(&conf->delayed_list)) {
			struct list_head *l = conf->delayed_list.next;
			struct stripe_head *sh;
			sh = list_entry(l, struct stripe_head, lru);
			list_del_init(l);
			clear_bit(STRIPE_DELAYED, &sh->state);
			if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
				atomic_inc(&conf->preread_active_stripes);
4575
			list_add_tail(&sh->lru, &conf->hold_list);
4576
			raid5_wakeup_stripe_thread(sh);
4577
		}
N
NeilBrown 已提交
4578
	}
4579 4580
}

4581 4582
static void activate_bit_delay(struct r5conf *conf,
	struct list_head *temp_inactive_list)
4583 4584 4585 4586 4587 4588 4589
{
	/* device_lock is held */
	struct list_head head;
	list_add(&head, &conf->bitmap_list);
	list_del_init(&conf->bitmap_list);
	while (!list_empty(&head)) {
		struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru);
4590
		int hash;
4591 4592
		list_del_init(&sh->lru);
		atomic_inc(&sh->count);
4593 4594
		hash = sh->hash_lock_index;
		__release_stripe(conf, sh, &temp_inactive_list[hash]);
4595 4596 4597
	}
}

4598
static int raid5_congested(struct mddev *mddev, int bits)
4599
{
4600
	struct r5conf *conf = mddev->private;
4601 4602 4603 4604

	/* No difference between reads and writes.  Just check
	 * how busy the stripe_cache is
	 */
4605

4606 4607 4608 4609
	if (conf->inactive_blocked)
		return 1;
	if (conf->quiesce)
		return 1;
4610
	if (atomic_read(&conf->empty_inactive_list_nr))
4611 4612 4613 4614 4615
		return 1;

	return 0;
}

4616 4617 4618
/* We want read requests to align with chunks where possible,
 * but write requests don't need to.
 */
4619
static int raid5_mergeable_bvec(struct mddev *mddev,
4620 4621
				struct bvec_merge_data *bvm,
				struct bio_vec *biovec)
4622
{
4623
	sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev);
4624
	int max;
4625
	unsigned int chunk_sectors = mddev->chunk_sectors;
4626
	unsigned int bio_sectors = bvm->bi_size >> 9;
4627

4628
	if ((bvm->bi_rw & 1) == WRITE)
4629 4630
		return biovec->bv_len; /* always allow writes to be mergeable */

4631 4632
	if (mddev->new_chunk_sectors < mddev->chunk_sectors)
		chunk_sectors = mddev->new_chunk_sectors;
4633 4634 4635 4636 4637 4638 4639 4640
	max =  (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9;
	if (max < 0) max = 0;
	if (max <= biovec->bv_len && bio_sectors == 0)
		return biovec->bv_len;
	else
		return max;
}

4641
static int in_chunk_boundary(struct mddev *mddev, struct bio *bio)
4642
{
4643
	sector_t sector = bio->bi_iter.bi_sector + get_start_sect(bio->bi_bdev);
4644
	unsigned int chunk_sectors = mddev->chunk_sectors;
4645
	unsigned int bio_sectors = bio_sectors(bio);
4646

4647 4648
	if (mddev->new_chunk_sectors < mddev->chunk_sectors)
		chunk_sectors = mddev->new_chunk_sectors;
4649 4650 4651 4652
	return  chunk_sectors >=
		((sector & (chunk_sectors - 1)) + bio_sectors);
}

4653 4654 4655 4656
/*
 *  add bio to the retry LIFO  ( in O(1) ... we are in interrupt )
 *  later sampled by raid5d.
 */
4657
static void add_bio_to_retry(struct bio *bi,struct r5conf *conf)
4658 4659 4660 4661 4662 4663 4664 4665 4666 4667 4668 4669
{
	unsigned long flags;

	spin_lock_irqsave(&conf->device_lock, flags);

	bi->bi_next = conf->retry_read_aligned_list;
	conf->retry_read_aligned_list = bi;

	spin_unlock_irqrestore(&conf->device_lock, flags);
	md_wakeup_thread(conf->mddev->thread);
}

4670
static struct bio *remove_bio_from_retry(struct r5conf *conf)
4671 4672 4673 4674 4675 4676 4677 4678 4679 4680
{
	struct bio *bi;

	bi = conf->retry_read_aligned;
	if (bi) {
		conf->retry_read_aligned = NULL;
		return bi;
	}
	bi = conf->retry_read_aligned_list;
	if(bi) {
4681
		conf->retry_read_aligned_list = bi->bi_next;
4682
		bi->bi_next = NULL;
4683 4684 4685 4686
		/*
		 * this sets the active strip count to 1 and the processed
		 * strip count to zero (upper 8 bits)
		 */
4687
		raid5_set_bi_stripes(bi, 1); /* biased count of active stripes */
4688 4689 4690 4691 4692
	}

	return bi;
}

4693 4694 4695 4696 4697 4698
/*
 *  The "raid5_align_endio" should check if the read succeeded and if it
 *  did, call bio_endio on the original bio (having bio_put the new bio
 *  first).
 *  If the read failed..
 */
4699
static void raid5_align_endio(struct bio *bi, int error)
4700 4701
{
	struct bio* raid_bi  = bi->bi_private;
4702
	struct mddev *mddev;
4703
	struct r5conf *conf;
4704
	int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags);
4705
	struct md_rdev *rdev;
4706

4707
	bio_put(bi);
4708 4709 4710

	rdev = (void*)raid_bi->bi_next;
	raid_bi->bi_next = NULL;
4711 4712
	mddev = rdev->mddev;
	conf = mddev->private;
4713 4714 4715 4716

	rdev_dec_pending(rdev, conf->mddev);

	if (!error && uptodate) {
4717 4718
		trace_block_bio_complete(bdev_get_queue(raid_bi->bi_bdev),
					 raid_bi, 0);
4719
		bio_endio(raid_bi, 0);
4720 4721
		if (atomic_dec_and_test(&conf->active_aligned_reads))
			wake_up(&conf->wait_for_stripe);
4722
		return;
4723 4724
	}

4725
	pr_debug("raid5_align_endio : io error...handing IO for a retry\n");
4726 4727

	add_bio_to_retry(raid_bi, conf);
4728 4729
}

4730 4731
static int bio_fits_rdev(struct bio *bi)
{
4732
	struct request_queue *q = bdev_get_queue(bi->bi_bdev);
4733

4734
	if (bio_sectors(bi) > queue_max_sectors(q))
4735 4736
		return 0;
	blk_recount_segments(q, bi);
4737
	if (bi->bi_phys_segments > queue_max_segments(q))
4738 4739 4740 4741 4742 4743 4744 4745 4746 4747 4748
		return 0;

	if (q->merge_bvec_fn)
		/* it's too hard to apply the merge_bvec_fn at this stage,
		 * just just give up
		 */
		return 0;

	return 1;
}

4749
static int chunk_aligned_read(struct mddev *mddev, struct bio * raid_bio)
4750
{
4751
	struct r5conf *conf = mddev->private;
N
NeilBrown 已提交
4752
	int dd_idx;
4753
	struct bio* align_bi;
4754
	struct md_rdev *rdev;
4755
	sector_t end_sector;
4756 4757

	if (!in_chunk_boundary(mddev, raid_bio)) {
4758
		pr_debug("chunk_aligned_read : non aligned\n");
4759 4760 4761
		return 0;
	}
	/*
4762
	 * use bio_clone_mddev to make a copy of the bio
4763
	 */
4764
	align_bi = bio_clone_mddev(raid_bio, GFP_NOIO, mddev);
4765 4766 4767 4768 4769 4770 4771 4772 4773 4774 4775
	if (!align_bi)
		return 0;
	/*
	 *   set bi_end_io to a new function, and set bi_private to the
	 *     original bio.
	 */
	align_bi->bi_end_io  = raid5_align_endio;
	align_bi->bi_private = raid_bio;
	/*
	 *	compute position
	 */
4776 4777 4778
	align_bi->bi_iter.bi_sector =
		raid5_compute_sector(conf, raid_bio->bi_iter.bi_sector,
				     0, &dd_idx, NULL);
4779

K
Kent Overstreet 已提交
4780
	end_sector = bio_end_sector(align_bi);
4781
	rcu_read_lock();
4782 4783 4784 4785 4786 4787 4788 4789 4790 4791 4792
	rdev = rcu_dereference(conf->disks[dd_idx].replacement);
	if (!rdev || test_bit(Faulty, &rdev->flags) ||
	    rdev->recovery_offset < end_sector) {
		rdev = rcu_dereference(conf->disks[dd_idx].rdev);
		if (rdev &&
		    (test_bit(Faulty, &rdev->flags) ||
		    !(test_bit(In_sync, &rdev->flags) ||
		      rdev->recovery_offset >= end_sector)))
			rdev = NULL;
	}
	if (rdev) {
4793 4794 4795
		sector_t first_bad;
		int bad_sectors;

4796 4797
		atomic_inc(&rdev->nr_pending);
		rcu_read_unlock();
4798 4799
		raid_bio->bi_next = (void*)rdev;
		align_bi->bi_bdev =  rdev->bdev;
4800
		__clear_bit(BIO_SEG_VALID, &align_bi->bi_flags);
4801

4802
		if (!bio_fits_rdev(align_bi) ||
4803 4804
		    is_badblock(rdev, align_bi->bi_iter.bi_sector,
				bio_sectors(align_bi),
4805 4806
				&first_bad, &bad_sectors)) {
			/* too big in some way, or has a known bad block */
4807 4808 4809 4810 4811
			bio_put(align_bi);
			rdev_dec_pending(rdev, mddev);
			return 0;
		}

4812
		/* No reshape active, so we can trust rdev->data_offset */
4813
		align_bi->bi_iter.bi_sector += rdev->data_offset;
4814

4815 4816 4817
		spin_lock_irq(&conf->device_lock);
		wait_event_lock_irq(conf->wait_for_stripe,
				    conf->quiesce == 0,
4818
				    conf->device_lock);
4819 4820 4821
		atomic_inc(&conf->active_aligned_reads);
		spin_unlock_irq(&conf->device_lock);

4822 4823 4824
		if (mddev->gendisk)
			trace_block_bio_remap(bdev_get_queue(align_bi->bi_bdev),
					      align_bi, disk_devt(mddev->gendisk),
4825
					      raid_bio->bi_iter.bi_sector);
4826 4827 4828 4829
		generic_make_request(align_bi);
		return 1;
	} else {
		rcu_read_unlock();
4830
		bio_put(align_bi);
4831 4832 4833 4834
		return 0;
	}
}

4835 4836 4837 4838 4839 4840 4841 4842 4843 4844
/* __get_priority_stripe - get the next stripe to process
 *
 * Full stripe writes are allowed to pass preread active stripes up until
 * the bypass_threshold is exceeded.  In general the bypass_count
 * increments when the handle_list is handled before the hold_list; however, it
 * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a
 * stripe with in flight i/o.  The bypass_count will be reset when the
 * head of the hold_list has changed, i.e. the head was promoted to the
 * handle_list.
 */
4845
static struct stripe_head *__get_priority_stripe(struct r5conf *conf, int group)
4846
{
4847 4848
	struct stripe_head *sh = NULL, *tmp;
	struct list_head *handle_list = NULL;
4849
	struct r5worker_group *wg = NULL;
4850 4851 4852 4853 4854

	if (conf->worker_cnt_per_group == 0) {
		handle_list = &conf->handle_list;
	} else if (group != ANY_GROUP) {
		handle_list = &conf->worker_groups[group].handle_list;
4855
		wg = &conf->worker_groups[group];
4856 4857 4858 4859
	} else {
		int i;
		for (i = 0; i < conf->group_cnt; i++) {
			handle_list = &conf->worker_groups[i].handle_list;
4860
			wg = &conf->worker_groups[i];
4861 4862 4863 4864
			if (!list_empty(handle_list))
				break;
		}
	}
4865 4866 4867

	pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n",
		  __func__,
4868
		  list_empty(handle_list) ? "empty" : "busy",
4869 4870 4871
		  list_empty(&conf->hold_list) ? "empty" : "busy",
		  atomic_read(&conf->pending_full_writes), conf->bypass_count);

4872 4873
	if (!list_empty(handle_list)) {
		sh = list_entry(handle_list->next, typeof(*sh), lru);
4874 4875 4876 4877 4878 4879 4880 4881 4882 4883 4884 4885 4886 4887 4888 4889 4890

		if (list_empty(&conf->hold_list))
			conf->bypass_count = 0;
		else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) {
			if (conf->hold_list.next == conf->last_hold)
				conf->bypass_count++;
			else {
				conf->last_hold = conf->hold_list.next;
				conf->bypass_count -= conf->bypass_threshold;
				if (conf->bypass_count < 0)
					conf->bypass_count = 0;
			}
		}
	} else if (!list_empty(&conf->hold_list) &&
		   ((conf->bypass_threshold &&
		     conf->bypass_count > conf->bypass_threshold) ||
		    atomic_read(&conf->pending_full_writes) == 0)) {
4891 4892 4893 4894 4895 4896 4897 4898 4899 4900 4901 4902 4903 4904 4905 4906

		list_for_each_entry(tmp, &conf->hold_list,  lru) {
			if (conf->worker_cnt_per_group == 0 ||
			    group == ANY_GROUP ||
			    !cpu_online(tmp->cpu) ||
			    cpu_to_group(tmp->cpu) == group) {
				sh = tmp;
				break;
			}
		}

		if (sh) {
			conf->bypass_count -= conf->bypass_threshold;
			if (conf->bypass_count < 0)
				conf->bypass_count = 0;
		}
4907
		wg = NULL;
4908 4909 4910
	}

	if (!sh)
4911 4912
		return NULL;

4913 4914 4915 4916
	if (wg) {
		wg->stripes_cnt--;
		sh->group = NULL;
	}
4917
	list_del_init(&sh->lru);
4918
	BUG_ON(atomic_inc_return(&sh->count) != 1);
4919 4920
	return sh;
}
4921

4922 4923 4924
struct raid5_plug_cb {
	struct blk_plug_cb	cb;
	struct list_head	list;
4925
	struct list_head	temp_inactive_list[NR_STRIPE_HASH_LOCKS];
4926 4927 4928 4929 4930 4931 4932 4933 4934
};

static void raid5_unplug(struct blk_plug_cb *blk_cb, bool from_schedule)
{
	struct raid5_plug_cb *cb = container_of(
		blk_cb, struct raid5_plug_cb, cb);
	struct stripe_head *sh;
	struct mddev *mddev = cb->cb.data;
	struct r5conf *conf = mddev->private;
N
NeilBrown 已提交
4935
	int cnt = 0;
4936
	int hash;
4937 4938 4939 4940 4941 4942 4943 4944 4945 4946 4947

	if (cb->list.next && !list_empty(&cb->list)) {
		spin_lock_irq(&conf->device_lock);
		while (!list_empty(&cb->list)) {
			sh = list_first_entry(&cb->list, struct stripe_head, lru);
			list_del_init(&sh->lru);
			/*
			 * avoid race release_stripe_plug() sees
			 * STRIPE_ON_UNPLUG_LIST clear but the stripe
			 * is still in our list
			 */
4948
			smp_mb__before_atomic();
4949
			clear_bit(STRIPE_ON_UNPLUG_LIST, &sh->state);
S
Shaohua Li 已提交
4950 4951 4952 4953
			/*
			 * STRIPE_ON_RELEASE_LIST could be set here. In that
			 * case, the count is always > 1 here
			 */
4954 4955
			hash = sh->hash_lock_index;
			__release_stripe(conf, sh, &cb->temp_inactive_list[hash]);
N
NeilBrown 已提交
4956
			cnt++;
4957 4958 4959
		}
		spin_unlock_irq(&conf->device_lock);
	}
4960 4961
	release_inactive_stripe_list(conf, cb->temp_inactive_list,
				     NR_STRIPE_HASH_LOCKS);
4962 4963
	if (mddev->queue)
		trace_block_unplug(mddev->queue, cnt, !from_schedule);
4964 4965 4966 4967 4968 4969 4970 4971 4972 4973 4974 4975 4976 4977 4978 4979 4980 4981
	kfree(cb);
}

static void release_stripe_plug(struct mddev *mddev,
				struct stripe_head *sh)
{
	struct blk_plug_cb *blk_cb = blk_check_plugged(
		raid5_unplug, mddev,
		sizeof(struct raid5_plug_cb));
	struct raid5_plug_cb *cb;

	if (!blk_cb) {
		release_stripe(sh);
		return;
	}

	cb = container_of(blk_cb, struct raid5_plug_cb, cb);

4982 4983
	if (cb->list.next == NULL) {
		int i;
4984
		INIT_LIST_HEAD(&cb->list);
4985 4986 4987
		for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
			INIT_LIST_HEAD(cb->temp_inactive_list + i);
	}
4988 4989 4990 4991 4992 4993 4994

	if (!test_and_set_bit(STRIPE_ON_UNPLUG_LIST, &sh->state))
		list_add_tail(&sh->lru, &cb->list);
	else
		release_stripe(sh);
}

S
Shaohua Li 已提交
4995 4996 4997 4998 4999 5000 5001 5002 5003 5004 5005 5006
static void make_discard_request(struct mddev *mddev, struct bio *bi)
{
	struct r5conf *conf = mddev->private;
	sector_t logical_sector, last_sector;
	struct stripe_head *sh;
	int remaining;
	int stripe_sectors;

	if (mddev->reshape_position != MaxSector)
		/* Skip discard while reshape is happening */
		return;

5007 5008
	logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1);
	last_sector = bi->bi_iter.bi_sector + (bi->bi_iter.bi_size>>9);
S
Shaohua Li 已提交
5009 5010 5011 5012 5013 5014 5015 5016 5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029

	bi->bi_next = NULL;
	bi->bi_phys_segments = 1; /* over-loaded to count active stripes */

	stripe_sectors = conf->chunk_sectors *
		(conf->raid_disks - conf->max_degraded);
	logical_sector = DIV_ROUND_UP_SECTOR_T(logical_sector,
					       stripe_sectors);
	sector_div(last_sector, stripe_sectors);

	logical_sector *= conf->chunk_sectors;
	last_sector *= conf->chunk_sectors;

	for (; logical_sector < last_sector;
	     logical_sector += STRIPE_SECTORS) {
		DEFINE_WAIT(w);
		int d;
	again:
		sh = get_active_stripe(conf, logical_sector, 0, 0, 0);
		prepare_to_wait(&conf->wait_for_overlap, &w,
				TASK_UNINTERRUPTIBLE);
5030 5031 5032 5033 5034 5035 5036
		set_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);
		if (test_bit(STRIPE_SYNCING, &sh->state)) {
			release_stripe(sh);
			schedule();
			goto again;
		}
		clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);
S
Shaohua Li 已提交
5037 5038 5039 5040 5041 5042 5043 5044 5045 5046 5047 5048
		spin_lock_irq(&sh->stripe_lock);
		for (d = 0; d < conf->raid_disks; d++) {
			if (d == sh->pd_idx || d == sh->qd_idx)
				continue;
			if (sh->dev[d].towrite || sh->dev[d].toread) {
				set_bit(R5_Overlap, &sh->dev[d].flags);
				spin_unlock_irq(&sh->stripe_lock);
				release_stripe(sh);
				schedule();
				goto again;
			}
		}
5049
		set_bit(STRIPE_DISCARD, &sh->state);
S
Shaohua Li 已提交
5050
		finish_wait(&conf->wait_for_overlap, &w);
5051
		sh->overwrite_disks = 0;
S
Shaohua Li 已提交
5052 5053 5054 5055 5056 5057
		for (d = 0; d < conf->raid_disks; d++) {
			if (d == sh->pd_idx || d == sh->qd_idx)
				continue;
			sh->dev[d].towrite = bi;
			set_bit(R5_OVERWRITE, &sh->dev[d].flags);
			raid5_inc_bi_active_stripes(bi);
5058
			sh->overwrite_disks++;
S
Shaohua Li 已提交
5059 5060 5061 5062 5063 5064 5065 5066 5067 5068 5069 5070 5071 5072 5073 5074 5075 5076 5077 5078 5079 5080 5081 5082 5083 5084 5085 5086
		}
		spin_unlock_irq(&sh->stripe_lock);
		if (conf->mddev->bitmap) {
			for (d = 0;
			     d < conf->raid_disks - conf->max_degraded;
			     d++)
				bitmap_startwrite(mddev->bitmap,
						  sh->sector,
						  STRIPE_SECTORS,
						  0);
			sh->bm_seq = conf->seq_flush + 1;
			set_bit(STRIPE_BIT_DELAY, &sh->state);
		}

		set_bit(STRIPE_HANDLE, &sh->state);
		clear_bit(STRIPE_DELAYED, &sh->state);
		if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
			atomic_inc(&conf->preread_active_stripes);
		release_stripe_plug(mddev, sh);
	}

	remaining = raid5_dec_bi_active_stripes(bi);
	if (remaining == 0) {
		md_write_end(mddev);
		bio_endio(bi, 0);
	}
}

5087
static void make_request(struct mddev *mddev, struct bio * bi)
L
Linus Torvalds 已提交
5088
{
5089
	struct r5conf *conf = mddev->private;
5090
	int dd_idx;
L
Linus Torvalds 已提交
5091 5092 5093
	sector_t new_sector;
	sector_t logical_sector, last_sector;
	struct stripe_head *sh;
5094
	const int rw = bio_data_dir(bi);
5095
	int remaining;
5096 5097
	DEFINE_WAIT(w);
	bool do_prepare;
L
Linus Torvalds 已提交
5098

T
Tejun Heo 已提交
5099 5100
	if (unlikely(bi->bi_rw & REQ_FLUSH)) {
		md_flush_request(mddev, bi);
5101
		return;
5102 5103
	}

5104
	md_write_start(mddev, bi);
5105

5106
	if (rw == READ &&
5107
	     mddev->reshape_position == MaxSector &&
5108
	     chunk_aligned_read(mddev,bi))
5109
		return;
5110

S
Shaohua Li 已提交
5111 5112 5113 5114 5115
	if (unlikely(bi->bi_rw & REQ_DISCARD)) {
		make_discard_request(mddev, bi);
		return;
	}

5116
	logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1);
K
Kent Overstreet 已提交
5117
	last_sector = bio_end_sector(bi);
L
Linus Torvalds 已提交
5118 5119
	bi->bi_next = NULL;
	bi->bi_phys_segments = 1;	/* over-loaded to count active stripes */
5120

5121
	prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE);
L
Linus Torvalds 已提交
5122
	for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) {
5123
		int previous;
5124
		int seq;
5125

5126
		do_prepare = false;
5127
	retry:
5128
		seq = read_seqcount_begin(&conf->gen_lock);
5129
		previous = 0;
5130 5131 5132
		if (do_prepare)
			prepare_to_wait(&conf->wait_for_overlap, &w,
				TASK_UNINTERRUPTIBLE);
5133
		if (unlikely(conf->reshape_progress != MaxSector)) {
5134
			/* spinlock is needed as reshape_progress may be
5135 5136
			 * 64bit on a 32bit platform, and so it might be
			 * possible to see a half-updated value
5137
			 * Of course reshape_progress could change after
5138 5139 5140 5141
			 * the lock is dropped, so once we get a reference
			 * to the stripe that we think it is, we will have
			 * to check again.
			 */
5142
			spin_lock_irq(&conf->device_lock);
5143
			if (mddev->reshape_backwards
5144 5145
			    ? logical_sector < conf->reshape_progress
			    : logical_sector >= conf->reshape_progress) {
5146 5147
				previous = 1;
			} else {
5148
				if (mddev->reshape_backwards
5149 5150
				    ? logical_sector < conf->reshape_safe
				    : logical_sector >= conf->reshape_safe) {
5151 5152
					spin_unlock_irq(&conf->device_lock);
					schedule();
5153
					do_prepare = true;
5154 5155 5156
					goto retry;
				}
			}
5157 5158
			spin_unlock_irq(&conf->device_lock);
		}
5159

5160 5161
		new_sector = raid5_compute_sector(conf, logical_sector,
						  previous,
5162
						  &dd_idx, NULL);
5163
		pr_debug("raid456: make_request, sector %llu logical %llu\n",
5164
			(unsigned long long)new_sector,
L
Linus Torvalds 已提交
5165 5166
			(unsigned long long)logical_sector);

5167
		sh = get_active_stripe(conf, new_sector, previous,
5168
				       (bi->bi_rw&RWA_MASK), 0);
L
Linus Torvalds 已提交
5169
		if (sh) {
5170
			if (unlikely(previous)) {
5171
				/* expansion might have moved on while waiting for a
5172 5173 5174 5175 5176 5177
				 * stripe, so we must do the range check again.
				 * Expansion could still move past after this
				 * test, but as we are holding a reference to
				 * 'sh', we know that if that happens,
				 *  STRIPE_EXPANDING will get set and the expansion
				 * won't proceed until we finish with the stripe.
5178 5179 5180
				 */
				int must_retry = 0;
				spin_lock_irq(&conf->device_lock);
5181
				if (mddev->reshape_backwards
5182 5183
				    ? logical_sector >= conf->reshape_progress
				    : logical_sector < conf->reshape_progress)
5184 5185 5186 5187 5188
					/* mismatch, need to try again */
					must_retry = 1;
				spin_unlock_irq(&conf->device_lock);
				if (must_retry) {
					release_stripe(sh);
5189
					schedule();
5190
					do_prepare = true;
5191 5192 5193
					goto retry;
				}
			}
5194 5195 5196 5197 5198 5199 5200
			if (read_seqcount_retry(&conf->gen_lock, seq)) {
				/* Might have got the wrong stripe_head
				 * by accident
				 */
				release_stripe(sh);
				goto retry;
			}
5201

5202
			if (rw == WRITE &&
5203
			    logical_sector >= mddev->suspend_lo &&
5204 5205
			    logical_sector < mddev->suspend_hi) {
				release_stripe(sh);
5206 5207 5208 5209 5210 5211 5212 5213
				/* As the suspend_* range is controlled by
				 * userspace, we want an interruptible
				 * wait.
				 */
				flush_signals(current);
				prepare_to_wait(&conf->wait_for_overlap,
						&w, TASK_INTERRUPTIBLE);
				if (logical_sector >= mddev->suspend_lo &&
5214
				    logical_sector < mddev->suspend_hi) {
5215
					schedule();
5216 5217
					do_prepare = true;
				}
5218 5219
				goto retry;
			}
5220 5221

			if (test_bit(STRIPE_EXPANDING, &sh->state) ||
5222
			    !add_stripe_bio(sh, bi, dd_idx, rw, previous)) {
5223 5224
				/* Stripe is busy expanding or
				 * add failed due to overlap.  Flush everything
L
Linus Torvalds 已提交
5225 5226
				 * and wait a while
				 */
N
NeilBrown 已提交
5227
				md_wakeup_thread(mddev->thread);
L
Linus Torvalds 已提交
5228 5229
				release_stripe(sh);
				schedule();
5230
				do_prepare = true;
L
Linus Torvalds 已提交
5231 5232
				goto retry;
			}
5233 5234
			set_bit(STRIPE_HANDLE, &sh->state);
			clear_bit(STRIPE_DELAYED, &sh->state);
5235 5236
			if ((!sh->batch_head || sh == sh->batch_head) &&
			    (bi->bi_rw & REQ_SYNC) &&
5237 5238
			    !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
				atomic_inc(&conf->preread_active_stripes);
5239
			release_stripe_plug(mddev, sh);
L
Linus Torvalds 已提交
5240 5241 5242 5243 5244 5245
		} else {
			/* cannot get stripe for read-ahead, just give-up */
			clear_bit(BIO_UPTODATE, &bi->bi_flags);
			break;
		}
	}
5246
	finish_wait(&conf->wait_for_overlap, &w);
5247

5248
	remaining = raid5_dec_bi_active_stripes(bi);
5249
	if (remaining == 0) {
L
Linus Torvalds 已提交
5250

5251
		if ( rw == WRITE )
L
Linus Torvalds 已提交
5252
			md_write_end(mddev);
5253

5254 5255
		trace_block_bio_complete(bdev_get_queue(bi->bi_bdev),
					 bi, 0);
5256
		bio_endio(bi, 0);
L
Linus Torvalds 已提交
5257 5258 5259
	}
}

5260
static sector_t raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks);
D
Dan Williams 已提交
5261

5262
static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *skipped)
L
Linus Torvalds 已提交
5263
{
5264 5265 5266 5267 5268 5269 5270 5271 5272
	/* reshaping is quite different to recovery/resync so it is
	 * handled quite separately ... here.
	 *
	 * On each call to sync_request, we gather one chunk worth of
	 * destination stripes and flag them as expanding.
	 * Then we find all the source stripes and request reads.
	 * As the reads complete, handle_stripe will copy the data
	 * into the destination stripe and release that stripe.
	 */
5273
	struct r5conf *conf = mddev->private;
L
Linus Torvalds 已提交
5274
	struct stripe_head *sh;
5275
	sector_t first_sector, last_sector;
5276 5277 5278
	int raid_disks = conf->previous_raid_disks;
	int data_disks = raid_disks - conf->max_degraded;
	int new_data_disks = conf->raid_disks - conf->max_degraded;
5279 5280
	int i;
	int dd_idx;
5281
	sector_t writepos, readpos, safepos;
5282
	sector_t stripe_addr;
5283
	int reshape_sectors;
5284
	struct list_head stripes;
5285

5286 5287
	if (sector_nr == 0) {
		/* If restarting in the middle, skip the initial sectors */
5288
		if (mddev->reshape_backwards &&
5289 5290 5291
		    conf->reshape_progress < raid5_size(mddev, 0, 0)) {
			sector_nr = raid5_size(mddev, 0, 0)
				- conf->reshape_progress;
5292
		} else if (!mddev->reshape_backwards &&
5293 5294
			   conf->reshape_progress > 0)
			sector_nr = conf->reshape_progress;
5295
		sector_div(sector_nr, new_data_disks);
5296
		if (sector_nr) {
5297 5298
			mddev->curr_resync_completed = sector_nr;
			sysfs_notify(&mddev->kobj, NULL, "sync_completed");
5299 5300 5301
			*skipped = 1;
			return sector_nr;
		}
5302 5303
	}

5304 5305 5306 5307
	/* We need to process a full chunk at a time.
	 * If old and new chunk sizes differ, we need to process the
	 * largest of these
	 */
5308 5309
	if (mddev->new_chunk_sectors > mddev->chunk_sectors)
		reshape_sectors = mddev->new_chunk_sectors;
5310
	else
5311
		reshape_sectors = mddev->chunk_sectors;
5312

5313 5314 5315 5316 5317
	/* We update the metadata at least every 10 seconds, or when
	 * the data about to be copied would over-write the source of
	 * the data at the front of the range.  i.e. one new_stripe
	 * along from reshape_progress new_maps to after where
	 * reshape_safe old_maps to
5318
	 */
5319
	writepos = conf->reshape_progress;
5320
	sector_div(writepos, new_data_disks);
5321 5322
	readpos = conf->reshape_progress;
	sector_div(readpos, data_disks);
5323
	safepos = conf->reshape_safe;
5324
	sector_div(safepos, data_disks);
5325
	if (mddev->reshape_backwards) {
5326
		writepos -= min_t(sector_t, reshape_sectors, writepos);
5327
		readpos += reshape_sectors;
5328
		safepos += reshape_sectors;
5329
	} else {
5330
		writepos += reshape_sectors;
5331 5332
		readpos -= min_t(sector_t, reshape_sectors, readpos);
		safepos -= min_t(sector_t, reshape_sectors, safepos);
5333
	}
5334

5335 5336 5337 5338 5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349
	/* Having calculated the 'writepos' possibly use it
	 * to set 'stripe_addr' which is where we will write to.
	 */
	if (mddev->reshape_backwards) {
		BUG_ON(conf->reshape_progress == 0);
		stripe_addr = writepos;
		BUG_ON((mddev->dev_sectors &
			~((sector_t)reshape_sectors - 1))
		       - reshape_sectors - stripe_addr
		       != sector_nr);
	} else {
		BUG_ON(writepos != sector_nr + reshape_sectors);
		stripe_addr = sector_nr;
	}

5350 5351 5352 5353
	/* 'writepos' is the most advanced device address we might write.
	 * 'readpos' is the least advanced device address we might read.
	 * 'safepos' is the least address recorded in the metadata as having
	 *     been reshaped.
5354 5355 5356 5357
	 * If there is a min_offset_diff, these are adjusted either by
	 * increasing the safepos/readpos if diff is negative, or
	 * increasing writepos if diff is positive.
	 * If 'readpos' is then behind 'writepos', there is no way that we can
5358 5359 5360 5361 5362 5363 5364 5365 5366 5367 5368 5369
	 * ensure safety in the face of a crash - that must be done by userspace
	 * making a backup of the data.  So in that case there is no particular
	 * rush to update metadata.
	 * Otherwise if 'safepos' is behind 'writepos', then we really need to
	 * update the metadata to advance 'safepos' to match 'readpos' so that
	 * we can be safe in the event of a crash.
	 * So we insist on updating metadata if safepos is behind writepos and
	 * readpos is beyond writepos.
	 * In any case, update the metadata every 10 seconds.
	 * Maybe that number should be configurable, but I'm not sure it is
	 * worth it.... maybe it could be a multiple of safemode_delay???
	 */
5370 5371 5372 5373 5374 5375
	if (conf->min_offset_diff < 0) {
		safepos += -conf->min_offset_diff;
		readpos += -conf->min_offset_diff;
	} else
		writepos += conf->min_offset_diff;

5376
	if ((mddev->reshape_backwards
5377 5378 5379
	     ? (safepos > writepos && readpos < writepos)
	     : (safepos < writepos && readpos > writepos)) ||
	    time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
5380 5381
		/* Cannot proceed until we've updated the superblock... */
		wait_event(conf->wait_for_overlap,
5382 5383 5384 5385
			   atomic_read(&conf->reshape_stripes)==0
			   || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
		if (atomic_read(&conf->reshape_stripes) != 0)
			return 0;
5386
		mddev->reshape_position = conf->reshape_progress;
5387
		mddev->curr_resync_completed = sector_nr;
5388
		conf->reshape_checkpoint = jiffies;
5389
		set_bit(MD_CHANGE_DEVS, &mddev->flags);
5390
		md_wakeup_thread(mddev->thread);
5391
		wait_event(mddev->sb_wait, mddev->flags == 0 ||
5392 5393 5394
			   test_bit(MD_RECOVERY_INTR, &mddev->recovery));
		if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
			return 0;
5395
		spin_lock_irq(&conf->device_lock);
5396
		conf->reshape_safe = mddev->reshape_position;
5397 5398
		spin_unlock_irq(&conf->device_lock);
		wake_up(&conf->wait_for_overlap);
5399
		sysfs_notify(&mddev->kobj, NULL, "sync_completed");
5400 5401
	}

5402
	INIT_LIST_HEAD(&stripes);
5403
	for (i = 0; i < reshape_sectors; i += STRIPE_SECTORS) {
5404
		int j;
5405
		int skipped_disk = 0;
5406
		sh = get_active_stripe(conf, stripe_addr+i, 0, 0, 1);
5407 5408 5409 5410 5411 5412 5413 5414 5415
		set_bit(STRIPE_EXPANDING, &sh->state);
		atomic_inc(&conf->reshape_stripes);
		/* If any of this stripe is beyond the end of the old
		 * array, then we need to zero those blocks
		 */
		for (j=sh->disks; j--;) {
			sector_t s;
			if (j == sh->pd_idx)
				continue;
5416
			if (conf->level == 6 &&
5417
			    j == sh->qd_idx)
5418
				continue;
5419
			s = compute_blocknr(sh, j, 0);
D
Dan Williams 已提交
5420
			if (s < raid5_size(mddev, 0, 0)) {
5421
				skipped_disk = 1;
5422 5423 5424 5425 5426 5427
				continue;
			}
			memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE);
			set_bit(R5_Expanded, &sh->dev[j].flags);
			set_bit(R5_UPTODATE, &sh->dev[j].flags);
		}
5428
		if (!skipped_disk) {
5429 5430 5431
			set_bit(STRIPE_EXPAND_READY, &sh->state);
			set_bit(STRIPE_HANDLE, &sh->state);
		}
5432
		list_add(&sh->lru, &stripes);
5433 5434
	}
	spin_lock_irq(&conf->device_lock);
5435
	if (mddev->reshape_backwards)
5436
		conf->reshape_progress -= reshape_sectors * new_data_disks;
5437
	else
5438
		conf->reshape_progress += reshape_sectors * new_data_disks;
5439 5440 5441 5442 5443 5444 5445
	spin_unlock_irq(&conf->device_lock);
	/* Ok, those stripe are ready. We can start scheduling
	 * reads on the source stripes.
	 * The source stripes are determined by mapping the first and last
	 * block on the destination stripes.
	 */
	first_sector =
5446
		raid5_compute_sector(conf, stripe_addr*(new_data_disks),
5447
				     1, &dd_idx, NULL);
5448
	last_sector =
5449
		raid5_compute_sector(conf, ((stripe_addr+reshape_sectors)
5450
					    * new_data_disks - 1),
5451
				     1, &dd_idx, NULL);
A
Andre Noll 已提交
5452 5453
	if (last_sector >= mddev->dev_sectors)
		last_sector = mddev->dev_sectors - 1;
5454
	while (first_sector <= last_sector) {
5455
		sh = get_active_stripe(conf, first_sector, 1, 0, 1);
5456 5457 5458 5459 5460
		set_bit(STRIPE_EXPAND_SOURCE, &sh->state);
		set_bit(STRIPE_HANDLE, &sh->state);
		release_stripe(sh);
		first_sector += STRIPE_SECTORS;
	}
5461 5462 5463 5464 5465 5466 5467 5468
	/* Now that the sources are clearly marked, we can release
	 * the destination stripes
	 */
	while (!list_empty(&stripes)) {
		sh = list_entry(stripes.next, struct stripe_head, lru);
		list_del_init(&sh->lru);
		release_stripe(sh);
	}
5469 5470 5471
	/* If this takes us to the resync_max point where we have to pause,
	 * then we need to write out the superblock.
	 */
5472
	sector_nr += reshape_sectors;
5473 5474
	if ((sector_nr - mddev->curr_resync_completed) * 2
	    >= mddev->resync_max - mddev->curr_resync_completed) {
5475 5476
		/* Cannot proceed until we've updated the superblock... */
		wait_event(conf->wait_for_overlap,
5477 5478 5479 5480
			   atomic_read(&conf->reshape_stripes) == 0
			   || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
		if (atomic_read(&conf->reshape_stripes) != 0)
			goto ret;
5481
		mddev->reshape_position = conf->reshape_progress;
5482
		mddev->curr_resync_completed = sector_nr;
5483
		conf->reshape_checkpoint = jiffies;
5484 5485 5486 5487
		set_bit(MD_CHANGE_DEVS, &mddev->flags);
		md_wakeup_thread(mddev->thread);
		wait_event(mddev->sb_wait,
			   !test_bit(MD_CHANGE_DEVS, &mddev->flags)
5488 5489 5490
			   || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
		if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
			goto ret;
5491
		spin_lock_irq(&conf->device_lock);
5492
		conf->reshape_safe = mddev->reshape_position;
5493 5494
		spin_unlock_irq(&conf->device_lock);
		wake_up(&conf->wait_for_overlap);
5495
		sysfs_notify(&mddev->kobj, NULL, "sync_completed");
5496
	}
5497
ret:
5498
	return reshape_sectors;
5499 5500
}

5501
static inline sector_t sync_request(struct mddev *mddev, sector_t sector_nr, int *skipped)
5502
{
5503
	struct r5conf *conf = mddev->private;
5504
	struct stripe_head *sh;
A
Andre Noll 已提交
5505
	sector_t max_sector = mddev->dev_sectors;
N
NeilBrown 已提交
5506
	sector_t sync_blocks;
5507 5508
	int still_degraded = 0;
	int i;
L
Linus Torvalds 已提交
5509

5510
	if (sector_nr >= max_sector) {
L
Linus Torvalds 已提交
5511
		/* just being told to finish up .. nothing much to do */
5512

5513 5514 5515 5516
		if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {
			end_reshape(conf);
			return 0;
		}
5517 5518 5519 5520

		if (mddev->curr_resync < max_sector) /* aborted */
			bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
					&sync_blocks, 1);
5521
		else /* completed sync */
5522 5523 5524
			conf->fullsync = 0;
		bitmap_close_sync(mddev->bitmap);

L
Linus Torvalds 已提交
5525 5526
		return 0;
	}
5527

5528 5529 5530
	/* Allow raid5_quiesce to complete */
	wait_event(conf->wait_for_overlap, conf->quiesce != 2);

5531 5532
	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
		return reshape_request(mddev, sector_nr, skipped);
5533

5534 5535 5536 5537 5538 5539
	/* No need to check resync_max as we never do more than one
	 * stripe, and as resync_max will always be on a chunk boundary,
	 * if the check in md_do_sync didn't fire, there is no chance
	 * of overstepping resync_max here
	 */

5540
	/* if there is too many failed drives and we are trying
L
Linus Torvalds 已提交
5541 5542 5543
	 * to resync, then assert that we are finished, because there is
	 * nothing we can do.
	 */
5544
	if (mddev->degraded >= conf->max_degraded &&
5545
	    test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
A
Andre Noll 已提交
5546
		sector_t rv = mddev->dev_sectors - sector_nr;
5547
		*skipped = 1;
L
Linus Torvalds 已提交
5548 5549
		return rv;
	}
5550 5551 5552 5553
	if (!test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
	    !conf->fullsync &&
	    !bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) &&
	    sync_blocks >= STRIPE_SECTORS) {
5554 5555 5556 5557 5558
		/* we can skip this block, and probably more */
		sync_blocks /= STRIPE_SECTORS;
		*skipped = 1;
		return sync_blocks * STRIPE_SECTORS; /* keep things rounded to whole stripes */
	}
L
Linus Torvalds 已提交
5559

N
NeilBrown 已提交
5560 5561
	bitmap_cond_end_sync(mddev->bitmap, sector_nr);

5562
	sh = get_active_stripe(conf, sector_nr, 0, 1, 0);
L
Linus Torvalds 已提交
5563
	if (sh == NULL) {
5564
		sh = get_active_stripe(conf, sector_nr, 0, 0, 0);
L
Linus Torvalds 已提交
5565
		/* make sure we don't swamp the stripe cache if someone else
5566
		 * is trying to get access
L
Linus Torvalds 已提交
5567
		 */
5568
		schedule_timeout_uninterruptible(1);
L
Linus Torvalds 已提交
5569
	}
5570
	/* Need to check if array will still be degraded after recovery/resync
5571 5572
	 * Note in case of > 1 drive failures it's possible we're rebuilding
	 * one drive while leaving another faulty drive in array.
5573
	 */
5574 5575 5576 5577 5578
	rcu_read_lock();
	for (i = 0; i < conf->raid_disks; i++) {
		struct md_rdev *rdev = ACCESS_ONCE(conf->disks[i].rdev);

		if (rdev == NULL || test_bit(Faulty, &rdev->flags))
5579
			still_degraded = 1;
5580 5581
	}
	rcu_read_unlock();
5582 5583 5584

	bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, still_degraded);

5585
	set_bit(STRIPE_SYNC_REQUESTED, &sh->state);
5586
	set_bit(STRIPE_HANDLE, &sh->state);
L
Linus Torvalds 已提交
5587 5588 5589 5590 5591 5592

	release_stripe(sh);

	return STRIPE_SECTORS;
}

5593
static int  retry_aligned_read(struct r5conf *conf, struct bio *raid_bio)
5594 5595 5596 5597 5598 5599 5600 5601 5602 5603 5604 5605
{
	/* We may not be able to submit a whole bio at once as there
	 * may not be enough stripe_heads available.
	 * We cannot pre-allocate enough stripe_heads as we may need
	 * more than exist in the cache (if we allow ever large chunks).
	 * So we do one stripe head at a time and record in
	 * ->bi_hw_segments how many have been done.
	 *
	 * We *know* that this entire raid_bio is in one chunk, so
	 * it will be only one 'dd_idx' and only need one call to raid5_compute_sector.
	 */
	struct stripe_head *sh;
5606
	int dd_idx;
5607 5608 5609 5610 5611
	sector_t sector, logical_sector, last_sector;
	int scnt = 0;
	int remaining;
	int handled = 0;

5612 5613
	logical_sector = raid_bio->bi_iter.bi_sector &
		~((sector_t)STRIPE_SECTORS-1);
5614
	sector = raid5_compute_sector(conf, logical_sector,
5615
				      0, &dd_idx, NULL);
K
Kent Overstreet 已提交
5616
	last_sector = bio_end_sector(raid_bio);
5617 5618

	for (; logical_sector < last_sector;
5619 5620 5621
	     logical_sector += STRIPE_SECTORS,
		     sector += STRIPE_SECTORS,
		     scnt++) {
5622

5623
		if (scnt < raid5_bi_processed_stripes(raid_bio))
5624 5625 5626
			/* already done this stripe */
			continue;

5627
		sh = get_active_stripe(conf, sector, 0, 1, 1);
5628 5629 5630

		if (!sh) {
			/* failed to get a stripe - must wait */
5631
			raid5_set_bi_processed_stripes(raid_bio, scnt);
5632 5633 5634 5635
			conf->retry_read_aligned = raid_bio;
			return handled;
		}

5636
		if (!add_stripe_bio(sh, raid_bio, dd_idx, 0, 0)) {
5637
			release_stripe(sh);
5638
			raid5_set_bi_processed_stripes(raid_bio, scnt);
5639 5640 5641 5642
			conf->retry_read_aligned = raid_bio;
			return handled;
		}

5643
		set_bit(R5_ReadNoMerge, &sh->dev[dd_idx].flags);
5644
		handle_stripe(sh);
5645 5646 5647
		release_stripe(sh);
		handled++;
	}
5648
	remaining = raid5_dec_bi_active_stripes(raid_bio);
5649 5650 5651
	if (remaining == 0) {
		trace_block_bio_complete(bdev_get_queue(raid_bio->bi_bdev),
					 raid_bio, 0);
5652
		bio_endio(raid_bio, 0);
5653
	}
5654 5655 5656 5657 5658
	if (atomic_dec_and_test(&conf->active_aligned_reads))
		wake_up(&conf->wait_for_stripe);
	return handled;
}

5659
static int handle_active_stripes(struct r5conf *conf, int group,
5660 5661
				 struct r5worker *worker,
				 struct list_head *temp_inactive_list)
5662 5663
{
	struct stripe_head *batch[MAX_STRIPE_BATCH], *sh;
5664 5665
	int i, batch_size = 0, hash;
	bool release_inactive = false;
5666 5667

	while (batch_size < MAX_STRIPE_BATCH &&
5668
			(sh = __get_priority_stripe(conf, group)) != NULL)
5669 5670
		batch[batch_size++] = sh;

5671 5672 5673 5674 5675 5676 5677 5678
	if (batch_size == 0) {
		for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
			if (!list_empty(temp_inactive_list + i))
				break;
		if (i == NR_STRIPE_HASH_LOCKS)
			return batch_size;
		release_inactive = true;
	}
5679 5680
	spin_unlock_irq(&conf->device_lock);

5681 5682 5683 5684 5685 5686 5687 5688
	release_inactive_stripe_list(conf, temp_inactive_list,
				     NR_STRIPE_HASH_LOCKS);

	if (release_inactive) {
		spin_lock_irq(&conf->device_lock);
		return 0;
	}

5689 5690 5691 5692 5693 5694
	for (i = 0; i < batch_size; i++)
		handle_stripe(batch[i]);

	cond_resched();

	spin_lock_irq(&conf->device_lock);
5695 5696 5697 5698
	for (i = 0; i < batch_size; i++) {
		hash = batch[i]->hash_lock_index;
		__release_stripe(conf, batch[i], &temp_inactive_list[hash]);
	}
5699 5700
	return batch_size;
}
5701

5702 5703 5704 5705 5706 5707 5708 5709 5710 5711 5712 5713 5714 5715 5716 5717 5718
static void raid5_do_work(struct work_struct *work)
{
	struct r5worker *worker = container_of(work, struct r5worker, work);
	struct r5worker_group *group = worker->group;
	struct r5conf *conf = group->conf;
	int group_id = group - conf->worker_groups;
	int handled;
	struct blk_plug plug;

	pr_debug("+++ raid5worker active\n");

	blk_start_plug(&plug);
	handled = 0;
	spin_lock_irq(&conf->device_lock);
	while (1) {
		int batch_size, released;

5719
		released = release_stripe_list(conf, worker->temp_inactive_list);
5720

5721 5722
		batch_size = handle_active_stripes(conf, group_id, worker,
						   worker->temp_inactive_list);
5723
		worker->working = false;
5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 5735
		if (!batch_size && !released)
			break;
		handled += batch_size;
	}
	pr_debug("%d stripes handled\n", handled);

	spin_unlock_irq(&conf->device_lock);
	blk_finish_plug(&plug);

	pr_debug("--- raid5worker inactive\n");
}

L
Linus Torvalds 已提交
5736 5737 5738 5739 5740 5741 5742
/*
 * This is our raid5 kernel thread.
 *
 * We scan the hash table for stripes which can be handled now.
 * During the scan, completed stripes are saved for us by the interrupt
 * handler, so that they will not have to wait for our next wakeup.
 */
S
Shaohua Li 已提交
5743
static void raid5d(struct md_thread *thread)
L
Linus Torvalds 已提交
5744
{
S
Shaohua Li 已提交
5745
	struct mddev *mddev = thread->mddev;
5746
	struct r5conf *conf = mddev->private;
L
Linus Torvalds 已提交
5747
	int handled;
5748
	struct blk_plug plug;
L
Linus Torvalds 已提交
5749

5750
	pr_debug("+++ raid5d active\n");
L
Linus Torvalds 已提交
5751 5752 5753

	md_check_recovery(mddev);

5754
	blk_start_plug(&plug);
L
Linus Torvalds 已提交
5755 5756 5757
	handled = 0;
	spin_lock_irq(&conf->device_lock);
	while (1) {
5758
		struct bio *bio;
S
Shaohua Li 已提交
5759 5760
		int batch_size, released;

5761
		released = release_stripe_list(conf, conf->temp_inactive_list);
L
Linus Torvalds 已提交
5762

5763
		if (
5764 5765 5766
		    !list_empty(&conf->bitmap_list)) {
			/* Now is a good time to flush some bitmap updates */
			conf->seq_flush++;
5767
			spin_unlock_irq(&conf->device_lock);
5768
			bitmap_unplug(mddev->bitmap);
5769
			spin_lock_irq(&conf->device_lock);
5770
			conf->seq_write = conf->seq_flush;
5771
			activate_bit_delay(conf, conf->temp_inactive_list);
5772
		}
5773
		raid5_activate_delayed(conf);
5774

5775 5776 5777 5778 5779 5780 5781 5782 5783 5784
		while ((bio = remove_bio_from_retry(conf))) {
			int ok;
			spin_unlock_irq(&conf->device_lock);
			ok = retry_aligned_read(conf, bio);
			spin_lock_irq(&conf->device_lock);
			if (!ok)
				break;
			handled++;
		}

5785 5786
		batch_size = handle_active_stripes(conf, ANY_GROUP, NULL,
						   conf->temp_inactive_list);
S
Shaohua Li 已提交
5787
		if (!batch_size && !released)
L
Linus Torvalds 已提交
5788
			break;
5789
		handled += batch_size;
L
Linus Torvalds 已提交
5790

5791 5792
		if (mddev->flags & ~(1<<MD_CHANGE_PENDING)) {
			spin_unlock_irq(&conf->device_lock);
5793
			md_check_recovery(mddev);
5794 5795
			spin_lock_irq(&conf->device_lock);
		}
L
Linus Torvalds 已提交
5796
	}
5797
	pr_debug("%d stripes handled\n", handled);
L
Linus Torvalds 已提交
5798 5799 5800

	spin_unlock_irq(&conf->device_lock);

5801
	async_tx_issue_pending_all();
5802
	blk_finish_plug(&plug);
L
Linus Torvalds 已提交
5803

5804
	pr_debug("--- raid5d inactive\n");
L
Linus Torvalds 已提交
5805 5806
}

5807
static ssize_t
5808
raid5_show_stripe_cache_size(struct mddev *mddev, char *page)
5809
{
5810 5811 5812 5813
	struct r5conf *conf;
	int ret = 0;
	spin_lock(&mddev->lock);
	conf = mddev->private;
5814
	if (conf)
5815 5816 5817
		ret = sprintf(page, "%d\n", conf->max_nr_stripes);
	spin_unlock(&mddev->lock);
	return ret;
5818 5819
}

5820
int
5821
raid5_set_cache_size(struct mddev *mddev, int size)
5822
{
5823
	struct r5conf *conf = mddev->private;
5824
	int err;
5825
	int hash;
5826

5827
	if (size <= 16 || size > 32768)
5828
		return -EINVAL;
5829
	hash = (conf->max_nr_stripes - 1) % NR_STRIPE_HASH_LOCKS;
5830
	while (size < conf->max_nr_stripes) {
5831
		if (drop_one_stripe(conf, hash))
5832 5833 5834
			conf->max_nr_stripes--;
		else
			break;
5835 5836 5837
		hash--;
		if (hash < 0)
			hash = NR_STRIPE_HASH_LOCKS - 1;
5838
	}
5839 5840 5841
	err = md_allow_write(mddev);
	if (err)
		return err;
5842
	hash = conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS;
5843
	while (size > conf->max_nr_stripes) {
5844
		if (grow_one_stripe(conf, hash, GFP_KERNEL))
5845 5846
			conf->max_nr_stripes++;
		else break;
5847
		hash = (hash + 1) % NR_STRIPE_HASH_LOCKS;
5848
	}
5849 5850 5851 5852 5853
	return 0;
}
EXPORT_SYMBOL(raid5_set_cache_size);

static ssize_t
5854
raid5_store_stripe_cache_size(struct mddev *mddev, const char *page, size_t len)
5855
{
5856
	struct r5conf *conf;
5857 5858 5859 5860 5861
	unsigned long new;
	int err;

	if (len >= PAGE_SIZE)
		return -EINVAL;
5862
	if (kstrtoul(page, 10, &new))
5863
		return -EINVAL;
5864
	err = mddev_lock(mddev);
5865 5866
	if (err)
		return err;
5867 5868 5869 5870 5871 5872 5873 5874
	conf = mddev->private;
	if (!conf)
		err = -ENODEV;
	else
		err = raid5_set_cache_size(mddev, new);
	mddev_unlock(mddev);

	return err ?: len;
5875
}
5876

5877 5878 5879 5880
static struct md_sysfs_entry
raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR,
				raid5_show_stripe_cache_size,
				raid5_store_stripe_cache_size);
5881

5882 5883 5884 5885 5886 5887 5888 5889 5890 5891 5892 5893 5894 5895 5896 5897 5898 5899 5900 5901 5902 5903 5904 5905 5906 5907 5908 5909 5910 5911 5912 5913 5914 5915 5916 5917 5918 5919 5920 5921 5922 5923 5924
static ssize_t
raid5_show_rmw_level(struct mddev  *mddev, char *page)
{
	struct r5conf *conf = mddev->private;
	if (conf)
		return sprintf(page, "%d\n", conf->rmw_level);
	else
		return 0;
}

static ssize_t
raid5_store_rmw_level(struct mddev  *mddev, const char *page, size_t len)
{
	struct r5conf *conf = mddev->private;
	unsigned long new;

	if (!conf)
		return -ENODEV;

	if (len >= PAGE_SIZE)
		return -EINVAL;

	if (kstrtoul(page, 10, &new))
		return -EINVAL;

	if (new != PARITY_DISABLE_RMW && !raid6_call.xor_syndrome)
		return -EINVAL;

	if (new != PARITY_DISABLE_RMW &&
	    new != PARITY_ENABLE_RMW &&
	    new != PARITY_PREFER_RMW)
		return -EINVAL;

	conf->rmw_level = new;
	return len;
}

static struct md_sysfs_entry
raid5_rmw_level = __ATTR(rmw_level, S_IRUGO | S_IWUSR,
			 raid5_show_rmw_level,
			 raid5_store_rmw_level);


5925
static ssize_t
5926
raid5_show_preread_threshold(struct mddev *mddev, char *page)
5927
{
5928 5929 5930 5931
	struct r5conf *conf;
	int ret = 0;
	spin_lock(&mddev->lock);
	conf = mddev->private;
5932
	if (conf)
5933 5934 5935
		ret = sprintf(page, "%d\n", conf->bypass_threshold);
	spin_unlock(&mddev->lock);
	return ret;
5936 5937 5938
}

static ssize_t
5939
raid5_store_preread_threshold(struct mddev *mddev, const char *page, size_t len)
5940
{
5941
	struct r5conf *conf;
5942
	unsigned long new;
5943 5944
	int err;

5945 5946
	if (len >= PAGE_SIZE)
		return -EINVAL;
5947
	if (kstrtoul(page, 10, &new))
5948
		return -EINVAL;
5949 5950 5951 5952 5953 5954 5955 5956 5957 5958 5959 5960 5961

	err = mddev_lock(mddev);
	if (err)
		return err;
	conf = mddev->private;
	if (!conf)
		err = -ENODEV;
	else if (new > conf->max_nr_stripes)
		err = -EINVAL;
	else
		conf->bypass_threshold = new;
	mddev_unlock(mddev);
	return err ?: len;
5962 5963 5964 5965 5966 5967 5968 5969
}

static struct md_sysfs_entry
raid5_preread_bypass_threshold = __ATTR(preread_bypass_threshold,
					S_IRUGO | S_IWUSR,
					raid5_show_preread_threshold,
					raid5_store_preread_threshold);

5970 5971 5972
static ssize_t
raid5_show_skip_copy(struct mddev *mddev, char *page)
{
5973 5974 5975 5976
	struct r5conf *conf;
	int ret = 0;
	spin_lock(&mddev->lock);
	conf = mddev->private;
5977
	if (conf)
5978 5979 5980
		ret = sprintf(page, "%d\n", conf->skip_copy);
	spin_unlock(&mddev->lock);
	return ret;
5981 5982 5983 5984 5985
}

static ssize_t
raid5_store_skip_copy(struct mddev *mddev, const char *page, size_t len)
{
5986
	struct r5conf *conf;
5987
	unsigned long new;
5988 5989
	int err;

5990 5991 5992 5993 5994
	if (len >= PAGE_SIZE)
		return -EINVAL;
	if (kstrtoul(page, 10, &new))
		return -EINVAL;
	new = !!new;
5995 5996 5997 5998 5999 6000 6001 6002 6003 6004 6005 6006 6007 6008 6009 6010 6011 6012 6013 6014

	err = mddev_lock(mddev);
	if (err)
		return err;
	conf = mddev->private;
	if (!conf)
		err = -ENODEV;
	else if (new != conf->skip_copy) {
		mddev_suspend(mddev);
		conf->skip_copy = new;
		if (new)
			mddev->queue->backing_dev_info.capabilities |=
				BDI_CAP_STABLE_WRITES;
		else
			mddev->queue->backing_dev_info.capabilities &=
				~BDI_CAP_STABLE_WRITES;
		mddev_resume(mddev);
	}
	mddev_unlock(mddev);
	return err ?: len;
6015 6016 6017 6018 6019 6020 6021
}

static struct md_sysfs_entry
raid5_skip_copy = __ATTR(skip_copy, S_IRUGO | S_IWUSR,
					raid5_show_skip_copy,
					raid5_store_skip_copy);

6022
static ssize_t
6023
stripe_cache_active_show(struct mddev *mddev, char *page)
6024
{
6025
	struct r5conf *conf = mddev->private;
6026 6027 6028 6029
	if (conf)
		return sprintf(page, "%d\n", atomic_read(&conf->active_stripes));
	else
		return 0;
6030 6031
}

6032 6033
static struct md_sysfs_entry
raid5_stripecache_active = __ATTR_RO(stripe_cache_active);
6034

6035 6036 6037
static ssize_t
raid5_show_group_thread_cnt(struct mddev *mddev, char *page)
{
6038 6039 6040 6041
	struct r5conf *conf;
	int ret = 0;
	spin_lock(&mddev->lock);
	conf = mddev->private;
6042
	if (conf)
6043 6044 6045
		ret = sprintf(page, "%d\n", conf->worker_cnt_per_group);
	spin_unlock(&mddev->lock);
	return ret;
6046 6047
}

6048 6049 6050 6051
static int alloc_thread_groups(struct r5conf *conf, int cnt,
			       int *group_cnt,
			       int *worker_cnt_per_group,
			       struct r5worker_group **worker_groups);
6052 6053 6054
static ssize_t
raid5_store_group_thread_cnt(struct mddev *mddev, const char *page, size_t len)
{
6055
	struct r5conf *conf;
6056 6057
	unsigned long new;
	int err;
6058 6059
	struct r5worker_group *new_groups, *old_groups;
	int group_cnt, worker_cnt_per_group;
6060 6061 6062 6063 6064 6065

	if (len >= PAGE_SIZE)
		return -EINVAL;
	if (kstrtoul(page, 10, &new))
		return -EINVAL;

6066 6067 6068 6069 6070 6071 6072 6073
	err = mddev_lock(mddev);
	if (err)
		return err;
	conf = mddev->private;
	if (!conf)
		err = -ENODEV;
	else if (new != conf->worker_cnt_per_group) {
		mddev_suspend(mddev);
6074

6075 6076 6077
		old_groups = conf->worker_groups;
		if (old_groups)
			flush_workqueue(raid5_wq);
6078

6079 6080 6081 6082 6083 6084 6085 6086 6087
		err = alloc_thread_groups(conf, new,
					  &group_cnt, &worker_cnt_per_group,
					  &new_groups);
		if (!err) {
			spin_lock_irq(&conf->device_lock);
			conf->group_cnt = group_cnt;
			conf->worker_cnt_per_group = worker_cnt_per_group;
			conf->worker_groups = new_groups;
			spin_unlock_irq(&conf->device_lock);
6088

6089 6090 6091 6092 6093
			if (old_groups)
				kfree(old_groups[0].workers);
			kfree(old_groups);
		}
		mddev_resume(mddev);
6094
	}
6095
	mddev_unlock(mddev);
6096

6097
	return err ?: len;
6098 6099 6100 6101 6102 6103 6104
}

static struct md_sysfs_entry
raid5_group_thread_cnt = __ATTR(group_thread_cnt, S_IRUGO | S_IWUSR,
				raid5_show_group_thread_cnt,
				raid5_store_group_thread_cnt);

6105
static struct attribute *raid5_attrs[] =  {
6106 6107
	&raid5_stripecache_size.attr,
	&raid5_stripecache_active.attr,
6108
	&raid5_preread_bypass_threshold.attr,
6109
	&raid5_group_thread_cnt.attr,
6110
	&raid5_skip_copy.attr,
6111
	&raid5_rmw_level.attr,
6112 6113
	NULL,
};
6114 6115 6116
static struct attribute_group raid5_attrs_group = {
	.name = NULL,
	.attrs = raid5_attrs,
6117 6118
};

6119 6120 6121 6122
static int alloc_thread_groups(struct r5conf *conf, int cnt,
			       int *group_cnt,
			       int *worker_cnt_per_group,
			       struct r5worker_group **worker_groups)
6123
{
6124
	int i, j, k;
6125 6126 6127
	ssize_t size;
	struct r5worker *workers;

6128
	*worker_cnt_per_group = cnt;
6129
	if (cnt == 0) {
6130 6131
		*group_cnt = 0;
		*worker_groups = NULL;
6132 6133
		return 0;
	}
6134
	*group_cnt = num_possible_nodes();
6135
	size = sizeof(struct r5worker) * cnt;
6136 6137 6138 6139
	workers = kzalloc(size * *group_cnt, GFP_NOIO);
	*worker_groups = kzalloc(sizeof(struct r5worker_group) *
				*group_cnt, GFP_NOIO);
	if (!*worker_groups || !workers) {
6140
		kfree(workers);
6141
		kfree(*worker_groups);
6142 6143 6144
		return -ENOMEM;
	}

6145
	for (i = 0; i < *group_cnt; i++) {
6146 6147
		struct r5worker_group *group;

6148
		group = &(*worker_groups)[i];
6149 6150 6151 6152 6153
		INIT_LIST_HEAD(&group->handle_list);
		group->conf = conf;
		group->workers = workers + i * cnt;

		for (j = 0; j < cnt; j++) {
6154 6155 6156 6157 6158 6159
			struct r5worker *worker = group->workers + j;
			worker->group = group;
			INIT_WORK(&worker->work, raid5_do_work);

			for (k = 0; k < NR_STRIPE_HASH_LOCKS; k++)
				INIT_LIST_HEAD(worker->temp_inactive_list + k);
6160 6161 6162 6163 6164 6165 6166 6167 6168 6169 6170 6171 6172 6173
		}
	}

	return 0;
}

static void free_thread_groups(struct r5conf *conf)
{
	if (conf->worker_groups)
		kfree(conf->worker_groups[0].workers);
	kfree(conf->worker_groups);
	conf->worker_groups = NULL;
}

6174
static sector_t
6175
raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks)
6176
{
6177
	struct r5conf *conf = mddev->private;
6178 6179 6180

	if (!sectors)
		sectors = mddev->dev_sectors;
6181
	if (!raid_disks)
6182
		/* size is defined by the smallest of previous and new size */
6183
		raid_disks = min(conf->raid_disks, conf->previous_raid_disks);
6184

6185
	sectors &= ~((sector_t)mddev->chunk_sectors - 1);
6186
	sectors &= ~((sector_t)mddev->new_chunk_sectors - 1);
6187 6188 6189
	return sectors * (raid_disks - conf->max_degraded);
}

6190 6191 6192
static void free_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu)
{
	safe_put_page(percpu->spare_page);
6193 6194
	if (percpu->scribble)
		flex_array_free(percpu->scribble);
6195 6196 6197 6198 6199 6200 6201 6202 6203
	percpu->spare_page = NULL;
	percpu->scribble = NULL;
}

static int alloc_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu)
{
	if (conf->level == 6 && !percpu->spare_page)
		percpu->spare_page = alloc_page(GFP_KERNEL);
	if (!percpu->scribble)
6204 6205 6206
		percpu->scribble = scribble_alloc(max(conf->raid_disks,
			conf->previous_raid_disks), conf->chunk_sectors /
			STRIPE_SECTORS, GFP_KERNEL);
6207 6208 6209 6210 6211 6212 6213 6214 6215

	if (!percpu->scribble || (conf->level == 6 && !percpu->spare_page)) {
		free_scratch_buffer(conf, percpu);
		return -ENOMEM;
	}

	return 0;
}

6216
static void raid5_free_percpu(struct r5conf *conf)
6217 6218 6219 6220 6221 6222 6223 6224 6225
{
	unsigned long cpu;

	if (!conf->percpu)
		return;

#ifdef CONFIG_HOTPLUG_CPU
	unregister_cpu_notifier(&conf->cpu_notify);
#endif
6226 6227 6228 6229

	get_online_cpus();
	for_each_possible_cpu(cpu)
		free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));
6230 6231 6232 6233 6234
	put_online_cpus();

	free_percpu(conf->percpu);
}

6235
static void free_conf(struct r5conf *conf)
6236
{
6237
	free_thread_groups(conf);
6238
	shrink_stripes(conf);
6239
	raid5_free_percpu(conf);
6240 6241 6242 6243 6244
	kfree(conf->disks);
	kfree(conf->stripe_hashtbl);
	kfree(conf);
}

6245 6246 6247 6248
#ifdef CONFIG_HOTPLUG_CPU
static int raid456_cpu_notify(struct notifier_block *nfb, unsigned long action,
			      void *hcpu)
{
6249
	struct r5conf *conf = container_of(nfb, struct r5conf, cpu_notify);
6250 6251 6252 6253 6254 6255
	long cpu = (long)hcpu;
	struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu);

	switch (action) {
	case CPU_UP_PREPARE:
	case CPU_UP_PREPARE_FROZEN:
6256
		if (alloc_scratch_buffer(conf, percpu)) {
6257 6258
			pr_err("%s: failed memory allocation for cpu%ld\n",
			       __func__, cpu);
6259
			return notifier_from_errno(-ENOMEM);
6260 6261 6262 6263
		}
		break;
	case CPU_DEAD:
	case CPU_DEAD_FROZEN:
6264
		free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));
6265 6266 6267 6268 6269 6270 6271 6272
		break;
	default:
		break;
	}
	return NOTIFY_OK;
}
#endif

6273
static int raid5_alloc_percpu(struct r5conf *conf)
6274 6275
{
	unsigned long cpu;
6276
	int err = 0;
6277

6278 6279
	conf->percpu = alloc_percpu(struct raid5_percpu);
	if (!conf->percpu)
6280
		return -ENOMEM;
6281 6282 6283 6284 6285 6286 6287 6288

#ifdef CONFIG_HOTPLUG_CPU
	conf->cpu_notify.notifier_call = raid456_cpu_notify;
	conf->cpu_notify.priority = 0;
	err = register_cpu_notifier(&conf->cpu_notify);
	if (err)
		return err;
#endif
6289 6290 6291

	get_online_cpus();
	for_each_present_cpu(cpu) {
6292 6293 6294 6295
		err = alloc_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));
		if (err) {
			pr_err("%s: failed memory allocation for cpu%ld\n",
			       __func__, cpu);
6296 6297 6298 6299 6300 6301 6302 6303
			break;
		}
	}
	put_online_cpus();

	return err;
}

6304
static struct r5conf *setup_conf(struct mddev *mddev)
L
Linus Torvalds 已提交
6305
{
6306
	struct r5conf *conf;
6307
	int raid_disk, memory, max_disks;
6308
	struct md_rdev *rdev;
L
Linus Torvalds 已提交
6309
	struct disk_info *disk;
6310
	char pers_name[6];
6311
	int i;
6312 6313
	int group_cnt, worker_cnt_per_group;
	struct r5worker_group *new_group;
L
Linus Torvalds 已提交
6314

N
NeilBrown 已提交
6315 6316 6317
	if (mddev->new_level != 5
	    && mddev->new_level != 4
	    && mddev->new_level != 6) {
6318
		printk(KERN_ERR "md/raid:%s: raid level not set to 4/5/6 (%d)\n",
N
NeilBrown 已提交
6319 6320
		       mdname(mddev), mddev->new_level);
		return ERR_PTR(-EIO);
L
Linus Torvalds 已提交
6321
	}
N
NeilBrown 已提交
6322 6323 6324 6325
	if ((mddev->new_level == 5
	     && !algorithm_valid_raid5(mddev->new_layout)) ||
	    (mddev->new_level == 6
	     && !algorithm_valid_raid6(mddev->new_layout))) {
6326
		printk(KERN_ERR "md/raid:%s: layout %d not supported\n",
N
NeilBrown 已提交
6327 6328
		       mdname(mddev), mddev->new_layout);
		return ERR_PTR(-EIO);
6329
	}
N
NeilBrown 已提交
6330
	if (mddev->new_level == 6 && mddev->raid_disks < 4) {
6331
		printk(KERN_ERR "md/raid:%s: not enough configured devices (%d, minimum 4)\n",
N
NeilBrown 已提交
6332 6333
		       mdname(mddev), mddev->raid_disks);
		return ERR_PTR(-EINVAL);
6334 6335
	}

6336 6337 6338
	if (!mddev->new_chunk_sectors ||
	    (mddev->new_chunk_sectors << 9) % PAGE_SIZE ||
	    !is_power_of_2(mddev->new_chunk_sectors)) {
6339 6340
		printk(KERN_ERR "md/raid:%s: invalid chunk size %d\n",
		       mdname(mddev), mddev->new_chunk_sectors << 9);
N
NeilBrown 已提交
6341
		return ERR_PTR(-EINVAL);
6342 6343
	}

6344
	conf = kzalloc(sizeof(struct r5conf), GFP_KERNEL);
N
NeilBrown 已提交
6345
	if (conf == NULL)
L
Linus Torvalds 已提交
6346
		goto abort;
6347
	/* Don't enable multi-threading by default*/
6348 6349 6350 6351 6352 6353
	if (!alloc_thread_groups(conf, 0, &group_cnt, &worker_cnt_per_group,
				 &new_group)) {
		conf->group_cnt = group_cnt;
		conf->worker_cnt_per_group = worker_cnt_per_group;
		conf->worker_groups = new_group;
	} else
6354
		goto abort;
6355
	spin_lock_init(&conf->device_lock);
6356
	seqcount_init(&conf->gen_lock);
6357 6358 6359 6360 6361 6362
	init_waitqueue_head(&conf->wait_for_stripe);
	init_waitqueue_head(&conf->wait_for_overlap);
	INIT_LIST_HEAD(&conf->handle_list);
	INIT_LIST_HEAD(&conf->hold_list);
	INIT_LIST_HEAD(&conf->delayed_list);
	INIT_LIST_HEAD(&conf->bitmap_list);
S
Shaohua Li 已提交
6363
	init_llist_head(&conf->released_stripes);
6364 6365 6366 6367
	atomic_set(&conf->active_stripes, 0);
	atomic_set(&conf->preread_active_stripes, 0);
	atomic_set(&conf->active_aligned_reads, 0);
	conf->bypass_threshold = BYPASS_THRESHOLD;
6368
	conf->recovery_disabled = mddev->recovery_disabled - 1;
N
NeilBrown 已提交
6369 6370 6371 6372 6373

	conf->raid_disks = mddev->raid_disks;
	if (mddev->reshape_position == MaxSector)
		conf->previous_raid_disks = mddev->raid_disks;
	else
6374
		conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks;
6375
	max_disks = max(conf->raid_disks, conf->previous_raid_disks);
6376

6377
	conf->disks = kzalloc(max_disks * sizeof(struct disk_info),
6378 6379 6380
			      GFP_KERNEL);
	if (!conf->disks)
		goto abort;
6381

L
Linus Torvalds 已提交
6382 6383
	conf->mddev = mddev;

6384
	if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL)
L
Linus Torvalds 已提交
6385 6386
		goto abort;

6387 6388 6389 6390 6391 6392 6393 6394 6395 6396 6397 6398 6399 6400 6401
	/* We init hash_locks[0] separately to that it can be used
	 * as the reference lock in the spin_lock_nest_lock() call
	 * in lock_all_device_hash_locks_irq in order to convince
	 * lockdep that we know what we are doing.
	 */
	spin_lock_init(conf->hash_locks);
	for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++)
		spin_lock_init(conf->hash_locks + i);

	for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
		INIT_LIST_HEAD(conf->inactive_list + i);

	for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
		INIT_LIST_HEAD(conf->temp_inactive_list + i);

6402
	conf->level = mddev->new_level;
6403
	conf->chunk_sectors = mddev->new_chunk_sectors;
6404 6405 6406
	if (raid5_alloc_percpu(conf) != 0)
		goto abort;

6407
	pr_debug("raid456: run(%s) called.\n", mdname(mddev));
L
Linus Torvalds 已提交
6408

N
NeilBrown 已提交
6409
	rdev_for_each(rdev, mddev) {
L
Linus Torvalds 已提交
6410
		raid_disk = rdev->raid_disk;
6411
		if (raid_disk >= max_disks
L
Linus Torvalds 已提交
6412 6413 6414 6415
		    || raid_disk < 0)
			continue;
		disk = conf->disks + raid_disk;

6416 6417 6418 6419 6420 6421 6422 6423 6424
		if (test_bit(Replacement, &rdev->flags)) {
			if (disk->replacement)
				goto abort;
			disk->replacement = rdev;
		} else {
			if (disk->rdev)
				goto abort;
			disk->rdev = rdev;
		}
L
Linus Torvalds 已提交
6425

6426
		if (test_bit(In_sync, &rdev->flags)) {
L
Linus Torvalds 已提交
6427
			char b[BDEVNAME_SIZE];
6428 6429 6430
			printk(KERN_INFO "md/raid:%s: device %s operational as raid"
			       " disk %d\n",
			       mdname(mddev), bdevname(rdev->bdev, b), raid_disk);
J
Jonathan Brassow 已提交
6431
		} else if (rdev->saved_raid_disk != raid_disk)
6432 6433
			/* Cannot rely on bitmap to complete recovery */
			conf->fullsync = 1;
L
Linus Torvalds 已提交
6434 6435
	}

N
NeilBrown 已提交
6436
	conf->level = mddev->new_level;
6437
	if (conf->level == 6) {
6438
		conf->max_degraded = 2;
6439 6440 6441 6442 6443
		if (raid6_call.xor_syndrome)
			conf->rmw_level = PARITY_ENABLE_RMW;
		else
			conf->rmw_level = PARITY_DISABLE_RMW;
	} else {
6444
		conf->max_degraded = 1;
6445 6446
		conf->rmw_level = PARITY_ENABLE_RMW;
	}
N
NeilBrown 已提交
6447
	conf->algorithm = mddev->new_layout;
6448
	conf->reshape_progress = mddev->reshape_position;
6449
	if (conf->reshape_progress != MaxSector) {
6450
		conf->prev_chunk_sectors = mddev->chunk_sectors;
6451 6452
		conf->prev_algo = mddev->layout;
	}
L
Linus Torvalds 已提交
6453

N
NeilBrown 已提交
6454
	memory = conf->max_nr_stripes * (sizeof(struct stripe_head) +
6455
		 max_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024;
6456
	atomic_set(&conf->empty_inactive_list_nr, NR_STRIPE_HASH_LOCKS);
6457
	if (grow_stripes(conf, NR_STRIPES)) {
N
NeilBrown 已提交
6458
		printk(KERN_ERR
6459 6460
		       "md/raid:%s: couldn't allocate %dkB for buffers\n",
		       mdname(mddev), memory);
N
NeilBrown 已提交
6461 6462
		goto abort;
	} else
6463 6464
		printk(KERN_INFO "md/raid:%s: allocated %dkB\n",
		       mdname(mddev), memory);
L
Linus Torvalds 已提交
6465

6466 6467
	sprintf(pers_name, "raid%d", mddev->new_level);
	conf->thread = md_register_thread(raid5d, mddev, pers_name);
N
NeilBrown 已提交
6468 6469
	if (!conf->thread) {
		printk(KERN_ERR
6470
		       "md/raid:%s: couldn't allocate thread.\n",
N
NeilBrown 已提交
6471
		       mdname(mddev));
6472 6473
		goto abort;
	}
N
NeilBrown 已提交
6474 6475 6476 6477 6478

	return conf;

 abort:
	if (conf) {
6479
		free_conf(conf);
N
NeilBrown 已提交
6480 6481 6482 6483 6484
		return ERR_PTR(-EIO);
	} else
		return ERR_PTR(-ENOMEM);
}

6485 6486 6487 6488 6489 6490 6491 6492 6493 6494 6495 6496
static int only_parity(int raid_disk, int algo, int raid_disks, int max_degraded)
{
	switch (algo) {
	case ALGORITHM_PARITY_0:
		if (raid_disk < max_degraded)
			return 1;
		break;
	case ALGORITHM_PARITY_N:
		if (raid_disk >= raid_disks - max_degraded)
			return 1;
		break;
	case ALGORITHM_PARITY_0_6:
6497
		if (raid_disk == 0 ||
6498 6499 6500 6501 6502 6503 6504 6505 6506 6507 6508 6509 6510
		    raid_disk == raid_disks - 1)
			return 1;
		break;
	case ALGORITHM_LEFT_ASYMMETRIC_6:
	case ALGORITHM_RIGHT_ASYMMETRIC_6:
	case ALGORITHM_LEFT_SYMMETRIC_6:
	case ALGORITHM_RIGHT_SYMMETRIC_6:
		if (raid_disk == raid_disks - 1)
			return 1;
	}
	return 0;
}

6511
static int run(struct mddev *mddev)
N
NeilBrown 已提交
6512
{
6513
	struct r5conf *conf;
6514
	int working_disks = 0;
6515
	int dirty_parity_disks = 0;
6516
	struct md_rdev *rdev;
6517
	sector_t reshape_offset = 0;
6518
	int i;
6519 6520
	long long min_offset_diff = 0;
	int first = 1;
N
NeilBrown 已提交
6521

6522
	if (mddev->recovery_cp != MaxSector)
6523
		printk(KERN_NOTICE "md/raid:%s: not clean"
6524 6525
		       " -- starting background reconstruction\n",
		       mdname(mddev));
6526 6527 6528 6529 6530 6531 6532 6533 6534 6535 6536 6537 6538 6539 6540 6541 6542

	rdev_for_each(rdev, mddev) {
		long long diff;
		if (rdev->raid_disk < 0)
			continue;
		diff = (rdev->new_data_offset - rdev->data_offset);
		if (first) {
			min_offset_diff = diff;
			first = 0;
		} else if (mddev->reshape_backwards &&
			 diff < min_offset_diff)
			min_offset_diff = diff;
		else if (!mddev->reshape_backwards &&
			 diff > min_offset_diff)
			min_offset_diff = diff;
	}

N
NeilBrown 已提交
6543 6544
	if (mddev->reshape_position != MaxSector) {
		/* Check that we can continue the reshape.
6545 6546 6547 6548 6549 6550 6551 6552 6553 6554
		 * Difficulties arise if the stripe we would write to
		 * next is at or after the stripe we would read from next.
		 * For a reshape that changes the number of devices, this
		 * is only possible for a very short time, and mdadm makes
		 * sure that time appears to have past before assembling
		 * the array.  So we fail if that time hasn't passed.
		 * For a reshape that keeps the number of devices the same
		 * mdadm must be monitoring the reshape can keeping the
		 * critical areas read-only and backed up.  It will start
		 * the array in read-only mode, so we check for that.
N
NeilBrown 已提交
6555 6556 6557
		 */
		sector_t here_new, here_old;
		int old_disks;
6558
		int max_degraded = (mddev->level == 6 ? 2 : 1);
N
NeilBrown 已提交
6559

6560
		if (mddev->new_level != mddev->level) {
6561
			printk(KERN_ERR "md/raid:%s: unsupported reshape "
N
NeilBrown 已提交
6562 6563 6564 6565 6566 6567 6568 6569 6570 6571
			       "required - aborting.\n",
			       mdname(mddev));
			return -EINVAL;
		}
		old_disks = mddev->raid_disks - mddev->delta_disks;
		/* reshape_position must be on a new-stripe boundary, and one
		 * further up in new geometry must map after here in old
		 * geometry.
		 */
		here_new = mddev->reshape_position;
6572
		if (sector_div(here_new, mddev->new_chunk_sectors *
N
NeilBrown 已提交
6573
			       (mddev->raid_disks - max_degraded))) {
6574 6575
			printk(KERN_ERR "md/raid:%s: reshape_position not "
			       "on a stripe boundary\n", mdname(mddev));
N
NeilBrown 已提交
6576 6577
			return -EINVAL;
		}
6578
		reshape_offset = here_new * mddev->new_chunk_sectors;
N
NeilBrown 已提交
6579 6580
		/* here_new is the stripe we will write to */
		here_old = mddev->reshape_position;
6581
		sector_div(here_old, mddev->chunk_sectors *
N
NeilBrown 已提交
6582 6583 6584
			   (old_disks-max_degraded));
		/* here_old is the first stripe that we might need to read
		 * from */
6585
		if (mddev->delta_disks == 0) {
6586 6587 6588 6589 6590 6591
			if ((here_new * mddev->new_chunk_sectors !=
			     here_old * mddev->chunk_sectors)) {
				printk(KERN_ERR "md/raid:%s: reshape position is"
				       " confused - aborting\n", mdname(mddev));
				return -EINVAL;
			}
6592
			/* We cannot be sure it is safe to start an in-place
6593
			 * reshape.  It is only safe if user-space is monitoring
6594 6595 6596 6597 6598
			 * and taking constant backups.
			 * mdadm always starts a situation like this in
			 * readonly mode so it can take control before
			 * allowing any writes.  So just check for that.
			 */
6599 6600 6601 6602 6603 6604 6605
			if (abs(min_offset_diff) >= mddev->chunk_sectors &&
			    abs(min_offset_diff) >= mddev->new_chunk_sectors)
				/* not really in-place - so OK */;
			else if (mddev->ro == 0) {
				printk(KERN_ERR "md/raid:%s: in-place reshape "
				       "must be started in read-only mode "
				       "- aborting\n",
6606
				       mdname(mddev));
6607 6608
				return -EINVAL;
			}
6609
		} else if (mddev->reshape_backwards
6610
		    ? (here_new * mddev->new_chunk_sectors + min_offset_diff <=
6611 6612
		       here_old * mddev->chunk_sectors)
		    : (here_new * mddev->new_chunk_sectors >=
6613
		       here_old * mddev->chunk_sectors + (-min_offset_diff))) {
N
NeilBrown 已提交
6614
			/* Reading from the same stripe as writing to - bad */
6615 6616 6617
			printk(KERN_ERR "md/raid:%s: reshape_position too early for "
			       "auto-recovery - aborting.\n",
			       mdname(mddev));
N
NeilBrown 已提交
6618 6619
			return -EINVAL;
		}
6620 6621
		printk(KERN_INFO "md/raid:%s: reshape will continue\n",
		       mdname(mddev));
N
NeilBrown 已提交
6622 6623 6624 6625
		/* OK, we should be able to continue; */
	} else {
		BUG_ON(mddev->level != mddev->new_level);
		BUG_ON(mddev->layout != mddev->new_layout);
6626
		BUG_ON(mddev->chunk_sectors != mddev->new_chunk_sectors);
N
NeilBrown 已提交
6627
		BUG_ON(mddev->delta_disks != 0);
L
Linus Torvalds 已提交
6628
	}
N
NeilBrown 已提交
6629

6630 6631 6632 6633 6634
	if (mddev->private == NULL)
		conf = setup_conf(mddev);
	else
		conf = mddev->private;

N
NeilBrown 已提交
6635 6636 6637
	if (IS_ERR(conf))
		return PTR_ERR(conf);

6638
	conf->min_offset_diff = min_offset_diff;
N
NeilBrown 已提交
6639 6640 6641 6642
	mddev->thread = conf->thread;
	conf->thread = NULL;
	mddev->private = conf;

6643 6644 6645 6646 6647 6648 6649 6650 6651 6652 6653
	for (i = 0; i < conf->raid_disks && conf->previous_raid_disks;
	     i++) {
		rdev = conf->disks[i].rdev;
		if (!rdev && conf->disks[i].replacement) {
			/* The replacement is all we have yet */
			rdev = conf->disks[i].replacement;
			conf->disks[i].replacement = NULL;
			clear_bit(Replacement, &rdev->flags);
			conf->disks[i].rdev = rdev;
		}
		if (!rdev)
6654
			continue;
6655 6656 6657 6658 6659 6660 6661
		if (conf->disks[i].replacement &&
		    conf->reshape_progress != MaxSector) {
			/* replacements and reshape simply do not mix. */
			printk(KERN_ERR "md: cannot handle concurrent "
			       "replacement and reshape.\n");
			goto abort;
		}
6662
		if (test_bit(In_sync, &rdev->flags)) {
N
NeilBrown 已提交
6663
			working_disks++;
6664 6665
			continue;
		}
6666 6667 6668 6669 6670 6671 6672 6673 6674 6675 6676 6677
		/* This disc is not fully in-sync.  However if it
		 * just stored parity (beyond the recovery_offset),
		 * when we don't need to be concerned about the
		 * array being dirty.
		 * When reshape goes 'backwards', we never have
		 * partially completed devices, so we only need
		 * to worry about reshape going forwards.
		 */
		/* Hack because v0.91 doesn't store recovery_offset properly. */
		if (mddev->major_version == 0 &&
		    mddev->minor_version > 90)
			rdev->recovery_offset = reshape_offset;
6678

6679 6680 6681 6682 6683 6684 6685 6686 6687 6688 6689 6690 6691 6692 6693
		if (rdev->recovery_offset < reshape_offset) {
			/* We need to check old and new layout */
			if (!only_parity(rdev->raid_disk,
					 conf->algorithm,
					 conf->raid_disks,
					 conf->max_degraded))
				continue;
		}
		if (!only_parity(rdev->raid_disk,
				 conf->prev_algo,
				 conf->previous_raid_disks,
				 conf->max_degraded))
			continue;
		dirty_parity_disks++;
	}
N
NeilBrown 已提交
6694

6695 6696 6697
	/*
	 * 0 for a fully functional array, 1 or 2 for a degraded array.
	 */
6698
	mddev->degraded = calc_degraded(conf);
N
NeilBrown 已提交
6699

6700
	if (has_failed(conf)) {
6701
		printk(KERN_ERR "md/raid:%s: not enough operational devices"
L
Linus Torvalds 已提交
6702
			" (%d/%d failed)\n",
6703
			mdname(mddev), mddev->degraded, conf->raid_disks);
L
Linus Torvalds 已提交
6704 6705 6706
		goto abort;
	}

N
NeilBrown 已提交
6707
	/* device size must be a multiple of chunk size */
6708
	mddev->dev_sectors &= ~(mddev->chunk_sectors - 1);
N
NeilBrown 已提交
6709 6710
	mddev->resync_max_sectors = mddev->dev_sectors;

6711
	if (mddev->degraded > dirty_parity_disks &&
L
Linus Torvalds 已提交
6712
	    mddev->recovery_cp != MaxSector) {
6713 6714
		if (mddev->ok_start_degraded)
			printk(KERN_WARNING
6715 6716
			       "md/raid:%s: starting dirty degraded array"
			       " - data corruption possible.\n",
6717 6718 6719
			       mdname(mddev));
		else {
			printk(KERN_ERR
6720
			       "md/raid:%s: cannot start dirty degraded array.\n",
6721 6722 6723
			       mdname(mddev));
			goto abort;
		}
L
Linus Torvalds 已提交
6724 6725 6726
	}

	if (mddev->degraded == 0)
6727 6728
		printk(KERN_INFO "md/raid:%s: raid level %d active with %d out of %d"
		       " devices, algorithm %d\n", mdname(mddev), conf->level,
6729 6730
		       mddev->raid_disks-mddev->degraded, mddev->raid_disks,
		       mddev->new_layout);
L
Linus Torvalds 已提交
6731
	else
6732 6733 6734 6735 6736
		printk(KERN_ALERT "md/raid:%s: raid level %d active with %d"
		       " out of %d devices, algorithm %d\n",
		       mdname(mddev), conf->level,
		       mddev->raid_disks - mddev->degraded,
		       mddev->raid_disks, mddev->new_layout);
L
Linus Torvalds 已提交
6737 6738 6739

	print_raid5_conf(conf);

6740 6741
	if (conf->reshape_progress != MaxSector) {
		conf->reshape_safe = conf->reshape_progress;
6742 6743 6744 6745 6746 6747
		atomic_set(&conf->reshape_stripes, 0);
		clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
		clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
		set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
		set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
		mddev->sync_thread = md_register_thread(md_do_sync, mddev,
6748
							"reshape");
6749 6750
	}

L
Linus Torvalds 已提交
6751
	/* Ok, everything is just fine now */
6752 6753
	if (mddev->to_remove == &raid5_attrs_group)
		mddev->to_remove = NULL;
N
NeilBrown 已提交
6754 6755
	else if (mddev->kobj.sd &&
	    sysfs_create_group(&mddev->kobj, &raid5_attrs_group))
6756
		printk(KERN_WARNING
6757
		       "raid5: failed to create sysfs attributes for %s\n",
6758
		       mdname(mddev));
6759
	md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
6760

6761
	if (mddev->queue) {
6762
		int chunk_size;
S
Shaohua Li 已提交
6763
		bool discard_supported = true;
6764 6765 6766 6767 6768 6769 6770 6771 6772
		/* read-ahead size must cover two whole stripes, which
		 * is 2 * (datadisks) * chunksize where 'n' is the
		 * number of raid devices
		 */
		int data_disks = conf->previous_raid_disks - conf->max_degraded;
		int stripe = data_disks *
			((mddev->chunk_sectors << 9) / PAGE_SIZE);
		if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
			mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
N
NeilBrown 已提交
6773

6774 6775 6776 6777
		chunk_size = mddev->chunk_sectors << 9;
		blk_queue_io_min(mddev->queue, chunk_size);
		blk_queue_io_opt(mddev->queue, chunk_size *
				 (conf->raid_disks - conf->max_degraded));
6778
		mddev->queue->limits.raid_partial_stripes_expensive = 1;
S
Shaohua Li 已提交
6779 6780 6781 6782 6783
		/*
		 * We can only discard a whole stripe. It doesn't make sense to
		 * discard data disk but write parity disk
		 */
		stripe = stripe * PAGE_SIZE;
6784 6785 6786 6787
		/* Round up to power of 2, as discard handling
		 * currently assumes that */
		while ((stripe-1) & stripe)
			stripe = (stripe | (stripe-1)) + 1;
S
Shaohua Li 已提交
6788 6789 6790 6791
		mddev->queue->limits.discard_alignment = stripe;
		mddev->queue->limits.discard_granularity = stripe;
		/*
		 * unaligned part of discard request will be ignored, so can't
6792
		 * guarantee discard_zeroes_data
S
Shaohua Li 已提交
6793 6794
		 */
		mddev->queue->limits.discard_zeroes_data = 0;
6795

6796 6797
		blk_queue_max_write_same_sectors(mddev->queue, 0);

6798
		rdev_for_each(rdev, mddev) {
6799 6800
			disk_stack_limits(mddev->gendisk, rdev->bdev,
					  rdev->data_offset << 9);
6801 6802
			disk_stack_limits(mddev->gendisk, rdev->bdev,
					  rdev->new_data_offset << 9);
S
Shaohua Li 已提交
6803 6804 6805 6806 6807 6808 6809 6810 6811 6812 6813 6814 6815 6816
			/*
			 * discard_zeroes_data is required, otherwise data
			 * could be lost. Consider a scenario: discard a stripe
			 * (the stripe could be inconsistent if
			 * discard_zeroes_data is 0); write one disk of the
			 * stripe (the stripe could be inconsistent again
			 * depending on which disks are used to calculate
			 * parity); the disk is broken; The stripe data of this
			 * disk is lost.
			 */
			if (!blk_queue_discard(bdev_get_queue(rdev->bdev)) ||
			    !bdev_get_queue(rdev->bdev)->
						limits.discard_zeroes_data)
				discard_supported = false;
6817 6818 6819 6820 6821 6822 6823 6824 6825 6826 6827 6828
			/* Unfortunately, discard_zeroes_data is not currently
			 * a guarantee - just a hint.  So we only allow DISCARD
			 * if the sysadmin has confirmed that only safe devices
			 * are in use by setting a module parameter.
			 */
			if (!devices_handle_discard_safely) {
				if (discard_supported) {
					pr_info("md/raid456: discard support disabled due to uncertainty.\n");
					pr_info("Set raid456.devices_handle_discard_safely=Y to override.\n");
				}
				discard_supported = false;
			}
6829
		}
S
Shaohua Li 已提交
6830 6831 6832 6833 6834 6835 6836 6837 6838

		if (discard_supported &&
		   mddev->queue->limits.max_discard_sectors >= stripe &&
		   mddev->queue->limits.discard_granularity >= stripe)
			queue_flag_set_unlocked(QUEUE_FLAG_DISCARD,
						mddev->queue);
		else
			queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD,
						mddev->queue);
6839
	}
6840

L
Linus Torvalds 已提交
6841 6842
	return 0;
abort:
6843
	md_unregister_thread(&mddev->thread);
N
NeilBrown 已提交
6844 6845
	print_raid5_conf(conf);
	free_conf(conf);
L
Linus Torvalds 已提交
6846
	mddev->private = NULL;
6847
	printk(KERN_ALERT "md/raid:%s: failed to run raid set.\n", mdname(mddev));
L
Linus Torvalds 已提交
6848 6849 6850
	return -EIO;
}

N
NeilBrown 已提交
6851
static void raid5_free(struct mddev *mddev, void *priv)
L
Linus Torvalds 已提交
6852
{
N
NeilBrown 已提交
6853
	struct r5conf *conf = priv;
L
Linus Torvalds 已提交
6854

6855
	free_conf(conf);
6856
	mddev->to_remove = &raid5_attrs_group;
L
Linus Torvalds 已提交
6857 6858
}

6859
static void status(struct seq_file *seq, struct mddev *mddev)
L
Linus Torvalds 已提交
6860
{
6861
	struct r5conf *conf = mddev->private;
L
Linus Torvalds 已提交
6862 6863
	int i;

6864 6865
	seq_printf(seq, " level %d, %dk chunk, algorithm %d", mddev->level,
		mddev->chunk_sectors / 2, mddev->layout);
6866
	seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded);
L
Linus Torvalds 已提交
6867 6868 6869
	for (i = 0; i < conf->raid_disks; i++)
		seq_printf (seq, "%s",
			       conf->disks[i].rdev &&
6870
			       test_bit(In_sync, &conf->disks[i].rdev->flags) ? "U" : "_");
L
Linus Torvalds 已提交
6871 6872 6873
	seq_printf (seq, "]");
}

6874
static void print_raid5_conf (struct r5conf *conf)
L
Linus Torvalds 已提交
6875 6876 6877 6878
{
	int i;
	struct disk_info *tmp;

6879
	printk(KERN_DEBUG "RAID conf printout:\n");
L
Linus Torvalds 已提交
6880 6881 6882 6883
	if (!conf) {
		printk("(conf==NULL)\n");
		return;
	}
6884 6885 6886
	printk(KERN_DEBUG " --- level:%d rd:%d wd:%d\n", conf->level,
	       conf->raid_disks,
	       conf->raid_disks - conf->mddev->degraded);
L
Linus Torvalds 已提交
6887 6888 6889 6890 6891

	for (i = 0; i < conf->raid_disks; i++) {
		char b[BDEVNAME_SIZE];
		tmp = conf->disks + i;
		if (tmp->rdev)
6892 6893 6894
			printk(KERN_DEBUG " disk %d, o:%d, dev:%s\n",
			       i, !test_bit(Faulty, &tmp->rdev->flags),
			       bdevname(tmp->rdev->bdev, b));
L
Linus Torvalds 已提交
6895 6896 6897
	}
}

6898
static int raid5_spare_active(struct mddev *mddev)
L
Linus Torvalds 已提交
6899 6900
{
	int i;
6901
	struct r5conf *conf = mddev->private;
L
Linus Torvalds 已提交
6902
	struct disk_info *tmp;
6903 6904
	int count = 0;
	unsigned long flags;
L
Linus Torvalds 已提交
6905 6906 6907

	for (i = 0; i < conf->raid_disks; i++) {
		tmp = conf->disks + i;
6908 6909 6910 6911 6912 6913 6914 6915 6916 6917 6918 6919 6920 6921 6922 6923 6924 6925 6926
		if (tmp->replacement
		    && tmp->replacement->recovery_offset == MaxSector
		    && !test_bit(Faulty, &tmp->replacement->flags)
		    && !test_and_set_bit(In_sync, &tmp->replacement->flags)) {
			/* Replacement has just become active. */
			if (!tmp->rdev
			    || !test_and_clear_bit(In_sync, &tmp->rdev->flags))
				count++;
			if (tmp->rdev) {
				/* Replaced device not technically faulty,
				 * but we need to be sure it gets removed
				 * and never re-added.
				 */
				set_bit(Faulty, &tmp->rdev->flags);
				sysfs_notify_dirent_safe(
					tmp->rdev->sysfs_state);
			}
			sysfs_notify_dirent_safe(tmp->replacement->sysfs_state);
		} else if (tmp->rdev
6927
		    && tmp->rdev->recovery_offset == MaxSector
6928
		    && !test_bit(Faulty, &tmp->rdev->flags)
6929
		    && !test_and_set_bit(In_sync, &tmp->rdev->flags)) {
6930
			count++;
6931
			sysfs_notify_dirent_safe(tmp->rdev->sysfs_state);
L
Linus Torvalds 已提交
6932 6933
		}
	}
6934
	spin_lock_irqsave(&conf->device_lock, flags);
6935
	mddev->degraded = calc_degraded(conf);
6936
	spin_unlock_irqrestore(&conf->device_lock, flags);
L
Linus Torvalds 已提交
6937
	print_raid5_conf(conf);
6938
	return count;
L
Linus Torvalds 已提交
6939 6940
}

6941
static int raid5_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
L
Linus Torvalds 已提交
6942
{
6943
	struct r5conf *conf = mddev->private;
L
Linus Torvalds 已提交
6944
	int err = 0;
6945
	int number = rdev->raid_disk;
6946
	struct md_rdev **rdevp;
L
Linus Torvalds 已提交
6947 6948 6949
	struct disk_info *p = conf->disks + number;

	print_raid5_conf(conf);
6950 6951 6952 6953 6954 6955 6956 6957 6958 6959 6960 6961 6962 6963 6964 6965 6966 6967 6968 6969 6970 6971
	if (rdev == p->rdev)
		rdevp = &p->rdev;
	else if (rdev == p->replacement)
		rdevp = &p->replacement;
	else
		return 0;

	if (number >= conf->raid_disks &&
	    conf->reshape_progress == MaxSector)
		clear_bit(In_sync, &rdev->flags);

	if (test_bit(In_sync, &rdev->flags) ||
	    atomic_read(&rdev->nr_pending)) {
		err = -EBUSY;
		goto abort;
	}
	/* Only remove non-faulty devices if recovery
	 * isn't possible.
	 */
	if (!test_bit(Faulty, &rdev->flags) &&
	    mddev->recovery_disabled != conf->recovery_disabled &&
	    !has_failed(conf) &&
6972
	    (!p->replacement || p->replacement == rdev) &&
6973 6974 6975 6976 6977 6978 6979 6980 6981 6982
	    number < conf->raid_disks) {
		err = -EBUSY;
		goto abort;
	}
	*rdevp = NULL;
	synchronize_rcu();
	if (atomic_read(&rdev->nr_pending)) {
		/* lost the race, try later */
		err = -EBUSY;
		*rdevp = rdev;
6983 6984 6985 6986 6987 6988 6989 6990 6991 6992 6993 6994 6995 6996
	} else if (p->replacement) {
		/* We must have just cleared 'rdev' */
		p->rdev = p->replacement;
		clear_bit(Replacement, &p->replacement->flags);
		smp_mb(); /* Make sure other CPUs may see both as identical
			   * but will never see neither - if they are careful
			   */
		p->replacement = NULL;
		clear_bit(WantReplacement, &rdev->flags);
	} else
		/* We might have just removed the Replacement as faulty-
		 * clear the bit just in case
		 */
		clear_bit(WantReplacement, &rdev->flags);
L
Linus Torvalds 已提交
6997 6998 6999 7000 7001 7002
abort:

	print_raid5_conf(conf);
	return err;
}

7003
static int raid5_add_disk(struct mddev *mddev, struct md_rdev *rdev)
L
Linus Torvalds 已提交
7004
{
7005
	struct r5conf *conf = mddev->private;
7006
	int err = -EEXIST;
L
Linus Torvalds 已提交
7007 7008
	int disk;
	struct disk_info *p;
7009 7010
	int first = 0;
	int last = conf->raid_disks - 1;
L
Linus Torvalds 已提交
7011

7012 7013 7014
	if (mddev->recovery_disabled == conf->recovery_disabled)
		return -EBUSY;

N
NeilBrown 已提交
7015
	if (rdev->saved_raid_disk < 0 && has_failed(conf))
L
Linus Torvalds 已提交
7016
		/* no point adding a device */
7017
		return -EINVAL;
L
Linus Torvalds 已提交
7018

7019 7020
	if (rdev->raid_disk >= 0)
		first = last = rdev->raid_disk;
L
Linus Torvalds 已提交
7021 7022

	/*
7023 7024
	 * find the disk ... but prefer rdev->saved_raid_disk
	 * if possible.
L
Linus Torvalds 已提交
7025
	 */
7026
	if (rdev->saved_raid_disk >= 0 &&
7027
	    rdev->saved_raid_disk >= first &&
7028
	    conf->disks[rdev->saved_raid_disk].rdev == NULL)
7029 7030 7031
		first = rdev->saved_raid_disk;

	for (disk = first; disk <= last; disk++) {
7032 7033
		p = conf->disks + disk;
		if (p->rdev == NULL) {
7034
			clear_bit(In_sync, &rdev->flags);
L
Linus Torvalds 已提交
7035
			rdev->raid_disk = disk;
7036
			err = 0;
7037 7038
			if (rdev->saved_raid_disk != disk)
				conf->fullsync = 1;
7039
			rcu_assign_pointer(p->rdev, rdev);
7040
			goto out;
L
Linus Torvalds 已提交
7041
		}
7042 7043 7044
	}
	for (disk = first; disk <= last; disk++) {
		p = conf->disks + disk;
7045 7046 7047 7048 7049 7050 7051 7052 7053 7054 7055
		if (test_bit(WantReplacement, &p->rdev->flags) &&
		    p->replacement == NULL) {
			clear_bit(In_sync, &rdev->flags);
			set_bit(Replacement, &rdev->flags);
			rdev->raid_disk = disk;
			err = 0;
			conf->fullsync = 1;
			rcu_assign_pointer(p->replacement, rdev);
			break;
		}
	}
7056
out:
L
Linus Torvalds 已提交
7057
	print_raid5_conf(conf);
7058
	return err;
L
Linus Torvalds 已提交
7059 7060
}

7061
static int raid5_resize(struct mddev *mddev, sector_t sectors)
L
Linus Torvalds 已提交
7062 7063 7064 7065 7066 7067 7068 7069
{
	/* no resync is happening, and there is enough space
	 * on all devices, so we can resize.
	 * We need to make sure resync covers any new space.
	 * If the array is shrinking we should possibly wait until
	 * any io in the removed space completes, but it hardly seems
	 * worth it.
	 */
7070
	sector_t newsize;
7071
	sectors &= ~((sector_t)mddev->chunk_sectors - 1);
7072 7073 7074
	newsize = raid5_size(mddev, sectors, mddev->raid_disks);
	if (mddev->external_size &&
	    mddev->array_sectors > newsize)
D
Dan Williams 已提交
7075
		return -EINVAL;
7076 7077 7078 7079 7080 7081
	if (mddev->bitmap) {
		int ret = bitmap_resize(mddev->bitmap, sectors, 0, 0);
		if (ret)
			return ret;
	}
	md_set_array_sectors(mddev, newsize);
7082
	set_capacity(mddev->gendisk, mddev->array_sectors);
7083
	revalidate_disk(mddev->gendisk);
7084 7085
	if (sectors > mddev->dev_sectors &&
	    mddev->recovery_cp > mddev->dev_sectors) {
A
Andre Noll 已提交
7086
		mddev->recovery_cp = mddev->dev_sectors;
L
Linus Torvalds 已提交
7087 7088
		set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
	}
A
Andre Noll 已提交
7089
	mddev->dev_sectors = sectors;
7090
	mddev->resync_max_sectors = sectors;
L
Linus Torvalds 已提交
7091 7092 7093
	return 0;
}

7094
static int check_stripe_cache(struct mddev *mddev)
7095 7096 7097 7098 7099 7100 7101 7102 7103
{
	/* Can only proceed if there are plenty of stripe_heads.
	 * We need a minimum of one full stripe,, and for sensible progress
	 * it is best to have about 4 times that.
	 * If we require 4 times, then the default 256 4K stripe_heads will
	 * allow for chunk sizes up to 256K, which is probably OK.
	 * If the chunk size is greater, user-space should request more
	 * stripe_heads first.
	 */
7104
	struct r5conf *conf = mddev->private;
7105 7106 7107 7108
	if (((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4
	    > conf->max_nr_stripes ||
	    ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4
	    > conf->max_nr_stripes) {
7109 7110
		printk(KERN_WARNING "md/raid:%s: reshape: not enough stripes.  Needed %lu\n",
		       mdname(mddev),
7111 7112 7113 7114 7115 7116 7117
		       ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9)
			/ STRIPE_SIZE)*4);
		return 0;
	}
	return 1;
}

7118
static int check_reshape(struct mddev *mddev)
7119
{
7120
	struct r5conf *conf = mddev->private;
7121

7122 7123
	if (mddev->delta_disks == 0 &&
	    mddev->new_layout == mddev->layout &&
7124
	    mddev->new_chunk_sectors == mddev->chunk_sectors)
7125
		return 0; /* nothing to do */
7126
	if (has_failed(conf))
7127
		return -EINVAL;
7128
	if (mddev->delta_disks < 0 && mddev->reshape_position == MaxSector) {
7129 7130 7131 7132 7133 7134 7135 7136 7137 7138 7139
		/* We might be able to shrink, but the devices must
		 * be made bigger first.
		 * For raid6, 4 is the minimum size.
		 * Otherwise 2 is the minimum
		 */
		int min = 2;
		if (mddev->level == 6)
			min = 4;
		if (mddev->raid_disks + mddev->delta_disks < min)
			return -EINVAL;
	}
7140

7141
	if (!check_stripe_cache(mddev))
7142 7143
		return -ENOSPC;

7144 7145
	return resize_stripes(conf, (conf->previous_raid_disks
				     + mddev->delta_disks));
7146 7147
}

7148
static int raid5_start_reshape(struct mddev *mddev)
7149
{
7150
	struct r5conf *conf = mddev->private;
7151
	struct md_rdev *rdev;
7152
	int spares = 0;
7153
	unsigned long flags;
7154

7155
	if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
7156 7157
		return -EBUSY;

7158 7159 7160
	if (!check_stripe_cache(mddev))
		return -ENOSPC;

7161 7162 7163
	if (has_failed(conf))
		return -EINVAL;

7164
	rdev_for_each(rdev, mddev) {
7165 7166
		if (!test_bit(In_sync, &rdev->flags)
		    && !test_bit(Faulty, &rdev->flags))
7167
			spares++;
7168
	}
7169

7170
	if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded)
7171 7172 7173 7174 7175
		/* Not enough devices even to make a degraded array
		 * of that size
		 */
		return -EINVAL;

7176 7177 7178 7179 7180 7181
	/* Refuse to reduce size of the array.  Any reductions in
	 * array size must be through explicit setting of array_size
	 * attribute.
	 */
	if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks)
	    < mddev->array_sectors) {
7182
		printk(KERN_ERR "md/raid:%s: array size must be reduced "
7183 7184 7185 7186
		       "before number of disks\n", mdname(mddev));
		return -EINVAL;
	}

7187
	atomic_set(&conf->reshape_stripes, 0);
7188
	spin_lock_irq(&conf->device_lock);
7189
	write_seqcount_begin(&conf->gen_lock);
7190
	conf->previous_raid_disks = conf->raid_disks;
7191
	conf->raid_disks += mddev->delta_disks;
7192 7193
	conf->prev_chunk_sectors = conf->chunk_sectors;
	conf->chunk_sectors = mddev->new_chunk_sectors;
7194 7195
	conf->prev_algo = conf->algorithm;
	conf->algorithm = mddev->new_layout;
7196 7197 7198 7199 7200
	conf->generation++;
	/* Code that selects data_offset needs to see the generation update
	 * if reshape_progress has been set - so a memory barrier needed.
	 */
	smp_mb();
7201
	if (mddev->reshape_backwards)
7202 7203 7204 7205
		conf->reshape_progress = raid5_size(mddev, 0, 0);
	else
		conf->reshape_progress = 0;
	conf->reshape_safe = conf->reshape_progress;
7206
	write_seqcount_end(&conf->gen_lock);
7207 7208
	spin_unlock_irq(&conf->device_lock);

7209 7210 7211 7212 7213 7214 7215
	/* Now make sure any requests that proceeded on the assumption
	 * the reshape wasn't running - like Discard or Read - have
	 * completed.
	 */
	mddev_suspend(mddev);
	mddev_resume(mddev);

7216 7217
	/* Add some new drives, as many as will fit.
	 * We know there are enough to make the newly sized array work.
7218 7219 7220 7221
	 * Don't add devices if we are reducing the number of
	 * devices in the array.  This is because it is not possible
	 * to correctly record the "partially reconstructed" state of
	 * such devices during the reshape and confusion could result.
7222
	 */
7223
	if (mddev->delta_disks >= 0) {
N
NeilBrown 已提交
7224
		rdev_for_each(rdev, mddev)
7225 7226 7227 7228
			if (rdev->raid_disk < 0 &&
			    !test_bit(Faulty, &rdev->flags)) {
				if (raid5_add_disk(mddev, rdev) == 0) {
					if (rdev->raid_disk
7229
					    >= conf->previous_raid_disks)
7230
						set_bit(In_sync, &rdev->flags);
7231
					else
7232
						rdev->recovery_offset = 0;
7233 7234

					if (sysfs_link_rdev(mddev, rdev))
7235
						/* Failure here is OK */;
7236
				}
7237 7238 7239 7240 7241
			} else if (rdev->raid_disk >= conf->previous_raid_disks
				   && !test_bit(Faulty, &rdev->flags)) {
				/* This is a spare that was manually added */
				set_bit(In_sync, &rdev->flags);
			}
7242

7243 7244 7245 7246
		/* When a reshape changes the number of devices,
		 * ->degraded is measured against the larger of the
		 * pre and post number of devices.
		 */
7247
		spin_lock_irqsave(&conf->device_lock, flags);
7248
		mddev->degraded = calc_degraded(conf);
7249 7250
		spin_unlock_irqrestore(&conf->device_lock, flags);
	}
7251
	mddev->raid_disks = conf->raid_disks;
7252
	mddev->reshape_position = conf->reshape_progress;
7253
	set_bit(MD_CHANGE_DEVS, &mddev->flags);
7254

7255 7256 7257 7258 7259
	clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
	clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
	set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
	set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
	mddev->sync_thread = md_register_thread(md_do_sync, mddev,
7260
						"reshape");
7261 7262 7263
	if (!mddev->sync_thread) {
		mddev->recovery = 0;
		spin_lock_irq(&conf->device_lock);
7264
		write_seqcount_begin(&conf->gen_lock);
7265
		mddev->raid_disks = conf->raid_disks = conf->previous_raid_disks;
7266 7267 7268
		mddev->new_chunk_sectors =
			conf->chunk_sectors = conf->prev_chunk_sectors;
		mddev->new_layout = conf->algorithm = conf->prev_algo;
7269 7270 7271
		rdev_for_each(rdev, mddev)
			rdev->new_data_offset = rdev->data_offset;
		smp_wmb();
7272
		conf->generation --;
7273
		conf->reshape_progress = MaxSector;
7274
		mddev->reshape_position = MaxSector;
7275
		write_seqcount_end(&conf->gen_lock);
7276 7277 7278
		spin_unlock_irq(&conf->device_lock);
		return -EAGAIN;
	}
7279
	conf->reshape_checkpoint = jiffies;
7280 7281 7282 7283 7284
	md_wakeup_thread(mddev->sync_thread);
	md_new_event(mddev);
	return 0;
}

7285 7286 7287
/* This is called from the reshape thread and should make any
 * changes needed in 'conf'
 */
7288
static void end_reshape(struct r5conf *conf)
7289 7290
{

7291
	if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) {
7292
		struct md_rdev *rdev;
7293 7294

		spin_lock_irq(&conf->device_lock);
7295
		conf->previous_raid_disks = conf->raid_disks;
7296 7297 7298
		rdev_for_each(rdev, conf->mddev)
			rdev->data_offset = rdev->new_data_offset;
		smp_wmb();
7299
		conf->reshape_progress = MaxSector;
7300
		spin_unlock_irq(&conf->device_lock);
7301
		wake_up(&conf->wait_for_overlap);
7302 7303 7304 7305

		/* read-ahead size must cover two whole stripes, which is
		 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices
		 */
7306
		if (conf->mddev->queue) {
7307
			int data_disks = conf->raid_disks - conf->max_degraded;
7308
			int stripe = data_disks * ((conf->chunk_sectors << 9)
7309
						   / PAGE_SIZE);
7310 7311 7312
			if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
				conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
		}
7313 7314 7315
	}
}

7316 7317 7318
/* This is called from the raid5d thread with mddev_lock held.
 * It makes config changes to the device.
 */
7319
static void raid5_finish_reshape(struct mddev *mddev)
7320
{
7321
	struct r5conf *conf = mddev->private;
7322 7323 7324

	if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {

7325 7326 7327
		if (mddev->delta_disks > 0) {
			md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
			set_capacity(mddev->gendisk, mddev->array_sectors);
7328
			revalidate_disk(mddev->gendisk);
7329 7330
		} else {
			int d;
7331 7332 7333
			spin_lock_irq(&conf->device_lock);
			mddev->degraded = calc_degraded(conf);
			spin_unlock_irq(&conf->device_lock);
7334 7335
			for (d = conf->raid_disks ;
			     d < conf->raid_disks - mddev->delta_disks;
7336
			     d++) {
7337
				struct md_rdev *rdev = conf->disks[d].rdev;
7338 7339 7340 7341 7342
				if (rdev)
					clear_bit(In_sync, &rdev->flags);
				rdev = conf->disks[d].replacement;
				if (rdev)
					clear_bit(In_sync, &rdev->flags);
7343
			}
7344
		}
7345
		mddev->layout = conf->algorithm;
7346
		mddev->chunk_sectors = conf->chunk_sectors;
7347 7348
		mddev->reshape_position = MaxSector;
		mddev->delta_disks = 0;
7349
		mddev->reshape_backwards = 0;
7350 7351 7352
	}
}

7353
static void raid5_quiesce(struct mddev *mddev, int state)
7354
{
7355
	struct r5conf *conf = mddev->private;
7356 7357

	switch(state) {
7358 7359 7360 7361
	case 2: /* resume for a suspend */
		wake_up(&conf->wait_for_overlap);
		break;

7362
	case 1: /* stop all writes */
7363
		lock_all_device_hash_locks_irq(conf);
7364 7365 7366 7367
		/* '2' tells resync/reshape to pause so that all
		 * active stripes can drain
		 */
		conf->quiesce = 2;
7368
		wait_event_cmd(conf->wait_for_stripe,
7369 7370
				    atomic_read(&conf->active_stripes) == 0 &&
				    atomic_read(&conf->active_aligned_reads) == 0,
7371 7372
				    unlock_all_device_hash_locks_irq(conf),
				    lock_all_device_hash_locks_irq(conf));
7373
		conf->quiesce = 1;
7374
		unlock_all_device_hash_locks_irq(conf);
7375 7376
		/* allow reshape to continue */
		wake_up(&conf->wait_for_overlap);
7377 7378 7379
		break;

	case 0: /* re-enable writes */
7380
		lock_all_device_hash_locks_irq(conf);
7381 7382
		conf->quiesce = 0;
		wake_up(&conf->wait_for_stripe);
7383
		wake_up(&conf->wait_for_overlap);
7384
		unlock_all_device_hash_locks_irq(conf);
7385 7386 7387
		break;
	}
}
7388

7389
static void *raid45_takeover_raid0(struct mddev *mddev, int level)
7390
{
7391
	struct r0conf *raid0_conf = mddev->private;
7392
	sector_t sectors;
7393

D
Dan Williams 已提交
7394
	/* for raid0 takeover only one zone is supported */
7395
	if (raid0_conf->nr_strip_zones > 1) {
7396 7397
		printk(KERN_ERR "md/raid:%s: cannot takeover raid0 with more than one zone.\n",
		       mdname(mddev));
D
Dan Williams 已提交
7398 7399 7400
		return ERR_PTR(-EINVAL);
	}

7401 7402
	sectors = raid0_conf->strip_zone[0].zone_end;
	sector_div(sectors, raid0_conf->strip_zone[0].nb_dev);
7403
	mddev->dev_sectors = sectors;
D
Dan Williams 已提交
7404
	mddev->new_level = level;
7405 7406 7407 7408 7409 7410 7411 7412 7413 7414
	mddev->new_layout = ALGORITHM_PARITY_N;
	mddev->new_chunk_sectors = mddev->chunk_sectors;
	mddev->raid_disks += 1;
	mddev->delta_disks = 1;
	/* make sure it will be not marked as dirty */
	mddev->recovery_cp = MaxSector;

	return setup_conf(mddev);
}

7415
static void *raid5_takeover_raid1(struct mddev *mddev)
7416 7417 7418 7419 7420 7421 7422 7423 7424 7425 7426 7427 7428 7429 7430 7431 7432 7433 7434 7435 7436
{
	int chunksect;

	if (mddev->raid_disks != 2 ||
	    mddev->degraded > 1)
		return ERR_PTR(-EINVAL);

	/* Should check if there are write-behind devices? */

	chunksect = 64*2; /* 64K by default */

	/* The array must be an exact multiple of chunksize */
	while (chunksect && (mddev->array_sectors & (chunksect-1)))
		chunksect >>= 1;

	if ((chunksect<<9) < STRIPE_SIZE)
		/* array size does not allow a suitable chunk size */
		return ERR_PTR(-EINVAL);

	mddev->new_level = 5;
	mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC;
7437
	mddev->new_chunk_sectors = chunksect;
7438 7439 7440 7441

	return setup_conf(mddev);
}

7442
static void *raid5_takeover_raid6(struct mddev *mddev)
7443 7444 7445 7446 7447 7448 7449 7450 7451 7452 7453 7454 7455 7456 7457 7458 7459 7460 7461 7462 7463 7464 7465 7466 7467 7468 7469 7470 7471 7472 7473 7474
{
	int new_layout;

	switch (mddev->layout) {
	case ALGORITHM_LEFT_ASYMMETRIC_6:
		new_layout = ALGORITHM_LEFT_ASYMMETRIC;
		break;
	case ALGORITHM_RIGHT_ASYMMETRIC_6:
		new_layout = ALGORITHM_RIGHT_ASYMMETRIC;
		break;
	case ALGORITHM_LEFT_SYMMETRIC_6:
		new_layout = ALGORITHM_LEFT_SYMMETRIC;
		break;
	case ALGORITHM_RIGHT_SYMMETRIC_6:
		new_layout = ALGORITHM_RIGHT_SYMMETRIC;
		break;
	case ALGORITHM_PARITY_0_6:
		new_layout = ALGORITHM_PARITY_0;
		break;
	case ALGORITHM_PARITY_N:
		new_layout = ALGORITHM_PARITY_N;
		break;
	default:
		return ERR_PTR(-EINVAL);
	}
	mddev->new_level = 5;
	mddev->new_layout = new_layout;
	mddev->delta_disks = -1;
	mddev->raid_disks -= 1;
	return setup_conf(mddev);
}

7475
static int raid5_check_reshape(struct mddev *mddev)
7476
{
7477 7478 7479 7480
	/* For a 2-drive array, the layout and chunk size can be changed
	 * immediately as not restriping is needed.
	 * For larger arrays we record the new value - after validation
	 * to be used by a reshape pass.
7481
	 */
7482
	struct r5conf *conf = mddev->private;
7483
	int new_chunk = mddev->new_chunk_sectors;
7484

7485
	if (mddev->new_layout >= 0 && !algorithm_valid_raid5(mddev->new_layout))
7486 7487
		return -EINVAL;
	if (new_chunk > 0) {
7488
		if (!is_power_of_2(new_chunk))
7489
			return -EINVAL;
7490
		if (new_chunk < (PAGE_SIZE>>9))
7491
			return -EINVAL;
7492
		if (mddev->array_sectors & (new_chunk-1))
7493 7494 7495 7496 7497 7498
			/* not factor of array size */
			return -EINVAL;
	}

	/* They look valid */

7499
	if (mddev->raid_disks == 2) {
7500 7501 7502 7503
		/* can make the change immediately */
		if (mddev->new_layout >= 0) {
			conf->algorithm = mddev->new_layout;
			mddev->layout = mddev->new_layout;
7504 7505
		}
		if (new_chunk > 0) {
7506 7507
			conf->chunk_sectors = new_chunk ;
			mddev->chunk_sectors = new_chunk;
7508 7509 7510
		}
		set_bit(MD_CHANGE_DEVS, &mddev->flags);
		md_wakeup_thread(mddev->thread);
7511
	}
7512
	return check_reshape(mddev);
7513 7514
}

7515
static int raid6_check_reshape(struct mddev *mddev)
7516
{
7517
	int new_chunk = mddev->new_chunk_sectors;
7518

7519
	if (mddev->new_layout >= 0 && !algorithm_valid_raid6(mddev->new_layout))
7520
		return -EINVAL;
7521
	if (new_chunk > 0) {
7522
		if (!is_power_of_2(new_chunk))
7523
			return -EINVAL;
7524
		if (new_chunk < (PAGE_SIZE >> 9))
7525
			return -EINVAL;
7526
		if (mddev->array_sectors & (new_chunk-1))
7527 7528
			/* not factor of array size */
			return -EINVAL;
7529
	}
7530 7531

	/* They look valid */
7532
	return check_reshape(mddev);
7533 7534
}

7535
static void *raid5_takeover(struct mddev *mddev)
7536 7537
{
	/* raid5 can take over:
D
Dan Williams 已提交
7538
	 *  raid0 - if there is only one strip zone - make it a raid4 layout
7539 7540 7541 7542
	 *  raid1 - if there are two drives.  We need to know the chunk size
	 *  raid4 - trivial - just use a raid4 layout.
	 *  raid6 - Providing it is a *_6 layout
	 */
D
Dan Williams 已提交
7543 7544
	if (mddev->level == 0)
		return raid45_takeover_raid0(mddev, 5);
7545 7546
	if (mddev->level == 1)
		return raid5_takeover_raid1(mddev);
7547 7548 7549 7550 7551
	if (mddev->level == 4) {
		mddev->new_layout = ALGORITHM_PARITY_N;
		mddev->new_level = 5;
		return setup_conf(mddev);
	}
7552 7553
	if (mddev->level == 6)
		return raid5_takeover_raid6(mddev);
7554 7555 7556 7557

	return ERR_PTR(-EINVAL);
}

7558
static void *raid4_takeover(struct mddev *mddev)
7559
{
D
Dan Williams 已提交
7560 7561 7562
	/* raid4 can take over:
	 *  raid0 - if there is only one strip zone
	 *  raid5 - if layout is right
7563
	 */
D
Dan Williams 已提交
7564 7565
	if (mddev->level == 0)
		return raid45_takeover_raid0(mddev, 4);
7566 7567 7568 7569 7570 7571 7572 7573
	if (mddev->level == 5 &&
	    mddev->layout == ALGORITHM_PARITY_N) {
		mddev->new_layout = 0;
		mddev->new_level = 4;
		return setup_conf(mddev);
	}
	return ERR_PTR(-EINVAL);
}
7574

7575
static struct md_personality raid5_personality;
7576

7577
static void *raid6_takeover(struct mddev *mddev)
7578 7579 7580 7581 7582 7583 7584 7585 7586 7587 7588 7589 7590 7591 7592 7593 7594 7595 7596 7597 7598 7599 7600 7601 7602 7603 7604 7605 7606 7607 7608 7609 7610 7611 7612 7613 7614 7615 7616 7617 7618 7619 7620 7621 7622
{
	/* Currently can only take over a raid5.  We map the
	 * personality to an equivalent raid6 personality
	 * with the Q block at the end.
	 */
	int new_layout;

	if (mddev->pers != &raid5_personality)
		return ERR_PTR(-EINVAL);
	if (mddev->degraded > 1)
		return ERR_PTR(-EINVAL);
	if (mddev->raid_disks > 253)
		return ERR_PTR(-EINVAL);
	if (mddev->raid_disks < 3)
		return ERR_PTR(-EINVAL);

	switch (mddev->layout) {
	case ALGORITHM_LEFT_ASYMMETRIC:
		new_layout = ALGORITHM_LEFT_ASYMMETRIC_6;
		break;
	case ALGORITHM_RIGHT_ASYMMETRIC:
		new_layout = ALGORITHM_RIGHT_ASYMMETRIC_6;
		break;
	case ALGORITHM_LEFT_SYMMETRIC:
		new_layout = ALGORITHM_LEFT_SYMMETRIC_6;
		break;
	case ALGORITHM_RIGHT_SYMMETRIC:
		new_layout = ALGORITHM_RIGHT_SYMMETRIC_6;
		break;
	case ALGORITHM_PARITY_0:
		new_layout = ALGORITHM_PARITY_0_6;
		break;
	case ALGORITHM_PARITY_N:
		new_layout = ALGORITHM_PARITY_N;
		break;
	default:
		return ERR_PTR(-EINVAL);
	}
	mddev->new_level = 6;
	mddev->new_layout = new_layout;
	mddev->delta_disks = 1;
	mddev->raid_disks += 1;
	return setup_conf(mddev);
}

7623
static struct md_personality raid6_personality =
7624 7625 7626 7627 7628 7629
{
	.name		= "raid6",
	.level		= 6,
	.owner		= THIS_MODULE,
	.make_request	= make_request,
	.run		= run,
N
NeilBrown 已提交
7630
	.free		= raid5_free,
7631 7632 7633 7634 7635 7636 7637
	.status		= status,
	.error_handler	= error,
	.hot_add_disk	= raid5_add_disk,
	.hot_remove_disk= raid5_remove_disk,
	.spare_active	= raid5_spare_active,
	.sync_request	= sync_request,
	.resize		= raid5_resize,
7638
	.size		= raid5_size,
7639
	.check_reshape	= raid6_check_reshape,
7640
	.start_reshape  = raid5_start_reshape,
7641
	.finish_reshape = raid5_finish_reshape,
7642
	.quiesce	= raid5_quiesce,
7643
	.takeover	= raid6_takeover,
7644
	.congested	= raid5_congested,
7645
	.mergeable_bvec	= raid5_mergeable_bvec,
7646
};
7647
static struct md_personality raid5_personality =
L
Linus Torvalds 已提交
7648 7649
{
	.name		= "raid5",
7650
	.level		= 5,
L
Linus Torvalds 已提交
7651 7652 7653
	.owner		= THIS_MODULE,
	.make_request	= make_request,
	.run		= run,
N
NeilBrown 已提交
7654
	.free		= raid5_free,
L
Linus Torvalds 已提交
7655 7656 7657 7658 7659 7660 7661
	.status		= status,
	.error_handler	= error,
	.hot_add_disk	= raid5_add_disk,
	.hot_remove_disk= raid5_remove_disk,
	.spare_active	= raid5_spare_active,
	.sync_request	= sync_request,
	.resize		= raid5_resize,
7662
	.size		= raid5_size,
7663 7664
	.check_reshape	= raid5_check_reshape,
	.start_reshape  = raid5_start_reshape,
7665
	.finish_reshape = raid5_finish_reshape,
7666
	.quiesce	= raid5_quiesce,
7667
	.takeover	= raid5_takeover,
7668
	.congested	= raid5_congested,
7669
	.mergeable_bvec	= raid5_mergeable_bvec,
L
Linus Torvalds 已提交
7670 7671
};

7672
static struct md_personality raid4_personality =
L
Linus Torvalds 已提交
7673
{
7674 7675 7676 7677 7678
	.name		= "raid4",
	.level		= 4,
	.owner		= THIS_MODULE,
	.make_request	= make_request,
	.run		= run,
N
NeilBrown 已提交
7679
	.free		= raid5_free,
7680 7681 7682 7683 7684 7685 7686
	.status		= status,
	.error_handler	= error,
	.hot_add_disk	= raid5_add_disk,
	.hot_remove_disk= raid5_remove_disk,
	.spare_active	= raid5_spare_active,
	.sync_request	= sync_request,
	.resize		= raid5_resize,
7687
	.size		= raid5_size,
7688 7689
	.check_reshape	= raid5_check_reshape,
	.start_reshape  = raid5_start_reshape,
7690
	.finish_reshape = raid5_finish_reshape,
7691
	.quiesce	= raid5_quiesce,
7692
	.takeover	= raid4_takeover,
7693
	.congested	= raid5_congested,
7694
	.mergeable_bvec	= raid5_mergeable_bvec,
7695 7696 7697 7698
};

static int __init raid5_init(void)
{
7699 7700 7701 7702
	raid5_wq = alloc_workqueue("raid5wq",
		WQ_UNBOUND|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE|WQ_SYSFS, 0);
	if (!raid5_wq)
		return -ENOMEM;
7703
	register_md_personality(&raid6_personality);
7704 7705 7706
	register_md_personality(&raid5_personality);
	register_md_personality(&raid4_personality);
	return 0;
L
Linus Torvalds 已提交
7707 7708
}

7709
static void raid5_exit(void)
L
Linus Torvalds 已提交
7710
{
7711
	unregister_md_personality(&raid6_personality);
7712 7713
	unregister_md_personality(&raid5_personality);
	unregister_md_personality(&raid4_personality);
7714
	destroy_workqueue(raid5_wq);
L
Linus Torvalds 已提交
7715 7716 7717 7718 7719
}

module_init(raid5_init);
module_exit(raid5_exit);
MODULE_LICENSE("GPL");
7720
MODULE_DESCRIPTION("RAID4/5/6 (striping with parity) personality for MD");
L
Linus Torvalds 已提交
7721
MODULE_ALIAS("md-personality-4"); /* RAID5 */
7722 7723
MODULE_ALIAS("md-raid5");
MODULE_ALIAS("md-raid4");
7724 7725
MODULE_ALIAS("md-level-5");
MODULE_ALIAS("md-level-4");
7726 7727 7728 7729 7730 7731 7732
MODULE_ALIAS("md-personality-8"); /* RAID6 */
MODULE_ALIAS("md-raid6");
MODULE_ALIAS("md-level-6");

/* This used to be two separate modules, they were: */
MODULE_ALIAS("raid5");
MODULE_ALIAS("raid6");