raid10.c 133.3 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7
/*
 * raid10.c : Multiple Devices driver for Linux
 *
 * Copyright (C) 2000-2004 Neil Brown
 *
 * RAID-10 support for md.
 *
L
Lucas De Marchi 已提交
8
 * Base on code in raid1.c.  See raid1.c for further copyright information.
L
Linus Torvalds 已提交
9 10 11 12 13 14 15 16 17 18 19 20
 *
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2, or (at your option)
 * any later version.
 *
 * You should have received a copy of the GNU General Public License
 * (for example /usr/src/linux/COPYING); if not, write to the Free
 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

21
#include <linux/slab.h>
22
#include <linux/delay.h>
23
#include <linux/blkdev.h>
24
#include <linux/module.h>
25
#include <linux/seq_file.h>
26
#include <linux/ratelimit.h>
N
NeilBrown 已提交
27
#include <linux/kthread.h>
28
#include <trace/events/block.h>
29
#include "md.h"
30
#include "raid10.h"
31
#include "raid0.h"
32
#include "bitmap.h"
L
Linus Torvalds 已提交
33 34 35 36 37 38 39 40

/*
 * RAID10 provides a combination of RAID0 and RAID1 functionality.
 * The layout of data is defined by
 *    chunk_size
 *    raid_disks
 *    near_copies (stored in low byte of layout)
 *    far_copies (stored in second byte of layout)
41
 *    far_offset (stored in bit 16 of layout )
42
 *    use_far_sets (stored in bit 17 of layout )
43
 *    use_far_sets_bugfixed (stored in bit 18 of layout )
L
Linus Torvalds 已提交
44
 *
45 46 47 48 49 50 51 52
 * The data to be stored is divided into chunks using chunksize.  Each device
 * is divided into far_copies sections.   In each section, chunks are laid out
 * in a style similar to raid0, but near_copies copies of each chunk is stored
 * (each on a different drive).  The starting device for each section is offset
 * near_copies from the starting device of the previous section.  Thus there
 * are (near_copies * far_copies) of each chunk, and each is on a different
 * drive.  near_copies and far_copies must be at least one, and their product
 * is at most raid_disks.
53 54
 *
 * If far_offset is true, then the far_copies are handled a bit differently.
55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72
 * The copies are still in different stripes, but instead of being very far
 * apart on disk, there are adjacent stripes.
 *
 * The far and offset algorithms are handled slightly differently if
 * 'use_far_sets' is true.  In this case, the array's devices are grouped into
 * sets that are (near_copies * far_copies) in size.  The far copied stripes
 * are still shifted by 'near_copies' devices, but this shifting stays confined
 * to the set rather than the entire array.  This is done to improve the number
 * of device combinations that can fail without causing the array to fail.
 * Example 'far' algorithm w/o 'use_far_sets' (each letter represents a chunk
 * on a device):
 *    A B C D    A B C D E
 *      ...         ...
 *    D A B C    E A B C D
 * Example 'far' algorithm w/ 'use_far_sets' enabled (sets illustrated w/ []'s):
 *    [A B] [C D]    [A B] [C D E]
 *    |...| |...|    |...| | ... |
 *    [B A] [D C]    [B A] [E C D]
L
Linus Torvalds 已提交
73 74 75 76 77 78 79
 */

/*
 * Number of guaranteed r10bios in case of extreme VM load:
 */
#define	NR_RAID10_BIOS 256

80 81 82 83 84 85 86 87 88 89 90 91 92 93 94
/* when we get a read error on a read-only array, we redirect to another
 * device without failing the first device, or trying to over-write to
 * correct the read error.  To keep track of bad blocks on a per-bio
 * level, we store IO_BLOCKED in the appropriate 'bios' pointer
 */
#define IO_BLOCKED ((struct bio *)1)
/* When we successfully write to a known bad-block, we need to remove the
 * bad-block marking which must be done from process context.  So we record
 * the success by setting devs[n].bio to IO_MADE_GOOD
 */
#define IO_MADE_GOOD ((struct bio *)2)

#define BIO_SPECIAL(bio) ((unsigned long)bio <= 2)

/* When there are this many requests queued to be written by
95 96 97 98 99
 * the raid10 thread, we become 'congested' to provide back-pressure
 * for writeback.
 */
static int max_queued_requests = 1024;

100 101
static void allow_barrier(struct r10conf *conf);
static void lower_barrier(struct r10conf *conf);
102
static int _enough(struct r10conf *conf, int previous, int ignore);
103
static int enough(struct r10conf *conf, int ignore);
N
NeilBrown 已提交
104 105 106
static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
				int *skipped);
static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio);
107
static void end_reshape_write(struct bio *bio);
N
NeilBrown 已提交
108
static void end_reshape(struct r10conf *conf);
109

110 111 112
#define raid10_log(md, fmt, args...)				\
	do { if ((md)->queue) blk_add_trace_msg((md)->queue, "raid10 " fmt, ##args); } while (0)

113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130
/*
 * 'strct resync_pages' stores actual pages used for doing the resync
 *  IO, and it is per-bio, so make .bi_private points to it.
 */
static inline struct resync_pages *get_resync_pages(struct bio *bio)
{
	return bio->bi_private;
}

/*
 * for resync bio, r10bio pointer can be retrieved from the per-bio
 * 'struct resync_pages'.
 */
static inline struct r10bio *get_resync_r10bio(struct bio *bio)
{
	return get_resync_pages(bio)->raid_bio;
}

A
Al Viro 已提交
131
static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data)
L
Linus Torvalds 已提交
132
{
133
	struct r10conf *conf = data;
134
	int size = offsetof(struct r10bio, devs[conf->copies]);
L
Linus Torvalds 已提交
135

136 137
	/* allocate a r10bio with room for raid_disks entries in the
	 * bios array */
J
Jens Axboe 已提交
138
	return kzalloc(size, gfp_flags);
L
Linus Torvalds 已提交
139 140 141 142 143 144 145
}

static void r10bio_pool_free(void *r10_bio, void *data)
{
	kfree(r10_bio);
}

146 147 148 149
/* amount of memory to reserve for resync requests */
#define RESYNC_WINDOW (1024*1024)
/* maximum number of concurrent requests, memory permitting */
#define RESYNC_DEPTH (32*1024*1024/RESYNC_BLOCK_SIZE)
L
Linus Torvalds 已提交
150 151 152 153 154 155 156 157

/*
 * When performing a resync, we need to read and compare, so
 * we need as many pages are there are copies.
 * When performing a recovery, we need 2 bios, one for read,
 * one for write (we recover only one drive per r10buf)
 *
 */
A
Al Viro 已提交
158
static void * r10buf_pool_alloc(gfp_t gfp_flags, void *data)
L
Linus Torvalds 已提交
159
{
160
	struct r10conf *conf = data;
161
	struct r10bio *r10_bio;
L
Linus Torvalds 已提交
162
	struct bio *bio;
163 164 165
	int j;
	int nalloc, nalloc_rp;
	struct resync_pages *rps;
L
Linus Torvalds 已提交
166 167

	r10_bio = r10bio_pool_alloc(gfp_flags, conf);
J
Jens Axboe 已提交
168
	if (!r10_bio)
L
Linus Torvalds 已提交
169 170
		return NULL;

N
NeilBrown 已提交
171 172
	if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery) ||
	    test_bit(MD_RECOVERY_RESHAPE, &conf->mddev->recovery))
L
Linus Torvalds 已提交
173 174 175 176
		nalloc = conf->copies; /* resync */
	else
		nalloc = 2; /* recovery */

177 178 179 180 181 182 183 184 185
	/* allocate once for all bios */
	if (!conf->have_replacement)
		nalloc_rp = nalloc;
	else
		nalloc_rp = nalloc * 2;
	rps = kmalloc(sizeof(struct resync_pages) * nalloc_rp, gfp_flags);
	if (!rps)
		goto out_free_r10bio;

L
Linus Torvalds 已提交
186 187 188 189
	/*
	 * Allocate bios.
	 */
	for (j = nalloc ; j-- ; ) {
190
		bio = bio_kmalloc(gfp_flags, RESYNC_PAGES);
L
Linus Torvalds 已提交
191 192 193
		if (!bio)
			goto out_free_bio;
		r10_bio->devs[j].bio = bio;
194 195 196 197 198 199
		if (!conf->have_replacement)
			continue;
		bio = bio_kmalloc(gfp_flags, RESYNC_PAGES);
		if (!bio)
			goto out_free_bio;
		r10_bio->devs[j].repl_bio = bio;
L
Linus Torvalds 已提交
200 201 202 203 204
	}
	/*
	 * Allocate RESYNC_PAGES data pages and attach them
	 * where needed.
	 */
205
	for (j = 0; j < nalloc; j++) {
206
		struct bio *rbio = r10_bio->devs[j].repl_bio;
207 208 209 210 211 212
		struct resync_pages *rp, *rp_repl;

		rp = &rps[j];
		if (rbio)
			rp_repl = &rps[nalloc + j];

L
Linus Torvalds 已提交
213
		bio = r10_bio->devs[j].bio;
214 215 216 217

		if (!j || test_bit(MD_RECOVERY_SYNC,
				   &conf->mddev->recovery)) {
			if (resync_alloc_pages(rp, gfp_flags))
L
Linus Torvalds 已提交
218
				goto out_free_pages;
219 220 221 222
		} else {
			memcpy(rp, &rps[0], sizeof(*rp));
			resync_get_all_pages(rp);
		}
L
Linus Torvalds 已提交
223

224 225 226 227 228 229
		rp->idx = 0;
		rp->raid_bio = r10_bio;
		bio->bi_private = rp;
		if (rbio) {
			memcpy(rp_repl, rp, sizeof(*rp));
			rbio->bi_private = rp_repl;
L
Linus Torvalds 已提交
230 231 232 233 234 235
		}
	}

	return r10_bio;

out_free_pages:
236 237 238
	while (--j >= 0)
		resync_free_pages(&rps[j * 2]);

239
	j = 0;
L
Linus Torvalds 已提交
240
out_free_bio:
241 242 243
	for ( ; j < nalloc; j++) {
		if (r10_bio->devs[j].bio)
			bio_put(r10_bio->devs[j].bio);
244 245 246
		if (r10_bio->devs[j].repl_bio)
			bio_put(r10_bio->devs[j].repl_bio);
	}
247 248
	kfree(rps);
out_free_r10bio:
L
Linus Torvalds 已提交
249 250 251 252 253 254
	r10bio_pool_free(r10_bio, conf);
	return NULL;
}

static void r10buf_pool_free(void *__r10_bio, void *data)
{
255
	struct r10conf *conf = data;
256
	struct r10bio *r10bio = __r10_bio;
L
Linus Torvalds 已提交
257
	int j;
258
	struct resync_pages *rp = NULL;
L
Linus Torvalds 已提交
259

260
	for (j = conf->copies; j--; ) {
L
Linus Torvalds 已提交
261
		struct bio *bio = r10bio->devs[j].bio;
262 263 264 265 266

		rp = get_resync_pages(bio);
		resync_free_pages(rp);
		bio_put(bio);

267 268 269
		bio = r10bio->devs[j].repl_bio;
		if (bio)
			bio_put(bio);
L
Linus Torvalds 已提交
270
	}
271 272 273 274

	/* resync pages array stored in the 1st bio's .bi_private */
	kfree(rp);

L
Linus Torvalds 已提交
275 276 277
	r10bio_pool_free(r10bio, conf);
}

278
static void put_all_bios(struct r10conf *conf, struct r10bio *r10_bio)
L
Linus Torvalds 已提交
279 280 281 282 283
{
	int i;

	for (i = 0; i < conf->copies; i++) {
		struct bio **bio = & r10_bio->devs[i].bio;
284
		if (!BIO_SPECIAL(*bio))
L
Linus Torvalds 已提交
285 286
			bio_put(*bio);
		*bio = NULL;
287 288 289 290
		bio = &r10_bio->devs[i].repl_bio;
		if (r10_bio->read_slot < 0 && !BIO_SPECIAL(*bio))
			bio_put(*bio);
		*bio = NULL;
L
Linus Torvalds 已提交
291 292 293
	}
}

294
static void free_r10bio(struct r10bio *r10_bio)
L
Linus Torvalds 已提交
295
{
296
	struct r10conf *conf = r10_bio->mddev->private;
L
Linus Torvalds 已提交
297 298 299 300 301

	put_all_bios(conf, r10_bio);
	mempool_free(r10_bio, conf->r10bio_pool);
}

302
static void put_buf(struct r10bio *r10_bio)
L
Linus Torvalds 已提交
303
{
304
	struct r10conf *conf = r10_bio->mddev->private;
L
Linus Torvalds 已提交
305 306 307

	mempool_free(r10_bio, conf->r10buf_pool);

308
	lower_barrier(conf);
L
Linus Torvalds 已提交
309 310
}

311
static void reschedule_retry(struct r10bio *r10_bio)
L
Linus Torvalds 已提交
312 313
{
	unsigned long flags;
314
	struct mddev *mddev = r10_bio->mddev;
315
	struct r10conf *conf = mddev->private;
L
Linus Torvalds 已提交
316 317 318

	spin_lock_irqsave(&conf->device_lock, flags);
	list_add(&r10_bio->retry_list, &conf->retry_list);
319
	conf->nr_queued ++;
L
Linus Torvalds 已提交
320 321
	spin_unlock_irqrestore(&conf->device_lock, flags);

A
Arthur Jones 已提交
322 323 324
	/* wake up frozen array... */
	wake_up(&conf->wait_barrier);

L
Linus Torvalds 已提交
325 326 327 328 329 330 331 332
	md_wakeup_thread(mddev->thread);
}

/*
 * raid_end_bio_io() is called when we have finished servicing a mirrored
 * operation and are ready to return a success/failure code to the buffer
 * cache layer.
 */
333
static void raid_end_bio_io(struct r10bio *r10_bio)
L
Linus Torvalds 已提交
334 335
{
	struct bio *bio = r10_bio->master_bio;
336
	struct r10conf *conf = r10_bio->mddev->private;
L
Linus Torvalds 已提交
337

338
	if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
339
		bio->bi_error = -EIO;
340 341 342 343 344 345 346 347

	bio_endio(bio);
	/*
	 * Wake up any possible resync thread that waits for the device
	 * to go idle.
	 */
	allow_barrier(conf);

L
Linus Torvalds 已提交
348 349 350 351 352 353
	free_r10bio(r10_bio);
}

/*
 * Update disk head position estimator based on IRQ completion info.
 */
354
static inline void update_head_pos(int slot, struct r10bio *r10_bio)
L
Linus Torvalds 已提交
355
{
356
	struct r10conf *conf = r10_bio->mddev->private;
L
Linus Torvalds 已提交
357 358 359 360 361

	conf->mirrors[r10_bio->devs[slot].devnum].head_position =
		r10_bio->devs[slot].addr + (r10_bio->sectors);
}

362 363 364
/*
 * Find the disk number which triggered given bio
 */
365
static int find_bio_disk(struct r10conf *conf, struct r10bio *r10_bio,
366
			 struct bio *bio, int *slotp, int *replp)
367 368
{
	int slot;
369
	int repl = 0;
370

371
	for (slot = 0; slot < conf->copies; slot++) {
372 373
		if (r10_bio->devs[slot].bio == bio)
			break;
374 375 376 377 378
		if (r10_bio->devs[slot].repl_bio == bio) {
			repl = 1;
			break;
		}
	}
379 380 381 382

	BUG_ON(slot == conf->copies);
	update_head_pos(slot, r10_bio);

383 384
	if (slotp)
		*slotp = slot;
385 386
	if (replp)
		*replp = repl;
387 388 389
	return r10_bio->devs[slot].devnum;
}

390
static void raid10_end_read_request(struct bio *bio)
L
Linus Torvalds 已提交
391
{
392
	int uptodate = !bio->bi_error;
393
	struct r10bio *r10_bio = bio->bi_private;
L
Linus Torvalds 已提交
394
	int slot, dev;
395
	struct md_rdev *rdev;
396
	struct r10conf *conf = r10_bio->mddev->private;
L
Linus Torvalds 已提交
397 398 399

	slot = r10_bio->read_slot;
	dev = r10_bio->devs[slot].devnum;
400
	rdev = r10_bio->devs[slot].rdev;
L
Linus Torvalds 已提交
401 402 403
	/*
	 * this branch is our 'one mirror IO has finished' event handler:
	 */
404 405 406
	update_head_pos(slot, r10_bio);

	if (uptodate) {
L
Linus Torvalds 已提交
407 408 409 410 411 412 413 414 415 416
		/*
		 * Set R10BIO_Uptodate in our master bio, so that
		 * we will return a good error code to the higher
		 * levels even if IO on some other mirrored buffer fails.
		 *
		 * The 'master' represents the composite IO operation to
		 * user-side. So if something waits for IO, then it will
		 * wait for the 'master' bio.
		 */
		set_bit(R10BIO_Uptodate, &r10_bio->state);
417 418 419 420 421 422
	} else {
		/* If all other devices that store this block have
		 * failed, we want to return the error upwards rather
		 * than fail the last device.  Here we redefine
		 * "uptodate" to mean "Don't want to retry"
		 */
423 424
		if (!_enough(conf, test_bit(R10BIO_Previous, &r10_bio->state),
			     rdev->raid_disk))
425 426 427
			uptodate = 1;
	}
	if (uptodate) {
L
Linus Torvalds 已提交
428
		raid_end_bio_io(r10_bio);
429
		rdev_dec_pending(rdev, conf->mddev);
430
	} else {
L
Linus Torvalds 已提交
431
		/*
432
		 * oops, read error - keep the refcount on the rdev
L
Linus Torvalds 已提交
433 434
		 */
		char b[BDEVNAME_SIZE];
N
NeilBrown 已提交
435
		pr_err_ratelimited("md/raid10:%s: %s: rescheduling sector %llu\n",
436
				   mdname(conf->mddev),
437
				   bdevname(rdev->bdev, b),
438
				   (unsigned long long)r10_bio->sector);
439
		set_bit(R10BIO_ReadError, &r10_bio->state);
L
Linus Torvalds 已提交
440 441 442 443
		reschedule_retry(r10_bio);
	}
}

444
static void close_write(struct r10bio *r10_bio)
445 446 447 448 449 450 451 452 453
{
	/* clear the bitmap if all writes complete successfully */
	bitmap_endwrite(r10_bio->mddev->bitmap, r10_bio->sector,
			r10_bio->sectors,
			!test_bit(R10BIO_Degraded, &r10_bio->state),
			0);
	md_write_end(r10_bio->mddev);
}

454
static void one_write_done(struct r10bio *r10_bio)
455 456 457 458 459 460 461 462 463 464 465 466 467 468
{
	if (atomic_dec_and_test(&r10_bio->remaining)) {
		if (test_bit(R10BIO_WriteError, &r10_bio->state))
			reschedule_retry(r10_bio);
		else {
			close_write(r10_bio);
			if (test_bit(R10BIO_MadeGood, &r10_bio->state))
				reschedule_retry(r10_bio);
			else
				raid_end_bio_io(r10_bio);
		}
	}
}

469
static void raid10_end_write_request(struct bio *bio)
L
Linus Torvalds 已提交
470
{
471
	struct r10bio *r10_bio = bio->bi_private;
472
	int dev;
473
	int dec_rdev = 1;
474
	struct r10conf *conf = r10_bio->mddev->private;
475
	int slot, repl;
476
	struct md_rdev *rdev = NULL;
477
	struct bio *to_put = NULL;
S
Shaohua Li 已提交
478 479 480
	bool discard_error;

	discard_error = bio->bi_error && bio_op(bio) == REQ_OP_DISCARD;
L
Linus Torvalds 已提交
481

482
	dev = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
L
Linus Torvalds 已提交
483

484 485
	if (repl)
		rdev = conf->mirrors[dev].replacement;
486 487 488
	if (!rdev) {
		smp_rmb();
		repl = 0;
489
		rdev = conf->mirrors[dev].rdev;
490
	}
L
Linus Torvalds 已提交
491 492 493
	/*
	 * this branch is our 'one mirror IO has finished' event handler:
	 */
S
Shaohua Li 已提交
494
	if (bio->bi_error && !discard_error) {
495 496 497 498 499 500 501
		if (repl)
			/* Never record new bad blocks to replacement,
			 * just fail it.
			 */
			md_error(rdev->mddev, rdev);
		else {
			set_bit(WriteErrorSeen,	&rdev->flags);
502 503 504
			if (!test_and_set_bit(WantReplacement, &rdev->flags))
				set_bit(MD_RECOVERY_NEEDED,
					&rdev->mddev->recovery);
505

506
			dec_rdev = 0;
507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522
			if (test_bit(FailFast, &rdev->flags) &&
			    (bio->bi_opf & MD_FAILFAST)) {
				md_error(rdev->mddev, rdev);
				if (!test_bit(Faulty, &rdev->flags))
					/* This is the only remaining device,
					 * We need to retry the write without
					 * FailFast
					 */
					set_bit(R10BIO_WriteError, &r10_bio->state);
				else {
					r10_bio->devs[slot].bio = NULL;
					to_put = bio;
					dec_rdev = 1;
				}
			} else
				set_bit(R10BIO_WriteError, &r10_bio->state);
523
		}
524
	} else {
L
Linus Torvalds 已提交
525 526 527 528 529 530 531 532 533
		/*
		 * Set R10BIO_Uptodate in our master bio, so that
		 * we will return a good error code for to the higher
		 * levels even if IO on some other mirrored buffer fails.
		 *
		 * The 'master' represents the composite IO operation to
		 * user-side. So if something waits for IO, then it will
		 * wait for the 'master' bio.
		 */
534 535 536
		sector_t first_bad;
		int bad_sectors;

537 538 539 540 541 542 543 544 545 546 547
		/*
		 * Do not set R10BIO_Uptodate if the current device is
		 * rebuilding or Faulty. This is because we cannot use
		 * such device for properly reading the data back (we could
		 * potentially use it, if the current write would have felt
		 * before rdev->recovery_offset, but for simplicity we don't
		 * check this here.
		 */
		if (test_bit(In_sync, &rdev->flags) &&
		    !test_bit(Faulty, &rdev->flags))
			set_bit(R10BIO_Uptodate, &r10_bio->state);
L
Linus Torvalds 已提交
548

549
		/* Maybe we can clear some bad blocks. */
550
		if (is_badblock(rdev,
551 552
				r10_bio->devs[slot].addr,
				r10_bio->sectors,
S
Shaohua Li 已提交
553
				&first_bad, &bad_sectors) && !discard_error) {
554
			bio_put(bio);
555 556 557 558
			if (repl)
				r10_bio->devs[slot].repl_bio = IO_MADE_GOOD;
			else
				r10_bio->devs[slot].bio = IO_MADE_GOOD;
559 560 561 562 563
			dec_rdev = 0;
			set_bit(R10BIO_MadeGood, &r10_bio->state);
		}
	}

L
Linus Torvalds 已提交
564 565 566 567 568
	/*
	 *
	 * Let's see if all mirrored write operations have finished
	 * already.
	 */
569
	one_write_done(r10_bio);
570
	if (dec_rdev)
571
		rdev_dec_pending(rdev, conf->mddev);
572 573
	if (to_put)
		bio_put(to_put);
L
Linus Torvalds 已提交
574 575 576 577
}

/*
 * RAID10 layout manager
L
Lucas De Marchi 已提交
578
 * As well as the chunksize and raid_disks count, there are two
L
Linus Torvalds 已提交
579 580 581 582 583 584
 * parameters: near_copies and far_copies.
 * near_copies * far_copies must be <= raid_disks.
 * Normally one of these will be 1.
 * If both are 1, we get raid0.
 * If near_copies == raid_disks, we get raid1.
 *
L
Lucas De Marchi 已提交
585
 * Chunks are laid out in raid0 style with near_copies copies of the
L
Linus Torvalds 已提交
586 587 588 589 590 591 592 593 594
 * first chunk, followed by near_copies copies of the next chunk and
 * so on.
 * If far_copies > 1, then after 1/far_copies of the array has been assigned
 * as described above, we start again with a device offset of near_copies.
 * So we effectively have another copy of the whole array further down all
 * the drives, but with blocks on different drives.
 * With this layout, and block is never stored twice on the one device.
 *
 * raid10_find_phys finds the sector offset of a given virtual sector
595
 * on each device that it is on.
L
Linus Torvalds 已提交
596 597 598 599 600
 *
 * raid10_find_virt does the reverse mapping, from a device and a
 * sector offset to a virtual address
 */

601
static void __raid10_find_phys(struct geom *geo, struct r10bio *r10bio)
L
Linus Torvalds 已提交
602 603 604 605 606 607 608
{
	int n,f;
	sector_t sector;
	sector_t chunk;
	sector_t stripe;
	int dev;
	int slot = 0;
609 610 611 612 613 614 615
	int last_far_set_start, last_far_set_size;

	last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1;
	last_far_set_start *= geo->far_set_size;

	last_far_set_size = geo->far_set_size;
	last_far_set_size += (geo->raid_disks % geo->far_set_size);
L
Linus Torvalds 已提交
616 617

	/* now calculate first sector/dev */
618 619
	chunk = r10bio->sector >> geo->chunk_shift;
	sector = r10bio->sector & geo->chunk_mask;
L
Linus Torvalds 已提交
620

621
	chunk *= geo->near_copies;
L
Linus Torvalds 已提交
622
	stripe = chunk;
623 624 625
	dev = sector_div(stripe, geo->raid_disks);
	if (geo->far_offset)
		stripe *= geo->far_copies;
L
Linus Torvalds 已提交
626

627
	sector += stripe << geo->chunk_shift;
L
Linus Torvalds 已提交
628 629

	/* and calculate all the others */
630
	for (n = 0; n < geo->near_copies; n++) {
L
Linus Torvalds 已提交
631
		int d = dev;
632
		int set;
L
Linus Torvalds 已提交
633 634
		sector_t s = sector;
		r10bio->devs[slot].devnum = d;
635
		r10bio->devs[slot].addr = s;
L
Linus Torvalds 已提交
636 637
		slot++;

638
		for (f = 1; f < geo->far_copies; f++) {
639
			set = d / geo->far_set_size;
640
			d += geo->near_copies;
641

642 643 644 645 646 647 648 649 650
			if ((geo->raid_disks % geo->far_set_size) &&
			    (d > last_far_set_start)) {
				d -= last_far_set_start;
				d %= last_far_set_size;
				d += last_far_set_start;
			} else {
				d %= geo->far_set_size;
				d += geo->far_set_size * set;
			}
651
			s += geo->stride;
L
Linus Torvalds 已提交
652 653 654 655 656
			r10bio->devs[slot].devnum = d;
			r10bio->devs[slot].addr = s;
			slot++;
		}
		dev++;
657
		if (dev >= geo->raid_disks) {
L
Linus Torvalds 已提交
658
			dev = 0;
659
			sector += (geo->chunk_mask + 1);
L
Linus Torvalds 已提交
660 661
		}
	}
662 663 664 665 666 667 668 669 670 671 672 673 674 675 676
}

static void raid10_find_phys(struct r10conf *conf, struct r10bio *r10bio)
{
	struct geom *geo = &conf->geo;

	if (conf->reshape_progress != MaxSector &&
	    ((r10bio->sector >= conf->reshape_progress) !=
	     conf->mddev->reshape_backwards)) {
		set_bit(R10BIO_Previous, &r10bio->state);
		geo = &conf->prev;
	} else
		clear_bit(R10BIO_Previous, &r10bio->state);

	__raid10_find_phys(geo, r10bio);
L
Linus Torvalds 已提交
677 678
}

679
static sector_t raid10_find_virt(struct r10conf *conf, sector_t sector, int dev)
L
Linus Torvalds 已提交
680 681
{
	sector_t offset, chunk, vchunk;
682 683 684
	/* Never use conf->prev as this is only called during resync
	 * or recovery, so reshape isn't happening
	 */
685
	struct geom *geo = &conf->geo;
686 687
	int far_set_start = (dev / geo->far_set_size) * geo->far_set_size;
	int far_set_size = geo->far_set_size;
688 689 690 691 692 693 694 695 696 697 698 699
	int last_far_set_start;

	if (geo->raid_disks % geo->far_set_size) {
		last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1;
		last_far_set_start *= geo->far_set_size;

		if (dev >= last_far_set_start) {
			far_set_size = geo->far_set_size;
			far_set_size += (geo->raid_disks % geo->far_set_size);
			far_set_start = last_far_set_start;
		}
	}
L
Linus Torvalds 已提交
700

701 702
	offset = sector & geo->chunk_mask;
	if (geo->far_offset) {
703
		int fc;
704 705 706
		chunk = sector >> geo->chunk_shift;
		fc = sector_div(chunk, geo->far_copies);
		dev -= fc * geo->near_copies;
707 708
		if (dev < far_set_start)
			dev += far_set_size;
709
	} else {
710 711
		while (sector >= geo->stride) {
			sector -= geo->stride;
712 713
			if (dev < (geo->near_copies + far_set_start))
				dev += far_set_size - geo->near_copies;
714
			else
715
				dev -= geo->near_copies;
716
		}
717
		chunk = sector >> geo->chunk_shift;
718
	}
719 720 721
	vchunk = chunk * geo->raid_disks + dev;
	sector_div(vchunk, geo->near_copies);
	return (vchunk << geo->chunk_shift) + offset;
L
Linus Torvalds 已提交
722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742
}

/*
 * This routine returns the disk from which the requested read should
 * be done. There is a per-array 'next expected sequential IO' sector
 * number - if this matches on the next IO then we use the last disk.
 * There is also a per-disk 'last know head position' sector that is
 * maintained from IRQ contexts, both the normal and the resync IO
 * completion handlers update this position correctly. If there is no
 * perfect sequential match then we pick the disk whose head is closest.
 *
 * If there are 2 mirrors in the same 2 devices, performance degrades
 * because position is mirror, not device based.
 *
 * The rdev for the device selected will have nr_pending incremented.
 */

/*
 * FIXME: possibly should rethink readbalancing and do it differently
 * depending on near_copies / far_copies geometry.
 */
743 744 745
static struct md_rdev *read_balance(struct r10conf *conf,
				    struct r10bio *r10_bio,
				    int *max_sectors)
L
Linus Torvalds 已提交
746
{
747
	const sector_t this_sector = r10_bio->sector;
N
NeilBrown 已提交
748
	int disk, slot;
749 750
	int sectors = r10_bio->sectors;
	int best_good_sectors;
N
NeilBrown 已提交
751
	sector_t new_distance, best_dist;
752
	struct md_rdev *best_rdev, *rdev = NULL;
N
NeilBrown 已提交
753 754
	int do_balance;
	int best_slot;
755
	struct geom *geo = &conf->geo;
L
Linus Torvalds 已提交
756 757 758

	raid10_find_phys(conf, r10_bio);
	rcu_read_lock();
759
	sectors = r10_bio->sectors;
N
NeilBrown 已提交
760
	best_slot = -1;
761
	best_rdev = NULL;
N
NeilBrown 已提交
762
	best_dist = MaxSector;
763
	best_good_sectors = 0;
N
NeilBrown 已提交
764
	do_balance = 1;
765
	clear_bit(R10BIO_FailFast, &r10_bio->state);
L
Linus Torvalds 已提交
766 767
	/*
	 * Check if we can balance. We can balance on the whole
768 769 770
	 * device if no resync is going on (recovery is ok), or below
	 * the resync window. We take the first readable disk when
	 * above the resync window.
L
Linus Torvalds 已提交
771 772
	 */
	if (conf->mddev->recovery_cp < MaxSector
N
NeilBrown 已提交
773 774
	    && (this_sector + sectors >= conf->next_resync))
		do_balance = 0;
L
Linus Torvalds 已提交
775

N
NeilBrown 已提交
776
	for (slot = 0; slot < conf->copies ; slot++) {
777 778 779 780
		sector_t first_bad;
		int bad_sectors;
		sector_t dev_sector;

N
NeilBrown 已提交
781 782
		if (r10_bio->devs[slot].bio == IO_BLOCKED)
			continue;
L
Linus Torvalds 已提交
783
		disk = r10_bio->devs[slot].devnum;
784 785 786 787
		rdev = rcu_dereference(conf->mirrors[disk].replacement);
		if (rdev == NULL || test_bit(Faulty, &rdev->flags) ||
		    r10_bio->devs[slot].addr + sectors > rdev->recovery_offset)
			rdev = rcu_dereference(conf->mirrors[disk].rdev);
788
		if (rdev == NULL ||
789
		    test_bit(Faulty, &rdev->flags))
790 791 792
			continue;
		if (!test_bit(In_sync, &rdev->flags) &&
		    r10_bio->devs[slot].addr + sectors > rdev->recovery_offset)
N
NeilBrown 已提交
793 794
			continue;

795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816
		dev_sector = r10_bio->devs[slot].addr;
		if (is_badblock(rdev, dev_sector, sectors,
				&first_bad, &bad_sectors)) {
			if (best_dist < MaxSector)
				/* Already have a better slot */
				continue;
			if (first_bad <= dev_sector) {
				/* Cannot read here.  If this is the
				 * 'primary' device, then we must not read
				 * beyond 'bad_sectors' from another device.
				 */
				bad_sectors -= (dev_sector - first_bad);
				if (!do_balance && sectors > bad_sectors)
					sectors = bad_sectors;
				if (best_good_sectors > sectors)
					best_good_sectors = sectors;
			} else {
				sector_t good_sectors =
					first_bad - dev_sector;
				if (good_sectors > best_good_sectors) {
					best_good_sectors = good_sectors;
					best_slot = slot;
817
					best_rdev = rdev;
818 819 820 821 822 823 824 825 826
				}
				if (!do_balance)
					/* Must read from here */
					break;
			}
			continue;
		} else
			best_good_sectors = sectors;

N
NeilBrown 已提交
827 828
		if (!do_balance)
			break;
L
Linus Torvalds 已提交
829

830 831 832
		if (best_slot >= 0)
			/* At least 2 disks to choose from so failfast is OK */
			set_bit(R10BIO_FailFast, &r10_bio->state);
833 834 835 836
		/* This optimisation is debatable, and completely destroys
		 * sequential read speed for 'far copies' arrays.  So only
		 * keep it for 'near' arrays, and review those later.
		 */
837
		if (geo->near_copies > 1 && !atomic_read(&rdev->nr_pending))
838
			new_distance = 0;
839 840

		/* for far > 1 always use the lowest address */
841
		else if (geo->far_copies > 1)
N
NeilBrown 已提交
842
			new_distance = r10_bio->devs[slot].addr;
843
		else
N
NeilBrown 已提交
844 845 846 847 848
			new_distance = abs(r10_bio->devs[slot].addr -
					   conf->mirrors[disk].head_position);
		if (new_distance < best_dist) {
			best_dist = new_distance;
			best_slot = slot;
849
			best_rdev = rdev;
L
Linus Torvalds 已提交
850 851
		}
	}
852
	if (slot >= conf->copies) {
N
NeilBrown 已提交
853
		slot = best_slot;
854 855
		rdev = best_rdev;
	}
L
Linus Torvalds 已提交
856

N
NeilBrown 已提交
857 858 859 860
	if (slot >= 0) {
		atomic_inc(&rdev->nr_pending);
		r10_bio->read_slot = slot;
	} else
861
		rdev = NULL;
L
Linus Torvalds 已提交
862
	rcu_read_unlock();
863
	*max_sectors = best_good_sectors;
L
Linus Torvalds 已提交
864

865
	return rdev;
L
Linus Torvalds 已提交
866 867
}

868
static int raid10_congested(struct mddev *mddev, int bits)
869
{
870
	struct r10conf *conf = mddev->private;
871 872
	int i, ret = 0;

873
	if ((bits & (1 << WB_async_congested)) &&
874 875 876
	    conf->pending_count >= max_queued_requests)
		return 1;

877
	rcu_read_lock();
878 879 880 881
	for (i = 0;
	     (i < conf->geo.raid_disks || i < conf->prev.raid_disks)
		     && ret == 0;
	     i++) {
882
		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
883
		if (rdev && !test_bit(Faulty, &rdev->flags)) {
884
			struct request_queue *q = bdev_get_queue(rdev->bdev);
885

886
			ret |= bdi_congested(q->backing_dev_info, bits);
887 888 889 890 891 892
		}
	}
	rcu_read_unlock();
	return ret;
}

893
static void flush_pending_writes(struct r10conf *conf)
894 895 896 897 898 899 900 901 902
{
	/* Any writes that have been queued but are awaiting
	 * bitmap updates get flushed here.
	 */
	spin_lock_irq(&conf->device_lock);

	if (conf->pending_bio_list.head) {
		struct bio *bio;
		bio = bio_list_get(&conf->pending_bio_list);
903
		conf->pending_count = 0;
904 905 906 907
		spin_unlock_irq(&conf->device_lock);
		/* flush any pending bitmap writes to disk
		 * before proceeding w/ I/O */
		bitmap_unplug(conf->mddev->bitmap);
908
		wake_up(&conf->wait_barrier);
909 910 911

		while (bio) { /* submit pending writes */
			struct bio *next = bio->bi_next;
912
			struct md_rdev *rdev = (void*)bio->bi_bdev;
913
			bio->bi_next = NULL;
914 915 916 917 918 919
			bio->bi_bdev = rdev->bdev;
			if (test_bit(Faulty, &rdev->flags)) {
				bio->bi_error = -EIO;
				bio_endio(bio);
			} else if (unlikely((bio_op(bio) ==  REQ_OP_DISCARD) &&
					    !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
S
Shaohua Li 已提交
920
				/* Just ignore it */
921
				bio_endio(bio);
S
Shaohua Li 已提交
922 923
			else
				generic_make_request(bio);
924 925 926 927 928
			bio = next;
		}
	} else
		spin_unlock_irq(&conf->device_lock);
}
J
Jens Axboe 已提交
929

930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949
/* Barriers....
 * Sometimes we need to suspend IO while we do something else,
 * either some resync/recovery, or reconfigure the array.
 * To do this we raise a 'barrier'.
 * The 'barrier' is a counter that can be raised multiple times
 * to count how many activities are happening which preclude
 * normal IO.
 * We can only raise the barrier if there is no pending IO.
 * i.e. if nr_pending == 0.
 * We choose only to raise the barrier if no-one is waiting for the
 * barrier to go down.  This means that as soon as an IO request
 * is ready, no other operations which require a barrier will start
 * until the IO request has had a chance.
 *
 * So: regular IO calls 'wait_barrier'.  When that returns there
 *    is no backgroup IO happening,  It must arrange to call
 *    allow_barrier when it has finished its IO.
 * backgroup IO calls must call raise_barrier.  Once that returns
 *    there is no normal IO happeing.  It must arrange to call
 *    lower_barrier when the particular background IO completes.
L
Linus Torvalds 已提交
950 951
 */

952
static void raise_barrier(struct r10conf *conf, int force)
L
Linus Torvalds 已提交
953
{
954
	BUG_ON(force && !conf->barrier);
L
Linus Torvalds 已提交
955
	spin_lock_irq(&conf->resync_lock);
956

957 958
	/* Wait until no block IO is waiting (unless 'force') */
	wait_event_lock_irq(conf->wait_barrier, force || !conf->nr_waiting,
959
			    conf->resync_lock);
960 961 962 963

	/* block any new IO from starting */
	conf->barrier++;

N
NeilBrown 已提交
964
	/* Now wait for all pending IO to complete */
965
	wait_event_lock_irq(conf->wait_barrier,
966
			    !atomic_read(&conf->nr_pending) && conf->barrier < RESYNC_DEPTH,
967
			    conf->resync_lock);
968 969 970 971

	spin_unlock_irq(&conf->resync_lock);
}

972
static void lower_barrier(struct r10conf *conf)
973 974 975 976 977 978 979 980
{
	unsigned long flags;
	spin_lock_irqsave(&conf->resync_lock, flags);
	conf->barrier--;
	spin_unlock_irqrestore(&conf->resync_lock, flags);
	wake_up(&conf->wait_barrier);
}

981
static void wait_barrier(struct r10conf *conf)
982 983 984 985
{
	spin_lock_irq(&conf->resync_lock);
	if (conf->barrier) {
		conf->nr_waiting++;
986 987 988 989 990 991 992 993 994
		/* Wait for the barrier to drop.
		 * However if there are already pending
		 * requests (preventing the barrier from
		 * rising completely), and the
		 * pre-process bio queue isn't empty,
		 * then don't wait, as we need to empty
		 * that queue to get the nr_pending
		 * count down.
		 */
995
		raid10_log(conf->mddev, "wait barrier");
996 997
		wait_event_lock_irq(conf->wait_barrier,
				    !conf->barrier ||
998
				    (atomic_read(&conf->nr_pending) &&
999
				     current->bio_list &&
1000 1001
				     (!bio_list_empty(&current->bio_list[0]) ||
				      !bio_list_empty(&current->bio_list[1]))),
1002
				    conf->resync_lock);
1003
		conf->nr_waiting--;
1004 1005
		if (!conf->nr_waiting)
			wake_up(&conf->wait_barrier);
L
Linus Torvalds 已提交
1006
	}
1007
	atomic_inc(&conf->nr_pending);
L
Linus Torvalds 已提交
1008 1009 1010
	spin_unlock_irq(&conf->resync_lock);
}

1011 1012 1013 1014 1015 1016 1017 1018 1019
static void inc_pending(struct r10conf *conf)
{
	/* The current request requires multiple r10_bio, so
	 * we need to increment the pending count.
	 */
	WARN_ON(!atomic_read(&conf->nr_pending));
	atomic_inc(&conf->nr_pending);
}

1020
static void allow_barrier(struct r10conf *conf)
1021
{
1022 1023 1024
	if ((atomic_dec_and_test(&conf->nr_pending)) ||
			(conf->array_freeze_pending))
		wake_up(&conf->wait_barrier);
1025 1026
}

1027
static void freeze_array(struct r10conf *conf, int extra)
1028 1029
{
	/* stop syncio and normal IO and wait for everything to
N
NeilBrown 已提交
1030
	 * go quiet.
1031
	 * We increment barrier and nr_waiting, and then
1032
	 * wait until nr_pending match nr_queued+extra
1033 1034 1035 1036
	 * This is called in the context of one normal IO request
	 * that has failed. Thus any sync request that might be pending
	 * will be blocked by nr_pending, and we need to wait for
	 * pending IO requests to complete or be queued for re-try.
1037
	 * Thus the number queued (nr_queued) plus this request (extra)
1038 1039
	 * must match the number of pending IOs (nr_pending) before
	 * we continue.
1040 1041
	 */
	spin_lock_irq(&conf->resync_lock);
1042
	conf->array_freeze_pending++;
1043 1044
	conf->barrier++;
	conf->nr_waiting++;
1045
	wait_event_lock_irq_cmd(conf->wait_barrier,
1046
				atomic_read(&conf->nr_pending) == conf->nr_queued+extra,
1047 1048
				conf->resync_lock,
				flush_pending_writes(conf));
N
NeilBrown 已提交
1049

1050
	conf->array_freeze_pending--;
1051 1052 1053
	spin_unlock_irq(&conf->resync_lock);
}

1054
static void unfreeze_array(struct r10conf *conf)
1055 1056 1057 1058 1059 1060 1061 1062 1063
{
	/* reverse the effect of the freeze */
	spin_lock_irq(&conf->resync_lock);
	conf->barrier--;
	conf->nr_waiting--;
	wake_up(&conf->wait_barrier);
	spin_unlock_irq(&conf->resync_lock);
}

1064 1065 1066 1067 1068 1069 1070 1071 1072 1073
static sector_t choose_data_offset(struct r10bio *r10_bio,
				   struct md_rdev *rdev)
{
	if (!test_bit(MD_RECOVERY_RESHAPE, &rdev->mddev->recovery) ||
	    test_bit(R10BIO_Previous, &r10_bio->state))
		return rdev->data_offset;
	else
		return rdev->new_data_offset;
}

1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087
struct raid10_plug_cb {
	struct blk_plug_cb	cb;
	struct bio_list		pending;
	int			pending_cnt;
};

static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule)
{
	struct raid10_plug_cb *plug = container_of(cb, struct raid10_plug_cb,
						   cb);
	struct mddev *mddev = plug->cb.data;
	struct r10conf *conf = mddev->private;
	struct bio *bio;

1088
	if (from_schedule || current->bio_list) {
1089 1090 1091 1092
		spin_lock_irq(&conf->device_lock);
		bio_list_merge(&conf->pending_bio_list, &plug->pending);
		conf->pending_count += plug->pending_cnt;
		spin_unlock_irq(&conf->device_lock);
1093
		wake_up(&conf->wait_barrier);
1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105
		md_wakeup_thread(mddev->thread);
		kfree(plug);
		return;
	}

	/* we aren't scheduling, so we can do the write-out directly. */
	bio = bio_list_get(&plug->pending);
	bitmap_unplug(mddev->bitmap);
	wake_up(&conf->wait_barrier);

	while (bio) { /* submit pending writes */
		struct bio *next = bio->bi_next;
1106
		struct md_rdev *rdev = (void*)bio->bi_bdev;
1107
		bio->bi_next = NULL;
1108 1109 1110 1111 1112 1113
		bio->bi_bdev = rdev->bdev;
		if (test_bit(Faulty, &rdev->flags)) {
			bio->bi_error = -EIO;
			bio_endio(bio);
		} else if (unlikely((bio_op(bio) ==  REQ_OP_DISCARD) &&
				    !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
1114
			/* Just ignore it */
1115
			bio_endio(bio);
1116 1117
		else
			generic_make_request(bio);
1118 1119 1120 1121 1122
		bio = next;
	}
	kfree(plug);
}

1123 1124
static void raid10_read_request(struct mddev *mddev, struct bio *bio,
				struct r10bio *r10_bio)
L
Linus Torvalds 已提交
1125
{
1126
	struct r10conf *conf = mddev->private;
L
Linus Torvalds 已提交
1127
	struct bio *read_bio;
1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167
	const int op = bio_op(bio);
	const unsigned long do_sync = (bio->bi_opf & REQ_SYNC);
	int sectors_handled;
	int max_sectors;
	sector_t sectors;
	struct md_rdev *rdev;
	int slot;

	/*
	 * Register the new request and wait if the reconstruction
	 * thread has put up a bar for new requests.
	 * Continue immediately if no resync is active currently.
	 */
	wait_barrier(conf);

	sectors = bio_sectors(bio);
	while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
	    bio->bi_iter.bi_sector < conf->reshape_progress &&
	    bio->bi_iter.bi_sector + sectors > conf->reshape_progress) {
		/*
		 * IO spans the reshape position.  Need to wait for reshape to
		 * pass
		 */
		raid10_log(conf->mddev, "wait reshape");
		allow_barrier(conf);
		wait_event(conf->wait_barrier,
			   conf->reshape_progress <= bio->bi_iter.bi_sector ||
			   conf->reshape_progress >= bio->bi_iter.bi_sector +
			   sectors);
		wait_barrier(conf);
	}

read_again:
	rdev = read_balance(conf, r10_bio, &max_sectors);
	if (!rdev) {
		raid_end_bio_io(r10_bio);
		return;
	}
	slot = r10_bio->read_slot;

1168
	read_bio = bio_clone_fast(bio, GFP_NOIO, mddev->bio_set);
1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196
	bio_trim(read_bio, r10_bio->sector - bio->bi_iter.bi_sector,
		 max_sectors);

	r10_bio->devs[slot].bio = read_bio;
	r10_bio->devs[slot].rdev = rdev;

	read_bio->bi_iter.bi_sector = r10_bio->devs[slot].addr +
		choose_data_offset(r10_bio, rdev);
	read_bio->bi_bdev = rdev->bdev;
	read_bio->bi_end_io = raid10_end_read_request;
	bio_set_op_attrs(read_bio, op, do_sync);
	if (test_bit(FailFast, &rdev->flags) &&
	    test_bit(R10BIO_FailFast, &r10_bio->state))
	        read_bio->bi_opf |= MD_FAILFAST;
	read_bio->bi_private = r10_bio;

	if (mddev->gendisk)
	        trace_block_bio_remap(bdev_get_queue(read_bio->bi_bdev),
	                              read_bio, disk_devt(mddev->gendisk),
	                              r10_bio->sector);
	if (max_sectors < r10_bio->sectors) {
		/*
		 * Could not read all from this device, so we will need another
		 * r10_bio.
		 */
		sectors_handled = (r10_bio->sector + max_sectors
				   - bio->bi_iter.bi_sector);
		r10_bio->sectors = max_sectors;
1197 1198
		inc_pending(conf);
		bio_inc_remaining(bio);
1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219
		/*
		 * Cannot call generic_make_request directly as that will be
		 * queued in __generic_make_request and subsequent
		 * mempool_alloc might block waiting for it.  so hand bio over
		 * to raid10d.
		 */
		reschedule_retry(r10_bio);

		r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);

		r10_bio->master_bio = bio;
		r10_bio->sectors = bio_sectors(bio) - sectors_handled;
		r10_bio->state = 0;
		r10_bio->mddev = mddev;
		r10_bio->sector = bio->bi_iter.bi_sector + sectors_handled;
		goto read_again;
	} else
		generic_make_request(read_bio);
	return;
}

1220 1221 1222
static void raid10_write_one_disk(struct mddev *mddev, struct r10bio *r10_bio,
				  struct bio *bio, bool replacement,
				  int n_copy, int max_sectors)
1223
{
M
Mike Christie 已提交
1224
	const int op = bio_op(bio);
J
Jens Axboe 已提交
1225 1226
	const unsigned long do_sync = (bio->bi_opf & REQ_SYNC);
	const unsigned long do_fua = (bio->bi_opf & REQ_FUA);
1227
	unsigned long flags;
1228 1229
	struct blk_plug_cb *cb;
	struct raid10_plug_cb *plug = NULL;
1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295
	struct r10conf *conf = mddev->private;
	struct md_rdev *rdev;
	int devnum = r10_bio->devs[n_copy].devnum;
	struct bio *mbio;

	if (replacement) {
		rdev = conf->mirrors[devnum].replacement;
		if (rdev == NULL) {
			/* Replacement just got moved to main 'rdev' */
			smp_mb();
			rdev = conf->mirrors[devnum].rdev;
		}
	} else
		rdev = conf->mirrors[devnum].rdev;

	mbio = bio_clone_fast(bio, GFP_NOIO, mddev->bio_set);
	bio_trim(mbio, r10_bio->sector - bio->bi_iter.bi_sector, max_sectors);
	if (replacement)
		r10_bio->devs[n_copy].repl_bio = mbio;
	else
		r10_bio->devs[n_copy].bio = mbio;

	mbio->bi_iter.bi_sector	= (r10_bio->devs[n_copy].addr +
				   choose_data_offset(r10_bio, rdev));
	mbio->bi_bdev = rdev->bdev;
	mbio->bi_end_io	= raid10_end_write_request;
	bio_set_op_attrs(mbio, op, do_sync | do_fua);
	if (!replacement && test_bit(FailFast,
				     &conf->mirrors[devnum].rdev->flags)
			 && enough(conf, devnum))
		mbio->bi_opf |= MD_FAILFAST;
	mbio->bi_private = r10_bio;

	if (conf->mddev->gendisk)
		trace_block_bio_remap(bdev_get_queue(mbio->bi_bdev),
				      mbio, disk_devt(conf->mddev->gendisk),
				      r10_bio->sector);
	/* flush_pending_writes() needs access to the rdev so...*/
	mbio->bi_bdev = (void *)rdev;

	atomic_inc(&r10_bio->remaining);

	cb = blk_check_plugged(raid10_unplug, mddev, sizeof(*plug));
	if (cb)
		plug = container_of(cb, struct raid10_plug_cb, cb);
	else
		plug = NULL;
	spin_lock_irqsave(&conf->device_lock, flags);
	if (plug) {
		bio_list_add(&plug->pending, mbio);
		plug->pending_cnt++;
	} else {
		bio_list_add(&conf->pending_bio_list, mbio);
		conf->pending_count++;
	}
	spin_unlock_irqrestore(&conf->device_lock, flags);
	if (!plug)
		md_wakeup_thread(mddev->thread);
}

static void raid10_write_request(struct mddev *mddev, struct bio *bio,
				 struct r10bio *r10_bio)
{
	struct r10conf *conf = mddev->private;
	int i;
	struct md_rdev *blocked_rdev;
1296
	sector_t sectors;
1297 1298
	int sectors_handled;
	int max_sectors;
L
Linus Torvalds 已提交
1299

1300 1301
	md_write_start(mddev, bio);

1302 1303 1304 1305 1306 1307 1308
	/*
	 * Register the new request and wait if the reconstruction
	 * thread has put up a bar for new requests.
	 * Continue immediately if no resync is active currently.
	 */
	wait_barrier(conf);

1309
	sectors = bio_sectors(bio);
N
NeilBrown 已提交
1310
	while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
1311 1312
	    bio->bi_iter.bi_sector < conf->reshape_progress &&
	    bio->bi_iter.bi_sector + sectors > conf->reshape_progress) {
1313 1314 1315
		/*
		 * IO spans the reshape position.  Need to wait for reshape to
		 * pass
N
NeilBrown 已提交
1316
		 */
1317
		raid10_log(conf->mddev, "wait reshape");
N
NeilBrown 已提交
1318 1319
		allow_barrier(conf);
		wait_event(conf->wait_barrier,
1320 1321 1322
			   conf->reshape_progress <= bio->bi_iter.bi_sector ||
			   conf->reshape_progress >= bio->bi_iter.bi_sector +
			   sectors);
N
NeilBrown 已提交
1323 1324
		wait_barrier(conf);
	}
1325

N
NeilBrown 已提交
1326 1327
	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
	    (mddev->reshape_backwards
1328 1329 1330 1331
	     ? (bio->bi_iter.bi_sector < conf->reshape_safe &&
		bio->bi_iter.bi_sector + sectors > conf->reshape_progress)
	     : (bio->bi_iter.bi_sector + sectors > conf->reshape_safe &&
		bio->bi_iter.bi_sector < conf->reshape_progress))) {
N
NeilBrown 已提交
1332 1333
		/* Need to update reshape_position in metadata */
		mddev->reshape_position = conf->reshape_progress;
1334 1335
		set_mask_bits(&mddev->sb_flags, 0,
			      BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
N
NeilBrown 已提交
1336
		md_wakeup_thread(mddev->thread);
1337
		raid10_log(conf->mddev, "wait reshape metadata");
N
NeilBrown 已提交
1338
		wait_event(mddev->sb_wait,
1339
			   !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags));
N
NeilBrown 已提交
1340 1341 1342 1343

		conf->reshape_safe = mddev->reshape_position;
	}

1344 1345
	if (conf->pending_count >= max_queued_requests) {
		md_wakeup_thread(mddev->thread);
1346
		raid10_log(mddev, "wait queued");
1347 1348 1349
		wait_event(conf->wait_barrier,
			   conf->pending_count < max_queued_requests);
	}
1350
	/* first select target devices under rcu_lock and
L
Linus Torvalds 已提交
1351 1352
	 * inc refcount on their rdev.  Record them by setting
	 * bios[x] to bio
1353 1354 1355 1356
	 * If there are known/acknowledged bad blocks on any device
	 * on which we have seen a write error, we want to avoid
	 * writing to those blocks.  This potentially requires several
	 * writes to write around the bad blocks.  Each set of writes
1357
	 * gets its own r10_bio with a set of bios attached.
L
Linus Torvalds 已提交
1358
	 */
N
NeilBrown 已提交
1359

1360
	r10_bio->read_slot = -1; /* make sure repl_bio gets freed */
L
Linus Torvalds 已提交
1361
	raid10_find_phys(conf, r10_bio);
1362
retry_write:
1363
	blocked_rdev = NULL;
L
Linus Torvalds 已提交
1364
	rcu_read_lock();
1365 1366
	max_sectors = r10_bio->sectors;

L
Linus Torvalds 已提交
1367 1368
	for (i = 0;  i < conf->copies; i++) {
		int d = r10_bio->devs[i].devnum;
1369
		struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev);
1370 1371
		struct md_rdev *rrdev = rcu_dereference(
			conf->mirrors[d].replacement);
1372 1373
		if (rdev == rrdev)
			rrdev = NULL;
1374 1375 1376 1377 1378
		if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) {
			atomic_inc(&rdev->nr_pending);
			blocked_rdev = rdev;
			break;
		}
1379 1380 1381 1382 1383
		if (rrdev && unlikely(test_bit(Blocked, &rrdev->flags))) {
			atomic_inc(&rrdev->nr_pending);
			blocked_rdev = rrdev;
			break;
		}
1384
		if (rdev && (test_bit(Faulty, &rdev->flags)))
1385
			rdev = NULL;
1386
		if (rrdev && (test_bit(Faulty, &rrdev->flags)))
1387 1388
			rrdev = NULL;

1389
		r10_bio->devs[i].bio = NULL;
1390
		r10_bio->devs[i].repl_bio = NULL;
1391 1392

		if (!rdev && !rrdev) {
1393
			set_bit(R10BIO_Degraded, &r10_bio->state);
1394 1395
			continue;
		}
1396
		if (rdev && test_bit(WriteErrorSeen, &rdev->flags)) {
1397 1398 1399 1400 1401
			sector_t first_bad;
			sector_t dev_sector = r10_bio->devs[i].addr;
			int bad_sectors;
			int is_bad;

1402
			is_bad = is_badblock(rdev, dev_sector, max_sectors,
1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435
					     &first_bad, &bad_sectors);
			if (is_bad < 0) {
				/* Mustn't write here until the bad block
				 * is acknowledged
				 */
				atomic_inc(&rdev->nr_pending);
				set_bit(BlockedBadBlocks, &rdev->flags);
				blocked_rdev = rdev;
				break;
			}
			if (is_bad && first_bad <= dev_sector) {
				/* Cannot write here at all */
				bad_sectors -= (dev_sector - first_bad);
				if (bad_sectors < max_sectors)
					/* Mustn't write more than bad_sectors
					 * to other devices yet
					 */
					max_sectors = bad_sectors;
				/* We don't set R10BIO_Degraded as that
				 * only applies if the disk is missing,
				 * so it might be re-added, and we want to
				 * know to recover this chunk.
				 * In this case the device is here, and the
				 * fact that this chunk is not in-sync is
				 * recorded in the bad block log.
				 */
				continue;
			}
			if (is_bad) {
				int good_sectors = first_bad - dev_sector;
				if (good_sectors < max_sectors)
					max_sectors = good_sectors;
			}
1436
		}
1437 1438 1439 1440
		if (rdev) {
			r10_bio->devs[i].bio = bio;
			atomic_inc(&rdev->nr_pending);
		}
1441 1442 1443 1444
		if (rrdev) {
			r10_bio->devs[i].repl_bio = bio;
			atomic_inc(&rrdev->nr_pending);
		}
L
Linus Torvalds 已提交
1445 1446 1447
	}
	rcu_read_unlock();

1448 1449 1450 1451 1452
	if (unlikely(blocked_rdev)) {
		/* Have to wait for this device to get unblocked, then retry */
		int j;
		int d;

1453
		for (j = 0; j < i; j++) {
1454 1455 1456 1457
			if (r10_bio->devs[j].bio) {
				d = r10_bio->devs[j].devnum;
				rdev_dec_pending(conf->mirrors[d].rdev, mddev);
			}
1458
			if (r10_bio->devs[j].repl_bio) {
1459
				struct md_rdev *rdev;
1460
				d = r10_bio->devs[j].devnum;
1461 1462 1463 1464 1465 1466 1467
				rdev = conf->mirrors[d].replacement;
				if (!rdev) {
					/* Race with remove_disk */
					smp_mb();
					rdev = conf->mirrors[d].rdev;
				}
				rdev_dec_pending(rdev, mddev);
1468 1469
			}
		}
1470
		allow_barrier(conf);
1471
		raid10_log(conf->mddev, "wait rdev %d blocked", blocked_rdev->raid_disk);
1472 1473 1474 1475 1476
		md_wait_for_blocked_rdev(blocked_rdev, mddev);
		wait_barrier(conf);
		goto retry_write;
	}

1477
	if (max_sectors < r10_bio->sectors)
1478
		r10_bio->sectors = max_sectors;
1479 1480
	sectors_handled = r10_bio->sector + max_sectors -
		bio->bi_iter.bi_sector;
1481

1482
	atomic_set(&r10_bio->remaining, 1);
1483
	bitmap_startwrite(mddev->bitmap, r10_bio->sector, r10_bio->sectors, 0);
1484

L
Linus Torvalds 已提交
1485
	for (i = 0; i < conf->copies; i++) {
1486 1487 1488 1489 1490 1491
		if (r10_bio->devs[i].bio)
			raid10_write_one_disk(mddev, r10_bio, bio, false,
					      i, max_sectors);
		if (r10_bio->devs[i].repl_bio)
			raid10_write_one_disk(mddev, r10_bio, bio, true,
					      i, max_sectors);
L
Linus Torvalds 已提交
1492 1493
	}

1494 1495 1496
	/* Don't remove the bias on 'remaining' (one_write_done) until
	 * after checking if we need to go around again.
	 */
1497

1498
	if (sectors_handled < bio_sectors(bio)) {
1499 1500 1501
		/* We need another r10_bio and it needs to be counted */
		inc_pending(conf);
		bio_inc_remaining(bio);
1502
		one_write_done(r10_bio);
1503 1504 1505
		r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);

		r10_bio->master_bio = bio;
1506
		r10_bio->sectors = bio_sectors(bio) - sectors_handled;
1507 1508

		r10_bio->mddev = mddev;
1509
		r10_bio->sector = bio->bi_iter.bi_sector + sectors_handled;
1510 1511 1512
		r10_bio->state = 0;
		goto retry_write;
	}
1513
	one_write_done(r10_bio);
K
Kent Overstreet 已提交
1514 1515
}

1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535
static void __make_request(struct mddev *mddev, struct bio *bio)
{
	struct r10conf *conf = mddev->private;
	struct r10bio *r10_bio;

	r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);

	r10_bio->master_bio = bio;
	r10_bio->sectors = bio_sectors(bio);

	r10_bio->mddev = mddev;
	r10_bio->sector = bio->bi_iter.bi_sector;
	r10_bio->state = 0;

	if (bio_data_dir(bio) == READ)
		raid10_read_request(mddev, bio, r10_bio);
	else
		raid10_write_request(mddev, bio, r10_bio);
}

S
Shaohua Li 已提交
1536
static void raid10_make_request(struct mddev *mddev, struct bio *bio)
K
Kent Overstreet 已提交
1537 1538 1539 1540 1541 1542 1543
{
	struct r10conf *conf = mddev->private;
	sector_t chunk_mask = (conf->geo.chunk_mask & conf->prev.chunk_mask);
	int chunk_sects = chunk_mask + 1;

	struct bio *split;

J
Jens Axboe 已提交
1544
	if (unlikely(bio->bi_opf & REQ_PREFLUSH)) {
K
Kent Overstreet 已提交
1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568
		md_flush_request(mddev, bio);
		return;
	}

	do {

		/*
		 * If this request crosses a chunk boundary, we need to split
		 * it.
		 */
		if (unlikely((bio->bi_iter.bi_sector & chunk_mask) +
			     bio_sectors(bio) > chunk_sects
			     && (conf->geo.near_copies < conf->geo.raid_disks
				 || conf->prev.near_copies <
				 conf->prev.raid_disks))) {
			split = bio_split(bio, chunk_sects -
					  (bio->bi_iter.bi_sector &
					   (chunk_sects - 1)),
					  GFP_NOIO, fs_bio_set);
			bio_chain(split, bio);
		} else {
			split = bio;
		}

S
Shaohua Li 已提交
1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582
		/*
		 * If a bio is splitted, the first part of bio will pass
		 * barrier but the bio is queued in current->bio_list (see
		 * generic_make_request). If there is a raise_barrier() called
		 * here, the second part of bio can't pass barrier. But since
		 * the first part bio isn't dispatched to underlaying disks
		 * yet, the barrier is never released, hence raise_barrier will
		 * alays wait. We have a deadlock.
		 * Note, this only happens in read path. For write path, the
		 * first part of bio is dispatched in a schedule() call
		 * (because of blk plug) or offloaded to raid10d.
		 * Quitting from the function immediately can change the bio
		 * order queued in bio_list and avoid the deadlock.
		 */
K
Kent Overstreet 已提交
1583
		__make_request(mddev, split);
S
Shaohua Li 已提交
1584 1585 1586 1587
		if (split != bio && bio_data_dir(bio) == READ) {
			generic_make_request(bio);
			break;
		}
K
Kent Overstreet 已提交
1588
	} while (split != bio);
1589 1590 1591

	/* In case raid10d snuck in to freeze_array */
	wake_up(&conf->wait_barrier);
L
Linus Torvalds 已提交
1592 1593
}

S
Shaohua Li 已提交
1594
static void raid10_status(struct seq_file *seq, struct mddev *mddev)
L
Linus Torvalds 已提交
1595
{
1596
	struct r10conf *conf = mddev->private;
L
Linus Torvalds 已提交
1597 1598
	int i;

1599
	if (conf->geo.near_copies < conf->geo.raid_disks)
1600
		seq_printf(seq, " %dK chunks", mddev->chunk_sectors / 2);
1601 1602 1603 1604 1605
	if (conf->geo.near_copies > 1)
		seq_printf(seq, " %d near-copies", conf->geo.near_copies);
	if (conf->geo.far_copies > 1) {
		if (conf->geo.far_offset)
			seq_printf(seq, " %d offset-copies", conf->geo.far_copies);
1606
		else
1607
			seq_printf(seq, " %d far-copies", conf->geo.far_copies);
1608 1609
		if (conf->geo.far_set_size != conf->geo.raid_disks)
			seq_printf(seq, " %d devices per set", conf->geo.far_set_size);
1610
	}
1611 1612
	seq_printf(seq, " [%d/%d] [", conf->geo.raid_disks,
					conf->geo.raid_disks - mddev->degraded);
1613 1614 1615 1616 1617 1618
	rcu_read_lock();
	for (i = 0; i < conf->geo.raid_disks; i++) {
		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
		seq_printf(seq, "%s", rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_");
	}
	rcu_read_unlock();
L
Linus Torvalds 已提交
1619 1620 1621
	seq_printf(seq, "]");
}

1622 1623 1624 1625 1626
/* check if there are enough drives for
 * every block to appear on atleast one.
 * Don't consider the device numbered 'ignore'
 * as we might be about to remove it.
 */
1627
static int _enough(struct r10conf *conf, int previous, int ignore)
1628 1629
{
	int first = 0;
1630
	int has_enough = 0;
1631 1632 1633 1634 1635 1636 1637 1638
	int disks, ncopies;
	if (previous) {
		disks = conf->prev.raid_disks;
		ncopies = conf->prev.near_copies;
	} else {
		disks = conf->geo.raid_disks;
		ncopies = conf->geo.near_copies;
	}
1639

1640
	rcu_read_lock();
1641 1642 1643
	do {
		int n = conf->copies;
		int cnt = 0;
1644
		int this = first;
1645
		while (n--) {
1646 1647 1648 1649
			struct md_rdev *rdev;
			if (this != ignore &&
			    (rdev = rcu_dereference(conf->mirrors[this].rdev)) &&
			    test_bit(In_sync, &rdev->flags))
1650
				cnt++;
1651
			this = (this+1) % disks;
1652 1653
		}
		if (cnt == 0)
1654
			goto out;
1655
		first = (first + ncopies) % disks;
1656
	} while (first != 0);
1657 1658 1659 1660
	has_enough = 1;
out:
	rcu_read_unlock();
	return has_enough;
1661 1662
}

1663 1664
static int enough(struct r10conf *conf, int ignore)
{
1665 1666 1667 1668 1669 1670 1671
	/* when calling 'enough', both 'prev' and 'geo' must
	 * be stable.
	 * This is ensured if ->reconfig_mutex or ->device_lock
	 * is held.
	 */
	return _enough(conf, 0, ignore) &&
		_enough(conf, 1, ignore);
1672 1673
}

S
Shaohua Li 已提交
1674
static void raid10_error(struct mddev *mddev, struct md_rdev *rdev)
L
Linus Torvalds 已提交
1675 1676
{
	char b[BDEVNAME_SIZE];
1677
	struct r10conf *conf = mddev->private;
1678
	unsigned long flags;
L
Linus Torvalds 已提交
1679 1680 1681 1682 1683 1684 1685

	/*
	 * If it is not operational, then we have already marked it as dead
	 * else if it is the last working disks, ignore the error, let the
	 * next level up know.
	 * else mark the drive as failed
	 */
1686
	spin_lock_irqsave(&conf->device_lock, flags);
1687
	if (test_bit(In_sync, &rdev->flags)
1688
	    && !enough(conf, rdev->raid_disk)) {
L
Linus Torvalds 已提交
1689 1690 1691
		/*
		 * Don't fail the drive, just return an IO error.
		 */
1692
		spin_unlock_irqrestore(&conf->device_lock, flags);
L
Linus Torvalds 已提交
1693
		return;
1694
	}
1695
	if (test_and_clear_bit(In_sync, &rdev->flags))
L
Linus Torvalds 已提交
1696
		mddev->degraded++;
1697 1698 1699 1700
	/*
	 * If recovery is running, make sure it aborts.
	 */
	set_bit(MD_RECOVERY_INTR, &mddev->recovery);
1701
	set_bit(Blocked, &rdev->flags);
1702
	set_bit(Faulty, &rdev->flags);
1703 1704
	set_mask_bits(&mddev->sb_flags, 0,
		      BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
1705
	spin_unlock_irqrestore(&conf->device_lock, flags);
N
NeilBrown 已提交
1706 1707 1708 1709
	pr_crit("md/raid10:%s: Disk failure on %s, disabling device.\n"
		"md/raid10:%s: Operation continuing on %d devices.\n",
		mdname(mddev), bdevname(rdev->bdev, b),
		mdname(mddev), conf->geo.raid_disks - mddev->degraded);
L
Linus Torvalds 已提交
1710 1711
}

1712
static void print_conf(struct r10conf *conf)
L
Linus Torvalds 已提交
1713 1714
{
	int i;
1715
	struct md_rdev *rdev;
L
Linus Torvalds 已提交
1716

N
NeilBrown 已提交
1717
	pr_debug("RAID10 conf printout:\n");
L
Linus Torvalds 已提交
1718
	if (!conf) {
N
NeilBrown 已提交
1719
		pr_debug("(!conf)\n");
L
Linus Torvalds 已提交
1720 1721
		return;
	}
N
NeilBrown 已提交
1722 1723
	pr_debug(" --- wd:%d rd:%d\n", conf->geo.raid_disks - conf->mddev->degraded,
		 conf->geo.raid_disks);
L
Linus Torvalds 已提交
1724

1725 1726
	/* This is only called with ->reconfix_mutex held, so
	 * rcu protection of rdev is not needed */
1727
	for (i = 0; i < conf->geo.raid_disks; i++) {
L
Linus Torvalds 已提交
1728
		char b[BDEVNAME_SIZE];
1729 1730
		rdev = conf->mirrors[i].rdev;
		if (rdev)
N
NeilBrown 已提交
1731 1732 1733 1734
			pr_debug(" disk %d, wo:%d, o:%d, dev:%s\n",
				 i, !test_bit(In_sync, &rdev->flags),
				 !test_bit(Faulty, &rdev->flags),
				 bdevname(rdev->bdev,b));
L
Linus Torvalds 已提交
1735 1736 1737
	}
}

1738
static void close_sync(struct r10conf *conf)
L
Linus Torvalds 已提交
1739
{
1740 1741
	wait_barrier(conf);
	allow_barrier(conf);
L
Linus Torvalds 已提交
1742 1743 1744 1745 1746

	mempool_destroy(conf->r10buf_pool);
	conf->r10buf_pool = NULL;
}

1747
static int raid10_spare_active(struct mddev *mddev)
L
Linus Torvalds 已提交
1748 1749
{
	int i;
1750
	struct r10conf *conf = mddev->private;
1751
	struct raid10_info *tmp;
1752 1753
	int count = 0;
	unsigned long flags;
L
Linus Torvalds 已提交
1754 1755 1756 1757 1758

	/*
	 * Find all non-in_sync disks within the RAID10 configuration
	 * and mark them in_sync
	 */
1759
	for (i = 0; i < conf->geo.raid_disks; i++) {
L
Linus Torvalds 已提交
1760
		tmp = conf->mirrors + i;
1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779
		if (tmp->replacement
		    && tmp->replacement->recovery_offset == MaxSector
		    && !test_bit(Faulty, &tmp->replacement->flags)
		    && !test_and_set_bit(In_sync, &tmp->replacement->flags)) {
			/* Replacement has just become active */
			if (!tmp->rdev
			    || !test_and_clear_bit(In_sync, &tmp->rdev->flags))
				count++;
			if (tmp->rdev) {
				/* Replaced device not technically faulty,
				 * but we need to be sure it gets removed
				 * and never re-added.
				 */
				set_bit(Faulty, &tmp->rdev->flags);
				sysfs_notify_dirent_safe(
					tmp->rdev->sysfs_state);
			}
			sysfs_notify_dirent_safe(tmp->replacement->sysfs_state);
		} else if (tmp->rdev
1780
			   && tmp->rdev->recovery_offset == MaxSector
1781 1782
			   && !test_bit(Faulty, &tmp->rdev->flags)
			   && !test_and_set_bit(In_sync, &tmp->rdev->flags)) {
1783
			count++;
1784
			sysfs_notify_dirent_safe(tmp->rdev->sysfs_state);
L
Linus Torvalds 已提交
1785 1786
		}
	}
1787 1788 1789
	spin_lock_irqsave(&conf->device_lock, flags);
	mddev->degraded -= count;
	spin_unlock_irqrestore(&conf->device_lock, flags);
L
Linus Torvalds 已提交
1790 1791

	print_conf(conf);
1792
	return count;
L
Linus Torvalds 已提交
1793 1794
}

1795
static int raid10_add_disk(struct mddev *mddev, struct md_rdev *rdev)
L
Linus Torvalds 已提交
1796
{
1797
	struct r10conf *conf = mddev->private;
1798
	int err = -EEXIST;
L
Linus Torvalds 已提交
1799
	int mirror;
1800
	int first = 0;
1801
	int last = conf->geo.raid_disks - 1;
L
Linus Torvalds 已提交
1802 1803 1804 1805 1806

	if (mddev->recovery_cp < MaxSector)
		/* only hot-add to in-sync arrays, as recovery is
		 * very different from resync
		 */
1807
		return -EBUSY;
1808
	if (rdev->saved_raid_disk < 0 && !_enough(conf, 1, -1))
1809
		return -EINVAL;
L
Linus Torvalds 已提交
1810

1811 1812 1813
	if (md_integrity_add_rdev(rdev, mddev))
		return -ENXIO;

N
NeilBrown 已提交
1814
	if (rdev->raid_disk >= 0)
1815
		first = last = rdev->raid_disk;
L
Linus Torvalds 已提交
1816

1817
	if (rdev->saved_raid_disk >= first &&
1818 1819 1820
	    conf->mirrors[rdev->saved_raid_disk].rdev == NULL)
		mirror = rdev->saved_raid_disk;
	else
1821
		mirror = first;
1822
	for ( ; mirror <= last ; mirror++) {
1823
		struct raid10_info *p = &conf->mirrors[mirror];
1824 1825
		if (p->recovery_disabled == mddev->recovery_disabled)
			continue;
1826 1827 1828 1829 1830 1831 1832 1833
		if (p->rdev) {
			if (!test_bit(WantReplacement, &p->rdev->flags) ||
			    p->replacement != NULL)
				continue;
			clear_bit(In_sync, &rdev->flags);
			set_bit(Replacement, &rdev->flags);
			rdev->raid_disk = mirror;
			err = 0;
1834 1835 1836
			if (mddev->gendisk)
				disk_stack_limits(mddev->gendisk, rdev->bdev,
						  rdev->data_offset << 9);
1837 1838 1839 1840
			conf->fullsync = 1;
			rcu_assign_pointer(p->replacement, rdev);
			break;
		}
L
Linus Torvalds 已提交
1841

1842 1843 1844
		if (mddev->gendisk)
			disk_stack_limits(mddev->gendisk, rdev->bdev,
					  rdev->data_offset << 9);
L
Linus Torvalds 已提交
1845

1846
		p->head_position = 0;
1847
		p->recovery_disabled = mddev->recovery_disabled - 1;
1848 1849 1850 1851 1852 1853 1854
		rdev->raid_disk = mirror;
		err = 0;
		if (rdev->saved_raid_disk != mirror)
			conf->fullsync = 1;
		rcu_assign_pointer(p->rdev, rdev);
		break;
	}
1855
	if (mddev->queue && blk_queue_discard(bdev_get_queue(rdev->bdev)))
S
Shaohua Li 已提交
1856 1857
		queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, mddev->queue);

L
Linus Torvalds 已提交
1858
	print_conf(conf);
1859
	return err;
L
Linus Torvalds 已提交
1860 1861
}

1862
static int raid10_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
L
Linus Torvalds 已提交
1863
{
1864
	struct r10conf *conf = mddev->private;
L
Linus Torvalds 已提交
1865
	int err = 0;
1866
	int number = rdev->raid_disk;
1867
	struct md_rdev **rdevp;
1868
	struct raid10_info *p = conf->mirrors + number;
L
Linus Torvalds 已提交
1869 1870

	print_conf(conf);
1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882
	if (rdev == p->rdev)
		rdevp = &p->rdev;
	else if (rdev == p->replacement)
		rdevp = &p->replacement;
	else
		return 0;

	if (test_bit(In_sync, &rdev->flags) ||
	    atomic_read(&rdev->nr_pending)) {
		err = -EBUSY;
		goto abort;
	}
1883
	/* Only remove non-faulty devices if recovery
1884 1885 1886 1887
	 * is not possible.
	 */
	if (!test_bit(Faulty, &rdev->flags) &&
	    mddev->recovery_disabled != p->recovery_disabled &&
1888
	    (!p->replacement || p->replacement == rdev) &&
1889
	    number < conf->geo.raid_disks &&
1890 1891 1892
	    enough(conf, -1)) {
		err = -EBUSY;
		goto abort;
L
Linus Torvalds 已提交
1893
	}
1894
	*rdevp = NULL;
1895 1896 1897 1898 1899 1900 1901 1902 1903 1904
	if (!test_bit(RemoveSynchronized, &rdev->flags)) {
		synchronize_rcu();
		if (atomic_read(&rdev->nr_pending)) {
			/* lost the race, try later */
			err = -EBUSY;
			*rdevp = rdev;
			goto abort;
		}
	}
	if (p->replacement) {
1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918
		/* We must have just cleared 'rdev' */
		p->rdev = p->replacement;
		clear_bit(Replacement, &p->replacement->flags);
		smp_mb(); /* Make sure other CPUs may see both as identical
			   * but will never see neither -- if they are careful.
			   */
		p->replacement = NULL;
		clear_bit(WantReplacement, &rdev->flags);
	} else
		/* We might have just remove the Replacement as faulty
		 * Clear the flag just in case
		 */
		clear_bit(WantReplacement, &rdev->flags);

1919 1920
	err = md_integrity_register(mddev);

L
Linus Torvalds 已提交
1921 1922 1923 1924 1925 1926
abort:

	print_conf(conf);
	return err;
}

1927
static void __end_sync_read(struct r10bio *r10_bio, struct bio *bio, int d)
L
Linus Torvalds 已提交
1928
{
1929
	struct r10conf *conf = r10_bio->mddev->private;
1930

1931
	if (!bio->bi_error)
1932
		set_bit(R10BIO_Uptodate, &r10_bio->state);
1933 1934 1935 1936
	else
		/* The write handler will notice the lack of
		 * R10BIO_Uptodate and record any errors etc
		 */
1937 1938
		atomic_add(r10_bio->sectors,
			   &conf->mirrors[d].rdev->corrected_errors);
L
Linus Torvalds 已提交
1939 1940 1941 1942

	/* for reconstruct, we always reschedule after a read.
	 * for resync, only after all reads
	 */
1943
	rdev_dec_pending(conf->mirrors[d].rdev, conf->mddev);
L
Linus Torvalds 已提交
1944 1945 1946 1947 1948 1949 1950 1951 1952
	if (test_bit(R10BIO_IsRecover, &r10_bio->state) ||
	    atomic_dec_and_test(&r10_bio->remaining)) {
		/* we have read all the blocks,
		 * do the comparison in process context in raid10d
		 */
		reschedule_retry(r10_bio);
	}
}

1953 1954
static void end_sync_read(struct bio *bio)
{
1955
	struct r10bio *r10_bio = get_resync_r10bio(bio);
1956 1957 1958 1959 1960 1961 1962 1963
	struct r10conf *conf = r10_bio->mddev->private;
	int d = find_bio_disk(conf, r10_bio, bio, NULL, NULL);

	__end_sync_read(r10_bio, bio, d);
}

static void end_reshape_read(struct bio *bio)
{
1964
	/* reshape read bio isn't allocated from r10buf_pool */
1965 1966 1967 1968 1969
	struct r10bio *r10_bio = bio->bi_private;

	__end_sync_read(r10_bio, bio, r10_bio->read_slot);
}

1970
static void end_sync_request(struct r10bio *r10_bio)
L
Linus Torvalds 已提交
1971
{
1972
	struct mddev *mddev = r10_bio->mddev;
1973

L
Linus Torvalds 已提交
1974 1975 1976
	while (atomic_dec_and_test(&r10_bio->remaining)) {
		if (r10_bio->master_bio == NULL) {
			/* the primary of several recovery bios */
1977
			sector_t s = r10_bio->sectors;
1978 1979
			if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
			    test_bit(R10BIO_WriteError, &r10_bio->state))
1980 1981 1982
				reschedule_retry(r10_bio);
			else
				put_buf(r10_bio);
1983
			md_done_sync(mddev, s, 1);
L
Linus Torvalds 已提交
1984 1985
			break;
		} else {
1986
			struct r10bio *r10_bio2 = (struct r10bio *)r10_bio->master_bio;
1987 1988
			if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
			    test_bit(R10BIO_WriteError, &r10_bio->state))
1989 1990 1991
				reschedule_retry(r10_bio);
			else
				put_buf(r10_bio);
L
Linus Torvalds 已提交
1992 1993 1994 1995 1996
			r10_bio = r10_bio2;
		}
	}
}

1997
static void end_sync_write(struct bio *bio)
1998
{
1999
	struct r10bio *r10_bio = get_resync_r10bio(bio);
2000
	struct mddev *mddev = r10_bio->mddev;
2001
	struct r10conf *conf = mddev->private;
2002 2003 2004 2005
	int d;
	sector_t first_bad;
	int bad_sectors;
	int slot;
2006
	int repl;
2007
	struct md_rdev *rdev = NULL;
2008

2009 2010 2011
	d = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
	if (repl)
		rdev = conf->mirrors[d].replacement;
2012
	else
2013
		rdev = conf->mirrors[d].rdev;
2014

2015
	if (bio->bi_error) {
2016 2017 2018 2019
		if (repl)
			md_error(mddev, rdev);
		else {
			set_bit(WriteErrorSeen, &rdev->flags);
2020 2021 2022
			if (!test_and_set_bit(WantReplacement, &rdev->flags))
				set_bit(MD_RECOVERY_NEEDED,
					&rdev->mddev->recovery);
2023 2024 2025
			set_bit(R10BIO_WriteError, &r10_bio->state);
		}
	} else if (is_badblock(rdev,
2026 2027 2028 2029 2030
			     r10_bio->devs[slot].addr,
			     r10_bio->sectors,
			     &first_bad, &bad_sectors))
		set_bit(R10BIO_MadeGood, &r10_bio->state);

2031
	rdev_dec_pending(rdev, mddev);
2032 2033 2034 2035

	end_sync_request(r10_bio);
}

L
Linus Torvalds 已提交
2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051
/*
 * Note: sync and recover and handled very differently for raid10
 * This code is for resync.
 * For resync, we read through virtual addresses and read all blocks.
 * If there is any error, we schedule a write.  The lowest numbered
 * drive is authoritative.
 * However requests come for physical address, so we need to map.
 * For every physical address there are raid_disks/copies virtual addresses,
 * which is always are least one, but is not necessarly an integer.
 * This means that a physical address can span multiple chunks, so we may
 * have to submit multiple io requests for a single sync request.
 */
/*
 * We check if all blocks are in-sync and only write to blocks that
 * aren't in sync
 */
2052
static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio)
L
Linus Torvalds 已提交
2053
{
2054
	struct r10conf *conf = mddev->private;
L
Linus Torvalds 已提交
2055 2056
	int i, first;
	struct bio *tbio, *fbio;
2057
	int vcnt;
2058
	struct page **tpages, **fpages;
L
Linus Torvalds 已提交
2059 2060 2061 2062 2063

	atomic_set(&r10_bio->remaining, 1);

	/* find the first device with a block */
	for (i=0; i<conf->copies; i++)
2064
		if (!r10_bio->devs[i].bio->bi_error)
L
Linus Torvalds 已提交
2065 2066 2067 2068 2069 2070 2071
			break;

	if (i == conf->copies)
		goto done;

	first = i;
	fbio = r10_bio->devs[i].bio;
2072 2073
	fbio->bi_iter.bi_size = r10_bio->sectors << 9;
	fbio->bi_iter.bi_idx = 0;
2074
	fpages = get_resync_pages(fbio)->pages;
L
Linus Torvalds 已提交
2075

2076
	vcnt = (r10_bio->sectors + (PAGE_SIZE >> 9) - 1) >> (PAGE_SHIFT - 9);
L
Linus Torvalds 已提交
2077
	/* now find blocks with errors */
2078 2079
	for (i=0 ; i < conf->copies ; i++) {
		int  j, d;
2080
		struct md_rdev *rdev;
2081
		struct resync_pages *rp;
L
Linus Torvalds 已提交
2082 2083

		tbio = r10_bio->devs[i].bio;
2084 2085 2086 2087

		if (tbio->bi_end_io != end_sync_read)
			continue;
		if (i == first)
L
Linus Torvalds 已提交
2088
			continue;
2089 2090

		tpages = get_resync_pages(tbio)->pages;
2091 2092
		d = r10_bio->devs[i].devnum;
		rdev = conf->mirrors[d].rdev;
2093
		if (!r10_bio->devs[i].bio->bi_error) {
2094 2095 2096 2097
			/* We know that the bi_io_vec layout is the same for
			 * both 'first' and 'i', so we just compare them.
			 * All vec entries are PAGE_SIZE;
			 */
2098 2099 2100 2101 2102
			int sectors = r10_bio->sectors;
			for (j = 0; j < vcnt; j++) {
				int len = PAGE_SIZE;
				if (sectors < (len / 512))
					len = sectors * 512;
2103 2104
				if (memcmp(page_address(fpages[j]),
					   page_address(tpages[j]),
2105
					   len))
2106
					break;
2107 2108
				sectors -= len/512;
			}
2109 2110
			if (j == vcnt)
				continue;
2111
			atomic64_add(r10_bio->sectors, &mddev->resync_mismatches);
2112 2113 2114
			if (test_bit(MD_RECOVERY_CHECK, &mddev->recovery))
				/* Don't fix anything. */
				continue;
2115 2116 2117 2118
		} else if (test_bit(FailFast, &rdev->flags)) {
			/* Just give up on this device */
			md_error(rdev->mddev, rdev);
			continue;
2119
		}
2120 2121
		/* Ok, we need to write this bio, either to correct an
		 * inconsistency or to correct an unreadable block.
L
Linus Torvalds 已提交
2122 2123 2124
		 * First we need to fixup bv_offset, bv_len and
		 * bi_vecs, as the read request might have corrupted these
		 */
2125
		rp = get_resync_pages(tbio);
K
Kent Overstreet 已提交
2126 2127
		bio_reset(tbio);

L
Linus Torvalds 已提交
2128
		tbio->bi_vcnt = vcnt;
2129
		tbio->bi_iter.bi_size = fbio->bi_iter.bi_size;
2130 2131
		rp->raid_bio = r10_bio;
		tbio->bi_private = rp;
2132
		tbio->bi_iter.bi_sector = r10_bio->devs[i].addr;
L
Linus Torvalds 已提交
2133
		tbio->bi_end_io = end_sync_write;
M
Mike Christie 已提交
2134
		bio_set_op_attrs(tbio, REQ_OP_WRITE, 0);
L
Linus Torvalds 已提交
2135

2136 2137
		bio_copy_data(tbio, fbio);

L
Linus Torvalds 已提交
2138 2139
		atomic_inc(&conf->mirrors[d].rdev->nr_pending);
		atomic_inc(&r10_bio->remaining);
2140
		md_sync_acct(conf->mirrors[d].rdev->bdev, bio_sectors(tbio));
L
Linus Torvalds 已提交
2141

2142 2143
		if (test_bit(FailFast, &conf->mirrors[d].rdev->flags))
			tbio->bi_opf |= MD_FAILFAST;
2144
		tbio->bi_iter.bi_sector += conf->mirrors[d].rdev->data_offset;
L
Linus Torvalds 已提交
2145 2146 2147 2148
		tbio->bi_bdev = conf->mirrors[d].rdev->bdev;
		generic_make_request(tbio);
	}

2149 2150 2151 2152
	/* Now write out to any replacement devices
	 * that are active
	 */
	for (i = 0; i < conf->copies; i++) {
2153
		int d;
2154 2155 2156 2157 2158 2159

		tbio = r10_bio->devs[i].repl_bio;
		if (!tbio || !tbio->bi_end_io)
			continue;
		if (r10_bio->devs[i].bio->bi_end_io != end_sync_write
		    && r10_bio->devs[i].bio != fbio)
2160
			bio_copy_data(tbio, fbio);
2161 2162 2163
		d = r10_bio->devs[i].devnum;
		atomic_inc(&r10_bio->remaining);
		md_sync_acct(conf->mirrors[d].replacement->bdev,
2164
			     bio_sectors(tbio));
2165 2166 2167
		generic_make_request(tbio);
	}

L
Linus Torvalds 已提交
2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184
done:
	if (atomic_dec_and_test(&r10_bio->remaining)) {
		md_done_sync(mddev, r10_bio->sectors, 1);
		put_buf(r10_bio);
	}
}

/*
 * Now for the recovery code.
 * Recovery happens across physical sectors.
 * We recover all non-is_sync drives by finding the virtual address of
 * each, and then choose a working drive that also has that virt address.
 * There is a separate r10_bio for each non-in_sync drive.
 * Only the first two slots are in use. The first for reading,
 * The second for writing.
 *
 */
2185
static void fix_recovery_read_error(struct r10bio *r10_bio)
2186 2187 2188 2189 2190 2191 2192 2193
{
	/* We got a read error during recovery.
	 * We repeat the read in smaller page-sized sections.
	 * If a read succeeds, write it to the new device or record
	 * a bad block if we cannot.
	 * If a read fails, record a bad block on both old and
	 * new devices.
	 */
2194
	struct mddev *mddev = r10_bio->mddev;
2195
	struct r10conf *conf = mddev->private;
2196 2197 2198 2199 2200 2201
	struct bio *bio = r10_bio->devs[0].bio;
	sector_t sect = 0;
	int sectors = r10_bio->sectors;
	int idx = 0;
	int dr = r10_bio->devs[0].devnum;
	int dw = r10_bio->devs[1].devnum;
2202
	struct page **pages = get_resync_pages(bio)->pages;
2203 2204 2205

	while (sectors) {
		int s = sectors;
2206
		struct md_rdev *rdev;
2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217
		sector_t addr;
		int ok;

		if (s > (PAGE_SIZE>>9))
			s = PAGE_SIZE >> 9;

		rdev = conf->mirrors[dr].rdev;
		addr = r10_bio->devs[0].addr + sect,
		ok = sync_page_io(rdev,
				  addr,
				  s << 9,
2218
				  pages[idx],
M
Mike Christie 已提交
2219
				  REQ_OP_READ, 0, false);
2220 2221 2222 2223 2224 2225
		if (ok) {
			rdev = conf->mirrors[dw].rdev;
			addr = r10_bio->devs[1].addr + sect;
			ok = sync_page_io(rdev,
					  addr,
					  s << 9,
2226
					  pages[idx],
M
Mike Christie 已提交
2227
					  REQ_OP_WRITE, 0, false);
2228
			if (!ok) {
2229
				set_bit(WriteErrorSeen, &rdev->flags);
2230 2231 2232 2233 2234
				if (!test_and_set_bit(WantReplacement,
						      &rdev->flags))
					set_bit(MD_RECOVERY_NEEDED,
						&rdev->mddev->recovery);
			}
2235 2236 2237 2238 2239 2240 2241 2242 2243 2244
		}
		if (!ok) {
			/* We don't worry if we cannot set a bad block -
			 * it really is bad so there is no loss in not
			 * recording it yet
			 */
			rdev_set_badblocks(rdev, addr, s, 0);

			if (rdev != conf->mirrors[dw].rdev) {
				/* need bad block on destination too */
2245
				struct md_rdev *rdev2 = conf->mirrors[dw].rdev;
2246 2247 2248 2249
				addr = r10_bio->devs[1].addr + sect;
				ok = rdev_set_badblocks(rdev2, addr, s, 0);
				if (!ok) {
					/* just abort the recovery */
N
NeilBrown 已提交
2250 2251
					pr_notice("md/raid10:%s: recovery aborted due to read error\n",
						  mdname(mddev));
2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266

					conf->mirrors[dw].recovery_disabled
						= mddev->recovery_disabled;
					set_bit(MD_RECOVERY_INTR,
						&mddev->recovery);
					break;
				}
			}
		}

		sectors -= s;
		sect += s;
		idx++;
	}
}
L
Linus Torvalds 已提交
2267

2268
static void recovery_request_write(struct mddev *mddev, struct r10bio *r10_bio)
L
Linus Torvalds 已提交
2269
{
2270
	struct r10conf *conf = mddev->private;
2271
	int d;
2272
	struct bio *wbio, *wbio2;
L
Linus Torvalds 已提交
2273

2274 2275 2276 2277 2278 2279
	if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) {
		fix_recovery_read_error(r10_bio);
		end_sync_request(r10_bio);
		return;
	}

2280 2281
	/*
	 * share the pages with the first bio
L
Linus Torvalds 已提交
2282 2283 2284
	 * and submit the write request
	 */
	d = r10_bio->devs[1].devnum;
2285 2286
	wbio = r10_bio->devs[1].bio;
	wbio2 = r10_bio->devs[1].repl_bio;
2287 2288 2289 2290 2291 2292
	/* Need to test wbio2->bi_end_io before we call
	 * generic_make_request as if the former is NULL,
	 * the latter is free to free wbio2.
	 */
	if (wbio2 && !wbio2->bi_end_io)
		wbio2 = NULL;
2293 2294
	if (wbio->bi_end_io) {
		atomic_inc(&conf->mirrors[d].rdev->nr_pending);
2295
		md_sync_acct(conf->mirrors[d].rdev->bdev, bio_sectors(wbio));
2296 2297
		generic_make_request(wbio);
	}
2298
	if (wbio2) {
2299 2300
		atomic_inc(&conf->mirrors[d].replacement->nr_pending);
		md_sync_acct(conf->mirrors[d].replacement->bdev,
2301
			     bio_sectors(wbio2));
2302 2303
		generic_make_request(wbio2);
	}
L
Linus Torvalds 已提交
2304 2305
}

2306 2307 2308 2309 2310 2311
/*
 * Used by fix_read_error() to decay the per rdev read_errors.
 * We halve the read error count for every hour that has elapsed
 * since the last recorded read error.
 *
 */
2312
static void check_decay_read_errors(struct mddev *mddev, struct md_rdev *rdev)
2313
{
2314
	long cur_time_mon;
2315 2316 2317
	unsigned long hours_since_last;
	unsigned int read_errors = atomic_read(&rdev->read_errors);

2318
	cur_time_mon = ktime_get_seconds();
2319

2320
	if (rdev->last_read_error == 0) {
2321 2322 2323 2324 2325
		/* first time we've seen a read error */
		rdev->last_read_error = cur_time_mon;
		return;
	}

2326 2327
	hours_since_last = (long)(cur_time_mon -
			    rdev->last_read_error) / 3600;
2328 2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341

	rdev->last_read_error = cur_time_mon;

	/*
	 * if hours_since_last is > the number of bits in read_errors
	 * just set read errors to 0. We do this to avoid
	 * overflowing the shift of read_errors by hours_since_last.
	 */
	if (hours_since_last >= 8 * sizeof(read_errors))
		atomic_set(&rdev->read_errors, 0);
	else
		atomic_set(&rdev->read_errors, read_errors >> hours_since_last);
}

2342
static int r10_sync_page_io(struct md_rdev *rdev, sector_t sector,
2343 2344 2345 2346 2347 2348 2349 2350
			    int sectors, struct page *page, int rw)
{
	sector_t first_bad;
	int bad_sectors;

	if (is_badblock(rdev, sector, sectors, &first_bad, &bad_sectors)
	    && (rw == READ || test_bit(WriteErrorSeen, &rdev->flags)))
		return -1;
M
Mike Christie 已提交
2351
	if (sync_page_io(rdev, sector, sectors << 9, page, rw, 0, false))
2352 2353
		/* success */
		return 1;
2354
	if (rw == WRITE) {
2355
		set_bit(WriteErrorSeen, &rdev->flags);
2356 2357 2358 2359
		if (!test_and_set_bit(WantReplacement, &rdev->flags))
			set_bit(MD_RECOVERY_NEEDED,
				&rdev->mddev->recovery);
	}
2360 2361 2362 2363 2364 2365
	/* need to record an error - either for the block or the device */
	if (!rdev_set_badblocks(rdev, sector, sectors, 0))
		md_error(rdev->mddev, rdev);
	return 0;
}

L
Linus Torvalds 已提交
2366 2367 2368 2369 2370
/*
 * This is a kernel thread which:
 *
 *	1.	Retries failed read operations on working mirrors.
 *	2.	Updates the raid superblock when problems encounter.
2371
 *	3.	Performs writes following reads for array synchronising.
L
Linus Torvalds 已提交
2372 2373
 */

2374
static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10bio *r10_bio)
2375 2376 2377
{
	int sect = 0; /* Offset from r10_bio->sector */
	int sectors = r10_bio->sectors;
2378
	struct md_rdev*rdev;
2379
	int max_read_errors = atomic_read(&mddev->max_corr_read_errors);
2380
	int d = r10_bio->devs[r10_bio->read_slot].devnum;
2381

2382 2383 2384 2385
	/* still own a reference to this rdev, so it cannot
	 * have been cleared recently.
	 */
	rdev = conf->mirrors[d].rdev;
2386

2387 2388 2389 2390
	if (test_bit(Faulty, &rdev->flags))
		/* drive has already been failed, just ignore any
		   more fix_read_error() attempts */
		return;
2391

2392 2393 2394 2395 2396
	check_decay_read_errors(mddev, rdev);
	atomic_inc(&rdev->read_errors);
	if (atomic_read(&rdev->read_errors) > max_read_errors) {
		char b[BDEVNAME_SIZE];
		bdevname(rdev->bdev, b);
2397

N
NeilBrown 已提交
2398 2399 2400 2401 2402
		pr_notice("md/raid10:%s: %s: Raid device exceeded read_error threshold [cur %d:max %d]\n",
			  mdname(mddev), b,
			  atomic_read(&rdev->read_errors), max_read_errors);
		pr_notice("md/raid10:%s: %s: Failing raid device\n",
			  mdname(mddev), b);
2403
		md_error(mddev, rdev);
2404
		r10_bio->devs[r10_bio->read_slot].bio = IO_BLOCKED;
2405
		return;
2406 2407
	}

2408 2409 2410 2411 2412 2413 2414 2415 2416 2417 2418
	while(sectors) {
		int s = sectors;
		int sl = r10_bio->read_slot;
		int success = 0;
		int start;

		if (s > (PAGE_SIZE>>9))
			s = PAGE_SIZE >> 9;

		rcu_read_lock();
		do {
2419 2420 2421
			sector_t first_bad;
			int bad_sectors;

2422
			d = r10_bio->devs[sl].devnum;
2423 2424
			rdev = rcu_dereference(conf->mirrors[d].rdev);
			if (rdev &&
2425
			    test_bit(In_sync, &rdev->flags) &&
2426
			    !test_bit(Faulty, &rdev->flags) &&
2427 2428
			    is_badblock(rdev, r10_bio->devs[sl].addr + sect, s,
					&first_bad, &bad_sectors) == 0) {
2429 2430
				atomic_inc(&rdev->nr_pending);
				rcu_read_unlock();
2431
				success = sync_page_io(rdev,
2432
						       r10_bio->devs[sl].addr +
J
Jonathan Brassow 已提交
2433
						       sect,
2434
						       s<<9,
M
Mike Christie 已提交
2435 2436
						       conf->tmppage,
						       REQ_OP_READ, 0, false);
2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448
				rdev_dec_pending(rdev, mddev);
				rcu_read_lock();
				if (success)
					break;
			}
			sl++;
			if (sl == conf->copies)
				sl = 0;
		} while (!success && sl != r10_bio->read_slot);
		rcu_read_unlock();

		if (!success) {
2449 2450 2451 2452
			/* Cannot read from anywhere, just mark the block
			 * as bad on the first device to discourage future
			 * reads.
			 */
2453
			int dn = r10_bio->devs[r10_bio->read_slot].devnum;
2454 2455 2456 2457 2458 2459
			rdev = conf->mirrors[dn].rdev;

			if (!rdev_set_badblocks(
				    rdev,
				    r10_bio->devs[r10_bio->read_slot].addr
				    + sect,
2460
				    s, 0)) {
2461
				md_error(mddev, rdev);
2462 2463 2464
				r10_bio->devs[r10_bio->read_slot].bio
					= IO_BLOCKED;
			}
2465 2466 2467 2468 2469 2470 2471
			break;
		}

		start = sl;
		/* write it back and re-read */
		rcu_read_lock();
		while (sl != r10_bio->read_slot) {
2472
			char b[BDEVNAME_SIZE];
2473

2474 2475 2476 2477 2478
			if (sl==0)
				sl = conf->copies;
			sl--;
			d = r10_bio->devs[sl].devnum;
			rdev = rcu_dereference(conf->mirrors[d].rdev);
2479
			if (!rdev ||
2480
			    test_bit(Faulty, &rdev->flags) ||
2481 2482 2483 2484 2485
			    !test_bit(In_sync, &rdev->flags))
				continue;

			atomic_inc(&rdev->nr_pending);
			rcu_read_unlock();
2486 2487 2488
			if (r10_sync_page_io(rdev,
					     r10_bio->devs[sl].addr +
					     sect,
2489
					     s, conf->tmppage, WRITE)
2490 2491
			    == 0) {
				/* Well, this device is dead */
N
NeilBrown 已提交
2492 2493 2494 2495 2496 2497 2498 2499 2500 2501
				pr_notice("md/raid10:%s: read correction write failed (%d sectors at %llu on %s)\n",
					  mdname(mddev), s,
					  (unsigned long long)(
						  sect +
						  choose_data_offset(r10_bio,
								     rdev)),
					  bdevname(rdev->bdev, b));
				pr_notice("md/raid10:%s: %s: failing drive\n",
					  mdname(mddev),
					  bdevname(rdev->bdev, b));
2502
			}
2503 2504
			rdev_dec_pending(rdev, mddev);
			rcu_read_lock();
2505 2506 2507
		}
		sl = start;
		while (sl != r10_bio->read_slot) {
2508
			char b[BDEVNAME_SIZE];
2509

2510 2511 2512 2513 2514
			if (sl==0)
				sl = conf->copies;
			sl--;
			d = r10_bio->devs[sl].devnum;
			rdev = rcu_dereference(conf->mirrors[d].rdev);
2515
			if (!rdev ||
2516
			    test_bit(Faulty, &rdev->flags) ||
2517 2518
			    !test_bit(In_sync, &rdev->flags))
				continue;
2519

2520 2521
			atomic_inc(&rdev->nr_pending);
			rcu_read_unlock();
2522 2523 2524
			switch (r10_sync_page_io(rdev,
					     r10_bio->devs[sl].addr +
					     sect,
2525
					     s, conf->tmppage,
2526 2527
						 READ)) {
			case 0:
2528
				/* Well, this device is dead */
N
NeilBrown 已提交
2529
				pr_notice("md/raid10:%s: unable to read back corrected sectors (%d sectors at %llu on %s)\n",
2530 2531
				       mdname(mddev), s,
				       (unsigned long long)(
2532 2533
					       sect +
					       choose_data_offset(r10_bio, rdev)),
2534
				       bdevname(rdev->bdev, b));
N
NeilBrown 已提交
2535
				pr_notice("md/raid10:%s: %s: failing drive\n",
2536 2537
				       mdname(mddev),
				       bdevname(rdev->bdev, b));
2538 2539
				break;
			case 1:
N
NeilBrown 已提交
2540
				pr_info("md/raid10:%s: read error corrected (%d sectors at %llu on %s)\n",
2541 2542
				       mdname(mddev), s,
				       (unsigned long long)(
2543 2544
					       sect +
					       choose_data_offset(r10_bio, rdev)),
2545 2546
				       bdevname(rdev->bdev, b));
				atomic_add(s, &rdev->corrected_errors);
2547
			}
2548 2549 2550

			rdev_dec_pending(rdev, mddev);
			rcu_read_lock();
2551 2552 2553 2554 2555 2556 2557 2558
		}
		rcu_read_unlock();

		sectors -= s;
		sect += s;
	}
}

2559
static int narrow_write_error(struct r10bio *r10_bio, int i)
2560 2561
{
	struct bio *bio = r10_bio->master_bio;
2562
	struct mddev *mddev = r10_bio->mddev;
2563
	struct r10conf *conf = mddev->private;
2564
	struct md_rdev *rdev = conf->mirrors[r10_bio->devs[i].devnum].rdev;
2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584
	/* bio has the data to be written to slot 'i' where
	 * we just recently had a write error.
	 * We repeatedly clone the bio and trim down to one block,
	 * then try the write.  Where the write fails we record
	 * a bad block.
	 * It is conceivable that the bio doesn't exactly align with
	 * blocks.  We must handle this.
	 *
	 * We currently own a reference to the rdev.
	 */

	int block_sectors;
	sector_t sector;
	int sectors;
	int sect_to_write = r10_bio->sectors;
	int ok = 1;

	if (rdev->badblocks.shift < 0)
		return 0;

2585 2586
	block_sectors = roundup(1 << rdev->badblocks.shift,
				bdev_logical_block_size(rdev->bdev) >> 9);
2587 2588 2589 2590 2591 2592 2593
	sector = r10_bio->sector;
	sectors = ((r10_bio->sector + block_sectors)
		   & ~(sector_t)(block_sectors - 1))
		- sector;

	while (sect_to_write) {
		struct bio *wbio;
2594
		sector_t wsector;
2595 2596 2597
		if (sectors > sect_to_write)
			sectors = sect_to_write;
		/* Write at 'sector' for 'sectors' */
2598
		wbio = bio_clone_fast(bio, GFP_NOIO, mddev->bio_set);
2599
		bio_trim(wbio, sector - bio->bi_iter.bi_sector, sectors);
2600 2601 2602
		wsector = r10_bio->devs[i].addr + (sector - r10_bio->sector);
		wbio->bi_iter.bi_sector = wsector +
				   choose_data_offset(r10_bio, rdev);
2603
		wbio->bi_bdev = rdev->bdev;
M
Mike Christie 已提交
2604
		bio_set_op_attrs(wbio, REQ_OP_WRITE, 0);
2605 2606

		if (submit_bio_wait(wbio) < 0)
2607
			/* Failure! */
2608
			ok = rdev_set_badblocks(rdev, wsector,
2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619
						sectors, 0)
				&& ok;

		bio_put(wbio);
		sect_to_write -= sectors;
		sector += sectors;
		sectors = block_sectors;
	}
	return ok;
}

2620
static void handle_read_error(struct mddev *mddev, struct r10bio *r10_bio)
2621 2622 2623
{
	int slot = r10_bio->read_slot;
	struct bio *bio;
2624
	struct r10conf *conf = mddev->private;
2625
	struct md_rdev *rdev = r10_bio->devs[slot].rdev;
2626 2627
	char b[BDEVNAME_SIZE];
	unsigned long do_sync;
2628
	int max_sectors;
2629 2630
	dev_t bio_dev;
	sector_t bio_last_sector;
2631 2632 2633 2634 2635 2636 2637 2638 2639

	/* we got a read error. Maybe the drive is bad.  Maybe just
	 * the block and we can fix it.
	 * We freeze all other IO, and try reading the block from
	 * other devices.  When we find one, we re-write
	 * and check it that fixes the read error.
	 * This is all done synchronously while the array is
	 * frozen.
	 */
2640 2641
	bio = r10_bio->devs[slot].bio;
	bdevname(bio->bi_bdev, b);
2642 2643
	bio_dev = bio->bi_bdev->bd_dev;
	bio_last_sector = r10_bio->devs[slot].addr + rdev->data_offset + r10_bio->sectors;
2644 2645 2646
	bio_put(bio);
	r10_bio->devs[slot].bio = NULL;

2647 2648 2649
	if (mddev->ro)
		r10_bio->devs[slot].bio = IO_BLOCKED;
	else if (!test_bit(FailFast, &rdev->flags)) {
2650
		freeze_array(conf, 1);
2651 2652
		fix_read_error(conf, mddev, r10_bio);
		unfreeze_array(conf);
2653
	} else
2654
		md_error(mddev, rdev);
2655

2656
	rdev_dec_pending(rdev, mddev);
2657

2658
read_more:
2659 2660
	rdev = read_balance(conf, r10_bio, &max_sectors);
	if (rdev == NULL) {
N
NeilBrown 已提交
2661 2662 2663
		pr_crit_ratelimited("md/raid10:%s: %s: unrecoverable I/O read error for block %llu\n",
				    mdname(mddev), b,
				    (unsigned long long)r10_bio->sector);
2664 2665 2666 2667
		raid_end_bio_io(r10_bio);
		return;
	}

J
Jens Axboe 已提交
2668
	do_sync = (r10_bio->master_bio->bi_opf & REQ_SYNC);
2669
	slot = r10_bio->read_slot;
N
NeilBrown 已提交
2670 2671 2672 2673
	pr_err_ratelimited("md/raid10:%s: %s: redirecting sector %llu to another mirror\n",
			   mdname(mddev),
			   bdevname(rdev->bdev, b),
			   (unsigned long long)r10_bio->sector);
2674
	bio = bio_clone_fast(r10_bio->master_bio, GFP_NOIO, mddev->bio_set);
2675
	bio_trim(bio, r10_bio->sector - bio->bi_iter.bi_sector, max_sectors);
2676
	r10_bio->devs[slot].bio = bio;
2677
	r10_bio->devs[slot].rdev = rdev;
2678
	bio->bi_iter.bi_sector = r10_bio->devs[slot].addr
2679
		+ choose_data_offset(r10_bio, rdev);
2680
	bio->bi_bdev = rdev->bdev;
M
Mike Christie 已提交
2681
	bio_set_op_attrs(bio, REQ_OP_READ, do_sync);
2682 2683 2684
	if (test_bit(FailFast, &rdev->flags) &&
	    test_bit(R10BIO_FailFast, &r10_bio->state))
		bio->bi_opf |= MD_FAILFAST;
2685 2686
	bio->bi_private = r10_bio;
	bio->bi_end_io = raid10_end_read_request;
2687 2688 2689 2690
	trace_block_bio_remap(bdev_get_queue(bio->bi_bdev),
			      bio, bio_dev,
			      bio_last_sector - r10_bio->sectors);

2691 2692 2693 2694 2695
	if (max_sectors < r10_bio->sectors) {
		/* Drat - have to split this up more */
		struct bio *mbio = r10_bio->master_bio;
		int sectors_handled =
			r10_bio->sector + max_sectors
2696
			- mbio->bi_iter.bi_sector;
2697
		r10_bio->sectors = max_sectors;
2698 2699
		bio_inc_remaining(mbio);
		inc_pending(conf);
2700 2701 2702 2703 2704
		generic_make_request(bio);

		r10_bio = mempool_alloc(conf->r10bio_pool,
					GFP_NOIO);
		r10_bio->master_bio = mbio;
2705
		r10_bio->sectors = bio_sectors(mbio) - sectors_handled;
2706 2707 2708 2709
		r10_bio->state = 0;
		set_bit(R10BIO_ReadError,
			&r10_bio->state);
		r10_bio->mddev = mddev;
2710
		r10_bio->sector = mbio->bi_iter.bi_sector
2711 2712 2713 2714 2715
			+ sectors_handled;

		goto read_more;
	} else
		generic_make_request(bio);
2716 2717
}

2718
static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio)
2719 2720 2721 2722
{
	/* Some sort of write request has finished and it
	 * succeeded in writing where we thought there was a
	 * bad block.  So forget the bad block.
2723 2724
	 * Or possibly if failed and we need to record
	 * a bad block.
2725 2726
	 */
	int m;
2727
	struct md_rdev *rdev;
2728 2729 2730

	if (test_bit(R10BIO_IsSync, &r10_bio->state) ||
	    test_bit(R10BIO_IsRecover, &r10_bio->state)) {
2731 2732 2733 2734 2735
		for (m = 0; m < conf->copies; m++) {
			int dev = r10_bio->devs[m].devnum;
			rdev = conf->mirrors[dev].rdev;
			if (r10_bio->devs[m].bio == NULL)
				continue;
2736
			if (!r10_bio->devs[m].bio->bi_error) {
2737 2738 2739
				rdev_clear_badblocks(
					rdev,
					r10_bio->devs[m].addr,
2740
					r10_bio->sectors, 0);
2741 2742 2743 2744 2745 2746
			} else {
				if (!rdev_set_badblocks(
					    rdev,
					    r10_bio->devs[m].addr,
					    r10_bio->sectors, 0))
					md_error(conf->mddev, rdev);
2747
			}
2748 2749 2750
			rdev = conf->mirrors[dev].replacement;
			if (r10_bio->devs[m].repl_bio == NULL)
				continue;
2751 2752

			if (!r10_bio->devs[m].repl_bio->bi_error) {
2753 2754 2755
				rdev_clear_badblocks(
					rdev,
					r10_bio->devs[m].addr,
2756
					r10_bio->sectors, 0);
2757 2758 2759 2760 2761 2762 2763
			} else {
				if (!rdev_set_badblocks(
					    rdev,
					    r10_bio->devs[m].addr,
					    r10_bio->sectors, 0))
					md_error(conf->mddev, rdev);
			}
2764
		}
2765 2766
		put_buf(r10_bio);
	} else {
2767
		bool fail = false;
2768 2769 2770 2771 2772
		for (m = 0; m < conf->copies; m++) {
			int dev = r10_bio->devs[m].devnum;
			struct bio *bio = r10_bio->devs[m].bio;
			rdev = conf->mirrors[dev].rdev;
			if (bio == IO_MADE_GOOD) {
2773 2774 2775
				rdev_clear_badblocks(
					rdev,
					r10_bio->devs[m].addr,
2776
					r10_bio->sectors, 0);
2777
				rdev_dec_pending(rdev, conf->mddev);
2778
			} else if (bio != NULL && bio->bi_error) {
2779
				fail = true;
2780 2781 2782 2783 2784 2785
				if (!narrow_write_error(r10_bio, m)) {
					md_error(conf->mddev, rdev);
					set_bit(R10BIO_Degraded,
						&r10_bio->state);
				}
				rdev_dec_pending(rdev, conf->mddev);
2786
			}
2787 2788
			bio = r10_bio->devs[m].repl_bio;
			rdev = conf->mirrors[dev].replacement;
2789
			if (rdev && bio == IO_MADE_GOOD) {
2790 2791 2792
				rdev_clear_badblocks(
					rdev,
					r10_bio->devs[m].addr,
2793
					r10_bio->sectors, 0);
2794 2795
				rdev_dec_pending(rdev, conf->mddev);
			}
2796
		}
2797 2798 2799
		if (fail) {
			spin_lock_irq(&conf->device_lock);
			list_add(&r10_bio->retry_list, &conf->bio_end_io_list);
2800
			conf->nr_queued++;
2801 2802
			spin_unlock_irq(&conf->device_lock);
			md_wakeup_thread(conf->mddev->thread);
2803 2804 2805 2806
		} else {
			if (test_bit(R10BIO_WriteError,
				     &r10_bio->state))
				close_write(r10_bio);
2807
			raid_end_bio_io(r10_bio);
2808
		}
2809 2810 2811
	}
}

S
Shaohua Li 已提交
2812
static void raid10d(struct md_thread *thread)
L
Linus Torvalds 已提交
2813
{
S
Shaohua Li 已提交
2814
	struct mddev *mddev = thread->mddev;
2815
	struct r10bio *r10_bio;
L
Linus Torvalds 已提交
2816
	unsigned long flags;
2817
	struct r10conf *conf = mddev->private;
L
Linus Torvalds 已提交
2818
	struct list_head *head = &conf->retry_list;
2819
	struct blk_plug plug;
L
Linus Torvalds 已提交
2820 2821 2822

	md_check_recovery(mddev);

2823
	if (!list_empty_careful(&conf->bio_end_io_list) &&
2824
	    !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
2825 2826
		LIST_HEAD(tmp);
		spin_lock_irqsave(&conf->device_lock, flags);
2827
		if (!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
2828 2829 2830 2831
			while (!list_empty(&conf->bio_end_io_list)) {
				list_move(conf->bio_end_io_list.prev, &tmp);
				conf->nr_queued--;
			}
2832 2833 2834
		}
		spin_unlock_irqrestore(&conf->device_lock, flags);
		while (!list_empty(&tmp)) {
2835 2836
			r10_bio = list_first_entry(&tmp, struct r10bio,
						   retry_list);
2837
			list_del(&r10_bio->retry_list);
2838 2839 2840 2841 2842 2843
			if (mddev->degraded)
				set_bit(R10BIO_Degraded, &r10_bio->state);

			if (test_bit(R10BIO_WriteError,
				     &r10_bio->state))
				close_write(r10_bio);
2844 2845 2846 2847
			raid_end_bio_io(r10_bio);
		}
	}

2848
	blk_start_plug(&plug);
L
Linus Torvalds 已提交
2849
	for (;;) {
2850

2851
		flush_pending_writes(conf);
2852

2853 2854 2855
		spin_lock_irqsave(&conf->device_lock, flags);
		if (list_empty(head)) {
			spin_unlock_irqrestore(&conf->device_lock, flags);
L
Linus Torvalds 已提交
2856
			break;
2857
		}
2858
		r10_bio = list_entry(head->prev, struct r10bio, retry_list);
L
Linus Torvalds 已提交
2859
		list_del(head->prev);
2860
		conf->nr_queued--;
L
Linus Torvalds 已提交
2861 2862 2863
		spin_unlock_irqrestore(&conf->device_lock, flags);

		mddev = r10_bio->mddev;
2864
		conf = mddev->private;
2865 2866
		if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
		    test_bit(R10BIO_WriteError, &r10_bio->state))
2867
			handle_write_completed(conf, r10_bio);
N
NeilBrown 已提交
2868 2869
		else if (test_bit(R10BIO_IsReshape, &r10_bio->state))
			reshape_request_write(mddev, r10_bio);
2870
		else if (test_bit(R10BIO_IsSync, &r10_bio->state))
L
Linus Torvalds 已提交
2871
			sync_request_write(mddev, r10_bio);
J
Jens Axboe 已提交
2872
		else if (test_bit(R10BIO_IsRecover, &r10_bio->state))
L
Linus Torvalds 已提交
2873
			recovery_request_write(mddev, r10_bio);
2874
		else if (test_bit(R10BIO_ReadError, &r10_bio->state))
2875
			handle_read_error(mddev, r10_bio);
2876 2877 2878 2879 2880 2881 2882
		else {
			/* just a partial read to be scheduled from a
			 * separate context
			 */
			int slot = r10_bio->read_slot;
			generic_make_request(r10_bio->devs[slot].bio);
		}
2883

N
NeilBrown 已提交
2884
		cond_resched();
2885
		if (mddev->sb_flags & ~(1<<MD_SB_CHANGE_PENDING))
2886
			md_check_recovery(mddev);
L
Linus Torvalds 已提交
2887
	}
2888
	blk_finish_plug(&plug);
L
Linus Torvalds 已提交
2889 2890
}

2891
static int init_resync(struct r10conf *conf)
L
Linus Torvalds 已提交
2892 2893
{
	int buffs;
2894
	int i;
L
Linus Torvalds 已提交
2895 2896

	buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE;
2897
	BUG_ON(conf->r10buf_pool);
2898
	conf->have_replacement = 0;
2899
	for (i = 0; i < conf->geo.raid_disks; i++)
2900 2901
		if (conf->mirrors[i].replacement)
			conf->have_replacement = 1;
L
Linus Torvalds 已提交
2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940
	conf->r10buf_pool = mempool_create(buffs, r10buf_pool_alloc, r10buf_pool_free, conf);
	if (!conf->r10buf_pool)
		return -ENOMEM;
	conf->next_resync = 0;
	return 0;
}

/*
 * perform a "sync" on one "block"
 *
 * We need to make sure that no normal I/O request - particularly write
 * requests - conflict with active sync requests.
 *
 * This is achieved by tracking pending requests and a 'barrier' concept
 * that can be installed to exclude normal IO requests.
 *
 * Resync and recovery are handled very differently.
 * We differentiate by looking at MD_RECOVERY_SYNC in mddev->recovery.
 *
 * For resync, we iterate over virtual addresses, read all copies,
 * and update if there are differences.  If only one copy is live,
 * skip it.
 * For recovery, we iterate over physical addresses, read a good
 * value for each non-in_sync drive, and over-write.
 *
 * So, for recovery we may have several outstanding complex requests for a
 * given address, one for each out-of-sync device.  We model this by allocating
 * a number of r10_bio structures, one for each out-of-sync device.
 * As we setup these structures, we collect all bio's together into a list
 * which we then process collectively to add pages, and then process again
 * to pass to generic_make_request.
 *
 * The r10_bio structures are linked using a borrowed master_bio pointer.
 * This link is counted in ->remaining.  When the r10_bio that points to NULL
 * has its remaining count decremented to 0, the whole complex operation
 * is complete.
 *
 */

S
Shaohua Li 已提交
2941
static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr,
2942
			     int *skipped)
L
Linus Torvalds 已提交
2943
{
2944
	struct r10conf *conf = mddev->private;
2945
	struct r10bio *r10_bio;
L
Linus Torvalds 已提交
2946 2947 2948
	struct bio *biolist = NULL, *bio;
	sector_t max_sector, nr_sectors;
	int i;
2949
	int max_sync;
N
NeilBrown 已提交
2950
	sector_t sync_blocks;
L
Linus Torvalds 已提交
2951 2952
	sector_t sectors_skipped = 0;
	int chunks_skipped = 0;
2953
	sector_t chunk_mask = conf->geo.chunk_mask;
L
Linus Torvalds 已提交
2954 2955 2956

	if (!conf->r10buf_pool)
		if (init_resync(conf))
2957
			return 0;
L
Linus Torvalds 已提交
2958

2959 2960 2961 2962 2963 2964
	/*
	 * Allow skipping a full rebuild for incremental assembly
	 * of a clean array, like RAID1 does.
	 */
	if (mddev->bitmap == NULL &&
	    mddev->recovery_cp == MaxSector &&
2965 2966
	    mddev->reshape_position == MaxSector &&
	    !test_bit(MD_RECOVERY_SYNC, &mddev->recovery) &&
2967
	    !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
2968
	    !test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
2969 2970
	    conf->fullsync == 0) {
		*skipped = 1;
2971
		return mddev->dev_sectors - sector_nr;
2972 2973
	}

L
Linus Torvalds 已提交
2974
 skipped:
A
Andre Noll 已提交
2975
	max_sector = mddev->dev_sectors;
N
NeilBrown 已提交
2976 2977
	if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) ||
	    test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
L
Linus Torvalds 已提交
2978 2979
		max_sector = mddev->resync_max_sectors;
	if (sector_nr >= max_sector) {
2980 2981 2982 2983 2984 2985 2986 2987 2988
		/* If we aborted, we need to abort the
		 * sync on the 'current' bitmap chucks (there can
		 * be several when recovering multiple devices).
		 * as we may have started syncing it but not finished.
		 * We can find the current address in
		 * mddev->curr_resync, but for recovery,
		 * we need to convert that to several
		 * virtual addresses.
		 */
N
NeilBrown 已提交
2989 2990
		if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {
			end_reshape(conf);
2991
			close_sync(conf);
N
NeilBrown 已提交
2992 2993 2994
			return 0;
		}

2995 2996 2997 2998
		if (mddev->curr_resync < max_sector) { /* aborted */
			if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery))
				bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
						&sync_blocks, 1);
2999
			else for (i = 0; i < conf->geo.raid_disks; i++) {
3000 3001 3002 3003 3004
				sector_t sect =
					raid10_find_virt(conf, mddev->curr_resync, i);
				bitmap_end_sync(mddev->bitmap, sect,
						&sync_blocks, 1);
			}
3005 3006 3007 3008 3009 3010 3011 3012
		} else {
			/* completed sync */
			if ((!mddev->bitmap || conf->fullsync)
			    && conf->have_replacement
			    && test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
				/* Completed a full sync so the replacements
				 * are now fully recovered.
				 */
3013 3014 3015 3016 3017 3018 3019 3020
				rcu_read_lock();
				for (i = 0; i < conf->geo.raid_disks; i++) {
					struct md_rdev *rdev =
						rcu_dereference(conf->mirrors[i].replacement);
					if (rdev)
						rdev->recovery_offset = MaxSector;
				}
				rcu_read_unlock();
3021
			}
3022
			conf->fullsync = 0;
3023
		}
3024
		bitmap_close_sync(mddev->bitmap);
L
Linus Torvalds 已提交
3025
		close_sync(conf);
3026
		*skipped = 1;
L
Linus Torvalds 已提交
3027 3028
		return sectors_skipped;
	}
N
NeilBrown 已提交
3029 3030 3031 3032

	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
		return reshape_request(mddev, sector_nr, skipped);

3033
	if (chunks_skipped >= conf->geo.raid_disks) {
L
Linus Torvalds 已提交
3034 3035 3036
		/* if there has been nothing to do on any drive,
		 * then there is nothing to do at all..
		 */
3037 3038
		*skipped = 1;
		return (max_sector - sector_nr) + sectors_skipped;
L
Linus Torvalds 已提交
3039 3040
	}

3041 3042 3043
	if (max_sector > mddev->resync_max)
		max_sector = mddev->resync_max; /* Don't do IO beyond here */

L
Linus Torvalds 已提交
3044 3045 3046
	/* make sure whole request will fit in a chunk - if chunks
	 * are meaningful
	 */
3047 3048 3049
	if (conf->geo.near_copies < conf->geo.raid_disks &&
	    max_sector > (sector_nr | chunk_mask))
		max_sector = (sector_nr | chunk_mask) + 1;
L
Linus Torvalds 已提交
3050

3051 3052 3053 3054 3055 3056 3057
	/*
	 * If there is non-resync activity waiting for a turn, then let it
	 * though before starting on this new sync request.
	 */
	if (conf->nr_waiting)
		schedule_timeout_uninterruptible(1);

L
Linus Torvalds 已提交
3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072
	/* Again, very different code for resync and recovery.
	 * Both must result in an r10bio with a list of bios that
	 * have bi_end_io, bi_sector, bi_bdev set,
	 * and bi_private set to the r10bio.
	 * For recovery, we may actually create several r10bios
	 * with 2 bios in each, that correspond to the bios in the main one.
	 * In this case, the subordinate r10bios link back through a
	 * borrowed master_bio pointer, and the counter in the master
	 * includes a ref from each subordinate.
	 */
	/* First, we decide what to do and set ->bi_end_io
	 * To end_sync_read if we want to read, and
	 * end_sync_write if we will want to write.
	 */

3073
	max_sync = RESYNC_PAGES << (PAGE_SHIFT-9);
L
Linus Torvalds 已提交
3074 3075
	if (!test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
		/* recovery... the complicated one */
3076
		int j;
L
Linus Torvalds 已提交
3077 3078
		r10_bio = NULL;

3079
		for (i = 0 ; i < conf->geo.raid_disks; i++) {
3080
			int still_degraded;
3081
			struct r10bio *rb2;
3082 3083
			sector_t sect;
			int must_sync;
3084
			int any_working;
3085
			struct raid10_info *mirror = &conf->mirrors[i];
3086
			struct md_rdev *mrdev, *mreplace;
3087

3088 3089 3090 3091 3092
			rcu_read_lock();
			mrdev = rcu_dereference(mirror->rdev);
			mreplace = rcu_dereference(mirror->replacement);

			if ((mrdev == NULL ||
3093
			     test_bit(Faulty, &mrdev->flags) ||
3094 3095 3096 3097
			     test_bit(In_sync, &mrdev->flags)) &&
			    (mreplace == NULL ||
			     test_bit(Faulty, &mreplace->flags))) {
				rcu_read_unlock();
3098
				continue;
3099
			}
L
Linus Torvalds 已提交
3100

3101 3102 3103 3104
			still_degraded = 0;
			/* want to reconstruct this device */
			rb2 = r10_bio;
			sect = raid10_find_virt(conf, sector_nr, i);
3105 3106 3107 3108
			if (sect >= mddev->resync_max_sectors) {
				/* last stripe is not complete - don't
				 * try to recover this sector.
				 */
3109
				rcu_read_unlock();
3110 3111
				continue;
			}
3112 3113
			if (mreplace && test_bit(Faulty, &mreplace->flags))
				mreplace = NULL;
3114 3115 3116
			/* Unless we are doing a full sync, or a replacement
			 * we only need to recover the block if it is set in
			 * the bitmap
3117 3118 3119 3120 3121 3122
			 */
			must_sync = bitmap_start_sync(mddev->bitmap, sect,
						      &sync_blocks, 1);
			if (sync_blocks < max_sync)
				max_sync = sync_blocks;
			if (!must_sync &&
3123
			    mreplace == NULL &&
3124 3125 3126 3127 3128
			    !conf->fullsync) {
				/* yep, skip the sync_blocks here, but don't assume
				 * that there will never be anything to do here
				 */
				chunks_skipped = -1;
3129
				rcu_read_unlock();
3130 3131
				continue;
			}
3132 3133 3134 3135
			atomic_inc(&mrdev->nr_pending);
			if (mreplace)
				atomic_inc(&mreplace->nr_pending);
			rcu_read_unlock();
3136

3137
			r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
3138
			r10_bio->state = 0;
3139 3140
			raise_barrier(conf, rb2 != NULL);
			atomic_set(&r10_bio->remaining, 0);
3141

3142 3143 3144 3145 3146 3147
			r10_bio->master_bio = (struct bio*)rb2;
			if (rb2)
				atomic_inc(&rb2->remaining);
			r10_bio->mddev = mddev;
			set_bit(R10BIO_IsRecover, &r10_bio->state);
			r10_bio->sector = sect;
L
Linus Torvalds 已提交
3148

3149 3150 3151 3152 3153
			raid10_find_phys(conf, r10_bio);

			/* Need to check if the array will still be
			 * degraded
			 */
3154 3155 3156 3157 3158
			rcu_read_lock();
			for (j = 0; j < conf->geo.raid_disks; j++) {
				struct md_rdev *rdev = rcu_dereference(
					conf->mirrors[j].rdev);
				if (rdev == NULL || test_bit(Faulty, &rdev->flags)) {
3159
					still_degraded = 1;
3160
					break;
L
Linus Torvalds 已提交
3161
				}
3162
			}
3163 3164 3165 3166

			must_sync = bitmap_start_sync(mddev->bitmap, sect,
						      &sync_blocks, still_degraded);

3167
			any_working = 0;
3168
			for (j=0; j<conf->copies;j++) {
3169
				int k;
3170
				int d = r10_bio->devs[j].devnum;
3171
				sector_t from_addr, to_addr;
3172 3173
				struct md_rdev *rdev =
					rcu_dereference(conf->mirrors[d].rdev);
3174 3175
				sector_t sector, first_bad;
				int bad_sectors;
3176 3177
				if (!rdev ||
				    !test_bit(In_sync, &rdev->flags))
3178 3179
					continue;
				/* This is where we read from */
3180
				any_working = 1;
3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194
				sector = r10_bio->devs[j].addr;

				if (is_badblock(rdev, sector, max_sync,
						&first_bad, &bad_sectors)) {
					if (first_bad > sector)
						max_sync = first_bad - sector;
					else {
						bad_sectors -= (sector
								- first_bad);
						if (max_sync > bad_sectors)
							max_sync = bad_sectors;
						continue;
					}
				}
3195 3196 3197 3198
				bio = r10_bio->devs[0].bio;
				bio->bi_next = biolist;
				biolist = bio;
				bio->bi_end_io = end_sync_read;
M
Mike Christie 已提交
3199
				bio_set_op_attrs(bio, REQ_OP_READ, 0);
3200 3201
				if (test_bit(FailFast, &rdev->flags))
					bio->bi_opf |= MD_FAILFAST;
3202
				from_addr = r10_bio->devs[j].addr;
3203 3204
				bio->bi_iter.bi_sector = from_addr +
					rdev->data_offset;
3205 3206 3207
				bio->bi_bdev = rdev->bdev;
				atomic_inc(&rdev->nr_pending);
				/* and we write to 'i' (if not in_sync) */
3208 3209 3210 3211 3212

				for (k=0; k<conf->copies; k++)
					if (r10_bio->devs[k].devnum == i)
						break;
				BUG_ON(k == conf->copies);
3213
				to_addr = r10_bio->devs[k].addr;
3214
				r10_bio->devs[0].devnum = d;
3215
				r10_bio->devs[0].addr = from_addr;
3216
				r10_bio->devs[1].devnum = i;
3217
				r10_bio->devs[1].addr = to_addr;
3218

3219
				if (!test_bit(In_sync, &mrdev->flags)) {
3220 3221 3222 3223
					bio = r10_bio->devs[1].bio;
					bio->bi_next = biolist;
					biolist = bio;
					bio->bi_end_io = end_sync_write;
M
Mike Christie 已提交
3224
					bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
3225
					bio->bi_iter.bi_sector = to_addr
3226 3227
						+ mrdev->data_offset;
					bio->bi_bdev = mrdev->bdev;
3228 3229 3230 3231 3232 3233 3234 3235
					atomic_inc(&r10_bio->remaining);
				} else
					r10_bio->devs[1].bio->bi_end_io = NULL;

				/* and maybe write to replacement */
				bio = r10_bio->devs[1].repl_bio;
				if (bio)
					bio->bi_end_io = NULL;
3236
				/* Note: if mreplace != NULL, then bio
3237 3238 3239 3240 3241 3242 3243
				 * cannot be NULL as r10buf_pool_alloc will
				 * have allocated it.
				 * So the second test here is pointless.
				 * But it keeps semantic-checkers happy, and
				 * this comment keeps human reviewers
				 * happy.
				 */
3244 3245
				if (mreplace == NULL || bio == NULL ||
				    test_bit(Faulty, &mreplace->flags))
3246 3247 3248 3249
					break;
				bio->bi_next = biolist;
				biolist = bio;
				bio->bi_end_io = end_sync_write;
M
Mike Christie 已提交
3250
				bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
3251
				bio->bi_iter.bi_sector = to_addr +
3252 3253
					mreplace->data_offset;
				bio->bi_bdev = mreplace->bdev;
3254
				atomic_inc(&r10_bio->remaining);
3255 3256
				break;
			}
3257
			rcu_read_unlock();
3258
			if (j == conf->copies) {
3259 3260 3261 3262 3263 3264 3265 3266 3267 3268
				/* Cannot recover, so abort the recovery or
				 * record a bad block */
				if (any_working) {
					/* problem is that there are bad blocks
					 * on other device(s)
					 */
					int k;
					for (k = 0; k < conf->copies; k++)
						if (r10_bio->devs[k].devnum == i)
							break;
3269
					if (!test_bit(In_sync,
3270
						      &mrdev->flags)
3271
					    && !rdev_set_badblocks(
3272
						    mrdev,
3273 3274 3275
						    r10_bio->devs[k].addr,
						    max_sync, 0))
						any_working = 0;
3276
					if (mreplace &&
3277
					    !rdev_set_badblocks(
3278
						    mreplace,
3279 3280 3281 3282 3283 3284 3285
						    r10_bio->devs[k].addr,
						    max_sync, 0))
						any_working = 0;
				}
				if (!any_working)  {
					if (!test_and_set_bit(MD_RECOVERY_INTR,
							      &mddev->recovery))
N
NeilBrown 已提交
3286
						pr_warn("md/raid10:%s: insufficient working devices for recovery.\n",
3287
						       mdname(mddev));
3288
					mirror->recovery_disabled
3289 3290
						= mddev->recovery_disabled;
				}
3291 3292 3293 3294
				put_buf(r10_bio);
				if (rb2)
					atomic_dec(&rb2->remaining);
				r10_bio = rb2;
3295 3296 3297
				rdev_dec_pending(mrdev, mddev);
				if (mreplace)
					rdev_dec_pending(mreplace, mddev);
3298
				break;
L
Linus Torvalds 已提交
3299
			}
3300 3301 3302
			rdev_dec_pending(mrdev, mddev);
			if (mreplace)
				rdev_dec_pending(mreplace, mddev);
3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319
			if (r10_bio->devs[0].bio->bi_opf & MD_FAILFAST) {
				/* Only want this if there is elsewhere to
				 * read from. 'j' is currently the first
				 * readable copy.
				 */
				int targets = 1;
				for (; j < conf->copies; j++) {
					int d = r10_bio->devs[j].devnum;
					if (conf->mirrors[d].rdev &&
					    test_bit(In_sync,
						      &conf->mirrors[d].rdev->flags))
						targets++;
				}
				if (targets == 1)
					r10_bio->devs[0].bio->bi_opf
						&= ~MD_FAILFAST;
			}
3320
		}
L
Linus Torvalds 已提交
3321 3322
		if (biolist == NULL) {
			while (r10_bio) {
3323 3324
				struct r10bio *rb2 = r10_bio;
				r10_bio = (struct r10bio*) rb2->master_bio;
L
Linus Torvalds 已提交
3325 3326 3327 3328 3329 3330 3331 3332
				rb2->master_bio = NULL;
				put_buf(rb2);
			}
			goto giveup;
		}
	} else {
		/* resync. Schedule a read for every block at this virt offset */
		int count = 0;
3333

3334
		bitmap_cond_end_sync(mddev->bitmap, sector_nr, 0);
3335

3336 3337
		if (!bitmap_start_sync(mddev->bitmap, sector_nr,
				       &sync_blocks, mddev->degraded) &&
3338 3339
		    !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED,
						 &mddev->recovery)) {
3340 3341 3342 3343 3344 3345
			/* We can skip this block */
			*skipped = 1;
			return sync_blocks + sectors_skipped;
		}
		if (sync_blocks < max_sync)
			max_sync = sync_blocks;
L
Linus Torvalds 已提交
3346
		r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
3347
		r10_bio->state = 0;
L
Linus Torvalds 已提交
3348 3349 3350

		r10_bio->mddev = mddev;
		atomic_set(&r10_bio->remaining, 0);
3351 3352
		raise_barrier(conf, 0);
		conf->next_resync = sector_nr;
L
Linus Torvalds 已提交
3353 3354 3355 3356 3357

		r10_bio->master_bio = NULL;
		r10_bio->sector = sector_nr;
		set_bit(R10BIO_IsSync, &r10_bio->state);
		raid10_find_phys(conf, r10_bio);
3358
		r10_bio->sectors = (sector_nr | chunk_mask) - sector_nr + 1;
L
Linus Torvalds 已提交
3359

3360
		for (i = 0; i < conf->copies; i++) {
L
Linus Torvalds 已提交
3361
			int d = r10_bio->devs[i].devnum;
3362 3363
			sector_t first_bad, sector;
			int bad_sectors;
3364
			struct md_rdev *rdev;
3365

3366 3367 3368
			if (r10_bio->devs[i].repl_bio)
				r10_bio->devs[i].repl_bio->bi_end_io = NULL;

L
Linus Torvalds 已提交
3369
			bio = r10_bio->devs[i].bio;
3370
			bio->bi_error = -EIO;
3371 3372 3373 3374
			rcu_read_lock();
			rdev = rcu_dereference(conf->mirrors[d].rdev);
			if (rdev == NULL || test_bit(Faulty, &rdev->flags)) {
				rcu_read_unlock();
L
Linus Torvalds 已提交
3375
				continue;
3376
			}
3377
			sector = r10_bio->devs[i].addr;
3378
			if (is_badblock(rdev, sector, max_sync,
3379 3380 3381 3382 3383 3384
					&first_bad, &bad_sectors)) {
				if (first_bad > sector)
					max_sync = first_bad - sector;
				else {
					bad_sectors -= (sector - first_bad);
					if (max_sync > bad_sectors)
3385
						max_sync = bad_sectors;
3386
					rcu_read_unlock();
3387 3388 3389
					continue;
				}
			}
3390
			atomic_inc(&rdev->nr_pending);
L
Linus Torvalds 已提交
3391 3392 3393 3394
			atomic_inc(&r10_bio->remaining);
			bio->bi_next = biolist;
			biolist = bio;
			bio->bi_end_io = end_sync_read;
M
Mike Christie 已提交
3395
			bio_set_op_attrs(bio, REQ_OP_READ, 0);
3396 3397
			if (test_bit(FailFast, &conf->mirrors[d].rdev->flags))
				bio->bi_opf |= MD_FAILFAST;
3398 3399
			bio->bi_iter.bi_sector = sector + rdev->data_offset;
			bio->bi_bdev = rdev->bdev;
L
Linus Torvalds 已提交
3400
			count++;
3401

3402 3403 3404
			rdev = rcu_dereference(conf->mirrors[d].replacement);
			if (rdev == NULL || test_bit(Faulty, &rdev->flags)) {
				rcu_read_unlock();
3405
				continue;
3406 3407 3408
			}
			atomic_inc(&rdev->nr_pending);
			rcu_read_unlock();
3409 3410 3411

			/* Need to set up for writing to the replacement */
			bio = r10_bio->devs[i].repl_bio;
3412
			bio->bi_error = -EIO;
3413 3414 3415 3416 3417

			sector = r10_bio->devs[i].addr;
			bio->bi_next = biolist;
			biolist = bio;
			bio->bi_end_io = end_sync_write;
M
Mike Christie 已提交
3418
			bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
3419 3420
			if (test_bit(FailFast, &conf->mirrors[d].rdev->flags))
				bio->bi_opf |= MD_FAILFAST;
3421 3422
			bio->bi_iter.bi_sector = sector + rdev->data_offset;
			bio->bi_bdev = rdev->bdev;
3423
			count++;
L
Linus Torvalds 已提交
3424 3425 3426 3427 3428 3429
		}

		if (count < 2) {
			for (i=0; i<conf->copies; i++) {
				int d = r10_bio->devs[i].devnum;
				if (r10_bio->devs[i].bio->bi_end_io)
3430 3431
					rdev_dec_pending(conf->mirrors[d].rdev,
							 mddev);
3432 3433 3434 3435 3436
				if (r10_bio->devs[i].repl_bio &&
				    r10_bio->devs[i].repl_bio->bi_end_io)
					rdev_dec_pending(
						conf->mirrors[d].replacement,
						mddev);
L
Linus Torvalds 已提交
3437 3438 3439 3440 3441 3442 3443 3444
			}
			put_buf(r10_bio);
			biolist = NULL;
			goto giveup;
		}
	}

	nr_sectors = 0;
3445 3446
	if (sector_nr + max_sync < max_sector)
		max_sector = sector_nr + max_sync;
L
Linus Torvalds 已提交
3447 3448 3449 3450 3451 3452 3453 3454
	do {
		struct page *page;
		int len = PAGE_SIZE;
		if (sector_nr + (len>>9) > max_sector)
			len = (max_sector - sector_nr) << 9;
		if (len == 0)
			break;
		for (bio= biolist ; bio ; bio=bio->bi_next) {
3455 3456
			struct resync_pages *rp = get_resync_pages(bio);
			page = resync_fetch_page(rp, rp->idx++);
3457 3458 3459 3460 3461
			/*
			 * won't fail because the vec table is big enough
			 * to hold all these pages
			 */
			bio_add_page(bio, page, len, 0);
L
Linus Torvalds 已提交
3462 3463 3464
		}
		nr_sectors += len>>9;
		sector_nr += len>>9;
3465
	} while (get_resync_pages(biolist)->idx < RESYNC_PAGES);
L
Linus Torvalds 已提交
3466 3467 3468 3469 3470 3471 3472
	r10_bio->sectors = nr_sectors;

	while (biolist) {
		bio = biolist;
		biolist = biolist->bi_next;

		bio->bi_next = NULL;
3473
		r10_bio = get_resync_r10bio(bio);
L
Linus Torvalds 已提交
3474 3475 3476 3477
		r10_bio->sectors = nr_sectors;

		if (bio->bi_end_io == end_sync_read) {
			md_sync_acct(bio->bi_bdev, nr_sectors);
3478
			bio->bi_error = 0;
L
Linus Torvalds 已提交
3479 3480 3481 3482
			generic_make_request(bio);
		}
	}

3483 3484 3485 3486 3487 3488
	if (sectors_skipped)
		/* pretend they weren't skipped, it makes
		 * no important difference in this case
		 */
		md_done_sync(mddev, sectors_skipped, 1);

L
Linus Torvalds 已提交
3489 3490 3491
	return sectors_skipped + nr_sectors;
 giveup:
	/* There is nowhere to write, so all non-sync
3492 3493
	 * drives must be failed or in resync, all drives
	 * have a bad block, so try the next chunk...
L
Linus Torvalds 已提交
3494
	 */
3495 3496 3497 3498
	if (sector_nr + max_sync < max_sector)
		max_sector = sector_nr + max_sync;

	sectors_skipped += (max_sector - sector_nr);
L
Linus Torvalds 已提交
3499 3500 3501 3502 3503
	chunks_skipped ++;
	sector_nr = max_sector;
	goto skipped;
}

3504
static sector_t
3505
raid10_size(struct mddev *mddev, sector_t sectors, int raid_disks)
3506 3507
{
	sector_t size;
3508
	struct r10conf *conf = mddev->private;
3509 3510

	if (!raid_disks)
N
NeilBrown 已提交
3511 3512
		raid_disks = min(conf->geo.raid_disks,
				 conf->prev.raid_disks);
3513
	if (!sectors)
3514
		sectors = conf->dev_sectors;
3515

3516 3517
	size = sectors >> conf->geo.chunk_shift;
	sector_div(size, conf->geo.far_copies);
3518
	size = size * raid_disks;
3519
	sector_div(size, conf->geo.near_copies);
3520

3521
	return size << conf->geo.chunk_shift;
3522 3523
}

3524 3525 3526 3527 3528 3529 3530
static void calc_sectors(struct r10conf *conf, sector_t size)
{
	/* Calculate the number of sectors-per-device that will
	 * actually be used, and set conf->dev_sectors and
	 * conf->stride
	 */

3531 3532 3533 3534
	size = size >> conf->geo.chunk_shift;
	sector_div(size, conf->geo.far_copies);
	size = size * conf->geo.raid_disks;
	sector_div(size, conf->geo.near_copies);
3535 3536 3537 3538 3539 3540 3541
	/* 'size' is now the number of chunks in the array */
	/* calculate "used chunks per device" */
	size = size * conf->copies;

	/* We need to round up when dividing by raid_disks to
	 * get the stride size.
	 */
3542
	size = DIV_ROUND_UP_SECTOR_T(size, conf->geo.raid_disks);
3543

3544
	conf->dev_sectors = size << conf->geo.chunk_shift;
3545

3546 3547
	if (conf->geo.far_offset)
		conf->geo.stride = 1 << conf->geo.chunk_shift;
3548
	else {
3549 3550
		sector_div(size, conf->geo.far_copies);
		conf->geo.stride = size << conf->geo.chunk_shift;
3551 3552
	}
}
3553

3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576 3577
enum geo_type {geo_new, geo_old, geo_start};
static int setup_geo(struct geom *geo, struct mddev *mddev, enum geo_type new)
{
	int nc, fc, fo;
	int layout, chunk, disks;
	switch (new) {
	case geo_old:
		layout = mddev->layout;
		chunk = mddev->chunk_sectors;
		disks = mddev->raid_disks - mddev->delta_disks;
		break;
	case geo_new:
		layout = mddev->new_layout;
		chunk = mddev->new_chunk_sectors;
		disks = mddev->raid_disks;
		break;
	default: /* avoid 'may be unused' warnings */
	case geo_start: /* new when starting reshape - raid_disks not
			 * updated yet. */
		layout = mddev->new_layout;
		chunk = mddev->new_chunk_sectors;
		disks = mddev->raid_disks + mddev->delta_disks;
		break;
	}
3578
	if (layout >> 19)
3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589
		return -1;
	if (chunk < (PAGE_SIZE >> 9) ||
	    !is_power_of_2(chunk))
		return -2;
	nc = layout & 255;
	fc = (layout >> 8) & 255;
	fo = layout & (1<<16);
	geo->raid_disks = disks;
	geo->near_copies = nc;
	geo->far_copies = fc;
	geo->far_offset = fo;
3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605
	switch (layout >> 17) {
	case 0:	/* original layout.  simple but not always optimal */
		geo->far_set_size = disks;
		break;
	case 1: /* "improved" layout which was buggy.  Hopefully no-one is
		 * actually using this, but leave code here just in case.*/
		geo->far_set_size = disks/fc;
		WARN(geo->far_set_size < fc,
		     "This RAID10 layout does not provide data safety - please backup and create new array\n");
		break;
	case 2: /* "improved" layout fixed to match documentation */
		geo->far_set_size = fc * nc;
		break;
	default: /* Not a valid layout */
		return -1;
	}
3606 3607 3608 3609 3610
	geo->chunk_mask = chunk - 1;
	geo->chunk_shift = ffz(~chunk);
	return nc*fc;
}

3611
static struct r10conf *setup_conf(struct mddev *mddev)
L
Linus Torvalds 已提交
3612
{
3613
	struct r10conf *conf = NULL;
3614
	int err = -EINVAL;
3615 3616 3617 3618
	struct geom geo;
	int copies;

	copies = setup_geo(&geo, mddev, geo_new);
L
Linus Torvalds 已提交
3619

3620
	if (copies == -2) {
N
NeilBrown 已提交
3621 3622
		pr_warn("md/raid10:%s: chunk size must be at least PAGE_SIZE(%ld) and be a power of 2.\n",
			mdname(mddev), PAGE_SIZE);
3623
		goto out;
L
Linus Torvalds 已提交
3624
	}
3625

3626
	if (copies < 2 || copies > mddev->raid_disks) {
N
NeilBrown 已提交
3627 3628
		pr_warn("md/raid10:%s: unsupported raid10 layout: 0x%8x\n",
			mdname(mddev), mddev->new_layout);
L
Linus Torvalds 已提交
3629 3630
		goto out;
	}
3631 3632

	err = -ENOMEM;
3633
	conf = kzalloc(sizeof(struct r10conf), GFP_KERNEL);
3634
	if (!conf)
L
Linus Torvalds 已提交
3635
		goto out;
3636

N
NeilBrown 已提交
3637
	/* FIXME calc properly */
3638
	conf->mirrors = kzalloc(sizeof(struct raid10_info)*(mddev->raid_disks +
3639
							    max(0,-mddev->delta_disks)),
3640 3641 3642
				GFP_KERNEL);
	if (!conf->mirrors)
		goto out;
3643 3644 3645

	conf->tmppage = alloc_page(GFP_KERNEL);
	if (!conf->tmppage)
3646 3647
		goto out;

3648 3649
	conf->geo = geo;
	conf->copies = copies;
3650 3651 3652 3653 3654
	conf->r10bio_pool = mempool_create(NR_RAID10_BIOS, r10bio_pool_alloc,
					   r10bio_pool_free, conf);
	if (!conf->r10bio_pool)
		goto out;

3655
	calc_sectors(conf, mddev->dev_sectors);
N
NeilBrown 已提交
3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670
	if (mddev->reshape_position == MaxSector) {
		conf->prev = conf->geo;
		conf->reshape_progress = MaxSector;
	} else {
		if (setup_geo(&conf->prev, mddev, geo_old) != conf->copies) {
			err = -EINVAL;
			goto out;
		}
		conf->reshape_progress = mddev->reshape_position;
		if (conf->prev.far_offset)
			conf->prev.stride = 1 << conf->prev.chunk_shift;
		else
			/* far_copies must be 1 */
			conf->prev.stride = conf->dev_sectors;
	}
3671
	conf->reshape_safe = conf->reshape_progress;
3672
	spin_lock_init(&conf->device_lock);
3673
	INIT_LIST_HEAD(&conf->retry_list);
3674
	INIT_LIST_HEAD(&conf->bio_end_io_list);
3675 3676 3677

	spin_lock_init(&conf->resync_lock);
	init_waitqueue_head(&conf->wait_barrier);
3678
	atomic_set(&conf->nr_pending, 0);
3679

3680
	conf->thread = md_register_thread(raid10d, mddev, "raid10");
3681 3682 3683 3684 3685 3686 3687 3688
	if (!conf->thread)
		goto out;

	conf->mddev = mddev;
	return conf;

 out:
	if (conf) {
3689
		mempool_destroy(conf->r10bio_pool);
3690 3691 3692 3693 3694 3695 3696
		kfree(conf->mirrors);
		safe_put_page(conf->tmppage);
		kfree(conf);
	}
	return ERR_PTR(err);
}

S
Shaohua Li 已提交
3697
static int raid10_run(struct mddev *mddev)
3698
{
3699
	struct r10conf *conf;
3700
	int i, disk_idx, chunk_size;
3701
	struct raid10_info *disk;
3702
	struct md_rdev *rdev;
3703
	sector_t size;
N
NeilBrown 已提交
3704 3705
	sector_t min_offset_diff = 0;
	int first = 1;
S
Shaohua Li 已提交
3706
	bool discard_supported = false;
3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720

	if (mddev->private == NULL) {
		conf = setup_conf(mddev);
		if (IS_ERR(conf))
			return PTR_ERR(conf);
		mddev->private = conf;
	}
	conf = mddev->private;
	if (!conf)
		goto out;

	mddev->thread = conf->thread;
	conf->thread = NULL;

3721
	chunk_size = mddev->chunk_sectors << 9;
3722
	if (mddev->queue) {
S
Shaohua Li 已提交
3723 3724
		blk_queue_max_discard_sectors(mddev->queue,
					      mddev->chunk_sectors);
3725
		blk_queue_max_write_same_sectors(mddev->queue, 0);
3726 3727 3728 3729 3730 3731 3732
		blk_queue_io_min(mddev->queue, chunk_size);
		if (conf->geo.raid_disks % conf->geo.near_copies)
			blk_queue_io_opt(mddev->queue, chunk_size * conf->geo.raid_disks);
		else
			blk_queue_io_opt(mddev->queue, chunk_size *
					 (conf->geo.raid_disks / conf->geo.near_copies));
	}
3733

N
NeilBrown 已提交
3734
	rdev_for_each(rdev, mddev) {
N
NeilBrown 已提交
3735
		long long diff;
3736
		struct request_queue *q;
3737

L
Linus Torvalds 已提交
3738
		disk_idx = rdev->raid_disk;
3739 3740 3741 3742
		if (disk_idx < 0)
			continue;
		if (disk_idx >= conf->geo.raid_disks &&
		    disk_idx >= conf->prev.raid_disks)
L
Linus Torvalds 已提交
3743 3744 3745
			continue;
		disk = conf->mirrors + disk_idx;

3746 3747 3748 3749 3750 3751 3752 3753 3754
		if (test_bit(Replacement, &rdev->flags)) {
			if (disk->replacement)
				goto out_free_conf;
			disk->replacement = rdev;
		} else {
			if (disk->rdev)
				goto out_free_conf;
			disk->rdev = rdev;
		}
3755
		q = bdev_get_queue(rdev->bdev);
N
NeilBrown 已提交
3756 3757 3758 3759 3760 3761 3762
		diff = (rdev->new_data_offset - rdev->data_offset);
		if (!mddev->reshape_backwards)
			diff = -diff;
		if (diff < 0)
			diff = 0;
		if (first || diff < min_offset_diff)
			min_offset_diff = diff;
3763

3764 3765 3766
		if (mddev->gendisk)
			disk_stack_limits(mddev->gendisk, rdev->bdev,
					  rdev->data_offset << 9);
L
Linus Torvalds 已提交
3767 3768

		disk->head_position = 0;
S
Shaohua Li 已提交
3769 3770 3771

		if (blk_queue_discard(bdev_get_queue(rdev->bdev)))
			discard_supported = true;
L
Linus Torvalds 已提交
3772
	}
N
NeilBrown 已提交
3773

3774 3775 3776 3777 3778 3779 3780 3781
	if (mddev->queue) {
		if (discard_supported)
			queue_flag_set_unlocked(QUEUE_FLAG_DISCARD,
						mddev->queue);
		else
			queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD,
						  mddev->queue);
	}
3782
	/* need to check that every block has at least one working mirror */
3783
	if (!enough(conf, -1)) {
N
NeilBrown 已提交
3784
		pr_err("md/raid10:%s: not enough operational mirrors.\n",
3785
		       mdname(mddev));
L
Linus Torvalds 已提交
3786 3787 3788
		goto out_free_conf;
	}

N
NeilBrown 已提交
3789 3790 3791 3792 3793 3794
	if (conf->reshape_progress != MaxSector) {
		/* must ensure that shape change is supported */
		if (conf->geo.far_copies != 1 &&
		    conf->geo.far_offset == 0)
			goto out_free_conf;
		if (conf->prev.far_copies != 1 &&
3795
		    conf->prev.far_offset == 0)
N
NeilBrown 已提交
3796 3797 3798
			goto out_free_conf;
	}

L
Linus Torvalds 已提交
3799
	mddev->degraded = 0;
3800 3801 3802 3803
	for (i = 0;
	     i < conf->geo.raid_disks
		     || i < conf->prev.raid_disks;
	     i++) {
L
Linus Torvalds 已提交
3804 3805 3806

		disk = conf->mirrors + i;

3807 3808 3809 3810 3811 3812 3813
		if (!disk->rdev && disk->replacement) {
			/* The replacement is all we have - use it */
			disk->rdev = disk->replacement;
			disk->replacement = NULL;
			clear_bit(Replacement, &disk->rdev->flags);
		}

3814
		if (!disk->rdev ||
3815
		    !test_bit(In_sync, &disk->rdev->flags)) {
L
Linus Torvalds 已提交
3816 3817
			disk->head_position = 0;
			mddev->degraded++;
3818 3819
			if (disk->rdev &&
			    disk->rdev->saved_raid_disk < 0)
3820
				conf->fullsync = 1;
L
Linus Torvalds 已提交
3821
		}
3822
		disk->recovery_disabled = mddev->recovery_disabled - 1;
L
Linus Torvalds 已提交
3823 3824
	}

3825
	if (mddev->recovery_cp != MaxSector)
N
NeilBrown 已提交
3826 3827 3828
		pr_notice("md/raid10:%s: not clean -- starting background reconstruction\n",
			  mdname(mddev));
	pr_info("md/raid10:%s: active with %d out of %d devices\n",
3829 3830
		mdname(mddev), conf->geo.raid_disks - mddev->degraded,
		conf->geo.raid_disks);
L
Linus Torvalds 已提交
3831 3832 3833
	/*
	 * Ok, everything is just fine now
	 */
3834 3835 3836 3837
	mddev->dev_sectors = conf->dev_sectors;
	size = raid10_size(mddev, 0, 0);
	md_set_array_sectors(mddev, size);
	mddev->resync_max_sectors = size;
3838
	set_bit(MD_FAILFAST_SUPPORTED, &mddev->flags);
L
Linus Torvalds 已提交
3839

3840
	if (mddev->queue) {
3841
		int stripe = conf->geo.raid_disks *
3842
			((mddev->chunk_sectors << 9) / PAGE_SIZE);
3843 3844 3845 3846 3847

		/* Calculate max read-ahead size.
		 * We need to readahead at least twice a whole stripe....
		 * maybe...
		 */
3848
		stripe /= conf->geo.near_copies;
3849 3850
		if (mddev->queue->backing_dev_info->ra_pages < 2 * stripe)
			mddev->queue->backing_dev_info->ra_pages = 2 * stripe;
L
Linus Torvalds 已提交
3851 3852
	}

3853 3854 3855
	if (md_integrity_register(mddev))
		goto out_free_conf;

N
NeilBrown 已提交
3856 3857 3858 3859 3860 3861 3862 3863 3864 3865
	if (conf->reshape_progress != MaxSector) {
		unsigned long before_length, after_length;

		before_length = ((1 << conf->prev.chunk_shift) *
				 conf->prev.far_copies);
		after_length = ((1 << conf->geo.chunk_shift) *
				conf->geo.far_copies);

		if (max(before_length, after_length) > min_offset_diff) {
			/* This cannot work */
N
NeilBrown 已提交
3866
			pr_warn("md/raid10: offset difference not enough to continue reshape\n");
N
NeilBrown 已提交
3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878
			goto out_free_conf;
		}
		conf->offset_diff = min_offset_diff;

		clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
		clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
		set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
		set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
		mddev->sync_thread = md_register_thread(md_do_sync, mddev,
							"reshape");
	}

L
Linus Torvalds 已提交
3879 3880 3881
	return 0;

out_free_conf:
3882
	md_unregister_thread(&mddev->thread);
3883
	mempool_destroy(conf->r10bio_pool);
3884
	safe_put_page(conf->tmppage);
3885
	kfree(conf->mirrors);
L
Linus Torvalds 已提交
3886 3887 3888 3889 3890 3891
	kfree(conf);
	mddev->private = NULL;
out:
	return -EIO;
}

N
NeilBrown 已提交
3892
static void raid10_free(struct mddev *mddev, void *priv)
L
Linus Torvalds 已提交
3893
{
N
NeilBrown 已提交
3894
	struct r10conf *conf = priv;
L
Linus Torvalds 已提交
3895

3896
	mempool_destroy(conf->r10bio_pool);
3897
	safe_put_page(conf->tmppage);
3898
	kfree(conf->mirrors);
3899 3900
	kfree(conf->mirrors_old);
	kfree(conf->mirrors_new);
L
Linus Torvalds 已提交
3901 3902 3903
	kfree(conf);
}

3904
static void raid10_quiesce(struct mddev *mddev, int state)
3905
{
3906
	struct r10conf *conf = mddev->private;
3907 3908 3909 3910 3911 3912 3913 3914 3915 3916

	switch(state) {
	case 1:
		raise_barrier(conf, 0);
		break;
	case 0:
		lower_barrier(conf);
		break;
	}
}
L
Linus Torvalds 已提交
3917

3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934
static int raid10_resize(struct mddev *mddev, sector_t sectors)
{
	/* Resize of 'far' arrays is not supported.
	 * For 'near' and 'offset' arrays we can set the
	 * number of sectors used to be an appropriate multiple
	 * of the chunk size.
	 * For 'offset', this is far_copies*chunksize.
	 * For 'near' the multiplier is the LCM of
	 * near_copies and raid_disks.
	 * So if far_copies > 1 && !far_offset, fail.
	 * Else find LCM(raid_disks, near_copy)*far_copies and
	 * multiply by chunk_size.  Then round to this number.
	 * This is mostly done by raid10_size()
	 */
	struct r10conf *conf = mddev->private;
	sector_t oldsize, size;

3935 3936 3937
	if (mddev->reshape_position != MaxSector)
		return -EBUSY;

3938
	if (conf->geo.far_copies > 1 && !conf->geo.far_offset)
3939 3940 3941 3942
		return -EINVAL;

	oldsize = raid10_size(mddev, 0, 0);
	size = raid10_size(mddev, sectors, 0);
3943 3944
	if (mddev->external_size &&
	    mddev->array_sectors > size)
3945
		return -EINVAL;
3946 3947 3948 3949 3950 3951
	if (mddev->bitmap) {
		int ret = bitmap_resize(mddev->bitmap, size, 0, 0);
		if (ret)
			return ret;
	}
	md_set_array_sectors(mddev, size);
3952 3953 3954 3955 3956
	if (sectors > mddev->dev_sectors &&
	    mddev->recovery_cp > oldsize) {
		mddev->recovery_cp = oldsize;
		set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
	}
3957 3958
	calc_sectors(conf, sectors);
	mddev->dev_sectors = conf->dev_sectors;
3959 3960 3961 3962
	mddev->resync_max_sectors = size;
	return 0;
}

3963
static void *raid10_takeover_raid0(struct mddev *mddev, sector_t size, int devs)
3964
{
3965
	struct md_rdev *rdev;
3966
	struct r10conf *conf;
3967 3968

	if (mddev->degraded > 0) {
N
NeilBrown 已提交
3969 3970
		pr_warn("md/raid10:%s: Error: degraded raid0!\n",
			mdname(mddev));
3971 3972
		return ERR_PTR(-EINVAL);
	}
3973
	sector_div(size, devs);
3974 3975 3976 3977 3978 3979 3980 3981 3982 3983

	/* Set new parameters */
	mddev->new_level = 10;
	/* new layout: far_copies = 1, near_copies = 2 */
	mddev->new_layout = (1<<8) + 2;
	mddev->new_chunk_sectors = mddev->chunk_sectors;
	mddev->delta_disks = mddev->raid_disks;
	mddev->raid_disks *= 2;
	/* make sure it will be not marked as dirty */
	mddev->recovery_cp = MaxSector;
3984
	mddev->dev_sectors = size;
3985 3986

	conf = setup_conf(mddev);
3987
	if (!IS_ERR(conf)) {
N
NeilBrown 已提交
3988
		rdev_for_each(rdev, mddev)
3989
			if (rdev->raid_disk >= 0) {
3990
				rdev->new_raid_disk = rdev->raid_disk * 2;
3991 3992
				rdev->sectors = size;
			}
3993 3994 3995
		conf->barrier = 1;
	}

3996 3997 3998
	return conf;
}

3999
static void *raid10_takeover(struct mddev *mddev)
4000
{
4001
	struct r0conf *raid0_conf;
4002 4003 4004 4005 4006 4007

	/* raid10 can take over:
	 *  raid0 - providing it has only two drives
	 */
	if (mddev->level == 0) {
		/* for raid0 takeover only one zone is supported */
4008 4009
		raid0_conf = mddev->private;
		if (raid0_conf->nr_strip_zones > 1) {
N
NeilBrown 已提交
4010 4011
			pr_warn("md/raid10:%s: cannot takeover raid 0 with more than one zone.\n",
				mdname(mddev));
4012 4013
			return ERR_PTR(-EINVAL);
		}
4014 4015 4016
		return raid10_takeover_raid0(mddev,
			raid0_conf->strip_zone->zone_end,
			raid0_conf->strip_zone->nb_dev);
4017 4018 4019 4020
	}
	return ERR_PTR(-EINVAL);
}

N
NeilBrown 已提交
4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061
static int raid10_check_reshape(struct mddev *mddev)
{
	/* Called when there is a request to change
	 * - layout (to ->new_layout)
	 * - chunk size (to ->new_chunk_sectors)
	 * - raid_disks (by delta_disks)
	 * or when trying to restart a reshape that was ongoing.
	 *
	 * We need to validate the request and possibly allocate
	 * space if that might be an issue later.
	 *
	 * Currently we reject any reshape of a 'far' mode array,
	 * allow chunk size to change if new is generally acceptable,
	 * allow raid_disks to increase, and allow
	 * a switch between 'near' mode and 'offset' mode.
	 */
	struct r10conf *conf = mddev->private;
	struct geom geo;

	if (conf->geo.far_copies != 1 && !conf->geo.far_offset)
		return -EINVAL;

	if (setup_geo(&geo, mddev, geo_start) != conf->copies)
		/* mustn't change number of copies */
		return -EINVAL;
	if (geo.far_copies > 1 && !geo.far_offset)
		/* Cannot switch to 'far' mode */
		return -EINVAL;

	if (mddev->array_sectors & geo.chunk_mask)
			/* not factor of array size */
			return -EINVAL;

	if (!enough(conf, -1))
		return -EINVAL;

	kfree(conf->mirrors_new);
	conf->mirrors_new = NULL;
	if (mddev->delta_disks > 0) {
		/* allocate new 'mirrors' list */
		conf->mirrors_new = kzalloc(
4062
			sizeof(struct raid10_info)
N
NeilBrown 已提交
4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147
			*(mddev->raid_disks +
			  mddev->delta_disks),
			GFP_KERNEL);
		if (!conf->mirrors_new)
			return -ENOMEM;
	}
	return 0;
}

/*
 * Need to check if array has failed when deciding whether to:
 *  - start an array
 *  - remove non-faulty devices
 *  - add a spare
 *  - allow a reshape
 * This determination is simple when no reshape is happening.
 * However if there is a reshape, we need to carefully check
 * both the before and after sections.
 * This is because some failed devices may only affect one
 * of the two sections, and some non-in_sync devices may
 * be insync in the section most affected by failed devices.
 */
static int calc_degraded(struct r10conf *conf)
{
	int degraded, degraded2;
	int i;

	rcu_read_lock();
	degraded = 0;
	/* 'prev' section first */
	for (i = 0; i < conf->prev.raid_disks; i++) {
		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
		if (!rdev || test_bit(Faulty, &rdev->flags))
			degraded++;
		else if (!test_bit(In_sync, &rdev->flags))
			/* When we can reduce the number of devices in
			 * an array, this might not contribute to
			 * 'degraded'.  It does now.
			 */
			degraded++;
	}
	rcu_read_unlock();
	if (conf->geo.raid_disks == conf->prev.raid_disks)
		return degraded;
	rcu_read_lock();
	degraded2 = 0;
	for (i = 0; i < conf->geo.raid_disks; i++) {
		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
		if (!rdev || test_bit(Faulty, &rdev->flags))
			degraded2++;
		else if (!test_bit(In_sync, &rdev->flags)) {
			/* If reshape is increasing the number of devices,
			 * this section has already been recovered, so
			 * it doesn't contribute to degraded.
			 * else it does.
			 */
			if (conf->geo.raid_disks <= conf->prev.raid_disks)
				degraded2++;
		}
	}
	rcu_read_unlock();
	if (degraded2 > degraded)
		return degraded2;
	return degraded;
}

static int raid10_start_reshape(struct mddev *mddev)
{
	/* A 'reshape' has been requested. This commits
	 * the various 'new' fields and sets MD_RECOVER_RESHAPE
	 * This also checks if there are enough spares and adds them
	 * to the array.
	 * We currently require enough spares to make the final
	 * array non-degraded.  We also require that the difference
	 * between old and new data_offset - on each device - is
	 * enough that we never risk over-writing.
	 */

	unsigned long before_length, after_length;
	sector_t min_offset_diff = 0;
	int first = 1;
	struct geom new;
	struct r10conf *conf = mddev->private;
	struct md_rdev *rdev;
	int spares = 0;
4148
	int ret;
N
NeilBrown 已提交
4149 4150 4151 4152 4153 4154 4155 4156 4157 4158 4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186

	if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
		return -EBUSY;

	if (setup_geo(&new, mddev, geo_start) != conf->copies)
		return -EINVAL;

	before_length = ((1 << conf->prev.chunk_shift) *
			 conf->prev.far_copies);
	after_length = ((1 << conf->geo.chunk_shift) *
			conf->geo.far_copies);

	rdev_for_each(rdev, mddev) {
		if (!test_bit(In_sync, &rdev->flags)
		    && !test_bit(Faulty, &rdev->flags))
			spares++;
		if (rdev->raid_disk >= 0) {
			long long diff = (rdev->new_data_offset
					  - rdev->data_offset);
			if (!mddev->reshape_backwards)
				diff = -diff;
			if (diff < 0)
				diff = 0;
			if (first || diff < min_offset_diff)
				min_offset_diff = diff;
		}
	}

	if (max(before_length, after_length) > min_offset_diff)
		return -EINVAL;

	if (spares < mddev->delta_disks)
		return -EINVAL;

	conf->offset_diff = min_offset_diff;
	spin_lock_irq(&conf->device_lock);
	if (conf->mirrors_new) {
		memcpy(conf->mirrors_new, conf->mirrors,
4187
		       sizeof(struct raid10_info)*conf->prev.raid_disks);
N
NeilBrown 已提交
4188
		smp_mb();
4189
		kfree(conf->mirrors_old);
N
NeilBrown 已提交
4190 4191 4192 4193 4194 4195 4196 4197 4198 4199
		conf->mirrors_old = conf->mirrors;
		conf->mirrors = conf->mirrors_new;
		conf->mirrors_new = NULL;
	}
	setup_geo(&conf->geo, mddev, geo_start);
	smp_mb();
	if (mddev->reshape_backwards) {
		sector_t size = raid10_size(mddev, 0, 0);
		if (size < mddev->array_sectors) {
			spin_unlock_irq(&conf->device_lock);
N
NeilBrown 已提交
4200 4201
			pr_warn("md/raid10:%s: array size must be reduce before number of disks\n",
				mdname(mddev));
N
NeilBrown 已提交
4202 4203 4204 4205 4206 4207
			return -EINVAL;
		}
		mddev->resync_max_sectors = size;
		conf->reshape_progress = size;
	} else
		conf->reshape_progress = 0;
4208
	conf->reshape_safe = conf->reshape_progress;
N
NeilBrown 已提交
4209 4210
	spin_unlock_irq(&conf->device_lock);

4211 4212 4213 4214 4215 4216 4217 4218
	if (mddev->delta_disks && mddev->bitmap) {
		ret = bitmap_resize(mddev->bitmap,
				    raid10_size(mddev, 0,
						conf->geo.raid_disks),
				    0, 0);
		if (ret)
			goto abort;
	}
N
NeilBrown 已提交
4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247
	if (mddev->delta_disks > 0) {
		rdev_for_each(rdev, mddev)
			if (rdev->raid_disk < 0 &&
			    !test_bit(Faulty, &rdev->flags)) {
				if (raid10_add_disk(mddev, rdev) == 0) {
					if (rdev->raid_disk >=
					    conf->prev.raid_disks)
						set_bit(In_sync, &rdev->flags);
					else
						rdev->recovery_offset = 0;

					if (sysfs_link_rdev(mddev, rdev))
						/* Failure here  is OK */;
				}
			} else if (rdev->raid_disk >= conf->prev.raid_disks
				   && !test_bit(Faulty, &rdev->flags)) {
				/* This is a spare that was manually added */
				set_bit(In_sync, &rdev->flags);
			}
	}
	/* When a reshape changes the number of devices,
	 * ->degraded is measured against the larger of the
	 * pre and  post numbers.
	 */
	spin_lock_irq(&conf->device_lock);
	mddev->degraded = calc_degraded(conf);
	spin_unlock_irq(&conf->device_lock);
	mddev->raid_disks = conf->geo.raid_disks;
	mddev->reshape_position = conf->reshape_progress;
4248
	set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
N
NeilBrown 已提交
4249 4250 4251

	clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
	clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
4252
	clear_bit(MD_RECOVERY_DONE, &mddev->recovery);
N
NeilBrown 已提交
4253 4254 4255 4256 4257 4258
	set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
	set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);

	mddev->sync_thread = md_register_thread(md_do_sync, mddev,
						"reshape");
	if (!mddev->sync_thread) {
4259 4260
		ret = -EAGAIN;
		goto abort;
N
NeilBrown 已提交
4261 4262 4263 4264 4265
	}
	conf->reshape_checkpoint = jiffies;
	md_wakeup_thread(mddev->sync_thread);
	md_new_event(mddev);
	return 0;
4266 4267 4268 4269 4270 4271 4272 4273 4274 4275

abort:
	mddev->recovery = 0;
	spin_lock_irq(&conf->device_lock);
	conf->geo = conf->prev;
	mddev->raid_disks = conf->geo.raid_disks;
	rdev_for_each(rdev, mddev)
		rdev->new_data_offset = rdev->data_offset;
	smp_wmb();
	conf->reshape_progress = MaxSector;
4276
	conf->reshape_safe = MaxSector;
4277 4278 4279
	mddev->reshape_position = MaxSector;
	spin_unlock_irq(&conf->device_lock);
	return ret;
N
NeilBrown 已提交
4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319
}

/* Calculate the last device-address that could contain
 * any block from the chunk that includes the array-address 's'
 * and report the next address.
 * i.e. the address returned will be chunk-aligned and after
 * any data that is in the chunk containing 's'.
 */
static sector_t last_dev_address(sector_t s, struct geom *geo)
{
	s = (s | geo->chunk_mask) + 1;
	s >>= geo->chunk_shift;
	s *= geo->near_copies;
	s = DIV_ROUND_UP_SECTOR_T(s, geo->raid_disks);
	s *= geo->far_copies;
	s <<= geo->chunk_shift;
	return s;
}

/* Calculate the first device-address that could contain
 * any block from the chunk that includes the array-address 's'.
 * This too will be the start of a chunk
 */
static sector_t first_dev_address(sector_t s, struct geom *geo)
{
	s >>= geo->chunk_shift;
	s *= geo->near_copies;
	sector_div(s, geo->raid_disks);
	s *= geo->far_copies;
	s <<= geo->chunk_shift;
	return s;
}

static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
				int *skipped)
{
	/* We simply copy at most one chunk (smallest of old and new)
	 * at a time, possibly less if that exceeds RESYNC_PAGES,
	 * or we hit a bad block or something.
	 * This might mean we pause for normal IO in the middle of
4320
	 * a chunk, but that is not a problem as mddev->reshape_position
N
NeilBrown 已提交
4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343
	 * can record any location.
	 *
	 * If we will want to write to a location that isn't
	 * yet recorded as 'safe' (i.e. in metadata on disk) then
	 * we need to flush all reshape requests and update the metadata.
	 *
	 * When reshaping forwards (e.g. to more devices), we interpret
	 * 'safe' as the earliest block which might not have been copied
	 * down yet.  We divide this by previous stripe size and multiply
	 * by previous stripe length to get lowest device offset that we
	 * cannot write to yet.
	 * We interpret 'sector_nr' as an address that we want to write to.
	 * From this we use last_device_address() to find where we might
	 * write to, and first_device_address on the  'safe' position.
	 * If this 'next' write position is after the 'safe' position,
	 * we must update the metadata to increase the 'safe' position.
	 *
	 * When reshaping backwards, we round in the opposite direction
	 * and perform the reverse test:  next write position must not be
	 * less than current safe position.
	 *
	 * In all this the minimum difference in data offsets
	 * (conf->offset_diff - always positive) allows a bit of slack,
4344
	 * so next can be after 'safe', but not by more than offset_diff
N
NeilBrown 已提交
4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363
	 *
	 * We need to prepare all the bios here before we start any IO
	 * to ensure the size we choose is acceptable to all devices.
	 * The means one for each copy for write-out and an extra one for
	 * read-in.
	 * We store the read-in bio in ->master_bio and the others in
	 * ->devs[x].bio and ->devs[x].repl_bio.
	 */
	struct r10conf *conf = mddev->private;
	struct r10bio *r10_bio;
	sector_t next, safe, last;
	int max_sectors;
	int nr_sectors;
	int s;
	struct md_rdev *rdev;
	int need_flush = 0;
	struct bio *blist;
	struct bio *bio, *read_bio;
	int sectors_done = 0;
4364
	struct page **pages;
N
NeilBrown 已提交
4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443

	if (sector_nr == 0) {
		/* If restarting in the middle, skip the initial sectors */
		if (mddev->reshape_backwards &&
		    conf->reshape_progress < raid10_size(mddev, 0, 0)) {
			sector_nr = (raid10_size(mddev, 0, 0)
				     - conf->reshape_progress);
		} else if (!mddev->reshape_backwards &&
			   conf->reshape_progress > 0)
			sector_nr = conf->reshape_progress;
		if (sector_nr) {
			mddev->curr_resync_completed = sector_nr;
			sysfs_notify(&mddev->kobj, NULL, "sync_completed");
			*skipped = 1;
			return sector_nr;
		}
	}

	/* We don't use sector_nr to track where we are up to
	 * as that doesn't work well for ->reshape_backwards.
	 * So just use ->reshape_progress.
	 */
	if (mddev->reshape_backwards) {
		/* 'next' is the earliest device address that we might
		 * write to for this chunk in the new layout
		 */
		next = first_dev_address(conf->reshape_progress - 1,
					 &conf->geo);

		/* 'safe' is the last device address that we might read from
		 * in the old layout after a restart
		 */
		safe = last_dev_address(conf->reshape_safe - 1,
					&conf->prev);

		if (next + conf->offset_diff < safe)
			need_flush = 1;

		last = conf->reshape_progress - 1;
		sector_nr = last & ~(sector_t)(conf->geo.chunk_mask
					       & conf->prev.chunk_mask);
		if (sector_nr + RESYNC_BLOCK_SIZE/512 < last)
			sector_nr = last + 1 - RESYNC_BLOCK_SIZE/512;
	} else {
		/* 'next' is after the last device address that we
		 * might write to for this chunk in the new layout
		 */
		next = last_dev_address(conf->reshape_progress, &conf->geo);

		/* 'safe' is the earliest device address that we might
		 * read from in the old layout after a restart
		 */
		safe = first_dev_address(conf->reshape_safe, &conf->prev);

		/* Need to update metadata if 'next' might be beyond 'safe'
		 * as that would possibly corrupt data
		 */
		if (next > safe + conf->offset_diff)
			need_flush = 1;

		sector_nr = conf->reshape_progress;
		last  = sector_nr | (conf->geo.chunk_mask
				     & conf->prev.chunk_mask);

		if (sector_nr + RESYNC_BLOCK_SIZE/512 <= last)
			last = sector_nr + RESYNC_BLOCK_SIZE/512 - 1;
	}

	if (need_flush ||
	    time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
		/* Need to update reshape_position in metadata */
		wait_barrier(conf);
		mddev->reshape_position = conf->reshape_progress;
		if (mddev->reshape_backwards)
			mddev->curr_resync_completed = raid10_size(mddev, 0, 0)
				- conf->reshape_progress;
		else
			mddev->curr_resync_completed = conf->reshape_progress;
		conf->reshape_checkpoint = jiffies;
4444
		set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
N
NeilBrown 已提交
4445
		md_wakeup_thread(mddev->thread);
4446
		wait_event(mddev->sb_wait, mddev->sb_flags == 0 ||
4447 4448 4449 4450 4451
			   test_bit(MD_RECOVERY_INTR, &mddev->recovery));
		if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
			allow_barrier(conf);
			return sectors_done;
		}
N
NeilBrown 已提交
4452 4453 4454 4455 4456 4457 4458
		conf->reshape_safe = mddev->reshape_position;
		allow_barrier(conf);
	}

read_more:
	/* Now schedule reads for blocks from sector_nr to last */
	r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
4459
	r10_bio->state = 0;
N
NeilBrown 已提交
4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473
	raise_barrier(conf, sectors_done != 0);
	atomic_set(&r10_bio->remaining, 0);
	r10_bio->mddev = mddev;
	r10_bio->sector = sector_nr;
	set_bit(R10BIO_IsReshape, &r10_bio->state);
	r10_bio->sectors = last - sector_nr + 1;
	rdev = read_balance(conf, r10_bio, &max_sectors);
	BUG_ON(!test_bit(R10BIO_Previous, &r10_bio->state));

	if (!rdev) {
		/* Cannot read from here, so need to record bad blocks
		 * on all the target devices.
		 */
		// FIXME
4474
		mempool_free(r10_bio, conf->r10buf_pool);
N
NeilBrown 已提交
4475 4476 4477 4478 4479 4480 4481
		set_bit(MD_RECOVERY_INTR, &mddev->recovery);
		return sectors_done;
	}

	read_bio = bio_alloc_mddev(GFP_KERNEL, RESYNC_PAGES, mddev);

	read_bio->bi_bdev = rdev->bdev;
4482
	read_bio->bi_iter.bi_sector = (r10_bio->devs[r10_bio->read_slot].addr
N
NeilBrown 已提交
4483 4484
			       + rdev->data_offset);
	read_bio->bi_private = r10_bio;
4485
	read_bio->bi_end_io = end_reshape_read;
M
Mike Christie 已提交
4486
	bio_set_op_attrs(read_bio, REQ_OP_READ, 0);
4487
	read_bio->bi_flags &= (~0UL << BIO_RESET_BITS);
4488
	read_bio->bi_error = 0;
N
NeilBrown 已提交
4489
	read_bio->bi_vcnt = 0;
4490
	read_bio->bi_iter.bi_size = 0;
N
NeilBrown 已提交
4491 4492 4493 4494 4495 4496 4497 4498 4499
	r10_bio->master_bio = read_bio;
	r10_bio->read_slot = r10_bio->devs[r10_bio->read_slot].devnum;

	/* Now find the locations in the new layout */
	__raid10_find_phys(&conf->geo, r10_bio);

	blist = read_bio;
	read_bio->bi_next = NULL;

4500
	rcu_read_lock();
N
NeilBrown 已提交
4501 4502 4503 4504 4505
	for (s = 0; s < conf->copies*2; s++) {
		struct bio *b;
		int d = r10_bio->devs[s/2].devnum;
		struct md_rdev *rdev2;
		if (s&1) {
4506
			rdev2 = rcu_dereference(conf->mirrors[d].replacement);
N
NeilBrown 已提交
4507 4508
			b = r10_bio->devs[s/2].repl_bio;
		} else {
4509
			rdev2 = rcu_dereference(conf->mirrors[d].rdev);
N
NeilBrown 已提交
4510 4511 4512 4513
			b = r10_bio->devs[s/2].bio;
		}
		if (!rdev2 || test_bit(Faulty, &rdev2->flags))
			continue;
K
Kent Overstreet 已提交
4514

N
NeilBrown 已提交
4515
		b->bi_bdev = rdev2->bdev;
4516 4517
		b->bi_iter.bi_sector = r10_bio->devs[s/2].addr +
			rdev2->new_data_offset;
N
NeilBrown 已提交
4518
		b->bi_end_io = end_reshape_write;
M
Mike Christie 已提交
4519
		bio_set_op_attrs(b, REQ_OP_WRITE, 0);
N
NeilBrown 已提交
4520 4521 4522 4523 4524 4525 4526
		b->bi_next = blist;
		blist = b;
	}

	/* Now add as many pages as possible to all of these bios. */

	nr_sectors = 0;
4527
	pages = get_resync_pages(r10_bio->devs[0].bio)->pages;
N
NeilBrown 已提交
4528
	for (s = 0 ; s < max_sectors; s += PAGE_SIZE >> 9) {
4529
		struct page *page = pages[s / (PAGE_SIZE >> 9)];
N
NeilBrown 已提交
4530 4531 4532 4533
		int len = (max_sectors - s) << 9;
		if (len > PAGE_SIZE)
			len = PAGE_SIZE;
		for (bio = blist; bio ; bio = bio->bi_next) {
4534 4535 4536 4537 4538
			/*
			 * won't fail because the vec table is big enough
			 * to hold all these pages
			 */
			bio_add_page(bio, page, len, 0);
N
NeilBrown 已提交
4539 4540 4541 4542
		}
		sector_nr += len >> 9;
		nr_sectors += len >> 9;
	}
4543
	rcu_read_unlock();
N
NeilBrown 已提交
4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561 4562 4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594
	r10_bio->sectors = nr_sectors;

	/* Now submit the read */
	md_sync_acct(read_bio->bi_bdev, r10_bio->sectors);
	atomic_inc(&r10_bio->remaining);
	read_bio->bi_next = NULL;
	generic_make_request(read_bio);
	sector_nr += nr_sectors;
	sectors_done += nr_sectors;
	if (sector_nr <= last)
		goto read_more;

	/* Now that we have done the whole section we can
	 * update reshape_progress
	 */
	if (mddev->reshape_backwards)
		conf->reshape_progress -= sectors_done;
	else
		conf->reshape_progress += sectors_done;

	return sectors_done;
}

static void end_reshape_request(struct r10bio *r10_bio);
static int handle_reshape_read_error(struct mddev *mddev,
				     struct r10bio *r10_bio);
static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio)
{
	/* Reshape read completed.  Hopefully we have a block
	 * to write out.
	 * If we got a read error then we do sync 1-page reads from
	 * elsewhere until we find the data - or give up.
	 */
	struct r10conf *conf = mddev->private;
	int s;

	if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
		if (handle_reshape_read_error(mddev, r10_bio) < 0) {
			/* Reshape has been aborted */
			md_done_sync(mddev, r10_bio->sectors, 0);
			return;
		}

	/* We definitely have the data in the pages, schedule the
	 * writes.
	 */
	atomic_set(&r10_bio->remaining, 1);
	for (s = 0; s < conf->copies*2; s++) {
		struct bio *b;
		int d = r10_bio->devs[s/2].devnum;
		struct md_rdev *rdev;
4595
		rcu_read_lock();
N
NeilBrown 已提交
4596
		if (s&1) {
4597
			rdev = rcu_dereference(conf->mirrors[d].replacement);
N
NeilBrown 已提交
4598 4599
			b = r10_bio->devs[s/2].repl_bio;
		} else {
4600
			rdev = rcu_dereference(conf->mirrors[d].rdev);
N
NeilBrown 已提交
4601 4602
			b = r10_bio->devs[s/2].bio;
		}
4603 4604
		if (!rdev || test_bit(Faulty, &rdev->flags)) {
			rcu_read_unlock();
N
NeilBrown 已提交
4605
			continue;
4606
		}
N
NeilBrown 已提交
4607
		atomic_inc(&rdev->nr_pending);
4608
		rcu_read_unlock();
N
NeilBrown 已提交
4609 4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623 4624 4625 4626
		md_sync_acct(b->bi_bdev, r10_bio->sectors);
		atomic_inc(&r10_bio->remaining);
		b->bi_next = NULL;
		generic_make_request(b);
	}
	end_reshape_request(r10_bio);
}

static void end_reshape(struct r10conf *conf)
{
	if (test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery))
		return;

	spin_lock_irq(&conf->device_lock);
	conf->prev = conf->geo;
	md_finish_reshape(conf->mddev);
	smp_wmb();
	conf->reshape_progress = MaxSector;
4627
	conf->reshape_safe = MaxSector;
N
NeilBrown 已提交
4628 4629 4630 4631 4632 4633 4634 4635 4636
	spin_unlock_irq(&conf->device_lock);

	/* read-ahead size must cover two whole stripes, which is
	 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices
	 */
	if (conf->mddev->queue) {
		int stripe = conf->geo.raid_disks *
			((conf->mddev->chunk_sectors << 9) / PAGE_SIZE);
		stripe /= conf->geo.near_copies;
4637 4638
		if (conf->mddev->queue->backing_dev_info->ra_pages < 2 * stripe)
			conf->mddev->queue->backing_dev_info->ra_pages = 2 * stripe;
N
NeilBrown 已提交
4639 4640 4641 4642 4643 4644 4645 4646 4647 4648
	}
	conf->fullsync = 0;
}

static int handle_reshape_read_error(struct mddev *mddev,
				     struct r10bio *r10_bio)
{
	/* Use sync reads to get the blocks from somewhere else */
	int sectors = r10_bio->sectors;
	struct r10conf *conf = mddev->private;
4649 4650 4651 4652 4653
	struct {
		struct r10bio r10_bio;
		struct r10dev devs[conf->copies];
	} on_stack;
	struct r10bio *r10b = &on_stack.r10_bio;
N
NeilBrown 已提交
4654 4655 4656 4657
	int slot = 0;
	int idx = 0;
	struct bio_vec *bvec = r10_bio->master_bio->bi_io_vec;

4658 4659
	r10b->sector = r10_bio->sector;
	__raid10_find_phys(&conf->prev, r10b);
N
NeilBrown 已提交
4660 4661 4662 4663 4664 4665 4666 4667 4668

	while (sectors) {
		int s = sectors;
		int success = 0;
		int first_slot = slot;

		if (s > (PAGE_SIZE >> 9))
			s = PAGE_SIZE >> 9;

4669
		rcu_read_lock();
N
NeilBrown 已提交
4670
		while (!success) {
4671
			int d = r10b->devs[slot].devnum;
4672
			struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev);
N
NeilBrown 已提交
4673 4674 4675 4676 4677 4678
			sector_t addr;
			if (rdev == NULL ||
			    test_bit(Faulty, &rdev->flags) ||
			    !test_bit(In_sync, &rdev->flags))
				goto failed;

4679
			addr = r10b->devs[slot].addr + idx * PAGE_SIZE;
4680 4681
			atomic_inc(&rdev->nr_pending);
			rcu_read_unlock();
N
NeilBrown 已提交
4682 4683 4684 4685
			success = sync_page_io(rdev,
					       addr,
					       s << 9,
					       bvec[idx].bv_page,
M
Mike Christie 已提交
4686
					       REQ_OP_READ, 0, false);
4687 4688
			rdev_dec_pending(rdev, mddev);
			rcu_read_lock();
N
NeilBrown 已提交
4689 4690 4691 4692 4693 4694 4695 4696 4697
			if (success)
				break;
		failed:
			slot++;
			if (slot >= conf->copies)
				slot = 0;
			if (slot == first_slot)
				break;
		}
4698
		rcu_read_unlock();
N
NeilBrown 已提交
4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710
		if (!success) {
			/* couldn't read this block, must give up */
			set_bit(MD_RECOVERY_INTR,
				&mddev->recovery);
			return -EIO;
		}
		sectors -= s;
		idx++;
	}
	return 0;
}

4711
static void end_reshape_write(struct bio *bio)
N
NeilBrown 已提交
4712
{
4713
	struct r10bio *r10_bio = get_resync_r10bio(bio);
N
NeilBrown 已提交
4714 4715 4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 4726 4727 4728
	struct mddev *mddev = r10_bio->mddev;
	struct r10conf *conf = mddev->private;
	int d;
	int slot;
	int repl;
	struct md_rdev *rdev = NULL;

	d = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
	if (repl)
		rdev = conf->mirrors[d].replacement;
	if (!rdev) {
		smp_mb();
		rdev = conf->mirrors[d].rdev;
	}

4729
	if (bio->bi_error) {
N
NeilBrown 已提交
4730 4731 4732 4733 4734 4735 4736 4737 4738 4739 4740 4741 4742 4743 4744 4745 4746 4747 4748 4749 4750 4751 4752 4753 4754 4755 4756 4757 4758 4759 4760 4761
		/* FIXME should record badblock */
		md_error(mddev, rdev);
	}

	rdev_dec_pending(rdev, mddev);
	end_reshape_request(r10_bio);
}

static void end_reshape_request(struct r10bio *r10_bio)
{
	if (!atomic_dec_and_test(&r10_bio->remaining))
		return;
	md_done_sync(r10_bio->mddev, r10_bio->sectors, 1);
	bio_put(r10_bio->master_bio);
	put_buf(r10_bio);
}

static void raid10_finish_reshape(struct mddev *mddev)
{
	struct r10conf *conf = mddev->private;

	if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
		return;

	if (mddev->delta_disks > 0) {
		sector_t size = raid10_size(mddev, 0, 0);
		md_set_array_sectors(mddev, size);
		if (mddev->recovery_cp > mddev->resync_max_sectors) {
			mddev->recovery_cp = mddev->resync_max_sectors;
			set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
		}
		mddev->resync_max_sectors = size;
4762 4763 4764 4765
		if (mddev->queue) {
			set_capacity(mddev->gendisk, mddev->array_sectors);
			revalidate_disk(mddev->gendisk);
		}
4766 4767
	} else {
		int d;
4768
		rcu_read_lock();
4769 4770 4771
		for (d = conf->geo.raid_disks ;
		     d < conf->geo.raid_disks - mddev->delta_disks;
		     d++) {
4772
			struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev);
4773 4774
			if (rdev)
				clear_bit(In_sync, &rdev->flags);
4775
			rdev = rcu_dereference(conf->mirrors[d].replacement);
4776 4777 4778
			if (rdev)
				clear_bit(In_sync, &rdev->flags);
		}
4779
		rcu_read_unlock();
N
NeilBrown 已提交
4780 4781 4782 4783 4784 4785 4786 4787
	}
	mddev->layout = mddev->new_layout;
	mddev->chunk_sectors = 1 << conf->geo.chunk_shift;
	mddev->reshape_position = MaxSector;
	mddev->delta_disks = 0;
	mddev->reshape_backwards = 0;
}

4788
static struct md_personality raid10_personality =
L
Linus Torvalds 已提交
4789 4790
{
	.name		= "raid10",
4791
	.level		= 10,
L
Linus Torvalds 已提交
4792
	.owner		= THIS_MODULE,
S
Shaohua Li 已提交
4793 4794
	.make_request	= raid10_make_request,
	.run		= raid10_run,
N
NeilBrown 已提交
4795
	.free		= raid10_free,
S
Shaohua Li 已提交
4796 4797
	.status		= raid10_status,
	.error_handler	= raid10_error,
L
Linus Torvalds 已提交
4798 4799 4800
	.hot_add_disk	= raid10_add_disk,
	.hot_remove_disk= raid10_remove_disk,
	.spare_active	= raid10_spare_active,
S
Shaohua Li 已提交
4801
	.sync_request	= raid10_sync_request,
4802
	.quiesce	= raid10_quiesce,
4803
	.size		= raid10_size,
4804
	.resize		= raid10_resize,
4805
	.takeover	= raid10_takeover,
N
NeilBrown 已提交
4806 4807 4808
	.check_reshape	= raid10_check_reshape,
	.start_reshape	= raid10_start_reshape,
	.finish_reshape	= raid10_finish_reshape,
4809
	.congested	= raid10_congested,
L
Linus Torvalds 已提交
4810 4811 4812 4813
};

static int __init raid_init(void)
{
4814
	return register_md_personality(&raid10_personality);
L
Linus Torvalds 已提交
4815 4816 4817 4818
}

static void raid_exit(void)
{
4819
	unregister_md_personality(&raid10_personality);
L
Linus Torvalds 已提交
4820 4821 4822 4823 4824
}

module_init(raid_init);
module_exit(raid_exit);
MODULE_LICENSE("GPL");
4825
MODULE_DESCRIPTION("RAID10 (striped mirror) personality for MD");
L
Linus Torvalds 已提交
4826
MODULE_ALIAS("md-personality-9"); /* RAID10 */
4827
MODULE_ALIAS("md-raid10");
4828
MODULE_ALIAS("md-level-10");
4829 4830

module_param(max_queued_requests, int, S_IRUGO|S_IWUSR);