raid10.c 130.4 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7
/*
 * raid10.c : Multiple Devices driver for Linux
 *
 * Copyright (C) 2000-2004 Neil Brown
 *
 * RAID-10 support for md.
 *
L
Lucas De Marchi 已提交
8
 * Base on code in raid1.c.  See raid1.c for further copyright information.
L
Linus Torvalds 已提交
9 10 11 12 13 14 15 16 17 18 19 20
 *
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2, or (at your option)
 * any later version.
 *
 * You should have received a copy of the GNU General Public License
 * (for example /usr/src/linux/COPYING); if not, write to the Free
 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

21
#include <linux/slab.h>
22
#include <linux/delay.h>
23
#include <linux/blkdev.h>
24
#include <linux/module.h>
25
#include <linux/seq_file.h>
26
#include <linux/ratelimit.h>
N
NeilBrown 已提交
27
#include <linux/kthread.h>
28
#include <trace/events/block.h>
29
#include "md.h"
30
#include "raid10.h"
31
#include "raid0.h"
32
#include "bitmap.h"
L
Linus Torvalds 已提交
33 34 35 36 37 38 39 40

/*
 * RAID10 provides a combination of RAID0 and RAID1 functionality.
 * The layout of data is defined by
 *    chunk_size
 *    raid_disks
 *    near_copies (stored in low byte of layout)
 *    far_copies (stored in second byte of layout)
41
 *    far_offset (stored in bit 16 of layout )
42
 *    use_far_sets (stored in bit 17 of layout )
43
 *    use_far_sets_bugfixed (stored in bit 18 of layout )
L
Linus Torvalds 已提交
44
 *
45 46 47 48 49 50 51 52
 * The data to be stored is divided into chunks using chunksize.  Each device
 * is divided into far_copies sections.   In each section, chunks are laid out
 * in a style similar to raid0, but near_copies copies of each chunk is stored
 * (each on a different drive).  The starting device for each section is offset
 * near_copies from the starting device of the previous section.  Thus there
 * are (near_copies * far_copies) of each chunk, and each is on a different
 * drive.  near_copies and far_copies must be at least one, and their product
 * is at most raid_disks.
53 54
 *
 * If far_offset is true, then the far_copies are handled a bit differently.
55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72
 * The copies are still in different stripes, but instead of being very far
 * apart on disk, there are adjacent stripes.
 *
 * The far and offset algorithms are handled slightly differently if
 * 'use_far_sets' is true.  In this case, the array's devices are grouped into
 * sets that are (near_copies * far_copies) in size.  The far copied stripes
 * are still shifted by 'near_copies' devices, but this shifting stays confined
 * to the set rather than the entire array.  This is done to improve the number
 * of device combinations that can fail without causing the array to fail.
 * Example 'far' algorithm w/o 'use_far_sets' (each letter represents a chunk
 * on a device):
 *    A B C D    A B C D E
 *      ...         ...
 *    D A B C    E A B C D
 * Example 'far' algorithm w/ 'use_far_sets' enabled (sets illustrated w/ []'s):
 *    [A B] [C D]    [A B] [C D E]
 *    |...| |...|    |...| | ... |
 *    [B A] [D C]    [B A] [E C D]
L
Linus Torvalds 已提交
73 74 75 76 77 78 79
 */

/*
 * Number of guaranteed r10bios in case of extreme VM load:
 */
#define	NR_RAID10_BIOS 256

80 81 82 83 84 85 86 87 88 89 90 91 92 93 94
/* when we get a read error on a read-only array, we redirect to another
 * device without failing the first device, or trying to over-write to
 * correct the read error.  To keep track of bad blocks on a per-bio
 * level, we store IO_BLOCKED in the appropriate 'bios' pointer
 */
#define IO_BLOCKED ((struct bio *)1)
/* When we successfully write to a known bad-block, we need to remove the
 * bad-block marking which must be done from process context.  So we record
 * the success by setting devs[n].bio to IO_MADE_GOOD
 */
#define IO_MADE_GOOD ((struct bio *)2)

#define BIO_SPECIAL(bio) ((unsigned long)bio <= 2)

/* When there are this many requests queued to be written by
95 96 97 98 99
 * the raid10 thread, we become 'congested' to provide back-pressure
 * for writeback.
 */
static int max_queued_requests = 1024;

100 101
static void allow_barrier(struct r10conf *conf);
static void lower_barrier(struct r10conf *conf);
102
static int _enough(struct r10conf *conf, int previous, int ignore);
103
static int enough(struct r10conf *conf, int ignore);
N
NeilBrown 已提交
104 105 106
static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
				int *skipped);
static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio);
107
static void end_reshape_write(struct bio *bio);
N
NeilBrown 已提交
108
static void end_reshape(struct r10conf *conf);
109

110 111 112
#define raid10_log(md, fmt, args...)				\
	do { if ((md)->queue) blk_add_trace_msg((md)->queue, "raid10 " fmt, ##args); } while (0)

113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130
/*
 * 'strct resync_pages' stores actual pages used for doing the resync
 *  IO, and it is per-bio, so make .bi_private points to it.
 */
static inline struct resync_pages *get_resync_pages(struct bio *bio)
{
	return bio->bi_private;
}

/*
 * for resync bio, r10bio pointer can be retrieved from the per-bio
 * 'struct resync_pages'.
 */
static inline struct r10bio *get_resync_r10bio(struct bio *bio)
{
	return get_resync_pages(bio)->raid_bio;
}

A
Al Viro 已提交
131
static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data)
L
Linus Torvalds 已提交
132
{
133
	struct r10conf *conf = data;
134
	int size = offsetof(struct r10bio, devs[conf->copies]);
L
Linus Torvalds 已提交
135

136 137
	/* allocate a r10bio with room for raid_disks entries in the
	 * bios array */
J
Jens Axboe 已提交
138
	return kzalloc(size, gfp_flags);
L
Linus Torvalds 已提交
139 140 141 142 143 144 145
}

static void r10bio_pool_free(void *r10_bio, void *data)
{
	kfree(r10_bio);
}

146 147 148 149
/* amount of memory to reserve for resync requests */
#define RESYNC_WINDOW (1024*1024)
/* maximum number of concurrent requests, memory permitting */
#define RESYNC_DEPTH (32*1024*1024/RESYNC_BLOCK_SIZE)
L
Linus Torvalds 已提交
150 151 152 153 154 155 156 157

/*
 * When performing a resync, we need to read and compare, so
 * we need as many pages are there are copies.
 * When performing a recovery, we need 2 bios, one for read,
 * one for write (we recover only one drive per r10buf)
 *
 */
A
Al Viro 已提交
158
static void * r10buf_pool_alloc(gfp_t gfp_flags, void *data)
L
Linus Torvalds 已提交
159
{
160
	struct r10conf *conf = data;
161
	struct r10bio *r10_bio;
L
Linus Torvalds 已提交
162
	struct bio *bio;
163 164 165
	int j;
	int nalloc, nalloc_rp;
	struct resync_pages *rps;
L
Linus Torvalds 已提交
166 167

	r10_bio = r10bio_pool_alloc(gfp_flags, conf);
J
Jens Axboe 已提交
168
	if (!r10_bio)
L
Linus Torvalds 已提交
169 170
		return NULL;

N
NeilBrown 已提交
171 172
	if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery) ||
	    test_bit(MD_RECOVERY_RESHAPE, &conf->mddev->recovery))
L
Linus Torvalds 已提交
173 174 175 176
		nalloc = conf->copies; /* resync */
	else
		nalloc = 2; /* recovery */

177 178 179 180 181 182 183 184 185
	/* allocate once for all bios */
	if (!conf->have_replacement)
		nalloc_rp = nalloc;
	else
		nalloc_rp = nalloc * 2;
	rps = kmalloc(sizeof(struct resync_pages) * nalloc_rp, gfp_flags);
	if (!rps)
		goto out_free_r10bio;

L
Linus Torvalds 已提交
186 187 188 189
	/*
	 * Allocate bios.
	 */
	for (j = nalloc ; j-- ; ) {
190
		bio = bio_kmalloc(gfp_flags, RESYNC_PAGES);
L
Linus Torvalds 已提交
191 192 193
		if (!bio)
			goto out_free_bio;
		r10_bio->devs[j].bio = bio;
194 195 196 197 198 199
		if (!conf->have_replacement)
			continue;
		bio = bio_kmalloc(gfp_flags, RESYNC_PAGES);
		if (!bio)
			goto out_free_bio;
		r10_bio->devs[j].repl_bio = bio;
L
Linus Torvalds 已提交
200 201 202 203 204
	}
	/*
	 * Allocate RESYNC_PAGES data pages and attach them
	 * where needed.
	 */
205
	for (j = 0; j < nalloc; j++) {
206
		struct bio *rbio = r10_bio->devs[j].repl_bio;
207 208 209 210 211 212
		struct resync_pages *rp, *rp_repl;

		rp = &rps[j];
		if (rbio)
			rp_repl = &rps[nalloc + j];

L
Linus Torvalds 已提交
213
		bio = r10_bio->devs[j].bio;
214 215 216 217

		if (!j || test_bit(MD_RECOVERY_SYNC,
				   &conf->mddev->recovery)) {
			if (resync_alloc_pages(rp, gfp_flags))
L
Linus Torvalds 已提交
218
				goto out_free_pages;
219 220 221 222
		} else {
			memcpy(rp, &rps[0], sizeof(*rp));
			resync_get_all_pages(rp);
		}
L
Linus Torvalds 已提交
223

224 225 226 227 228 229
		rp->idx = 0;
		rp->raid_bio = r10_bio;
		bio->bi_private = rp;
		if (rbio) {
			memcpy(rp_repl, rp, sizeof(*rp));
			rbio->bi_private = rp_repl;
L
Linus Torvalds 已提交
230 231 232 233 234 235
		}
	}

	return r10_bio;

out_free_pages:
236 237 238
	while (--j >= 0)
		resync_free_pages(&rps[j * 2]);

239
	j = 0;
L
Linus Torvalds 已提交
240
out_free_bio:
241 242 243
	for ( ; j < nalloc; j++) {
		if (r10_bio->devs[j].bio)
			bio_put(r10_bio->devs[j].bio);
244 245 246
		if (r10_bio->devs[j].repl_bio)
			bio_put(r10_bio->devs[j].repl_bio);
	}
247 248
	kfree(rps);
out_free_r10bio:
L
Linus Torvalds 已提交
249 250 251 252 253 254
	r10bio_pool_free(r10_bio, conf);
	return NULL;
}

static void r10buf_pool_free(void *__r10_bio, void *data)
{
255
	struct r10conf *conf = data;
256
	struct r10bio *r10bio = __r10_bio;
L
Linus Torvalds 已提交
257
	int j;
258
	struct resync_pages *rp = NULL;
L
Linus Torvalds 已提交
259

260
	for (j = conf->copies; j--; ) {
L
Linus Torvalds 已提交
261
		struct bio *bio = r10bio->devs[j].bio;
262 263 264 265 266

		rp = get_resync_pages(bio);
		resync_free_pages(rp);
		bio_put(bio);

267 268 269
		bio = r10bio->devs[j].repl_bio;
		if (bio)
			bio_put(bio);
L
Linus Torvalds 已提交
270
	}
271 272 273 274

	/* resync pages array stored in the 1st bio's .bi_private */
	kfree(rp);

L
Linus Torvalds 已提交
275 276 277
	r10bio_pool_free(r10bio, conf);
}

278
static void put_all_bios(struct r10conf *conf, struct r10bio *r10_bio)
L
Linus Torvalds 已提交
279 280 281 282 283
{
	int i;

	for (i = 0; i < conf->copies; i++) {
		struct bio **bio = & r10_bio->devs[i].bio;
284
		if (!BIO_SPECIAL(*bio))
L
Linus Torvalds 已提交
285 286
			bio_put(*bio);
		*bio = NULL;
287 288 289 290
		bio = &r10_bio->devs[i].repl_bio;
		if (r10_bio->read_slot < 0 && !BIO_SPECIAL(*bio))
			bio_put(*bio);
		*bio = NULL;
L
Linus Torvalds 已提交
291 292 293
	}
}

294
static void free_r10bio(struct r10bio *r10_bio)
L
Linus Torvalds 已提交
295
{
296
	struct r10conf *conf = r10_bio->mddev->private;
L
Linus Torvalds 已提交
297 298 299 300 301

	put_all_bios(conf, r10_bio);
	mempool_free(r10_bio, conf->r10bio_pool);
}

302
static void put_buf(struct r10bio *r10_bio)
L
Linus Torvalds 已提交
303
{
304
	struct r10conf *conf = r10_bio->mddev->private;
L
Linus Torvalds 已提交
305 306 307

	mempool_free(r10_bio, conf->r10buf_pool);

308
	lower_barrier(conf);
L
Linus Torvalds 已提交
309 310
}

311
static void reschedule_retry(struct r10bio *r10_bio)
L
Linus Torvalds 已提交
312 313
{
	unsigned long flags;
314
	struct mddev *mddev = r10_bio->mddev;
315
	struct r10conf *conf = mddev->private;
L
Linus Torvalds 已提交
316 317 318

	spin_lock_irqsave(&conf->device_lock, flags);
	list_add(&r10_bio->retry_list, &conf->retry_list);
319
	conf->nr_queued ++;
L
Linus Torvalds 已提交
320 321
	spin_unlock_irqrestore(&conf->device_lock, flags);

A
Arthur Jones 已提交
322 323 324
	/* wake up frozen array... */
	wake_up(&conf->wait_barrier);

L
Linus Torvalds 已提交
325 326 327 328 329 330 331 332
	md_wakeup_thread(mddev->thread);
}

/*
 * raid_end_bio_io() is called when we have finished servicing a mirrored
 * operation and are ready to return a success/failure code to the buffer
 * cache layer.
 */
333
static void raid_end_bio_io(struct r10bio *r10_bio)
L
Linus Torvalds 已提交
334 335
{
	struct bio *bio = r10_bio->master_bio;
336
	struct r10conf *conf = r10_bio->mddev->private;
L
Linus Torvalds 已提交
337

338
	if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
339
		bio->bi_error = -EIO;
340 341 342 343 344 345 346 347

	bio_endio(bio);
	/*
	 * Wake up any possible resync thread that waits for the device
	 * to go idle.
	 */
	allow_barrier(conf);

L
Linus Torvalds 已提交
348 349 350 351 352 353
	free_r10bio(r10_bio);
}

/*
 * Update disk head position estimator based on IRQ completion info.
 */
354
static inline void update_head_pos(int slot, struct r10bio *r10_bio)
L
Linus Torvalds 已提交
355
{
356
	struct r10conf *conf = r10_bio->mddev->private;
L
Linus Torvalds 已提交
357 358 359 360 361

	conf->mirrors[r10_bio->devs[slot].devnum].head_position =
		r10_bio->devs[slot].addr + (r10_bio->sectors);
}

362 363 364
/*
 * Find the disk number which triggered given bio
 */
365
static int find_bio_disk(struct r10conf *conf, struct r10bio *r10_bio,
366
			 struct bio *bio, int *slotp, int *replp)
367 368
{
	int slot;
369
	int repl = 0;
370

371
	for (slot = 0; slot < conf->copies; slot++) {
372 373
		if (r10_bio->devs[slot].bio == bio)
			break;
374 375 376 377 378
		if (r10_bio->devs[slot].repl_bio == bio) {
			repl = 1;
			break;
		}
	}
379 380 381 382

	BUG_ON(slot == conf->copies);
	update_head_pos(slot, r10_bio);

383 384
	if (slotp)
		*slotp = slot;
385 386
	if (replp)
		*replp = repl;
387 388 389
	return r10_bio->devs[slot].devnum;
}

390
static void raid10_end_read_request(struct bio *bio)
L
Linus Torvalds 已提交
391
{
392
	int uptodate = !bio->bi_error;
393
	struct r10bio *r10_bio = bio->bi_private;
L
Linus Torvalds 已提交
394
	int slot, dev;
395
	struct md_rdev *rdev;
396
	struct r10conf *conf = r10_bio->mddev->private;
L
Linus Torvalds 已提交
397 398 399

	slot = r10_bio->read_slot;
	dev = r10_bio->devs[slot].devnum;
400
	rdev = r10_bio->devs[slot].rdev;
L
Linus Torvalds 已提交
401 402 403
	/*
	 * this branch is our 'one mirror IO has finished' event handler:
	 */
404 405 406
	update_head_pos(slot, r10_bio);

	if (uptodate) {
L
Linus Torvalds 已提交
407 408 409 410 411 412 413 414 415 416
		/*
		 * Set R10BIO_Uptodate in our master bio, so that
		 * we will return a good error code to the higher
		 * levels even if IO on some other mirrored buffer fails.
		 *
		 * The 'master' represents the composite IO operation to
		 * user-side. So if something waits for IO, then it will
		 * wait for the 'master' bio.
		 */
		set_bit(R10BIO_Uptodate, &r10_bio->state);
417 418 419 420 421 422
	} else {
		/* If all other devices that store this block have
		 * failed, we want to return the error upwards rather
		 * than fail the last device.  Here we redefine
		 * "uptodate" to mean "Don't want to retry"
		 */
423 424
		if (!_enough(conf, test_bit(R10BIO_Previous, &r10_bio->state),
			     rdev->raid_disk))
425 426 427
			uptodate = 1;
	}
	if (uptodate) {
L
Linus Torvalds 已提交
428
		raid_end_bio_io(r10_bio);
429
		rdev_dec_pending(rdev, conf->mddev);
430
	} else {
L
Linus Torvalds 已提交
431
		/*
432
		 * oops, read error - keep the refcount on the rdev
L
Linus Torvalds 已提交
433 434
		 */
		char b[BDEVNAME_SIZE];
N
NeilBrown 已提交
435
		pr_err_ratelimited("md/raid10:%s: %s: rescheduling sector %llu\n",
436
				   mdname(conf->mddev),
437
				   bdevname(rdev->bdev, b),
438
				   (unsigned long long)r10_bio->sector);
439
		set_bit(R10BIO_ReadError, &r10_bio->state);
L
Linus Torvalds 已提交
440 441 442 443
		reschedule_retry(r10_bio);
	}
}

444
static void close_write(struct r10bio *r10_bio)
445 446 447 448 449 450 451 452 453
{
	/* clear the bitmap if all writes complete successfully */
	bitmap_endwrite(r10_bio->mddev->bitmap, r10_bio->sector,
			r10_bio->sectors,
			!test_bit(R10BIO_Degraded, &r10_bio->state),
			0);
	md_write_end(r10_bio->mddev);
}

454
static void one_write_done(struct r10bio *r10_bio)
455 456 457 458 459 460 461 462 463 464 465 466 467 468
{
	if (atomic_dec_and_test(&r10_bio->remaining)) {
		if (test_bit(R10BIO_WriteError, &r10_bio->state))
			reschedule_retry(r10_bio);
		else {
			close_write(r10_bio);
			if (test_bit(R10BIO_MadeGood, &r10_bio->state))
				reschedule_retry(r10_bio);
			else
				raid_end_bio_io(r10_bio);
		}
	}
}

469
static void raid10_end_write_request(struct bio *bio)
L
Linus Torvalds 已提交
470
{
471
	struct r10bio *r10_bio = bio->bi_private;
472
	int dev;
473
	int dec_rdev = 1;
474
	struct r10conf *conf = r10_bio->mddev->private;
475
	int slot, repl;
476
	struct md_rdev *rdev = NULL;
477
	struct bio *to_put = NULL;
S
Shaohua Li 已提交
478 479 480
	bool discard_error;

	discard_error = bio->bi_error && bio_op(bio) == REQ_OP_DISCARD;
L
Linus Torvalds 已提交
481

482
	dev = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
L
Linus Torvalds 已提交
483

484 485
	if (repl)
		rdev = conf->mirrors[dev].replacement;
486 487 488
	if (!rdev) {
		smp_rmb();
		repl = 0;
489
		rdev = conf->mirrors[dev].rdev;
490
	}
L
Linus Torvalds 已提交
491 492 493
	/*
	 * this branch is our 'one mirror IO has finished' event handler:
	 */
S
Shaohua Li 已提交
494
	if (bio->bi_error && !discard_error) {
495 496 497 498 499 500 501
		if (repl)
			/* Never record new bad blocks to replacement,
			 * just fail it.
			 */
			md_error(rdev->mddev, rdev);
		else {
			set_bit(WriteErrorSeen,	&rdev->flags);
502 503 504
			if (!test_and_set_bit(WantReplacement, &rdev->flags))
				set_bit(MD_RECOVERY_NEEDED,
					&rdev->mddev->recovery);
505

506
			dec_rdev = 0;
507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522
			if (test_bit(FailFast, &rdev->flags) &&
			    (bio->bi_opf & MD_FAILFAST)) {
				md_error(rdev->mddev, rdev);
				if (!test_bit(Faulty, &rdev->flags))
					/* This is the only remaining device,
					 * We need to retry the write without
					 * FailFast
					 */
					set_bit(R10BIO_WriteError, &r10_bio->state);
				else {
					r10_bio->devs[slot].bio = NULL;
					to_put = bio;
					dec_rdev = 1;
				}
			} else
				set_bit(R10BIO_WriteError, &r10_bio->state);
523
		}
524
	} else {
L
Linus Torvalds 已提交
525 526 527 528 529 530 531 532 533
		/*
		 * Set R10BIO_Uptodate in our master bio, so that
		 * we will return a good error code for to the higher
		 * levels even if IO on some other mirrored buffer fails.
		 *
		 * The 'master' represents the composite IO operation to
		 * user-side. So if something waits for IO, then it will
		 * wait for the 'master' bio.
		 */
534 535 536
		sector_t first_bad;
		int bad_sectors;

537 538 539 540 541 542 543 544 545 546 547
		/*
		 * Do not set R10BIO_Uptodate if the current device is
		 * rebuilding or Faulty. This is because we cannot use
		 * such device for properly reading the data back (we could
		 * potentially use it, if the current write would have felt
		 * before rdev->recovery_offset, but for simplicity we don't
		 * check this here.
		 */
		if (test_bit(In_sync, &rdev->flags) &&
		    !test_bit(Faulty, &rdev->flags))
			set_bit(R10BIO_Uptodate, &r10_bio->state);
L
Linus Torvalds 已提交
548

549
		/* Maybe we can clear some bad blocks. */
550
		if (is_badblock(rdev,
551 552
				r10_bio->devs[slot].addr,
				r10_bio->sectors,
S
Shaohua Li 已提交
553
				&first_bad, &bad_sectors) && !discard_error) {
554
			bio_put(bio);
555 556 557 558
			if (repl)
				r10_bio->devs[slot].repl_bio = IO_MADE_GOOD;
			else
				r10_bio->devs[slot].bio = IO_MADE_GOOD;
559 560 561 562 563
			dec_rdev = 0;
			set_bit(R10BIO_MadeGood, &r10_bio->state);
		}
	}

L
Linus Torvalds 已提交
564 565 566 567 568
	/*
	 *
	 * Let's see if all mirrored write operations have finished
	 * already.
	 */
569
	one_write_done(r10_bio);
570
	if (dec_rdev)
571
		rdev_dec_pending(rdev, conf->mddev);
572 573
	if (to_put)
		bio_put(to_put);
L
Linus Torvalds 已提交
574 575 576 577
}

/*
 * RAID10 layout manager
L
Lucas De Marchi 已提交
578
 * As well as the chunksize and raid_disks count, there are two
L
Linus Torvalds 已提交
579 580 581 582 583 584
 * parameters: near_copies and far_copies.
 * near_copies * far_copies must be <= raid_disks.
 * Normally one of these will be 1.
 * If both are 1, we get raid0.
 * If near_copies == raid_disks, we get raid1.
 *
L
Lucas De Marchi 已提交
585
 * Chunks are laid out in raid0 style with near_copies copies of the
L
Linus Torvalds 已提交
586 587 588 589 590 591 592 593 594
 * first chunk, followed by near_copies copies of the next chunk and
 * so on.
 * If far_copies > 1, then after 1/far_copies of the array has been assigned
 * as described above, we start again with a device offset of near_copies.
 * So we effectively have another copy of the whole array further down all
 * the drives, but with blocks on different drives.
 * With this layout, and block is never stored twice on the one device.
 *
 * raid10_find_phys finds the sector offset of a given virtual sector
595
 * on each device that it is on.
L
Linus Torvalds 已提交
596 597 598 599 600
 *
 * raid10_find_virt does the reverse mapping, from a device and a
 * sector offset to a virtual address
 */

601
static void __raid10_find_phys(struct geom *geo, struct r10bio *r10bio)
L
Linus Torvalds 已提交
602 603 604 605 606 607 608
{
	int n,f;
	sector_t sector;
	sector_t chunk;
	sector_t stripe;
	int dev;
	int slot = 0;
609 610 611 612 613 614 615
	int last_far_set_start, last_far_set_size;

	last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1;
	last_far_set_start *= geo->far_set_size;

	last_far_set_size = geo->far_set_size;
	last_far_set_size += (geo->raid_disks % geo->far_set_size);
L
Linus Torvalds 已提交
616 617

	/* now calculate first sector/dev */
618 619
	chunk = r10bio->sector >> geo->chunk_shift;
	sector = r10bio->sector & geo->chunk_mask;
L
Linus Torvalds 已提交
620

621
	chunk *= geo->near_copies;
L
Linus Torvalds 已提交
622
	stripe = chunk;
623 624 625
	dev = sector_div(stripe, geo->raid_disks);
	if (geo->far_offset)
		stripe *= geo->far_copies;
L
Linus Torvalds 已提交
626

627
	sector += stripe << geo->chunk_shift;
L
Linus Torvalds 已提交
628 629

	/* and calculate all the others */
630
	for (n = 0; n < geo->near_copies; n++) {
L
Linus Torvalds 已提交
631
		int d = dev;
632
		int set;
L
Linus Torvalds 已提交
633 634
		sector_t s = sector;
		r10bio->devs[slot].devnum = d;
635
		r10bio->devs[slot].addr = s;
L
Linus Torvalds 已提交
636 637
		slot++;

638
		for (f = 1; f < geo->far_copies; f++) {
639
			set = d / geo->far_set_size;
640
			d += geo->near_copies;
641

642 643 644 645 646 647 648 649 650
			if ((geo->raid_disks % geo->far_set_size) &&
			    (d > last_far_set_start)) {
				d -= last_far_set_start;
				d %= last_far_set_size;
				d += last_far_set_start;
			} else {
				d %= geo->far_set_size;
				d += geo->far_set_size * set;
			}
651
			s += geo->stride;
L
Linus Torvalds 已提交
652 653 654 655 656
			r10bio->devs[slot].devnum = d;
			r10bio->devs[slot].addr = s;
			slot++;
		}
		dev++;
657
		if (dev >= geo->raid_disks) {
L
Linus Torvalds 已提交
658
			dev = 0;
659
			sector += (geo->chunk_mask + 1);
L
Linus Torvalds 已提交
660 661
		}
	}
662 663 664 665 666 667 668 669 670 671 672 673 674 675 676
}

static void raid10_find_phys(struct r10conf *conf, struct r10bio *r10bio)
{
	struct geom *geo = &conf->geo;

	if (conf->reshape_progress != MaxSector &&
	    ((r10bio->sector >= conf->reshape_progress) !=
	     conf->mddev->reshape_backwards)) {
		set_bit(R10BIO_Previous, &r10bio->state);
		geo = &conf->prev;
	} else
		clear_bit(R10BIO_Previous, &r10bio->state);

	__raid10_find_phys(geo, r10bio);
L
Linus Torvalds 已提交
677 678
}

679
static sector_t raid10_find_virt(struct r10conf *conf, sector_t sector, int dev)
L
Linus Torvalds 已提交
680 681
{
	sector_t offset, chunk, vchunk;
682 683 684
	/* Never use conf->prev as this is only called during resync
	 * or recovery, so reshape isn't happening
	 */
685
	struct geom *geo = &conf->geo;
686 687
	int far_set_start = (dev / geo->far_set_size) * geo->far_set_size;
	int far_set_size = geo->far_set_size;
688 689 690 691 692 693 694 695 696 697 698 699
	int last_far_set_start;

	if (geo->raid_disks % geo->far_set_size) {
		last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1;
		last_far_set_start *= geo->far_set_size;

		if (dev >= last_far_set_start) {
			far_set_size = geo->far_set_size;
			far_set_size += (geo->raid_disks % geo->far_set_size);
			far_set_start = last_far_set_start;
		}
	}
L
Linus Torvalds 已提交
700

701 702
	offset = sector & geo->chunk_mask;
	if (geo->far_offset) {
703
		int fc;
704 705 706
		chunk = sector >> geo->chunk_shift;
		fc = sector_div(chunk, geo->far_copies);
		dev -= fc * geo->near_copies;
707 708
		if (dev < far_set_start)
			dev += far_set_size;
709
	} else {
710 711
		while (sector >= geo->stride) {
			sector -= geo->stride;
712 713
			if (dev < (geo->near_copies + far_set_start))
				dev += far_set_size - geo->near_copies;
714
			else
715
				dev -= geo->near_copies;
716
		}
717
		chunk = sector >> geo->chunk_shift;
718
	}
719 720 721
	vchunk = chunk * geo->raid_disks + dev;
	sector_div(vchunk, geo->near_copies);
	return (vchunk << geo->chunk_shift) + offset;
L
Linus Torvalds 已提交
722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742
}

/*
 * This routine returns the disk from which the requested read should
 * be done. There is a per-array 'next expected sequential IO' sector
 * number - if this matches on the next IO then we use the last disk.
 * There is also a per-disk 'last know head position' sector that is
 * maintained from IRQ contexts, both the normal and the resync IO
 * completion handlers update this position correctly. If there is no
 * perfect sequential match then we pick the disk whose head is closest.
 *
 * If there are 2 mirrors in the same 2 devices, performance degrades
 * because position is mirror, not device based.
 *
 * The rdev for the device selected will have nr_pending incremented.
 */

/*
 * FIXME: possibly should rethink readbalancing and do it differently
 * depending on near_copies / far_copies geometry.
 */
743 744 745
static struct md_rdev *read_balance(struct r10conf *conf,
				    struct r10bio *r10_bio,
				    int *max_sectors)
L
Linus Torvalds 已提交
746
{
747
	const sector_t this_sector = r10_bio->sector;
N
NeilBrown 已提交
748
	int disk, slot;
749 750
	int sectors = r10_bio->sectors;
	int best_good_sectors;
N
NeilBrown 已提交
751
	sector_t new_distance, best_dist;
752
	struct md_rdev *best_rdev, *rdev = NULL;
N
NeilBrown 已提交
753 754
	int do_balance;
	int best_slot;
755
	struct geom *geo = &conf->geo;
L
Linus Torvalds 已提交
756 757 758

	raid10_find_phys(conf, r10_bio);
	rcu_read_lock();
759
	sectors = r10_bio->sectors;
N
NeilBrown 已提交
760
	best_slot = -1;
761
	best_rdev = NULL;
N
NeilBrown 已提交
762
	best_dist = MaxSector;
763
	best_good_sectors = 0;
N
NeilBrown 已提交
764
	do_balance = 1;
765
	clear_bit(R10BIO_FailFast, &r10_bio->state);
L
Linus Torvalds 已提交
766 767
	/*
	 * Check if we can balance. We can balance on the whole
768 769 770
	 * device if no resync is going on (recovery is ok), or below
	 * the resync window. We take the first readable disk when
	 * above the resync window.
L
Linus Torvalds 已提交
771 772
	 */
	if (conf->mddev->recovery_cp < MaxSector
N
NeilBrown 已提交
773 774
	    && (this_sector + sectors >= conf->next_resync))
		do_balance = 0;
L
Linus Torvalds 已提交
775

N
NeilBrown 已提交
776
	for (slot = 0; slot < conf->copies ; slot++) {
777 778 779 780
		sector_t first_bad;
		int bad_sectors;
		sector_t dev_sector;

N
NeilBrown 已提交
781 782
		if (r10_bio->devs[slot].bio == IO_BLOCKED)
			continue;
L
Linus Torvalds 已提交
783
		disk = r10_bio->devs[slot].devnum;
784 785 786 787
		rdev = rcu_dereference(conf->mirrors[disk].replacement);
		if (rdev == NULL || test_bit(Faulty, &rdev->flags) ||
		    r10_bio->devs[slot].addr + sectors > rdev->recovery_offset)
			rdev = rcu_dereference(conf->mirrors[disk].rdev);
788
		if (rdev == NULL ||
789
		    test_bit(Faulty, &rdev->flags))
790 791 792
			continue;
		if (!test_bit(In_sync, &rdev->flags) &&
		    r10_bio->devs[slot].addr + sectors > rdev->recovery_offset)
N
NeilBrown 已提交
793 794
			continue;

795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816
		dev_sector = r10_bio->devs[slot].addr;
		if (is_badblock(rdev, dev_sector, sectors,
				&first_bad, &bad_sectors)) {
			if (best_dist < MaxSector)
				/* Already have a better slot */
				continue;
			if (first_bad <= dev_sector) {
				/* Cannot read here.  If this is the
				 * 'primary' device, then we must not read
				 * beyond 'bad_sectors' from another device.
				 */
				bad_sectors -= (dev_sector - first_bad);
				if (!do_balance && sectors > bad_sectors)
					sectors = bad_sectors;
				if (best_good_sectors > sectors)
					best_good_sectors = sectors;
			} else {
				sector_t good_sectors =
					first_bad - dev_sector;
				if (good_sectors > best_good_sectors) {
					best_good_sectors = good_sectors;
					best_slot = slot;
817
					best_rdev = rdev;
818 819 820 821 822 823 824 825 826
				}
				if (!do_balance)
					/* Must read from here */
					break;
			}
			continue;
		} else
			best_good_sectors = sectors;

N
NeilBrown 已提交
827 828
		if (!do_balance)
			break;
L
Linus Torvalds 已提交
829

830 831 832
		if (best_slot >= 0)
			/* At least 2 disks to choose from so failfast is OK */
			set_bit(R10BIO_FailFast, &r10_bio->state);
833 834 835 836
		/* This optimisation is debatable, and completely destroys
		 * sequential read speed for 'far copies' arrays.  So only
		 * keep it for 'near' arrays, and review those later.
		 */
837
		if (geo->near_copies > 1 && !atomic_read(&rdev->nr_pending))
838
			new_distance = 0;
839 840

		/* for far > 1 always use the lowest address */
841
		else if (geo->far_copies > 1)
N
NeilBrown 已提交
842
			new_distance = r10_bio->devs[slot].addr;
843
		else
N
NeilBrown 已提交
844 845 846 847 848
			new_distance = abs(r10_bio->devs[slot].addr -
					   conf->mirrors[disk].head_position);
		if (new_distance < best_dist) {
			best_dist = new_distance;
			best_slot = slot;
849
			best_rdev = rdev;
L
Linus Torvalds 已提交
850 851
		}
	}
852
	if (slot >= conf->copies) {
N
NeilBrown 已提交
853
		slot = best_slot;
854 855
		rdev = best_rdev;
	}
L
Linus Torvalds 已提交
856

N
NeilBrown 已提交
857 858 859 860
	if (slot >= 0) {
		atomic_inc(&rdev->nr_pending);
		r10_bio->read_slot = slot;
	} else
861
		rdev = NULL;
L
Linus Torvalds 已提交
862
	rcu_read_unlock();
863
	*max_sectors = best_good_sectors;
L
Linus Torvalds 已提交
864

865
	return rdev;
L
Linus Torvalds 已提交
866 867
}

868
static int raid10_congested(struct mddev *mddev, int bits)
869
{
870
	struct r10conf *conf = mddev->private;
871 872
	int i, ret = 0;

873
	if ((bits & (1 << WB_async_congested)) &&
874 875 876
	    conf->pending_count >= max_queued_requests)
		return 1;

877
	rcu_read_lock();
878 879 880 881
	for (i = 0;
	     (i < conf->geo.raid_disks || i < conf->prev.raid_disks)
		     && ret == 0;
	     i++) {
882
		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
883
		if (rdev && !test_bit(Faulty, &rdev->flags)) {
884
			struct request_queue *q = bdev_get_queue(rdev->bdev);
885

886
			ret |= bdi_congested(q->backing_dev_info, bits);
887 888 889 890 891 892
		}
	}
	rcu_read_unlock();
	return ret;
}

893
static void flush_pending_writes(struct r10conf *conf)
894 895 896 897 898 899 900 901 902
{
	/* Any writes that have been queued but are awaiting
	 * bitmap updates get flushed here.
	 */
	spin_lock_irq(&conf->device_lock);

	if (conf->pending_bio_list.head) {
		struct bio *bio;
		bio = bio_list_get(&conf->pending_bio_list);
903
		conf->pending_count = 0;
904 905 906 907
		spin_unlock_irq(&conf->device_lock);
		/* flush any pending bitmap writes to disk
		 * before proceeding w/ I/O */
		bitmap_unplug(conf->mddev->bitmap);
908
		wake_up(&conf->wait_barrier);
909 910 911

		while (bio) { /* submit pending writes */
			struct bio *next = bio->bi_next;
912
			struct md_rdev *rdev = (void*)bio->bi_bdev;
913
			bio->bi_next = NULL;
914 915 916 917 918 919
			bio->bi_bdev = rdev->bdev;
			if (test_bit(Faulty, &rdev->flags)) {
				bio->bi_error = -EIO;
				bio_endio(bio);
			} else if (unlikely((bio_op(bio) ==  REQ_OP_DISCARD) &&
					    !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
S
Shaohua Li 已提交
920
				/* Just ignore it */
921
				bio_endio(bio);
S
Shaohua Li 已提交
922 923
			else
				generic_make_request(bio);
924 925 926 927 928
			bio = next;
		}
	} else
		spin_unlock_irq(&conf->device_lock);
}
J
Jens Axboe 已提交
929

930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949
/* Barriers....
 * Sometimes we need to suspend IO while we do something else,
 * either some resync/recovery, or reconfigure the array.
 * To do this we raise a 'barrier'.
 * The 'barrier' is a counter that can be raised multiple times
 * to count how many activities are happening which preclude
 * normal IO.
 * We can only raise the barrier if there is no pending IO.
 * i.e. if nr_pending == 0.
 * We choose only to raise the barrier if no-one is waiting for the
 * barrier to go down.  This means that as soon as an IO request
 * is ready, no other operations which require a barrier will start
 * until the IO request has had a chance.
 *
 * So: regular IO calls 'wait_barrier'.  When that returns there
 *    is no backgroup IO happening,  It must arrange to call
 *    allow_barrier when it has finished its IO.
 * backgroup IO calls must call raise_barrier.  Once that returns
 *    there is no normal IO happeing.  It must arrange to call
 *    lower_barrier when the particular background IO completes.
L
Linus Torvalds 已提交
950 951
 */

952
static void raise_barrier(struct r10conf *conf, int force)
L
Linus Torvalds 已提交
953
{
954
	BUG_ON(force && !conf->barrier);
L
Linus Torvalds 已提交
955
	spin_lock_irq(&conf->resync_lock);
956

957 958
	/* Wait until no block IO is waiting (unless 'force') */
	wait_event_lock_irq(conf->wait_barrier, force || !conf->nr_waiting,
959
			    conf->resync_lock);
960 961 962 963

	/* block any new IO from starting */
	conf->barrier++;

N
NeilBrown 已提交
964
	/* Now wait for all pending IO to complete */
965
	wait_event_lock_irq(conf->wait_barrier,
966
			    !atomic_read(&conf->nr_pending) && conf->barrier < RESYNC_DEPTH,
967
			    conf->resync_lock);
968 969 970 971

	spin_unlock_irq(&conf->resync_lock);
}

972
static void lower_barrier(struct r10conf *conf)
973 974 975 976 977 978 979 980
{
	unsigned long flags;
	spin_lock_irqsave(&conf->resync_lock, flags);
	conf->barrier--;
	spin_unlock_irqrestore(&conf->resync_lock, flags);
	wake_up(&conf->wait_barrier);
}

981
static void wait_barrier(struct r10conf *conf)
982 983 984 985
{
	spin_lock_irq(&conf->resync_lock);
	if (conf->barrier) {
		conf->nr_waiting++;
986 987 988 989 990 991 992 993 994
		/* Wait for the barrier to drop.
		 * However if there are already pending
		 * requests (preventing the barrier from
		 * rising completely), and the
		 * pre-process bio queue isn't empty,
		 * then don't wait, as we need to empty
		 * that queue to get the nr_pending
		 * count down.
		 */
995
		raid10_log(conf->mddev, "wait barrier");
996 997
		wait_event_lock_irq(conf->wait_barrier,
				    !conf->barrier ||
998
				    (atomic_read(&conf->nr_pending) &&
999
				     current->bio_list &&
1000 1001
				     (!bio_list_empty(&current->bio_list[0]) ||
				      !bio_list_empty(&current->bio_list[1]))),
1002
				    conf->resync_lock);
1003
		conf->nr_waiting--;
1004 1005
		if (!conf->nr_waiting)
			wake_up(&conf->wait_barrier);
L
Linus Torvalds 已提交
1006
	}
1007
	atomic_inc(&conf->nr_pending);
L
Linus Torvalds 已提交
1008 1009 1010
	spin_unlock_irq(&conf->resync_lock);
}

1011
static void allow_barrier(struct r10conf *conf)
1012
{
1013 1014 1015
	if ((atomic_dec_and_test(&conf->nr_pending)) ||
			(conf->array_freeze_pending))
		wake_up(&conf->wait_barrier);
1016 1017
}

1018
static void freeze_array(struct r10conf *conf, int extra)
1019 1020
{
	/* stop syncio and normal IO and wait for everything to
N
NeilBrown 已提交
1021
	 * go quiet.
1022
	 * We increment barrier and nr_waiting, and then
1023
	 * wait until nr_pending match nr_queued+extra
1024 1025 1026 1027
	 * This is called in the context of one normal IO request
	 * that has failed. Thus any sync request that might be pending
	 * will be blocked by nr_pending, and we need to wait for
	 * pending IO requests to complete or be queued for re-try.
1028
	 * Thus the number queued (nr_queued) plus this request (extra)
1029 1030
	 * must match the number of pending IOs (nr_pending) before
	 * we continue.
1031 1032
	 */
	spin_lock_irq(&conf->resync_lock);
1033
	conf->array_freeze_pending++;
1034 1035
	conf->barrier++;
	conf->nr_waiting++;
1036
	wait_event_lock_irq_cmd(conf->wait_barrier,
1037
				atomic_read(&conf->nr_pending) == conf->nr_queued+extra,
1038 1039
				conf->resync_lock,
				flush_pending_writes(conf));
N
NeilBrown 已提交
1040

1041
	conf->array_freeze_pending--;
1042 1043 1044
	spin_unlock_irq(&conf->resync_lock);
}

1045
static void unfreeze_array(struct r10conf *conf)
1046 1047 1048 1049 1050 1051 1052 1053 1054
{
	/* reverse the effect of the freeze */
	spin_lock_irq(&conf->resync_lock);
	conf->barrier--;
	conf->nr_waiting--;
	wake_up(&conf->wait_barrier);
	spin_unlock_irq(&conf->resync_lock);
}

1055 1056 1057 1058 1059 1060 1061 1062 1063 1064
static sector_t choose_data_offset(struct r10bio *r10_bio,
				   struct md_rdev *rdev)
{
	if (!test_bit(MD_RECOVERY_RESHAPE, &rdev->mddev->recovery) ||
	    test_bit(R10BIO_Previous, &r10_bio->state))
		return rdev->data_offset;
	else
		return rdev->new_data_offset;
}

1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078
struct raid10_plug_cb {
	struct blk_plug_cb	cb;
	struct bio_list		pending;
	int			pending_cnt;
};

static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule)
{
	struct raid10_plug_cb *plug = container_of(cb, struct raid10_plug_cb,
						   cb);
	struct mddev *mddev = plug->cb.data;
	struct r10conf *conf = mddev->private;
	struct bio *bio;

1079
	if (from_schedule || current->bio_list) {
1080 1081 1082 1083
		spin_lock_irq(&conf->device_lock);
		bio_list_merge(&conf->pending_bio_list, &plug->pending);
		conf->pending_count += plug->pending_cnt;
		spin_unlock_irq(&conf->device_lock);
1084
		wake_up(&conf->wait_barrier);
1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096
		md_wakeup_thread(mddev->thread);
		kfree(plug);
		return;
	}

	/* we aren't scheduling, so we can do the write-out directly. */
	bio = bio_list_get(&plug->pending);
	bitmap_unplug(mddev->bitmap);
	wake_up(&conf->wait_barrier);

	while (bio) { /* submit pending writes */
		struct bio *next = bio->bi_next;
1097
		struct md_rdev *rdev = (void*)bio->bi_bdev;
1098
		bio->bi_next = NULL;
1099 1100 1101 1102 1103 1104
		bio->bi_bdev = rdev->bdev;
		if (test_bit(Faulty, &rdev->flags)) {
			bio->bi_error = -EIO;
			bio_endio(bio);
		} else if (unlikely((bio_op(bio) ==  REQ_OP_DISCARD) &&
				    !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
1105
			/* Just ignore it */
1106
			bio_endio(bio);
1107 1108
		else
			generic_make_request(bio);
1109 1110 1111 1112 1113
		bio = next;
	}
	kfree(plug);
}

1114 1115
static void raid10_read_request(struct mddev *mddev, struct bio *bio,
				struct r10bio *r10_bio)
L
Linus Torvalds 已提交
1116
{
1117
	struct r10conf *conf = mddev->private;
L
Linus Torvalds 已提交
1118
	struct bio *read_bio;
1119 1120 1121 1122 1123
	const int op = bio_op(bio);
	const unsigned long do_sync = (bio->bi_opf & REQ_SYNC);
	int max_sectors;
	sector_t sectors;
	struct md_rdev *rdev;
1124 1125 1126 1127
	char b[BDEVNAME_SIZE];
	int slot = r10_bio->read_slot;
	struct md_rdev *err_rdev = NULL;
	gfp_t gfp = GFP_NOIO;
1128

1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155
	if (r10_bio->devs[slot].rdev) {
		/*
		 * This is an error retry, but we cannot
		 * safely dereference the rdev in the r10_bio,
		 * we must use the one in conf.
		 * If it has already been disconnected (unlikely)
		 * we lose the device name in error messages.
		 */
		int disk;
		/*
		 * As we are blocking raid10, it is a little safer to
		 * use __GFP_HIGH.
		 */
		gfp = GFP_NOIO | __GFP_HIGH;

		rcu_read_lock();
		disk = r10_bio->devs[slot].devnum;
		err_rdev = rcu_dereference(conf->mirrors[disk].rdev);
		if (err_rdev)
			bdevname(err_rdev->bdev, b);
		else {
			strcpy(b, "???");
			/* This never gets dereferenced */
			err_rdev = r10_bio->devs[slot].rdev;
		}
		rcu_read_unlock();
	}
1156 1157 1158 1159 1160 1161 1162
	/*
	 * Register the new request and wait if the reconstruction
	 * thread has put up a bar for new requests.
	 * Continue immediately if no resync is active currently.
	 */
	wait_barrier(conf);

1163
	sectors = r10_bio->sectors;
1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181
	while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
	    bio->bi_iter.bi_sector < conf->reshape_progress &&
	    bio->bi_iter.bi_sector + sectors > conf->reshape_progress) {
		/*
		 * IO spans the reshape position.  Need to wait for reshape to
		 * pass
		 */
		raid10_log(conf->mddev, "wait reshape");
		allow_barrier(conf);
		wait_event(conf->wait_barrier,
			   conf->reshape_progress <= bio->bi_iter.bi_sector ||
			   conf->reshape_progress >= bio->bi_iter.bi_sector +
			   sectors);
		wait_barrier(conf);
	}

	rdev = read_balance(conf, r10_bio, &max_sectors);
	if (!rdev) {
1182 1183 1184 1185 1186
		if (err_rdev) {
			pr_crit_ratelimited("md/raid10:%s: %s: unrecoverable I/O read error for block %llu\n",
					    mdname(mddev), b,
					    (unsigned long long)r10_bio->sector);
		}
1187 1188 1189
		raid_end_bio_io(r10_bio);
		return;
	}
1190 1191 1192 1193 1194
	if (err_rdev)
		pr_err_ratelimited("md/raid10:%s: %s: redirecting sector %llu to another mirror\n",
				   mdname(mddev),
				   bdevname(rdev->bdev, b),
				   (unsigned long long)r10_bio->sector);
1195 1196
	if (max_sectors < bio_sectors(bio)) {
		struct bio *split = bio_split(bio, max_sectors,
1197
					      gfp, conf->bio_split);
1198 1199 1200 1201 1202 1203
		bio_chain(split, bio);
		generic_make_request(bio);
		bio = split;
		r10_bio->master_bio = bio;
		r10_bio->sectors = max_sectors;
	}
1204 1205
	slot = r10_bio->read_slot;

1206
	read_bio = bio_clone_fast(bio, gfp, mddev->bio_set);
1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224

	r10_bio->devs[slot].bio = read_bio;
	r10_bio->devs[slot].rdev = rdev;

	read_bio->bi_iter.bi_sector = r10_bio->devs[slot].addr +
		choose_data_offset(r10_bio, rdev);
	read_bio->bi_bdev = rdev->bdev;
	read_bio->bi_end_io = raid10_end_read_request;
	bio_set_op_attrs(read_bio, op, do_sync);
	if (test_bit(FailFast, &rdev->flags) &&
	    test_bit(R10BIO_FailFast, &r10_bio->state))
	        read_bio->bi_opf |= MD_FAILFAST;
	read_bio->bi_private = r10_bio;

	if (mddev->gendisk)
	        trace_block_bio_remap(bdev_get_queue(read_bio->bi_bdev),
	                              read_bio, disk_devt(mddev->gendisk),
	                              r10_bio->sector);
1225
	generic_make_request(read_bio);
1226 1227 1228
	return;
}

1229 1230
static void raid10_write_one_disk(struct mddev *mddev, struct r10bio *r10_bio,
				  struct bio *bio, bool replacement,
1231
				  int n_copy)
1232
{
M
Mike Christie 已提交
1233
	const int op = bio_op(bio);
J
Jens Axboe 已提交
1234 1235
	const unsigned long do_sync = (bio->bi_opf & REQ_SYNC);
	const unsigned long do_fua = (bio->bi_opf & REQ_FUA);
1236
	unsigned long flags;
1237 1238
	struct blk_plug_cb *cb;
	struct raid10_plug_cb *plug = NULL;
1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303
	struct r10conf *conf = mddev->private;
	struct md_rdev *rdev;
	int devnum = r10_bio->devs[n_copy].devnum;
	struct bio *mbio;

	if (replacement) {
		rdev = conf->mirrors[devnum].replacement;
		if (rdev == NULL) {
			/* Replacement just got moved to main 'rdev' */
			smp_mb();
			rdev = conf->mirrors[devnum].rdev;
		}
	} else
		rdev = conf->mirrors[devnum].rdev;

	mbio = bio_clone_fast(bio, GFP_NOIO, mddev->bio_set);
	if (replacement)
		r10_bio->devs[n_copy].repl_bio = mbio;
	else
		r10_bio->devs[n_copy].bio = mbio;

	mbio->bi_iter.bi_sector	= (r10_bio->devs[n_copy].addr +
				   choose_data_offset(r10_bio, rdev));
	mbio->bi_bdev = rdev->bdev;
	mbio->bi_end_io	= raid10_end_write_request;
	bio_set_op_attrs(mbio, op, do_sync | do_fua);
	if (!replacement && test_bit(FailFast,
				     &conf->mirrors[devnum].rdev->flags)
			 && enough(conf, devnum))
		mbio->bi_opf |= MD_FAILFAST;
	mbio->bi_private = r10_bio;

	if (conf->mddev->gendisk)
		trace_block_bio_remap(bdev_get_queue(mbio->bi_bdev),
				      mbio, disk_devt(conf->mddev->gendisk),
				      r10_bio->sector);
	/* flush_pending_writes() needs access to the rdev so...*/
	mbio->bi_bdev = (void *)rdev;

	atomic_inc(&r10_bio->remaining);

	cb = blk_check_plugged(raid10_unplug, mddev, sizeof(*plug));
	if (cb)
		plug = container_of(cb, struct raid10_plug_cb, cb);
	else
		plug = NULL;
	spin_lock_irqsave(&conf->device_lock, flags);
	if (plug) {
		bio_list_add(&plug->pending, mbio);
		plug->pending_cnt++;
	} else {
		bio_list_add(&conf->pending_bio_list, mbio);
		conf->pending_count++;
	}
	spin_unlock_irqrestore(&conf->device_lock, flags);
	if (!plug)
		md_wakeup_thread(mddev->thread);
}

static void raid10_write_request(struct mddev *mddev, struct bio *bio,
				 struct r10bio *r10_bio)
{
	struct r10conf *conf = mddev->private;
	int i;
	struct md_rdev *blocked_rdev;
1304
	sector_t sectors;
1305
	int max_sectors;
L
Linus Torvalds 已提交
1306

1307 1308
	md_write_start(mddev, bio);

1309 1310 1311 1312 1313 1314 1315
	/*
	 * Register the new request and wait if the reconstruction
	 * thread has put up a bar for new requests.
	 * Continue immediately if no resync is active currently.
	 */
	wait_barrier(conf);

1316
	sectors = r10_bio->sectors;
N
NeilBrown 已提交
1317
	while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
1318 1319
	    bio->bi_iter.bi_sector < conf->reshape_progress &&
	    bio->bi_iter.bi_sector + sectors > conf->reshape_progress) {
1320 1321 1322
		/*
		 * IO spans the reshape position.  Need to wait for reshape to
		 * pass
N
NeilBrown 已提交
1323
		 */
1324
		raid10_log(conf->mddev, "wait reshape");
N
NeilBrown 已提交
1325 1326
		allow_barrier(conf);
		wait_event(conf->wait_barrier,
1327 1328 1329
			   conf->reshape_progress <= bio->bi_iter.bi_sector ||
			   conf->reshape_progress >= bio->bi_iter.bi_sector +
			   sectors);
N
NeilBrown 已提交
1330 1331
		wait_barrier(conf);
	}
1332

N
NeilBrown 已提交
1333 1334
	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
	    (mddev->reshape_backwards
1335 1336 1337 1338
	     ? (bio->bi_iter.bi_sector < conf->reshape_safe &&
		bio->bi_iter.bi_sector + sectors > conf->reshape_progress)
	     : (bio->bi_iter.bi_sector + sectors > conf->reshape_safe &&
		bio->bi_iter.bi_sector < conf->reshape_progress))) {
N
NeilBrown 已提交
1339 1340
		/* Need to update reshape_position in metadata */
		mddev->reshape_position = conf->reshape_progress;
1341 1342
		set_mask_bits(&mddev->sb_flags, 0,
			      BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
N
NeilBrown 已提交
1343
		md_wakeup_thread(mddev->thread);
1344
		raid10_log(conf->mddev, "wait reshape metadata");
N
NeilBrown 已提交
1345
		wait_event(mddev->sb_wait,
1346
			   !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags));
N
NeilBrown 已提交
1347 1348 1349 1350

		conf->reshape_safe = mddev->reshape_position;
	}

1351 1352
	if (conf->pending_count >= max_queued_requests) {
		md_wakeup_thread(mddev->thread);
1353
		raid10_log(mddev, "wait queued");
1354 1355 1356
		wait_event(conf->wait_barrier,
			   conf->pending_count < max_queued_requests);
	}
1357
	/* first select target devices under rcu_lock and
L
Linus Torvalds 已提交
1358 1359
	 * inc refcount on their rdev.  Record them by setting
	 * bios[x] to bio
1360 1361 1362 1363
	 * If there are known/acknowledged bad blocks on any device
	 * on which we have seen a write error, we want to avoid
	 * writing to those blocks.  This potentially requires several
	 * writes to write around the bad blocks.  Each set of writes
1364
	 * gets its own r10_bio with a set of bios attached.
L
Linus Torvalds 已提交
1365
	 */
N
NeilBrown 已提交
1366

1367
	r10_bio->read_slot = -1; /* make sure repl_bio gets freed */
L
Linus Torvalds 已提交
1368
	raid10_find_phys(conf, r10_bio);
1369
retry_write:
1370
	blocked_rdev = NULL;
L
Linus Torvalds 已提交
1371
	rcu_read_lock();
1372 1373
	max_sectors = r10_bio->sectors;

L
Linus Torvalds 已提交
1374 1375
	for (i = 0;  i < conf->copies; i++) {
		int d = r10_bio->devs[i].devnum;
1376
		struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev);
1377 1378
		struct md_rdev *rrdev = rcu_dereference(
			conf->mirrors[d].replacement);
1379 1380
		if (rdev == rrdev)
			rrdev = NULL;
1381 1382 1383 1384 1385
		if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) {
			atomic_inc(&rdev->nr_pending);
			blocked_rdev = rdev;
			break;
		}
1386 1387 1388 1389 1390
		if (rrdev && unlikely(test_bit(Blocked, &rrdev->flags))) {
			atomic_inc(&rrdev->nr_pending);
			blocked_rdev = rrdev;
			break;
		}
1391
		if (rdev && (test_bit(Faulty, &rdev->flags)))
1392
			rdev = NULL;
1393
		if (rrdev && (test_bit(Faulty, &rrdev->flags)))
1394 1395
			rrdev = NULL;

1396
		r10_bio->devs[i].bio = NULL;
1397
		r10_bio->devs[i].repl_bio = NULL;
1398 1399

		if (!rdev && !rrdev) {
1400
			set_bit(R10BIO_Degraded, &r10_bio->state);
1401 1402
			continue;
		}
1403
		if (rdev && test_bit(WriteErrorSeen, &rdev->flags)) {
1404 1405 1406 1407 1408
			sector_t first_bad;
			sector_t dev_sector = r10_bio->devs[i].addr;
			int bad_sectors;
			int is_bad;

1409
			is_bad = is_badblock(rdev, dev_sector, max_sectors,
1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442
					     &first_bad, &bad_sectors);
			if (is_bad < 0) {
				/* Mustn't write here until the bad block
				 * is acknowledged
				 */
				atomic_inc(&rdev->nr_pending);
				set_bit(BlockedBadBlocks, &rdev->flags);
				blocked_rdev = rdev;
				break;
			}
			if (is_bad && first_bad <= dev_sector) {
				/* Cannot write here at all */
				bad_sectors -= (dev_sector - first_bad);
				if (bad_sectors < max_sectors)
					/* Mustn't write more than bad_sectors
					 * to other devices yet
					 */
					max_sectors = bad_sectors;
				/* We don't set R10BIO_Degraded as that
				 * only applies if the disk is missing,
				 * so it might be re-added, and we want to
				 * know to recover this chunk.
				 * In this case the device is here, and the
				 * fact that this chunk is not in-sync is
				 * recorded in the bad block log.
				 */
				continue;
			}
			if (is_bad) {
				int good_sectors = first_bad - dev_sector;
				if (good_sectors < max_sectors)
					max_sectors = good_sectors;
			}
1443
		}
1444 1445 1446 1447
		if (rdev) {
			r10_bio->devs[i].bio = bio;
			atomic_inc(&rdev->nr_pending);
		}
1448 1449 1450 1451
		if (rrdev) {
			r10_bio->devs[i].repl_bio = bio;
			atomic_inc(&rrdev->nr_pending);
		}
L
Linus Torvalds 已提交
1452 1453 1454
	}
	rcu_read_unlock();

1455 1456 1457 1458 1459
	if (unlikely(blocked_rdev)) {
		/* Have to wait for this device to get unblocked, then retry */
		int j;
		int d;

1460
		for (j = 0; j < i; j++) {
1461 1462 1463 1464
			if (r10_bio->devs[j].bio) {
				d = r10_bio->devs[j].devnum;
				rdev_dec_pending(conf->mirrors[d].rdev, mddev);
			}
1465
			if (r10_bio->devs[j].repl_bio) {
1466
				struct md_rdev *rdev;
1467
				d = r10_bio->devs[j].devnum;
1468 1469 1470 1471 1472 1473 1474
				rdev = conf->mirrors[d].replacement;
				if (!rdev) {
					/* Race with remove_disk */
					smp_mb();
					rdev = conf->mirrors[d].rdev;
				}
				rdev_dec_pending(rdev, mddev);
1475 1476
			}
		}
1477
		allow_barrier(conf);
1478
		raid10_log(conf->mddev, "wait rdev %d blocked", blocked_rdev->raid_disk);
1479 1480 1481 1482 1483
		md_wait_for_blocked_rdev(blocked_rdev, mddev);
		wait_barrier(conf);
		goto retry_write;
	}

1484
	if (max_sectors < r10_bio->sectors)
1485
		r10_bio->sectors = max_sectors;
1486 1487 1488 1489 1490 1491 1492 1493

	if (r10_bio->sectors < bio_sectors(bio)) {
		struct bio *split = bio_split(bio, r10_bio->sectors,
					      GFP_NOIO, conf->bio_split);
		bio_chain(split, bio);
		generic_make_request(bio);
		bio = split;
		r10_bio->master_bio = bio;
1494 1495
	}

1496
	atomic_set(&r10_bio->remaining, 1);
1497
	bitmap_startwrite(mddev->bitmap, r10_bio->sector, r10_bio->sectors, 0);
1498

L
Linus Torvalds 已提交
1499
	for (i = 0; i < conf->copies; i++) {
1500
		if (r10_bio->devs[i].bio)
1501
			raid10_write_one_disk(mddev, r10_bio, bio, false, i);
1502
		if (r10_bio->devs[i].repl_bio)
1503
			raid10_write_one_disk(mddev, r10_bio, bio, true, i);
1504
	}
1505
	one_write_done(r10_bio);
K
Kent Overstreet 已提交
1506 1507
}

1508
static void __make_request(struct mddev *mddev, struct bio *bio, int sectors)
1509 1510 1511 1512 1513 1514 1515
{
	struct r10conf *conf = mddev->private;
	struct r10bio *r10_bio;

	r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);

	r10_bio->master_bio = bio;
1516
	r10_bio->sectors = sectors;
1517 1518 1519 1520

	r10_bio->mddev = mddev;
	r10_bio->sector = bio->bi_iter.bi_sector;
	r10_bio->state = 0;
1521
	memset(r10_bio->devs, 0, sizeof(r10_bio->devs[0]) * conf->copies);
1522 1523 1524 1525 1526 1527 1528

	if (bio_data_dir(bio) == READ)
		raid10_read_request(mddev, bio, r10_bio);
	else
		raid10_write_request(mddev, bio, r10_bio);
}

S
Shaohua Li 已提交
1529
static void raid10_make_request(struct mddev *mddev, struct bio *bio)
K
Kent Overstreet 已提交
1530 1531 1532 1533
{
	struct r10conf *conf = mddev->private;
	sector_t chunk_mask = (conf->geo.chunk_mask & conf->prev.chunk_mask);
	int chunk_sects = chunk_mask + 1;
1534
	int sectors = bio_sectors(bio);
K
Kent Overstreet 已提交
1535

J
Jens Axboe 已提交
1536
	if (unlikely(bio->bi_opf & REQ_PREFLUSH)) {
K
Kent Overstreet 已提交
1537 1538 1539 1540
		md_flush_request(mddev, bio);
		return;
	}

1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553
	/*
	 * If this request crosses a chunk boundary, we need to split
	 * it.
	 */
	if (unlikely((bio->bi_iter.bi_sector & chunk_mask) +
		     sectors > chunk_sects
		     && (conf->geo.near_copies < conf->geo.raid_disks
			 || conf->prev.near_copies <
			 conf->prev.raid_disks)))
		sectors = chunk_sects -
			(bio->bi_iter.bi_sector &
			 (chunk_sects - 1));
	__make_request(mddev, bio, sectors);
1554 1555 1556

	/* In case raid10d snuck in to freeze_array */
	wake_up(&conf->wait_barrier);
L
Linus Torvalds 已提交
1557 1558
}

S
Shaohua Li 已提交
1559
static void raid10_status(struct seq_file *seq, struct mddev *mddev)
L
Linus Torvalds 已提交
1560
{
1561
	struct r10conf *conf = mddev->private;
L
Linus Torvalds 已提交
1562 1563
	int i;

1564
	if (conf->geo.near_copies < conf->geo.raid_disks)
1565
		seq_printf(seq, " %dK chunks", mddev->chunk_sectors / 2);
1566 1567 1568 1569 1570
	if (conf->geo.near_copies > 1)
		seq_printf(seq, " %d near-copies", conf->geo.near_copies);
	if (conf->geo.far_copies > 1) {
		if (conf->geo.far_offset)
			seq_printf(seq, " %d offset-copies", conf->geo.far_copies);
1571
		else
1572
			seq_printf(seq, " %d far-copies", conf->geo.far_copies);
1573 1574
		if (conf->geo.far_set_size != conf->geo.raid_disks)
			seq_printf(seq, " %d devices per set", conf->geo.far_set_size);
1575
	}
1576 1577
	seq_printf(seq, " [%d/%d] [", conf->geo.raid_disks,
					conf->geo.raid_disks - mddev->degraded);
1578 1579 1580 1581 1582 1583
	rcu_read_lock();
	for (i = 0; i < conf->geo.raid_disks; i++) {
		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
		seq_printf(seq, "%s", rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_");
	}
	rcu_read_unlock();
L
Linus Torvalds 已提交
1584 1585 1586
	seq_printf(seq, "]");
}

1587 1588 1589 1590 1591
/* check if there are enough drives for
 * every block to appear on atleast one.
 * Don't consider the device numbered 'ignore'
 * as we might be about to remove it.
 */
1592
static int _enough(struct r10conf *conf, int previous, int ignore)
1593 1594
{
	int first = 0;
1595
	int has_enough = 0;
1596 1597 1598 1599 1600 1601 1602 1603
	int disks, ncopies;
	if (previous) {
		disks = conf->prev.raid_disks;
		ncopies = conf->prev.near_copies;
	} else {
		disks = conf->geo.raid_disks;
		ncopies = conf->geo.near_copies;
	}
1604

1605
	rcu_read_lock();
1606 1607 1608
	do {
		int n = conf->copies;
		int cnt = 0;
1609
		int this = first;
1610
		while (n--) {
1611 1612 1613 1614
			struct md_rdev *rdev;
			if (this != ignore &&
			    (rdev = rcu_dereference(conf->mirrors[this].rdev)) &&
			    test_bit(In_sync, &rdev->flags))
1615
				cnt++;
1616
			this = (this+1) % disks;
1617 1618
		}
		if (cnt == 0)
1619
			goto out;
1620
		first = (first + ncopies) % disks;
1621
	} while (first != 0);
1622 1623 1624 1625
	has_enough = 1;
out:
	rcu_read_unlock();
	return has_enough;
1626 1627
}

1628 1629
static int enough(struct r10conf *conf, int ignore)
{
1630 1631 1632 1633 1634 1635 1636
	/* when calling 'enough', both 'prev' and 'geo' must
	 * be stable.
	 * This is ensured if ->reconfig_mutex or ->device_lock
	 * is held.
	 */
	return _enough(conf, 0, ignore) &&
		_enough(conf, 1, ignore);
1637 1638
}

S
Shaohua Li 已提交
1639
static void raid10_error(struct mddev *mddev, struct md_rdev *rdev)
L
Linus Torvalds 已提交
1640 1641
{
	char b[BDEVNAME_SIZE];
1642
	struct r10conf *conf = mddev->private;
1643
	unsigned long flags;
L
Linus Torvalds 已提交
1644 1645 1646 1647 1648 1649 1650

	/*
	 * If it is not operational, then we have already marked it as dead
	 * else if it is the last working disks, ignore the error, let the
	 * next level up know.
	 * else mark the drive as failed
	 */
1651
	spin_lock_irqsave(&conf->device_lock, flags);
1652
	if (test_bit(In_sync, &rdev->flags)
1653
	    && !enough(conf, rdev->raid_disk)) {
L
Linus Torvalds 已提交
1654 1655 1656
		/*
		 * Don't fail the drive, just return an IO error.
		 */
1657
		spin_unlock_irqrestore(&conf->device_lock, flags);
L
Linus Torvalds 已提交
1658
		return;
1659
	}
1660
	if (test_and_clear_bit(In_sync, &rdev->flags))
L
Linus Torvalds 已提交
1661
		mddev->degraded++;
1662 1663 1664 1665
	/*
	 * If recovery is running, make sure it aborts.
	 */
	set_bit(MD_RECOVERY_INTR, &mddev->recovery);
1666
	set_bit(Blocked, &rdev->flags);
1667
	set_bit(Faulty, &rdev->flags);
1668 1669
	set_mask_bits(&mddev->sb_flags, 0,
		      BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
1670
	spin_unlock_irqrestore(&conf->device_lock, flags);
N
NeilBrown 已提交
1671 1672 1673 1674
	pr_crit("md/raid10:%s: Disk failure on %s, disabling device.\n"
		"md/raid10:%s: Operation continuing on %d devices.\n",
		mdname(mddev), bdevname(rdev->bdev, b),
		mdname(mddev), conf->geo.raid_disks - mddev->degraded);
L
Linus Torvalds 已提交
1675 1676
}

1677
static void print_conf(struct r10conf *conf)
L
Linus Torvalds 已提交
1678 1679
{
	int i;
1680
	struct md_rdev *rdev;
L
Linus Torvalds 已提交
1681

N
NeilBrown 已提交
1682
	pr_debug("RAID10 conf printout:\n");
L
Linus Torvalds 已提交
1683
	if (!conf) {
N
NeilBrown 已提交
1684
		pr_debug("(!conf)\n");
L
Linus Torvalds 已提交
1685 1686
		return;
	}
N
NeilBrown 已提交
1687 1688
	pr_debug(" --- wd:%d rd:%d\n", conf->geo.raid_disks - conf->mddev->degraded,
		 conf->geo.raid_disks);
L
Linus Torvalds 已提交
1689

1690 1691
	/* This is only called with ->reconfix_mutex held, so
	 * rcu protection of rdev is not needed */
1692
	for (i = 0; i < conf->geo.raid_disks; i++) {
L
Linus Torvalds 已提交
1693
		char b[BDEVNAME_SIZE];
1694 1695
		rdev = conf->mirrors[i].rdev;
		if (rdev)
N
NeilBrown 已提交
1696 1697 1698 1699
			pr_debug(" disk %d, wo:%d, o:%d, dev:%s\n",
				 i, !test_bit(In_sync, &rdev->flags),
				 !test_bit(Faulty, &rdev->flags),
				 bdevname(rdev->bdev,b));
L
Linus Torvalds 已提交
1700 1701 1702
	}
}

1703
static void close_sync(struct r10conf *conf)
L
Linus Torvalds 已提交
1704
{
1705 1706
	wait_barrier(conf);
	allow_barrier(conf);
L
Linus Torvalds 已提交
1707 1708 1709 1710 1711

	mempool_destroy(conf->r10buf_pool);
	conf->r10buf_pool = NULL;
}

1712
static int raid10_spare_active(struct mddev *mddev)
L
Linus Torvalds 已提交
1713 1714
{
	int i;
1715
	struct r10conf *conf = mddev->private;
1716
	struct raid10_info *tmp;
1717 1718
	int count = 0;
	unsigned long flags;
L
Linus Torvalds 已提交
1719 1720 1721 1722 1723

	/*
	 * Find all non-in_sync disks within the RAID10 configuration
	 * and mark them in_sync
	 */
1724
	for (i = 0; i < conf->geo.raid_disks; i++) {
L
Linus Torvalds 已提交
1725
		tmp = conf->mirrors + i;
1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744
		if (tmp->replacement
		    && tmp->replacement->recovery_offset == MaxSector
		    && !test_bit(Faulty, &tmp->replacement->flags)
		    && !test_and_set_bit(In_sync, &tmp->replacement->flags)) {
			/* Replacement has just become active */
			if (!tmp->rdev
			    || !test_and_clear_bit(In_sync, &tmp->rdev->flags))
				count++;
			if (tmp->rdev) {
				/* Replaced device not technically faulty,
				 * but we need to be sure it gets removed
				 * and never re-added.
				 */
				set_bit(Faulty, &tmp->rdev->flags);
				sysfs_notify_dirent_safe(
					tmp->rdev->sysfs_state);
			}
			sysfs_notify_dirent_safe(tmp->replacement->sysfs_state);
		} else if (tmp->rdev
1745
			   && tmp->rdev->recovery_offset == MaxSector
1746 1747
			   && !test_bit(Faulty, &tmp->rdev->flags)
			   && !test_and_set_bit(In_sync, &tmp->rdev->flags)) {
1748
			count++;
1749
			sysfs_notify_dirent_safe(tmp->rdev->sysfs_state);
L
Linus Torvalds 已提交
1750 1751
		}
	}
1752 1753 1754
	spin_lock_irqsave(&conf->device_lock, flags);
	mddev->degraded -= count;
	spin_unlock_irqrestore(&conf->device_lock, flags);
L
Linus Torvalds 已提交
1755 1756

	print_conf(conf);
1757
	return count;
L
Linus Torvalds 已提交
1758 1759
}

1760
static int raid10_add_disk(struct mddev *mddev, struct md_rdev *rdev)
L
Linus Torvalds 已提交
1761
{
1762
	struct r10conf *conf = mddev->private;
1763
	int err = -EEXIST;
L
Linus Torvalds 已提交
1764
	int mirror;
1765
	int first = 0;
1766
	int last = conf->geo.raid_disks - 1;
L
Linus Torvalds 已提交
1767 1768 1769 1770 1771

	if (mddev->recovery_cp < MaxSector)
		/* only hot-add to in-sync arrays, as recovery is
		 * very different from resync
		 */
1772
		return -EBUSY;
1773
	if (rdev->saved_raid_disk < 0 && !_enough(conf, 1, -1))
1774
		return -EINVAL;
L
Linus Torvalds 已提交
1775

1776 1777 1778
	if (md_integrity_add_rdev(rdev, mddev))
		return -ENXIO;

N
NeilBrown 已提交
1779
	if (rdev->raid_disk >= 0)
1780
		first = last = rdev->raid_disk;
L
Linus Torvalds 已提交
1781

1782
	if (rdev->saved_raid_disk >= first &&
1783 1784 1785
	    conf->mirrors[rdev->saved_raid_disk].rdev == NULL)
		mirror = rdev->saved_raid_disk;
	else
1786
		mirror = first;
1787
	for ( ; mirror <= last ; mirror++) {
1788
		struct raid10_info *p = &conf->mirrors[mirror];
1789 1790
		if (p->recovery_disabled == mddev->recovery_disabled)
			continue;
1791 1792 1793 1794 1795 1796 1797 1798
		if (p->rdev) {
			if (!test_bit(WantReplacement, &p->rdev->flags) ||
			    p->replacement != NULL)
				continue;
			clear_bit(In_sync, &rdev->flags);
			set_bit(Replacement, &rdev->flags);
			rdev->raid_disk = mirror;
			err = 0;
1799 1800 1801
			if (mddev->gendisk)
				disk_stack_limits(mddev->gendisk, rdev->bdev,
						  rdev->data_offset << 9);
1802 1803 1804 1805
			conf->fullsync = 1;
			rcu_assign_pointer(p->replacement, rdev);
			break;
		}
L
Linus Torvalds 已提交
1806

1807 1808 1809
		if (mddev->gendisk)
			disk_stack_limits(mddev->gendisk, rdev->bdev,
					  rdev->data_offset << 9);
L
Linus Torvalds 已提交
1810

1811
		p->head_position = 0;
1812
		p->recovery_disabled = mddev->recovery_disabled - 1;
1813 1814 1815 1816 1817 1818 1819
		rdev->raid_disk = mirror;
		err = 0;
		if (rdev->saved_raid_disk != mirror)
			conf->fullsync = 1;
		rcu_assign_pointer(p->rdev, rdev);
		break;
	}
1820
	if (mddev->queue && blk_queue_discard(bdev_get_queue(rdev->bdev)))
S
Shaohua Li 已提交
1821 1822
		queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, mddev->queue);

L
Linus Torvalds 已提交
1823
	print_conf(conf);
1824
	return err;
L
Linus Torvalds 已提交
1825 1826
}

1827
static int raid10_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
L
Linus Torvalds 已提交
1828
{
1829
	struct r10conf *conf = mddev->private;
L
Linus Torvalds 已提交
1830
	int err = 0;
1831
	int number = rdev->raid_disk;
1832
	struct md_rdev **rdevp;
1833
	struct raid10_info *p = conf->mirrors + number;
L
Linus Torvalds 已提交
1834 1835

	print_conf(conf);
1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847
	if (rdev == p->rdev)
		rdevp = &p->rdev;
	else if (rdev == p->replacement)
		rdevp = &p->replacement;
	else
		return 0;

	if (test_bit(In_sync, &rdev->flags) ||
	    atomic_read(&rdev->nr_pending)) {
		err = -EBUSY;
		goto abort;
	}
1848
	/* Only remove non-faulty devices if recovery
1849 1850 1851 1852
	 * is not possible.
	 */
	if (!test_bit(Faulty, &rdev->flags) &&
	    mddev->recovery_disabled != p->recovery_disabled &&
1853
	    (!p->replacement || p->replacement == rdev) &&
1854
	    number < conf->geo.raid_disks &&
1855 1856 1857
	    enough(conf, -1)) {
		err = -EBUSY;
		goto abort;
L
Linus Torvalds 已提交
1858
	}
1859
	*rdevp = NULL;
1860 1861 1862 1863 1864 1865 1866 1867 1868 1869
	if (!test_bit(RemoveSynchronized, &rdev->flags)) {
		synchronize_rcu();
		if (atomic_read(&rdev->nr_pending)) {
			/* lost the race, try later */
			err = -EBUSY;
			*rdevp = rdev;
			goto abort;
		}
	}
	if (p->replacement) {
1870 1871 1872 1873 1874 1875 1876
		/* We must have just cleared 'rdev' */
		p->rdev = p->replacement;
		clear_bit(Replacement, &p->replacement->flags);
		smp_mb(); /* Make sure other CPUs may see both as identical
			   * but will never see neither -- if they are careful.
			   */
		p->replacement = NULL;
1877
	}
1878

1879
	clear_bit(WantReplacement, &rdev->flags);
1880 1881
	err = md_integrity_register(mddev);

L
Linus Torvalds 已提交
1882 1883 1884 1885 1886 1887
abort:

	print_conf(conf);
	return err;
}

1888
static void __end_sync_read(struct r10bio *r10_bio, struct bio *bio, int d)
L
Linus Torvalds 已提交
1889
{
1890
	struct r10conf *conf = r10_bio->mddev->private;
1891

1892
	if (!bio->bi_error)
1893
		set_bit(R10BIO_Uptodate, &r10_bio->state);
1894 1895 1896 1897
	else
		/* The write handler will notice the lack of
		 * R10BIO_Uptodate and record any errors etc
		 */
1898 1899
		atomic_add(r10_bio->sectors,
			   &conf->mirrors[d].rdev->corrected_errors);
L
Linus Torvalds 已提交
1900 1901 1902 1903

	/* for reconstruct, we always reschedule after a read.
	 * for resync, only after all reads
	 */
1904
	rdev_dec_pending(conf->mirrors[d].rdev, conf->mddev);
L
Linus Torvalds 已提交
1905 1906 1907 1908 1909 1910 1911 1912 1913
	if (test_bit(R10BIO_IsRecover, &r10_bio->state) ||
	    atomic_dec_and_test(&r10_bio->remaining)) {
		/* we have read all the blocks,
		 * do the comparison in process context in raid10d
		 */
		reschedule_retry(r10_bio);
	}
}

1914 1915
static void end_sync_read(struct bio *bio)
{
1916
	struct r10bio *r10_bio = get_resync_r10bio(bio);
1917 1918 1919 1920 1921 1922 1923 1924
	struct r10conf *conf = r10_bio->mddev->private;
	int d = find_bio_disk(conf, r10_bio, bio, NULL, NULL);

	__end_sync_read(r10_bio, bio, d);
}

static void end_reshape_read(struct bio *bio)
{
1925
	/* reshape read bio isn't allocated from r10buf_pool */
1926 1927 1928 1929 1930
	struct r10bio *r10_bio = bio->bi_private;

	__end_sync_read(r10_bio, bio, r10_bio->read_slot);
}

1931
static void end_sync_request(struct r10bio *r10_bio)
L
Linus Torvalds 已提交
1932
{
1933
	struct mddev *mddev = r10_bio->mddev;
1934

L
Linus Torvalds 已提交
1935 1936 1937
	while (atomic_dec_and_test(&r10_bio->remaining)) {
		if (r10_bio->master_bio == NULL) {
			/* the primary of several recovery bios */
1938
			sector_t s = r10_bio->sectors;
1939 1940
			if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
			    test_bit(R10BIO_WriteError, &r10_bio->state))
1941 1942 1943
				reschedule_retry(r10_bio);
			else
				put_buf(r10_bio);
1944
			md_done_sync(mddev, s, 1);
L
Linus Torvalds 已提交
1945 1946
			break;
		} else {
1947
			struct r10bio *r10_bio2 = (struct r10bio *)r10_bio->master_bio;
1948 1949
			if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
			    test_bit(R10BIO_WriteError, &r10_bio->state))
1950 1951 1952
				reschedule_retry(r10_bio);
			else
				put_buf(r10_bio);
L
Linus Torvalds 已提交
1953 1954 1955 1956 1957
			r10_bio = r10_bio2;
		}
	}
}

1958
static void end_sync_write(struct bio *bio)
1959
{
1960
	struct r10bio *r10_bio = get_resync_r10bio(bio);
1961
	struct mddev *mddev = r10_bio->mddev;
1962
	struct r10conf *conf = mddev->private;
1963 1964 1965 1966
	int d;
	sector_t first_bad;
	int bad_sectors;
	int slot;
1967
	int repl;
1968
	struct md_rdev *rdev = NULL;
1969

1970 1971 1972
	d = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
	if (repl)
		rdev = conf->mirrors[d].replacement;
1973
	else
1974
		rdev = conf->mirrors[d].rdev;
1975

1976
	if (bio->bi_error) {
1977 1978 1979 1980
		if (repl)
			md_error(mddev, rdev);
		else {
			set_bit(WriteErrorSeen, &rdev->flags);
1981 1982 1983
			if (!test_and_set_bit(WantReplacement, &rdev->flags))
				set_bit(MD_RECOVERY_NEEDED,
					&rdev->mddev->recovery);
1984 1985 1986
			set_bit(R10BIO_WriteError, &r10_bio->state);
		}
	} else if (is_badblock(rdev,
1987 1988 1989 1990 1991
			     r10_bio->devs[slot].addr,
			     r10_bio->sectors,
			     &first_bad, &bad_sectors))
		set_bit(R10BIO_MadeGood, &r10_bio->state);

1992
	rdev_dec_pending(rdev, mddev);
1993 1994 1995 1996

	end_sync_request(r10_bio);
}

L
Linus Torvalds 已提交
1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012
/*
 * Note: sync and recover and handled very differently for raid10
 * This code is for resync.
 * For resync, we read through virtual addresses and read all blocks.
 * If there is any error, we schedule a write.  The lowest numbered
 * drive is authoritative.
 * However requests come for physical address, so we need to map.
 * For every physical address there are raid_disks/copies virtual addresses,
 * which is always are least one, but is not necessarly an integer.
 * This means that a physical address can span multiple chunks, so we may
 * have to submit multiple io requests for a single sync request.
 */
/*
 * We check if all blocks are in-sync and only write to blocks that
 * aren't in sync
 */
2013
static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio)
L
Linus Torvalds 已提交
2014
{
2015
	struct r10conf *conf = mddev->private;
L
Linus Torvalds 已提交
2016 2017
	int i, first;
	struct bio *tbio, *fbio;
2018
	int vcnt;
2019
	struct page **tpages, **fpages;
L
Linus Torvalds 已提交
2020 2021 2022 2023 2024

	atomic_set(&r10_bio->remaining, 1);

	/* find the first device with a block */
	for (i=0; i<conf->copies; i++)
2025
		if (!r10_bio->devs[i].bio->bi_error)
L
Linus Torvalds 已提交
2026 2027 2028 2029 2030 2031 2032
			break;

	if (i == conf->copies)
		goto done;

	first = i;
	fbio = r10_bio->devs[i].bio;
2033 2034
	fbio->bi_iter.bi_size = r10_bio->sectors << 9;
	fbio->bi_iter.bi_idx = 0;
2035
	fpages = get_resync_pages(fbio)->pages;
L
Linus Torvalds 已提交
2036

2037
	vcnt = (r10_bio->sectors + (PAGE_SIZE >> 9) - 1) >> (PAGE_SHIFT - 9);
L
Linus Torvalds 已提交
2038
	/* now find blocks with errors */
2039 2040
	for (i=0 ; i < conf->copies ; i++) {
		int  j, d;
2041
		struct md_rdev *rdev;
2042
		struct resync_pages *rp;
L
Linus Torvalds 已提交
2043 2044

		tbio = r10_bio->devs[i].bio;
2045 2046 2047 2048

		if (tbio->bi_end_io != end_sync_read)
			continue;
		if (i == first)
L
Linus Torvalds 已提交
2049
			continue;
2050 2051

		tpages = get_resync_pages(tbio)->pages;
2052 2053
		d = r10_bio->devs[i].devnum;
		rdev = conf->mirrors[d].rdev;
2054
		if (!r10_bio->devs[i].bio->bi_error) {
2055 2056 2057 2058
			/* We know that the bi_io_vec layout is the same for
			 * both 'first' and 'i', so we just compare them.
			 * All vec entries are PAGE_SIZE;
			 */
2059 2060 2061 2062 2063
			int sectors = r10_bio->sectors;
			for (j = 0; j < vcnt; j++) {
				int len = PAGE_SIZE;
				if (sectors < (len / 512))
					len = sectors * 512;
2064 2065
				if (memcmp(page_address(fpages[j]),
					   page_address(tpages[j]),
2066
					   len))
2067
					break;
2068 2069
				sectors -= len/512;
			}
2070 2071
			if (j == vcnt)
				continue;
2072
			atomic64_add(r10_bio->sectors, &mddev->resync_mismatches);
2073 2074 2075
			if (test_bit(MD_RECOVERY_CHECK, &mddev->recovery))
				/* Don't fix anything. */
				continue;
2076 2077 2078 2079
		} else if (test_bit(FailFast, &rdev->flags)) {
			/* Just give up on this device */
			md_error(rdev->mddev, rdev);
			continue;
2080
		}
2081 2082
		/* Ok, we need to write this bio, either to correct an
		 * inconsistency or to correct an unreadable block.
L
Linus Torvalds 已提交
2083 2084 2085
		 * First we need to fixup bv_offset, bv_len and
		 * bi_vecs, as the read request might have corrupted these
		 */
2086
		rp = get_resync_pages(tbio);
K
Kent Overstreet 已提交
2087 2088
		bio_reset(tbio);

L
Linus Torvalds 已提交
2089
		tbio->bi_vcnt = vcnt;
2090
		tbio->bi_iter.bi_size = fbio->bi_iter.bi_size;
2091 2092
		rp->raid_bio = r10_bio;
		tbio->bi_private = rp;
2093
		tbio->bi_iter.bi_sector = r10_bio->devs[i].addr;
L
Linus Torvalds 已提交
2094
		tbio->bi_end_io = end_sync_write;
M
Mike Christie 已提交
2095
		bio_set_op_attrs(tbio, REQ_OP_WRITE, 0);
L
Linus Torvalds 已提交
2096

2097 2098
		bio_copy_data(tbio, fbio);

L
Linus Torvalds 已提交
2099 2100
		atomic_inc(&conf->mirrors[d].rdev->nr_pending);
		atomic_inc(&r10_bio->remaining);
2101
		md_sync_acct(conf->mirrors[d].rdev->bdev, bio_sectors(tbio));
L
Linus Torvalds 已提交
2102

2103 2104
		if (test_bit(FailFast, &conf->mirrors[d].rdev->flags))
			tbio->bi_opf |= MD_FAILFAST;
2105
		tbio->bi_iter.bi_sector += conf->mirrors[d].rdev->data_offset;
L
Linus Torvalds 已提交
2106 2107 2108 2109
		tbio->bi_bdev = conf->mirrors[d].rdev->bdev;
		generic_make_request(tbio);
	}

2110 2111 2112 2113
	/* Now write out to any replacement devices
	 * that are active
	 */
	for (i = 0; i < conf->copies; i++) {
2114
		int d;
2115 2116 2117 2118 2119 2120

		tbio = r10_bio->devs[i].repl_bio;
		if (!tbio || !tbio->bi_end_io)
			continue;
		if (r10_bio->devs[i].bio->bi_end_io != end_sync_write
		    && r10_bio->devs[i].bio != fbio)
2121
			bio_copy_data(tbio, fbio);
2122 2123 2124
		d = r10_bio->devs[i].devnum;
		atomic_inc(&r10_bio->remaining);
		md_sync_acct(conf->mirrors[d].replacement->bdev,
2125
			     bio_sectors(tbio));
2126 2127 2128
		generic_make_request(tbio);
	}

L
Linus Torvalds 已提交
2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145
done:
	if (atomic_dec_and_test(&r10_bio->remaining)) {
		md_done_sync(mddev, r10_bio->sectors, 1);
		put_buf(r10_bio);
	}
}

/*
 * Now for the recovery code.
 * Recovery happens across physical sectors.
 * We recover all non-is_sync drives by finding the virtual address of
 * each, and then choose a working drive that also has that virt address.
 * There is a separate r10_bio for each non-in_sync drive.
 * Only the first two slots are in use. The first for reading,
 * The second for writing.
 *
 */
2146
static void fix_recovery_read_error(struct r10bio *r10_bio)
2147 2148 2149 2150 2151 2152 2153 2154
{
	/* We got a read error during recovery.
	 * We repeat the read in smaller page-sized sections.
	 * If a read succeeds, write it to the new device or record
	 * a bad block if we cannot.
	 * If a read fails, record a bad block on both old and
	 * new devices.
	 */
2155
	struct mddev *mddev = r10_bio->mddev;
2156
	struct r10conf *conf = mddev->private;
2157 2158 2159 2160 2161 2162
	struct bio *bio = r10_bio->devs[0].bio;
	sector_t sect = 0;
	int sectors = r10_bio->sectors;
	int idx = 0;
	int dr = r10_bio->devs[0].devnum;
	int dw = r10_bio->devs[1].devnum;
2163
	struct page **pages = get_resync_pages(bio)->pages;
2164 2165 2166

	while (sectors) {
		int s = sectors;
2167
		struct md_rdev *rdev;
2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178
		sector_t addr;
		int ok;

		if (s > (PAGE_SIZE>>9))
			s = PAGE_SIZE >> 9;

		rdev = conf->mirrors[dr].rdev;
		addr = r10_bio->devs[0].addr + sect,
		ok = sync_page_io(rdev,
				  addr,
				  s << 9,
2179
				  pages[idx],
M
Mike Christie 已提交
2180
				  REQ_OP_READ, 0, false);
2181 2182 2183 2184 2185 2186
		if (ok) {
			rdev = conf->mirrors[dw].rdev;
			addr = r10_bio->devs[1].addr + sect;
			ok = sync_page_io(rdev,
					  addr,
					  s << 9,
2187
					  pages[idx],
M
Mike Christie 已提交
2188
					  REQ_OP_WRITE, 0, false);
2189
			if (!ok) {
2190
				set_bit(WriteErrorSeen, &rdev->flags);
2191 2192 2193 2194 2195
				if (!test_and_set_bit(WantReplacement,
						      &rdev->flags))
					set_bit(MD_RECOVERY_NEEDED,
						&rdev->mddev->recovery);
			}
2196 2197 2198 2199 2200 2201 2202 2203 2204 2205
		}
		if (!ok) {
			/* We don't worry if we cannot set a bad block -
			 * it really is bad so there is no loss in not
			 * recording it yet
			 */
			rdev_set_badblocks(rdev, addr, s, 0);

			if (rdev != conf->mirrors[dw].rdev) {
				/* need bad block on destination too */
2206
				struct md_rdev *rdev2 = conf->mirrors[dw].rdev;
2207 2208 2209 2210
				addr = r10_bio->devs[1].addr + sect;
				ok = rdev_set_badblocks(rdev2, addr, s, 0);
				if (!ok) {
					/* just abort the recovery */
N
NeilBrown 已提交
2211 2212
					pr_notice("md/raid10:%s: recovery aborted due to read error\n",
						  mdname(mddev));
2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227

					conf->mirrors[dw].recovery_disabled
						= mddev->recovery_disabled;
					set_bit(MD_RECOVERY_INTR,
						&mddev->recovery);
					break;
				}
			}
		}

		sectors -= s;
		sect += s;
		idx++;
	}
}
L
Linus Torvalds 已提交
2228

2229
static void recovery_request_write(struct mddev *mddev, struct r10bio *r10_bio)
L
Linus Torvalds 已提交
2230
{
2231
	struct r10conf *conf = mddev->private;
2232
	int d;
2233
	struct bio *wbio, *wbio2;
L
Linus Torvalds 已提交
2234

2235 2236 2237 2238 2239 2240
	if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) {
		fix_recovery_read_error(r10_bio);
		end_sync_request(r10_bio);
		return;
	}

2241 2242
	/*
	 * share the pages with the first bio
L
Linus Torvalds 已提交
2243 2244 2245
	 * and submit the write request
	 */
	d = r10_bio->devs[1].devnum;
2246 2247
	wbio = r10_bio->devs[1].bio;
	wbio2 = r10_bio->devs[1].repl_bio;
2248 2249 2250 2251 2252 2253
	/* Need to test wbio2->bi_end_io before we call
	 * generic_make_request as if the former is NULL,
	 * the latter is free to free wbio2.
	 */
	if (wbio2 && !wbio2->bi_end_io)
		wbio2 = NULL;
2254 2255
	if (wbio->bi_end_io) {
		atomic_inc(&conf->mirrors[d].rdev->nr_pending);
2256
		md_sync_acct(conf->mirrors[d].rdev->bdev, bio_sectors(wbio));
2257 2258
		generic_make_request(wbio);
	}
2259
	if (wbio2) {
2260 2261
		atomic_inc(&conf->mirrors[d].replacement->nr_pending);
		md_sync_acct(conf->mirrors[d].replacement->bdev,
2262
			     bio_sectors(wbio2));
2263 2264
		generic_make_request(wbio2);
	}
L
Linus Torvalds 已提交
2265 2266
}

2267 2268 2269 2270 2271 2272
/*
 * Used by fix_read_error() to decay the per rdev read_errors.
 * We halve the read error count for every hour that has elapsed
 * since the last recorded read error.
 *
 */
2273
static void check_decay_read_errors(struct mddev *mddev, struct md_rdev *rdev)
2274
{
2275
	long cur_time_mon;
2276 2277 2278
	unsigned long hours_since_last;
	unsigned int read_errors = atomic_read(&rdev->read_errors);

2279
	cur_time_mon = ktime_get_seconds();
2280

2281
	if (rdev->last_read_error == 0) {
2282 2283 2284 2285 2286
		/* first time we've seen a read error */
		rdev->last_read_error = cur_time_mon;
		return;
	}

2287 2288
	hours_since_last = (long)(cur_time_mon -
			    rdev->last_read_error) / 3600;
2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302

	rdev->last_read_error = cur_time_mon;

	/*
	 * if hours_since_last is > the number of bits in read_errors
	 * just set read errors to 0. We do this to avoid
	 * overflowing the shift of read_errors by hours_since_last.
	 */
	if (hours_since_last >= 8 * sizeof(read_errors))
		atomic_set(&rdev->read_errors, 0);
	else
		atomic_set(&rdev->read_errors, read_errors >> hours_since_last);
}

2303
static int r10_sync_page_io(struct md_rdev *rdev, sector_t sector,
2304 2305 2306 2307 2308 2309 2310 2311
			    int sectors, struct page *page, int rw)
{
	sector_t first_bad;
	int bad_sectors;

	if (is_badblock(rdev, sector, sectors, &first_bad, &bad_sectors)
	    && (rw == READ || test_bit(WriteErrorSeen, &rdev->flags)))
		return -1;
M
Mike Christie 已提交
2312
	if (sync_page_io(rdev, sector, sectors << 9, page, rw, 0, false))
2313 2314
		/* success */
		return 1;
2315
	if (rw == WRITE) {
2316
		set_bit(WriteErrorSeen, &rdev->flags);
2317 2318 2319 2320
		if (!test_and_set_bit(WantReplacement, &rdev->flags))
			set_bit(MD_RECOVERY_NEEDED,
				&rdev->mddev->recovery);
	}
2321 2322 2323 2324 2325 2326
	/* need to record an error - either for the block or the device */
	if (!rdev_set_badblocks(rdev, sector, sectors, 0))
		md_error(rdev->mddev, rdev);
	return 0;
}

L
Linus Torvalds 已提交
2327 2328 2329 2330 2331
/*
 * This is a kernel thread which:
 *
 *	1.	Retries failed read operations on working mirrors.
 *	2.	Updates the raid superblock when problems encounter.
2332
 *	3.	Performs writes following reads for array synchronising.
L
Linus Torvalds 已提交
2333 2334
 */

2335
static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10bio *r10_bio)
2336 2337 2338
{
	int sect = 0; /* Offset from r10_bio->sector */
	int sectors = r10_bio->sectors;
2339
	struct md_rdev*rdev;
2340
	int max_read_errors = atomic_read(&mddev->max_corr_read_errors);
2341
	int d = r10_bio->devs[r10_bio->read_slot].devnum;
2342

2343 2344 2345 2346
	/* still own a reference to this rdev, so it cannot
	 * have been cleared recently.
	 */
	rdev = conf->mirrors[d].rdev;
2347

2348 2349 2350 2351
	if (test_bit(Faulty, &rdev->flags))
		/* drive has already been failed, just ignore any
		   more fix_read_error() attempts */
		return;
2352

2353 2354 2355 2356 2357
	check_decay_read_errors(mddev, rdev);
	atomic_inc(&rdev->read_errors);
	if (atomic_read(&rdev->read_errors) > max_read_errors) {
		char b[BDEVNAME_SIZE];
		bdevname(rdev->bdev, b);
2358

N
NeilBrown 已提交
2359 2360 2361 2362 2363
		pr_notice("md/raid10:%s: %s: Raid device exceeded read_error threshold [cur %d:max %d]\n",
			  mdname(mddev), b,
			  atomic_read(&rdev->read_errors), max_read_errors);
		pr_notice("md/raid10:%s: %s: Failing raid device\n",
			  mdname(mddev), b);
2364
		md_error(mddev, rdev);
2365
		r10_bio->devs[r10_bio->read_slot].bio = IO_BLOCKED;
2366
		return;
2367 2368
	}

2369 2370 2371 2372 2373 2374 2375 2376 2377 2378 2379
	while(sectors) {
		int s = sectors;
		int sl = r10_bio->read_slot;
		int success = 0;
		int start;

		if (s > (PAGE_SIZE>>9))
			s = PAGE_SIZE >> 9;

		rcu_read_lock();
		do {
2380 2381 2382
			sector_t first_bad;
			int bad_sectors;

2383
			d = r10_bio->devs[sl].devnum;
2384 2385
			rdev = rcu_dereference(conf->mirrors[d].rdev);
			if (rdev &&
2386
			    test_bit(In_sync, &rdev->flags) &&
2387
			    !test_bit(Faulty, &rdev->flags) &&
2388 2389
			    is_badblock(rdev, r10_bio->devs[sl].addr + sect, s,
					&first_bad, &bad_sectors) == 0) {
2390 2391
				atomic_inc(&rdev->nr_pending);
				rcu_read_unlock();
2392
				success = sync_page_io(rdev,
2393
						       r10_bio->devs[sl].addr +
J
Jonathan Brassow 已提交
2394
						       sect,
2395
						       s<<9,
M
Mike Christie 已提交
2396 2397
						       conf->tmppage,
						       REQ_OP_READ, 0, false);
2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409
				rdev_dec_pending(rdev, mddev);
				rcu_read_lock();
				if (success)
					break;
			}
			sl++;
			if (sl == conf->copies)
				sl = 0;
		} while (!success && sl != r10_bio->read_slot);
		rcu_read_unlock();

		if (!success) {
2410 2411 2412 2413
			/* Cannot read from anywhere, just mark the block
			 * as bad on the first device to discourage future
			 * reads.
			 */
2414
			int dn = r10_bio->devs[r10_bio->read_slot].devnum;
2415 2416 2417 2418 2419 2420
			rdev = conf->mirrors[dn].rdev;

			if (!rdev_set_badblocks(
				    rdev,
				    r10_bio->devs[r10_bio->read_slot].addr
				    + sect,
2421
				    s, 0)) {
2422
				md_error(mddev, rdev);
2423 2424 2425
				r10_bio->devs[r10_bio->read_slot].bio
					= IO_BLOCKED;
			}
2426 2427 2428 2429 2430 2431 2432
			break;
		}

		start = sl;
		/* write it back and re-read */
		rcu_read_lock();
		while (sl != r10_bio->read_slot) {
2433
			char b[BDEVNAME_SIZE];
2434

2435 2436 2437 2438 2439
			if (sl==0)
				sl = conf->copies;
			sl--;
			d = r10_bio->devs[sl].devnum;
			rdev = rcu_dereference(conf->mirrors[d].rdev);
2440
			if (!rdev ||
2441
			    test_bit(Faulty, &rdev->flags) ||
2442 2443 2444 2445 2446
			    !test_bit(In_sync, &rdev->flags))
				continue;

			atomic_inc(&rdev->nr_pending);
			rcu_read_unlock();
2447 2448 2449
			if (r10_sync_page_io(rdev,
					     r10_bio->devs[sl].addr +
					     sect,
2450
					     s, conf->tmppage, WRITE)
2451 2452
			    == 0) {
				/* Well, this device is dead */
N
NeilBrown 已提交
2453 2454 2455 2456 2457 2458 2459 2460 2461 2462
				pr_notice("md/raid10:%s: read correction write failed (%d sectors at %llu on %s)\n",
					  mdname(mddev), s,
					  (unsigned long long)(
						  sect +
						  choose_data_offset(r10_bio,
								     rdev)),
					  bdevname(rdev->bdev, b));
				pr_notice("md/raid10:%s: %s: failing drive\n",
					  mdname(mddev),
					  bdevname(rdev->bdev, b));
2463
			}
2464 2465
			rdev_dec_pending(rdev, mddev);
			rcu_read_lock();
2466 2467 2468
		}
		sl = start;
		while (sl != r10_bio->read_slot) {
2469
			char b[BDEVNAME_SIZE];
2470

2471 2472 2473 2474 2475
			if (sl==0)
				sl = conf->copies;
			sl--;
			d = r10_bio->devs[sl].devnum;
			rdev = rcu_dereference(conf->mirrors[d].rdev);
2476
			if (!rdev ||
2477
			    test_bit(Faulty, &rdev->flags) ||
2478 2479
			    !test_bit(In_sync, &rdev->flags))
				continue;
2480

2481 2482
			atomic_inc(&rdev->nr_pending);
			rcu_read_unlock();
2483 2484 2485
			switch (r10_sync_page_io(rdev,
					     r10_bio->devs[sl].addr +
					     sect,
2486
					     s, conf->tmppage,
2487 2488
						 READ)) {
			case 0:
2489
				/* Well, this device is dead */
N
NeilBrown 已提交
2490
				pr_notice("md/raid10:%s: unable to read back corrected sectors (%d sectors at %llu on %s)\n",
2491 2492
				       mdname(mddev), s,
				       (unsigned long long)(
2493 2494
					       sect +
					       choose_data_offset(r10_bio, rdev)),
2495
				       bdevname(rdev->bdev, b));
N
NeilBrown 已提交
2496
				pr_notice("md/raid10:%s: %s: failing drive\n",
2497 2498
				       mdname(mddev),
				       bdevname(rdev->bdev, b));
2499 2500
				break;
			case 1:
N
NeilBrown 已提交
2501
				pr_info("md/raid10:%s: read error corrected (%d sectors at %llu on %s)\n",
2502 2503
				       mdname(mddev), s,
				       (unsigned long long)(
2504 2505
					       sect +
					       choose_data_offset(r10_bio, rdev)),
2506 2507
				       bdevname(rdev->bdev, b));
				atomic_add(s, &rdev->corrected_errors);
2508
			}
2509 2510 2511

			rdev_dec_pending(rdev, mddev);
			rcu_read_lock();
2512 2513 2514 2515 2516 2517 2518 2519
		}
		rcu_read_unlock();

		sectors -= s;
		sect += s;
	}
}

2520
static int narrow_write_error(struct r10bio *r10_bio, int i)
2521 2522
{
	struct bio *bio = r10_bio->master_bio;
2523
	struct mddev *mddev = r10_bio->mddev;
2524
	struct r10conf *conf = mddev->private;
2525
	struct md_rdev *rdev = conf->mirrors[r10_bio->devs[i].devnum].rdev;
2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545
	/* bio has the data to be written to slot 'i' where
	 * we just recently had a write error.
	 * We repeatedly clone the bio and trim down to one block,
	 * then try the write.  Where the write fails we record
	 * a bad block.
	 * It is conceivable that the bio doesn't exactly align with
	 * blocks.  We must handle this.
	 *
	 * We currently own a reference to the rdev.
	 */

	int block_sectors;
	sector_t sector;
	int sectors;
	int sect_to_write = r10_bio->sectors;
	int ok = 1;

	if (rdev->badblocks.shift < 0)
		return 0;

2546 2547
	block_sectors = roundup(1 << rdev->badblocks.shift,
				bdev_logical_block_size(rdev->bdev) >> 9);
2548 2549 2550 2551 2552 2553 2554
	sector = r10_bio->sector;
	sectors = ((r10_bio->sector + block_sectors)
		   & ~(sector_t)(block_sectors - 1))
		- sector;

	while (sect_to_write) {
		struct bio *wbio;
2555
		sector_t wsector;
2556 2557 2558
		if (sectors > sect_to_write)
			sectors = sect_to_write;
		/* Write at 'sector' for 'sectors' */
2559
		wbio = bio_clone_fast(bio, GFP_NOIO, mddev->bio_set);
2560
		bio_trim(wbio, sector - bio->bi_iter.bi_sector, sectors);
2561 2562 2563
		wsector = r10_bio->devs[i].addr + (sector - r10_bio->sector);
		wbio->bi_iter.bi_sector = wsector +
				   choose_data_offset(r10_bio, rdev);
2564
		wbio->bi_bdev = rdev->bdev;
M
Mike Christie 已提交
2565
		bio_set_op_attrs(wbio, REQ_OP_WRITE, 0);
2566 2567

		if (submit_bio_wait(wbio) < 0)
2568
			/* Failure! */
2569
			ok = rdev_set_badblocks(rdev, wsector,
2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580
						sectors, 0)
				&& ok;

		bio_put(wbio);
		sect_to_write -= sectors;
		sector += sectors;
		sectors = block_sectors;
	}
	return ok;
}

2581
static void handle_read_error(struct mddev *mddev, struct r10bio *r10_bio)
2582 2583 2584
{
	int slot = r10_bio->read_slot;
	struct bio *bio;
2585
	struct r10conf *conf = mddev->private;
2586
	struct md_rdev *rdev = r10_bio->devs[slot].rdev;
2587 2588
	dev_t bio_dev;
	sector_t bio_last_sector;
2589 2590 2591 2592 2593 2594 2595 2596 2597

	/* we got a read error. Maybe the drive is bad.  Maybe just
	 * the block and we can fix it.
	 * We freeze all other IO, and try reading the block from
	 * other devices.  When we find one, we re-write
	 * and check it that fixes the read error.
	 * This is all done synchronously while the array is
	 * frozen.
	 */
2598
	bio = r10_bio->devs[slot].bio;
2599 2600
	bio_dev = bio->bi_bdev->bd_dev;
	bio_last_sector = r10_bio->devs[slot].addr + rdev->data_offset + r10_bio->sectors;
2601 2602 2603
	bio_put(bio);
	r10_bio->devs[slot].bio = NULL;

2604 2605 2606
	if (mddev->ro)
		r10_bio->devs[slot].bio = IO_BLOCKED;
	else if (!test_bit(FailFast, &rdev->flags)) {
2607
		freeze_array(conf, 1);
2608 2609
		fix_read_error(conf, mddev, r10_bio);
		unfreeze_array(conf);
2610
	} else
2611
		md_error(mddev, rdev);
2612

2613
	rdev_dec_pending(rdev, mddev);
2614 2615 2616
	allow_barrier(conf);
	r10_bio->state = 0;
	raid10_read_request(mddev, r10_bio->master_bio, r10_bio);
2617 2618
}

2619
static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio)
2620 2621 2622 2623
{
	/* Some sort of write request has finished and it
	 * succeeded in writing where we thought there was a
	 * bad block.  So forget the bad block.
2624 2625
	 * Or possibly if failed and we need to record
	 * a bad block.
2626 2627
	 */
	int m;
2628
	struct md_rdev *rdev;
2629 2630 2631

	if (test_bit(R10BIO_IsSync, &r10_bio->state) ||
	    test_bit(R10BIO_IsRecover, &r10_bio->state)) {
2632 2633 2634 2635 2636
		for (m = 0; m < conf->copies; m++) {
			int dev = r10_bio->devs[m].devnum;
			rdev = conf->mirrors[dev].rdev;
			if (r10_bio->devs[m].bio == NULL)
				continue;
2637
			if (!r10_bio->devs[m].bio->bi_error) {
2638 2639 2640
				rdev_clear_badblocks(
					rdev,
					r10_bio->devs[m].addr,
2641
					r10_bio->sectors, 0);
2642 2643 2644 2645 2646 2647
			} else {
				if (!rdev_set_badblocks(
					    rdev,
					    r10_bio->devs[m].addr,
					    r10_bio->sectors, 0))
					md_error(conf->mddev, rdev);
2648
			}
2649 2650 2651
			rdev = conf->mirrors[dev].replacement;
			if (r10_bio->devs[m].repl_bio == NULL)
				continue;
2652 2653

			if (!r10_bio->devs[m].repl_bio->bi_error) {
2654 2655 2656
				rdev_clear_badblocks(
					rdev,
					r10_bio->devs[m].addr,
2657
					r10_bio->sectors, 0);
2658 2659 2660 2661 2662 2663 2664
			} else {
				if (!rdev_set_badblocks(
					    rdev,
					    r10_bio->devs[m].addr,
					    r10_bio->sectors, 0))
					md_error(conf->mddev, rdev);
			}
2665
		}
2666 2667
		put_buf(r10_bio);
	} else {
2668
		bool fail = false;
2669 2670 2671 2672 2673
		for (m = 0; m < conf->copies; m++) {
			int dev = r10_bio->devs[m].devnum;
			struct bio *bio = r10_bio->devs[m].bio;
			rdev = conf->mirrors[dev].rdev;
			if (bio == IO_MADE_GOOD) {
2674 2675 2676
				rdev_clear_badblocks(
					rdev,
					r10_bio->devs[m].addr,
2677
					r10_bio->sectors, 0);
2678
				rdev_dec_pending(rdev, conf->mddev);
2679
			} else if (bio != NULL && bio->bi_error) {
2680
				fail = true;
2681 2682 2683 2684 2685 2686
				if (!narrow_write_error(r10_bio, m)) {
					md_error(conf->mddev, rdev);
					set_bit(R10BIO_Degraded,
						&r10_bio->state);
				}
				rdev_dec_pending(rdev, conf->mddev);
2687
			}
2688 2689
			bio = r10_bio->devs[m].repl_bio;
			rdev = conf->mirrors[dev].replacement;
2690
			if (rdev && bio == IO_MADE_GOOD) {
2691 2692 2693
				rdev_clear_badblocks(
					rdev,
					r10_bio->devs[m].addr,
2694
					r10_bio->sectors, 0);
2695 2696
				rdev_dec_pending(rdev, conf->mddev);
			}
2697
		}
2698 2699 2700
		if (fail) {
			spin_lock_irq(&conf->device_lock);
			list_add(&r10_bio->retry_list, &conf->bio_end_io_list);
2701
			conf->nr_queued++;
2702
			spin_unlock_irq(&conf->device_lock);
2703 2704 2705 2706 2707
			/*
			 * In case freeze_array() is waiting for condition
			 * nr_pending == nr_queued + extra to be true.
			 */
			wake_up(&conf->wait_barrier);
2708
			md_wakeup_thread(conf->mddev->thread);
2709 2710 2711 2712
		} else {
			if (test_bit(R10BIO_WriteError,
				     &r10_bio->state))
				close_write(r10_bio);
2713
			raid_end_bio_io(r10_bio);
2714
		}
2715 2716 2717
	}
}

S
Shaohua Li 已提交
2718
static void raid10d(struct md_thread *thread)
L
Linus Torvalds 已提交
2719
{
S
Shaohua Li 已提交
2720
	struct mddev *mddev = thread->mddev;
2721
	struct r10bio *r10_bio;
L
Linus Torvalds 已提交
2722
	unsigned long flags;
2723
	struct r10conf *conf = mddev->private;
L
Linus Torvalds 已提交
2724
	struct list_head *head = &conf->retry_list;
2725
	struct blk_plug plug;
L
Linus Torvalds 已提交
2726 2727 2728

	md_check_recovery(mddev);

2729
	if (!list_empty_careful(&conf->bio_end_io_list) &&
2730
	    !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
2731 2732
		LIST_HEAD(tmp);
		spin_lock_irqsave(&conf->device_lock, flags);
2733
		if (!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
2734 2735 2736 2737
			while (!list_empty(&conf->bio_end_io_list)) {
				list_move(conf->bio_end_io_list.prev, &tmp);
				conf->nr_queued--;
			}
2738 2739 2740
		}
		spin_unlock_irqrestore(&conf->device_lock, flags);
		while (!list_empty(&tmp)) {
2741 2742
			r10_bio = list_first_entry(&tmp, struct r10bio,
						   retry_list);
2743
			list_del(&r10_bio->retry_list);
2744 2745 2746 2747 2748 2749
			if (mddev->degraded)
				set_bit(R10BIO_Degraded, &r10_bio->state);

			if (test_bit(R10BIO_WriteError,
				     &r10_bio->state))
				close_write(r10_bio);
2750 2751 2752 2753
			raid_end_bio_io(r10_bio);
		}
	}

2754
	blk_start_plug(&plug);
L
Linus Torvalds 已提交
2755
	for (;;) {
2756

2757
		flush_pending_writes(conf);
2758

2759 2760 2761
		spin_lock_irqsave(&conf->device_lock, flags);
		if (list_empty(head)) {
			spin_unlock_irqrestore(&conf->device_lock, flags);
L
Linus Torvalds 已提交
2762
			break;
2763
		}
2764
		r10_bio = list_entry(head->prev, struct r10bio, retry_list);
L
Linus Torvalds 已提交
2765
		list_del(head->prev);
2766
		conf->nr_queued--;
L
Linus Torvalds 已提交
2767 2768 2769
		spin_unlock_irqrestore(&conf->device_lock, flags);

		mddev = r10_bio->mddev;
2770
		conf = mddev->private;
2771 2772
		if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
		    test_bit(R10BIO_WriteError, &r10_bio->state))
2773
			handle_write_completed(conf, r10_bio);
N
NeilBrown 已提交
2774 2775
		else if (test_bit(R10BIO_IsReshape, &r10_bio->state))
			reshape_request_write(mddev, r10_bio);
2776
		else if (test_bit(R10BIO_IsSync, &r10_bio->state))
L
Linus Torvalds 已提交
2777
			sync_request_write(mddev, r10_bio);
J
Jens Axboe 已提交
2778
		else if (test_bit(R10BIO_IsRecover, &r10_bio->state))
L
Linus Torvalds 已提交
2779
			recovery_request_write(mddev, r10_bio);
2780
		else if (test_bit(R10BIO_ReadError, &r10_bio->state))
2781
			handle_read_error(mddev, r10_bio);
2782 2783
		else
			WARN_ON_ONCE(1);
2784

N
NeilBrown 已提交
2785
		cond_resched();
2786
		if (mddev->sb_flags & ~(1<<MD_SB_CHANGE_PENDING))
2787
			md_check_recovery(mddev);
L
Linus Torvalds 已提交
2788
	}
2789
	blk_finish_plug(&plug);
L
Linus Torvalds 已提交
2790 2791
}

2792
static int init_resync(struct r10conf *conf)
L
Linus Torvalds 已提交
2793 2794
{
	int buffs;
2795
	int i;
L
Linus Torvalds 已提交
2796 2797

	buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE;
2798
	BUG_ON(conf->r10buf_pool);
2799
	conf->have_replacement = 0;
2800
	for (i = 0; i < conf->geo.raid_disks; i++)
2801 2802
		if (conf->mirrors[i].replacement)
			conf->have_replacement = 1;
L
Linus Torvalds 已提交
2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841
	conf->r10buf_pool = mempool_create(buffs, r10buf_pool_alloc, r10buf_pool_free, conf);
	if (!conf->r10buf_pool)
		return -ENOMEM;
	conf->next_resync = 0;
	return 0;
}

/*
 * perform a "sync" on one "block"
 *
 * We need to make sure that no normal I/O request - particularly write
 * requests - conflict with active sync requests.
 *
 * This is achieved by tracking pending requests and a 'barrier' concept
 * that can be installed to exclude normal IO requests.
 *
 * Resync and recovery are handled very differently.
 * We differentiate by looking at MD_RECOVERY_SYNC in mddev->recovery.
 *
 * For resync, we iterate over virtual addresses, read all copies,
 * and update if there are differences.  If only one copy is live,
 * skip it.
 * For recovery, we iterate over physical addresses, read a good
 * value for each non-in_sync drive, and over-write.
 *
 * So, for recovery we may have several outstanding complex requests for a
 * given address, one for each out-of-sync device.  We model this by allocating
 * a number of r10_bio structures, one for each out-of-sync device.
 * As we setup these structures, we collect all bio's together into a list
 * which we then process collectively to add pages, and then process again
 * to pass to generic_make_request.
 *
 * The r10_bio structures are linked using a borrowed master_bio pointer.
 * This link is counted in ->remaining.  When the r10_bio that points to NULL
 * has its remaining count decremented to 0, the whole complex operation
 * is complete.
 *
 */

S
Shaohua Li 已提交
2842
static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr,
2843
			     int *skipped)
L
Linus Torvalds 已提交
2844
{
2845
	struct r10conf *conf = mddev->private;
2846
	struct r10bio *r10_bio;
L
Linus Torvalds 已提交
2847 2848 2849
	struct bio *biolist = NULL, *bio;
	sector_t max_sector, nr_sectors;
	int i;
2850
	int max_sync;
N
NeilBrown 已提交
2851
	sector_t sync_blocks;
L
Linus Torvalds 已提交
2852 2853
	sector_t sectors_skipped = 0;
	int chunks_skipped = 0;
2854
	sector_t chunk_mask = conf->geo.chunk_mask;
L
Linus Torvalds 已提交
2855 2856 2857

	if (!conf->r10buf_pool)
		if (init_resync(conf))
2858
			return 0;
L
Linus Torvalds 已提交
2859

2860 2861 2862 2863 2864 2865
	/*
	 * Allow skipping a full rebuild for incremental assembly
	 * of a clean array, like RAID1 does.
	 */
	if (mddev->bitmap == NULL &&
	    mddev->recovery_cp == MaxSector &&
2866 2867
	    mddev->reshape_position == MaxSector &&
	    !test_bit(MD_RECOVERY_SYNC, &mddev->recovery) &&
2868
	    !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
2869
	    !test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
2870 2871
	    conf->fullsync == 0) {
		*skipped = 1;
2872
		return mddev->dev_sectors - sector_nr;
2873 2874
	}

L
Linus Torvalds 已提交
2875
 skipped:
A
Andre Noll 已提交
2876
	max_sector = mddev->dev_sectors;
N
NeilBrown 已提交
2877 2878
	if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) ||
	    test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
L
Linus Torvalds 已提交
2879 2880
		max_sector = mddev->resync_max_sectors;
	if (sector_nr >= max_sector) {
2881 2882 2883 2884 2885 2886 2887 2888 2889
		/* If we aborted, we need to abort the
		 * sync on the 'current' bitmap chucks (there can
		 * be several when recovering multiple devices).
		 * as we may have started syncing it but not finished.
		 * We can find the current address in
		 * mddev->curr_resync, but for recovery,
		 * we need to convert that to several
		 * virtual addresses.
		 */
N
NeilBrown 已提交
2890 2891
		if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {
			end_reshape(conf);
2892
			close_sync(conf);
N
NeilBrown 已提交
2893 2894 2895
			return 0;
		}

2896 2897 2898 2899
		if (mddev->curr_resync < max_sector) { /* aborted */
			if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery))
				bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
						&sync_blocks, 1);
2900
			else for (i = 0; i < conf->geo.raid_disks; i++) {
2901 2902 2903 2904 2905
				sector_t sect =
					raid10_find_virt(conf, mddev->curr_resync, i);
				bitmap_end_sync(mddev->bitmap, sect,
						&sync_blocks, 1);
			}
2906 2907 2908 2909 2910 2911 2912 2913
		} else {
			/* completed sync */
			if ((!mddev->bitmap || conf->fullsync)
			    && conf->have_replacement
			    && test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
				/* Completed a full sync so the replacements
				 * are now fully recovered.
				 */
2914 2915 2916 2917 2918 2919 2920 2921
				rcu_read_lock();
				for (i = 0; i < conf->geo.raid_disks; i++) {
					struct md_rdev *rdev =
						rcu_dereference(conf->mirrors[i].replacement);
					if (rdev)
						rdev->recovery_offset = MaxSector;
				}
				rcu_read_unlock();
2922
			}
2923
			conf->fullsync = 0;
2924
		}
2925
		bitmap_close_sync(mddev->bitmap);
L
Linus Torvalds 已提交
2926
		close_sync(conf);
2927
		*skipped = 1;
L
Linus Torvalds 已提交
2928 2929
		return sectors_skipped;
	}
N
NeilBrown 已提交
2930 2931 2932 2933

	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
		return reshape_request(mddev, sector_nr, skipped);

2934
	if (chunks_skipped >= conf->geo.raid_disks) {
L
Linus Torvalds 已提交
2935 2936 2937
		/* if there has been nothing to do on any drive,
		 * then there is nothing to do at all..
		 */
2938 2939
		*skipped = 1;
		return (max_sector - sector_nr) + sectors_skipped;
L
Linus Torvalds 已提交
2940 2941
	}

2942 2943 2944
	if (max_sector > mddev->resync_max)
		max_sector = mddev->resync_max; /* Don't do IO beyond here */

L
Linus Torvalds 已提交
2945 2946 2947
	/* make sure whole request will fit in a chunk - if chunks
	 * are meaningful
	 */
2948 2949 2950
	if (conf->geo.near_copies < conf->geo.raid_disks &&
	    max_sector > (sector_nr | chunk_mask))
		max_sector = (sector_nr | chunk_mask) + 1;
L
Linus Torvalds 已提交
2951

2952 2953 2954 2955 2956 2957 2958
	/*
	 * If there is non-resync activity waiting for a turn, then let it
	 * though before starting on this new sync request.
	 */
	if (conf->nr_waiting)
		schedule_timeout_uninterruptible(1);

L
Linus Torvalds 已提交
2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973
	/* Again, very different code for resync and recovery.
	 * Both must result in an r10bio with a list of bios that
	 * have bi_end_io, bi_sector, bi_bdev set,
	 * and bi_private set to the r10bio.
	 * For recovery, we may actually create several r10bios
	 * with 2 bios in each, that correspond to the bios in the main one.
	 * In this case, the subordinate r10bios link back through a
	 * borrowed master_bio pointer, and the counter in the master
	 * includes a ref from each subordinate.
	 */
	/* First, we decide what to do and set ->bi_end_io
	 * To end_sync_read if we want to read, and
	 * end_sync_write if we will want to write.
	 */

2974
	max_sync = RESYNC_PAGES << (PAGE_SHIFT-9);
L
Linus Torvalds 已提交
2975 2976
	if (!test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
		/* recovery... the complicated one */
2977
		int j;
L
Linus Torvalds 已提交
2978 2979
		r10_bio = NULL;

2980
		for (i = 0 ; i < conf->geo.raid_disks; i++) {
2981
			int still_degraded;
2982
			struct r10bio *rb2;
2983 2984
			sector_t sect;
			int must_sync;
2985
			int any_working;
2986
			struct raid10_info *mirror = &conf->mirrors[i];
2987
			struct md_rdev *mrdev, *mreplace;
2988

2989 2990 2991 2992 2993
			rcu_read_lock();
			mrdev = rcu_dereference(mirror->rdev);
			mreplace = rcu_dereference(mirror->replacement);

			if ((mrdev == NULL ||
2994
			     test_bit(Faulty, &mrdev->flags) ||
2995 2996 2997 2998
			     test_bit(In_sync, &mrdev->flags)) &&
			    (mreplace == NULL ||
			     test_bit(Faulty, &mreplace->flags))) {
				rcu_read_unlock();
2999
				continue;
3000
			}
L
Linus Torvalds 已提交
3001

3002 3003 3004 3005
			still_degraded = 0;
			/* want to reconstruct this device */
			rb2 = r10_bio;
			sect = raid10_find_virt(conf, sector_nr, i);
3006 3007 3008 3009
			if (sect >= mddev->resync_max_sectors) {
				/* last stripe is not complete - don't
				 * try to recover this sector.
				 */
3010
				rcu_read_unlock();
3011 3012
				continue;
			}
3013 3014
			if (mreplace && test_bit(Faulty, &mreplace->flags))
				mreplace = NULL;
3015 3016 3017
			/* Unless we are doing a full sync, or a replacement
			 * we only need to recover the block if it is set in
			 * the bitmap
3018 3019 3020 3021 3022 3023
			 */
			must_sync = bitmap_start_sync(mddev->bitmap, sect,
						      &sync_blocks, 1);
			if (sync_blocks < max_sync)
				max_sync = sync_blocks;
			if (!must_sync &&
3024
			    mreplace == NULL &&
3025 3026 3027 3028 3029
			    !conf->fullsync) {
				/* yep, skip the sync_blocks here, but don't assume
				 * that there will never be anything to do here
				 */
				chunks_skipped = -1;
3030
				rcu_read_unlock();
3031 3032
				continue;
			}
3033 3034 3035 3036
			atomic_inc(&mrdev->nr_pending);
			if (mreplace)
				atomic_inc(&mreplace->nr_pending);
			rcu_read_unlock();
3037

3038
			r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
3039
			r10_bio->state = 0;
3040 3041
			raise_barrier(conf, rb2 != NULL);
			atomic_set(&r10_bio->remaining, 0);
3042

3043 3044 3045 3046 3047 3048
			r10_bio->master_bio = (struct bio*)rb2;
			if (rb2)
				atomic_inc(&rb2->remaining);
			r10_bio->mddev = mddev;
			set_bit(R10BIO_IsRecover, &r10_bio->state);
			r10_bio->sector = sect;
L
Linus Torvalds 已提交
3049

3050 3051 3052 3053 3054
			raid10_find_phys(conf, r10_bio);

			/* Need to check if the array will still be
			 * degraded
			 */
3055 3056 3057 3058 3059
			rcu_read_lock();
			for (j = 0; j < conf->geo.raid_disks; j++) {
				struct md_rdev *rdev = rcu_dereference(
					conf->mirrors[j].rdev);
				if (rdev == NULL || test_bit(Faulty, &rdev->flags)) {
3060
					still_degraded = 1;
3061
					break;
L
Linus Torvalds 已提交
3062
				}
3063
			}
3064 3065 3066 3067

			must_sync = bitmap_start_sync(mddev->bitmap, sect,
						      &sync_blocks, still_degraded);

3068
			any_working = 0;
3069
			for (j=0; j<conf->copies;j++) {
3070
				int k;
3071
				int d = r10_bio->devs[j].devnum;
3072
				sector_t from_addr, to_addr;
3073 3074
				struct md_rdev *rdev =
					rcu_dereference(conf->mirrors[d].rdev);
3075 3076
				sector_t sector, first_bad;
				int bad_sectors;
3077 3078
				if (!rdev ||
				    !test_bit(In_sync, &rdev->flags))
3079 3080
					continue;
				/* This is where we read from */
3081
				any_working = 1;
3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095
				sector = r10_bio->devs[j].addr;

				if (is_badblock(rdev, sector, max_sync,
						&first_bad, &bad_sectors)) {
					if (first_bad > sector)
						max_sync = first_bad - sector;
					else {
						bad_sectors -= (sector
								- first_bad);
						if (max_sync > bad_sectors)
							max_sync = bad_sectors;
						continue;
					}
				}
3096 3097 3098 3099
				bio = r10_bio->devs[0].bio;
				bio->bi_next = biolist;
				biolist = bio;
				bio->bi_end_io = end_sync_read;
M
Mike Christie 已提交
3100
				bio_set_op_attrs(bio, REQ_OP_READ, 0);
3101 3102
				if (test_bit(FailFast, &rdev->flags))
					bio->bi_opf |= MD_FAILFAST;
3103
				from_addr = r10_bio->devs[j].addr;
3104 3105
				bio->bi_iter.bi_sector = from_addr +
					rdev->data_offset;
3106 3107 3108
				bio->bi_bdev = rdev->bdev;
				atomic_inc(&rdev->nr_pending);
				/* and we write to 'i' (if not in_sync) */
3109 3110 3111 3112 3113

				for (k=0; k<conf->copies; k++)
					if (r10_bio->devs[k].devnum == i)
						break;
				BUG_ON(k == conf->copies);
3114
				to_addr = r10_bio->devs[k].addr;
3115
				r10_bio->devs[0].devnum = d;
3116
				r10_bio->devs[0].addr = from_addr;
3117
				r10_bio->devs[1].devnum = i;
3118
				r10_bio->devs[1].addr = to_addr;
3119

3120
				if (!test_bit(In_sync, &mrdev->flags)) {
3121 3122 3123 3124
					bio = r10_bio->devs[1].bio;
					bio->bi_next = biolist;
					biolist = bio;
					bio->bi_end_io = end_sync_write;
M
Mike Christie 已提交
3125
					bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
3126
					bio->bi_iter.bi_sector = to_addr
3127 3128
						+ mrdev->data_offset;
					bio->bi_bdev = mrdev->bdev;
3129 3130 3131 3132 3133 3134 3135 3136
					atomic_inc(&r10_bio->remaining);
				} else
					r10_bio->devs[1].bio->bi_end_io = NULL;

				/* and maybe write to replacement */
				bio = r10_bio->devs[1].repl_bio;
				if (bio)
					bio->bi_end_io = NULL;
3137
				/* Note: if mreplace != NULL, then bio
3138 3139 3140 3141 3142 3143 3144
				 * cannot be NULL as r10buf_pool_alloc will
				 * have allocated it.
				 * So the second test here is pointless.
				 * But it keeps semantic-checkers happy, and
				 * this comment keeps human reviewers
				 * happy.
				 */
3145 3146
				if (mreplace == NULL || bio == NULL ||
				    test_bit(Faulty, &mreplace->flags))
3147 3148 3149 3150
					break;
				bio->bi_next = biolist;
				biolist = bio;
				bio->bi_end_io = end_sync_write;
M
Mike Christie 已提交
3151
				bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
3152
				bio->bi_iter.bi_sector = to_addr +
3153 3154
					mreplace->data_offset;
				bio->bi_bdev = mreplace->bdev;
3155
				atomic_inc(&r10_bio->remaining);
3156 3157
				break;
			}
3158
			rcu_read_unlock();
3159
			if (j == conf->copies) {
3160 3161 3162 3163 3164 3165 3166 3167 3168 3169
				/* Cannot recover, so abort the recovery or
				 * record a bad block */
				if (any_working) {
					/* problem is that there are bad blocks
					 * on other device(s)
					 */
					int k;
					for (k = 0; k < conf->copies; k++)
						if (r10_bio->devs[k].devnum == i)
							break;
3170
					if (!test_bit(In_sync,
3171
						      &mrdev->flags)
3172
					    && !rdev_set_badblocks(
3173
						    mrdev,
3174 3175 3176
						    r10_bio->devs[k].addr,
						    max_sync, 0))
						any_working = 0;
3177
					if (mreplace &&
3178
					    !rdev_set_badblocks(
3179
						    mreplace,
3180 3181 3182 3183 3184 3185 3186
						    r10_bio->devs[k].addr,
						    max_sync, 0))
						any_working = 0;
				}
				if (!any_working)  {
					if (!test_and_set_bit(MD_RECOVERY_INTR,
							      &mddev->recovery))
N
NeilBrown 已提交
3187
						pr_warn("md/raid10:%s: insufficient working devices for recovery.\n",
3188
						       mdname(mddev));
3189
					mirror->recovery_disabled
3190 3191
						= mddev->recovery_disabled;
				}
3192 3193 3194 3195
				put_buf(r10_bio);
				if (rb2)
					atomic_dec(&rb2->remaining);
				r10_bio = rb2;
3196 3197 3198
				rdev_dec_pending(mrdev, mddev);
				if (mreplace)
					rdev_dec_pending(mreplace, mddev);
3199
				break;
L
Linus Torvalds 已提交
3200
			}
3201 3202 3203
			rdev_dec_pending(mrdev, mddev);
			if (mreplace)
				rdev_dec_pending(mreplace, mddev);
3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220
			if (r10_bio->devs[0].bio->bi_opf & MD_FAILFAST) {
				/* Only want this if there is elsewhere to
				 * read from. 'j' is currently the first
				 * readable copy.
				 */
				int targets = 1;
				for (; j < conf->copies; j++) {
					int d = r10_bio->devs[j].devnum;
					if (conf->mirrors[d].rdev &&
					    test_bit(In_sync,
						      &conf->mirrors[d].rdev->flags))
						targets++;
				}
				if (targets == 1)
					r10_bio->devs[0].bio->bi_opf
						&= ~MD_FAILFAST;
			}
3221
		}
L
Linus Torvalds 已提交
3222 3223
		if (biolist == NULL) {
			while (r10_bio) {
3224 3225
				struct r10bio *rb2 = r10_bio;
				r10_bio = (struct r10bio*) rb2->master_bio;
L
Linus Torvalds 已提交
3226 3227 3228 3229 3230 3231 3232 3233
				rb2->master_bio = NULL;
				put_buf(rb2);
			}
			goto giveup;
		}
	} else {
		/* resync. Schedule a read for every block at this virt offset */
		int count = 0;
3234

3235
		bitmap_cond_end_sync(mddev->bitmap, sector_nr, 0);
3236

3237 3238
		if (!bitmap_start_sync(mddev->bitmap, sector_nr,
				       &sync_blocks, mddev->degraded) &&
3239 3240
		    !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED,
						 &mddev->recovery)) {
3241 3242 3243 3244 3245 3246
			/* We can skip this block */
			*skipped = 1;
			return sync_blocks + sectors_skipped;
		}
		if (sync_blocks < max_sync)
			max_sync = sync_blocks;
L
Linus Torvalds 已提交
3247
		r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
3248
		r10_bio->state = 0;
L
Linus Torvalds 已提交
3249 3250 3251

		r10_bio->mddev = mddev;
		atomic_set(&r10_bio->remaining, 0);
3252 3253
		raise_barrier(conf, 0);
		conf->next_resync = sector_nr;
L
Linus Torvalds 已提交
3254 3255 3256 3257 3258

		r10_bio->master_bio = NULL;
		r10_bio->sector = sector_nr;
		set_bit(R10BIO_IsSync, &r10_bio->state);
		raid10_find_phys(conf, r10_bio);
3259
		r10_bio->sectors = (sector_nr | chunk_mask) - sector_nr + 1;
L
Linus Torvalds 已提交
3260

3261
		for (i = 0; i < conf->copies; i++) {
L
Linus Torvalds 已提交
3262
			int d = r10_bio->devs[i].devnum;
3263 3264
			sector_t first_bad, sector;
			int bad_sectors;
3265
			struct md_rdev *rdev;
3266

3267 3268 3269
			if (r10_bio->devs[i].repl_bio)
				r10_bio->devs[i].repl_bio->bi_end_io = NULL;

L
Linus Torvalds 已提交
3270
			bio = r10_bio->devs[i].bio;
3271
			bio->bi_error = -EIO;
3272 3273 3274 3275
			rcu_read_lock();
			rdev = rcu_dereference(conf->mirrors[d].rdev);
			if (rdev == NULL || test_bit(Faulty, &rdev->flags)) {
				rcu_read_unlock();
L
Linus Torvalds 已提交
3276
				continue;
3277
			}
3278
			sector = r10_bio->devs[i].addr;
3279
			if (is_badblock(rdev, sector, max_sync,
3280 3281 3282 3283 3284 3285
					&first_bad, &bad_sectors)) {
				if (first_bad > sector)
					max_sync = first_bad - sector;
				else {
					bad_sectors -= (sector - first_bad);
					if (max_sync > bad_sectors)
3286
						max_sync = bad_sectors;
3287
					rcu_read_unlock();
3288 3289 3290
					continue;
				}
			}
3291
			atomic_inc(&rdev->nr_pending);
L
Linus Torvalds 已提交
3292 3293 3294 3295
			atomic_inc(&r10_bio->remaining);
			bio->bi_next = biolist;
			biolist = bio;
			bio->bi_end_io = end_sync_read;
M
Mike Christie 已提交
3296
			bio_set_op_attrs(bio, REQ_OP_READ, 0);
3297 3298
			if (test_bit(FailFast, &conf->mirrors[d].rdev->flags))
				bio->bi_opf |= MD_FAILFAST;
3299 3300
			bio->bi_iter.bi_sector = sector + rdev->data_offset;
			bio->bi_bdev = rdev->bdev;
L
Linus Torvalds 已提交
3301
			count++;
3302

3303 3304 3305
			rdev = rcu_dereference(conf->mirrors[d].replacement);
			if (rdev == NULL || test_bit(Faulty, &rdev->flags)) {
				rcu_read_unlock();
3306
				continue;
3307 3308 3309
			}
			atomic_inc(&rdev->nr_pending);
			rcu_read_unlock();
3310 3311 3312

			/* Need to set up for writing to the replacement */
			bio = r10_bio->devs[i].repl_bio;
3313
			bio->bi_error = -EIO;
3314 3315 3316 3317 3318

			sector = r10_bio->devs[i].addr;
			bio->bi_next = biolist;
			biolist = bio;
			bio->bi_end_io = end_sync_write;
M
Mike Christie 已提交
3319
			bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
3320 3321
			if (test_bit(FailFast, &conf->mirrors[d].rdev->flags))
				bio->bi_opf |= MD_FAILFAST;
3322 3323
			bio->bi_iter.bi_sector = sector + rdev->data_offset;
			bio->bi_bdev = rdev->bdev;
3324
			count++;
L
Linus Torvalds 已提交
3325 3326 3327 3328 3329 3330
		}

		if (count < 2) {
			for (i=0; i<conf->copies; i++) {
				int d = r10_bio->devs[i].devnum;
				if (r10_bio->devs[i].bio->bi_end_io)
3331 3332
					rdev_dec_pending(conf->mirrors[d].rdev,
							 mddev);
3333 3334 3335 3336 3337
				if (r10_bio->devs[i].repl_bio &&
				    r10_bio->devs[i].repl_bio->bi_end_io)
					rdev_dec_pending(
						conf->mirrors[d].replacement,
						mddev);
L
Linus Torvalds 已提交
3338 3339 3340 3341 3342 3343 3344 3345
			}
			put_buf(r10_bio);
			biolist = NULL;
			goto giveup;
		}
	}

	nr_sectors = 0;
3346 3347
	if (sector_nr + max_sync < max_sector)
		max_sector = sector_nr + max_sync;
L
Linus Torvalds 已提交
3348 3349 3350 3351 3352 3353 3354 3355
	do {
		struct page *page;
		int len = PAGE_SIZE;
		if (sector_nr + (len>>9) > max_sector)
			len = (max_sector - sector_nr) << 9;
		if (len == 0)
			break;
		for (bio= biolist ; bio ; bio=bio->bi_next) {
3356 3357
			struct resync_pages *rp = get_resync_pages(bio);
			page = resync_fetch_page(rp, rp->idx++);
3358 3359 3360 3361 3362
			/*
			 * won't fail because the vec table is big enough
			 * to hold all these pages
			 */
			bio_add_page(bio, page, len, 0);
L
Linus Torvalds 已提交
3363 3364 3365
		}
		nr_sectors += len>>9;
		sector_nr += len>>9;
3366
	} while (get_resync_pages(biolist)->idx < RESYNC_PAGES);
L
Linus Torvalds 已提交
3367 3368 3369 3370 3371 3372 3373
	r10_bio->sectors = nr_sectors;

	while (biolist) {
		bio = biolist;
		biolist = biolist->bi_next;

		bio->bi_next = NULL;
3374
		r10_bio = get_resync_r10bio(bio);
L
Linus Torvalds 已提交
3375 3376 3377 3378
		r10_bio->sectors = nr_sectors;

		if (bio->bi_end_io == end_sync_read) {
			md_sync_acct(bio->bi_bdev, nr_sectors);
3379
			bio->bi_error = 0;
L
Linus Torvalds 已提交
3380 3381 3382 3383
			generic_make_request(bio);
		}
	}

3384 3385 3386 3387 3388 3389
	if (sectors_skipped)
		/* pretend they weren't skipped, it makes
		 * no important difference in this case
		 */
		md_done_sync(mddev, sectors_skipped, 1);

L
Linus Torvalds 已提交
3390 3391 3392
	return sectors_skipped + nr_sectors;
 giveup:
	/* There is nowhere to write, so all non-sync
3393 3394
	 * drives must be failed or in resync, all drives
	 * have a bad block, so try the next chunk...
L
Linus Torvalds 已提交
3395
	 */
3396 3397 3398 3399
	if (sector_nr + max_sync < max_sector)
		max_sector = sector_nr + max_sync;

	sectors_skipped += (max_sector - sector_nr);
L
Linus Torvalds 已提交
3400 3401 3402 3403 3404
	chunks_skipped ++;
	sector_nr = max_sector;
	goto skipped;
}

3405
static sector_t
3406
raid10_size(struct mddev *mddev, sector_t sectors, int raid_disks)
3407 3408
{
	sector_t size;
3409
	struct r10conf *conf = mddev->private;
3410 3411

	if (!raid_disks)
N
NeilBrown 已提交
3412 3413
		raid_disks = min(conf->geo.raid_disks,
				 conf->prev.raid_disks);
3414
	if (!sectors)
3415
		sectors = conf->dev_sectors;
3416

3417 3418
	size = sectors >> conf->geo.chunk_shift;
	sector_div(size, conf->geo.far_copies);
3419
	size = size * raid_disks;
3420
	sector_div(size, conf->geo.near_copies);
3421

3422
	return size << conf->geo.chunk_shift;
3423 3424
}

3425 3426 3427 3428 3429 3430 3431
static void calc_sectors(struct r10conf *conf, sector_t size)
{
	/* Calculate the number of sectors-per-device that will
	 * actually be used, and set conf->dev_sectors and
	 * conf->stride
	 */

3432 3433 3434 3435
	size = size >> conf->geo.chunk_shift;
	sector_div(size, conf->geo.far_copies);
	size = size * conf->geo.raid_disks;
	sector_div(size, conf->geo.near_copies);
3436 3437 3438 3439 3440 3441 3442
	/* 'size' is now the number of chunks in the array */
	/* calculate "used chunks per device" */
	size = size * conf->copies;

	/* We need to round up when dividing by raid_disks to
	 * get the stride size.
	 */
3443
	size = DIV_ROUND_UP_SECTOR_T(size, conf->geo.raid_disks);
3444

3445
	conf->dev_sectors = size << conf->geo.chunk_shift;
3446

3447 3448
	if (conf->geo.far_offset)
		conf->geo.stride = 1 << conf->geo.chunk_shift;
3449
	else {
3450 3451
		sector_div(size, conf->geo.far_copies);
		conf->geo.stride = size << conf->geo.chunk_shift;
3452 3453
	}
}
3454

3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478
enum geo_type {geo_new, geo_old, geo_start};
static int setup_geo(struct geom *geo, struct mddev *mddev, enum geo_type new)
{
	int nc, fc, fo;
	int layout, chunk, disks;
	switch (new) {
	case geo_old:
		layout = mddev->layout;
		chunk = mddev->chunk_sectors;
		disks = mddev->raid_disks - mddev->delta_disks;
		break;
	case geo_new:
		layout = mddev->new_layout;
		chunk = mddev->new_chunk_sectors;
		disks = mddev->raid_disks;
		break;
	default: /* avoid 'may be unused' warnings */
	case geo_start: /* new when starting reshape - raid_disks not
			 * updated yet. */
		layout = mddev->new_layout;
		chunk = mddev->new_chunk_sectors;
		disks = mddev->raid_disks + mddev->delta_disks;
		break;
	}
3479
	if (layout >> 19)
3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490
		return -1;
	if (chunk < (PAGE_SIZE >> 9) ||
	    !is_power_of_2(chunk))
		return -2;
	nc = layout & 255;
	fc = (layout >> 8) & 255;
	fo = layout & (1<<16);
	geo->raid_disks = disks;
	geo->near_copies = nc;
	geo->far_copies = fc;
	geo->far_offset = fo;
3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506
	switch (layout >> 17) {
	case 0:	/* original layout.  simple but not always optimal */
		geo->far_set_size = disks;
		break;
	case 1: /* "improved" layout which was buggy.  Hopefully no-one is
		 * actually using this, but leave code here just in case.*/
		geo->far_set_size = disks/fc;
		WARN(geo->far_set_size < fc,
		     "This RAID10 layout does not provide data safety - please backup and create new array\n");
		break;
	case 2: /* "improved" layout fixed to match documentation */
		geo->far_set_size = fc * nc;
		break;
	default: /* Not a valid layout */
		return -1;
	}
3507 3508 3509 3510 3511
	geo->chunk_mask = chunk - 1;
	geo->chunk_shift = ffz(~chunk);
	return nc*fc;
}

3512
static struct r10conf *setup_conf(struct mddev *mddev)
L
Linus Torvalds 已提交
3513
{
3514
	struct r10conf *conf = NULL;
3515
	int err = -EINVAL;
3516 3517 3518 3519
	struct geom geo;
	int copies;

	copies = setup_geo(&geo, mddev, geo_new);
L
Linus Torvalds 已提交
3520

3521
	if (copies == -2) {
N
NeilBrown 已提交
3522 3523
		pr_warn("md/raid10:%s: chunk size must be at least PAGE_SIZE(%ld) and be a power of 2.\n",
			mdname(mddev), PAGE_SIZE);
3524
		goto out;
L
Linus Torvalds 已提交
3525
	}
3526

3527
	if (copies < 2 || copies > mddev->raid_disks) {
N
NeilBrown 已提交
3528 3529
		pr_warn("md/raid10:%s: unsupported raid10 layout: 0x%8x\n",
			mdname(mddev), mddev->new_layout);
L
Linus Torvalds 已提交
3530 3531
		goto out;
	}
3532 3533

	err = -ENOMEM;
3534
	conf = kzalloc(sizeof(struct r10conf), GFP_KERNEL);
3535
	if (!conf)
L
Linus Torvalds 已提交
3536
		goto out;
3537

N
NeilBrown 已提交
3538
	/* FIXME calc properly */
3539
	conf->mirrors = kzalloc(sizeof(struct raid10_info)*(mddev->raid_disks +
3540
							    max(0,-mddev->delta_disks)),
3541 3542 3543
				GFP_KERNEL);
	if (!conf->mirrors)
		goto out;
3544 3545 3546

	conf->tmppage = alloc_page(GFP_KERNEL);
	if (!conf->tmppage)
3547 3548
		goto out;

3549 3550
	conf->geo = geo;
	conf->copies = copies;
3551 3552 3553 3554 3555
	conf->r10bio_pool = mempool_create(NR_RAID10_BIOS, r10bio_pool_alloc,
					   r10bio_pool_free, conf);
	if (!conf->r10bio_pool)
		goto out;

3556 3557 3558 3559
	conf->bio_split = bioset_create(BIO_POOL_SIZE, 0);
	if (!conf->bio_split)
		goto out;

3560
	calc_sectors(conf, mddev->dev_sectors);
N
NeilBrown 已提交
3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575
	if (mddev->reshape_position == MaxSector) {
		conf->prev = conf->geo;
		conf->reshape_progress = MaxSector;
	} else {
		if (setup_geo(&conf->prev, mddev, geo_old) != conf->copies) {
			err = -EINVAL;
			goto out;
		}
		conf->reshape_progress = mddev->reshape_position;
		if (conf->prev.far_offset)
			conf->prev.stride = 1 << conf->prev.chunk_shift;
		else
			/* far_copies must be 1 */
			conf->prev.stride = conf->dev_sectors;
	}
3576
	conf->reshape_safe = conf->reshape_progress;
3577
	spin_lock_init(&conf->device_lock);
3578
	INIT_LIST_HEAD(&conf->retry_list);
3579
	INIT_LIST_HEAD(&conf->bio_end_io_list);
3580 3581 3582

	spin_lock_init(&conf->resync_lock);
	init_waitqueue_head(&conf->wait_barrier);
3583
	atomic_set(&conf->nr_pending, 0);
3584

3585
	conf->thread = md_register_thread(raid10d, mddev, "raid10");
3586 3587 3588 3589 3590 3591 3592 3593
	if (!conf->thread)
		goto out;

	conf->mddev = mddev;
	return conf;

 out:
	if (conf) {
3594
		mempool_destroy(conf->r10bio_pool);
3595 3596
		kfree(conf->mirrors);
		safe_put_page(conf->tmppage);
3597 3598
		if (conf->bio_split)
			bioset_free(conf->bio_split);
3599 3600 3601 3602 3603
		kfree(conf);
	}
	return ERR_PTR(err);
}

S
Shaohua Li 已提交
3604
static int raid10_run(struct mddev *mddev)
3605
{
3606
	struct r10conf *conf;
3607
	int i, disk_idx, chunk_size;
3608
	struct raid10_info *disk;
3609
	struct md_rdev *rdev;
3610
	sector_t size;
N
NeilBrown 已提交
3611 3612
	sector_t min_offset_diff = 0;
	int first = 1;
S
Shaohua Li 已提交
3613
	bool discard_supported = false;
3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627

	if (mddev->private == NULL) {
		conf = setup_conf(mddev);
		if (IS_ERR(conf))
			return PTR_ERR(conf);
		mddev->private = conf;
	}
	conf = mddev->private;
	if (!conf)
		goto out;

	mddev->thread = conf->thread;
	conf->thread = NULL;

3628
	chunk_size = mddev->chunk_sectors << 9;
3629
	if (mddev->queue) {
S
Shaohua Li 已提交
3630 3631
		blk_queue_max_discard_sectors(mddev->queue,
					      mddev->chunk_sectors);
3632
		blk_queue_max_write_same_sectors(mddev->queue, 0);
3633
		blk_queue_max_write_zeroes_sectors(mddev->queue, 0);
3634 3635 3636 3637 3638 3639 3640
		blk_queue_io_min(mddev->queue, chunk_size);
		if (conf->geo.raid_disks % conf->geo.near_copies)
			blk_queue_io_opt(mddev->queue, chunk_size * conf->geo.raid_disks);
		else
			blk_queue_io_opt(mddev->queue, chunk_size *
					 (conf->geo.raid_disks / conf->geo.near_copies));
	}
3641

N
NeilBrown 已提交
3642
	rdev_for_each(rdev, mddev) {
N
NeilBrown 已提交
3643
		long long diff;
3644

L
Linus Torvalds 已提交
3645
		disk_idx = rdev->raid_disk;
3646 3647 3648 3649
		if (disk_idx < 0)
			continue;
		if (disk_idx >= conf->geo.raid_disks &&
		    disk_idx >= conf->prev.raid_disks)
L
Linus Torvalds 已提交
3650 3651 3652
			continue;
		disk = conf->mirrors + disk_idx;

3653 3654 3655 3656 3657 3658 3659 3660 3661
		if (test_bit(Replacement, &rdev->flags)) {
			if (disk->replacement)
				goto out_free_conf;
			disk->replacement = rdev;
		} else {
			if (disk->rdev)
				goto out_free_conf;
			disk->rdev = rdev;
		}
N
NeilBrown 已提交
3662 3663 3664 3665 3666 3667 3668
		diff = (rdev->new_data_offset - rdev->data_offset);
		if (!mddev->reshape_backwards)
			diff = -diff;
		if (diff < 0)
			diff = 0;
		if (first || diff < min_offset_diff)
			min_offset_diff = diff;
3669

3670 3671 3672
		if (mddev->gendisk)
			disk_stack_limits(mddev->gendisk, rdev->bdev,
					  rdev->data_offset << 9);
L
Linus Torvalds 已提交
3673 3674

		disk->head_position = 0;
S
Shaohua Li 已提交
3675 3676 3677

		if (blk_queue_discard(bdev_get_queue(rdev->bdev)))
			discard_supported = true;
3678
		first = 0;
L
Linus Torvalds 已提交
3679
	}
N
NeilBrown 已提交
3680

3681 3682 3683 3684 3685 3686 3687 3688
	if (mddev->queue) {
		if (discard_supported)
			queue_flag_set_unlocked(QUEUE_FLAG_DISCARD,
						mddev->queue);
		else
			queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD,
						  mddev->queue);
	}
3689
	/* need to check that every block has at least one working mirror */
3690
	if (!enough(conf, -1)) {
N
NeilBrown 已提交
3691
		pr_err("md/raid10:%s: not enough operational mirrors.\n",
3692
		       mdname(mddev));
L
Linus Torvalds 已提交
3693 3694 3695
		goto out_free_conf;
	}

N
NeilBrown 已提交
3696 3697 3698 3699 3700 3701
	if (conf->reshape_progress != MaxSector) {
		/* must ensure that shape change is supported */
		if (conf->geo.far_copies != 1 &&
		    conf->geo.far_offset == 0)
			goto out_free_conf;
		if (conf->prev.far_copies != 1 &&
3702
		    conf->prev.far_offset == 0)
N
NeilBrown 已提交
3703 3704 3705
			goto out_free_conf;
	}

L
Linus Torvalds 已提交
3706
	mddev->degraded = 0;
3707 3708 3709 3710
	for (i = 0;
	     i < conf->geo.raid_disks
		     || i < conf->prev.raid_disks;
	     i++) {
L
Linus Torvalds 已提交
3711 3712 3713

		disk = conf->mirrors + i;

3714 3715 3716 3717 3718 3719 3720
		if (!disk->rdev && disk->replacement) {
			/* The replacement is all we have - use it */
			disk->rdev = disk->replacement;
			disk->replacement = NULL;
			clear_bit(Replacement, &disk->rdev->flags);
		}

3721
		if (!disk->rdev ||
3722
		    !test_bit(In_sync, &disk->rdev->flags)) {
L
Linus Torvalds 已提交
3723 3724
			disk->head_position = 0;
			mddev->degraded++;
3725 3726
			if (disk->rdev &&
			    disk->rdev->saved_raid_disk < 0)
3727
				conf->fullsync = 1;
L
Linus Torvalds 已提交
3728
		}
3729
		disk->recovery_disabled = mddev->recovery_disabled - 1;
L
Linus Torvalds 已提交
3730 3731
	}

3732
	if (mddev->recovery_cp != MaxSector)
N
NeilBrown 已提交
3733 3734 3735
		pr_notice("md/raid10:%s: not clean -- starting background reconstruction\n",
			  mdname(mddev));
	pr_info("md/raid10:%s: active with %d out of %d devices\n",
3736 3737
		mdname(mddev), conf->geo.raid_disks - mddev->degraded,
		conf->geo.raid_disks);
L
Linus Torvalds 已提交
3738 3739 3740
	/*
	 * Ok, everything is just fine now
	 */
3741 3742 3743 3744
	mddev->dev_sectors = conf->dev_sectors;
	size = raid10_size(mddev, 0, 0);
	md_set_array_sectors(mddev, size);
	mddev->resync_max_sectors = size;
3745
	set_bit(MD_FAILFAST_SUPPORTED, &mddev->flags);
L
Linus Torvalds 已提交
3746

3747
	if (mddev->queue) {
3748
		int stripe = conf->geo.raid_disks *
3749
			((mddev->chunk_sectors << 9) / PAGE_SIZE);
3750 3751 3752 3753 3754

		/* Calculate max read-ahead size.
		 * We need to readahead at least twice a whole stripe....
		 * maybe...
		 */
3755
		stripe /= conf->geo.near_copies;
3756 3757
		if (mddev->queue->backing_dev_info->ra_pages < 2 * stripe)
			mddev->queue->backing_dev_info->ra_pages = 2 * stripe;
L
Linus Torvalds 已提交
3758 3759
	}

3760 3761 3762
	if (md_integrity_register(mddev))
		goto out_free_conf;

N
NeilBrown 已提交
3763 3764 3765 3766 3767 3768 3769 3770 3771 3772
	if (conf->reshape_progress != MaxSector) {
		unsigned long before_length, after_length;

		before_length = ((1 << conf->prev.chunk_shift) *
				 conf->prev.far_copies);
		after_length = ((1 << conf->geo.chunk_shift) *
				conf->geo.far_copies);

		if (max(before_length, after_length) > min_offset_diff) {
			/* This cannot work */
N
NeilBrown 已提交
3773
			pr_warn("md/raid10: offset difference not enough to continue reshape\n");
N
NeilBrown 已提交
3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785
			goto out_free_conf;
		}
		conf->offset_diff = min_offset_diff;

		clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
		clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
		set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
		set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
		mddev->sync_thread = md_register_thread(md_do_sync, mddev,
							"reshape");
	}

L
Linus Torvalds 已提交
3786 3787 3788
	return 0;

out_free_conf:
3789
	md_unregister_thread(&mddev->thread);
3790
	mempool_destroy(conf->r10bio_pool);
3791
	safe_put_page(conf->tmppage);
3792
	kfree(conf->mirrors);
L
Linus Torvalds 已提交
3793 3794 3795 3796 3797 3798
	kfree(conf);
	mddev->private = NULL;
out:
	return -EIO;
}

N
NeilBrown 已提交
3799
static void raid10_free(struct mddev *mddev, void *priv)
L
Linus Torvalds 已提交
3800
{
N
NeilBrown 已提交
3801
	struct r10conf *conf = priv;
L
Linus Torvalds 已提交
3802

3803
	mempool_destroy(conf->r10bio_pool);
3804
	safe_put_page(conf->tmppage);
3805
	kfree(conf->mirrors);
3806 3807
	kfree(conf->mirrors_old);
	kfree(conf->mirrors_new);
3808 3809
	if (conf->bio_split)
		bioset_free(conf->bio_split);
L
Linus Torvalds 已提交
3810 3811 3812
	kfree(conf);
}

3813
static void raid10_quiesce(struct mddev *mddev, int state)
3814
{
3815
	struct r10conf *conf = mddev->private;
3816 3817 3818 3819 3820 3821 3822 3823 3824 3825

	switch(state) {
	case 1:
		raise_barrier(conf, 0);
		break;
	case 0:
		lower_barrier(conf);
		break;
	}
}
L
Linus Torvalds 已提交
3826

3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843
static int raid10_resize(struct mddev *mddev, sector_t sectors)
{
	/* Resize of 'far' arrays is not supported.
	 * For 'near' and 'offset' arrays we can set the
	 * number of sectors used to be an appropriate multiple
	 * of the chunk size.
	 * For 'offset', this is far_copies*chunksize.
	 * For 'near' the multiplier is the LCM of
	 * near_copies and raid_disks.
	 * So if far_copies > 1 && !far_offset, fail.
	 * Else find LCM(raid_disks, near_copy)*far_copies and
	 * multiply by chunk_size.  Then round to this number.
	 * This is mostly done by raid10_size()
	 */
	struct r10conf *conf = mddev->private;
	sector_t oldsize, size;

3844 3845 3846
	if (mddev->reshape_position != MaxSector)
		return -EBUSY;

3847
	if (conf->geo.far_copies > 1 && !conf->geo.far_offset)
3848 3849 3850 3851
		return -EINVAL;

	oldsize = raid10_size(mddev, 0, 0);
	size = raid10_size(mddev, sectors, 0);
3852 3853
	if (mddev->external_size &&
	    mddev->array_sectors > size)
3854
		return -EINVAL;
3855 3856 3857 3858 3859 3860
	if (mddev->bitmap) {
		int ret = bitmap_resize(mddev->bitmap, size, 0, 0);
		if (ret)
			return ret;
	}
	md_set_array_sectors(mddev, size);
3861 3862 3863 3864 3865
	if (sectors > mddev->dev_sectors &&
	    mddev->recovery_cp > oldsize) {
		mddev->recovery_cp = oldsize;
		set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
	}
3866 3867
	calc_sectors(conf, sectors);
	mddev->dev_sectors = conf->dev_sectors;
3868 3869 3870 3871
	mddev->resync_max_sectors = size;
	return 0;
}

3872
static void *raid10_takeover_raid0(struct mddev *mddev, sector_t size, int devs)
3873
{
3874
	struct md_rdev *rdev;
3875
	struct r10conf *conf;
3876 3877

	if (mddev->degraded > 0) {
N
NeilBrown 已提交
3878 3879
		pr_warn("md/raid10:%s: Error: degraded raid0!\n",
			mdname(mddev));
3880 3881
		return ERR_PTR(-EINVAL);
	}
3882
	sector_div(size, devs);
3883 3884 3885 3886 3887 3888 3889 3890 3891 3892

	/* Set new parameters */
	mddev->new_level = 10;
	/* new layout: far_copies = 1, near_copies = 2 */
	mddev->new_layout = (1<<8) + 2;
	mddev->new_chunk_sectors = mddev->chunk_sectors;
	mddev->delta_disks = mddev->raid_disks;
	mddev->raid_disks *= 2;
	/* make sure it will be not marked as dirty */
	mddev->recovery_cp = MaxSector;
3893
	mddev->dev_sectors = size;
3894 3895

	conf = setup_conf(mddev);
3896
	if (!IS_ERR(conf)) {
N
NeilBrown 已提交
3897
		rdev_for_each(rdev, mddev)
3898
			if (rdev->raid_disk >= 0) {
3899
				rdev->new_raid_disk = rdev->raid_disk * 2;
3900 3901
				rdev->sectors = size;
			}
3902 3903 3904
		conf->barrier = 1;
	}

3905 3906 3907
	return conf;
}

3908
static void *raid10_takeover(struct mddev *mddev)
3909
{
3910
	struct r0conf *raid0_conf;
3911 3912 3913 3914 3915 3916

	/* raid10 can take over:
	 *  raid0 - providing it has only two drives
	 */
	if (mddev->level == 0) {
		/* for raid0 takeover only one zone is supported */
3917 3918
		raid0_conf = mddev->private;
		if (raid0_conf->nr_strip_zones > 1) {
N
NeilBrown 已提交
3919 3920
			pr_warn("md/raid10:%s: cannot takeover raid 0 with more than one zone.\n",
				mdname(mddev));
3921 3922
			return ERR_PTR(-EINVAL);
		}
3923 3924 3925
		return raid10_takeover_raid0(mddev,
			raid0_conf->strip_zone->zone_end,
			raid0_conf->strip_zone->nb_dev);
3926 3927 3928 3929
	}
	return ERR_PTR(-EINVAL);
}

N
NeilBrown 已提交
3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970
static int raid10_check_reshape(struct mddev *mddev)
{
	/* Called when there is a request to change
	 * - layout (to ->new_layout)
	 * - chunk size (to ->new_chunk_sectors)
	 * - raid_disks (by delta_disks)
	 * or when trying to restart a reshape that was ongoing.
	 *
	 * We need to validate the request and possibly allocate
	 * space if that might be an issue later.
	 *
	 * Currently we reject any reshape of a 'far' mode array,
	 * allow chunk size to change if new is generally acceptable,
	 * allow raid_disks to increase, and allow
	 * a switch between 'near' mode and 'offset' mode.
	 */
	struct r10conf *conf = mddev->private;
	struct geom geo;

	if (conf->geo.far_copies != 1 && !conf->geo.far_offset)
		return -EINVAL;

	if (setup_geo(&geo, mddev, geo_start) != conf->copies)
		/* mustn't change number of copies */
		return -EINVAL;
	if (geo.far_copies > 1 && !geo.far_offset)
		/* Cannot switch to 'far' mode */
		return -EINVAL;

	if (mddev->array_sectors & geo.chunk_mask)
			/* not factor of array size */
			return -EINVAL;

	if (!enough(conf, -1))
		return -EINVAL;

	kfree(conf->mirrors_new);
	conf->mirrors_new = NULL;
	if (mddev->delta_disks > 0) {
		/* allocate new 'mirrors' list */
		conf->mirrors_new = kzalloc(
3971
			sizeof(struct raid10_info)
N
NeilBrown 已提交
3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056
			*(mddev->raid_disks +
			  mddev->delta_disks),
			GFP_KERNEL);
		if (!conf->mirrors_new)
			return -ENOMEM;
	}
	return 0;
}

/*
 * Need to check if array has failed when deciding whether to:
 *  - start an array
 *  - remove non-faulty devices
 *  - add a spare
 *  - allow a reshape
 * This determination is simple when no reshape is happening.
 * However if there is a reshape, we need to carefully check
 * both the before and after sections.
 * This is because some failed devices may only affect one
 * of the two sections, and some non-in_sync devices may
 * be insync in the section most affected by failed devices.
 */
static int calc_degraded(struct r10conf *conf)
{
	int degraded, degraded2;
	int i;

	rcu_read_lock();
	degraded = 0;
	/* 'prev' section first */
	for (i = 0; i < conf->prev.raid_disks; i++) {
		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
		if (!rdev || test_bit(Faulty, &rdev->flags))
			degraded++;
		else if (!test_bit(In_sync, &rdev->flags))
			/* When we can reduce the number of devices in
			 * an array, this might not contribute to
			 * 'degraded'.  It does now.
			 */
			degraded++;
	}
	rcu_read_unlock();
	if (conf->geo.raid_disks == conf->prev.raid_disks)
		return degraded;
	rcu_read_lock();
	degraded2 = 0;
	for (i = 0; i < conf->geo.raid_disks; i++) {
		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
		if (!rdev || test_bit(Faulty, &rdev->flags))
			degraded2++;
		else if (!test_bit(In_sync, &rdev->flags)) {
			/* If reshape is increasing the number of devices,
			 * this section has already been recovered, so
			 * it doesn't contribute to degraded.
			 * else it does.
			 */
			if (conf->geo.raid_disks <= conf->prev.raid_disks)
				degraded2++;
		}
	}
	rcu_read_unlock();
	if (degraded2 > degraded)
		return degraded2;
	return degraded;
}

static int raid10_start_reshape(struct mddev *mddev)
{
	/* A 'reshape' has been requested. This commits
	 * the various 'new' fields and sets MD_RECOVER_RESHAPE
	 * This also checks if there are enough spares and adds them
	 * to the array.
	 * We currently require enough spares to make the final
	 * array non-degraded.  We also require that the difference
	 * between old and new data_offset - on each device - is
	 * enough that we never risk over-writing.
	 */

	unsigned long before_length, after_length;
	sector_t min_offset_diff = 0;
	int first = 1;
	struct geom new;
	struct r10conf *conf = mddev->private;
	struct md_rdev *rdev;
	int spares = 0;
4057
	int ret;
N
NeilBrown 已提交
4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082

	if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
		return -EBUSY;

	if (setup_geo(&new, mddev, geo_start) != conf->copies)
		return -EINVAL;

	before_length = ((1 << conf->prev.chunk_shift) *
			 conf->prev.far_copies);
	after_length = ((1 << conf->geo.chunk_shift) *
			conf->geo.far_copies);

	rdev_for_each(rdev, mddev) {
		if (!test_bit(In_sync, &rdev->flags)
		    && !test_bit(Faulty, &rdev->flags))
			spares++;
		if (rdev->raid_disk >= 0) {
			long long diff = (rdev->new_data_offset
					  - rdev->data_offset);
			if (!mddev->reshape_backwards)
				diff = -diff;
			if (diff < 0)
				diff = 0;
			if (first || diff < min_offset_diff)
				min_offset_diff = diff;
4083
			first = 0;
N
NeilBrown 已提交
4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096
		}
	}

	if (max(before_length, after_length) > min_offset_diff)
		return -EINVAL;

	if (spares < mddev->delta_disks)
		return -EINVAL;

	conf->offset_diff = min_offset_diff;
	spin_lock_irq(&conf->device_lock);
	if (conf->mirrors_new) {
		memcpy(conf->mirrors_new, conf->mirrors,
4097
		       sizeof(struct raid10_info)*conf->prev.raid_disks);
N
NeilBrown 已提交
4098
		smp_mb();
4099
		kfree(conf->mirrors_old);
N
NeilBrown 已提交
4100 4101 4102 4103 4104 4105 4106 4107 4108 4109
		conf->mirrors_old = conf->mirrors;
		conf->mirrors = conf->mirrors_new;
		conf->mirrors_new = NULL;
	}
	setup_geo(&conf->geo, mddev, geo_start);
	smp_mb();
	if (mddev->reshape_backwards) {
		sector_t size = raid10_size(mddev, 0, 0);
		if (size < mddev->array_sectors) {
			spin_unlock_irq(&conf->device_lock);
N
NeilBrown 已提交
4110 4111
			pr_warn("md/raid10:%s: array size must be reduce before number of disks\n",
				mdname(mddev));
N
NeilBrown 已提交
4112 4113 4114 4115 4116 4117
			return -EINVAL;
		}
		mddev->resync_max_sectors = size;
		conf->reshape_progress = size;
	} else
		conf->reshape_progress = 0;
4118
	conf->reshape_safe = conf->reshape_progress;
N
NeilBrown 已提交
4119 4120
	spin_unlock_irq(&conf->device_lock);

4121 4122 4123 4124 4125 4126 4127 4128
	if (mddev->delta_disks && mddev->bitmap) {
		ret = bitmap_resize(mddev->bitmap,
				    raid10_size(mddev, 0,
						conf->geo.raid_disks),
				    0, 0);
		if (ret)
			goto abort;
	}
N
NeilBrown 已提交
4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155 4156 4157
	if (mddev->delta_disks > 0) {
		rdev_for_each(rdev, mddev)
			if (rdev->raid_disk < 0 &&
			    !test_bit(Faulty, &rdev->flags)) {
				if (raid10_add_disk(mddev, rdev) == 0) {
					if (rdev->raid_disk >=
					    conf->prev.raid_disks)
						set_bit(In_sync, &rdev->flags);
					else
						rdev->recovery_offset = 0;

					if (sysfs_link_rdev(mddev, rdev))
						/* Failure here  is OK */;
				}
			} else if (rdev->raid_disk >= conf->prev.raid_disks
				   && !test_bit(Faulty, &rdev->flags)) {
				/* This is a spare that was manually added */
				set_bit(In_sync, &rdev->flags);
			}
	}
	/* When a reshape changes the number of devices,
	 * ->degraded is measured against the larger of the
	 * pre and  post numbers.
	 */
	spin_lock_irq(&conf->device_lock);
	mddev->degraded = calc_degraded(conf);
	spin_unlock_irq(&conf->device_lock);
	mddev->raid_disks = conf->geo.raid_disks;
	mddev->reshape_position = conf->reshape_progress;
4158
	set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
N
NeilBrown 已提交
4159 4160 4161

	clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
	clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
4162
	clear_bit(MD_RECOVERY_DONE, &mddev->recovery);
N
NeilBrown 已提交
4163 4164 4165 4166 4167 4168
	set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
	set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);

	mddev->sync_thread = md_register_thread(md_do_sync, mddev,
						"reshape");
	if (!mddev->sync_thread) {
4169 4170
		ret = -EAGAIN;
		goto abort;
N
NeilBrown 已提交
4171 4172 4173 4174 4175
	}
	conf->reshape_checkpoint = jiffies;
	md_wakeup_thread(mddev->sync_thread);
	md_new_event(mddev);
	return 0;
4176 4177 4178 4179 4180 4181 4182 4183 4184 4185

abort:
	mddev->recovery = 0;
	spin_lock_irq(&conf->device_lock);
	conf->geo = conf->prev;
	mddev->raid_disks = conf->geo.raid_disks;
	rdev_for_each(rdev, mddev)
		rdev->new_data_offset = rdev->data_offset;
	smp_wmb();
	conf->reshape_progress = MaxSector;
4186
	conf->reshape_safe = MaxSector;
4187 4188 4189
	mddev->reshape_position = MaxSector;
	spin_unlock_irq(&conf->device_lock);
	return ret;
N
NeilBrown 已提交
4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229
}

/* Calculate the last device-address that could contain
 * any block from the chunk that includes the array-address 's'
 * and report the next address.
 * i.e. the address returned will be chunk-aligned and after
 * any data that is in the chunk containing 's'.
 */
static sector_t last_dev_address(sector_t s, struct geom *geo)
{
	s = (s | geo->chunk_mask) + 1;
	s >>= geo->chunk_shift;
	s *= geo->near_copies;
	s = DIV_ROUND_UP_SECTOR_T(s, geo->raid_disks);
	s *= geo->far_copies;
	s <<= geo->chunk_shift;
	return s;
}

/* Calculate the first device-address that could contain
 * any block from the chunk that includes the array-address 's'.
 * This too will be the start of a chunk
 */
static sector_t first_dev_address(sector_t s, struct geom *geo)
{
	s >>= geo->chunk_shift;
	s *= geo->near_copies;
	sector_div(s, geo->raid_disks);
	s *= geo->far_copies;
	s <<= geo->chunk_shift;
	return s;
}

static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
				int *skipped)
{
	/* We simply copy at most one chunk (smallest of old and new)
	 * at a time, possibly less if that exceeds RESYNC_PAGES,
	 * or we hit a bad block or something.
	 * This might mean we pause for normal IO in the middle of
4230
	 * a chunk, but that is not a problem as mddev->reshape_position
N
NeilBrown 已提交
4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253
	 * can record any location.
	 *
	 * If we will want to write to a location that isn't
	 * yet recorded as 'safe' (i.e. in metadata on disk) then
	 * we need to flush all reshape requests and update the metadata.
	 *
	 * When reshaping forwards (e.g. to more devices), we interpret
	 * 'safe' as the earliest block which might not have been copied
	 * down yet.  We divide this by previous stripe size and multiply
	 * by previous stripe length to get lowest device offset that we
	 * cannot write to yet.
	 * We interpret 'sector_nr' as an address that we want to write to.
	 * From this we use last_device_address() to find where we might
	 * write to, and first_device_address on the  'safe' position.
	 * If this 'next' write position is after the 'safe' position,
	 * we must update the metadata to increase the 'safe' position.
	 *
	 * When reshaping backwards, we round in the opposite direction
	 * and perform the reverse test:  next write position must not be
	 * less than current safe position.
	 *
	 * In all this the minimum difference in data offsets
	 * (conf->offset_diff - always positive) allows a bit of slack,
4254
	 * so next can be after 'safe', but not by more than offset_diff
N
NeilBrown 已提交
4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273
	 *
	 * We need to prepare all the bios here before we start any IO
	 * to ensure the size we choose is acceptable to all devices.
	 * The means one for each copy for write-out and an extra one for
	 * read-in.
	 * We store the read-in bio in ->master_bio and the others in
	 * ->devs[x].bio and ->devs[x].repl_bio.
	 */
	struct r10conf *conf = mddev->private;
	struct r10bio *r10_bio;
	sector_t next, safe, last;
	int max_sectors;
	int nr_sectors;
	int s;
	struct md_rdev *rdev;
	int need_flush = 0;
	struct bio *blist;
	struct bio *bio, *read_bio;
	int sectors_done = 0;
4274
	struct page **pages;
N
NeilBrown 已提交
4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353

	if (sector_nr == 0) {
		/* If restarting in the middle, skip the initial sectors */
		if (mddev->reshape_backwards &&
		    conf->reshape_progress < raid10_size(mddev, 0, 0)) {
			sector_nr = (raid10_size(mddev, 0, 0)
				     - conf->reshape_progress);
		} else if (!mddev->reshape_backwards &&
			   conf->reshape_progress > 0)
			sector_nr = conf->reshape_progress;
		if (sector_nr) {
			mddev->curr_resync_completed = sector_nr;
			sysfs_notify(&mddev->kobj, NULL, "sync_completed");
			*skipped = 1;
			return sector_nr;
		}
	}

	/* We don't use sector_nr to track where we are up to
	 * as that doesn't work well for ->reshape_backwards.
	 * So just use ->reshape_progress.
	 */
	if (mddev->reshape_backwards) {
		/* 'next' is the earliest device address that we might
		 * write to for this chunk in the new layout
		 */
		next = first_dev_address(conf->reshape_progress - 1,
					 &conf->geo);

		/* 'safe' is the last device address that we might read from
		 * in the old layout after a restart
		 */
		safe = last_dev_address(conf->reshape_safe - 1,
					&conf->prev);

		if (next + conf->offset_diff < safe)
			need_flush = 1;

		last = conf->reshape_progress - 1;
		sector_nr = last & ~(sector_t)(conf->geo.chunk_mask
					       & conf->prev.chunk_mask);
		if (sector_nr + RESYNC_BLOCK_SIZE/512 < last)
			sector_nr = last + 1 - RESYNC_BLOCK_SIZE/512;
	} else {
		/* 'next' is after the last device address that we
		 * might write to for this chunk in the new layout
		 */
		next = last_dev_address(conf->reshape_progress, &conf->geo);

		/* 'safe' is the earliest device address that we might
		 * read from in the old layout after a restart
		 */
		safe = first_dev_address(conf->reshape_safe, &conf->prev);

		/* Need to update metadata if 'next' might be beyond 'safe'
		 * as that would possibly corrupt data
		 */
		if (next > safe + conf->offset_diff)
			need_flush = 1;

		sector_nr = conf->reshape_progress;
		last  = sector_nr | (conf->geo.chunk_mask
				     & conf->prev.chunk_mask);

		if (sector_nr + RESYNC_BLOCK_SIZE/512 <= last)
			last = sector_nr + RESYNC_BLOCK_SIZE/512 - 1;
	}

	if (need_flush ||
	    time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
		/* Need to update reshape_position in metadata */
		wait_barrier(conf);
		mddev->reshape_position = conf->reshape_progress;
		if (mddev->reshape_backwards)
			mddev->curr_resync_completed = raid10_size(mddev, 0, 0)
				- conf->reshape_progress;
		else
			mddev->curr_resync_completed = conf->reshape_progress;
		conf->reshape_checkpoint = jiffies;
4354
		set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
N
NeilBrown 已提交
4355
		md_wakeup_thread(mddev->thread);
4356
		wait_event(mddev->sb_wait, mddev->sb_flags == 0 ||
4357 4358 4359 4360 4361
			   test_bit(MD_RECOVERY_INTR, &mddev->recovery));
		if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
			allow_barrier(conf);
			return sectors_done;
		}
N
NeilBrown 已提交
4362 4363 4364 4365 4366 4367 4368
		conf->reshape_safe = mddev->reshape_position;
		allow_barrier(conf);
	}

read_more:
	/* Now schedule reads for blocks from sector_nr to last */
	r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
4369
	r10_bio->state = 0;
N
NeilBrown 已提交
4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383
	raise_barrier(conf, sectors_done != 0);
	atomic_set(&r10_bio->remaining, 0);
	r10_bio->mddev = mddev;
	r10_bio->sector = sector_nr;
	set_bit(R10BIO_IsReshape, &r10_bio->state);
	r10_bio->sectors = last - sector_nr + 1;
	rdev = read_balance(conf, r10_bio, &max_sectors);
	BUG_ON(!test_bit(R10BIO_Previous, &r10_bio->state));

	if (!rdev) {
		/* Cannot read from here, so need to record bad blocks
		 * on all the target devices.
		 */
		// FIXME
4384
		mempool_free(r10_bio, conf->r10buf_pool);
N
NeilBrown 已提交
4385 4386 4387 4388 4389 4390 4391
		set_bit(MD_RECOVERY_INTR, &mddev->recovery);
		return sectors_done;
	}

	read_bio = bio_alloc_mddev(GFP_KERNEL, RESYNC_PAGES, mddev);

	read_bio->bi_bdev = rdev->bdev;
4392
	read_bio->bi_iter.bi_sector = (r10_bio->devs[r10_bio->read_slot].addr
N
NeilBrown 已提交
4393 4394
			       + rdev->data_offset);
	read_bio->bi_private = r10_bio;
4395
	read_bio->bi_end_io = end_reshape_read;
M
Mike Christie 已提交
4396
	bio_set_op_attrs(read_bio, REQ_OP_READ, 0);
4397
	read_bio->bi_flags &= (~0UL << BIO_RESET_BITS);
4398
	read_bio->bi_error = 0;
N
NeilBrown 已提交
4399
	read_bio->bi_vcnt = 0;
4400
	read_bio->bi_iter.bi_size = 0;
N
NeilBrown 已提交
4401 4402 4403 4404 4405 4406 4407 4408 4409
	r10_bio->master_bio = read_bio;
	r10_bio->read_slot = r10_bio->devs[r10_bio->read_slot].devnum;

	/* Now find the locations in the new layout */
	__raid10_find_phys(&conf->geo, r10_bio);

	blist = read_bio;
	read_bio->bi_next = NULL;

4410
	rcu_read_lock();
N
NeilBrown 已提交
4411 4412 4413 4414 4415
	for (s = 0; s < conf->copies*2; s++) {
		struct bio *b;
		int d = r10_bio->devs[s/2].devnum;
		struct md_rdev *rdev2;
		if (s&1) {
4416
			rdev2 = rcu_dereference(conf->mirrors[d].replacement);
N
NeilBrown 已提交
4417 4418
			b = r10_bio->devs[s/2].repl_bio;
		} else {
4419
			rdev2 = rcu_dereference(conf->mirrors[d].rdev);
N
NeilBrown 已提交
4420 4421 4422 4423
			b = r10_bio->devs[s/2].bio;
		}
		if (!rdev2 || test_bit(Faulty, &rdev2->flags))
			continue;
K
Kent Overstreet 已提交
4424

N
NeilBrown 已提交
4425
		b->bi_bdev = rdev2->bdev;
4426 4427
		b->bi_iter.bi_sector = r10_bio->devs[s/2].addr +
			rdev2->new_data_offset;
N
NeilBrown 已提交
4428
		b->bi_end_io = end_reshape_write;
M
Mike Christie 已提交
4429
		bio_set_op_attrs(b, REQ_OP_WRITE, 0);
N
NeilBrown 已提交
4430 4431 4432 4433 4434 4435 4436
		b->bi_next = blist;
		blist = b;
	}

	/* Now add as many pages as possible to all of these bios. */

	nr_sectors = 0;
4437
	pages = get_resync_pages(r10_bio->devs[0].bio)->pages;
N
NeilBrown 已提交
4438
	for (s = 0 ; s < max_sectors; s += PAGE_SIZE >> 9) {
4439
		struct page *page = pages[s / (PAGE_SIZE >> 9)];
N
NeilBrown 已提交
4440 4441 4442 4443
		int len = (max_sectors - s) << 9;
		if (len > PAGE_SIZE)
			len = PAGE_SIZE;
		for (bio = blist; bio ; bio = bio->bi_next) {
4444 4445 4446 4447 4448
			/*
			 * won't fail because the vec table is big enough
			 * to hold all these pages
			 */
			bio_add_page(bio, page, len, 0);
N
NeilBrown 已提交
4449 4450 4451 4452
		}
		sector_nr += len >> 9;
		nr_sectors += len >> 9;
	}
4453
	rcu_read_unlock();
N
NeilBrown 已提交
4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504
	r10_bio->sectors = nr_sectors;

	/* Now submit the read */
	md_sync_acct(read_bio->bi_bdev, r10_bio->sectors);
	atomic_inc(&r10_bio->remaining);
	read_bio->bi_next = NULL;
	generic_make_request(read_bio);
	sector_nr += nr_sectors;
	sectors_done += nr_sectors;
	if (sector_nr <= last)
		goto read_more;

	/* Now that we have done the whole section we can
	 * update reshape_progress
	 */
	if (mddev->reshape_backwards)
		conf->reshape_progress -= sectors_done;
	else
		conf->reshape_progress += sectors_done;

	return sectors_done;
}

static void end_reshape_request(struct r10bio *r10_bio);
static int handle_reshape_read_error(struct mddev *mddev,
				     struct r10bio *r10_bio);
static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio)
{
	/* Reshape read completed.  Hopefully we have a block
	 * to write out.
	 * If we got a read error then we do sync 1-page reads from
	 * elsewhere until we find the data - or give up.
	 */
	struct r10conf *conf = mddev->private;
	int s;

	if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
		if (handle_reshape_read_error(mddev, r10_bio) < 0) {
			/* Reshape has been aborted */
			md_done_sync(mddev, r10_bio->sectors, 0);
			return;
		}

	/* We definitely have the data in the pages, schedule the
	 * writes.
	 */
	atomic_set(&r10_bio->remaining, 1);
	for (s = 0; s < conf->copies*2; s++) {
		struct bio *b;
		int d = r10_bio->devs[s/2].devnum;
		struct md_rdev *rdev;
4505
		rcu_read_lock();
N
NeilBrown 已提交
4506
		if (s&1) {
4507
			rdev = rcu_dereference(conf->mirrors[d].replacement);
N
NeilBrown 已提交
4508 4509
			b = r10_bio->devs[s/2].repl_bio;
		} else {
4510
			rdev = rcu_dereference(conf->mirrors[d].rdev);
N
NeilBrown 已提交
4511 4512
			b = r10_bio->devs[s/2].bio;
		}
4513 4514
		if (!rdev || test_bit(Faulty, &rdev->flags)) {
			rcu_read_unlock();
N
NeilBrown 已提交
4515
			continue;
4516
		}
N
NeilBrown 已提交
4517
		atomic_inc(&rdev->nr_pending);
4518
		rcu_read_unlock();
N
NeilBrown 已提交
4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536
		md_sync_acct(b->bi_bdev, r10_bio->sectors);
		atomic_inc(&r10_bio->remaining);
		b->bi_next = NULL;
		generic_make_request(b);
	}
	end_reshape_request(r10_bio);
}

static void end_reshape(struct r10conf *conf)
{
	if (test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery))
		return;

	spin_lock_irq(&conf->device_lock);
	conf->prev = conf->geo;
	md_finish_reshape(conf->mddev);
	smp_wmb();
	conf->reshape_progress = MaxSector;
4537
	conf->reshape_safe = MaxSector;
N
NeilBrown 已提交
4538 4539 4540 4541 4542 4543 4544 4545 4546
	spin_unlock_irq(&conf->device_lock);

	/* read-ahead size must cover two whole stripes, which is
	 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices
	 */
	if (conf->mddev->queue) {
		int stripe = conf->geo.raid_disks *
			((conf->mddev->chunk_sectors << 9) / PAGE_SIZE);
		stripe /= conf->geo.near_copies;
4547 4548
		if (conf->mddev->queue->backing_dev_info->ra_pages < 2 * stripe)
			conf->mddev->queue->backing_dev_info->ra_pages = 2 * stripe;
N
NeilBrown 已提交
4549 4550 4551 4552 4553 4554 4555 4556 4557 4558
	}
	conf->fullsync = 0;
}

static int handle_reshape_read_error(struct mddev *mddev,
				     struct r10bio *r10_bio)
{
	/* Use sync reads to get the blocks from somewhere else */
	int sectors = r10_bio->sectors;
	struct r10conf *conf = mddev->private;
4559 4560 4561 4562 4563
	struct {
		struct r10bio r10_bio;
		struct r10dev devs[conf->copies];
	} on_stack;
	struct r10bio *r10b = &on_stack.r10_bio;
N
NeilBrown 已提交
4564 4565
	int slot = 0;
	int idx = 0;
4566 4567 4568 4569
	struct page **pages;

	/* reshape IOs share pages from .devs[0].bio */
	pages = get_resync_pages(r10_bio->devs[0].bio)->pages;
N
NeilBrown 已提交
4570

4571 4572
	r10b->sector = r10_bio->sector;
	__raid10_find_phys(&conf->prev, r10b);
N
NeilBrown 已提交
4573 4574 4575 4576 4577 4578 4579 4580 4581

	while (sectors) {
		int s = sectors;
		int success = 0;
		int first_slot = slot;

		if (s > (PAGE_SIZE >> 9))
			s = PAGE_SIZE >> 9;

4582
		rcu_read_lock();
N
NeilBrown 已提交
4583
		while (!success) {
4584
			int d = r10b->devs[slot].devnum;
4585
			struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev);
N
NeilBrown 已提交
4586 4587 4588 4589 4590 4591
			sector_t addr;
			if (rdev == NULL ||
			    test_bit(Faulty, &rdev->flags) ||
			    !test_bit(In_sync, &rdev->flags))
				goto failed;

4592
			addr = r10b->devs[slot].addr + idx * PAGE_SIZE;
4593 4594
			atomic_inc(&rdev->nr_pending);
			rcu_read_unlock();
N
NeilBrown 已提交
4595 4596 4597
			success = sync_page_io(rdev,
					       addr,
					       s << 9,
4598
					       pages[idx],
M
Mike Christie 已提交
4599
					       REQ_OP_READ, 0, false);
4600 4601
			rdev_dec_pending(rdev, mddev);
			rcu_read_lock();
N
NeilBrown 已提交
4602 4603 4604 4605 4606 4607 4608 4609 4610
			if (success)
				break;
		failed:
			slot++;
			if (slot >= conf->copies)
				slot = 0;
			if (slot == first_slot)
				break;
		}
4611
		rcu_read_unlock();
N
NeilBrown 已提交
4612 4613 4614 4615 4616 4617 4618 4619 4620 4621 4622 4623
		if (!success) {
			/* couldn't read this block, must give up */
			set_bit(MD_RECOVERY_INTR,
				&mddev->recovery);
			return -EIO;
		}
		sectors -= s;
		idx++;
	}
	return 0;
}

4624
static void end_reshape_write(struct bio *bio)
N
NeilBrown 已提交
4625
{
4626
	struct r10bio *r10_bio = get_resync_r10bio(bio);
N
NeilBrown 已提交
4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641
	struct mddev *mddev = r10_bio->mddev;
	struct r10conf *conf = mddev->private;
	int d;
	int slot;
	int repl;
	struct md_rdev *rdev = NULL;

	d = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
	if (repl)
		rdev = conf->mirrors[d].replacement;
	if (!rdev) {
		smp_mb();
		rdev = conf->mirrors[d].rdev;
	}

4642
	if (bio->bi_error) {
N
NeilBrown 已提交
4643 4644 4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661 4662 4663 4664 4665 4666 4667 4668 4669 4670 4671 4672 4673 4674
		/* FIXME should record badblock */
		md_error(mddev, rdev);
	}

	rdev_dec_pending(rdev, mddev);
	end_reshape_request(r10_bio);
}

static void end_reshape_request(struct r10bio *r10_bio)
{
	if (!atomic_dec_and_test(&r10_bio->remaining))
		return;
	md_done_sync(r10_bio->mddev, r10_bio->sectors, 1);
	bio_put(r10_bio->master_bio);
	put_buf(r10_bio);
}

static void raid10_finish_reshape(struct mddev *mddev)
{
	struct r10conf *conf = mddev->private;

	if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
		return;

	if (mddev->delta_disks > 0) {
		sector_t size = raid10_size(mddev, 0, 0);
		md_set_array_sectors(mddev, size);
		if (mddev->recovery_cp > mddev->resync_max_sectors) {
			mddev->recovery_cp = mddev->resync_max_sectors;
			set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
		}
		mddev->resync_max_sectors = size;
4675 4676 4677 4678
		if (mddev->queue) {
			set_capacity(mddev->gendisk, mddev->array_sectors);
			revalidate_disk(mddev->gendisk);
		}
4679 4680
	} else {
		int d;
4681
		rcu_read_lock();
4682 4683 4684
		for (d = conf->geo.raid_disks ;
		     d < conf->geo.raid_disks - mddev->delta_disks;
		     d++) {
4685
			struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev);
4686 4687
			if (rdev)
				clear_bit(In_sync, &rdev->flags);
4688
			rdev = rcu_dereference(conf->mirrors[d].replacement);
4689 4690 4691
			if (rdev)
				clear_bit(In_sync, &rdev->flags);
		}
4692
		rcu_read_unlock();
N
NeilBrown 已提交
4693 4694 4695 4696 4697 4698 4699 4700
	}
	mddev->layout = mddev->new_layout;
	mddev->chunk_sectors = 1 << conf->geo.chunk_shift;
	mddev->reshape_position = MaxSector;
	mddev->delta_disks = 0;
	mddev->reshape_backwards = 0;
}

4701
static struct md_personality raid10_personality =
L
Linus Torvalds 已提交
4702 4703
{
	.name		= "raid10",
4704
	.level		= 10,
L
Linus Torvalds 已提交
4705
	.owner		= THIS_MODULE,
S
Shaohua Li 已提交
4706 4707
	.make_request	= raid10_make_request,
	.run		= raid10_run,
N
NeilBrown 已提交
4708
	.free		= raid10_free,
S
Shaohua Li 已提交
4709 4710
	.status		= raid10_status,
	.error_handler	= raid10_error,
L
Linus Torvalds 已提交
4711 4712 4713
	.hot_add_disk	= raid10_add_disk,
	.hot_remove_disk= raid10_remove_disk,
	.spare_active	= raid10_spare_active,
S
Shaohua Li 已提交
4714
	.sync_request	= raid10_sync_request,
4715
	.quiesce	= raid10_quiesce,
4716
	.size		= raid10_size,
4717
	.resize		= raid10_resize,
4718
	.takeover	= raid10_takeover,
N
NeilBrown 已提交
4719 4720 4721
	.check_reshape	= raid10_check_reshape,
	.start_reshape	= raid10_start_reshape,
	.finish_reshape	= raid10_finish_reshape,
4722
	.congested	= raid10_congested,
L
Linus Torvalds 已提交
4723 4724 4725 4726
};

static int __init raid_init(void)
{
4727
	return register_md_personality(&raid10_personality);
L
Linus Torvalds 已提交
4728 4729 4730 4731
}

static void raid_exit(void)
{
4732
	unregister_md_personality(&raid10_personality);
L
Linus Torvalds 已提交
4733 4734 4735 4736 4737
}

module_init(raid_init);
module_exit(raid_exit);
MODULE_LICENSE("GPL");
4738
MODULE_DESCRIPTION("RAID10 (striped mirror) personality for MD");
L
Linus Torvalds 已提交
4739
MODULE_ALIAS("md-personality-9"); /* RAID10 */
4740
MODULE_ALIAS("md-raid10");
4741
MODULE_ALIAS("md-level-10");
4742 4743

module_param(max_queued_requests, int, S_IRUGO|S_IWUSR);