raid10.c 127.8 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7
/*
 * raid10.c : Multiple Devices driver for Linux
 *
 * Copyright (C) 2000-2004 Neil Brown
 *
 * RAID-10 support for md.
 *
L
Lucas De Marchi 已提交
8
 * Base on code in raid1.c.  See raid1.c for further copyright information.
L
Linus Torvalds 已提交
9 10 11 12 13 14 15 16 17 18 19 20
 *
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2, or (at your option)
 * any later version.
 *
 * You should have received a copy of the GNU General Public License
 * (for example /usr/src/linux/COPYING); if not, write to the Free
 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

21
#include <linux/slab.h>
22
#include <linux/delay.h>
23
#include <linux/blkdev.h>
24
#include <linux/module.h>
25
#include <linux/seq_file.h>
26
#include <linux/ratelimit.h>
27
#include <linux/kthread.h>
28
#include "md.h"
29
#include "raid10.h"
30
#include "raid0.h"
31
#include "bitmap.h"
L
Linus Torvalds 已提交
32 33 34 35 36 37 38 39

/*
 * RAID10 provides a combination of RAID0 and RAID1 functionality.
 * The layout of data is defined by
 *    chunk_size
 *    raid_disks
 *    near_copies (stored in low byte of layout)
 *    far_copies (stored in second byte of layout)
40
 *    far_offset (stored in bit 16 of layout )
41
 *    use_far_sets (stored in bit 17 of layout )
42
 *    use_far_sets_bugfixed (stored in bit 18 of layout )
L
Linus Torvalds 已提交
43
 *
44 45 46 47 48 49 50 51
 * The data to be stored is divided into chunks using chunksize.  Each device
 * is divided into far_copies sections.   In each section, chunks are laid out
 * in a style similar to raid0, but near_copies copies of each chunk is stored
 * (each on a different drive).  The starting device for each section is offset
 * near_copies from the starting device of the previous section.  Thus there
 * are (near_copies * far_copies) of each chunk, and each is on a different
 * drive.  near_copies and far_copies must be at least one, and their product
 * is at most raid_disks.
52 53
 *
 * If far_offset is true, then the far_copies are handled a bit differently.
54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71
 * The copies are still in different stripes, but instead of being very far
 * apart on disk, there are adjacent stripes.
 *
 * The far and offset algorithms are handled slightly differently if
 * 'use_far_sets' is true.  In this case, the array's devices are grouped into
 * sets that are (near_copies * far_copies) in size.  The far copied stripes
 * are still shifted by 'near_copies' devices, but this shifting stays confined
 * to the set rather than the entire array.  This is done to improve the number
 * of device combinations that can fail without causing the array to fail.
 * Example 'far' algorithm w/o 'use_far_sets' (each letter represents a chunk
 * on a device):
 *    A B C D    A B C D E
 *      ...         ...
 *    D A B C    E A B C D
 * Example 'far' algorithm w/ 'use_far_sets' enabled (sets illustrated w/ []'s):
 *    [A B] [C D]    [A B] [C D E]
 *    |...| |...|    |...| | ... |
 *    [B A] [D C]    [B A] [E C D]
L
Linus Torvalds 已提交
72 73 74 75 76 77 78
 */

/*
 * Number of guaranteed r10bios in case of extreme VM load:
 */
#define	NR_RAID10_BIOS 256

79 80 81 82 83 84 85 86 87 88 89 90 91 92 93
/* when we get a read error on a read-only array, we redirect to another
 * device without failing the first device, or trying to over-write to
 * correct the read error.  To keep track of bad blocks on a per-bio
 * level, we store IO_BLOCKED in the appropriate 'bios' pointer
 */
#define IO_BLOCKED ((struct bio *)1)
/* When we successfully write to a known bad-block, we need to remove the
 * bad-block marking which must be done from process context.  So we record
 * the success by setting devs[n].bio to IO_MADE_GOOD
 */
#define IO_MADE_GOOD ((struct bio *)2)

#define BIO_SPECIAL(bio) ((unsigned long)bio <= 2)

/* When there are this many requests queued to be written by
94 95 96 97 98
 * the raid10 thread, we become 'congested' to provide back-pressure
 * for writeback.
 */
static int max_queued_requests = 1024;

99 100
static void allow_barrier(struct r10conf *conf);
static void lower_barrier(struct r10conf *conf);
101
static int _enough(struct r10conf *conf, int previous, int ignore);
102 103 104
static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
				int *skipped);
static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio);
105
static void end_reshape_write(struct bio *bio);
106
static void end_reshape(struct r10conf *conf);
107

108
static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data)
L
Linus Torvalds 已提交
109
{
110
	struct r10conf *conf = data;
111
	int size = offsetof(struct r10bio, devs[conf->copies]);
L
Linus Torvalds 已提交
112

113 114
	/* allocate a r10bio with room for raid_disks entries in the
	 * bios array */
115
	return kzalloc(size, gfp_flags);
L
Linus Torvalds 已提交
116 117 118 119 120 121 122
}

static void r10bio_pool_free(void *r10_bio, void *data)
{
	kfree(r10_bio);
}

123
/* Maximum size of each resync request */
L
Linus Torvalds 已提交
124 125
#define RESYNC_BLOCK_SIZE (64*1024)
#define RESYNC_PAGES ((RESYNC_BLOCK_SIZE + PAGE_SIZE-1) / PAGE_SIZE)
126 127 128 129
/* amount of memory to reserve for resync requests */
#define RESYNC_WINDOW (1024*1024)
/* maximum number of concurrent requests, memory permitting */
#define RESYNC_DEPTH (32*1024*1024/RESYNC_BLOCK_SIZE)
L
Linus Torvalds 已提交
130 131 132 133 134 135 136 137

/*
 * When performing a resync, we need to read and compare, so
 * we need as many pages are there are copies.
 * When performing a recovery, we need 2 bios, one for read,
 * one for write (we recover only one drive per r10buf)
 *
 */
138
static void * r10buf_pool_alloc(gfp_t gfp_flags, void *data)
L
Linus Torvalds 已提交
139
{
140
	struct r10conf *conf = data;
L
Linus Torvalds 已提交
141
	struct page *page;
142
	struct r10bio *r10_bio;
L
Linus Torvalds 已提交
143 144 145 146 147
	struct bio *bio;
	int i, j;
	int nalloc;

	r10_bio = r10bio_pool_alloc(gfp_flags, conf);
148
	if (!r10_bio)
L
Linus Torvalds 已提交
149 150
		return NULL;

151 152
	if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery) ||
	    test_bit(MD_RECOVERY_RESHAPE, &conf->mddev->recovery))
L
Linus Torvalds 已提交
153 154 155 156 157 158 159 160
		nalloc = conf->copies; /* resync */
	else
		nalloc = 2; /* recovery */

	/*
	 * Allocate bios.
	 */
	for (j = nalloc ; j-- ; ) {
161
		bio = bio_kmalloc(gfp_flags, RESYNC_PAGES);
L
Linus Torvalds 已提交
162 163 164
		if (!bio)
			goto out_free_bio;
		r10_bio->devs[j].bio = bio;
165 166 167 168 169 170
		if (!conf->have_replacement)
			continue;
		bio = bio_kmalloc(gfp_flags, RESYNC_PAGES);
		if (!bio)
			goto out_free_bio;
		r10_bio->devs[j].repl_bio = bio;
L
Linus Torvalds 已提交
171 172 173 174 175 176
	}
	/*
	 * Allocate RESYNC_PAGES data pages and attach them
	 * where needed.
	 */
	for (j = 0 ; j < nalloc; j++) {
177
		struct bio *rbio = r10_bio->devs[j].repl_bio;
L
Linus Torvalds 已提交
178 179
		bio = r10_bio->devs[j].bio;
		for (i = 0; i < RESYNC_PAGES; i++) {
180 181 182 183
			if (j > 0 && !test_bit(MD_RECOVERY_SYNC,
					       &conf->mddev->recovery)) {
				/* we can share bv_page's during recovery
				 * and reshape */
184 185 186 187 188
				struct bio *rbio = r10_bio->devs[0].bio;
				page = rbio->bi_io_vec[i].bv_page;
				get_page(page);
			} else
				page = alloc_page(gfp_flags);
L
Linus Torvalds 已提交
189 190 191 192
			if (unlikely(!page))
				goto out_free_pages;

			bio->bi_io_vec[i].bv_page = page;
193 194
			if (rbio)
				rbio->bi_io_vec[i].bv_page = page;
L
Linus Torvalds 已提交
195 196 197 198 199 200 201
		}
	}

	return r10_bio;

out_free_pages:
	for ( ; i > 0 ; i--)
202
		safe_put_page(bio->bi_io_vec[i-1].bv_page);
L
Linus Torvalds 已提交
203 204
	while (j--)
		for (i = 0; i < RESYNC_PAGES ; i++)
205
			safe_put_page(r10_bio->devs[j].bio->bi_io_vec[i].bv_page);
206
	j = 0;
L
Linus Torvalds 已提交
207
out_free_bio:
208 209 210
	for ( ; j < nalloc; j++) {
		if (r10_bio->devs[j].bio)
			bio_put(r10_bio->devs[j].bio);
211 212 213
		if (r10_bio->devs[j].repl_bio)
			bio_put(r10_bio->devs[j].repl_bio);
	}
L
Linus Torvalds 已提交
214 215 216 217 218 219 220
	r10bio_pool_free(r10_bio, conf);
	return NULL;
}

static void r10buf_pool_free(void *__r10_bio, void *data)
{
	int i;
221
	struct r10conf *conf = data;
222
	struct r10bio *r10bio = __r10_bio;
L
Linus Torvalds 已提交
223 224 225 226 227 228
	int j;

	for (j=0; j < conf->copies; j++) {
		struct bio *bio = r10bio->devs[j].bio;
		if (bio) {
			for (i = 0; i < RESYNC_PAGES; i++) {
229
				safe_put_page(bio->bi_io_vec[i].bv_page);
L
Linus Torvalds 已提交
230 231 232 233
				bio->bi_io_vec[i].bv_page = NULL;
			}
			bio_put(bio);
		}
234 235 236
		bio = r10bio->devs[j].repl_bio;
		if (bio)
			bio_put(bio);
L
Linus Torvalds 已提交
237 238 239 240
	}
	r10bio_pool_free(r10bio, conf);
}

241
static void put_all_bios(struct r10conf *conf, struct r10bio *r10_bio)
L
Linus Torvalds 已提交
242 243 244 245 246
{
	int i;

	for (i = 0; i < conf->copies; i++) {
		struct bio **bio = & r10_bio->devs[i].bio;
247
		if (!BIO_SPECIAL(*bio))
L
Linus Torvalds 已提交
248 249
			bio_put(*bio);
		*bio = NULL;
250 251 252 253
		bio = &r10_bio->devs[i].repl_bio;
		if (r10_bio->read_slot < 0 && !BIO_SPECIAL(*bio))
			bio_put(*bio);
		*bio = NULL;
L
Linus Torvalds 已提交
254 255 256
	}
}

257
static void free_r10bio(struct r10bio *r10_bio)
L
Linus Torvalds 已提交
258
{
259
	struct r10conf *conf = r10_bio->mddev->private;
L
Linus Torvalds 已提交
260 261 262 263 264

	put_all_bios(conf, r10_bio);
	mempool_free(r10_bio, conf->r10bio_pool);
}

265
static void put_buf(struct r10bio *r10_bio)
L
Linus Torvalds 已提交
266
{
267
	struct r10conf *conf = r10_bio->mddev->private;
L
Linus Torvalds 已提交
268 269 270

	mempool_free(r10_bio, conf->r10buf_pool);

271
	lower_barrier(conf);
L
Linus Torvalds 已提交
272 273
}

274
static void reschedule_retry(struct r10bio *r10_bio)
L
Linus Torvalds 已提交
275 276
{
	unsigned long flags;
277
	struct mddev *mddev = r10_bio->mddev;
278
	struct r10conf *conf = mddev->private;
L
Linus Torvalds 已提交
279 280 281

	spin_lock_irqsave(&conf->device_lock, flags);
	list_add(&r10_bio->retry_list, &conf->retry_list);
282
	conf->nr_queued ++;
L
Linus Torvalds 已提交
283 284
	spin_unlock_irqrestore(&conf->device_lock, flags);

285 286 287
	/* wake up frozen array... */
	wake_up(&conf->wait_barrier);

L
Linus Torvalds 已提交
288 289 290 291 292 293 294 295
	md_wakeup_thread(mddev->thread);
}

/*
 * raid_end_bio_io() is called when we have finished servicing a mirrored
 * operation and are ready to return a success/failure code to the buffer
 * cache layer.
 */
296
static void raid_end_bio_io(struct r10bio *r10_bio)
L
Linus Torvalds 已提交
297 298
{
	struct bio *bio = r10_bio->master_bio;
299
	int done;
300
	struct r10conf *conf = r10_bio->mddev->private;
L
Linus Torvalds 已提交
301

302 303 304 305 306 307 308 309 310
	if (bio->bi_phys_segments) {
		unsigned long flags;
		spin_lock_irqsave(&conf->device_lock, flags);
		bio->bi_phys_segments--;
		done = (bio->bi_phys_segments == 0);
		spin_unlock_irqrestore(&conf->device_lock, flags);
	} else
		done = 1;
	if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
311
		bio->bi_error = -EIO;
312
	if (done) {
313
		bio_endio(bio);
314 315 316 317 318 319
		/*
		 * Wake up any possible resync thread that waits for the device
		 * to go idle.
		 */
		allow_barrier(conf);
	}
L
Linus Torvalds 已提交
320 321 322 323 324 325
	free_r10bio(r10_bio);
}

/*
 * Update disk head position estimator based on IRQ completion info.
 */
326
static inline void update_head_pos(int slot, struct r10bio *r10_bio)
L
Linus Torvalds 已提交
327
{
328
	struct r10conf *conf = r10_bio->mddev->private;
L
Linus Torvalds 已提交
329 330 331 332 333

	conf->mirrors[r10_bio->devs[slot].devnum].head_position =
		r10_bio->devs[slot].addr + (r10_bio->sectors);
}

334 335 336
/*
 * Find the disk number which triggered given bio
 */
337
static int find_bio_disk(struct r10conf *conf, struct r10bio *r10_bio,
338
			 struct bio *bio, int *slotp, int *replp)
339 340
{
	int slot;
341
	int repl = 0;
342

343
	for (slot = 0; slot < conf->copies; slot++) {
344 345
		if (r10_bio->devs[slot].bio == bio)
			break;
346 347 348 349 350
		if (r10_bio->devs[slot].repl_bio == bio) {
			repl = 1;
			break;
		}
	}
351 352 353 354

	BUG_ON(slot == conf->copies);
	update_head_pos(slot, r10_bio);

355 356
	if (slotp)
		*slotp = slot;
357 358
	if (replp)
		*replp = repl;
359 360 361
	return r10_bio->devs[slot].devnum;
}

362
static void raid10_end_read_request(struct bio *bio)
L
Linus Torvalds 已提交
363
{
364
	int uptodate = !bio->bi_error;
365
	struct r10bio *r10_bio = bio->bi_private;
L
Linus Torvalds 已提交
366
	int slot, dev;
367
	struct md_rdev *rdev;
368
	struct r10conf *conf = r10_bio->mddev->private;
L
Linus Torvalds 已提交
369 370 371

	slot = r10_bio->read_slot;
	dev = r10_bio->devs[slot].devnum;
372
	rdev = r10_bio->devs[slot].rdev;
L
Linus Torvalds 已提交
373 374 375
	/*
	 * this branch is our 'one mirror IO has finished' event handler:
	 */
376 377 378
	update_head_pos(slot, r10_bio);

	if (uptodate) {
L
Linus Torvalds 已提交
379 380 381 382 383 384 385 386 387 388
		/*
		 * Set R10BIO_Uptodate in our master bio, so that
		 * we will return a good error code to the higher
		 * levels even if IO on some other mirrored buffer fails.
		 *
		 * The 'master' represents the composite IO operation to
		 * user-side. So if something waits for IO, then it will
		 * wait for the 'master' bio.
		 */
		set_bit(R10BIO_Uptodate, &r10_bio->state);
389 390 391 392 393 394
	} else {
		/* If all other devices that store this block have
		 * failed, we want to return the error upwards rather
		 * than fail the last device.  Here we redefine
		 * "uptodate" to mean "Don't want to retry"
		 */
395 396
		if (!_enough(conf, test_bit(R10BIO_Previous, &r10_bio->state),
			     rdev->raid_disk))
397 398 399
			uptodate = 1;
	}
	if (uptodate) {
L
Linus Torvalds 已提交
400
		raid_end_bio_io(r10_bio);
401
		rdev_dec_pending(rdev, conf->mddev);
402
	} else {
L
Linus Torvalds 已提交
403
		/*
404
		 * oops, read error - keep the refcount on the rdev
L
Linus Torvalds 已提交
405 406
		 */
		char b[BDEVNAME_SIZE];
407 408 409
		printk_ratelimited(KERN_ERR
				   "md/raid10:%s: %s: rescheduling sector %llu\n",
				   mdname(conf->mddev),
410
				   bdevname(rdev->bdev, b),
411
				   (unsigned long long)r10_bio->sector);
412
		set_bit(R10BIO_ReadError, &r10_bio->state);
L
Linus Torvalds 已提交
413 414 415 416
		reschedule_retry(r10_bio);
	}
}

417
static void close_write(struct r10bio *r10_bio)
418 419 420 421 422 423 424 425 426
{
	/* clear the bitmap if all writes complete successfully */
	bitmap_endwrite(r10_bio->mddev->bitmap, r10_bio->sector,
			r10_bio->sectors,
			!test_bit(R10BIO_Degraded, &r10_bio->state),
			0);
	md_write_end(r10_bio->mddev);
}

427
static void one_write_done(struct r10bio *r10_bio)
428 429 430 431 432 433 434 435 436 437 438 439 440 441
{
	if (atomic_dec_and_test(&r10_bio->remaining)) {
		if (test_bit(R10BIO_WriteError, &r10_bio->state))
			reschedule_retry(r10_bio);
		else {
			close_write(r10_bio);
			if (test_bit(R10BIO_MadeGood, &r10_bio->state))
				reschedule_retry(r10_bio);
			else
				raid_end_bio_io(r10_bio);
		}
	}
}

442
static void raid10_end_write_request(struct bio *bio)
L
Linus Torvalds 已提交
443
{
444
	struct r10bio *r10_bio = bio->bi_private;
445
	int dev;
446
	int dec_rdev = 1;
447
	struct r10conf *conf = r10_bio->mddev->private;
448
	int slot, repl;
449
	struct md_rdev *rdev = NULL;
L
Linus Torvalds 已提交
450

451
	dev = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
L
Linus Torvalds 已提交
452

453 454
	if (repl)
		rdev = conf->mirrors[dev].replacement;
455 456 457
	if (!rdev) {
		smp_rmb();
		repl = 0;
458
		rdev = conf->mirrors[dev].rdev;
459
	}
L
Linus Torvalds 已提交
460 461 462
	/*
	 * this branch is our 'one mirror IO has finished' event handler:
	 */
463
	if (bio->bi_error) {
464 465 466 467 468 469 470
		if (repl)
			/* Never record new bad blocks to replacement,
			 * just fail it.
			 */
			md_error(rdev->mddev, rdev);
		else {
			set_bit(WriteErrorSeen,	&rdev->flags);
471 472 473
			if (!test_and_set_bit(WantReplacement, &rdev->flags))
				set_bit(MD_RECOVERY_NEEDED,
					&rdev->mddev->recovery);
474 475 476
			set_bit(R10BIO_WriteError, &r10_bio->state);
			dec_rdev = 0;
		}
477
	} else {
L
Linus Torvalds 已提交
478 479 480 481 482 483 484 485 486
		/*
		 * Set R10BIO_Uptodate in our master bio, so that
		 * we will return a good error code for to the higher
		 * levels even if IO on some other mirrored buffer fails.
		 *
		 * The 'master' represents the composite IO operation to
		 * user-side. So if something waits for IO, then it will
		 * wait for the 'master' bio.
		 */
487 488 489
		sector_t first_bad;
		int bad_sectors;

490 491 492 493 494 495 496 497 498 499 500
		/*
		 * Do not set R10BIO_Uptodate if the current device is
		 * rebuilding or Faulty. This is because we cannot use
		 * such device for properly reading the data back (we could
		 * potentially use it, if the current write would have felt
		 * before rdev->recovery_offset, but for simplicity we don't
		 * check this here.
		 */
		if (test_bit(In_sync, &rdev->flags) &&
		    !test_bit(Faulty, &rdev->flags))
			set_bit(R10BIO_Uptodate, &r10_bio->state);
L
Linus Torvalds 已提交
501

502
		/* Maybe we can clear some bad blocks. */
503
		if (is_badblock(rdev,
504 505 506 507
				r10_bio->devs[slot].addr,
				r10_bio->sectors,
				&first_bad, &bad_sectors)) {
			bio_put(bio);
508 509 510 511
			if (repl)
				r10_bio->devs[slot].repl_bio = IO_MADE_GOOD;
			else
				r10_bio->devs[slot].bio = IO_MADE_GOOD;
512 513 514 515 516
			dec_rdev = 0;
			set_bit(R10BIO_MadeGood, &r10_bio->state);
		}
	}

L
Linus Torvalds 已提交
517 518 519 520 521
	/*
	 *
	 * Let's see if all mirrored write operations have finished
	 * already.
	 */
522
	one_write_done(r10_bio);
523
	if (dec_rdev)
524
		rdev_dec_pending(rdev, conf->mddev);
L
Linus Torvalds 已提交
525 526 527 528
}

/*
 * RAID10 layout manager
L
Lucas De Marchi 已提交
529
 * As well as the chunksize and raid_disks count, there are two
L
Linus Torvalds 已提交
530 531 532 533 534 535
 * parameters: near_copies and far_copies.
 * near_copies * far_copies must be <= raid_disks.
 * Normally one of these will be 1.
 * If both are 1, we get raid0.
 * If near_copies == raid_disks, we get raid1.
 *
L
Lucas De Marchi 已提交
536
 * Chunks are laid out in raid0 style with near_copies copies of the
L
Linus Torvalds 已提交
537 538 539 540 541 542 543 544 545
 * first chunk, followed by near_copies copies of the next chunk and
 * so on.
 * If far_copies > 1, then after 1/far_copies of the array has been assigned
 * as described above, we start again with a device offset of near_copies.
 * So we effectively have another copy of the whole array further down all
 * the drives, but with blocks on different drives.
 * With this layout, and block is never stored twice on the one device.
 *
 * raid10_find_phys finds the sector offset of a given virtual sector
546
 * on each device that it is on.
L
Linus Torvalds 已提交
547 548 549 550 551
 *
 * raid10_find_virt does the reverse mapping, from a device and a
 * sector offset to a virtual address
 */

552
static void __raid10_find_phys(struct geom *geo, struct r10bio *r10bio)
L
Linus Torvalds 已提交
553 554 555 556 557 558 559
{
	int n,f;
	sector_t sector;
	sector_t chunk;
	sector_t stripe;
	int dev;
	int slot = 0;
560 561 562 563 564 565 566
	int last_far_set_start, last_far_set_size;

	last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1;
	last_far_set_start *= geo->far_set_size;

	last_far_set_size = geo->far_set_size;
	last_far_set_size += (geo->raid_disks % geo->far_set_size);
L
Linus Torvalds 已提交
567 568

	/* now calculate first sector/dev */
569 570
	chunk = r10bio->sector >> geo->chunk_shift;
	sector = r10bio->sector & geo->chunk_mask;
L
Linus Torvalds 已提交
571

572
	chunk *= geo->near_copies;
L
Linus Torvalds 已提交
573
	stripe = chunk;
574 575 576
	dev = sector_div(stripe, geo->raid_disks);
	if (geo->far_offset)
		stripe *= geo->far_copies;
L
Linus Torvalds 已提交
577

578
	sector += stripe << geo->chunk_shift;
L
Linus Torvalds 已提交
579 580

	/* and calculate all the others */
581
	for (n = 0; n < geo->near_copies; n++) {
L
Linus Torvalds 已提交
582
		int d = dev;
583
		int set;
L
Linus Torvalds 已提交
584 585
		sector_t s = sector;
		r10bio->devs[slot].devnum = d;
586
		r10bio->devs[slot].addr = s;
L
Linus Torvalds 已提交
587 588
		slot++;

589
		for (f = 1; f < geo->far_copies; f++) {
590
			set = d / geo->far_set_size;
591
			d += geo->near_copies;
592

593 594 595 596 597 598 599 600 601
			if ((geo->raid_disks % geo->far_set_size) &&
			    (d > last_far_set_start)) {
				d -= last_far_set_start;
				d %= last_far_set_size;
				d += last_far_set_start;
			} else {
				d %= geo->far_set_size;
				d += geo->far_set_size * set;
			}
602
			s += geo->stride;
L
Linus Torvalds 已提交
603 604 605 606 607
			r10bio->devs[slot].devnum = d;
			r10bio->devs[slot].addr = s;
			slot++;
		}
		dev++;
608
		if (dev >= geo->raid_disks) {
L
Linus Torvalds 已提交
609
			dev = 0;
610
			sector += (geo->chunk_mask + 1);
L
Linus Torvalds 已提交
611 612
		}
	}
613 614 615 616 617 618 619 620 621 622 623 624 625 626 627
}

static void raid10_find_phys(struct r10conf *conf, struct r10bio *r10bio)
{
	struct geom *geo = &conf->geo;

	if (conf->reshape_progress != MaxSector &&
	    ((r10bio->sector >= conf->reshape_progress) !=
	     conf->mddev->reshape_backwards)) {
		set_bit(R10BIO_Previous, &r10bio->state);
		geo = &conf->prev;
	} else
		clear_bit(R10BIO_Previous, &r10bio->state);

	__raid10_find_phys(geo, r10bio);
L
Linus Torvalds 已提交
628 629
}

630
static sector_t raid10_find_virt(struct r10conf *conf, sector_t sector, int dev)
L
Linus Torvalds 已提交
631 632
{
	sector_t offset, chunk, vchunk;
633 634 635
	/* Never use conf->prev as this is only called during resync
	 * or recovery, so reshape isn't happening
	 */
636
	struct geom *geo = &conf->geo;
637 638
	int far_set_start = (dev / geo->far_set_size) * geo->far_set_size;
	int far_set_size = geo->far_set_size;
639 640 641 642 643 644 645 646 647 648 649 650
	int last_far_set_start;

	if (geo->raid_disks % geo->far_set_size) {
		last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1;
		last_far_set_start *= geo->far_set_size;

		if (dev >= last_far_set_start) {
			far_set_size = geo->far_set_size;
			far_set_size += (geo->raid_disks % geo->far_set_size);
			far_set_start = last_far_set_start;
		}
	}
L
Linus Torvalds 已提交
651

652 653
	offset = sector & geo->chunk_mask;
	if (geo->far_offset) {
654
		int fc;
655 656 657
		chunk = sector >> geo->chunk_shift;
		fc = sector_div(chunk, geo->far_copies);
		dev -= fc * geo->near_copies;
658 659
		if (dev < far_set_start)
			dev += far_set_size;
660
	} else {
661 662
		while (sector >= geo->stride) {
			sector -= geo->stride;
663 664
			if (dev < (geo->near_copies + far_set_start))
				dev += far_set_size - geo->near_copies;
665
			else
666
				dev -= geo->near_copies;
667
		}
668
		chunk = sector >> geo->chunk_shift;
669
	}
670 671 672
	vchunk = chunk * geo->raid_disks + dev;
	sector_div(vchunk, geo->near_copies);
	return (vchunk << geo->chunk_shift) + offset;
L
Linus Torvalds 已提交
673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693
}

/*
 * This routine returns the disk from which the requested read should
 * be done. There is a per-array 'next expected sequential IO' sector
 * number - if this matches on the next IO then we use the last disk.
 * There is also a per-disk 'last know head position' sector that is
 * maintained from IRQ contexts, both the normal and the resync IO
 * completion handlers update this position correctly. If there is no
 * perfect sequential match then we pick the disk whose head is closest.
 *
 * If there are 2 mirrors in the same 2 devices, performance degrades
 * because position is mirror, not device based.
 *
 * The rdev for the device selected will have nr_pending incremented.
 */

/*
 * FIXME: possibly should rethink readbalancing and do it differently
 * depending on near_copies / far_copies geometry.
 */
694 695 696
static struct md_rdev *read_balance(struct r10conf *conf,
				    struct r10bio *r10_bio,
				    int *max_sectors)
L
Linus Torvalds 已提交
697
{
698
	const sector_t this_sector = r10_bio->sector;
699
	int disk, slot;
700 701
	int sectors = r10_bio->sectors;
	int best_good_sectors;
702
	sector_t new_distance, best_dist;
703
	struct md_rdev *best_rdev, *rdev = NULL;
704 705
	int do_balance;
	int best_slot;
706
	struct geom *geo = &conf->geo;
L
Linus Torvalds 已提交
707 708 709

	raid10_find_phys(conf, r10_bio);
	rcu_read_lock();
710
retry:
711
	sectors = r10_bio->sectors;
712
	best_slot = -1;
713
	best_rdev = NULL;
714
	best_dist = MaxSector;
715
	best_good_sectors = 0;
716
	do_balance = 1;
L
Linus Torvalds 已提交
717 718
	/*
	 * Check if we can balance. We can balance on the whole
719 720 721
	 * device if no resync is going on (recovery is ok), or below
	 * the resync window. We take the first readable disk when
	 * above the resync window.
L
Linus Torvalds 已提交
722 723
	 */
	if (conf->mddev->recovery_cp < MaxSector
724 725
	    && (this_sector + sectors >= conf->next_resync))
		do_balance = 0;
L
Linus Torvalds 已提交
726

727
	for (slot = 0; slot < conf->copies ; slot++) {
728 729 730 731
		sector_t first_bad;
		int bad_sectors;
		sector_t dev_sector;

732 733
		if (r10_bio->devs[slot].bio == IO_BLOCKED)
			continue;
L
Linus Torvalds 已提交
734
		disk = r10_bio->devs[slot].devnum;
735 736 737 738
		rdev = rcu_dereference(conf->mirrors[disk].replacement);
		if (rdev == NULL || test_bit(Faulty, &rdev->flags) ||
		    r10_bio->devs[slot].addr + sectors > rdev->recovery_offset)
			rdev = rcu_dereference(conf->mirrors[disk].rdev);
739
		if (rdev == NULL ||
740
		    test_bit(Faulty, &rdev->flags))
741 742 743
			continue;
		if (!test_bit(In_sync, &rdev->flags) &&
		    r10_bio->devs[slot].addr + sectors > rdev->recovery_offset)
744 745
			continue;

746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767
		dev_sector = r10_bio->devs[slot].addr;
		if (is_badblock(rdev, dev_sector, sectors,
				&first_bad, &bad_sectors)) {
			if (best_dist < MaxSector)
				/* Already have a better slot */
				continue;
			if (first_bad <= dev_sector) {
				/* Cannot read here.  If this is the
				 * 'primary' device, then we must not read
				 * beyond 'bad_sectors' from another device.
				 */
				bad_sectors -= (dev_sector - first_bad);
				if (!do_balance && sectors > bad_sectors)
					sectors = bad_sectors;
				if (best_good_sectors > sectors)
					best_good_sectors = sectors;
			} else {
				sector_t good_sectors =
					first_bad - dev_sector;
				if (good_sectors > best_good_sectors) {
					best_good_sectors = good_sectors;
					best_slot = slot;
768
					best_rdev = rdev;
769 770 771 772 773 774 775 776 777
				}
				if (!do_balance)
					/* Must read from here */
					break;
			}
			continue;
		} else
			best_good_sectors = sectors;

778 779
		if (!do_balance)
			break;
L
Linus Torvalds 已提交
780

781 782 783 784
		/* This optimisation is debatable, and completely destroys
		 * sequential read speed for 'far copies' arrays.  So only
		 * keep it for 'near' arrays, and review those later.
		 */
785
		if (geo->near_copies > 1 && !atomic_read(&rdev->nr_pending))
L
Linus Torvalds 已提交
786
			break;
787 788

		/* for far > 1 always use the lowest address */
789
		if (geo->far_copies > 1)
790
			new_distance = r10_bio->devs[slot].addr;
791
		else
792 793 794 795 796
			new_distance = abs(r10_bio->devs[slot].addr -
					   conf->mirrors[disk].head_position);
		if (new_distance < best_dist) {
			best_dist = new_distance;
			best_slot = slot;
797
			best_rdev = rdev;
L
Linus Torvalds 已提交
798 799
		}
	}
800
	if (slot >= conf->copies) {
801
		slot = best_slot;
802 803
		rdev = best_rdev;
	}
L
Linus Torvalds 已提交
804

805 806 807 808 809 810 811 812 813 814 815
	if (slot >= 0) {
		atomic_inc(&rdev->nr_pending);
		if (test_bit(Faulty, &rdev->flags)) {
			/* Cannot risk returning a device that failed
			 * before we inc'ed nr_pending
			 */
			rdev_dec_pending(rdev, conf->mddev);
			goto retry;
		}
		r10_bio->read_slot = slot;
	} else
816
		rdev = NULL;
L
Linus Torvalds 已提交
817
	rcu_read_unlock();
818
	*max_sectors = best_good_sectors;
L
Linus Torvalds 已提交
819

820
	return rdev;
L
Linus Torvalds 已提交
821 822
}

823
static int raid10_congested(struct mddev *mddev, int bits)
824
{
825
	struct r10conf *conf = mddev->private;
826 827
	int i, ret = 0;

828
	if ((bits & (1 << WB_async_congested)) &&
829 830 831
	    conf->pending_count >= max_queued_requests)
		return 1;

832
	rcu_read_lock();
833 834 835 836
	for (i = 0;
	     (i < conf->geo.raid_disks || i < conf->prev.raid_disks)
		     && ret == 0;
	     i++) {
837
		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
838
		if (rdev && !test_bit(Faulty, &rdev->flags)) {
839
			struct request_queue *q = bdev_get_queue(rdev->bdev);
840 841 842 843 844 845 846 847

			ret |= bdi_congested(&q->backing_dev_info, bits);
		}
	}
	rcu_read_unlock();
	return ret;
}

848
static void flush_pending_writes(struct r10conf *conf)
849 850 851 852 853 854 855 856 857
{
	/* Any writes that have been queued but are awaiting
	 * bitmap updates get flushed here.
	 */
	spin_lock_irq(&conf->device_lock);

	if (conf->pending_bio_list.head) {
		struct bio *bio;
		bio = bio_list_get(&conf->pending_bio_list);
858
		conf->pending_count = 0;
859 860 861 862
		spin_unlock_irq(&conf->device_lock);
		/* flush any pending bitmap writes to disk
		 * before proceeding w/ I/O */
		bitmap_unplug(conf->mddev->bitmap);
863
		wake_up(&conf->wait_barrier);
864 865 866 867

		while (bio) { /* submit pending writes */
			struct bio *next = bio->bi_next;
			bio->bi_next = NULL;
S
Shaohua Li 已提交
868 869 870
			if (unlikely((bio->bi_rw & REQ_DISCARD) &&
			    !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
				/* Just ignore it */
871
				bio_endio(bio);
S
Shaohua Li 已提交
872 873
			else
				generic_make_request(bio);
874 875 876 877 878
			bio = next;
		}
	} else
		spin_unlock_irq(&conf->device_lock);
}
879

880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899
/* Barriers....
 * Sometimes we need to suspend IO while we do something else,
 * either some resync/recovery, or reconfigure the array.
 * To do this we raise a 'barrier'.
 * The 'barrier' is a counter that can be raised multiple times
 * to count how many activities are happening which preclude
 * normal IO.
 * We can only raise the barrier if there is no pending IO.
 * i.e. if nr_pending == 0.
 * We choose only to raise the barrier if no-one is waiting for the
 * barrier to go down.  This means that as soon as an IO request
 * is ready, no other operations which require a barrier will start
 * until the IO request has had a chance.
 *
 * So: regular IO calls 'wait_barrier'.  When that returns there
 *    is no backgroup IO happening,  It must arrange to call
 *    allow_barrier when it has finished its IO.
 * backgroup IO calls must call raise_barrier.  Once that returns
 *    there is no normal IO happeing.  It must arrange to call
 *    lower_barrier when the particular background IO completes.
L
Linus Torvalds 已提交
900 901
 */

902
static void raise_barrier(struct r10conf *conf, int force)
L
Linus Torvalds 已提交
903
{
904
	BUG_ON(force && !conf->barrier);
L
Linus Torvalds 已提交
905
	spin_lock_irq(&conf->resync_lock);
906

907 908
	/* Wait until no block IO is waiting (unless 'force') */
	wait_event_lock_irq(conf->wait_barrier, force || !conf->nr_waiting,
909
			    conf->resync_lock);
910 911 912 913

	/* block any new IO from starting */
	conf->barrier++;

914
	/* Now wait for all pending IO to complete */
915 916
	wait_event_lock_irq(conf->wait_barrier,
			    !conf->nr_pending && conf->barrier < RESYNC_DEPTH,
917
			    conf->resync_lock);
918 919 920 921

	spin_unlock_irq(&conf->resync_lock);
}

922
static void lower_barrier(struct r10conf *conf)
923 924 925 926 927 928 929 930
{
	unsigned long flags;
	spin_lock_irqsave(&conf->resync_lock, flags);
	conf->barrier--;
	spin_unlock_irqrestore(&conf->resync_lock, flags);
	wake_up(&conf->wait_barrier);
}

931
static void wait_barrier(struct r10conf *conf)
932 933 934 935
{
	spin_lock_irq(&conf->resync_lock);
	if (conf->barrier) {
		conf->nr_waiting++;
936 937 938 939 940 941 942 943 944 945 946 947 948 949
		/* Wait for the barrier to drop.
		 * However if there are already pending
		 * requests (preventing the barrier from
		 * rising completely), and the
		 * pre-process bio queue isn't empty,
		 * then don't wait, as we need to empty
		 * that queue to get the nr_pending
		 * count down.
		 */
		wait_event_lock_irq(conf->wait_barrier,
				    !conf->barrier ||
				    (conf->nr_pending &&
				     current->bio_list &&
				     !bio_list_empty(current->bio_list)),
950
				    conf->resync_lock);
951
		conf->nr_waiting--;
L
Linus Torvalds 已提交
952
	}
953
	conf->nr_pending++;
L
Linus Torvalds 已提交
954 955 956
	spin_unlock_irq(&conf->resync_lock);
}

957
static void allow_barrier(struct r10conf *conf)
958 959 960 961 962 963 964 965
{
	unsigned long flags;
	spin_lock_irqsave(&conf->resync_lock, flags);
	conf->nr_pending--;
	spin_unlock_irqrestore(&conf->resync_lock, flags);
	wake_up(&conf->wait_barrier);
}

966
static void freeze_array(struct r10conf *conf, int extra)
967 968
{
	/* stop syncio and normal IO and wait for everything to
969
	 * go quiet.
970
	 * We increment barrier and nr_waiting, and then
971
	 * wait until nr_pending match nr_queued+extra
972 973 974 975
	 * This is called in the context of one normal IO request
	 * that has failed. Thus any sync request that might be pending
	 * will be blocked by nr_pending, and we need to wait for
	 * pending IO requests to complete or be queued for re-try.
976
	 * Thus the number queued (nr_queued) plus this request (extra)
977 978
	 * must match the number of pending IOs (nr_pending) before
	 * we continue.
979 980 981 982
	 */
	spin_lock_irq(&conf->resync_lock);
	conf->barrier++;
	conf->nr_waiting++;
983
	wait_event_lock_irq_cmd(conf->wait_barrier,
984
				conf->nr_pending == conf->nr_queued+extra,
985 986
				conf->resync_lock,
				flush_pending_writes(conf));
987

988 989 990
	spin_unlock_irq(&conf->resync_lock);
}

991
static void unfreeze_array(struct r10conf *conf)
992 993 994 995 996 997 998 999 1000
{
	/* reverse the effect of the freeze */
	spin_lock_irq(&conf->resync_lock);
	conf->barrier--;
	conf->nr_waiting--;
	wake_up(&conf->wait_barrier);
	spin_unlock_irq(&conf->resync_lock);
}

1001 1002 1003 1004 1005 1006 1007 1008 1009 1010
static sector_t choose_data_offset(struct r10bio *r10_bio,
				   struct md_rdev *rdev)
{
	if (!test_bit(MD_RECOVERY_RESHAPE, &rdev->mddev->recovery) ||
	    test_bit(R10BIO_Previous, &r10_bio->state))
		return rdev->data_offset;
	else
		return rdev->new_data_offset;
}

1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024
struct raid10_plug_cb {
	struct blk_plug_cb	cb;
	struct bio_list		pending;
	int			pending_cnt;
};

static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule)
{
	struct raid10_plug_cb *plug = container_of(cb, struct raid10_plug_cb,
						   cb);
	struct mddev *mddev = plug->cb.data;
	struct r10conf *conf = mddev->private;
	struct bio *bio;

1025
	if (from_schedule || current->bio_list) {
1026 1027 1028 1029
		spin_lock_irq(&conf->device_lock);
		bio_list_merge(&conf->pending_bio_list, &plug->pending);
		conf->pending_count += plug->pending_cnt;
		spin_unlock_irq(&conf->device_lock);
1030
		wake_up(&conf->wait_barrier);
1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043
		md_wakeup_thread(mddev->thread);
		kfree(plug);
		return;
	}

	/* we aren't scheduling, so we can do the write-out directly. */
	bio = bio_list_get(&plug->pending);
	bitmap_unplug(mddev->bitmap);
	wake_up(&conf->wait_barrier);

	while (bio) { /* submit pending writes */
		struct bio *next = bio->bi_next;
		bio->bi_next = NULL;
1044 1045 1046
		if (unlikely((bio->bi_rw & REQ_DISCARD) &&
		    !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
			/* Just ignore it */
1047
			bio_endio(bio);
1048 1049
		else
			generic_make_request(bio);
1050 1051 1052 1053 1054
		bio = next;
	}
	kfree(plug);
}

1055
static void __make_request(struct mddev *mddev, struct bio *bio)
L
Linus Torvalds 已提交
1056
{
1057
	struct r10conf *conf = mddev->private;
1058
	struct r10bio *r10_bio;
L
Linus Torvalds 已提交
1059 1060
	struct bio *read_bio;
	int i;
1061
	const int rw = bio_data_dir(bio);
1062
	const unsigned long do_sync = (bio->bi_rw & REQ_SYNC);
1063
	const unsigned long do_fua = (bio->bi_rw & REQ_FUA);
S
Shaohua Li 已提交
1064 1065
	const unsigned long do_discard = (bio->bi_rw
					  & (REQ_DISCARD | REQ_SECURE));
1066
	const unsigned long do_same = (bio->bi_rw & REQ_WRITE_SAME);
1067
	unsigned long flags;
1068
	struct md_rdev *blocked_rdev;
1069 1070
	struct blk_plug_cb *cb;
	struct raid10_plug_cb *plug = NULL;
1071 1072
	int sectors_handled;
	int max_sectors;
1073
	int sectors;
L
Linus Torvalds 已提交
1074

1075 1076 1077 1078 1079 1080 1081
	/*
	 * Register the new request and wait if the reconstruction
	 * thread has put up a bar for new requests.
	 * Continue immediately if no resync is active currently.
	 */
	wait_barrier(conf);

1082
	sectors = bio_sectors(bio);
1083
	while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
1084 1085
	    bio->bi_iter.bi_sector < conf->reshape_progress &&
	    bio->bi_iter.bi_sector + sectors > conf->reshape_progress) {
1086 1087 1088 1089 1090
		/* IO spans the reshape position.  Need to wait for
		 * reshape to pass
		 */
		allow_barrier(conf);
		wait_event(conf->wait_barrier,
1091 1092 1093
			   conf->reshape_progress <= bio->bi_iter.bi_sector ||
			   conf->reshape_progress >= bio->bi_iter.bi_sector +
			   sectors);
1094 1095 1096 1097 1098
		wait_barrier(conf);
	}
	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
	    bio_data_dir(bio) == WRITE &&
	    (mddev->reshape_backwards
1099 1100 1101 1102
	     ? (bio->bi_iter.bi_sector < conf->reshape_safe &&
		bio->bi_iter.bi_sector + sectors > conf->reshape_progress)
	     : (bio->bi_iter.bi_sector + sectors > conf->reshape_safe &&
		bio->bi_iter.bi_sector < conf->reshape_progress))) {
1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113
		/* Need to update reshape_position in metadata */
		mddev->reshape_position = conf->reshape_progress;
		set_bit(MD_CHANGE_DEVS, &mddev->flags);
		set_bit(MD_CHANGE_PENDING, &mddev->flags);
		md_wakeup_thread(mddev->thread);
		wait_event(mddev->sb_wait,
			   !test_bit(MD_CHANGE_PENDING, &mddev->flags));

		conf->reshape_safe = mddev->reshape_position;
	}

L
Linus Torvalds 已提交
1114 1115 1116
	r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);

	r10_bio->master_bio = bio;
1117
	r10_bio->sectors = sectors;
L
Linus Torvalds 已提交
1118 1119

	r10_bio->mddev = mddev;
1120
	r10_bio->sector = bio->bi_iter.bi_sector;
1121
	r10_bio->state = 0;
L
Linus Torvalds 已提交
1122

1123 1124 1125 1126 1127 1128 1129 1130
	/* We might need to issue multiple reads to different
	 * devices if there are bad blocks around, so we keep
	 * track of the number of reads in bio->bi_phys_segments.
	 * If this is 0, there is only one r10_bio and no locking
	 * will be needed when the request completes.  If it is
	 * non-zero, then it is the number of not-completed requests.
	 */
	bio->bi_phys_segments = 0;
1131
	bio_clear_flag(bio, BIO_SEG_VALID);
1132

1133
	if (rw == READ) {
L
Linus Torvalds 已提交
1134 1135 1136
		/*
		 * read balancing logic:
		 */
1137
		struct md_rdev *rdev;
1138 1139 1140
		int slot;

read_again:
1141 1142
		rdev = read_balance(conf, r10_bio, &max_sectors);
		if (!rdev) {
L
Linus Torvalds 已提交
1143
			raid_end_bio_io(r10_bio);
1144
			return;
L
Linus Torvalds 已提交
1145
		}
1146
		slot = r10_bio->read_slot;
L
Linus Torvalds 已提交
1147

1148
		read_bio = bio_clone_mddev(bio, GFP_NOIO, mddev);
1149
		bio_trim(read_bio, r10_bio->sector - bio->bi_iter.bi_sector,
1150
			 max_sectors);
L
Linus Torvalds 已提交
1151 1152

		r10_bio->devs[slot].bio = read_bio;
1153
		r10_bio->devs[slot].rdev = rdev;
L
Linus Torvalds 已提交
1154

1155
		read_bio->bi_iter.bi_sector = r10_bio->devs[slot].addr +
1156
			choose_data_offset(r10_bio, rdev);
1157
		read_bio->bi_bdev = rdev->bdev;
L
Linus Torvalds 已提交
1158
		read_bio->bi_end_io = raid10_end_read_request;
1159
		read_bio->bi_rw = READ | do_sync;
L
Linus Torvalds 已提交
1160 1161
		read_bio->bi_private = r10_bio;

1162 1163 1164 1165
		if (max_sectors < r10_bio->sectors) {
			/* Could not read all from this device, so we will
			 * need another r10_bio.
			 */
1166
			sectors_handled = (r10_bio->sector + max_sectors
1167
					   - bio->bi_iter.bi_sector);
1168 1169 1170 1171 1172 1173
			r10_bio->sectors = max_sectors;
			spin_lock_irq(&conf->device_lock);
			if (bio->bi_phys_segments == 0)
				bio->bi_phys_segments = 2;
			else
				bio->bi_phys_segments++;
1174
			spin_unlock_irq(&conf->device_lock);
1175 1176 1177 1178 1179 1180 1181 1182 1183 1184
			/* Cannot call generic_make_request directly
			 * as that will be queued in __generic_make_request
			 * and subsequent mempool_alloc might block
			 * waiting for it.  so hand bio over to raid10d.
			 */
			reschedule_retry(r10_bio);

			r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);

			r10_bio->master_bio = bio;
1185
			r10_bio->sectors = bio_sectors(bio) - sectors_handled;
1186 1187
			r10_bio->state = 0;
			r10_bio->mddev = mddev;
1188 1189
			r10_bio->sector = bio->bi_iter.bi_sector +
				sectors_handled;
1190 1191 1192
			goto read_again;
		} else
			generic_make_request(read_bio);
1193
		return;
L
Linus Torvalds 已提交
1194 1195 1196 1197 1198
	}

	/*
	 * WRITE:
	 */
1199 1200 1201 1202 1203
	if (conf->pending_count >= max_queued_requests) {
		md_wakeup_thread(mddev->thread);
		wait_event(conf->wait_barrier,
			   conf->pending_count < max_queued_requests);
	}
1204
	/* first select target devices under rcu_lock and
L
Linus Torvalds 已提交
1205 1206
	 * inc refcount on their rdev.  Record them by setting
	 * bios[x] to bio
1207 1208 1209 1210 1211 1212 1213
	 * If there are known/acknowledged bad blocks on any device
	 * on which we have seen a write error, we want to avoid
	 * writing to those blocks.  This potentially requires several
	 * writes to write around the bad blocks.  Each set of writes
	 * gets its own r10_bio with a set of bios attached.  The number
	 * of r10_bios is recored in bio->bi_phys_segments just as with
	 * the read case.
L
Linus Torvalds 已提交
1214
	 */
1215

1216
	r10_bio->read_slot = -1; /* make sure repl_bio gets freed */
L
Linus Torvalds 已提交
1217
	raid10_find_phys(conf, r10_bio);
1218
retry_write:
1219
	blocked_rdev = NULL;
L
Linus Torvalds 已提交
1220
	rcu_read_lock();
1221 1222
	max_sectors = r10_bio->sectors;

L
Linus Torvalds 已提交
1223 1224
	for (i = 0;  i < conf->copies; i++) {
		int d = r10_bio->devs[i].devnum;
1225
		struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev);
1226 1227
		struct md_rdev *rrdev = rcu_dereference(
			conf->mirrors[d].replacement);
1228 1229
		if (rdev == rrdev)
			rrdev = NULL;
1230 1231 1232 1233 1234
		if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) {
			atomic_inc(&rdev->nr_pending);
			blocked_rdev = rdev;
			break;
		}
1235 1236 1237 1238 1239
		if (rrdev && unlikely(test_bit(Blocked, &rrdev->flags))) {
			atomic_inc(&rrdev->nr_pending);
			blocked_rdev = rrdev;
			break;
		}
1240
		if (rdev && (test_bit(Faulty, &rdev->flags)))
1241
			rdev = NULL;
1242
		if (rrdev && (test_bit(Faulty, &rrdev->flags)))
1243 1244
			rrdev = NULL;

1245
		r10_bio->devs[i].bio = NULL;
1246
		r10_bio->devs[i].repl_bio = NULL;
1247 1248

		if (!rdev && !rrdev) {
1249
			set_bit(R10BIO_Degraded, &r10_bio->state);
1250 1251
			continue;
		}
1252
		if (rdev && test_bit(WriteErrorSeen, &rdev->flags)) {
1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292
			sector_t first_bad;
			sector_t dev_sector = r10_bio->devs[i].addr;
			int bad_sectors;
			int is_bad;

			is_bad = is_badblock(rdev, dev_sector,
					     max_sectors,
					     &first_bad, &bad_sectors);
			if (is_bad < 0) {
				/* Mustn't write here until the bad block
				 * is acknowledged
				 */
				atomic_inc(&rdev->nr_pending);
				set_bit(BlockedBadBlocks, &rdev->flags);
				blocked_rdev = rdev;
				break;
			}
			if (is_bad && first_bad <= dev_sector) {
				/* Cannot write here at all */
				bad_sectors -= (dev_sector - first_bad);
				if (bad_sectors < max_sectors)
					/* Mustn't write more than bad_sectors
					 * to other devices yet
					 */
					max_sectors = bad_sectors;
				/* We don't set R10BIO_Degraded as that
				 * only applies if the disk is missing,
				 * so it might be re-added, and we want to
				 * know to recover this chunk.
				 * In this case the device is here, and the
				 * fact that this chunk is not in-sync is
				 * recorded in the bad block log.
				 */
				continue;
			}
			if (is_bad) {
				int good_sectors = first_bad - dev_sector;
				if (good_sectors < max_sectors)
					max_sectors = good_sectors;
			}
1293
		}
1294 1295 1296 1297
		if (rdev) {
			r10_bio->devs[i].bio = bio;
			atomic_inc(&rdev->nr_pending);
		}
1298 1299 1300 1301
		if (rrdev) {
			r10_bio->devs[i].repl_bio = bio;
			atomic_inc(&rrdev->nr_pending);
		}
L
Linus Torvalds 已提交
1302 1303 1304
	}
	rcu_read_unlock();

1305 1306 1307 1308 1309
	if (unlikely(blocked_rdev)) {
		/* Have to wait for this device to get unblocked, then retry */
		int j;
		int d;

1310
		for (j = 0; j < i; j++) {
1311 1312 1313 1314
			if (r10_bio->devs[j].bio) {
				d = r10_bio->devs[j].devnum;
				rdev_dec_pending(conf->mirrors[d].rdev, mddev);
			}
1315
			if (r10_bio->devs[j].repl_bio) {
1316
				struct md_rdev *rdev;
1317
				d = r10_bio->devs[j].devnum;
1318 1319 1320 1321 1322 1323 1324
				rdev = conf->mirrors[d].replacement;
				if (!rdev) {
					/* Race with remove_disk */
					smp_mb();
					rdev = conf->mirrors[d].rdev;
				}
				rdev_dec_pending(rdev, mddev);
1325 1326
			}
		}
1327 1328 1329 1330 1331 1332
		allow_barrier(conf);
		md_wait_for_blocked_rdev(blocked_rdev, mddev);
		wait_barrier(conf);
		goto retry_write;
	}

1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344
	if (max_sectors < r10_bio->sectors) {
		/* We are splitting this into multiple parts, so
		 * we need to prepare for allocating another r10_bio.
		 */
		r10_bio->sectors = max_sectors;
		spin_lock_irq(&conf->device_lock);
		if (bio->bi_phys_segments == 0)
			bio->bi_phys_segments = 2;
		else
			bio->bi_phys_segments++;
		spin_unlock_irq(&conf->device_lock);
	}
1345 1346
	sectors_handled = r10_bio->sector + max_sectors -
		bio->bi_iter.bi_sector;
1347

1348
	atomic_set(&r10_bio->remaining, 1);
1349
	bitmap_startwrite(mddev->bitmap, r10_bio->sector, r10_bio->sectors, 0);
1350

L
Linus Torvalds 已提交
1351 1352 1353
	for (i = 0; i < conf->copies; i++) {
		struct bio *mbio;
		int d = r10_bio->devs[i].devnum;
1354 1355 1356
		if (r10_bio->devs[i].bio) {
			struct md_rdev *rdev = conf->mirrors[d].rdev;
			mbio = bio_clone_mddev(bio, GFP_NOIO, mddev);
1357
			bio_trim(mbio, r10_bio->sector - bio->bi_iter.bi_sector,
1358
				 max_sectors);
1359 1360
			r10_bio->devs[i].bio = mbio;

1361
			mbio->bi_iter.bi_sector	= (r10_bio->devs[i].addr+
1362 1363 1364 1365
					   choose_data_offset(r10_bio,
							      rdev));
			mbio->bi_bdev = rdev->bdev;
			mbio->bi_end_io	= raid10_end_write_request;
1366 1367
			mbio->bi_rw =
				WRITE | do_sync | do_fua | do_discard | do_same;
1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390
			mbio->bi_private = r10_bio;

			atomic_inc(&r10_bio->remaining);

			cb = blk_check_plugged(raid10_unplug, mddev,
					       sizeof(*plug));
			if (cb)
				plug = container_of(cb, struct raid10_plug_cb,
						    cb);
			else
				plug = NULL;
			spin_lock_irqsave(&conf->device_lock, flags);
			if (plug) {
				bio_list_add(&plug->pending, mbio);
				plug->pending_cnt++;
			} else {
				bio_list_add(&conf->pending_bio_list, mbio);
				conf->pending_count++;
			}
			spin_unlock_irqrestore(&conf->device_lock, flags);
			if (!plug)
				md_wakeup_thread(mddev->thread);
		}
1391

1392 1393 1394 1395 1396 1397 1398 1399
		if (r10_bio->devs[i].repl_bio) {
			struct md_rdev *rdev = conf->mirrors[d].replacement;
			if (rdev == NULL) {
				/* Replacement just got moved to main 'rdev' */
				smp_mb();
				rdev = conf->mirrors[d].rdev;
			}
			mbio = bio_clone_mddev(bio, GFP_NOIO, mddev);
1400
			bio_trim(mbio, r10_bio->sector - bio->bi_iter.bi_sector,
1401
				 max_sectors);
1402 1403
			r10_bio->devs[i].repl_bio = mbio;

1404
			mbio->bi_iter.bi_sector	= (r10_bio->devs[i].addr +
1405 1406 1407 1408
					   choose_data_offset(
						   r10_bio, rdev));
			mbio->bi_bdev = rdev->bdev;
			mbio->bi_end_io	= raid10_end_write_request;
1409 1410
			mbio->bi_rw =
				WRITE | do_sync | do_fua | do_discard | do_same;
1411 1412 1413 1414
			mbio->bi_private = r10_bio;

			atomic_inc(&r10_bio->remaining);
			spin_lock_irqsave(&conf->device_lock, flags);
1415 1416
			bio_list_add(&conf->pending_bio_list, mbio);
			conf->pending_count++;
1417 1418 1419
			spin_unlock_irqrestore(&conf->device_lock, flags);
			if (!mddev_check_plugged(mddev))
				md_wakeup_thread(mddev->thread);
1420
		}
L
Linus Torvalds 已提交
1421 1422
	}

1423 1424 1425
	/* Don't remove the bias on 'remaining' (one_write_done) until
	 * after checking if we need to go around again.
	 */
1426

1427
	if (sectors_handled < bio_sectors(bio)) {
1428
		one_write_done(r10_bio);
1429
		/* We need another r10_bio.  It has already been counted
1430 1431 1432 1433 1434
		 * in bio->bi_phys_segments.
		 */
		r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);

		r10_bio->master_bio = bio;
1435
		r10_bio->sectors = bio_sectors(bio) - sectors_handled;
1436 1437

		r10_bio->mddev = mddev;
1438
		r10_bio->sector = bio->bi_iter.bi_sector + sectors_handled;
1439 1440 1441
		r10_bio->state = 0;
		goto retry_write;
	}
1442
	one_write_done(r10_bio);
1443 1444
}

S
Shaohua Li 已提交
1445
static void raid10_make_request(struct mddev *mddev, struct bio *bio)
1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481
{
	struct r10conf *conf = mddev->private;
	sector_t chunk_mask = (conf->geo.chunk_mask & conf->prev.chunk_mask);
	int chunk_sects = chunk_mask + 1;

	struct bio *split;

	if (unlikely(bio->bi_rw & REQ_FLUSH)) {
		md_flush_request(mddev, bio);
		return;
	}

	md_write_start(mddev, bio);

	do {

		/*
		 * If this request crosses a chunk boundary, we need to split
		 * it.
		 */
		if (unlikely((bio->bi_iter.bi_sector & chunk_mask) +
			     bio_sectors(bio) > chunk_sects
			     && (conf->geo.near_copies < conf->geo.raid_disks
				 || conf->prev.near_copies <
				 conf->prev.raid_disks))) {
			split = bio_split(bio, chunk_sects -
					  (bio->bi_iter.bi_sector &
					   (chunk_sects - 1)),
					  GFP_NOIO, fs_bio_set);
			bio_chain(split, bio);
		} else {
			split = bio;
		}

		__make_request(mddev, split);
	} while (split != bio);
1482 1483 1484

	/* In case raid10d snuck in to freeze_array */
	wake_up(&conf->wait_barrier);
L
Linus Torvalds 已提交
1485 1486
}

S
Shaohua Li 已提交
1487
static void raid10_status(struct seq_file *seq, struct mddev *mddev)
L
Linus Torvalds 已提交
1488
{
1489
	struct r10conf *conf = mddev->private;
L
Linus Torvalds 已提交
1490 1491
	int i;

1492
	if (conf->geo.near_copies < conf->geo.raid_disks)
1493
		seq_printf(seq, " %dK chunks", mddev->chunk_sectors / 2);
1494 1495 1496 1497 1498
	if (conf->geo.near_copies > 1)
		seq_printf(seq, " %d near-copies", conf->geo.near_copies);
	if (conf->geo.far_copies > 1) {
		if (conf->geo.far_offset)
			seq_printf(seq, " %d offset-copies", conf->geo.far_copies);
1499
		else
1500
			seq_printf(seq, " %d far-copies", conf->geo.far_copies);
1501 1502
		if (conf->geo.far_set_size != conf->geo.raid_disks)
			seq_printf(seq, " %d devices per set", conf->geo.far_set_size);
1503
	}
1504 1505 1506
	seq_printf(seq, " [%d/%d] [", conf->geo.raid_disks,
					conf->geo.raid_disks - mddev->degraded);
	for (i = 0; i < conf->geo.raid_disks; i++)
L
Linus Torvalds 已提交
1507 1508
		seq_printf(seq, "%s",
			      conf->mirrors[i].rdev &&
1509
			      test_bit(In_sync, &conf->mirrors[i].rdev->flags) ? "U" : "_");
L
Linus Torvalds 已提交
1510 1511 1512
	seq_printf(seq, "]");
}

1513 1514 1515 1516 1517
/* check if there are enough drives for
 * every block to appear on atleast one.
 * Don't consider the device numbered 'ignore'
 * as we might be about to remove it.
 */
1518
static int _enough(struct r10conf *conf, int previous, int ignore)
1519 1520
{
	int first = 0;
1521
	int has_enough = 0;
1522 1523 1524 1525 1526 1527 1528 1529
	int disks, ncopies;
	if (previous) {
		disks = conf->prev.raid_disks;
		ncopies = conf->prev.near_copies;
	} else {
		disks = conf->geo.raid_disks;
		ncopies = conf->geo.near_copies;
	}
1530

1531
	rcu_read_lock();
1532 1533 1534
	do {
		int n = conf->copies;
		int cnt = 0;
1535
		int this = first;
1536
		while (n--) {
1537 1538 1539 1540
			struct md_rdev *rdev;
			if (this != ignore &&
			    (rdev = rcu_dereference(conf->mirrors[this].rdev)) &&
			    test_bit(In_sync, &rdev->flags))
1541
				cnt++;
1542
			this = (this+1) % disks;
1543 1544
		}
		if (cnt == 0)
1545
			goto out;
1546
		first = (first + ncopies) % disks;
1547
	} while (first != 0);
1548 1549 1550 1551
	has_enough = 1;
out:
	rcu_read_unlock();
	return has_enough;
1552 1553
}

1554 1555
static int enough(struct r10conf *conf, int ignore)
{
1556 1557 1558 1559 1560 1561 1562
	/* when calling 'enough', both 'prev' and 'geo' must
	 * be stable.
	 * This is ensured if ->reconfig_mutex or ->device_lock
	 * is held.
	 */
	return _enough(conf, 0, ignore) &&
		_enough(conf, 1, ignore);
1563 1564
}

S
Shaohua Li 已提交
1565
static void raid10_error(struct mddev *mddev, struct md_rdev *rdev)
L
Linus Torvalds 已提交
1566 1567
{
	char b[BDEVNAME_SIZE];
1568
	struct r10conf *conf = mddev->private;
1569
	unsigned long flags;
L
Linus Torvalds 已提交
1570 1571 1572 1573 1574 1575 1576

	/*
	 * If it is not operational, then we have already marked it as dead
	 * else if it is the last working disks, ignore the error, let the
	 * next level up know.
	 * else mark the drive as failed
	 */
1577
	spin_lock_irqsave(&conf->device_lock, flags);
1578
	if (test_bit(In_sync, &rdev->flags)
1579
	    && !enough(conf, rdev->raid_disk)) {
L
Linus Torvalds 已提交
1580 1581 1582
		/*
		 * Don't fail the drive, just return an IO error.
		 */
1583
		spin_unlock_irqrestore(&conf->device_lock, flags);
L
Linus Torvalds 已提交
1584
		return;
1585
	}
1586
	if (test_and_clear_bit(In_sync, &rdev->flags))
L
Linus Torvalds 已提交
1587
		mddev->degraded++;
1588 1589 1590 1591
	/*
	 * If recovery is running, make sure it aborts.
	 */
	set_bit(MD_RECOVERY_INTR, &mddev->recovery);
1592
	set_bit(Blocked, &rdev->flags);
1593
	set_bit(Faulty, &rdev->flags);
1594
	set_bit(MD_CHANGE_DEVS, &mddev->flags);
1595
	set_bit(MD_CHANGE_PENDING, &mddev->flags);
1596
	spin_unlock_irqrestore(&conf->device_lock, flags);
1597 1598 1599
	printk(KERN_ALERT
	       "md/raid10:%s: Disk failure on %s, disabling device.\n"
	       "md/raid10:%s: Operation continuing on %d devices.\n",
1600
	       mdname(mddev), bdevname(rdev->bdev, b),
1601
	       mdname(mddev), conf->geo.raid_disks - mddev->degraded);
L
Linus Torvalds 已提交
1602 1603
}

1604
static void print_conf(struct r10conf *conf)
L
Linus Torvalds 已提交
1605 1606
{
	int i;
1607
	struct raid10_info *tmp;
L
Linus Torvalds 已提交
1608

1609
	printk(KERN_DEBUG "RAID10 conf printout:\n");
L
Linus Torvalds 已提交
1610
	if (!conf) {
1611
		printk(KERN_DEBUG "(!conf)\n");
L
Linus Torvalds 已提交
1612 1613
		return;
	}
1614 1615
	printk(KERN_DEBUG " --- wd:%d rd:%d\n", conf->geo.raid_disks - conf->mddev->degraded,
		conf->geo.raid_disks);
L
Linus Torvalds 已提交
1616

1617
	for (i = 0; i < conf->geo.raid_disks; i++) {
L
Linus Torvalds 已提交
1618 1619 1620
		char b[BDEVNAME_SIZE];
		tmp = conf->mirrors + i;
		if (tmp->rdev)
1621
			printk(KERN_DEBUG " disk %d, wo:%d, o:%d, dev:%s\n",
1622 1623
				i, !test_bit(In_sync, &tmp->rdev->flags),
			        !test_bit(Faulty, &tmp->rdev->flags),
L
Linus Torvalds 已提交
1624 1625 1626 1627
				bdevname(tmp->rdev->bdev,b));
	}
}

1628
static void close_sync(struct r10conf *conf)
L
Linus Torvalds 已提交
1629
{
1630 1631
	wait_barrier(conf);
	allow_barrier(conf);
L
Linus Torvalds 已提交
1632 1633 1634 1635 1636

	mempool_destroy(conf->r10buf_pool);
	conf->r10buf_pool = NULL;
}

1637
static int raid10_spare_active(struct mddev *mddev)
L
Linus Torvalds 已提交
1638 1639
{
	int i;
1640
	struct r10conf *conf = mddev->private;
1641
	struct raid10_info *tmp;
1642 1643
	int count = 0;
	unsigned long flags;
L
Linus Torvalds 已提交
1644 1645 1646 1647 1648

	/*
	 * Find all non-in_sync disks within the RAID10 configuration
	 * and mark them in_sync
	 */
1649
	for (i = 0; i < conf->geo.raid_disks; i++) {
L
Linus Torvalds 已提交
1650
		tmp = conf->mirrors + i;
1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669
		if (tmp->replacement
		    && tmp->replacement->recovery_offset == MaxSector
		    && !test_bit(Faulty, &tmp->replacement->flags)
		    && !test_and_set_bit(In_sync, &tmp->replacement->flags)) {
			/* Replacement has just become active */
			if (!tmp->rdev
			    || !test_and_clear_bit(In_sync, &tmp->rdev->flags))
				count++;
			if (tmp->rdev) {
				/* Replaced device not technically faulty,
				 * but we need to be sure it gets removed
				 * and never re-added.
				 */
				set_bit(Faulty, &tmp->rdev->flags);
				sysfs_notify_dirent_safe(
					tmp->rdev->sysfs_state);
			}
			sysfs_notify_dirent_safe(tmp->replacement->sysfs_state);
		} else if (tmp->rdev
1670
			   && tmp->rdev->recovery_offset == MaxSector
1671 1672
			   && !test_bit(Faulty, &tmp->rdev->flags)
			   && !test_and_set_bit(In_sync, &tmp->rdev->flags)) {
1673
			count++;
1674
			sysfs_notify_dirent_safe(tmp->rdev->sysfs_state);
L
Linus Torvalds 已提交
1675 1676
		}
	}
1677 1678 1679
	spin_lock_irqsave(&conf->device_lock, flags);
	mddev->degraded -= count;
	spin_unlock_irqrestore(&conf->device_lock, flags);
L
Linus Torvalds 已提交
1680 1681

	print_conf(conf);
1682
	return count;
L
Linus Torvalds 已提交
1683 1684
}

1685
static int raid10_add_disk(struct mddev *mddev, struct md_rdev *rdev)
L
Linus Torvalds 已提交
1686
{
1687
	struct r10conf *conf = mddev->private;
1688
	int err = -EEXIST;
L
Linus Torvalds 已提交
1689
	int mirror;
1690
	int first = 0;
1691
	int last = conf->geo.raid_disks - 1;
L
Linus Torvalds 已提交
1692 1693 1694 1695 1696

	if (mddev->recovery_cp < MaxSector)
		/* only hot-add to in-sync arrays, as recovery is
		 * very different from resync
		 */
1697
		return -EBUSY;
1698
	if (rdev->saved_raid_disk < 0 && !_enough(conf, 1, -1))
1699
		return -EINVAL;
L
Linus Torvalds 已提交
1700

1701 1702 1703
	if (md_integrity_add_rdev(rdev, mddev))
		return -ENXIO;

1704
	if (rdev->raid_disk >= 0)
1705
		first = last = rdev->raid_disk;
L
Linus Torvalds 已提交
1706

1707
	if (rdev->saved_raid_disk >= first &&
1708 1709 1710
	    conf->mirrors[rdev->saved_raid_disk].rdev == NULL)
		mirror = rdev->saved_raid_disk;
	else
1711
		mirror = first;
1712
	for ( ; mirror <= last ; mirror++) {
1713
		struct raid10_info *p = &conf->mirrors[mirror];
1714 1715
		if (p->recovery_disabled == mddev->recovery_disabled)
			continue;
1716 1717 1718 1719 1720 1721 1722 1723
		if (p->rdev) {
			if (!test_bit(WantReplacement, &p->rdev->flags) ||
			    p->replacement != NULL)
				continue;
			clear_bit(In_sync, &rdev->flags);
			set_bit(Replacement, &rdev->flags);
			rdev->raid_disk = mirror;
			err = 0;
1724 1725 1726
			if (mddev->gendisk)
				disk_stack_limits(mddev->gendisk, rdev->bdev,
						  rdev->data_offset << 9);
1727 1728 1729 1730
			conf->fullsync = 1;
			rcu_assign_pointer(p->replacement, rdev);
			break;
		}
L
Linus Torvalds 已提交
1731

1732 1733 1734
		if (mddev->gendisk)
			disk_stack_limits(mddev->gendisk, rdev->bdev,
					  rdev->data_offset << 9);
L
Linus Torvalds 已提交
1735

1736
		p->head_position = 0;
1737
		p->recovery_disabled = mddev->recovery_disabled - 1;
1738 1739 1740 1741 1742 1743 1744
		rdev->raid_disk = mirror;
		err = 0;
		if (rdev->saved_raid_disk != mirror)
			conf->fullsync = 1;
		rcu_assign_pointer(p->rdev, rdev);
		break;
	}
1745
	if (mddev->queue && blk_queue_discard(bdev_get_queue(rdev->bdev)))
S
Shaohua Li 已提交
1746 1747
		queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, mddev->queue);

L
Linus Torvalds 已提交
1748
	print_conf(conf);
1749
	return err;
L
Linus Torvalds 已提交
1750 1751
}

1752
static int raid10_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
L
Linus Torvalds 已提交
1753
{
1754
	struct r10conf *conf = mddev->private;
L
Linus Torvalds 已提交
1755
	int err = 0;
1756
	int number = rdev->raid_disk;
1757
	struct md_rdev **rdevp;
1758
	struct raid10_info *p = conf->mirrors + number;
L
Linus Torvalds 已提交
1759 1760

	print_conf(conf);
1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777
	if (rdev == p->rdev)
		rdevp = &p->rdev;
	else if (rdev == p->replacement)
		rdevp = &p->replacement;
	else
		return 0;

	if (test_bit(In_sync, &rdev->flags) ||
	    atomic_read(&rdev->nr_pending)) {
		err = -EBUSY;
		goto abort;
	}
	/* Only remove faulty devices if recovery
	 * is not possible.
	 */
	if (!test_bit(Faulty, &rdev->flags) &&
	    mddev->recovery_disabled != p->recovery_disabled &&
1778
	    (!p->replacement || p->replacement == rdev) &&
1779
	    number < conf->geo.raid_disks &&
1780 1781 1782
	    enough(conf, -1)) {
		err = -EBUSY;
		goto abort;
L
Linus Torvalds 已提交
1783
	}
1784 1785 1786 1787 1788 1789 1790
	*rdevp = NULL;
	synchronize_rcu();
	if (atomic_read(&rdev->nr_pending)) {
		/* lost the race, try later */
		err = -EBUSY;
		*rdevp = rdev;
		goto abort;
1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805
	} else if (p->replacement) {
		/* We must have just cleared 'rdev' */
		p->rdev = p->replacement;
		clear_bit(Replacement, &p->replacement->flags);
		smp_mb(); /* Make sure other CPUs may see both as identical
			   * but will never see neither -- if they are careful.
			   */
		p->replacement = NULL;
		clear_bit(WantReplacement, &rdev->flags);
	} else
		/* We might have just remove the Replacement as faulty
		 * Clear the flag just in case
		 */
		clear_bit(WantReplacement, &rdev->flags);

1806 1807
	err = md_integrity_register(mddev);

L
Linus Torvalds 已提交
1808 1809 1810 1811 1812 1813
abort:

	print_conf(conf);
	return err;
}

1814
static void end_sync_read(struct bio *bio)
L
Linus Torvalds 已提交
1815
{
1816
	struct r10bio *r10_bio = bio->bi_private;
1817
	struct r10conf *conf = r10_bio->mddev->private;
1818
	int d;
L
Linus Torvalds 已提交
1819

1820 1821 1822 1823 1824
	if (bio == r10_bio->master_bio) {
		/* this is a reshape read */
		d = r10_bio->read_slot; /* really the read dev */
	} else
		d = find_bio_disk(conf, r10_bio, bio, NULL, NULL);
1825

1826
	if (!bio->bi_error)
1827
		set_bit(R10BIO_Uptodate, &r10_bio->state);
1828 1829 1830 1831
	else
		/* The write handler will notice the lack of
		 * R10BIO_Uptodate and record any errors etc
		 */
1832 1833
		atomic_add(r10_bio->sectors,
			   &conf->mirrors[d].rdev->corrected_errors);
L
Linus Torvalds 已提交
1834 1835 1836 1837

	/* for reconstruct, we always reschedule after a read.
	 * for resync, only after all reads
	 */
1838
	rdev_dec_pending(conf->mirrors[d].rdev, conf->mddev);
L
Linus Torvalds 已提交
1839 1840 1841 1842 1843 1844 1845 1846 1847
	if (test_bit(R10BIO_IsRecover, &r10_bio->state) ||
	    atomic_dec_and_test(&r10_bio->remaining)) {
		/* we have read all the blocks,
		 * do the comparison in process context in raid10d
		 */
		reschedule_retry(r10_bio);
	}
}

1848
static void end_sync_request(struct r10bio *r10_bio)
L
Linus Torvalds 已提交
1849
{
1850
	struct mddev *mddev = r10_bio->mddev;
1851

L
Linus Torvalds 已提交
1852 1853 1854
	while (atomic_dec_and_test(&r10_bio->remaining)) {
		if (r10_bio->master_bio == NULL) {
			/* the primary of several recovery bios */
1855
			sector_t s = r10_bio->sectors;
1856 1857
			if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
			    test_bit(R10BIO_WriteError, &r10_bio->state))
1858 1859 1860
				reschedule_retry(r10_bio);
			else
				put_buf(r10_bio);
1861
			md_done_sync(mddev, s, 1);
L
Linus Torvalds 已提交
1862 1863
			break;
		} else {
1864
			struct r10bio *r10_bio2 = (struct r10bio *)r10_bio->master_bio;
1865 1866
			if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
			    test_bit(R10BIO_WriteError, &r10_bio->state))
1867 1868 1869
				reschedule_retry(r10_bio);
			else
				put_buf(r10_bio);
L
Linus Torvalds 已提交
1870 1871 1872 1873 1874
			r10_bio = r10_bio2;
		}
	}
}

1875
static void end_sync_write(struct bio *bio)
1876
{
1877
	struct r10bio *r10_bio = bio->bi_private;
1878
	struct mddev *mddev = r10_bio->mddev;
1879
	struct r10conf *conf = mddev->private;
1880 1881 1882 1883
	int d;
	sector_t first_bad;
	int bad_sectors;
	int slot;
1884
	int repl;
1885
	struct md_rdev *rdev = NULL;
1886

1887 1888 1889
	d = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
	if (repl)
		rdev = conf->mirrors[d].replacement;
1890
	else
1891
		rdev = conf->mirrors[d].rdev;
1892

1893
	if (bio->bi_error) {
1894 1895 1896 1897
		if (repl)
			md_error(mddev, rdev);
		else {
			set_bit(WriteErrorSeen, &rdev->flags);
1898 1899 1900
			if (!test_and_set_bit(WantReplacement, &rdev->flags))
				set_bit(MD_RECOVERY_NEEDED,
					&rdev->mddev->recovery);
1901 1902 1903
			set_bit(R10BIO_WriteError, &r10_bio->state);
		}
	} else if (is_badblock(rdev,
1904 1905 1906 1907 1908
			     r10_bio->devs[slot].addr,
			     r10_bio->sectors,
			     &first_bad, &bad_sectors))
		set_bit(R10BIO_MadeGood, &r10_bio->state);

1909
	rdev_dec_pending(rdev, mddev);
1910 1911 1912 1913

	end_sync_request(r10_bio);
}

L
Linus Torvalds 已提交
1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929
/*
 * Note: sync and recover and handled very differently for raid10
 * This code is for resync.
 * For resync, we read through virtual addresses and read all blocks.
 * If there is any error, we schedule a write.  The lowest numbered
 * drive is authoritative.
 * However requests come for physical address, so we need to map.
 * For every physical address there are raid_disks/copies virtual addresses,
 * which is always are least one, but is not necessarly an integer.
 * This means that a physical address can span multiple chunks, so we may
 * have to submit multiple io requests for a single sync request.
 */
/*
 * We check if all blocks are in-sync and only write to blocks that
 * aren't in sync
 */
1930
static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio)
L
Linus Torvalds 已提交
1931
{
1932
	struct r10conf *conf = mddev->private;
L
Linus Torvalds 已提交
1933 1934
	int i, first;
	struct bio *tbio, *fbio;
1935
	int vcnt;
L
Linus Torvalds 已提交
1936 1937 1938 1939 1940

	atomic_set(&r10_bio->remaining, 1);

	/* find the first device with a block */
	for (i=0; i<conf->copies; i++)
1941
		if (!r10_bio->devs[i].bio->bi_error)
L
Linus Torvalds 已提交
1942 1943 1944 1945 1946 1947 1948
			break;

	if (i == conf->copies)
		goto done;

	first = i;
	fbio = r10_bio->devs[i].bio;
1949 1950
	fbio->bi_iter.bi_size = r10_bio->sectors << 9;
	fbio->bi_iter.bi_idx = 0;
L
Linus Torvalds 已提交
1951

1952
	vcnt = (r10_bio->sectors + (PAGE_SIZE >> 9) - 1) >> (PAGE_SHIFT - 9);
L
Linus Torvalds 已提交
1953
	/* now find blocks with errors */
1954 1955
	for (i=0 ; i < conf->copies ; i++) {
		int  j, d;
L
Linus Torvalds 已提交
1956 1957

		tbio = r10_bio->devs[i].bio;
1958 1959 1960 1961

		if (tbio->bi_end_io != end_sync_read)
			continue;
		if (i == first)
L
Linus Torvalds 已提交
1962
			continue;
1963
		if (!r10_bio->devs[i].bio->bi_error) {
1964 1965 1966 1967
			/* We know that the bi_io_vec layout is the same for
			 * both 'first' and 'i', so we just compare them.
			 * All vec entries are PAGE_SIZE;
			 */
1968 1969 1970 1971 1972
			int sectors = r10_bio->sectors;
			for (j = 0; j < vcnt; j++) {
				int len = PAGE_SIZE;
				if (sectors < (len / 512))
					len = sectors * 512;
1973 1974
				if (memcmp(page_address(fbio->bi_io_vec[j].bv_page),
					   page_address(tbio->bi_io_vec[j].bv_page),
1975
					   len))
1976
					break;
1977 1978
				sectors -= len/512;
			}
1979 1980
			if (j == vcnt)
				continue;
1981
			atomic64_add(r10_bio->sectors, &mddev->resync_mismatches);
1982 1983 1984
			if (test_bit(MD_RECOVERY_CHECK, &mddev->recovery))
				/* Don't fix anything. */
				continue;
1985
		}
1986 1987
		/* Ok, we need to write this bio, either to correct an
		 * inconsistency or to correct an unreadable block.
L
Linus Torvalds 已提交
1988 1989 1990
		 * First we need to fixup bv_offset, bv_len and
		 * bi_vecs, as the read request might have corrupted these
		 */
K
Kent Overstreet 已提交
1991 1992
		bio_reset(tbio);

L
Linus Torvalds 已提交
1993
		tbio->bi_vcnt = vcnt;
1994
		tbio->bi_iter.bi_size = fbio->bi_iter.bi_size;
L
Linus Torvalds 已提交
1995 1996
		tbio->bi_rw = WRITE;
		tbio->bi_private = r10_bio;
1997
		tbio->bi_iter.bi_sector = r10_bio->devs[i].addr;
L
Linus Torvalds 已提交
1998 1999
		tbio->bi_end_io = end_sync_write;

2000 2001
		bio_copy_data(tbio, fbio);

L
Linus Torvalds 已提交
2002 2003 2004
		d = r10_bio->devs[i].devnum;
		atomic_inc(&conf->mirrors[d].rdev->nr_pending);
		atomic_inc(&r10_bio->remaining);
2005
		md_sync_acct(conf->mirrors[d].rdev->bdev, bio_sectors(tbio));
L
Linus Torvalds 已提交
2006

2007
		tbio->bi_iter.bi_sector += conf->mirrors[d].rdev->data_offset;
L
Linus Torvalds 已提交
2008 2009 2010 2011
		tbio->bi_bdev = conf->mirrors[d].rdev->bdev;
		generic_make_request(tbio);
	}

2012 2013 2014 2015
	/* Now write out to any replacement devices
	 * that are active
	 */
	for (i = 0; i < conf->copies; i++) {
2016
		int d;
2017 2018 2019 2020 2021 2022

		tbio = r10_bio->devs[i].repl_bio;
		if (!tbio || !tbio->bi_end_io)
			continue;
		if (r10_bio->devs[i].bio->bi_end_io != end_sync_write
		    && r10_bio->devs[i].bio != fbio)
2023
			bio_copy_data(tbio, fbio);
2024 2025 2026
		d = r10_bio->devs[i].devnum;
		atomic_inc(&r10_bio->remaining);
		md_sync_acct(conf->mirrors[d].replacement->bdev,
2027
			     bio_sectors(tbio));
2028 2029 2030
		generic_make_request(tbio);
	}

L
Linus Torvalds 已提交
2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047
done:
	if (atomic_dec_and_test(&r10_bio->remaining)) {
		md_done_sync(mddev, r10_bio->sectors, 1);
		put_buf(r10_bio);
	}
}

/*
 * Now for the recovery code.
 * Recovery happens across physical sectors.
 * We recover all non-is_sync drives by finding the virtual address of
 * each, and then choose a working drive that also has that virt address.
 * There is a separate r10_bio for each non-in_sync drive.
 * Only the first two slots are in use. The first for reading,
 * The second for writing.
 *
 */
2048
static void fix_recovery_read_error(struct r10bio *r10_bio)
2049 2050 2051 2052 2053 2054 2055 2056
{
	/* We got a read error during recovery.
	 * We repeat the read in smaller page-sized sections.
	 * If a read succeeds, write it to the new device or record
	 * a bad block if we cannot.
	 * If a read fails, record a bad block on both old and
	 * new devices.
	 */
2057
	struct mddev *mddev = r10_bio->mddev;
2058
	struct r10conf *conf = mddev->private;
2059 2060 2061 2062 2063 2064 2065 2066 2067
	struct bio *bio = r10_bio->devs[0].bio;
	sector_t sect = 0;
	int sectors = r10_bio->sectors;
	int idx = 0;
	int dr = r10_bio->devs[0].devnum;
	int dw = r10_bio->devs[1].devnum;

	while (sectors) {
		int s = sectors;
2068
		struct md_rdev *rdev;
2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089
		sector_t addr;
		int ok;

		if (s > (PAGE_SIZE>>9))
			s = PAGE_SIZE >> 9;

		rdev = conf->mirrors[dr].rdev;
		addr = r10_bio->devs[0].addr + sect,
		ok = sync_page_io(rdev,
				  addr,
				  s << 9,
				  bio->bi_io_vec[idx].bv_page,
				  READ, false);
		if (ok) {
			rdev = conf->mirrors[dw].rdev;
			addr = r10_bio->devs[1].addr + sect;
			ok = sync_page_io(rdev,
					  addr,
					  s << 9,
					  bio->bi_io_vec[idx].bv_page,
					  WRITE, false);
2090
			if (!ok) {
2091
				set_bit(WriteErrorSeen, &rdev->flags);
2092 2093 2094 2095 2096
				if (!test_and_set_bit(WantReplacement,
						      &rdev->flags))
					set_bit(MD_RECOVERY_NEEDED,
						&rdev->mddev->recovery);
			}
2097 2098 2099 2100 2101 2102 2103 2104 2105 2106
		}
		if (!ok) {
			/* We don't worry if we cannot set a bad block -
			 * it really is bad so there is no loss in not
			 * recording it yet
			 */
			rdev_set_badblocks(rdev, addr, s, 0);

			if (rdev != conf->mirrors[dw].rdev) {
				/* need bad block on destination too */
2107
				struct md_rdev *rdev2 = conf->mirrors[dw].rdev;
2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130
				addr = r10_bio->devs[1].addr + sect;
				ok = rdev_set_badblocks(rdev2, addr, s, 0);
				if (!ok) {
					/* just abort the recovery */
					printk(KERN_NOTICE
					       "md/raid10:%s: recovery aborted"
					       " due to read error\n",
					       mdname(mddev));

					conf->mirrors[dw].recovery_disabled
						= mddev->recovery_disabled;
					set_bit(MD_RECOVERY_INTR,
						&mddev->recovery);
					break;
				}
			}
		}

		sectors -= s;
		sect += s;
		idx++;
	}
}
L
Linus Torvalds 已提交
2131

2132
static void recovery_request_write(struct mddev *mddev, struct r10bio *r10_bio)
L
Linus Torvalds 已提交
2133
{
2134
	struct r10conf *conf = mddev->private;
2135
	int d;
2136
	struct bio *wbio, *wbio2;
L
Linus Torvalds 已提交
2137

2138 2139 2140 2141 2142 2143
	if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) {
		fix_recovery_read_error(r10_bio);
		end_sync_request(r10_bio);
		return;
	}

2144 2145
	/*
	 * share the pages with the first bio
L
Linus Torvalds 已提交
2146 2147 2148
	 * and submit the write request
	 */
	d = r10_bio->devs[1].devnum;
2149 2150
	wbio = r10_bio->devs[1].bio;
	wbio2 = r10_bio->devs[1].repl_bio;
2151 2152 2153 2154 2155 2156
	/* Need to test wbio2->bi_end_io before we call
	 * generic_make_request as if the former is NULL,
	 * the latter is free to free wbio2.
	 */
	if (wbio2 && !wbio2->bi_end_io)
		wbio2 = NULL;
2157 2158
	if (wbio->bi_end_io) {
		atomic_inc(&conf->mirrors[d].rdev->nr_pending);
2159
		md_sync_acct(conf->mirrors[d].rdev->bdev, bio_sectors(wbio));
2160 2161
		generic_make_request(wbio);
	}
2162
	if (wbio2) {
2163 2164
		atomic_inc(&conf->mirrors[d].replacement->nr_pending);
		md_sync_acct(conf->mirrors[d].replacement->bdev,
2165
			     bio_sectors(wbio2));
2166 2167
		generic_make_request(wbio2);
	}
L
Linus Torvalds 已提交
2168 2169
}

2170 2171 2172 2173 2174 2175
/*
 * Used by fix_read_error() to decay the per rdev read_errors.
 * We halve the read error count for every hour that has elapsed
 * since the last recorded read error.
 *
 */
2176
static void check_decay_read_errors(struct mddev *mddev, struct md_rdev *rdev)
2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206
{
	struct timespec cur_time_mon;
	unsigned long hours_since_last;
	unsigned int read_errors = atomic_read(&rdev->read_errors);

	ktime_get_ts(&cur_time_mon);

	if (rdev->last_read_error.tv_sec == 0 &&
	    rdev->last_read_error.tv_nsec == 0) {
		/* first time we've seen a read error */
		rdev->last_read_error = cur_time_mon;
		return;
	}

	hours_since_last = (cur_time_mon.tv_sec -
			    rdev->last_read_error.tv_sec) / 3600;

	rdev->last_read_error = cur_time_mon;

	/*
	 * if hours_since_last is > the number of bits in read_errors
	 * just set read errors to 0. We do this to avoid
	 * overflowing the shift of read_errors by hours_since_last.
	 */
	if (hours_since_last >= 8 * sizeof(read_errors))
		atomic_set(&rdev->read_errors, 0);
	else
		atomic_set(&rdev->read_errors, read_errors >> hours_since_last);
}

2207
static int r10_sync_page_io(struct md_rdev *rdev, sector_t sector,
2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218
			    int sectors, struct page *page, int rw)
{
	sector_t first_bad;
	int bad_sectors;

	if (is_badblock(rdev, sector, sectors, &first_bad, &bad_sectors)
	    && (rw == READ || test_bit(WriteErrorSeen, &rdev->flags)))
		return -1;
	if (sync_page_io(rdev, sector, sectors << 9, page, rw, false))
		/* success */
		return 1;
2219
	if (rw == WRITE) {
2220
		set_bit(WriteErrorSeen, &rdev->flags);
2221 2222 2223 2224
		if (!test_and_set_bit(WantReplacement, &rdev->flags))
			set_bit(MD_RECOVERY_NEEDED,
				&rdev->mddev->recovery);
	}
2225 2226 2227 2228 2229 2230
	/* need to record an error - either for the block or the device */
	if (!rdev_set_badblocks(rdev, sector, sectors, 0))
		md_error(rdev->mddev, rdev);
	return 0;
}

L
Linus Torvalds 已提交
2231 2232 2233 2234 2235
/*
 * This is a kernel thread which:
 *
 *	1.	Retries failed read operations on working mirrors.
 *	2.	Updates the raid superblock when problems encounter.
2236
 *	3.	Performs writes following reads for array synchronising.
L
Linus Torvalds 已提交
2237 2238
 */

2239
static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10bio *r10_bio)
2240 2241 2242
{
	int sect = 0; /* Offset from r10_bio->sector */
	int sectors = r10_bio->sectors;
2243
	struct md_rdev*rdev;
2244
	int max_read_errors = atomic_read(&mddev->max_corr_read_errors);
2245
	int d = r10_bio->devs[r10_bio->read_slot].devnum;
2246

2247 2248 2249 2250
	/* still own a reference to this rdev, so it cannot
	 * have been cleared recently.
	 */
	rdev = conf->mirrors[d].rdev;
2251

2252 2253 2254 2255
	if (test_bit(Faulty, &rdev->flags))
		/* drive has already been failed, just ignore any
		   more fix_read_error() attempts */
		return;
2256

2257 2258 2259 2260 2261
	check_decay_read_errors(mddev, rdev);
	atomic_inc(&rdev->read_errors);
	if (atomic_read(&rdev->read_errors) > max_read_errors) {
		char b[BDEVNAME_SIZE];
		bdevname(rdev->bdev, b);
2262

2263 2264 2265 2266 2267 2268 2269 2270 2271
		printk(KERN_NOTICE
		       "md/raid10:%s: %s: Raid device exceeded "
		       "read_error threshold [cur %d:max %d]\n",
		       mdname(mddev), b,
		       atomic_read(&rdev->read_errors), max_read_errors);
		printk(KERN_NOTICE
		       "md/raid10:%s: %s: Failing raid device\n",
		       mdname(mddev), b);
		md_error(mddev, conf->mirrors[d].rdev);
2272
		r10_bio->devs[r10_bio->read_slot].bio = IO_BLOCKED;
2273
		return;
2274 2275
	}

2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286
	while(sectors) {
		int s = sectors;
		int sl = r10_bio->read_slot;
		int success = 0;
		int start;

		if (s > (PAGE_SIZE>>9))
			s = PAGE_SIZE >> 9;

		rcu_read_lock();
		do {
2287 2288 2289
			sector_t first_bad;
			int bad_sectors;

2290
			d = r10_bio->devs[sl].devnum;
2291 2292
			rdev = rcu_dereference(conf->mirrors[d].rdev);
			if (rdev &&
2293 2294 2295
			    test_bit(In_sync, &rdev->flags) &&
			    is_badblock(rdev, r10_bio->devs[sl].addr + sect, s,
					&first_bad, &bad_sectors) == 0) {
2296 2297
				atomic_inc(&rdev->nr_pending);
				rcu_read_unlock();
2298
				success = sync_page_io(rdev,
2299
						       r10_bio->devs[sl].addr +
2300
						       sect,
2301
						       s<<9,
2302
						       conf->tmppage, READ, false);
2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314
				rdev_dec_pending(rdev, mddev);
				rcu_read_lock();
				if (success)
					break;
			}
			sl++;
			if (sl == conf->copies)
				sl = 0;
		} while (!success && sl != r10_bio->read_slot);
		rcu_read_unlock();

		if (!success) {
2315 2316 2317 2318
			/* Cannot read from anywhere, just mark the block
			 * as bad on the first device to discourage future
			 * reads.
			 */
2319
			int dn = r10_bio->devs[r10_bio->read_slot].devnum;
2320 2321 2322 2323 2324 2325
			rdev = conf->mirrors[dn].rdev;

			if (!rdev_set_badblocks(
				    rdev,
				    r10_bio->devs[r10_bio->read_slot].addr
				    + sect,
2326
				    s, 0)) {
2327
				md_error(mddev, rdev);
2328 2329 2330
				r10_bio->devs[r10_bio->read_slot].bio
					= IO_BLOCKED;
			}
2331 2332 2333 2334 2335 2336 2337
			break;
		}

		start = sl;
		/* write it back and re-read */
		rcu_read_lock();
		while (sl != r10_bio->read_slot) {
2338
			char b[BDEVNAME_SIZE];
2339

2340 2341 2342 2343 2344
			if (sl==0)
				sl = conf->copies;
			sl--;
			d = r10_bio->devs[sl].devnum;
			rdev = rcu_dereference(conf->mirrors[d].rdev);
2345 2346 2347 2348 2349 2350
			if (!rdev ||
			    !test_bit(In_sync, &rdev->flags))
				continue;

			atomic_inc(&rdev->nr_pending);
			rcu_read_unlock();
2351 2352 2353
			if (r10_sync_page_io(rdev,
					     r10_bio->devs[sl].addr +
					     sect,
2354
					     s, conf->tmppage, WRITE)
2355 2356 2357 2358 2359 2360 2361 2362
			    == 0) {
				/* Well, this device is dead */
				printk(KERN_NOTICE
				       "md/raid10:%s: read correction "
				       "write failed"
				       " (%d sectors at %llu on %s)\n",
				       mdname(mddev), s,
				       (unsigned long long)(
2363 2364 2365
					       sect +
					       choose_data_offset(r10_bio,
								  rdev)),
2366 2367 2368 2369 2370
				       bdevname(rdev->bdev, b));
				printk(KERN_NOTICE "md/raid10:%s: %s: failing "
				       "drive\n",
				       mdname(mddev),
				       bdevname(rdev->bdev, b));
2371
			}
2372 2373
			rdev_dec_pending(rdev, mddev);
			rcu_read_lock();
2374 2375 2376
		}
		sl = start;
		while (sl != r10_bio->read_slot) {
2377
			char b[BDEVNAME_SIZE];
2378

2379 2380 2381 2382 2383
			if (sl==0)
				sl = conf->copies;
			sl--;
			d = r10_bio->devs[sl].devnum;
			rdev = rcu_dereference(conf->mirrors[d].rdev);
2384 2385 2386
			if (!rdev ||
			    !test_bit(In_sync, &rdev->flags))
				continue;
2387

2388 2389
			atomic_inc(&rdev->nr_pending);
			rcu_read_unlock();
2390 2391 2392
			switch (r10_sync_page_io(rdev,
					     r10_bio->devs[sl].addr +
					     sect,
2393
					     s, conf->tmppage,
2394 2395
						 READ)) {
			case 0:
2396 2397 2398 2399 2400 2401 2402
				/* Well, this device is dead */
				printk(KERN_NOTICE
				       "md/raid10:%s: unable to read back "
				       "corrected sectors"
				       " (%d sectors at %llu on %s)\n",
				       mdname(mddev), s,
				       (unsigned long long)(
2403 2404
					       sect +
					       choose_data_offset(r10_bio, rdev)),
2405 2406 2407 2408 2409
				       bdevname(rdev->bdev, b));
				printk(KERN_NOTICE "md/raid10:%s: %s: failing "
				       "drive\n",
				       mdname(mddev),
				       bdevname(rdev->bdev, b));
2410 2411
				break;
			case 1:
2412 2413 2414 2415 2416
				printk(KERN_INFO
				       "md/raid10:%s: read error corrected"
				       " (%d sectors at %llu on %s)\n",
				       mdname(mddev), s,
				       (unsigned long long)(
2417 2418
					       sect +
					       choose_data_offset(r10_bio, rdev)),
2419 2420
				       bdevname(rdev->bdev, b));
				atomic_add(s, &rdev->corrected_errors);
2421
			}
2422 2423 2424

			rdev_dec_pending(rdev, mddev);
			rcu_read_lock();
2425 2426 2427 2428 2429 2430 2431 2432
		}
		rcu_read_unlock();

		sectors -= s;
		sect += s;
	}
}

2433
static int narrow_write_error(struct r10bio *r10_bio, int i)
2434 2435
{
	struct bio *bio = r10_bio->master_bio;
2436
	struct mddev *mddev = r10_bio->mddev;
2437
	struct r10conf *conf = mddev->private;
2438
	struct md_rdev *rdev = conf->mirrors[r10_bio->devs[i].devnum].rdev;
2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458
	/* bio has the data to be written to slot 'i' where
	 * we just recently had a write error.
	 * We repeatedly clone the bio and trim down to one block,
	 * then try the write.  Where the write fails we record
	 * a bad block.
	 * It is conceivable that the bio doesn't exactly align with
	 * blocks.  We must handle this.
	 *
	 * We currently own a reference to the rdev.
	 */

	int block_sectors;
	sector_t sector;
	int sectors;
	int sect_to_write = r10_bio->sectors;
	int ok = 1;

	if (rdev->badblocks.shift < 0)
		return 0;

2459 2460
	block_sectors = roundup(1 << rdev->badblocks.shift,
				bdev_logical_block_size(rdev->bdev) >> 9);
2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471
	sector = r10_bio->sector;
	sectors = ((r10_bio->sector + block_sectors)
		   & ~(sector_t)(block_sectors - 1))
		- sector;

	while (sect_to_write) {
		struct bio *wbio;
		if (sectors > sect_to_write)
			sectors = sect_to_write;
		/* Write at 'sector' for 'sectors' */
		wbio = bio_clone_mddev(bio, GFP_NOIO, mddev);
2472 2473
		bio_trim(wbio, sector - bio->bi_iter.bi_sector, sectors);
		wbio->bi_iter.bi_sector = (r10_bio->devs[i].addr+
2474
				   choose_data_offset(r10_bio, rdev) +
2475 2476
				   (sector - r10_bio->sector));
		wbio->bi_bdev = rdev->bdev;
2477
		if (submit_bio_wait(WRITE, wbio) < 0)
2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490
			/* Failure! */
			ok = rdev_set_badblocks(rdev, sector,
						sectors, 0)
				&& ok;

		bio_put(wbio);
		sect_to_write -= sectors;
		sector += sectors;
		sectors = block_sectors;
	}
	return ok;
}

2491
static void handle_read_error(struct mddev *mddev, struct r10bio *r10_bio)
2492 2493 2494
{
	int slot = r10_bio->read_slot;
	struct bio *bio;
2495
	struct r10conf *conf = mddev->private;
2496
	struct md_rdev *rdev = r10_bio->devs[slot].rdev;
2497 2498
	char b[BDEVNAME_SIZE];
	unsigned long do_sync;
2499
	int max_sectors;
2500 2501 2502 2503 2504 2505 2506 2507 2508

	/* we got a read error. Maybe the drive is bad.  Maybe just
	 * the block and we can fix it.
	 * We freeze all other IO, and try reading the block from
	 * other devices.  When we find one, we re-write
	 * and check it that fixes the read error.
	 * This is all done synchronously while the array is
	 * frozen.
	 */
2509 2510 2511 2512 2513
	bio = r10_bio->devs[slot].bio;
	bdevname(bio->bi_bdev, b);
	bio_put(bio);
	r10_bio->devs[slot].bio = NULL;

2514
	if (mddev->ro == 0) {
2515
		freeze_array(conf, 1);
2516 2517
		fix_read_error(conf, mddev, r10_bio);
		unfreeze_array(conf);
2518 2519 2520
	} else
		r10_bio->devs[slot].bio = IO_BLOCKED;

2521
	rdev_dec_pending(rdev, mddev);
2522

2523
read_more:
2524 2525
	rdev = read_balance(conf, r10_bio, &max_sectors);
	if (rdev == NULL) {
2526 2527
		printk(KERN_ALERT "md/raid10:%s: %s: unrecoverable I/O"
		       " read error for block %llu\n",
2528
		       mdname(mddev), b,
2529 2530 2531 2532 2533 2534 2535 2536 2537
		       (unsigned long long)r10_bio->sector);
		raid_end_bio_io(r10_bio);
		return;
	}

	do_sync = (r10_bio->master_bio->bi_rw & REQ_SYNC);
	slot = r10_bio->read_slot;
	printk_ratelimited(
		KERN_ERR
2538
		"md/raid10:%s: %s: redirecting "
2539 2540 2541 2542 2543 2544
		"sector %llu to another mirror\n",
		mdname(mddev),
		bdevname(rdev->bdev, b),
		(unsigned long long)r10_bio->sector);
	bio = bio_clone_mddev(r10_bio->master_bio,
			      GFP_NOIO, mddev);
2545
	bio_trim(bio, r10_bio->sector - bio->bi_iter.bi_sector, max_sectors);
2546
	r10_bio->devs[slot].bio = bio;
2547
	r10_bio->devs[slot].rdev = rdev;
2548
	bio->bi_iter.bi_sector = r10_bio->devs[slot].addr
2549
		+ choose_data_offset(r10_bio, rdev);
2550 2551 2552 2553
	bio->bi_bdev = rdev->bdev;
	bio->bi_rw = READ | do_sync;
	bio->bi_private = r10_bio;
	bio->bi_end_io = raid10_end_read_request;
2554 2555 2556 2557 2558
	if (max_sectors < r10_bio->sectors) {
		/* Drat - have to split this up more */
		struct bio *mbio = r10_bio->master_bio;
		int sectors_handled =
			r10_bio->sector + max_sectors
2559
			- mbio->bi_iter.bi_sector;
2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571
		r10_bio->sectors = max_sectors;
		spin_lock_irq(&conf->device_lock);
		if (mbio->bi_phys_segments == 0)
			mbio->bi_phys_segments = 2;
		else
			mbio->bi_phys_segments++;
		spin_unlock_irq(&conf->device_lock);
		generic_make_request(bio);

		r10_bio = mempool_alloc(conf->r10bio_pool,
					GFP_NOIO);
		r10_bio->master_bio = mbio;
2572
		r10_bio->sectors = bio_sectors(mbio) - sectors_handled;
2573 2574 2575 2576
		r10_bio->state = 0;
		set_bit(R10BIO_ReadError,
			&r10_bio->state);
		r10_bio->mddev = mddev;
2577
		r10_bio->sector = mbio->bi_iter.bi_sector
2578 2579 2580 2581 2582
			+ sectors_handled;

		goto read_more;
	} else
		generic_make_request(bio);
2583 2584
}

2585
static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio)
2586 2587 2588 2589
{
	/* Some sort of write request has finished and it
	 * succeeded in writing where we thought there was a
	 * bad block.  So forget the bad block.
2590 2591
	 * Or possibly if failed and we need to record
	 * a bad block.
2592 2593
	 */
	int m;
2594
	struct md_rdev *rdev;
2595 2596 2597

	if (test_bit(R10BIO_IsSync, &r10_bio->state) ||
	    test_bit(R10BIO_IsRecover, &r10_bio->state)) {
2598 2599 2600 2601 2602
		for (m = 0; m < conf->copies; m++) {
			int dev = r10_bio->devs[m].devnum;
			rdev = conf->mirrors[dev].rdev;
			if (r10_bio->devs[m].bio == NULL)
				continue;
2603
			if (!r10_bio->devs[m].bio->bi_error) {
2604 2605 2606
				rdev_clear_badblocks(
					rdev,
					r10_bio->devs[m].addr,
2607
					r10_bio->sectors, 0);
2608 2609 2610 2611 2612 2613
			} else {
				if (!rdev_set_badblocks(
					    rdev,
					    r10_bio->devs[m].addr,
					    r10_bio->sectors, 0))
					md_error(conf->mddev, rdev);
2614
			}
2615 2616 2617
			rdev = conf->mirrors[dev].replacement;
			if (r10_bio->devs[m].repl_bio == NULL)
				continue;
2618 2619

			if (!r10_bio->devs[m].repl_bio->bi_error) {
2620 2621 2622
				rdev_clear_badblocks(
					rdev,
					r10_bio->devs[m].addr,
2623
					r10_bio->sectors, 0);
2624 2625 2626 2627 2628 2629 2630
			} else {
				if (!rdev_set_badblocks(
					    rdev,
					    r10_bio->devs[m].addr,
					    r10_bio->sectors, 0))
					md_error(conf->mddev, rdev);
			}
2631
		}
2632 2633
		put_buf(r10_bio);
	} else {
2634
		bool fail = false;
2635 2636 2637 2638 2639
		for (m = 0; m < conf->copies; m++) {
			int dev = r10_bio->devs[m].devnum;
			struct bio *bio = r10_bio->devs[m].bio;
			rdev = conf->mirrors[dev].rdev;
			if (bio == IO_MADE_GOOD) {
2640 2641 2642
				rdev_clear_badblocks(
					rdev,
					r10_bio->devs[m].addr,
2643
					r10_bio->sectors, 0);
2644
				rdev_dec_pending(rdev, conf->mddev);
2645
			} else if (bio != NULL && bio->bi_error) {
2646
				fail = true;
2647 2648 2649 2650 2651 2652
				if (!narrow_write_error(r10_bio, m)) {
					md_error(conf->mddev, rdev);
					set_bit(R10BIO_Degraded,
						&r10_bio->state);
				}
				rdev_dec_pending(rdev, conf->mddev);
2653
			}
2654 2655
			bio = r10_bio->devs[m].repl_bio;
			rdev = conf->mirrors[dev].replacement;
2656
			if (rdev && bio == IO_MADE_GOOD) {
2657 2658 2659
				rdev_clear_badblocks(
					rdev,
					r10_bio->devs[m].addr,
2660
					r10_bio->sectors, 0);
2661 2662
				rdev_dec_pending(rdev, conf->mddev);
			}
2663
		}
2664 2665 2666
		if (fail) {
			spin_lock_irq(&conf->device_lock);
			list_add(&r10_bio->retry_list, &conf->bio_end_io_list);
2667
			conf->nr_queued++;
2668 2669
			spin_unlock_irq(&conf->device_lock);
			md_wakeup_thread(conf->mddev->thread);
2670 2671 2672 2673
		} else {
			if (test_bit(R10BIO_WriteError,
				     &r10_bio->state))
				close_write(r10_bio);
2674
			raid_end_bio_io(r10_bio);
2675
		}
2676 2677 2678
	}
}

2679
static void raid10d(struct md_thread *thread)
L
Linus Torvalds 已提交
2680
{
2681
	struct mddev *mddev = thread->mddev;
2682
	struct r10bio *r10_bio;
L
Linus Torvalds 已提交
2683
	unsigned long flags;
2684
	struct r10conf *conf = mddev->private;
L
Linus Torvalds 已提交
2685
	struct list_head *head = &conf->retry_list;
2686
	struct blk_plug plug;
L
Linus Torvalds 已提交
2687 2688 2689

	md_check_recovery(mddev);

2690 2691 2692 2693 2694
	if (!list_empty_careful(&conf->bio_end_io_list) &&
	    !test_bit(MD_CHANGE_PENDING, &mddev->flags)) {
		LIST_HEAD(tmp);
		spin_lock_irqsave(&conf->device_lock, flags);
		if (!test_bit(MD_CHANGE_PENDING, &mddev->flags)) {
2695 2696 2697 2698
			while (!list_empty(&conf->bio_end_io_list)) {
				list_move(conf->bio_end_io_list.prev, &tmp);
				conf->nr_queued--;
			}
2699 2700 2701
		}
		spin_unlock_irqrestore(&conf->device_lock, flags);
		while (!list_empty(&tmp)) {
2702 2703
			r10_bio = list_first_entry(&tmp, struct r10bio,
						   retry_list);
2704
			list_del(&r10_bio->retry_list);
2705 2706 2707 2708 2709 2710
			if (mddev->degraded)
				set_bit(R10BIO_Degraded, &r10_bio->state);

			if (test_bit(R10BIO_WriteError,
				     &r10_bio->state))
				close_write(r10_bio);
2711 2712 2713 2714
			raid_end_bio_io(r10_bio);
		}
	}

2715
	blk_start_plug(&plug);
L
Linus Torvalds 已提交
2716
	for (;;) {
2717

2718
		flush_pending_writes(conf);
2719

2720 2721 2722
		spin_lock_irqsave(&conf->device_lock, flags);
		if (list_empty(head)) {
			spin_unlock_irqrestore(&conf->device_lock, flags);
L
Linus Torvalds 已提交
2723
			break;
2724
		}
2725
		r10_bio = list_entry(head->prev, struct r10bio, retry_list);
L
Linus Torvalds 已提交
2726
		list_del(head->prev);
2727
		conf->nr_queued--;
L
Linus Torvalds 已提交
2728 2729 2730
		spin_unlock_irqrestore(&conf->device_lock, flags);

		mddev = r10_bio->mddev;
2731
		conf = mddev->private;
2732 2733
		if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
		    test_bit(R10BIO_WriteError, &r10_bio->state))
2734
			handle_write_completed(conf, r10_bio);
2735 2736
		else if (test_bit(R10BIO_IsReshape, &r10_bio->state))
			reshape_request_write(mddev, r10_bio);
2737
		else if (test_bit(R10BIO_IsSync, &r10_bio->state))
L
Linus Torvalds 已提交
2738
			sync_request_write(mddev, r10_bio);
2739
		else if (test_bit(R10BIO_IsRecover, &r10_bio->state))
L
Linus Torvalds 已提交
2740
			recovery_request_write(mddev, r10_bio);
2741
		else if (test_bit(R10BIO_ReadError, &r10_bio->state))
2742
			handle_read_error(mddev, r10_bio);
2743 2744 2745 2746 2747 2748 2749
		else {
			/* just a partial read to be scheduled from a
			 * separate context
			 */
			int slot = r10_bio->read_slot;
			generic_make_request(r10_bio->devs[slot].bio);
		}
2750

2751
		cond_resched();
2752 2753
		if (mddev->flags & ~(1<<MD_CHANGE_PENDING))
			md_check_recovery(mddev);
L
Linus Torvalds 已提交
2754
	}
2755
	blk_finish_plug(&plug);
L
Linus Torvalds 已提交
2756 2757
}

2758
static int init_resync(struct r10conf *conf)
L
Linus Torvalds 已提交
2759 2760
{
	int buffs;
2761
	int i;
L
Linus Torvalds 已提交
2762 2763

	buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE;
2764
	BUG_ON(conf->r10buf_pool);
2765
	conf->have_replacement = 0;
2766
	for (i = 0; i < conf->geo.raid_disks; i++)
2767 2768
		if (conf->mirrors[i].replacement)
			conf->have_replacement = 1;
L
Linus Torvalds 已提交
2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807
	conf->r10buf_pool = mempool_create(buffs, r10buf_pool_alloc, r10buf_pool_free, conf);
	if (!conf->r10buf_pool)
		return -ENOMEM;
	conf->next_resync = 0;
	return 0;
}

/*
 * perform a "sync" on one "block"
 *
 * We need to make sure that no normal I/O request - particularly write
 * requests - conflict with active sync requests.
 *
 * This is achieved by tracking pending requests and a 'barrier' concept
 * that can be installed to exclude normal IO requests.
 *
 * Resync and recovery are handled very differently.
 * We differentiate by looking at MD_RECOVERY_SYNC in mddev->recovery.
 *
 * For resync, we iterate over virtual addresses, read all copies,
 * and update if there are differences.  If only one copy is live,
 * skip it.
 * For recovery, we iterate over physical addresses, read a good
 * value for each non-in_sync drive, and over-write.
 *
 * So, for recovery we may have several outstanding complex requests for a
 * given address, one for each out-of-sync device.  We model this by allocating
 * a number of r10_bio structures, one for each out-of-sync device.
 * As we setup these structures, we collect all bio's together into a list
 * which we then process collectively to add pages, and then process again
 * to pass to generic_make_request.
 *
 * The r10_bio structures are linked using a borrowed master_bio pointer.
 * This link is counted in ->remaining.  When the r10_bio that points to NULL
 * has its remaining count decremented to 0, the whole complex operation
 * is complete.
 *
 */

S
Shaohua Li 已提交
2808
static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr,
2809
			     int *skipped)
L
Linus Torvalds 已提交
2810
{
2811
	struct r10conf *conf = mddev->private;
2812
	struct r10bio *r10_bio;
L
Linus Torvalds 已提交
2813 2814 2815
	struct bio *biolist = NULL, *bio;
	sector_t max_sector, nr_sectors;
	int i;
2816
	int max_sync;
2817
	sector_t sync_blocks;
L
Linus Torvalds 已提交
2818 2819
	sector_t sectors_skipped = 0;
	int chunks_skipped = 0;
2820
	sector_t chunk_mask = conf->geo.chunk_mask;
L
Linus Torvalds 已提交
2821 2822 2823

	if (!conf->r10buf_pool)
		if (init_resync(conf))
2824
			return 0;
L
Linus Torvalds 已提交
2825

2826 2827 2828 2829 2830 2831
	/*
	 * Allow skipping a full rebuild for incremental assembly
	 * of a clean array, like RAID1 does.
	 */
	if (mddev->bitmap == NULL &&
	    mddev->recovery_cp == MaxSector &&
2832 2833
	    mddev->reshape_position == MaxSector &&
	    !test_bit(MD_RECOVERY_SYNC, &mddev->recovery) &&
2834
	    !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
2835
	    !test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
2836 2837
	    conf->fullsync == 0) {
		*skipped = 1;
2838
		return mddev->dev_sectors - sector_nr;
2839 2840
	}

L
Linus Torvalds 已提交
2841
 skipped:
2842
	max_sector = mddev->dev_sectors;
2843 2844
	if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) ||
	    test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
L
Linus Torvalds 已提交
2845 2846
		max_sector = mddev->resync_max_sectors;
	if (sector_nr >= max_sector) {
2847 2848 2849 2850 2851 2852 2853 2854 2855
		/* If we aborted, we need to abort the
		 * sync on the 'current' bitmap chucks (there can
		 * be several when recovering multiple devices).
		 * as we may have started syncing it but not finished.
		 * We can find the current address in
		 * mddev->curr_resync, but for recovery,
		 * we need to convert that to several
		 * virtual addresses.
		 */
2856 2857
		if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {
			end_reshape(conf);
2858
			close_sync(conf);
2859 2860 2861
			return 0;
		}

2862 2863 2864 2865
		if (mddev->curr_resync < max_sector) { /* aborted */
			if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery))
				bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
						&sync_blocks, 1);
2866
			else for (i = 0; i < conf->geo.raid_disks; i++) {
2867 2868 2869 2870 2871
				sector_t sect =
					raid10_find_virt(conf, mddev->curr_resync, i);
				bitmap_end_sync(mddev->bitmap, sect,
						&sync_blocks, 1);
			}
2872 2873 2874 2875 2876 2877 2878 2879
		} else {
			/* completed sync */
			if ((!mddev->bitmap || conf->fullsync)
			    && conf->have_replacement
			    && test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
				/* Completed a full sync so the replacements
				 * are now fully recovered.
				 */
2880
				for (i = 0; i < conf->geo.raid_disks; i++)
2881 2882 2883 2884 2885
					if (conf->mirrors[i].replacement)
						conf->mirrors[i].replacement
							->recovery_offset
							= MaxSector;
			}
2886
			conf->fullsync = 0;
2887
		}
2888
		bitmap_close_sync(mddev->bitmap);
L
Linus Torvalds 已提交
2889
		close_sync(conf);
2890
		*skipped = 1;
L
Linus Torvalds 已提交
2891 2892
		return sectors_skipped;
	}
2893 2894 2895 2896

	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
		return reshape_request(mddev, sector_nr, skipped);

2897
	if (chunks_skipped >= conf->geo.raid_disks) {
L
Linus Torvalds 已提交
2898 2899 2900
		/* if there has been nothing to do on any drive,
		 * then there is nothing to do at all..
		 */
2901 2902
		*skipped = 1;
		return (max_sector - sector_nr) + sectors_skipped;
L
Linus Torvalds 已提交
2903 2904
	}

2905 2906 2907
	if (max_sector > mddev->resync_max)
		max_sector = mddev->resync_max; /* Don't do IO beyond here */

L
Linus Torvalds 已提交
2908 2909 2910
	/* make sure whole request will fit in a chunk - if chunks
	 * are meaningful
	 */
2911 2912 2913
	if (conf->geo.near_copies < conf->geo.raid_disks &&
	    max_sector > (sector_nr | chunk_mask))
		max_sector = (sector_nr | chunk_mask) + 1;
L
Linus Torvalds 已提交
2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929

	/* Again, very different code for resync and recovery.
	 * Both must result in an r10bio with a list of bios that
	 * have bi_end_io, bi_sector, bi_bdev set,
	 * and bi_private set to the r10bio.
	 * For recovery, we may actually create several r10bios
	 * with 2 bios in each, that correspond to the bios in the main one.
	 * In this case, the subordinate r10bios link back through a
	 * borrowed master_bio pointer, and the counter in the master
	 * includes a ref from each subordinate.
	 */
	/* First, we decide what to do and set ->bi_end_io
	 * To end_sync_read if we want to read, and
	 * end_sync_write if we will want to write.
	 */

2930
	max_sync = RESYNC_PAGES << (PAGE_SHIFT-9);
L
Linus Torvalds 已提交
2931 2932
	if (!test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
		/* recovery... the complicated one */
2933
		int j;
L
Linus Torvalds 已提交
2934 2935
		r10_bio = NULL;

2936
		for (i = 0 ; i < conf->geo.raid_disks; i++) {
2937
			int still_degraded;
2938
			struct r10bio *rb2;
2939 2940
			sector_t sect;
			int must_sync;
2941
			int any_working;
2942
			struct raid10_info *mirror = &conf->mirrors[i];
2943 2944 2945 2946 2947 2948 2949

			if ((mirror->rdev == NULL ||
			     test_bit(In_sync, &mirror->rdev->flags))
			    &&
			    (mirror->replacement == NULL ||
			     test_bit(Faulty,
				      &mirror->replacement->flags)))
2950
				continue;
L
Linus Torvalds 已提交
2951

2952 2953 2954 2955
			still_degraded = 0;
			/* want to reconstruct this device */
			rb2 = r10_bio;
			sect = raid10_find_virt(conf, sector_nr, i);
2956 2957 2958 2959 2960 2961
			if (sect >= mddev->resync_max_sectors) {
				/* last stripe is not complete - don't
				 * try to recover this sector.
				 */
				continue;
			}
2962 2963 2964
			/* Unless we are doing a full sync, or a replacement
			 * we only need to recover the block if it is set in
			 * the bitmap
2965 2966 2967 2968 2969 2970
			 */
			must_sync = bitmap_start_sync(mddev->bitmap, sect,
						      &sync_blocks, 1);
			if (sync_blocks < max_sync)
				max_sync = sync_blocks;
			if (!must_sync &&
2971
			    mirror->replacement == NULL &&
2972 2973 2974 2975 2976 2977 2978
			    !conf->fullsync) {
				/* yep, skip the sync_blocks here, but don't assume
				 * that there will never be anything to do here
				 */
				chunks_skipped = -1;
				continue;
			}
2979

2980
			r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
2981
			r10_bio->state = 0;
2982 2983
			raise_barrier(conf, rb2 != NULL);
			atomic_set(&r10_bio->remaining, 0);
2984

2985 2986 2987 2988 2989 2990
			r10_bio->master_bio = (struct bio*)rb2;
			if (rb2)
				atomic_inc(&rb2->remaining);
			r10_bio->mddev = mddev;
			set_bit(R10BIO_IsRecover, &r10_bio->state);
			r10_bio->sector = sect;
L
Linus Torvalds 已提交
2991

2992 2993 2994 2995 2996
			raid10_find_phys(conf, r10_bio);

			/* Need to check if the array will still be
			 * degraded
			 */
2997
			for (j = 0; j < conf->geo.raid_disks; j++)
2998 2999 3000
				if (conf->mirrors[j].rdev == NULL ||
				    test_bit(Faulty, &conf->mirrors[j].rdev->flags)) {
					still_degraded = 1;
3001
					break;
L
Linus Torvalds 已提交
3002
				}
3003 3004 3005 3006

			must_sync = bitmap_start_sync(mddev->bitmap, sect,
						      &sync_blocks, still_degraded);

3007
			any_working = 0;
3008
			for (j=0; j<conf->copies;j++) {
3009
				int k;
3010
				int d = r10_bio->devs[j].devnum;
3011
				sector_t from_addr, to_addr;
3012
				struct md_rdev *rdev;
3013 3014
				sector_t sector, first_bad;
				int bad_sectors;
3015 3016 3017 3018
				if (!conf->mirrors[d].rdev ||
				    !test_bit(In_sync, &conf->mirrors[d].rdev->flags))
					continue;
				/* This is where we read from */
3019
				any_working = 1;
3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034
				rdev = conf->mirrors[d].rdev;
				sector = r10_bio->devs[j].addr;

				if (is_badblock(rdev, sector, max_sync,
						&first_bad, &bad_sectors)) {
					if (first_bad > sector)
						max_sync = first_bad - sector;
					else {
						bad_sectors -= (sector
								- first_bad);
						if (max_sync > bad_sectors)
							max_sync = bad_sectors;
						continue;
					}
				}
3035
				bio = r10_bio->devs[0].bio;
K
Kent Overstreet 已提交
3036
				bio_reset(bio);
3037 3038 3039 3040 3041
				bio->bi_next = biolist;
				biolist = bio;
				bio->bi_private = r10_bio;
				bio->bi_end_io = end_sync_read;
				bio->bi_rw = READ;
3042
				from_addr = r10_bio->devs[j].addr;
3043 3044
				bio->bi_iter.bi_sector = from_addr +
					rdev->data_offset;
3045 3046 3047
				bio->bi_bdev = rdev->bdev;
				atomic_inc(&rdev->nr_pending);
				/* and we write to 'i' (if not in_sync) */
3048 3049 3050 3051 3052

				for (k=0; k<conf->copies; k++)
					if (r10_bio->devs[k].devnum == i)
						break;
				BUG_ON(k == conf->copies);
3053
				to_addr = r10_bio->devs[k].addr;
3054
				r10_bio->devs[0].devnum = d;
3055
				r10_bio->devs[0].addr = from_addr;
3056
				r10_bio->devs[1].devnum = i;
3057
				r10_bio->devs[1].addr = to_addr;
3058

3059 3060 3061
				rdev = mirror->rdev;
				if (!test_bit(In_sync, &rdev->flags)) {
					bio = r10_bio->devs[1].bio;
K
Kent Overstreet 已提交
3062
					bio_reset(bio);
3063 3064 3065 3066 3067
					bio->bi_next = biolist;
					biolist = bio;
					bio->bi_private = r10_bio;
					bio->bi_end_io = end_sync_write;
					bio->bi_rw = WRITE;
3068
					bio->bi_iter.bi_sector = to_addr
3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090
						+ rdev->data_offset;
					bio->bi_bdev = rdev->bdev;
					atomic_inc(&r10_bio->remaining);
				} else
					r10_bio->devs[1].bio->bi_end_io = NULL;

				/* and maybe write to replacement */
				bio = r10_bio->devs[1].repl_bio;
				if (bio)
					bio->bi_end_io = NULL;
				rdev = mirror->replacement;
				/* Note: if rdev != NULL, then bio
				 * cannot be NULL as r10buf_pool_alloc will
				 * have allocated it.
				 * So the second test here is pointless.
				 * But it keeps semantic-checkers happy, and
				 * this comment keeps human reviewers
				 * happy.
				 */
				if (rdev == NULL || bio == NULL ||
				    test_bit(Faulty, &rdev->flags))
					break;
K
Kent Overstreet 已提交
3091
				bio_reset(bio);
3092 3093 3094 3095 3096
				bio->bi_next = biolist;
				biolist = bio;
				bio->bi_private = r10_bio;
				bio->bi_end_io = end_sync_write;
				bio->bi_rw = WRITE;
3097 3098
				bio->bi_iter.bi_sector = to_addr +
					rdev->data_offset;
3099 3100
				bio->bi_bdev = rdev->bdev;
				atomic_inc(&r10_bio->remaining);
3101 3102 3103
				break;
			}
			if (j == conf->copies) {
3104 3105 3106 3107 3108 3109 3110 3111 3112 3113
				/* Cannot recover, so abort the recovery or
				 * record a bad block */
				if (any_working) {
					/* problem is that there are bad blocks
					 * on other device(s)
					 */
					int k;
					for (k = 0; k < conf->copies; k++)
						if (r10_bio->devs[k].devnum == i)
							break;
3114 3115 3116 3117 3118 3119 3120 3121 3122 3123
					if (!test_bit(In_sync,
						      &mirror->rdev->flags)
					    && !rdev_set_badblocks(
						    mirror->rdev,
						    r10_bio->devs[k].addr,
						    max_sync, 0))
						any_working = 0;
					if (mirror->replacement &&
					    !rdev_set_badblocks(
						    mirror->replacement,
3124 3125 3126 3127 3128 3129 3130 3131 3132 3133
						    r10_bio->devs[k].addr,
						    max_sync, 0))
						any_working = 0;
				}
				if (!any_working)  {
					if (!test_and_set_bit(MD_RECOVERY_INTR,
							      &mddev->recovery))
						printk(KERN_INFO "md/raid10:%s: insufficient "
						       "working devices for recovery.\n",
						       mdname(mddev));
3134
					mirror->recovery_disabled
3135 3136
						= mddev->recovery_disabled;
				}
3137 3138 3139 3140
				put_buf(r10_bio);
				if (rb2)
					atomic_dec(&rb2->remaining);
				r10_bio = rb2;
3141
				break;
L
Linus Torvalds 已提交
3142
			}
3143
		}
L
Linus Torvalds 已提交
3144 3145
		if (biolist == NULL) {
			while (r10_bio) {
3146 3147
				struct r10bio *rb2 = r10_bio;
				r10_bio = (struct r10bio*) rb2->master_bio;
L
Linus Torvalds 已提交
3148 3149 3150 3151 3152 3153 3154 3155
				rb2->master_bio = NULL;
				put_buf(rb2);
			}
			goto giveup;
		}
	} else {
		/* resync. Schedule a read for every block at this virt offset */
		int count = 0;
3156

3157
		bitmap_cond_end_sync(mddev->bitmap, sector_nr, 0);
3158

3159 3160
		if (!bitmap_start_sync(mddev->bitmap, sector_nr,
				       &sync_blocks, mddev->degraded) &&
3161 3162
		    !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED,
						 &mddev->recovery)) {
3163 3164 3165 3166 3167 3168
			/* We can skip this block */
			*skipped = 1;
			return sync_blocks + sectors_skipped;
		}
		if (sync_blocks < max_sync)
			max_sync = sync_blocks;
L
Linus Torvalds 已提交
3169
		r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
3170
		r10_bio->state = 0;
L
Linus Torvalds 已提交
3171 3172 3173

		r10_bio->mddev = mddev;
		atomic_set(&r10_bio->remaining, 0);
3174 3175
		raise_barrier(conf, 0);
		conf->next_resync = sector_nr;
L
Linus Torvalds 已提交
3176 3177 3178 3179 3180

		r10_bio->master_bio = NULL;
		r10_bio->sector = sector_nr;
		set_bit(R10BIO_IsSync, &r10_bio->state);
		raid10_find_phys(conf, r10_bio);
3181
		r10_bio->sectors = (sector_nr | chunk_mask) - sector_nr + 1;
L
Linus Torvalds 已提交
3182

3183
		for (i = 0; i < conf->copies; i++) {
L
Linus Torvalds 已提交
3184
			int d = r10_bio->devs[i].devnum;
3185 3186 3187
			sector_t first_bad, sector;
			int bad_sectors;

3188 3189 3190
			if (r10_bio->devs[i].repl_bio)
				r10_bio->devs[i].repl_bio->bi_end_io = NULL;

L
Linus Torvalds 已提交
3191
			bio = r10_bio->devs[i].bio;
K
Kent Overstreet 已提交
3192
			bio_reset(bio);
3193
			bio->bi_error = -EIO;
L
Linus Torvalds 已提交
3194
			if (conf->mirrors[d].rdev == NULL ||
3195
			    test_bit(Faulty, &conf->mirrors[d].rdev->flags))
L
Linus Torvalds 已提交
3196
				continue;
3197 3198 3199 3200 3201 3202 3203 3204 3205
			sector = r10_bio->devs[i].addr;
			if (is_badblock(conf->mirrors[d].rdev,
					sector, max_sync,
					&first_bad, &bad_sectors)) {
				if (first_bad > sector)
					max_sync = first_bad - sector;
				else {
					bad_sectors -= (sector - first_bad);
					if (max_sync > bad_sectors)
3206
						max_sync = bad_sectors;
3207 3208 3209
					continue;
				}
			}
L
Linus Torvalds 已提交
3210 3211 3212 3213 3214 3215
			atomic_inc(&conf->mirrors[d].rdev->nr_pending);
			atomic_inc(&r10_bio->remaining);
			bio->bi_next = biolist;
			biolist = bio;
			bio->bi_private = r10_bio;
			bio->bi_end_io = end_sync_read;
3216
			bio->bi_rw = READ;
3217
			bio->bi_iter.bi_sector = sector +
L
Linus Torvalds 已提交
3218 3219 3220
				conf->mirrors[d].rdev->data_offset;
			bio->bi_bdev = conf->mirrors[d].rdev->bdev;
			count++;
3221 3222 3223 3224 3225 3226 3227 3228

			if (conf->mirrors[d].replacement == NULL ||
			    test_bit(Faulty,
				     &conf->mirrors[d].replacement->flags))
				continue;

			/* Need to set up for writing to the replacement */
			bio = r10_bio->devs[i].repl_bio;
K
Kent Overstreet 已提交
3229
			bio_reset(bio);
3230
			bio->bi_error = -EIO;
3231 3232 3233 3234 3235 3236 3237 3238

			sector = r10_bio->devs[i].addr;
			atomic_inc(&conf->mirrors[d].rdev->nr_pending);
			bio->bi_next = biolist;
			biolist = bio;
			bio->bi_private = r10_bio;
			bio->bi_end_io = end_sync_write;
			bio->bi_rw = WRITE;
3239
			bio->bi_iter.bi_sector = sector +
3240 3241 3242
				conf->mirrors[d].replacement->data_offset;
			bio->bi_bdev = conf->mirrors[d].replacement->bdev;
			count++;
L
Linus Torvalds 已提交
3243 3244 3245 3246 3247 3248
		}

		if (count < 2) {
			for (i=0; i<conf->copies; i++) {
				int d = r10_bio->devs[i].devnum;
				if (r10_bio->devs[i].bio->bi_end_io)
3249 3250
					rdev_dec_pending(conf->mirrors[d].rdev,
							 mddev);
3251 3252 3253 3254 3255
				if (r10_bio->devs[i].repl_bio &&
				    r10_bio->devs[i].repl_bio->bi_end_io)
					rdev_dec_pending(
						conf->mirrors[d].replacement,
						mddev);
L
Linus Torvalds 已提交
3256 3257 3258 3259 3260 3261 3262 3263
			}
			put_buf(r10_bio);
			biolist = NULL;
			goto giveup;
		}
	}

	nr_sectors = 0;
3264 3265
	if (sector_nr + max_sync < max_sector)
		max_sector = sector_nr + max_sync;
L
Linus Torvalds 已提交
3266 3267 3268 3269 3270 3271 3272 3273
	do {
		struct page *page;
		int len = PAGE_SIZE;
		if (sector_nr + (len>>9) > max_sector)
			len = (max_sector - sector_nr) << 9;
		if (len == 0)
			break;
		for (bio= biolist ; bio ; bio=bio->bi_next) {
3274
			struct bio *bio2;
L
Linus Torvalds 已提交
3275
			page = bio->bi_io_vec[bio->bi_vcnt].bv_page;
3276 3277 3278 3279 3280 3281 3282 3283 3284 3285
			if (bio_add_page(bio, page, len, 0))
				continue;

			/* stop here */
			bio->bi_io_vec[bio->bi_vcnt].bv_page = page;
			for (bio2 = biolist;
			     bio2 && bio2 != bio;
			     bio2 = bio2->bi_next) {
				/* remove last page from this bio */
				bio2->bi_vcnt--;
3286
				bio2->bi_iter.bi_size -= len;
3287
				bio_clear_flag(bio2, BIO_SEG_VALID);
L
Linus Torvalds 已提交
3288
			}
3289
			goto bio_full;
L
Linus Torvalds 已提交
3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306
		}
		nr_sectors += len>>9;
		sector_nr += len>>9;
	} while (biolist->bi_vcnt < RESYNC_PAGES);
 bio_full:
	r10_bio->sectors = nr_sectors;

	while (biolist) {
		bio = biolist;
		biolist = biolist->bi_next;

		bio->bi_next = NULL;
		r10_bio = bio->bi_private;
		r10_bio->sectors = nr_sectors;

		if (bio->bi_end_io == end_sync_read) {
			md_sync_acct(bio->bi_bdev, nr_sectors);
3307
			bio->bi_error = 0;
L
Linus Torvalds 已提交
3308 3309 3310 3311
			generic_make_request(bio);
		}
	}

3312 3313 3314 3315 3316 3317
	if (sectors_skipped)
		/* pretend they weren't skipped, it makes
		 * no important difference in this case
		 */
		md_done_sync(mddev, sectors_skipped, 1);

L
Linus Torvalds 已提交
3318 3319 3320
	return sectors_skipped + nr_sectors;
 giveup:
	/* There is nowhere to write, so all non-sync
3321 3322
	 * drives must be failed or in resync, all drives
	 * have a bad block, so try the next chunk...
L
Linus Torvalds 已提交
3323
	 */
3324 3325 3326 3327
	if (sector_nr + max_sync < max_sector)
		max_sector = sector_nr + max_sync;

	sectors_skipped += (max_sector - sector_nr);
L
Linus Torvalds 已提交
3328 3329 3330 3331 3332
	chunks_skipped ++;
	sector_nr = max_sector;
	goto skipped;
}

3333
static sector_t
3334
raid10_size(struct mddev *mddev, sector_t sectors, int raid_disks)
3335 3336
{
	sector_t size;
3337
	struct r10conf *conf = mddev->private;
3338 3339

	if (!raid_disks)
3340 3341
		raid_disks = min(conf->geo.raid_disks,
				 conf->prev.raid_disks);
3342
	if (!sectors)
3343
		sectors = conf->dev_sectors;
3344

3345 3346
	size = sectors >> conf->geo.chunk_shift;
	sector_div(size, conf->geo.far_copies);
3347
	size = size * raid_disks;
3348
	sector_div(size, conf->geo.near_copies);
3349

3350
	return size << conf->geo.chunk_shift;
3351 3352
}

3353 3354 3355 3356 3357 3358 3359
static void calc_sectors(struct r10conf *conf, sector_t size)
{
	/* Calculate the number of sectors-per-device that will
	 * actually be used, and set conf->dev_sectors and
	 * conf->stride
	 */

3360 3361 3362 3363
	size = size >> conf->geo.chunk_shift;
	sector_div(size, conf->geo.far_copies);
	size = size * conf->geo.raid_disks;
	sector_div(size, conf->geo.near_copies);
3364 3365 3366 3367 3368 3369 3370
	/* 'size' is now the number of chunks in the array */
	/* calculate "used chunks per device" */
	size = size * conf->copies;

	/* We need to round up when dividing by raid_disks to
	 * get the stride size.
	 */
3371
	size = DIV_ROUND_UP_SECTOR_T(size, conf->geo.raid_disks);
3372

3373
	conf->dev_sectors = size << conf->geo.chunk_shift;
3374

3375 3376
	if (conf->geo.far_offset)
		conf->geo.stride = 1 << conf->geo.chunk_shift;
3377
	else {
3378 3379
		sector_div(size, conf->geo.far_copies);
		conf->geo.stride = size << conf->geo.chunk_shift;
3380 3381
	}
}
3382

3383 3384 3385 3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406
enum geo_type {geo_new, geo_old, geo_start};
static int setup_geo(struct geom *geo, struct mddev *mddev, enum geo_type new)
{
	int nc, fc, fo;
	int layout, chunk, disks;
	switch (new) {
	case geo_old:
		layout = mddev->layout;
		chunk = mddev->chunk_sectors;
		disks = mddev->raid_disks - mddev->delta_disks;
		break;
	case geo_new:
		layout = mddev->new_layout;
		chunk = mddev->new_chunk_sectors;
		disks = mddev->raid_disks;
		break;
	default: /* avoid 'may be unused' warnings */
	case geo_start: /* new when starting reshape - raid_disks not
			 * updated yet. */
		layout = mddev->new_layout;
		chunk = mddev->new_chunk_sectors;
		disks = mddev->raid_disks + mddev->delta_disks;
		break;
	}
3407
	if (layout >> 19)
3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418
		return -1;
	if (chunk < (PAGE_SIZE >> 9) ||
	    !is_power_of_2(chunk))
		return -2;
	nc = layout & 255;
	fc = (layout >> 8) & 255;
	fo = layout & (1<<16);
	geo->raid_disks = disks;
	geo->near_copies = nc;
	geo->far_copies = fc;
	geo->far_offset = fo;
3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434
	switch (layout >> 17) {
	case 0:	/* original layout.  simple but not always optimal */
		geo->far_set_size = disks;
		break;
	case 1: /* "improved" layout which was buggy.  Hopefully no-one is
		 * actually using this, but leave code here just in case.*/
		geo->far_set_size = disks/fc;
		WARN(geo->far_set_size < fc,
		     "This RAID10 layout does not provide data safety - please backup and create new array\n");
		break;
	case 2: /* "improved" layout fixed to match documentation */
		geo->far_set_size = fc * nc;
		break;
	default: /* Not a valid layout */
		return -1;
	}
3435 3436 3437 3438 3439
	geo->chunk_mask = chunk - 1;
	geo->chunk_shift = ffz(~chunk);
	return nc*fc;
}

3440
static struct r10conf *setup_conf(struct mddev *mddev)
L
Linus Torvalds 已提交
3441
{
3442
	struct r10conf *conf = NULL;
3443
	int err = -EINVAL;
3444 3445 3446 3447
	struct geom geo;
	int copies;

	copies = setup_geo(&geo, mddev, geo_new);
L
Linus Torvalds 已提交
3448

3449
	if (copies == -2) {
3450 3451 3452
		printk(KERN_ERR "md/raid10:%s: chunk size must be "
		       "at least PAGE_SIZE(%ld) and be a power of 2.\n",
		       mdname(mddev), PAGE_SIZE);
3453
		goto out;
L
Linus Torvalds 已提交
3454
	}
3455

3456
	if (copies < 2 || copies > mddev->raid_disks) {
3457
		printk(KERN_ERR "md/raid10:%s: unsupported raid10 layout: 0x%8x\n",
3458
		       mdname(mddev), mddev->new_layout);
L
Linus Torvalds 已提交
3459 3460
		goto out;
	}
3461 3462

	err = -ENOMEM;
3463
	conf = kzalloc(sizeof(struct r10conf), GFP_KERNEL);
3464
	if (!conf)
L
Linus Torvalds 已提交
3465
		goto out;
3466

3467
	/* FIXME calc properly */
3468
	conf->mirrors = kzalloc(sizeof(struct raid10_info)*(mddev->raid_disks +
3469
							    max(0,-mddev->delta_disks)),
3470 3471 3472
				GFP_KERNEL);
	if (!conf->mirrors)
		goto out;
3473 3474 3475

	conf->tmppage = alloc_page(GFP_KERNEL);
	if (!conf->tmppage)
3476 3477
		goto out;

3478 3479
	conf->geo = geo;
	conf->copies = copies;
3480 3481 3482 3483 3484
	conf->r10bio_pool = mempool_create(NR_RAID10_BIOS, r10bio_pool_alloc,
					   r10bio_pool_free, conf);
	if (!conf->r10bio_pool)
		goto out;

3485
	calc_sectors(conf, mddev->dev_sectors);
3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500
	if (mddev->reshape_position == MaxSector) {
		conf->prev = conf->geo;
		conf->reshape_progress = MaxSector;
	} else {
		if (setup_geo(&conf->prev, mddev, geo_old) != conf->copies) {
			err = -EINVAL;
			goto out;
		}
		conf->reshape_progress = mddev->reshape_position;
		if (conf->prev.far_offset)
			conf->prev.stride = 1 << conf->prev.chunk_shift;
		else
			/* far_copies must be 1 */
			conf->prev.stride = conf->dev_sectors;
	}
3501
	conf->reshape_safe = conf->reshape_progress;
3502
	spin_lock_init(&conf->device_lock);
3503
	INIT_LIST_HEAD(&conf->retry_list);
3504
	INIT_LIST_HEAD(&conf->bio_end_io_list);
3505 3506 3507 3508

	spin_lock_init(&conf->resync_lock);
	init_waitqueue_head(&conf->wait_barrier);

3509
	conf->thread = md_register_thread(raid10d, mddev, "raid10");
3510 3511 3512 3513 3514 3515 3516
	if (!conf->thread)
		goto out;

	conf->mddev = mddev;
	return conf;

 out:
3517 3518 3519
	if (err == -ENOMEM)
		printk(KERN_ERR "md/raid10:%s: couldn't allocate memory.\n",
		       mdname(mddev));
3520
	if (conf) {
3521
		mempool_destroy(conf->r10bio_pool);
3522 3523 3524 3525 3526 3527 3528
		kfree(conf->mirrors);
		safe_put_page(conf->tmppage);
		kfree(conf);
	}
	return ERR_PTR(err);
}

S
Shaohua Li 已提交
3529
static int raid10_run(struct mddev *mddev)
3530
{
3531
	struct r10conf *conf;
3532
	int i, disk_idx, chunk_size;
3533
	struct raid10_info *disk;
3534
	struct md_rdev *rdev;
3535
	sector_t size;
3536 3537
	sector_t min_offset_diff = 0;
	int first = 1;
S
Shaohua Li 已提交
3538
	bool discard_supported = false;
3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552

	if (mddev->private == NULL) {
		conf = setup_conf(mddev);
		if (IS_ERR(conf))
			return PTR_ERR(conf);
		mddev->private = conf;
	}
	conf = mddev->private;
	if (!conf)
		goto out;

	mddev->thread = conf->thread;
	conf->thread = NULL;

3553
	chunk_size = mddev->chunk_sectors << 9;
3554
	if (mddev->queue) {
S
Shaohua Li 已提交
3555 3556
		blk_queue_max_discard_sectors(mddev->queue,
					      mddev->chunk_sectors);
3557
		blk_queue_max_write_same_sectors(mddev->queue, 0);
3558 3559 3560 3561 3562 3563 3564
		blk_queue_io_min(mddev->queue, chunk_size);
		if (conf->geo.raid_disks % conf->geo.near_copies)
			blk_queue_io_opt(mddev->queue, chunk_size * conf->geo.raid_disks);
		else
			blk_queue_io_opt(mddev->queue, chunk_size *
					 (conf->geo.raid_disks / conf->geo.near_copies));
	}
3565

3566
	rdev_for_each(rdev, mddev) {
3567
		long long diff;
3568
		struct request_queue *q;
3569

L
Linus Torvalds 已提交
3570
		disk_idx = rdev->raid_disk;
3571 3572 3573 3574
		if (disk_idx < 0)
			continue;
		if (disk_idx >= conf->geo.raid_disks &&
		    disk_idx >= conf->prev.raid_disks)
L
Linus Torvalds 已提交
3575 3576 3577
			continue;
		disk = conf->mirrors + disk_idx;

3578 3579 3580 3581 3582 3583 3584 3585 3586
		if (test_bit(Replacement, &rdev->flags)) {
			if (disk->replacement)
				goto out_free_conf;
			disk->replacement = rdev;
		} else {
			if (disk->rdev)
				goto out_free_conf;
			disk->rdev = rdev;
		}
3587
		q = bdev_get_queue(rdev->bdev);
3588 3589 3590 3591 3592 3593 3594
		diff = (rdev->new_data_offset - rdev->data_offset);
		if (!mddev->reshape_backwards)
			diff = -diff;
		if (diff < 0)
			diff = 0;
		if (first || diff < min_offset_diff)
			min_offset_diff = diff;
3595

3596 3597 3598
		if (mddev->gendisk)
			disk_stack_limits(mddev->gendisk, rdev->bdev,
					  rdev->data_offset << 9);
L
Linus Torvalds 已提交
3599 3600

		disk->head_position = 0;
S
Shaohua Li 已提交
3601 3602 3603

		if (blk_queue_discard(bdev_get_queue(rdev->bdev)))
			discard_supported = true;
L
Linus Torvalds 已提交
3604
	}
3605

3606 3607 3608 3609 3610 3611 3612 3613
	if (mddev->queue) {
		if (discard_supported)
			queue_flag_set_unlocked(QUEUE_FLAG_DISCARD,
						mddev->queue);
		else
			queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD,
						  mddev->queue);
	}
3614
	/* need to check that every block has at least one working mirror */
3615
	if (!enough(conf, -1)) {
3616
		printk(KERN_ERR "md/raid10:%s: not enough operational mirrors.\n",
3617
		       mdname(mddev));
L
Linus Torvalds 已提交
3618 3619 3620
		goto out_free_conf;
	}

3621 3622 3623 3624 3625 3626
	if (conf->reshape_progress != MaxSector) {
		/* must ensure that shape change is supported */
		if (conf->geo.far_copies != 1 &&
		    conf->geo.far_offset == 0)
			goto out_free_conf;
		if (conf->prev.far_copies != 1 &&
3627
		    conf->prev.far_offset == 0)
3628 3629 3630
			goto out_free_conf;
	}

L
Linus Torvalds 已提交
3631
	mddev->degraded = 0;
3632 3633 3634 3635
	for (i = 0;
	     i < conf->geo.raid_disks
		     || i < conf->prev.raid_disks;
	     i++) {
L
Linus Torvalds 已提交
3636 3637 3638

		disk = conf->mirrors + i;

3639 3640 3641 3642 3643 3644 3645
		if (!disk->rdev && disk->replacement) {
			/* The replacement is all we have - use it */
			disk->rdev = disk->replacement;
			disk->replacement = NULL;
			clear_bit(Replacement, &disk->rdev->flags);
		}

3646
		if (!disk->rdev ||
3647
		    !test_bit(In_sync, &disk->rdev->flags)) {
L
Linus Torvalds 已提交
3648 3649
			disk->head_position = 0;
			mddev->degraded++;
3650 3651
			if (disk->rdev &&
			    disk->rdev->saved_raid_disk < 0)
3652
				conf->fullsync = 1;
L
Linus Torvalds 已提交
3653
		}
3654
		disk->recovery_disabled = mddev->recovery_disabled - 1;
L
Linus Torvalds 已提交
3655 3656
	}

3657
	if (mddev->recovery_cp != MaxSector)
3658
		printk(KERN_NOTICE "md/raid10:%s: not clean"
3659 3660
		       " -- starting background reconstruction\n",
		       mdname(mddev));
L
Linus Torvalds 已提交
3661
	printk(KERN_INFO
3662
		"md/raid10:%s: active with %d out of %d devices\n",
3663 3664
		mdname(mddev), conf->geo.raid_disks - mddev->degraded,
		conf->geo.raid_disks);
L
Linus Torvalds 已提交
3665 3666 3667
	/*
	 * Ok, everything is just fine now
	 */
3668 3669 3670 3671
	mddev->dev_sectors = conf->dev_sectors;
	size = raid10_size(mddev, 0, 0);
	md_set_array_sectors(mddev, size);
	mddev->resync_max_sectors = size;
L
Linus Torvalds 已提交
3672

3673
	if (mddev->queue) {
3674
		int stripe = conf->geo.raid_disks *
3675
			((mddev->chunk_sectors << 9) / PAGE_SIZE);
3676 3677 3678 3679 3680

		/* Calculate max read-ahead size.
		 * We need to readahead at least twice a whole stripe....
		 * maybe...
		 */
3681
		stripe /= conf->geo.near_copies;
3682 3683
		if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
			mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
L
Linus Torvalds 已提交
3684 3685
	}

3686 3687 3688
	if (md_integrity_register(mddev))
		goto out_free_conf;

3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711
	if (conf->reshape_progress != MaxSector) {
		unsigned long before_length, after_length;

		before_length = ((1 << conf->prev.chunk_shift) *
				 conf->prev.far_copies);
		after_length = ((1 << conf->geo.chunk_shift) *
				conf->geo.far_copies);

		if (max(before_length, after_length) > min_offset_diff) {
			/* This cannot work */
			printk("md/raid10: offset difference not enough to continue reshape\n");
			goto out_free_conf;
		}
		conf->offset_diff = min_offset_diff;

		clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
		clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
		set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
		set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
		mddev->sync_thread = md_register_thread(md_do_sync, mddev,
							"reshape");
	}

L
Linus Torvalds 已提交
3712 3713 3714
	return 0;

out_free_conf:
3715
	md_unregister_thread(&mddev->thread);
3716
	mempool_destroy(conf->r10bio_pool);
3717
	safe_put_page(conf->tmppage);
3718
	kfree(conf->mirrors);
L
Linus Torvalds 已提交
3719 3720 3721 3722 3723 3724
	kfree(conf);
	mddev->private = NULL;
out:
	return -EIO;
}

N
NeilBrown 已提交
3725
static void raid10_free(struct mddev *mddev, void *priv)
L
Linus Torvalds 已提交
3726
{
N
NeilBrown 已提交
3727
	struct r10conf *conf = priv;
L
Linus Torvalds 已提交
3728

3729
	mempool_destroy(conf->r10bio_pool);
3730
	safe_put_page(conf->tmppage);
3731
	kfree(conf->mirrors);
3732 3733
	kfree(conf->mirrors_old);
	kfree(conf->mirrors_new);
L
Linus Torvalds 已提交
3734 3735 3736
	kfree(conf);
}

3737
static void raid10_quiesce(struct mddev *mddev, int state)
3738
{
3739
	struct r10conf *conf = mddev->private;
3740 3741 3742 3743 3744 3745 3746 3747 3748 3749

	switch(state) {
	case 1:
		raise_barrier(conf, 0);
		break;
	case 0:
		lower_barrier(conf);
		break;
	}
}
L
Linus Torvalds 已提交
3750

3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767
static int raid10_resize(struct mddev *mddev, sector_t sectors)
{
	/* Resize of 'far' arrays is not supported.
	 * For 'near' and 'offset' arrays we can set the
	 * number of sectors used to be an appropriate multiple
	 * of the chunk size.
	 * For 'offset', this is far_copies*chunksize.
	 * For 'near' the multiplier is the LCM of
	 * near_copies and raid_disks.
	 * So if far_copies > 1 && !far_offset, fail.
	 * Else find LCM(raid_disks, near_copy)*far_copies and
	 * multiply by chunk_size.  Then round to this number.
	 * This is mostly done by raid10_size()
	 */
	struct r10conf *conf = mddev->private;
	sector_t oldsize, size;

3768 3769 3770
	if (mddev->reshape_position != MaxSector)
		return -EBUSY;

3771
	if (conf->geo.far_copies > 1 && !conf->geo.far_offset)
3772 3773 3774 3775
		return -EINVAL;

	oldsize = raid10_size(mddev, 0, 0);
	size = raid10_size(mddev, sectors, 0);
3776 3777
	if (mddev->external_size &&
	    mddev->array_sectors > size)
3778
		return -EINVAL;
3779 3780 3781 3782 3783 3784
	if (mddev->bitmap) {
		int ret = bitmap_resize(mddev->bitmap, size, 0, 0);
		if (ret)
			return ret;
	}
	md_set_array_sectors(mddev, size);
3785 3786 3787 3788 3789 3790 3791
	set_capacity(mddev->gendisk, mddev->array_sectors);
	revalidate_disk(mddev->gendisk);
	if (sectors > mddev->dev_sectors &&
	    mddev->recovery_cp > oldsize) {
		mddev->recovery_cp = oldsize;
		set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
	}
3792 3793
	calc_sectors(conf, sectors);
	mddev->dev_sectors = conf->dev_sectors;
3794 3795 3796 3797
	mddev->resync_max_sectors = size;
	return 0;
}

3798
static void *raid10_takeover_raid0(struct mddev *mddev, sector_t size, int devs)
3799
{
3800
	struct md_rdev *rdev;
3801
	struct r10conf *conf;
3802 3803

	if (mddev->degraded > 0) {
3804 3805
		printk(KERN_ERR "md/raid10:%s: Error: degraded raid0!\n",
		       mdname(mddev));
3806 3807
		return ERR_PTR(-EINVAL);
	}
3808
	sector_div(size, devs);
3809 3810 3811 3812 3813 3814 3815 3816 3817 3818

	/* Set new parameters */
	mddev->new_level = 10;
	/* new layout: far_copies = 1, near_copies = 2 */
	mddev->new_layout = (1<<8) + 2;
	mddev->new_chunk_sectors = mddev->chunk_sectors;
	mddev->delta_disks = mddev->raid_disks;
	mddev->raid_disks *= 2;
	/* make sure it will be not marked as dirty */
	mddev->recovery_cp = MaxSector;
3819
	mddev->dev_sectors = size;
3820 3821

	conf = setup_conf(mddev);
3822
	if (!IS_ERR(conf)) {
3823
		rdev_for_each(rdev, mddev)
3824
			if (rdev->raid_disk >= 0) {
3825
				rdev->new_raid_disk = rdev->raid_disk * 2;
3826 3827
				rdev->sectors = size;
			}
3828 3829 3830
		conf->barrier = 1;
	}

3831 3832 3833
	return conf;
}

3834
static void *raid10_takeover(struct mddev *mddev)
3835
{
3836
	struct r0conf *raid0_conf;
3837 3838 3839 3840 3841 3842

	/* raid10 can take over:
	 *  raid0 - providing it has only two drives
	 */
	if (mddev->level == 0) {
		/* for raid0 takeover only one zone is supported */
3843 3844
		raid0_conf = mddev->private;
		if (raid0_conf->nr_strip_zones > 1) {
3845 3846 3847
			printk(KERN_ERR "md/raid10:%s: cannot takeover raid 0"
			       " with more than one zone.\n",
			       mdname(mddev));
3848 3849
			return ERR_PTR(-EINVAL);
		}
3850 3851 3852
		return raid10_takeover_raid0(mddev,
			raid0_conf->strip_zone->zone_end,
			raid0_conf->strip_zone->nb_dev);
3853 3854 3855 3856
	}
	return ERR_PTR(-EINVAL);
}

3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897
static int raid10_check_reshape(struct mddev *mddev)
{
	/* Called when there is a request to change
	 * - layout (to ->new_layout)
	 * - chunk size (to ->new_chunk_sectors)
	 * - raid_disks (by delta_disks)
	 * or when trying to restart a reshape that was ongoing.
	 *
	 * We need to validate the request and possibly allocate
	 * space if that might be an issue later.
	 *
	 * Currently we reject any reshape of a 'far' mode array,
	 * allow chunk size to change if new is generally acceptable,
	 * allow raid_disks to increase, and allow
	 * a switch between 'near' mode and 'offset' mode.
	 */
	struct r10conf *conf = mddev->private;
	struct geom geo;

	if (conf->geo.far_copies != 1 && !conf->geo.far_offset)
		return -EINVAL;

	if (setup_geo(&geo, mddev, geo_start) != conf->copies)
		/* mustn't change number of copies */
		return -EINVAL;
	if (geo.far_copies > 1 && !geo.far_offset)
		/* Cannot switch to 'far' mode */
		return -EINVAL;

	if (mddev->array_sectors & geo.chunk_mask)
			/* not factor of array size */
			return -EINVAL;

	if (!enough(conf, -1))
		return -EINVAL;

	kfree(conf->mirrors_new);
	conf->mirrors_new = NULL;
	if (mddev->delta_disks > 0) {
		/* allocate new 'mirrors' list */
		conf->mirrors_new = kzalloc(
3898
			sizeof(struct raid10_info)
3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983
			*(mddev->raid_disks +
			  mddev->delta_disks),
			GFP_KERNEL);
		if (!conf->mirrors_new)
			return -ENOMEM;
	}
	return 0;
}

/*
 * Need to check if array has failed when deciding whether to:
 *  - start an array
 *  - remove non-faulty devices
 *  - add a spare
 *  - allow a reshape
 * This determination is simple when no reshape is happening.
 * However if there is a reshape, we need to carefully check
 * both the before and after sections.
 * This is because some failed devices may only affect one
 * of the two sections, and some non-in_sync devices may
 * be insync in the section most affected by failed devices.
 */
static int calc_degraded(struct r10conf *conf)
{
	int degraded, degraded2;
	int i;

	rcu_read_lock();
	degraded = 0;
	/* 'prev' section first */
	for (i = 0; i < conf->prev.raid_disks; i++) {
		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
		if (!rdev || test_bit(Faulty, &rdev->flags))
			degraded++;
		else if (!test_bit(In_sync, &rdev->flags))
			/* When we can reduce the number of devices in
			 * an array, this might not contribute to
			 * 'degraded'.  It does now.
			 */
			degraded++;
	}
	rcu_read_unlock();
	if (conf->geo.raid_disks == conf->prev.raid_disks)
		return degraded;
	rcu_read_lock();
	degraded2 = 0;
	for (i = 0; i < conf->geo.raid_disks; i++) {
		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
		if (!rdev || test_bit(Faulty, &rdev->flags))
			degraded2++;
		else if (!test_bit(In_sync, &rdev->flags)) {
			/* If reshape is increasing the number of devices,
			 * this section has already been recovered, so
			 * it doesn't contribute to degraded.
			 * else it does.
			 */
			if (conf->geo.raid_disks <= conf->prev.raid_disks)
				degraded2++;
		}
	}
	rcu_read_unlock();
	if (degraded2 > degraded)
		return degraded2;
	return degraded;
}

static int raid10_start_reshape(struct mddev *mddev)
{
	/* A 'reshape' has been requested. This commits
	 * the various 'new' fields and sets MD_RECOVER_RESHAPE
	 * This also checks if there are enough spares and adds them
	 * to the array.
	 * We currently require enough spares to make the final
	 * array non-degraded.  We also require that the difference
	 * between old and new data_offset - on each device - is
	 * enough that we never risk over-writing.
	 */

	unsigned long before_length, after_length;
	sector_t min_offset_diff = 0;
	int first = 1;
	struct geom new;
	struct r10conf *conf = mddev->private;
	struct md_rdev *rdev;
	int spares = 0;
3984
	int ret;
3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022

	if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
		return -EBUSY;

	if (setup_geo(&new, mddev, geo_start) != conf->copies)
		return -EINVAL;

	before_length = ((1 << conf->prev.chunk_shift) *
			 conf->prev.far_copies);
	after_length = ((1 << conf->geo.chunk_shift) *
			conf->geo.far_copies);

	rdev_for_each(rdev, mddev) {
		if (!test_bit(In_sync, &rdev->flags)
		    && !test_bit(Faulty, &rdev->flags))
			spares++;
		if (rdev->raid_disk >= 0) {
			long long diff = (rdev->new_data_offset
					  - rdev->data_offset);
			if (!mddev->reshape_backwards)
				diff = -diff;
			if (diff < 0)
				diff = 0;
			if (first || diff < min_offset_diff)
				min_offset_diff = diff;
		}
	}

	if (max(before_length, after_length) > min_offset_diff)
		return -EINVAL;

	if (spares < mddev->delta_disks)
		return -EINVAL;

	conf->offset_diff = min_offset_diff;
	spin_lock_irq(&conf->device_lock);
	if (conf->mirrors_new) {
		memcpy(conf->mirrors_new, conf->mirrors,
4023
		       sizeof(struct raid10_info)*conf->prev.raid_disks);
4024
		smp_mb();
4025
		kfree(conf->mirrors_old);
4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043
		conf->mirrors_old = conf->mirrors;
		conf->mirrors = conf->mirrors_new;
		conf->mirrors_new = NULL;
	}
	setup_geo(&conf->geo, mddev, geo_start);
	smp_mb();
	if (mddev->reshape_backwards) {
		sector_t size = raid10_size(mddev, 0, 0);
		if (size < mddev->array_sectors) {
			spin_unlock_irq(&conf->device_lock);
			printk(KERN_ERR "md/raid10:%s: array size must be reduce before number of disks\n",
			       mdname(mddev));
			return -EINVAL;
		}
		mddev->resync_max_sectors = size;
		conf->reshape_progress = size;
	} else
		conf->reshape_progress = 0;
4044
	conf->reshape_safe = conf->reshape_progress;
4045 4046
	spin_unlock_irq(&conf->device_lock);

4047 4048 4049 4050 4051 4052 4053 4054
	if (mddev->delta_disks && mddev->bitmap) {
		ret = bitmap_resize(mddev->bitmap,
				    raid10_size(mddev, 0,
						conf->geo.raid_disks),
				    0, 0);
		if (ret)
			goto abort;
	}
4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087
	if (mddev->delta_disks > 0) {
		rdev_for_each(rdev, mddev)
			if (rdev->raid_disk < 0 &&
			    !test_bit(Faulty, &rdev->flags)) {
				if (raid10_add_disk(mddev, rdev) == 0) {
					if (rdev->raid_disk >=
					    conf->prev.raid_disks)
						set_bit(In_sync, &rdev->flags);
					else
						rdev->recovery_offset = 0;

					if (sysfs_link_rdev(mddev, rdev))
						/* Failure here  is OK */;
				}
			} else if (rdev->raid_disk >= conf->prev.raid_disks
				   && !test_bit(Faulty, &rdev->flags)) {
				/* This is a spare that was manually added */
				set_bit(In_sync, &rdev->flags);
			}
	}
	/* When a reshape changes the number of devices,
	 * ->degraded is measured against the larger of the
	 * pre and  post numbers.
	 */
	spin_lock_irq(&conf->device_lock);
	mddev->degraded = calc_degraded(conf);
	spin_unlock_irq(&conf->device_lock);
	mddev->raid_disks = conf->geo.raid_disks;
	mddev->reshape_position = conf->reshape_progress;
	set_bit(MD_CHANGE_DEVS, &mddev->flags);

	clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
	clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
4088
	clear_bit(MD_RECOVERY_DONE, &mddev->recovery);
4089 4090 4091 4092 4093 4094
	set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
	set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);

	mddev->sync_thread = md_register_thread(md_do_sync, mddev,
						"reshape");
	if (!mddev->sync_thread) {
4095 4096
		ret = -EAGAIN;
		goto abort;
4097 4098 4099 4100 4101
	}
	conf->reshape_checkpoint = jiffies;
	md_wakeup_thread(mddev->sync_thread);
	md_new_event(mddev);
	return 0;
4102 4103 4104 4105 4106 4107 4108 4109 4110 4111

abort:
	mddev->recovery = 0;
	spin_lock_irq(&conf->device_lock);
	conf->geo = conf->prev;
	mddev->raid_disks = conf->geo.raid_disks;
	rdev_for_each(rdev, mddev)
		rdev->new_data_offset = rdev->data_offset;
	smp_wmb();
	conf->reshape_progress = MaxSector;
4112
	conf->reshape_safe = MaxSector;
4113 4114 4115
	mddev->reshape_position = MaxSector;
	spin_unlock_irq(&conf->device_lock);
	return ret;
4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155
}

/* Calculate the last device-address that could contain
 * any block from the chunk that includes the array-address 's'
 * and report the next address.
 * i.e. the address returned will be chunk-aligned and after
 * any data that is in the chunk containing 's'.
 */
static sector_t last_dev_address(sector_t s, struct geom *geo)
{
	s = (s | geo->chunk_mask) + 1;
	s >>= geo->chunk_shift;
	s *= geo->near_copies;
	s = DIV_ROUND_UP_SECTOR_T(s, geo->raid_disks);
	s *= geo->far_copies;
	s <<= geo->chunk_shift;
	return s;
}

/* Calculate the first device-address that could contain
 * any block from the chunk that includes the array-address 's'.
 * This too will be the start of a chunk
 */
static sector_t first_dev_address(sector_t s, struct geom *geo)
{
	s >>= geo->chunk_shift;
	s *= geo->near_copies;
	sector_div(s, geo->raid_disks);
	s *= geo->far_copies;
	s <<= geo->chunk_shift;
	return s;
}

static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
				int *skipped)
{
	/* We simply copy at most one chunk (smallest of old and new)
	 * at a time, possibly less if that exceeds RESYNC_PAGES,
	 * or we hit a bad block or something.
	 * This might mean we pause for normal IO in the middle of
4156
	 * a chunk, but that is not a problem as mddev->reshape_position
4157 4158 4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179
	 * can record any location.
	 *
	 * If we will want to write to a location that isn't
	 * yet recorded as 'safe' (i.e. in metadata on disk) then
	 * we need to flush all reshape requests and update the metadata.
	 *
	 * When reshaping forwards (e.g. to more devices), we interpret
	 * 'safe' as the earliest block which might not have been copied
	 * down yet.  We divide this by previous stripe size and multiply
	 * by previous stripe length to get lowest device offset that we
	 * cannot write to yet.
	 * We interpret 'sector_nr' as an address that we want to write to.
	 * From this we use last_device_address() to find where we might
	 * write to, and first_device_address on the  'safe' position.
	 * If this 'next' write position is after the 'safe' position,
	 * we must update the metadata to increase the 'safe' position.
	 *
	 * When reshaping backwards, we round in the opposite direction
	 * and perform the reverse test:  next write position must not be
	 * less than current safe position.
	 *
	 * In all this the minimum difference in data offsets
	 * (conf->offset_diff - always positive) allows a bit of slack,
4180
	 * so next can be after 'safe', but not by more than offset_diff
4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281
	 *
	 * We need to prepare all the bios here before we start any IO
	 * to ensure the size we choose is acceptable to all devices.
	 * The means one for each copy for write-out and an extra one for
	 * read-in.
	 * We store the read-in bio in ->master_bio and the others in
	 * ->devs[x].bio and ->devs[x].repl_bio.
	 */
	struct r10conf *conf = mddev->private;
	struct r10bio *r10_bio;
	sector_t next, safe, last;
	int max_sectors;
	int nr_sectors;
	int s;
	struct md_rdev *rdev;
	int need_flush = 0;
	struct bio *blist;
	struct bio *bio, *read_bio;
	int sectors_done = 0;

	if (sector_nr == 0) {
		/* If restarting in the middle, skip the initial sectors */
		if (mddev->reshape_backwards &&
		    conf->reshape_progress < raid10_size(mddev, 0, 0)) {
			sector_nr = (raid10_size(mddev, 0, 0)
				     - conf->reshape_progress);
		} else if (!mddev->reshape_backwards &&
			   conf->reshape_progress > 0)
			sector_nr = conf->reshape_progress;
		if (sector_nr) {
			mddev->curr_resync_completed = sector_nr;
			sysfs_notify(&mddev->kobj, NULL, "sync_completed");
			*skipped = 1;
			return sector_nr;
		}
	}

	/* We don't use sector_nr to track where we are up to
	 * as that doesn't work well for ->reshape_backwards.
	 * So just use ->reshape_progress.
	 */
	if (mddev->reshape_backwards) {
		/* 'next' is the earliest device address that we might
		 * write to for this chunk in the new layout
		 */
		next = first_dev_address(conf->reshape_progress - 1,
					 &conf->geo);

		/* 'safe' is the last device address that we might read from
		 * in the old layout after a restart
		 */
		safe = last_dev_address(conf->reshape_safe - 1,
					&conf->prev);

		if (next + conf->offset_diff < safe)
			need_flush = 1;

		last = conf->reshape_progress - 1;
		sector_nr = last & ~(sector_t)(conf->geo.chunk_mask
					       & conf->prev.chunk_mask);
		if (sector_nr + RESYNC_BLOCK_SIZE/512 < last)
			sector_nr = last + 1 - RESYNC_BLOCK_SIZE/512;
	} else {
		/* 'next' is after the last device address that we
		 * might write to for this chunk in the new layout
		 */
		next = last_dev_address(conf->reshape_progress, &conf->geo);

		/* 'safe' is the earliest device address that we might
		 * read from in the old layout after a restart
		 */
		safe = first_dev_address(conf->reshape_safe, &conf->prev);

		/* Need to update metadata if 'next' might be beyond 'safe'
		 * as that would possibly corrupt data
		 */
		if (next > safe + conf->offset_diff)
			need_flush = 1;

		sector_nr = conf->reshape_progress;
		last  = sector_nr | (conf->geo.chunk_mask
				     & conf->prev.chunk_mask);

		if (sector_nr + RESYNC_BLOCK_SIZE/512 <= last)
			last = sector_nr + RESYNC_BLOCK_SIZE/512 - 1;
	}

	if (need_flush ||
	    time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
		/* Need to update reshape_position in metadata */
		wait_barrier(conf);
		mddev->reshape_position = conf->reshape_progress;
		if (mddev->reshape_backwards)
			mddev->curr_resync_completed = raid10_size(mddev, 0, 0)
				- conf->reshape_progress;
		else
			mddev->curr_resync_completed = conf->reshape_progress;
		conf->reshape_checkpoint = jiffies;
		set_bit(MD_CHANGE_DEVS, &mddev->flags);
		md_wakeup_thread(mddev->thread);
		wait_event(mddev->sb_wait, mddev->flags == 0 ||
4282 4283 4284 4285 4286
			   test_bit(MD_RECOVERY_INTR, &mddev->recovery));
		if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
			allow_barrier(conf);
			return sectors_done;
		}
4287 4288 4289 4290 4291 4292 4293
		conf->reshape_safe = mddev->reshape_position;
		allow_barrier(conf);
	}

read_more:
	/* Now schedule reads for blocks from sector_nr to last */
	r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
4294
	r10_bio->state = 0;
4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308
	raise_barrier(conf, sectors_done != 0);
	atomic_set(&r10_bio->remaining, 0);
	r10_bio->mddev = mddev;
	r10_bio->sector = sector_nr;
	set_bit(R10BIO_IsReshape, &r10_bio->state);
	r10_bio->sectors = last - sector_nr + 1;
	rdev = read_balance(conf, r10_bio, &max_sectors);
	BUG_ON(!test_bit(R10BIO_Previous, &r10_bio->state));

	if (!rdev) {
		/* Cannot read from here, so need to record bad blocks
		 * on all the target devices.
		 */
		// FIXME
4309
		mempool_free(r10_bio, conf->r10buf_pool);
4310 4311 4312 4313 4314 4315 4316
		set_bit(MD_RECOVERY_INTR, &mddev->recovery);
		return sectors_done;
	}

	read_bio = bio_alloc_mddev(GFP_KERNEL, RESYNC_PAGES, mddev);

	read_bio->bi_bdev = rdev->bdev;
4317
	read_bio->bi_iter.bi_sector = (r10_bio->devs[r10_bio->read_slot].addr
4318 4319 4320 4321
			       + rdev->data_offset);
	read_bio->bi_private = r10_bio;
	read_bio->bi_end_io = end_sync_read;
	read_bio->bi_rw = READ;
4322
	read_bio->bi_flags &= (~0UL << BIO_RESET_BITS);
4323
	read_bio->bi_error = 0;
4324
	read_bio->bi_vcnt = 0;
4325
	read_bio->bi_iter.bi_size = 0;
4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346 4347
	r10_bio->master_bio = read_bio;
	r10_bio->read_slot = r10_bio->devs[r10_bio->read_slot].devnum;

	/* Now find the locations in the new layout */
	__raid10_find_phys(&conf->geo, r10_bio);

	blist = read_bio;
	read_bio->bi_next = NULL;

	for (s = 0; s < conf->copies*2; s++) {
		struct bio *b;
		int d = r10_bio->devs[s/2].devnum;
		struct md_rdev *rdev2;
		if (s&1) {
			rdev2 = conf->mirrors[d].replacement;
			b = r10_bio->devs[s/2].repl_bio;
		} else {
			rdev2 = conf->mirrors[d].rdev;
			b = r10_bio->devs[s/2].bio;
		}
		if (!rdev2 || test_bit(Faulty, &rdev2->flags))
			continue;
K
Kent Overstreet 已提交
4348 4349

		bio_reset(b);
4350
		b->bi_bdev = rdev2->bdev;
4351 4352
		b->bi_iter.bi_sector = r10_bio->devs[s/2].addr +
			rdev2->new_data_offset;
4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378
		b->bi_private = r10_bio;
		b->bi_end_io = end_reshape_write;
		b->bi_rw = WRITE;
		b->bi_next = blist;
		blist = b;
	}

	/* Now add as many pages as possible to all of these bios. */

	nr_sectors = 0;
	for (s = 0 ; s < max_sectors; s += PAGE_SIZE >> 9) {
		struct page *page = r10_bio->devs[0].bio->bi_io_vec[s/(PAGE_SIZE>>9)].bv_page;
		int len = (max_sectors - s) << 9;
		if (len > PAGE_SIZE)
			len = PAGE_SIZE;
		for (bio = blist; bio ; bio = bio->bi_next) {
			struct bio *bio2;
			if (bio_add_page(bio, page, len, 0))
				continue;

			/* Didn't fit, must stop */
			for (bio2 = blist;
			     bio2 && bio2 != bio;
			     bio2 = bio2->bi_next) {
				/* Remove last page from this bio */
				bio2->bi_vcnt--;
4379
				bio2->bi_iter.bi_size -= len;
4380
				bio_clear_flag(bio2, BIO_SEG_VALID);
4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466
			}
			goto bio_full;
		}
		sector_nr += len >> 9;
		nr_sectors += len >> 9;
	}
bio_full:
	r10_bio->sectors = nr_sectors;

	/* Now submit the read */
	md_sync_acct(read_bio->bi_bdev, r10_bio->sectors);
	atomic_inc(&r10_bio->remaining);
	read_bio->bi_next = NULL;
	generic_make_request(read_bio);
	sector_nr += nr_sectors;
	sectors_done += nr_sectors;
	if (sector_nr <= last)
		goto read_more;

	/* Now that we have done the whole section we can
	 * update reshape_progress
	 */
	if (mddev->reshape_backwards)
		conf->reshape_progress -= sectors_done;
	else
		conf->reshape_progress += sectors_done;

	return sectors_done;
}

static void end_reshape_request(struct r10bio *r10_bio);
static int handle_reshape_read_error(struct mddev *mddev,
				     struct r10bio *r10_bio);
static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio)
{
	/* Reshape read completed.  Hopefully we have a block
	 * to write out.
	 * If we got a read error then we do sync 1-page reads from
	 * elsewhere until we find the data - or give up.
	 */
	struct r10conf *conf = mddev->private;
	int s;

	if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
		if (handle_reshape_read_error(mddev, r10_bio) < 0) {
			/* Reshape has been aborted */
			md_done_sync(mddev, r10_bio->sectors, 0);
			return;
		}

	/* We definitely have the data in the pages, schedule the
	 * writes.
	 */
	atomic_set(&r10_bio->remaining, 1);
	for (s = 0; s < conf->copies*2; s++) {
		struct bio *b;
		int d = r10_bio->devs[s/2].devnum;
		struct md_rdev *rdev;
		if (s&1) {
			rdev = conf->mirrors[d].replacement;
			b = r10_bio->devs[s/2].repl_bio;
		} else {
			rdev = conf->mirrors[d].rdev;
			b = r10_bio->devs[s/2].bio;
		}
		if (!rdev || test_bit(Faulty, &rdev->flags))
			continue;
		atomic_inc(&rdev->nr_pending);
		md_sync_acct(b->bi_bdev, r10_bio->sectors);
		atomic_inc(&r10_bio->remaining);
		b->bi_next = NULL;
		generic_make_request(b);
	}
	end_reshape_request(r10_bio);
}

static void end_reshape(struct r10conf *conf)
{
	if (test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery))
		return;

	spin_lock_irq(&conf->device_lock);
	conf->prev = conf->geo;
	md_finish_reshape(conf->mddev);
	smp_wmb();
	conf->reshape_progress = MaxSector;
4467
	conf->reshape_safe = MaxSector;
4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488
	spin_unlock_irq(&conf->device_lock);

	/* read-ahead size must cover two whole stripes, which is
	 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices
	 */
	if (conf->mddev->queue) {
		int stripe = conf->geo.raid_disks *
			((conf->mddev->chunk_sectors << 9) / PAGE_SIZE);
		stripe /= conf->geo.near_copies;
		if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
			conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
	}
	conf->fullsync = 0;
}

static int handle_reshape_read_error(struct mddev *mddev,
				     struct r10bio *r10_bio)
{
	/* Use sync reads to get the blocks from somewhere else */
	int sectors = r10_bio->sectors;
	struct r10conf *conf = mddev->private;
4489 4490 4491 4492 4493
	struct {
		struct r10bio r10_bio;
		struct r10dev devs[conf->copies];
	} on_stack;
	struct r10bio *r10b = &on_stack.r10_bio;
4494 4495 4496 4497
	int slot = 0;
	int idx = 0;
	struct bio_vec *bvec = r10_bio->master_bio->bi_io_vec;

4498 4499
	r10b->sector = r10_bio->sector;
	__raid10_find_phys(&conf->prev, r10b);
4500 4501 4502 4503 4504 4505 4506 4507 4508 4509

	while (sectors) {
		int s = sectors;
		int success = 0;
		int first_slot = slot;

		if (s > (PAGE_SIZE >> 9))
			s = PAGE_SIZE >> 9;

		while (!success) {
4510
			int d = r10b->devs[slot].devnum;
4511 4512 4513 4514 4515 4516 4517
			struct md_rdev *rdev = conf->mirrors[d].rdev;
			sector_t addr;
			if (rdev == NULL ||
			    test_bit(Faulty, &rdev->flags) ||
			    !test_bit(In_sync, &rdev->flags))
				goto failed;

4518
			addr = r10b->devs[slot].addr + idx * PAGE_SIZE;
4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544
			success = sync_page_io(rdev,
					       addr,
					       s << 9,
					       bvec[idx].bv_page,
					       READ, false);
			if (success)
				break;
		failed:
			slot++;
			if (slot >= conf->copies)
				slot = 0;
			if (slot == first_slot)
				break;
		}
		if (!success) {
			/* couldn't read this block, must give up */
			set_bit(MD_RECOVERY_INTR,
				&mddev->recovery);
			return -EIO;
		}
		sectors -= s;
		idx++;
	}
	return 0;
}

4545
static void end_reshape_write(struct bio *bio)
4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559 4560 4561 4562
{
	struct r10bio *r10_bio = bio->bi_private;
	struct mddev *mddev = r10_bio->mddev;
	struct r10conf *conf = mddev->private;
	int d;
	int slot;
	int repl;
	struct md_rdev *rdev = NULL;

	d = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
	if (repl)
		rdev = conf->mirrors[d].replacement;
	if (!rdev) {
		smp_mb();
		rdev = conf->mirrors[d].rdev;
	}

4563
	if (bio->bi_error) {
4564 4565 4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597
		/* FIXME should record badblock */
		md_error(mddev, rdev);
	}

	rdev_dec_pending(rdev, mddev);
	end_reshape_request(r10_bio);
}

static void end_reshape_request(struct r10bio *r10_bio)
{
	if (!atomic_dec_and_test(&r10_bio->remaining))
		return;
	md_done_sync(r10_bio->mddev, r10_bio->sectors, 1);
	bio_put(r10_bio->master_bio);
	put_buf(r10_bio);
}

static void raid10_finish_reshape(struct mddev *mddev)
{
	struct r10conf *conf = mddev->private;

	if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
		return;

	if (mddev->delta_disks > 0) {
		sector_t size = raid10_size(mddev, 0, 0);
		md_set_array_sectors(mddev, size);
		if (mddev->recovery_cp > mddev->resync_max_sectors) {
			mddev->recovery_cp = mddev->resync_max_sectors;
			set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
		}
		mddev->resync_max_sectors = size;
		set_capacity(mddev->gendisk, mddev->array_sectors);
		revalidate_disk(mddev->gendisk);
4598 4599 4600 4601 4602 4603 4604 4605 4606 4607 4608 4609
	} else {
		int d;
		for (d = conf->geo.raid_disks ;
		     d < conf->geo.raid_disks - mddev->delta_disks;
		     d++) {
			struct md_rdev *rdev = conf->mirrors[d].rdev;
			if (rdev)
				clear_bit(In_sync, &rdev->flags);
			rdev = conf->mirrors[d].replacement;
			if (rdev)
				clear_bit(In_sync, &rdev->flags);
		}
4610 4611 4612 4613 4614 4615 4616 4617
	}
	mddev->layout = mddev->new_layout;
	mddev->chunk_sectors = 1 << conf->geo.chunk_shift;
	mddev->reshape_position = MaxSector;
	mddev->delta_disks = 0;
	mddev->reshape_backwards = 0;
}

4618
static struct md_personality raid10_personality =
L
Linus Torvalds 已提交
4619 4620
{
	.name		= "raid10",
4621
	.level		= 10,
L
Linus Torvalds 已提交
4622
	.owner		= THIS_MODULE,
S
Shaohua Li 已提交
4623 4624
	.make_request	= raid10_make_request,
	.run		= raid10_run,
N
NeilBrown 已提交
4625
	.free		= raid10_free,
S
Shaohua Li 已提交
4626 4627
	.status		= raid10_status,
	.error_handler	= raid10_error,
L
Linus Torvalds 已提交
4628 4629 4630
	.hot_add_disk	= raid10_add_disk,
	.hot_remove_disk= raid10_remove_disk,
	.spare_active	= raid10_spare_active,
S
Shaohua Li 已提交
4631
	.sync_request	= raid10_sync_request,
4632
	.quiesce	= raid10_quiesce,
4633
	.size		= raid10_size,
4634
	.resize		= raid10_resize,
4635
	.takeover	= raid10_takeover,
4636 4637 4638
	.check_reshape	= raid10_check_reshape,
	.start_reshape	= raid10_start_reshape,
	.finish_reshape	= raid10_finish_reshape,
4639
	.congested	= raid10_congested,
L
Linus Torvalds 已提交
4640 4641 4642 4643
};

static int __init raid_init(void)
{
4644
	return register_md_personality(&raid10_personality);
L
Linus Torvalds 已提交
4645 4646 4647 4648
}

static void raid_exit(void)
{
4649
	unregister_md_personality(&raid10_personality);
L
Linus Torvalds 已提交
4650 4651 4652 4653 4654
}

module_init(raid_init);
module_exit(raid_exit);
MODULE_LICENSE("GPL");
4655
MODULE_DESCRIPTION("RAID10 (striped mirror) personality for MD");
L
Linus Torvalds 已提交
4656
MODULE_ALIAS("md-personality-9"); /* RAID10 */
4657
MODULE_ALIAS("md-raid10");
4658
MODULE_ALIAS("md-level-10");
4659 4660

module_param(max_queued_requests, int, S_IRUGO|S_IWUSR);
新手
引导
客服 返回
顶部