raid10.c 123.9 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7
/*
 * raid10.c : Multiple Devices driver for Linux
 *
 * Copyright (C) 2000-2004 Neil Brown
 *
 * RAID-10 support for md.
 *
L
Lucas De Marchi 已提交
8
 * Base on code in raid1.c.  See raid1.c for further copyright information.
L
Linus Torvalds 已提交
9 10 11 12 13 14 15 16 17 18 19 20
 *
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2, or (at your option)
 * any later version.
 *
 * You should have received a copy of the GNU General Public License
 * (for example /usr/src/linux/COPYING); if not, write to the Free
 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

21
#include <linux/slab.h>
22
#include <linux/delay.h>
23
#include <linux/blkdev.h>
24
#include <linux/module.h>
25
#include <linux/seq_file.h>
26
#include <linux/ratelimit.h>
N
NeilBrown 已提交
27
#include <linux/kthread.h>
28
#include "md.h"
29
#include "raid10.h"
30
#include "raid0.h"
31
#include "bitmap.h"
L
Linus Torvalds 已提交
32 33 34 35 36 37 38 39

/*
 * RAID10 provides a combination of RAID0 and RAID1 functionality.
 * The layout of data is defined by
 *    chunk_size
 *    raid_disks
 *    near_copies (stored in low byte of layout)
 *    far_copies (stored in second byte of layout)
40
 *    far_offset (stored in bit 16 of layout )
L
Linus Torvalds 已提交
41 42 43 44 45 46 47
 *
 * The data to be stored is divided into chunks using chunksize.
 * Each device is divided into far_copies sections.
 * In each section, chunks are laid out in a style similar to raid0, but
 * near_copies copies of each chunk is stored (each on a different drive).
 * The starting device for each section is offset near_copies from the starting
 * device of the previous section.
48
 * Thus they are (near_copies*far_copies) of each chunk, and each is on a different
L
Linus Torvalds 已提交
49 50 51
 * drive.
 * near_copies and far_copies must be at least one, and their product is at most
 * raid_disks.
52 53 54 55
 *
 * If far_offset is true, then the far_copies are handled a bit differently.
 * The copies are still in different stripes, but instead of be very far apart
 * on disk, there are adjacent stripes.
L
Linus Torvalds 已提交
56 57 58 59 60 61 62
 */

/*
 * Number of guaranteed r10bios in case of extreme VM load:
 */
#define	NR_RAID10_BIOS 256

63 64 65 66 67 68
/* When there are this many requests queue to be written by
 * the raid10 thread, we become 'congested' to provide back-pressure
 * for writeback.
 */
static int max_queued_requests = 1024;

69 70
static void allow_barrier(struct r10conf *conf);
static void lower_barrier(struct r10conf *conf);
71
static int enough(struct r10conf *conf, int ignore);
N
NeilBrown 已提交
72 73 74 75 76
static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
				int *skipped);
static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio);
static void end_reshape_write(struct bio *bio, int error);
static void end_reshape(struct r10conf *conf);
77

A
Al Viro 已提交
78
static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data)
L
Linus Torvalds 已提交
79
{
80
	struct r10conf *conf = data;
81
	int size = offsetof(struct r10bio, devs[conf->copies]);
L
Linus Torvalds 已提交
82

83 84
	/* allocate a r10bio with room for raid_disks entries in the
	 * bios array */
J
Jens Axboe 已提交
85
	return kzalloc(size, gfp_flags);
L
Linus Torvalds 已提交
86 87 88 89 90 91 92
}

static void r10bio_pool_free(void *r10_bio, void *data)
{
	kfree(r10_bio);
}

93
/* Maximum size of each resync request */
L
Linus Torvalds 已提交
94 95
#define RESYNC_BLOCK_SIZE (64*1024)
#define RESYNC_PAGES ((RESYNC_BLOCK_SIZE + PAGE_SIZE-1) / PAGE_SIZE)
96 97 98 99
/* amount of memory to reserve for resync requests */
#define RESYNC_WINDOW (1024*1024)
/* maximum number of concurrent requests, memory permitting */
#define RESYNC_DEPTH (32*1024*1024/RESYNC_BLOCK_SIZE)
L
Linus Torvalds 已提交
100 101 102 103 104 105 106 107

/*
 * When performing a resync, we need to read and compare, so
 * we need as many pages are there are copies.
 * When performing a recovery, we need 2 bios, one for read,
 * one for write (we recover only one drive per r10buf)
 *
 */
A
Al Viro 已提交
108
static void * r10buf_pool_alloc(gfp_t gfp_flags, void *data)
L
Linus Torvalds 已提交
109
{
110
	struct r10conf *conf = data;
L
Linus Torvalds 已提交
111
	struct page *page;
112
	struct r10bio *r10_bio;
L
Linus Torvalds 已提交
113 114 115 116 117
	struct bio *bio;
	int i, j;
	int nalloc;

	r10_bio = r10bio_pool_alloc(gfp_flags, conf);
J
Jens Axboe 已提交
118
	if (!r10_bio)
L
Linus Torvalds 已提交
119 120
		return NULL;

N
NeilBrown 已提交
121 122
	if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery) ||
	    test_bit(MD_RECOVERY_RESHAPE, &conf->mddev->recovery))
L
Linus Torvalds 已提交
123 124 125 126 127 128 129 130
		nalloc = conf->copies; /* resync */
	else
		nalloc = 2; /* recovery */

	/*
	 * Allocate bios.
	 */
	for (j = nalloc ; j-- ; ) {
131
		bio = bio_kmalloc(gfp_flags, RESYNC_PAGES);
L
Linus Torvalds 已提交
132 133 134
		if (!bio)
			goto out_free_bio;
		r10_bio->devs[j].bio = bio;
135 136 137 138 139 140
		if (!conf->have_replacement)
			continue;
		bio = bio_kmalloc(gfp_flags, RESYNC_PAGES);
		if (!bio)
			goto out_free_bio;
		r10_bio->devs[j].repl_bio = bio;
L
Linus Torvalds 已提交
141 142 143 144 145 146
	}
	/*
	 * Allocate RESYNC_PAGES data pages and attach them
	 * where needed.
	 */
	for (j = 0 ; j < nalloc; j++) {
147
		struct bio *rbio = r10_bio->devs[j].repl_bio;
L
Linus Torvalds 已提交
148 149
		bio = r10_bio->devs[j].bio;
		for (i = 0; i < RESYNC_PAGES; i++) {
N
NeilBrown 已提交
150 151 152 153
			if (j > 0 && !test_bit(MD_RECOVERY_SYNC,
					       &conf->mddev->recovery)) {
				/* we can share bv_page's during recovery
				 * and reshape */
154 155 156 157 158
				struct bio *rbio = r10_bio->devs[0].bio;
				page = rbio->bi_io_vec[i].bv_page;
				get_page(page);
			} else
				page = alloc_page(gfp_flags);
L
Linus Torvalds 已提交
159 160 161 162
			if (unlikely(!page))
				goto out_free_pages;

			bio->bi_io_vec[i].bv_page = page;
163 164
			if (rbio)
				rbio->bi_io_vec[i].bv_page = page;
L
Linus Torvalds 已提交
165 166 167 168 169 170 171
		}
	}

	return r10_bio;

out_free_pages:
	for ( ; i > 0 ; i--)
172
		safe_put_page(bio->bi_io_vec[i-1].bv_page);
L
Linus Torvalds 已提交
173 174
	while (j--)
		for (i = 0; i < RESYNC_PAGES ; i++)
175
			safe_put_page(r10_bio->devs[j].bio->bi_io_vec[i].bv_page);
176
	j = 0;
L
Linus Torvalds 已提交
177
out_free_bio:
178 179 180
	for ( ; j < nalloc; j++) {
		if (r10_bio->devs[j].bio)
			bio_put(r10_bio->devs[j].bio);
181 182 183
		if (r10_bio->devs[j].repl_bio)
			bio_put(r10_bio->devs[j].repl_bio);
	}
L
Linus Torvalds 已提交
184 185 186 187 188 189 190
	r10bio_pool_free(r10_bio, conf);
	return NULL;
}

static void r10buf_pool_free(void *__r10_bio, void *data)
{
	int i;
191
	struct r10conf *conf = data;
192
	struct r10bio *r10bio = __r10_bio;
L
Linus Torvalds 已提交
193 194 195 196 197 198
	int j;

	for (j=0; j < conf->copies; j++) {
		struct bio *bio = r10bio->devs[j].bio;
		if (bio) {
			for (i = 0; i < RESYNC_PAGES; i++) {
199
				safe_put_page(bio->bi_io_vec[i].bv_page);
L
Linus Torvalds 已提交
200 201 202 203
				bio->bi_io_vec[i].bv_page = NULL;
			}
			bio_put(bio);
		}
204 205 206
		bio = r10bio->devs[j].repl_bio;
		if (bio)
			bio_put(bio);
L
Linus Torvalds 已提交
207 208 209 210
	}
	r10bio_pool_free(r10bio, conf);
}

211
static void put_all_bios(struct r10conf *conf, struct r10bio *r10_bio)
L
Linus Torvalds 已提交
212 213 214 215 216
{
	int i;

	for (i = 0; i < conf->copies; i++) {
		struct bio **bio = & r10_bio->devs[i].bio;
217
		if (!BIO_SPECIAL(*bio))
L
Linus Torvalds 已提交
218 219
			bio_put(*bio);
		*bio = NULL;
220 221 222 223
		bio = &r10_bio->devs[i].repl_bio;
		if (r10_bio->read_slot < 0 && !BIO_SPECIAL(*bio))
			bio_put(*bio);
		*bio = NULL;
L
Linus Torvalds 已提交
224 225 226
	}
}

227
static void free_r10bio(struct r10bio *r10_bio)
L
Linus Torvalds 已提交
228
{
229
	struct r10conf *conf = r10_bio->mddev->private;
L
Linus Torvalds 已提交
230 231 232 233 234

	put_all_bios(conf, r10_bio);
	mempool_free(r10_bio, conf->r10bio_pool);
}

235
static void put_buf(struct r10bio *r10_bio)
L
Linus Torvalds 已提交
236
{
237
	struct r10conf *conf = r10_bio->mddev->private;
L
Linus Torvalds 已提交
238 239 240

	mempool_free(r10_bio, conf->r10buf_pool);

241
	lower_barrier(conf);
L
Linus Torvalds 已提交
242 243
}

244
static void reschedule_retry(struct r10bio *r10_bio)
L
Linus Torvalds 已提交
245 246
{
	unsigned long flags;
247
	struct mddev *mddev = r10_bio->mddev;
248
	struct r10conf *conf = mddev->private;
L
Linus Torvalds 已提交
249 250 251

	spin_lock_irqsave(&conf->device_lock, flags);
	list_add(&r10_bio->retry_list, &conf->retry_list);
252
	conf->nr_queued ++;
L
Linus Torvalds 已提交
253 254
	spin_unlock_irqrestore(&conf->device_lock, flags);

A
Arthur Jones 已提交
255 256 257
	/* wake up frozen array... */
	wake_up(&conf->wait_barrier);

L
Linus Torvalds 已提交
258 259 260 261 262 263 264 265
	md_wakeup_thread(mddev->thread);
}

/*
 * raid_end_bio_io() is called when we have finished servicing a mirrored
 * operation and are ready to return a success/failure code to the buffer
 * cache layer.
 */
266
static void raid_end_bio_io(struct r10bio *r10_bio)
L
Linus Torvalds 已提交
267 268
{
	struct bio *bio = r10_bio->master_bio;
269
	int done;
270
	struct r10conf *conf = r10_bio->mddev->private;
L
Linus Torvalds 已提交
271

272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289
	if (bio->bi_phys_segments) {
		unsigned long flags;
		spin_lock_irqsave(&conf->device_lock, flags);
		bio->bi_phys_segments--;
		done = (bio->bi_phys_segments == 0);
		spin_unlock_irqrestore(&conf->device_lock, flags);
	} else
		done = 1;
	if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
		clear_bit(BIO_UPTODATE, &bio->bi_flags);
	if (done) {
		bio_endio(bio, 0);
		/*
		 * Wake up any possible resync thread that waits for the device
		 * to go idle.
		 */
		allow_barrier(conf);
	}
L
Linus Torvalds 已提交
290 291 292 293 294 295
	free_r10bio(r10_bio);
}

/*
 * Update disk head position estimator based on IRQ completion info.
 */
296
static inline void update_head_pos(int slot, struct r10bio *r10_bio)
L
Linus Torvalds 已提交
297
{
298
	struct r10conf *conf = r10_bio->mddev->private;
L
Linus Torvalds 已提交
299 300 301 302 303

	conf->mirrors[r10_bio->devs[slot].devnum].head_position =
		r10_bio->devs[slot].addr + (r10_bio->sectors);
}

304 305 306
/*
 * Find the disk number which triggered given bio
 */
307
static int find_bio_disk(struct r10conf *conf, struct r10bio *r10_bio,
308
			 struct bio *bio, int *slotp, int *replp)
309 310
{
	int slot;
311
	int repl = 0;
312

313
	for (slot = 0; slot < conf->copies; slot++) {
314 315
		if (r10_bio->devs[slot].bio == bio)
			break;
316 317 318 319 320
		if (r10_bio->devs[slot].repl_bio == bio) {
			repl = 1;
			break;
		}
	}
321 322 323 324

	BUG_ON(slot == conf->copies);
	update_head_pos(slot, r10_bio);

325 326
	if (slotp)
		*slotp = slot;
327 328
	if (replp)
		*replp = repl;
329 330 331
	return r10_bio->devs[slot].devnum;
}

332
static void raid10_end_read_request(struct bio *bio, int error)
L
Linus Torvalds 已提交
333 334
{
	int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
335
	struct r10bio *r10_bio = bio->bi_private;
L
Linus Torvalds 已提交
336
	int slot, dev;
337
	struct md_rdev *rdev;
338
	struct r10conf *conf = r10_bio->mddev->private;
L
Linus Torvalds 已提交
339 340 341 342


	slot = r10_bio->read_slot;
	dev = r10_bio->devs[slot].devnum;
343
	rdev = r10_bio->devs[slot].rdev;
L
Linus Torvalds 已提交
344 345 346
	/*
	 * this branch is our 'one mirror IO has finished' event handler:
	 */
347 348 349
	update_head_pos(slot, r10_bio);

	if (uptodate) {
L
Linus Torvalds 已提交
350 351 352 353 354 355 356 357 358 359
		/*
		 * Set R10BIO_Uptodate in our master bio, so that
		 * we will return a good error code to the higher
		 * levels even if IO on some other mirrored buffer fails.
		 *
		 * The 'master' represents the composite IO operation to
		 * user-side. So if something waits for IO, then it will
		 * wait for the 'master' bio.
		 */
		set_bit(R10BIO_Uptodate, &r10_bio->state);
360 361 362 363 364 365 366 367 368 369 370 371 372
	} else {
		/* If all other devices that store this block have
		 * failed, we want to return the error upwards rather
		 * than fail the last device.  Here we redefine
		 * "uptodate" to mean "Don't want to retry"
		 */
		unsigned long flags;
		spin_lock_irqsave(&conf->device_lock, flags);
		if (!enough(conf, rdev->raid_disk))
			uptodate = 1;
		spin_unlock_irqrestore(&conf->device_lock, flags);
	}
	if (uptodate) {
L
Linus Torvalds 已提交
373
		raid_end_bio_io(r10_bio);
374
		rdev_dec_pending(rdev, conf->mddev);
375
	} else {
L
Linus Torvalds 已提交
376
		/*
377
		 * oops, read error - keep the refcount on the rdev
L
Linus Torvalds 已提交
378 379
		 */
		char b[BDEVNAME_SIZE];
380 381 382
		printk_ratelimited(KERN_ERR
				   "md/raid10:%s: %s: rescheduling sector %llu\n",
				   mdname(conf->mddev),
383
				   bdevname(rdev->bdev, b),
384
				   (unsigned long long)r10_bio->sector);
385
		set_bit(R10BIO_ReadError, &r10_bio->state);
L
Linus Torvalds 已提交
386 387 388 389
		reschedule_retry(r10_bio);
	}
}

390
static void close_write(struct r10bio *r10_bio)
391 392 393 394 395 396 397 398 399
{
	/* clear the bitmap if all writes complete successfully */
	bitmap_endwrite(r10_bio->mddev->bitmap, r10_bio->sector,
			r10_bio->sectors,
			!test_bit(R10BIO_Degraded, &r10_bio->state),
			0);
	md_write_end(r10_bio->mddev);
}

400
static void one_write_done(struct r10bio *r10_bio)
401 402 403 404 405 406 407 408 409 410 411 412 413 414
{
	if (atomic_dec_and_test(&r10_bio->remaining)) {
		if (test_bit(R10BIO_WriteError, &r10_bio->state))
			reschedule_retry(r10_bio);
		else {
			close_write(r10_bio);
			if (test_bit(R10BIO_MadeGood, &r10_bio->state))
				reschedule_retry(r10_bio);
			else
				raid_end_bio_io(r10_bio);
		}
	}
}

415
static void raid10_end_write_request(struct bio *bio, int error)
L
Linus Torvalds 已提交
416 417
{
	int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
418
	struct r10bio *r10_bio = bio->bi_private;
419
	int dev;
420
	int dec_rdev = 1;
421
	struct r10conf *conf = r10_bio->mddev->private;
422
	int slot, repl;
423
	struct md_rdev *rdev = NULL;
L
Linus Torvalds 已提交
424

425
	dev = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
L
Linus Torvalds 已提交
426

427 428
	if (repl)
		rdev = conf->mirrors[dev].replacement;
429 430 431
	if (!rdev) {
		smp_rmb();
		repl = 0;
432
		rdev = conf->mirrors[dev].rdev;
433
	}
L
Linus Torvalds 已提交
434 435 436
	/*
	 * this branch is our 'one mirror IO has finished' event handler:
	 */
437
	if (!uptodate) {
438 439 440 441 442 443 444
		if (repl)
			/* Never record new bad blocks to replacement,
			 * just fail it.
			 */
			md_error(rdev->mddev, rdev);
		else {
			set_bit(WriteErrorSeen,	&rdev->flags);
445 446 447
			if (!test_and_set_bit(WantReplacement, &rdev->flags))
				set_bit(MD_RECOVERY_NEEDED,
					&rdev->mddev->recovery);
448 449 450
			set_bit(R10BIO_WriteError, &r10_bio->state);
			dec_rdev = 0;
		}
451
	} else {
L
Linus Torvalds 已提交
452 453 454 455 456 457 458 459 460
		/*
		 * Set R10BIO_Uptodate in our master bio, so that
		 * we will return a good error code for to the higher
		 * levels even if IO on some other mirrored buffer fails.
		 *
		 * The 'master' represents the composite IO operation to
		 * user-side. So if something waits for IO, then it will
		 * wait for the 'master' bio.
		 */
461 462 463
		sector_t first_bad;
		int bad_sectors;

L
Linus Torvalds 已提交
464 465
		set_bit(R10BIO_Uptodate, &r10_bio->state);

466
		/* Maybe we can clear some bad blocks. */
467
		if (is_badblock(rdev,
468 469 470 471
				r10_bio->devs[slot].addr,
				r10_bio->sectors,
				&first_bad, &bad_sectors)) {
			bio_put(bio);
472 473 474 475
			if (repl)
				r10_bio->devs[slot].repl_bio = IO_MADE_GOOD;
			else
				r10_bio->devs[slot].bio = IO_MADE_GOOD;
476 477 478 479 480
			dec_rdev = 0;
			set_bit(R10BIO_MadeGood, &r10_bio->state);
		}
	}

L
Linus Torvalds 已提交
481 482 483 484 485
	/*
	 *
	 * Let's see if all mirrored write operations have finished
	 * already.
	 */
486
	one_write_done(r10_bio);
487 488
	if (dec_rdev)
		rdev_dec_pending(conf->mirrors[dev].rdev, conf->mddev);
L
Linus Torvalds 已提交
489 490 491 492
}

/*
 * RAID10 layout manager
L
Lucas De Marchi 已提交
493
 * As well as the chunksize and raid_disks count, there are two
L
Linus Torvalds 已提交
494 495 496 497 498 499
 * parameters: near_copies and far_copies.
 * near_copies * far_copies must be <= raid_disks.
 * Normally one of these will be 1.
 * If both are 1, we get raid0.
 * If near_copies == raid_disks, we get raid1.
 *
L
Lucas De Marchi 已提交
500
 * Chunks are laid out in raid0 style with near_copies copies of the
L
Linus Torvalds 已提交
501 502 503 504 505 506 507 508 509
 * first chunk, followed by near_copies copies of the next chunk and
 * so on.
 * If far_copies > 1, then after 1/far_copies of the array has been assigned
 * as described above, we start again with a device offset of near_copies.
 * So we effectively have another copy of the whole array further down all
 * the drives, but with blocks on different drives.
 * With this layout, and block is never stored twice on the one device.
 *
 * raid10_find_phys finds the sector offset of a given virtual sector
510
 * on each device that it is on.
L
Linus Torvalds 已提交
511 512 513 514 515
 *
 * raid10_find_virt does the reverse mapping, from a device and a
 * sector offset to a virtual address
 */

516
static void __raid10_find_phys(struct geom *geo, struct r10bio *r10bio)
L
Linus Torvalds 已提交
517 518 519 520 521 522 523 524 525
{
	int n,f;
	sector_t sector;
	sector_t chunk;
	sector_t stripe;
	int dev;
	int slot = 0;

	/* now calculate first sector/dev */
526 527
	chunk = r10bio->sector >> geo->chunk_shift;
	sector = r10bio->sector & geo->chunk_mask;
L
Linus Torvalds 已提交
528

529
	chunk *= geo->near_copies;
L
Linus Torvalds 已提交
530
	stripe = chunk;
531 532 533
	dev = sector_div(stripe, geo->raid_disks);
	if (geo->far_offset)
		stripe *= geo->far_copies;
L
Linus Torvalds 已提交
534

535
	sector += stripe << geo->chunk_shift;
L
Linus Torvalds 已提交
536 537

	/* and calculate all the others */
538
	for (n = 0; n < geo->near_copies; n++) {
L
Linus Torvalds 已提交
539 540 541 542 543 544
		int d = dev;
		sector_t s = sector;
		r10bio->devs[slot].addr = sector;
		r10bio->devs[slot].devnum = d;
		slot++;

545 546 547 548 549
		for (f = 1; f < geo->far_copies; f++) {
			d += geo->near_copies;
			if (d >= geo->raid_disks)
				d -= geo->raid_disks;
			s += geo->stride;
L
Linus Torvalds 已提交
550 551 552 553 554
			r10bio->devs[slot].devnum = d;
			r10bio->devs[slot].addr = s;
			slot++;
		}
		dev++;
555
		if (dev >= geo->raid_disks) {
L
Linus Torvalds 已提交
556
			dev = 0;
557
			sector += (geo->chunk_mask + 1);
L
Linus Torvalds 已提交
558 559
		}
	}
560 561 562 563 564 565 566 567 568 569 570 571 572 573 574
}

static void raid10_find_phys(struct r10conf *conf, struct r10bio *r10bio)
{
	struct geom *geo = &conf->geo;

	if (conf->reshape_progress != MaxSector &&
	    ((r10bio->sector >= conf->reshape_progress) !=
	     conf->mddev->reshape_backwards)) {
		set_bit(R10BIO_Previous, &r10bio->state);
		geo = &conf->prev;
	} else
		clear_bit(R10BIO_Previous, &r10bio->state);

	__raid10_find_phys(geo, r10bio);
L
Linus Torvalds 已提交
575 576
}

577
static sector_t raid10_find_virt(struct r10conf *conf, sector_t sector, int dev)
L
Linus Torvalds 已提交
578 579
{
	sector_t offset, chunk, vchunk;
580 581 582
	/* Never use conf->prev as this is only called during resync
	 * or recovery, so reshape isn't happening
	 */
583
	struct geom *geo = &conf->geo;
L
Linus Torvalds 已提交
584

585 586
	offset = sector & geo->chunk_mask;
	if (geo->far_offset) {
587
		int fc;
588 589 590
		chunk = sector >> geo->chunk_shift;
		fc = sector_div(chunk, geo->far_copies);
		dev -= fc * geo->near_copies;
591
		if (dev < 0)
592
			dev += geo->raid_disks;
593
	} else {
594 595 596 597
		while (sector >= geo->stride) {
			sector -= geo->stride;
			if (dev < geo->near_copies)
				dev += geo->raid_disks - geo->near_copies;
598
			else
599
				dev -= geo->near_copies;
600
		}
601
		chunk = sector >> geo->chunk_shift;
602
	}
603 604 605
	vchunk = chunk * geo->raid_disks + dev;
	sector_div(vchunk, geo->near_copies);
	return (vchunk << geo->chunk_shift) + offset;
L
Linus Torvalds 已提交
606 607 608 609 610
}

/**
 *	raid10_mergeable_bvec -- tell bio layer if a two requests can be merged
 *	@q: request queue
611
 *	@bvm: properties of new bio
L
Linus Torvalds 已提交
612 613 614
 *	@biovec: the request that could be merged to it.
 *
 *	Return amount of bytes we can accept at this offset
615 616
 *	This requires checking for end-of-chunk if near_copies != raid_disks,
 *	and for subordinate merge_bvec_fns if merge_check_needed.
L
Linus Torvalds 已提交
617
 */
618 619 620
static int raid10_mergeable_bvec(struct request_queue *q,
				 struct bvec_merge_data *bvm,
				 struct bio_vec *biovec)
L
Linus Torvalds 已提交
621
{
622
	struct mddev *mddev = q->queuedata;
623
	struct r10conf *conf = mddev->private;
624
	sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev);
L
Linus Torvalds 已提交
625
	int max;
N
NeilBrown 已提交
626
	unsigned int chunk_sectors;
627
	unsigned int bio_sectors = bvm->bi_size >> 9;
628
	struct geom *geo = &conf->geo;
L
Linus Torvalds 已提交
629

N
NeilBrown 已提交
630
	chunk_sectors = (conf->geo.chunk_mask & conf->prev.chunk_mask) + 1;
631 632 633 634 635
	if (conf->reshape_progress != MaxSector &&
	    ((sector >= conf->reshape_progress) !=
	     conf->mddev->reshape_backwards))
		geo = &conf->prev;

636
	if (geo->near_copies < geo->raid_disks) {
637 638 639 640 641 642 643 644 645 646 647 648 649
		max = (chunk_sectors - ((sector & (chunk_sectors - 1))
					+ bio_sectors)) << 9;
		if (max < 0)
			/* bio_add cannot handle a negative return */
			max = 0;
		if (max <= biovec->bv_len && bio_sectors == 0)
			return biovec->bv_len;
	} else
		max = biovec->bv_len;

	if (mddev->merge_check_needed) {
		struct r10bio r10_bio;
		int s;
650 651 652 653 654 655
		if (conf->reshape_progress != MaxSector) {
			/* Cannot give any guidance during reshape */
			if (max <= biovec->bv_len && bio_sectors == 0)
				return biovec->bv_len;
			return 0;
		}
656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689
		r10_bio.sector = sector;
		raid10_find_phys(conf, &r10_bio);
		rcu_read_lock();
		for (s = 0; s < conf->copies; s++) {
			int disk = r10_bio.devs[s].devnum;
			struct md_rdev *rdev = rcu_dereference(
				conf->mirrors[disk].rdev);
			if (rdev && !test_bit(Faulty, &rdev->flags)) {
				struct request_queue *q =
					bdev_get_queue(rdev->bdev);
				if (q->merge_bvec_fn) {
					bvm->bi_sector = r10_bio.devs[s].addr
						+ rdev->data_offset;
					bvm->bi_bdev = rdev->bdev;
					max = min(max, q->merge_bvec_fn(
							  q, bvm, biovec));
				}
			}
			rdev = rcu_dereference(conf->mirrors[disk].replacement);
			if (rdev && !test_bit(Faulty, &rdev->flags)) {
				struct request_queue *q =
					bdev_get_queue(rdev->bdev);
				if (q->merge_bvec_fn) {
					bvm->bi_sector = r10_bio.devs[s].addr
						+ rdev->data_offset;
					bvm->bi_bdev = rdev->bdev;
					max = min(max, q->merge_bvec_fn(
							  q, bvm, biovec));
				}
			}
		}
		rcu_read_unlock();
	}
	return max;
L
Linus Torvalds 已提交
690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710
}

/*
 * This routine returns the disk from which the requested read should
 * be done. There is a per-array 'next expected sequential IO' sector
 * number - if this matches on the next IO then we use the last disk.
 * There is also a per-disk 'last know head position' sector that is
 * maintained from IRQ contexts, both the normal and the resync IO
 * completion handlers update this position correctly. If there is no
 * perfect sequential match then we pick the disk whose head is closest.
 *
 * If there are 2 mirrors in the same 2 devices, performance degrades
 * because position is mirror, not device based.
 *
 * The rdev for the device selected will have nr_pending incremented.
 */

/*
 * FIXME: possibly should rethink readbalancing and do it differently
 * depending on near_copies / far_copies geometry.
 */
711 712 713
static struct md_rdev *read_balance(struct r10conf *conf,
				    struct r10bio *r10_bio,
				    int *max_sectors)
L
Linus Torvalds 已提交
714
{
715
	const sector_t this_sector = r10_bio->sector;
N
NeilBrown 已提交
716
	int disk, slot;
717 718
	int sectors = r10_bio->sectors;
	int best_good_sectors;
N
NeilBrown 已提交
719
	sector_t new_distance, best_dist;
720
	struct md_rdev *rdev, *best_rdev;
N
NeilBrown 已提交
721 722
	int do_balance;
	int best_slot;
723
	struct geom *geo = &conf->geo;
L
Linus Torvalds 已提交
724 725 726

	raid10_find_phys(conf, r10_bio);
	rcu_read_lock();
N
NeilBrown 已提交
727
retry:
728
	sectors = r10_bio->sectors;
N
NeilBrown 已提交
729
	best_slot = -1;
730
	best_rdev = NULL;
N
NeilBrown 已提交
731
	best_dist = MaxSector;
732
	best_good_sectors = 0;
N
NeilBrown 已提交
733
	do_balance = 1;
L
Linus Torvalds 已提交
734 735
	/*
	 * Check if we can balance. We can balance on the whole
736 737 738
	 * device if no resync is going on (recovery is ok), or below
	 * the resync window. We take the first readable disk when
	 * above the resync window.
L
Linus Torvalds 已提交
739 740
	 */
	if (conf->mddev->recovery_cp < MaxSector
N
NeilBrown 已提交
741 742
	    && (this_sector + sectors >= conf->next_resync))
		do_balance = 0;
L
Linus Torvalds 已提交
743

N
NeilBrown 已提交
744
	for (slot = 0; slot < conf->copies ; slot++) {
745 746 747 748
		sector_t first_bad;
		int bad_sectors;
		sector_t dev_sector;

N
NeilBrown 已提交
749 750
		if (r10_bio->devs[slot].bio == IO_BLOCKED)
			continue;
L
Linus Torvalds 已提交
751
		disk = r10_bio->devs[slot].devnum;
752 753
		rdev = rcu_dereference(conf->mirrors[disk].replacement);
		if (rdev == NULL || test_bit(Faulty, &rdev->flags) ||
754
		    test_bit(Unmerged, &rdev->flags) ||
755 756
		    r10_bio->devs[slot].addr + sectors > rdev->recovery_offset)
			rdev = rcu_dereference(conf->mirrors[disk].rdev);
757 758 759
		if (rdev == NULL ||
		    test_bit(Faulty, &rdev->flags) ||
		    test_bit(Unmerged, &rdev->flags))
760 761 762
			continue;
		if (!test_bit(In_sync, &rdev->flags) &&
		    r10_bio->devs[slot].addr + sectors > rdev->recovery_offset)
N
NeilBrown 已提交
763 764
			continue;

765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786
		dev_sector = r10_bio->devs[slot].addr;
		if (is_badblock(rdev, dev_sector, sectors,
				&first_bad, &bad_sectors)) {
			if (best_dist < MaxSector)
				/* Already have a better slot */
				continue;
			if (first_bad <= dev_sector) {
				/* Cannot read here.  If this is the
				 * 'primary' device, then we must not read
				 * beyond 'bad_sectors' from another device.
				 */
				bad_sectors -= (dev_sector - first_bad);
				if (!do_balance && sectors > bad_sectors)
					sectors = bad_sectors;
				if (best_good_sectors > sectors)
					best_good_sectors = sectors;
			} else {
				sector_t good_sectors =
					first_bad - dev_sector;
				if (good_sectors > best_good_sectors) {
					best_good_sectors = good_sectors;
					best_slot = slot;
787
					best_rdev = rdev;
788 789 790 791 792 793 794 795 796
				}
				if (!do_balance)
					/* Must read from here */
					break;
			}
			continue;
		} else
			best_good_sectors = sectors;

N
NeilBrown 已提交
797 798
		if (!do_balance)
			break;
L
Linus Torvalds 已提交
799

800 801 802 803
		/* This optimisation is debatable, and completely destroys
		 * sequential read speed for 'far copies' arrays.  So only
		 * keep it for 'near' arrays, and review those later.
		 */
804
		if (geo->near_copies > 1 && !atomic_read(&rdev->nr_pending))
L
Linus Torvalds 已提交
805
			break;
806 807

		/* for far > 1 always use the lowest address */
808
		if (geo->far_copies > 1)
N
NeilBrown 已提交
809
			new_distance = r10_bio->devs[slot].addr;
810
		else
N
NeilBrown 已提交
811 812 813 814 815
			new_distance = abs(r10_bio->devs[slot].addr -
					   conf->mirrors[disk].head_position);
		if (new_distance < best_dist) {
			best_dist = new_distance;
			best_slot = slot;
816
			best_rdev = rdev;
L
Linus Torvalds 已提交
817 818
		}
	}
819
	if (slot >= conf->copies) {
N
NeilBrown 已提交
820
		slot = best_slot;
821 822
		rdev = best_rdev;
	}
L
Linus Torvalds 已提交
823

N
NeilBrown 已提交
824 825 826 827 828 829 830 831 832 833 834
	if (slot >= 0) {
		atomic_inc(&rdev->nr_pending);
		if (test_bit(Faulty, &rdev->flags)) {
			/* Cannot risk returning a device that failed
			 * before we inc'ed nr_pending
			 */
			rdev_dec_pending(rdev, conf->mddev);
			goto retry;
		}
		r10_bio->read_slot = slot;
	} else
835
		rdev = NULL;
L
Linus Torvalds 已提交
836
	rcu_read_unlock();
837
	*max_sectors = best_good_sectors;
L
Linus Torvalds 已提交
838

839
	return rdev;
L
Linus Torvalds 已提交
840 841
}

842 843
static int raid10_congested(void *data, int bits)
{
844
	struct mddev *mddev = data;
845
	struct r10conf *conf = mddev->private;
846 847
	int i, ret = 0;

848 849 850 851
	if ((bits & (1 << BDI_async_congested)) &&
	    conf->pending_count >= max_queued_requests)
		return 1;

852 853
	if (mddev_congested(mddev, bits))
		return 1;
854
	rcu_read_lock();
855 856 857 858
	for (i = 0;
	     (i < conf->geo.raid_disks || i < conf->prev.raid_disks)
		     && ret == 0;
	     i++) {
859
		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
860
		if (rdev && !test_bit(Faulty, &rdev->flags)) {
861
			struct request_queue *q = bdev_get_queue(rdev->bdev);
862 863 864 865 866 867 868 869

			ret |= bdi_congested(&q->backing_dev_info, bits);
		}
	}
	rcu_read_unlock();
	return ret;
}

870
static void flush_pending_writes(struct r10conf *conf)
871 872 873 874 875 876 877 878 879
{
	/* Any writes that have been queued but are awaiting
	 * bitmap updates get flushed here.
	 */
	spin_lock_irq(&conf->device_lock);

	if (conf->pending_bio_list.head) {
		struct bio *bio;
		bio = bio_list_get(&conf->pending_bio_list);
880
		conf->pending_count = 0;
881 882 883 884
		spin_unlock_irq(&conf->device_lock);
		/* flush any pending bitmap writes to disk
		 * before proceeding w/ I/O */
		bitmap_unplug(conf->mddev->bitmap);
885
		wake_up(&conf->wait_barrier);
886 887 888 889 890 891 892 893 894 895

		while (bio) { /* submit pending writes */
			struct bio *next = bio->bi_next;
			bio->bi_next = NULL;
			generic_make_request(bio);
			bio = next;
		}
	} else
		spin_unlock_irq(&conf->device_lock);
}
J
Jens Axboe 已提交
896

897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916
/* Barriers....
 * Sometimes we need to suspend IO while we do something else,
 * either some resync/recovery, or reconfigure the array.
 * To do this we raise a 'barrier'.
 * The 'barrier' is a counter that can be raised multiple times
 * to count how many activities are happening which preclude
 * normal IO.
 * We can only raise the barrier if there is no pending IO.
 * i.e. if nr_pending == 0.
 * We choose only to raise the barrier if no-one is waiting for the
 * barrier to go down.  This means that as soon as an IO request
 * is ready, no other operations which require a barrier will start
 * until the IO request has had a chance.
 *
 * So: regular IO calls 'wait_barrier'.  When that returns there
 *    is no backgroup IO happening,  It must arrange to call
 *    allow_barrier when it has finished its IO.
 * backgroup IO calls must call raise_barrier.  Once that returns
 *    there is no normal IO happeing.  It must arrange to call
 *    lower_barrier when the particular background IO completes.
L
Linus Torvalds 已提交
917 918
 */

919
static void raise_barrier(struct r10conf *conf, int force)
L
Linus Torvalds 已提交
920
{
921
	BUG_ON(force && !conf->barrier);
L
Linus Torvalds 已提交
922
	spin_lock_irq(&conf->resync_lock);
923

924 925
	/* Wait until no block IO is waiting (unless 'force') */
	wait_event_lock_irq(conf->wait_barrier, force || !conf->nr_waiting,
N
NeilBrown 已提交
926
			    conf->resync_lock, );
927 928 929 930

	/* block any new IO from starting */
	conf->barrier++;

N
NeilBrown 已提交
931
	/* Now wait for all pending IO to complete */
932 933
	wait_event_lock_irq(conf->wait_barrier,
			    !conf->nr_pending && conf->barrier < RESYNC_DEPTH,
N
NeilBrown 已提交
934
			    conf->resync_lock, );
935 936 937 938

	spin_unlock_irq(&conf->resync_lock);
}

939
static void lower_barrier(struct r10conf *conf)
940 941 942 943 944 945 946 947
{
	unsigned long flags;
	spin_lock_irqsave(&conf->resync_lock, flags);
	conf->barrier--;
	spin_unlock_irqrestore(&conf->resync_lock, flags);
	wake_up(&conf->wait_barrier);
}

948
static void wait_barrier(struct r10conf *conf)
949 950 951 952
{
	spin_lock_irq(&conf->resync_lock);
	if (conf->barrier) {
		conf->nr_waiting++;
953 954 955 956 957 958 959 960 961 962 963 964 965 966
		/* Wait for the barrier to drop.
		 * However if there are already pending
		 * requests (preventing the barrier from
		 * rising completely), and the
		 * pre-process bio queue isn't empty,
		 * then don't wait, as we need to empty
		 * that queue to get the nr_pending
		 * count down.
		 */
		wait_event_lock_irq(conf->wait_barrier,
				    !conf->barrier ||
				    (conf->nr_pending &&
				     current->bio_list &&
				     !bio_list_empty(current->bio_list)),
967
				    conf->resync_lock,
968
			);
969
		conf->nr_waiting--;
L
Linus Torvalds 已提交
970
	}
971
	conf->nr_pending++;
L
Linus Torvalds 已提交
972 973 974
	spin_unlock_irq(&conf->resync_lock);
}

975
static void allow_barrier(struct r10conf *conf)
976 977 978 979 980 981 982 983
{
	unsigned long flags;
	spin_lock_irqsave(&conf->resync_lock, flags);
	conf->nr_pending--;
	spin_unlock_irqrestore(&conf->resync_lock, flags);
	wake_up(&conf->wait_barrier);
}

984
static void freeze_array(struct r10conf *conf)
985 986
{
	/* stop syncio and normal IO and wait for everything to
N
NeilBrown 已提交
987
	 * go quiet.
988
	 * We increment barrier and nr_waiting, and then
989 990 991 992 993 994 995 996
	 * wait until nr_pending match nr_queued+1
	 * This is called in the context of one normal IO request
	 * that has failed. Thus any sync request that might be pending
	 * will be blocked by nr_pending, and we need to wait for
	 * pending IO requests to complete or be queued for re-try.
	 * Thus the number queued (nr_queued) plus this request (1)
	 * must match the number of pending IOs (nr_pending) before
	 * we continue.
997 998 999 1000 1001
	 */
	spin_lock_irq(&conf->resync_lock);
	conf->barrier++;
	conf->nr_waiting++;
	wait_event_lock_irq(conf->wait_barrier,
1002
			    conf->nr_pending == conf->nr_queued+1,
1003
			    conf->resync_lock,
N
NeilBrown 已提交
1004 1005
			    flush_pending_writes(conf));

1006 1007 1008
	spin_unlock_irq(&conf->resync_lock);
}

1009
static void unfreeze_array(struct r10conf *conf)
1010 1011 1012 1013 1014 1015 1016 1017 1018
{
	/* reverse the effect of the freeze */
	spin_lock_irq(&conf->resync_lock);
	conf->barrier--;
	conf->nr_waiting--;
	wake_up(&conf->wait_barrier);
	spin_unlock_irq(&conf->resync_lock);
}

1019 1020 1021 1022 1023 1024 1025 1026 1027 1028
static sector_t choose_data_offset(struct r10bio *r10_bio,
				   struct md_rdev *rdev)
{
	if (!test_bit(MD_RECOVERY_RESHAPE, &rdev->mddev->recovery) ||
	    test_bit(R10BIO_Previous, &r10_bio->state))
		return rdev->data_offset;
	else
		return rdev->new_data_offset;
}

1029
static void make_request(struct mddev *mddev, struct bio * bio)
L
Linus Torvalds 已提交
1030
{
1031
	struct r10conf *conf = mddev->private;
1032
	struct r10bio *r10_bio;
L
Linus Torvalds 已提交
1033 1034
	struct bio *read_bio;
	int i;
1035
	sector_t chunk_mask = (conf->geo.chunk_mask & conf->prev.chunk_mask);
1036
	int chunk_sects = chunk_mask + 1;
1037
	const int rw = bio_data_dir(bio);
1038
	const unsigned long do_sync = (bio->bi_rw & REQ_SYNC);
T
Tejun Heo 已提交
1039
	const unsigned long do_fua = (bio->bi_rw & REQ_FUA);
1040
	unsigned long flags;
1041
	struct md_rdev *blocked_rdev;
N
NeilBrown 已提交
1042
	int plugged;
1043 1044
	int sectors_handled;
	int max_sectors;
N
NeilBrown 已提交
1045
	int sectors;
L
Linus Torvalds 已提交
1046

T
Tejun Heo 已提交
1047 1048
	if (unlikely(bio->bi_rw & REQ_FLUSH)) {
		md_flush_request(mddev, bio);
1049
		return;
1050 1051
	}

L
Linus Torvalds 已提交
1052 1053 1054
	/* If this request crosses a chunk boundary, we need to
	 * split it.  This will only happen for 1 PAGE (or less) requests.
	 */
1055 1056
	if (unlikely((bio->bi_sector & chunk_mask) + (bio->bi_size >> 9)
		     > chunk_sects
1057 1058
		     && (conf->geo.near_copies < conf->geo.raid_disks
			 || conf->prev.near_copies < conf->prev.raid_disks))) {
L
Linus Torvalds 已提交
1059 1060 1061 1062 1063 1064 1065 1066
		struct bio_pair *bp;
		/* Sanity check -- queue functions should prevent this happening */
		if (bio->bi_vcnt != 1 ||
		    bio->bi_idx != 0)
			goto bad_map;
		/* This is a one page bio that upper layers
		 * refuse to split for us, so we need to split it.
		 */
D
Denis ChengRq 已提交
1067
		bp = bio_split(bio,
L
Linus Torvalds 已提交
1068
			       chunk_sects - (bio->bi_sector & (chunk_sects - 1)) );
1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081

		/* Each of these 'make_request' calls will call 'wait_barrier'.
		 * If the first succeeds but the second blocks due to the resync
		 * thread raising the barrier, we will deadlock because the
		 * IO to the underlying device will be queued in generic_make_request
		 * and will never complete, so will never reduce nr_pending.
		 * So increment nr_waiting here so no new raise_barriers will
		 * succeed, and so the second wait_barrier cannot block.
		 */
		spin_lock_irq(&conf->resync_lock);
		conf->nr_waiting++;
		spin_unlock_irq(&conf->resync_lock);

1082 1083
		make_request(mddev, &bp->bio1);
		make_request(mddev, &bp->bio2);
L
Linus Torvalds 已提交
1084

1085 1086 1087 1088 1089
		spin_lock_irq(&conf->resync_lock);
		conf->nr_waiting--;
		wake_up(&conf->wait_barrier);
		spin_unlock_irq(&conf->resync_lock);

L
Linus Torvalds 已提交
1090
		bio_pair_release(bp);
1091
		return;
L
Linus Torvalds 已提交
1092
	bad_map:
N
NeilBrown 已提交
1093 1094
		printk("md/raid10:%s: make_request bug: can't convert block across chunks"
		       " or bigger than %dk %llu %d\n", mdname(mddev), chunk_sects/2,
L
Linus Torvalds 已提交
1095 1096
		       (unsigned long long)bio->bi_sector, bio->bi_size >> 10);

1097
		bio_io_error(bio);
1098
		return;
L
Linus Torvalds 已提交
1099 1100
	}

1101
	md_write_start(mddev, bio);
1102

L
Linus Torvalds 已提交
1103 1104 1105 1106 1107
	/*
	 * Register the new request and wait if the reconstruction
	 * thread has put up a bar for new requests.
	 * Continue immediately if no resync is active currently.
	 */
1108
	wait_barrier(conf);
L
Linus Torvalds 已提交
1109

N
NeilBrown 已提交
1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140
	sectors = bio->bi_size >> 9;
	while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
	    bio->bi_sector < conf->reshape_progress &&
	    bio->bi_sector + sectors > conf->reshape_progress) {
		/* IO spans the reshape position.  Need to wait for
		 * reshape to pass
		 */
		allow_barrier(conf);
		wait_event(conf->wait_barrier,
			   conf->reshape_progress <= bio->bi_sector ||
			   conf->reshape_progress >= bio->bi_sector + sectors);
		wait_barrier(conf);
	}
	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
	    bio_data_dir(bio) == WRITE &&
	    (mddev->reshape_backwards
	     ? (bio->bi_sector < conf->reshape_safe &&
		bio->bi_sector + sectors > conf->reshape_progress)
	     : (bio->bi_sector + sectors > conf->reshape_safe &&
		bio->bi_sector < conf->reshape_progress))) {
		/* Need to update reshape_position in metadata */
		mddev->reshape_position = conf->reshape_progress;
		set_bit(MD_CHANGE_DEVS, &mddev->flags);
		set_bit(MD_CHANGE_PENDING, &mddev->flags);
		md_wakeup_thread(mddev->thread);
		wait_event(mddev->sb_wait,
			   !test_bit(MD_CHANGE_PENDING, &mddev->flags));

		conf->reshape_safe = mddev->reshape_position;
	}

L
Linus Torvalds 已提交
1141 1142 1143
	r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);

	r10_bio->master_bio = bio;
N
NeilBrown 已提交
1144
	r10_bio->sectors = sectors;
L
Linus Torvalds 已提交
1145 1146 1147

	r10_bio->mddev = mddev;
	r10_bio->sector = bio->bi_sector;
1148
	r10_bio->state = 0;
L
Linus Torvalds 已提交
1149

1150 1151 1152 1153 1154 1155 1156 1157 1158 1159
	/* We might need to issue multiple reads to different
	 * devices if there are bad blocks around, so we keep
	 * track of the number of reads in bio->bi_phys_segments.
	 * If this is 0, there is only one r10_bio and no locking
	 * will be needed when the request completes.  If it is
	 * non-zero, then it is the number of not-completed requests.
	 */
	bio->bi_phys_segments = 0;
	clear_bit(BIO_SEG_VALID, &bio->bi_flags);

1160
	if (rw == READ) {
L
Linus Torvalds 已提交
1161 1162 1163
		/*
		 * read balancing logic:
		 */
1164
		struct md_rdev *rdev;
1165 1166 1167
		int slot;

read_again:
1168 1169
		rdev = read_balance(conf, r10_bio, &max_sectors);
		if (!rdev) {
L
Linus Torvalds 已提交
1170
			raid_end_bio_io(r10_bio);
1171
			return;
L
Linus Torvalds 已提交
1172
		}
1173
		slot = r10_bio->read_slot;
L
Linus Torvalds 已提交
1174

1175
		read_bio = bio_clone_mddev(bio, GFP_NOIO, mddev);
1176 1177
		md_trim_bio(read_bio, r10_bio->sector - bio->bi_sector,
			    max_sectors);
L
Linus Torvalds 已提交
1178 1179

		r10_bio->devs[slot].bio = read_bio;
1180
		r10_bio->devs[slot].rdev = rdev;
L
Linus Torvalds 已提交
1181 1182

		read_bio->bi_sector = r10_bio->devs[slot].addr +
1183
			choose_data_offset(r10_bio, rdev);
1184
		read_bio->bi_bdev = rdev->bdev;
L
Linus Torvalds 已提交
1185
		read_bio->bi_end_io = raid10_end_read_request;
1186
		read_bio->bi_rw = READ | do_sync;
L
Linus Torvalds 已提交
1187 1188
		read_bio->bi_private = r10_bio;

1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219
		if (max_sectors < r10_bio->sectors) {
			/* Could not read all from this device, so we will
			 * need another r10_bio.
			 */
			sectors_handled = (r10_bio->sectors + max_sectors
					   - bio->bi_sector);
			r10_bio->sectors = max_sectors;
			spin_lock_irq(&conf->device_lock);
			if (bio->bi_phys_segments == 0)
				bio->bi_phys_segments = 2;
			else
				bio->bi_phys_segments++;
			spin_unlock(&conf->device_lock);
			/* Cannot call generic_make_request directly
			 * as that will be queued in __generic_make_request
			 * and subsequent mempool_alloc might block
			 * waiting for it.  so hand bio over to raid10d.
			 */
			reschedule_retry(r10_bio);

			r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);

			r10_bio->master_bio = bio;
			r10_bio->sectors = ((bio->bi_size >> 9)
					    - sectors_handled);
			r10_bio->state = 0;
			r10_bio->mddev = mddev;
			r10_bio->sector = bio->bi_sector + sectors_handled;
			goto read_again;
		} else
			generic_make_request(read_bio);
1220
		return;
L
Linus Torvalds 已提交
1221 1222 1223 1224 1225
	}

	/*
	 * WRITE:
	 */
1226 1227 1228 1229 1230
	if (conf->pending_count >= max_queued_requests) {
		md_wakeup_thread(mddev->thread);
		wait_event(conf->wait_barrier,
			   conf->pending_count < max_queued_requests);
	}
1231
	/* first select target devices under rcu_lock and
L
Linus Torvalds 已提交
1232 1233
	 * inc refcount on their rdev.  Record them by setting
	 * bios[x] to bio
1234 1235 1236 1237 1238 1239 1240
	 * If there are known/acknowledged bad blocks on any device
	 * on which we have seen a write error, we want to avoid
	 * writing to those blocks.  This potentially requires several
	 * writes to write around the bad blocks.  Each set of writes
	 * gets its own r10_bio with a set of bios attached.  The number
	 * of r10_bios is recored in bio->bi_phys_segments just as with
	 * the read case.
L
Linus Torvalds 已提交
1241
	 */
N
NeilBrown 已提交
1242 1243
	plugged = mddev_check_plugged(mddev);

1244
	r10_bio->read_slot = -1; /* make sure repl_bio gets freed */
L
Linus Torvalds 已提交
1245
	raid10_find_phys(conf, r10_bio);
1246
retry_write:
1247
	blocked_rdev = NULL;
L
Linus Torvalds 已提交
1248
	rcu_read_lock();
1249 1250
	max_sectors = r10_bio->sectors;

L
Linus Torvalds 已提交
1251 1252
	for (i = 0;  i < conf->copies; i++) {
		int d = r10_bio->devs[i].devnum;
1253
		struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev);
1254 1255
		struct md_rdev *rrdev = rcu_dereference(
			conf->mirrors[d].replacement);
1256 1257
		if (rdev == rrdev)
			rrdev = NULL;
1258 1259 1260 1261 1262
		if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) {
			atomic_inc(&rdev->nr_pending);
			blocked_rdev = rdev;
			break;
		}
1263 1264 1265 1266 1267
		if (rrdev && unlikely(test_bit(Blocked, &rrdev->flags))) {
			atomic_inc(&rrdev->nr_pending);
			blocked_rdev = rrdev;
			break;
		}
1268 1269
		if (rrdev && (test_bit(Faulty, &rrdev->flags)
			      || test_bit(Unmerged, &rrdev->flags)))
1270 1271
			rrdev = NULL;

1272
		r10_bio->devs[i].bio = NULL;
1273
		r10_bio->devs[i].repl_bio = NULL;
1274 1275
		if (!rdev || test_bit(Faulty, &rdev->flags) ||
		    test_bit(Unmerged, &rdev->flags)) {
1276
			set_bit(R10BIO_Degraded, &r10_bio->state);
1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319
			continue;
		}
		if (test_bit(WriteErrorSeen, &rdev->flags)) {
			sector_t first_bad;
			sector_t dev_sector = r10_bio->devs[i].addr;
			int bad_sectors;
			int is_bad;

			is_bad = is_badblock(rdev, dev_sector,
					     max_sectors,
					     &first_bad, &bad_sectors);
			if (is_bad < 0) {
				/* Mustn't write here until the bad block
				 * is acknowledged
				 */
				atomic_inc(&rdev->nr_pending);
				set_bit(BlockedBadBlocks, &rdev->flags);
				blocked_rdev = rdev;
				break;
			}
			if (is_bad && first_bad <= dev_sector) {
				/* Cannot write here at all */
				bad_sectors -= (dev_sector - first_bad);
				if (bad_sectors < max_sectors)
					/* Mustn't write more than bad_sectors
					 * to other devices yet
					 */
					max_sectors = bad_sectors;
				/* We don't set R10BIO_Degraded as that
				 * only applies if the disk is missing,
				 * so it might be re-added, and we want to
				 * know to recover this chunk.
				 * In this case the device is here, and the
				 * fact that this chunk is not in-sync is
				 * recorded in the bad block log.
				 */
				continue;
			}
			if (is_bad) {
				int good_sectors = first_bad - dev_sector;
				if (good_sectors < max_sectors)
					max_sectors = good_sectors;
			}
1320
		}
1321 1322
		r10_bio->devs[i].bio = bio;
		atomic_inc(&rdev->nr_pending);
1323 1324 1325 1326
		if (rrdev) {
			r10_bio->devs[i].repl_bio = bio;
			atomic_inc(&rrdev->nr_pending);
		}
L
Linus Torvalds 已提交
1327 1328 1329
	}
	rcu_read_unlock();

1330 1331 1332 1333 1334
	if (unlikely(blocked_rdev)) {
		/* Have to wait for this device to get unblocked, then retry */
		int j;
		int d;

1335
		for (j = 0; j < i; j++) {
1336 1337 1338 1339
			if (r10_bio->devs[j].bio) {
				d = r10_bio->devs[j].devnum;
				rdev_dec_pending(conf->mirrors[d].rdev, mddev);
			}
1340
			if (r10_bio->devs[j].repl_bio) {
1341
				struct md_rdev *rdev;
1342
				d = r10_bio->devs[j].devnum;
1343 1344 1345 1346 1347 1348 1349
				rdev = conf->mirrors[d].replacement;
				if (!rdev) {
					/* Race with remove_disk */
					smp_mb();
					rdev = conf->mirrors[d].rdev;
				}
				rdev_dec_pending(rdev, mddev);
1350 1351
			}
		}
1352 1353 1354 1355 1356 1357
		allow_barrier(conf);
		md_wait_for_blocked_rdev(blocked_rdev, mddev);
		wait_barrier(conf);
		goto retry_write;
	}

1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371
	if (max_sectors < r10_bio->sectors) {
		/* We are splitting this into multiple parts, so
		 * we need to prepare for allocating another r10_bio.
		 */
		r10_bio->sectors = max_sectors;
		spin_lock_irq(&conf->device_lock);
		if (bio->bi_phys_segments == 0)
			bio->bi_phys_segments = 2;
		else
			bio->bi_phys_segments++;
		spin_unlock_irq(&conf->device_lock);
	}
	sectors_handled = r10_bio->sector + max_sectors - bio->bi_sector;

1372
	atomic_set(&r10_bio->remaining, 1);
1373
	bitmap_startwrite(mddev->bitmap, r10_bio->sector, r10_bio->sectors, 0);
1374

L
Linus Torvalds 已提交
1375 1376 1377 1378 1379 1380
	for (i = 0; i < conf->copies; i++) {
		struct bio *mbio;
		int d = r10_bio->devs[i].devnum;
		if (!r10_bio->devs[i].bio)
			continue;

1381
		mbio = bio_clone_mddev(bio, GFP_NOIO, mddev);
1382 1383
		md_trim_bio(mbio, r10_bio->sector - bio->bi_sector,
			    max_sectors);
L
Linus Torvalds 已提交
1384 1385
		r10_bio->devs[i].bio = mbio;

1386
		mbio->bi_sector	= (r10_bio->devs[i].addr+
1387 1388
				   choose_data_offset(r10_bio,
						      conf->mirrors[d].rdev));
L
Linus Torvalds 已提交
1389 1390
		mbio->bi_bdev = conf->mirrors[d].rdev->bdev;
		mbio->bi_end_io	= raid10_end_write_request;
T
Tejun Heo 已提交
1391
		mbio->bi_rw = WRITE | do_sync | do_fua;
L
Linus Torvalds 已提交
1392 1393 1394
		mbio->bi_private = r10_bio;

		atomic_inc(&r10_bio->remaining);
1395 1396
		spin_lock_irqsave(&conf->device_lock, flags);
		bio_list_add(&conf->pending_bio_list, mbio);
1397
		conf->pending_count++;
1398
		spin_unlock_irqrestore(&conf->device_lock, flags);
1399 1400 1401 1402 1403 1404 1405 1406 1407

		if (!r10_bio->devs[i].repl_bio)
			continue;

		mbio = bio_clone_mddev(bio, GFP_NOIO, mddev);
		md_trim_bio(mbio, r10_bio->sector - bio->bi_sector,
			    max_sectors);
		r10_bio->devs[i].repl_bio = mbio;

1408 1409 1410 1411
		/* We are actively writing to the original device
		 * so it cannot disappear, so the replacement cannot
		 * become NULL here
		 */
1412 1413 1414 1415
		mbio->bi_sector	= (r10_bio->devs[i].addr +
				   choose_data_offset(
					   r10_bio,
					   conf->mirrors[d].replacement));
1416 1417 1418 1419 1420 1421 1422 1423 1424 1425
		mbio->bi_bdev = conf->mirrors[d].replacement->bdev;
		mbio->bi_end_io	= raid10_end_write_request;
		mbio->bi_rw = WRITE | do_sync | do_fua;
		mbio->bi_private = r10_bio;

		atomic_inc(&r10_bio->remaining);
		spin_lock_irqsave(&conf->device_lock, flags);
		bio_list_add(&conf->pending_bio_list, mbio);
		conf->pending_count++;
		spin_unlock_irqrestore(&conf->device_lock, flags);
L
Linus Torvalds 已提交
1426 1427
	}

1428 1429 1430
	/* Don't remove the bias on 'remaining' (one_write_done) until
	 * after checking if we need to go around again.
	 */
1431

1432
	if (sectors_handled < (bio->bi_size >> 9)) {
1433
		one_write_done(r10_bio);
1434
		/* We need another r10_bio.  It has already been counted
1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446
		 * in bio->bi_phys_segments.
		 */
		r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);

		r10_bio->master_bio = bio;
		r10_bio->sectors = (bio->bi_size >> 9) - sectors_handled;

		r10_bio->mddev = mddev;
		r10_bio->sector = bio->bi_sector + sectors_handled;
		r10_bio->state = 0;
		goto retry_write;
	}
1447 1448 1449 1450
	one_write_done(r10_bio);

	/* In case raid10d snuck in to freeze_array */
	wake_up(&conf->wait_barrier);
1451

N
NeilBrown 已提交
1452
	if (do_sync || !mddev->bitmap || !plugged)
1453
		md_wakeup_thread(mddev->thread);
L
Linus Torvalds 已提交
1454 1455
}

1456
static void status(struct seq_file *seq, struct mddev *mddev)
L
Linus Torvalds 已提交
1457
{
1458
	struct r10conf *conf = mddev->private;
L
Linus Torvalds 已提交
1459 1460
	int i;

1461
	if (conf->geo.near_copies < conf->geo.raid_disks)
1462
		seq_printf(seq, " %dK chunks", mddev->chunk_sectors / 2);
1463 1464 1465 1466 1467
	if (conf->geo.near_copies > 1)
		seq_printf(seq, " %d near-copies", conf->geo.near_copies);
	if (conf->geo.far_copies > 1) {
		if (conf->geo.far_offset)
			seq_printf(seq, " %d offset-copies", conf->geo.far_copies);
1468
		else
1469
			seq_printf(seq, " %d far-copies", conf->geo.far_copies);
1470
	}
1471 1472 1473
	seq_printf(seq, " [%d/%d] [", conf->geo.raid_disks,
					conf->geo.raid_disks - mddev->degraded);
	for (i = 0; i < conf->geo.raid_disks; i++)
L
Linus Torvalds 已提交
1474 1475
		seq_printf(seq, "%s",
			      conf->mirrors[i].rdev &&
1476
			      test_bit(In_sync, &conf->mirrors[i].rdev->flags) ? "U" : "_");
L
Linus Torvalds 已提交
1477 1478 1479
	seq_printf(seq, "]");
}

1480 1481 1482 1483 1484
/* check if there are enough drives for
 * every block to appear on atleast one.
 * Don't consider the device numbered 'ignore'
 * as we might be about to remove it.
 */
1485
static int _enough(struct r10conf *conf, struct geom *geo, int ignore)
1486 1487 1488 1489 1490 1491 1492 1493 1494 1495
{
	int first = 0;

	do {
		int n = conf->copies;
		int cnt = 0;
		while (n--) {
			if (conf->mirrors[first].rdev &&
			    first != ignore)
				cnt++;
1496
			first = (first+1) % geo->raid_disks;
1497 1498 1499 1500 1501 1502 1503
		}
		if (cnt == 0)
			return 0;
	} while (first != 0);
	return 1;
}

1504 1505 1506 1507 1508 1509
static int enough(struct r10conf *conf, int ignore)
{
	return _enough(conf, &conf->geo, ignore) &&
		_enough(conf, &conf->prev, ignore);
}

1510
static void error(struct mddev *mddev, struct md_rdev *rdev)
L
Linus Torvalds 已提交
1511 1512
{
	char b[BDEVNAME_SIZE];
1513
	struct r10conf *conf = mddev->private;
L
Linus Torvalds 已提交
1514 1515 1516 1517 1518 1519 1520

	/*
	 * If it is not operational, then we have already marked it as dead
	 * else if it is the last working disks, ignore the error, let the
	 * next level up know.
	 * else mark the drive as failed
	 */
1521
	if (test_bit(In_sync, &rdev->flags)
1522
	    && !enough(conf, rdev->raid_disk))
L
Linus Torvalds 已提交
1523 1524 1525 1526
		/*
		 * Don't fail the drive, just return an IO error.
		 */
		return;
1527 1528 1529
	if (test_and_clear_bit(In_sync, &rdev->flags)) {
		unsigned long flags;
		spin_lock_irqsave(&conf->device_lock, flags);
L
Linus Torvalds 已提交
1530
		mddev->degraded++;
1531
		spin_unlock_irqrestore(&conf->device_lock, flags);
L
Linus Torvalds 已提交
1532 1533 1534
		/*
		 * if recovery is running, make sure it aborts.
		 */
1535
		set_bit(MD_RECOVERY_INTR, &mddev->recovery);
L
Linus Torvalds 已提交
1536
	}
1537
	set_bit(Blocked, &rdev->flags);
1538
	set_bit(Faulty, &rdev->flags);
1539
	set_bit(MD_CHANGE_DEVS, &mddev->flags);
1540 1541 1542
	printk(KERN_ALERT
	       "md/raid10:%s: Disk failure on %s, disabling device.\n"
	       "md/raid10:%s: Operation continuing on %d devices.\n",
N
NeilBrown 已提交
1543
	       mdname(mddev), bdevname(rdev->bdev, b),
1544
	       mdname(mddev), conf->geo.raid_disks - mddev->degraded);
L
Linus Torvalds 已提交
1545 1546
}

1547
static void print_conf(struct r10conf *conf)
L
Linus Torvalds 已提交
1548 1549
{
	int i;
1550
	struct mirror_info *tmp;
L
Linus Torvalds 已提交
1551

N
NeilBrown 已提交
1552
	printk(KERN_DEBUG "RAID10 conf printout:\n");
L
Linus Torvalds 已提交
1553
	if (!conf) {
N
NeilBrown 已提交
1554
		printk(KERN_DEBUG "(!conf)\n");
L
Linus Torvalds 已提交
1555 1556
		return;
	}
1557 1558
	printk(KERN_DEBUG " --- wd:%d rd:%d\n", conf->geo.raid_disks - conf->mddev->degraded,
		conf->geo.raid_disks);
L
Linus Torvalds 已提交
1559

1560
	for (i = 0; i < conf->geo.raid_disks; i++) {
L
Linus Torvalds 已提交
1561 1562 1563
		char b[BDEVNAME_SIZE];
		tmp = conf->mirrors + i;
		if (tmp->rdev)
N
NeilBrown 已提交
1564
			printk(KERN_DEBUG " disk %d, wo:%d, o:%d, dev:%s\n",
1565 1566
				i, !test_bit(In_sync, &tmp->rdev->flags),
			        !test_bit(Faulty, &tmp->rdev->flags),
L
Linus Torvalds 已提交
1567 1568 1569 1570
				bdevname(tmp->rdev->bdev,b));
	}
}

1571
static void close_sync(struct r10conf *conf)
L
Linus Torvalds 已提交
1572
{
1573 1574
	wait_barrier(conf);
	allow_barrier(conf);
L
Linus Torvalds 已提交
1575 1576 1577 1578 1579

	mempool_destroy(conf->r10buf_pool);
	conf->r10buf_pool = NULL;
}

1580
static int raid10_spare_active(struct mddev *mddev)
L
Linus Torvalds 已提交
1581 1582
{
	int i;
1583
	struct r10conf *conf = mddev->private;
1584
	struct mirror_info *tmp;
1585 1586
	int count = 0;
	unsigned long flags;
L
Linus Torvalds 已提交
1587 1588 1589 1590 1591

	/*
	 * Find all non-in_sync disks within the RAID10 configuration
	 * and mark them in_sync
	 */
1592
	for (i = 0; i < conf->geo.raid_disks; i++) {
L
Linus Torvalds 已提交
1593
		tmp = conf->mirrors + i;
1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614
		if (tmp->replacement
		    && tmp->replacement->recovery_offset == MaxSector
		    && !test_bit(Faulty, &tmp->replacement->flags)
		    && !test_and_set_bit(In_sync, &tmp->replacement->flags)) {
			/* Replacement has just become active */
			if (!tmp->rdev
			    || !test_and_clear_bit(In_sync, &tmp->rdev->flags))
				count++;
			if (tmp->rdev) {
				/* Replaced device not technically faulty,
				 * but we need to be sure it gets removed
				 * and never re-added.
				 */
				set_bit(Faulty, &tmp->rdev->flags);
				sysfs_notify_dirent_safe(
					tmp->rdev->sysfs_state);
			}
			sysfs_notify_dirent_safe(tmp->replacement->sysfs_state);
		} else if (tmp->rdev
			   && !test_bit(Faulty, &tmp->rdev->flags)
			   && !test_and_set_bit(In_sync, &tmp->rdev->flags)) {
1615
			count++;
1616
			sysfs_notify_dirent(tmp->rdev->sysfs_state);
L
Linus Torvalds 已提交
1617 1618
		}
	}
1619 1620 1621
	spin_lock_irqsave(&conf->device_lock, flags);
	mddev->degraded -= count;
	spin_unlock_irqrestore(&conf->device_lock, flags);
L
Linus Torvalds 已提交
1622 1623

	print_conf(conf);
1624
	return count;
L
Linus Torvalds 已提交
1625 1626 1627
}


1628
static int raid10_add_disk(struct mddev *mddev, struct md_rdev *rdev)
L
Linus Torvalds 已提交
1629
{
1630
	struct r10conf *conf = mddev->private;
1631
	int err = -EEXIST;
L
Linus Torvalds 已提交
1632
	int mirror;
1633
	int first = 0;
1634
	int last = conf->geo.raid_disks - 1;
1635
	struct request_queue *q = bdev_get_queue(rdev->bdev);
L
Linus Torvalds 已提交
1636 1637 1638 1639 1640

	if (mddev->recovery_cp < MaxSector)
		/* only hot-add to in-sync arrays, as recovery is
		 * very different from resync
		 */
1641
		return -EBUSY;
1642
	if (rdev->saved_raid_disk < 0 && !_enough(conf, &conf->prev, -1))
1643
		return -EINVAL;
L
Linus Torvalds 已提交
1644

N
NeilBrown 已提交
1645
	if (rdev->raid_disk >= 0)
1646
		first = last = rdev->raid_disk;
L
Linus Torvalds 已提交
1647

1648 1649 1650 1651 1652
	if (q->merge_bvec_fn) {
		set_bit(Unmerged, &rdev->flags);
		mddev->merge_check_needed = 1;
	}

1653
	if (rdev->saved_raid_disk >= first &&
1654 1655 1656
	    conf->mirrors[rdev->saved_raid_disk].rdev == NULL)
		mirror = rdev->saved_raid_disk;
	else
1657
		mirror = first;
1658
	for ( ; mirror <= last ; mirror++) {
1659
		struct mirror_info *p = &conf->mirrors[mirror];
1660 1661
		if (p->recovery_disabled == mddev->recovery_disabled)
			continue;
1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675
		if (p->rdev) {
			if (!test_bit(WantReplacement, &p->rdev->flags) ||
			    p->replacement != NULL)
				continue;
			clear_bit(In_sync, &rdev->flags);
			set_bit(Replacement, &rdev->flags);
			rdev->raid_disk = mirror;
			err = 0;
			disk_stack_limits(mddev->gendisk, rdev->bdev,
					  rdev->data_offset << 9);
			conf->fullsync = 1;
			rcu_assign_pointer(p->replacement, rdev);
			break;
		}
L
Linus Torvalds 已提交
1676

1677 1678
		disk_stack_limits(mddev->gendisk, rdev->bdev,
				  rdev->data_offset << 9);
L
Linus Torvalds 已提交
1679

1680
		p->head_position = 0;
1681
		p->recovery_disabled = mddev->recovery_disabled - 1;
1682 1683 1684 1685 1686 1687 1688
		rdev->raid_disk = mirror;
		err = 0;
		if (rdev->saved_raid_disk != mirror)
			conf->fullsync = 1;
		rcu_assign_pointer(p->rdev, rdev);
		break;
	}
1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701
	if (err == 0 && test_bit(Unmerged, &rdev->flags)) {
		/* Some requests might not have seen this new
		 * merge_bvec_fn.  We must wait for them to complete
		 * before merging the device fully.
		 * First we make sure any code which has tested
		 * our function has submitted the request, then
		 * we wait for all outstanding requests to complete.
		 */
		synchronize_sched();
		raise_barrier(conf, 0);
		lower_barrier(conf);
		clear_bit(Unmerged, &rdev->flags);
	}
1702
	md_integrity_add_rdev(rdev, mddev);
L
Linus Torvalds 已提交
1703
	print_conf(conf);
1704
	return err;
L
Linus Torvalds 已提交
1705 1706
}

1707
static int raid10_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
L
Linus Torvalds 已提交
1708
{
1709
	struct r10conf *conf = mddev->private;
L
Linus Torvalds 已提交
1710
	int err = 0;
1711
	int number = rdev->raid_disk;
1712 1713
	struct md_rdev **rdevp;
	struct mirror_info *p = conf->mirrors + number;
L
Linus Torvalds 已提交
1714 1715

	print_conf(conf);
1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732
	if (rdev == p->rdev)
		rdevp = &p->rdev;
	else if (rdev == p->replacement)
		rdevp = &p->replacement;
	else
		return 0;

	if (test_bit(In_sync, &rdev->flags) ||
	    atomic_read(&rdev->nr_pending)) {
		err = -EBUSY;
		goto abort;
	}
	/* Only remove faulty devices if recovery
	 * is not possible.
	 */
	if (!test_bit(Faulty, &rdev->flags) &&
	    mddev->recovery_disabled != p->recovery_disabled &&
1733
	    (!p->replacement || p->replacement == rdev) &&
1734 1735 1736
	    enough(conf, -1)) {
		err = -EBUSY;
		goto abort;
L
Linus Torvalds 已提交
1737
	}
1738 1739 1740 1741 1742 1743 1744
	*rdevp = NULL;
	synchronize_rcu();
	if (atomic_read(&rdev->nr_pending)) {
		/* lost the race, try later */
		err = -EBUSY;
		*rdevp = rdev;
		goto abort;
1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759
	} else if (p->replacement) {
		/* We must have just cleared 'rdev' */
		p->rdev = p->replacement;
		clear_bit(Replacement, &p->replacement->flags);
		smp_mb(); /* Make sure other CPUs may see both as identical
			   * but will never see neither -- if they are careful.
			   */
		p->replacement = NULL;
		clear_bit(WantReplacement, &rdev->flags);
	} else
		/* We might have just remove the Replacement as faulty
		 * Clear the flag just in case
		 */
		clear_bit(WantReplacement, &rdev->flags);

1760 1761
	err = md_integrity_register(mddev);

L
Linus Torvalds 已提交
1762 1763 1764 1765 1766 1767 1768
abort:

	print_conf(conf);
	return err;
}


1769
static void end_sync_read(struct bio *bio, int error)
L
Linus Torvalds 已提交
1770
{
1771
	struct r10bio *r10_bio = bio->bi_private;
1772
	struct r10conf *conf = r10_bio->mddev->private;
1773
	int d;
L
Linus Torvalds 已提交
1774

N
NeilBrown 已提交
1775 1776 1777 1778 1779
	if (bio == r10_bio->master_bio) {
		/* this is a reshape read */
		d = r10_bio->read_slot; /* really the read dev */
	} else
		d = find_bio_disk(conf, r10_bio, bio, NULL, NULL);
1780 1781 1782

	if (test_bit(BIO_UPTODATE, &bio->bi_flags))
		set_bit(R10BIO_Uptodate, &r10_bio->state);
1783 1784 1785 1786
	else
		/* The write handler will notice the lack of
		 * R10BIO_Uptodate and record any errors etc
		 */
1787 1788
		atomic_add(r10_bio->sectors,
			   &conf->mirrors[d].rdev->corrected_errors);
L
Linus Torvalds 已提交
1789 1790 1791 1792

	/* for reconstruct, we always reschedule after a read.
	 * for resync, only after all reads
	 */
1793
	rdev_dec_pending(conf->mirrors[d].rdev, conf->mddev);
L
Linus Torvalds 已提交
1794 1795 1796 1797 1798 1799 1800 1801 1802
	if (test_bit(R10BIO_IsRecover, &r10_bio->state) ||
	    atomic_dec_and_test(&r10_bio->remaining)) {
		/* we have read all the blocks,
		 * do the comparison in process context in raid10d
		 */
		reschedule_retry(r10_bio);
	}
}

1803
static void end_sync_request(struct r10bio *r10_bio)
L
Linus Torvalds 已提交
1804
{
1805
	struct mddev *mddev = r10_bio->mddev;
1806

L
Linus Torvalds 已提交
1807 1808 1809
	while (atomic_dec_and_test(&r10_bio->remaining)) {
		if (r10_bio->master_bio == NULL) {
			/* the primary of several recovery bios */
1810
			sector_t s = r10_bio->sectors;
1811 1812
			if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
			    test_bit(R10BIO_WriteError, &r10_bio->state))
1813 1814 1815
				reschedule_retry(r10_bio);
			else
				put_buf(r10_bio);
1816
			md_done_sync(mddev, s, 1);
L
Linus Torvalds 已提交
1817 1818
			break;
		} else {
1819
			struct r10bio *r10_bio2 = (struct r10bio *)r10_bio->master_bio;
1820 1821
			if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
			    test_bit(R10BIO_WriteError, &r10_bio->state))
1822 1823 1824
				reschedule_retry(r10_bio);
			else
				put_buf(r10_bio);
L
Linus Torvalds 已提交
1825 1826 1827 1828 1829
			r10_bio = r10_bio2;
		}
	}
}

1830 1831 1832
static void end_sync_write(struct bio *bio, int error)
{
	int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
1833
	struct r10bio *r10_bio = bio->bi_private;
1834
	struct mddev *mddev = r10_bio->mddev;
1835
	struct r10conf *conf = mddev->private;
1836 1837 1838 1839
	int d;
	sector_t first_bad;
	int bad_sectors;
	int slot;
1840
	int repl;
1841
	struct md_rdev *rdev = NULL;
1842

1843 1844 1845
	d = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
	if (repl)
		rdev = conf->mirrors[d].replacement;
1846
	else
1847
		rdev = conf->mirrors[d].rdev;
1848 1849

	if (!uptodate) {
1850 1851 1852 1853
		if (repl)
			md_error(mddev, rdev);
		else {
			set_bit(WriteErrorSeen, &rdev->flags);
1854 1855 1856
			if (!test_and_set_bit(WantReplacement, &rdev->flags))
				set_bit(MD_RECOVERY_NEEDED,
					&rdev->mddev->recovery);
1857 1858 1859
			set_bit(R10BIO_WriteError, &r10_bio->state);
		}
	} else if (is_badblock(rdev,
1860 1861 1862 1863 1864
			     r10_bio->devs[slot].addr,
			     r10_bio->sectors,
			     &first_bad, &bad_sectors))
		set_bit(R10BIO_MadeGood, &r10_bio->state);

1865
	rdev_dec_pending(rdev, mddev);
1866 1867 1868 1869

	end_sync_request(r10_bio);
}

L
Linus Torvalds 已提交
1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885
/*
 * Note: sync and recover and handled very differently for raid10
 * This code is for resync.
 * For resync, we read through virtual addresses and read all blocks.
 * If there is any error, we schedule a write.  The lowest numbered
 * drive is authoritative.
 * However requests come for physical address, so we need to map.
 * For every physical address there are raid_disks/copies virtual addresses,
 * which is always are least one, but is not necessarly an integer.
 * This means that a physical address can span multiple chunks, so we may
 * have to submit multiple io requests for a single sync request.
 */
/*
 * We check if all blocks are in-sync and only write to blocks that
 * aren't in sync
 */
1886
static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio)
L
Linus Torvalds 已提交
1887
{
1888
	struct r10conf *conf = mddev->private;
L
Linus Torvalds 已提交
1889 1890
	int i, first;
	struct bio *tbio, *fbio;
1891
	int vcnt;
L
Linus Torvalds 已提交
1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905

	atomic_set(&r10_bio->remaining, 1);

	/* find the first device with a block */
	for (i=0; i<conf->copies; i++)
		if (test_bit(BIO_UPTODATE, &r10_bio->devs[i].bio->bi_flags))
			break;

	if (i == conf->copies)
		goto done;

	first = i;
	fbio = r10_bio->devs[i].bio;

1906
	vcnt = (r10_bio->sectors + (PAGE_SIZE >> 9) - 1) >> (PAGE_SHIFT - 9);
L
Linus Torvalds 已提交
1907
	/* now find blocks with errors */
1908 1909
	for (i=0 ; i < conf->copies ; i++) {
		int  j, d;
L
Linus Torvalds 已提交
1910 1911

		tbio = r10_bio->devs[i].bio;
1912 1913 1914 1915

		if (tbio->bi_end_io != end_sync_read)
			continue;
		if (i == first)
L
Linus Torvalds 已提交
1916
			continue;
1917 1918 1919 1920 1921 1922 1923 1924
		if (test_bit(BIO_UPTODATE, &r10_bio->devs[i].bio->bi_flags)) {
			/* We know that the bi_io_vec layout is the same for
			 * both 'first' and 'i', so we just compare them.
			 * All vec entries are PAGE_SIZE;
			 */
			for (j = 0; j < vcnt; j++)
				if (memcmp(page_address(fbio->bi_io_vec[j].bv_page),
					   page_address(tbio->bi_io_vec[j].bv_page),
1925
					   fbio->bi_io_vec[j].bv_len))
1926 1927 1928 1929
					break;
			if (j == vcnt)
				continue;
			mddev->resync_mismatches += r10_bio->sectors;
1930 1931 1932
			if (test_bit(MD_RECOVERY_CHECK, &mddev->recovery))
				/* Don't fix anything. */
				continue;
1933
		}
1934 1935
		/* Ok, we need to write this bio, either to correct an
		 * inconsistency or to correct an unreadable block.
L
Linus Torvalds 已提交
1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969
		 * First we need to fixup bv_offset, bv_len and
		 * bi_vecs, as the read request might have corrupted these
		 */
		tbio->bi_vcnt = vcnt;
		tbio->bi_size = r10_bio->sectors << 9;
		tbio->bi_idx = 0;
		tbio->bi_phys_segments = 0;
		tbio->bi_flags &= ~(BIO_POOL_MASK - 1);
		tbio->bi_flags |= 1 << BIO_UPTODATE;
		tbio->bi_next = NULL;
		tbio->bi_rw = WRITE;
		tbio->bi_private = r10_bio;
		tbio->bi_sector = r10_bio->devs[i].addr;

		for (j=0; j < vcnt ; j++) {
			tbio->bi_io_vec[j].bv_offset = 0;
			tbio->bi_io_vec[j].bv_len = PAGE_SIZE;

			memcpy(page_address(tbio->bi_io_vec[j].bv_page),
			       page_address(fbio->bi_io_vec[j].bv_page),
			       PAGE_SIZE);
		}
		tbio->bi_end_io = end_sync_write;

		d = r10_bio->devs[i].devnum;
		atomic_inc(&conf->mirrors[d].rdev->nr_pending);
		atomic_inc(&r10_bio->remaining);
		md_sync_acct(conf->mirrors[d].rdev->bdev, tbio->bi_size >> 9);

		tbio->bi_sector += conf->mirrors[d].rdev->data_offset;
		tbio->bi_bdev = conf->mirrors[d].rdev->bdev;
		generic_make_request(tbio);
	}

1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991
	/* Now write out to any replacement devices
	 * that are active
	 */
	for (i = 0; i < conf->copies; i++) {
		int j, d;

		tbio = r10_bio->devs[i].repl_bio;
		if (!tbio || !tbio->bi_end_io)
			continue;
		if (r10_bio->devs[i].bio->bi_end_io != end_sync_write
		    && r10_bio->devs[i].bio != fbio)
			for (j = 0; j < vcnt; j++)
				memcpy(page_address(tbio->bi_io_vec[j].bv_page),
				       page_address(fbio->bi_io_vec[j].bv_page),
				       PAGE_SIZE);
		d = r10_bio->devs[i].devnum;
		atomic_inc(&r10_bio->remaining);
		md_sync_acct(conf->mirrors[d].replacement->bdev,
			     tbio->bi_size >> 9);
		generic_make_request(tbio);
	}

L
Linus Torvalds 已提交
1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008
done:
	if (atomic_dec_and_test(&r10_bio->remaining)) {
		md_done_sync(mddev, r10_bio->sectors, 1);
		put_buf(r10_bio);
	}
}

/*
 * Now for the recovery code.
 * Recovery happens across physical sectors.
 * We recover all non-is_sync drives by finding the virtual address of
 * each, and then choose a working drive that also has that virt address.
 * There is a separate r10_bio for each non-in_sync drive.
 * Only the first two slots are in use. The first for reading,
 * The second for writing.
 *
 */
2009
static void fix_recovery_read_error(struct r10bio *r10_bio)
2010 2011 2012 2013 2014 2015 2016 2017
{
	/* We got a read error during recovery.
	 * We repeat the read in smaller page-sized sections.
	 * If a read succeeds, write it to the new device or record
	 * a bad block if we cannot.
	 * If a read fails, record a bad block on both old and
	 * new devices.
	 */
2018
	struct mddev *mddev = r10_bio->mddev;
2019
	struct r10conf *conf = mddev->private;
2020 2021 2022 2023 2024 2025 2026 2027 2028
	struct bio *bio = r10_bio->devs[0].bio;
	sector_t sect = 0;
	int sectors = r10_bio->sectors;
	int idx = 0;
	int dr = r10_bio->devs[0].devnum;
	int dw = r10_bio->devs[1].devnum;

	while (sectors) {
		int s = sectors;
2029
		struct md_rdev *rdev;
2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050
		sector_t addr;
		int ok;

		if (s > (PAGE_SIZE>>9))
			s = PAGE_SIZE >> 9;

		rdev = conf->mirrors[dr].rdev;
		addr = r10_bio->devs[0].addr + sect,
		ok = sync_page_io(rdev,
				  addr,
				  s << 9,
				  bio->bi_io_vec[idx].bv_page,
				  READ, false);
		if (ok) {
			rdev = conf->mirrors[dw].rdev;
			addr = r10_bio->devs[1].addr + sect;
			ok = sync_page_io(rdev,
					  addr,
					  s << 9,
					  bio->bi_io_vec[idx].bv_page,
					  WRITE, false);
2051
			if (!ok) {
2052
				set_bit(WriteErrorSeen, &rdev->flags);
2053 2054 2055 2056 2057
				if (!test_and_set_bit(WantReplacement,
						      &rdev->flags))
					set_bit(MD_RECOVERY_NEEDED,
						&rdev->mddev->recovery);
			}
2058 2059 2060 2061 2062 2063 2064 2065 2066 2067
		}
		if (!ok) {
			/* We don't worry if we cannot set a bad block -
			 * it really is bad so there is no loss in not
			 * recording it yet
			 */
			rdev_set_badblocks(rdev, addr, s, 0);

			if (rdev != conf->mirrors[dw].rdev) {
				/* need bad block on destination too */
2068
				struct md_rdev *rdev2 = conf->mirrors[dw].rdev;
2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091
				addr = r10_bio->devs[1].addr + sect;
				ok = rdev_set_badblocks(rdev2, addr, s, 0);
				if (!ok) {
					/* just abort the recovery */
					printk(KERN_NOTICE
					       "md/raid10:%s: recovery aborted"
					       " due to read error\n",
					       mdname(mddev));

					conf->mirrors[dw].recovery_disabled
						= mddev->recovery_disabled;
					set_bit(MD_RECOVERY_INTR,
						&mddev->recovery);
					break;
				}
			}
		}

		sectors -= s;
		sect += s;
		idx++;
	}
}
L
Linus Torvalds 已提交
2092

2093
static void recovery_request_write(struct mddev *mddev, struct r10bio *r10_bio)
L
Linus Torvalds 已提交
2094
{
2095
	struct r10conf *conf = mddev->private;
2096
	int d;
2097
	struct bio *wbio, *wbio2;
L
Linus Torvalds 已提交
2098

2099 2100 2101 2102 2103 2104
	if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) {
		fix_recovery_read_error(r10_bio);
		end_sync_request(r10_bio);
		return;
	}

2105 2106
	/*
	 * share the pages with the first bio
L
Linus Torvalds 已提交
2107 2108 2109
	 * and submit the write request
	 */
	d = r10_bio->devs[1].devnum;
2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122
	wbio = r10_bio->devs[1].bio;
	wbio2 = r10_bio->devs[1].repl_bio;
	if (wbio->bi_end_io) {
		atomic_inc(&conf->mirrors[d].rdev->nr_pending);
		md_sync_acct(conf->mirrors[d].rdev->bdev, wbio->bi_size >> 9);
		generic_make_request(wbio);
	}
	if (wbio2 && wbio2->bi_end_io) {
		atomic_inc(&conf->mirrors[d].replacement->nr_pending);
		md_sync_acct(conf->mirrors[d].replacement->bdev,
			     wbio2->bi_size >> 9);
		generic_make_request(wbio2);
	}
L
Linus Torvalds 已提交
2123 2124 2125
}


2126 2127 2128 2129 2130 2131
/*
 * Used by fix_read_error() to decay the per rdev read_errors.
 * We halve the read error count for every hour that has elapsed
 * since the last recorded read error.
 *
 */
2132
static void check_decay_read_errors(struct mddev *mddev, struct md_rdev *rdev)
2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162
{
	struct timespec cur_time_mon;
	unsigned long hours_since_last;
	unsigned int read_errors = atomic_read(&rdev->read_errors);

	ktime_get_ts(&cur_time_mon);

	if (rdev->last_read_error.tv_sec == 0 &&
	    rdev->last_read_error.tv_nsec == 0) {
		/* first time we've seen a read error */
		rdev->last_read_error = cur_time_mon;
		return;
	}

	hours_since_last = (cur_time_mon.tv_sec -
			    rdev->last_read_error.tv_sec) / 3600;

	rdev->last_read_error = cur_time_mon;

	/*
	 * if hours_since_last is > the number of bits in read_errors
	 * just set read errors to 0. We do this to avoid
	 * overflowing the shift of read_errors by hours_since_last.
	 */
	if (hours_since_last >= 8 * sizeof(read_errors))
		atomic_set(&rdev->read_errors, 0);
	else
		atomic_set(&rdev->read_errors, read_errors >> hours_since_last);
}

2163
static int r10_sync_page_io(struct md_rdev *rdev, sector_t sector,
2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174
			    int sectors, struct page *page, int rw)
{
	sector_t first_bad;
	int bad_sectors;

	if (is_badblock(rdev, sector, sectors, &first_bad, &bad_sectors)
	    && (rw == READ || test_bit(WriteErrorSeen, &rdev->flags)))
		return -1;
	if (sync_page_io(rdev, sector, sectors << 9, page, rw, false))
		/* success */
		return 1;
2175
	if (rw == WRITE) {
2176
		set_bit(WriteErrorSeen, &rdev->flags);
2177 2178 2179 2180
		if (!test_and_set_bit(WantReplacement, &rdev->flags))
			set_bit(MD_RECOVERY_NEEDED,
				&rdev->mddev->recovery);
	}
2181 2182 2183 2184 2185 2186
	/* need to record an error - either for the block or the device */
	if (!rdev_set_badblocks(rdev, sector, sectors, 0))
		md_error(rdev->mddev, rdev);
	return 0;
}

L
Linus Torvalds 已提交
2187 2188 2189 2190 2191
/*
 * This is a kernel thread which:
 *
 *	1.	Retries failed read operations on working mirrors.
 *	2.	Updates the raid superblock when problems encounter.
2192
 *	3.	Performs writes following reads for array synchronising.
L
Linus Torvalds 已提交
2193 2194
 */

2195
static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10bio *r10_bio)
2196 2197 2198
{
	int sect = 0; /* Offset from r10_bio->sector */
	int sectors = r10_bio->sectors;
2199
	struct md_rdev*rdev;
2200
	int max_read_errors = atomic_read(&mddev->max_corr_read_errors);
2201
	int d = r10_bio->devs[r10_bio->read_slot].devnum;
2202

2203 2204 2205 2206
	/* still own a reference to this rdev, so it cannot
	 * have been cleared recently.
	 */
	rdev = conf->mirrors[d].rdev;
2207

2208 2209 2210 2211
	if (test_bit(Faulty, &rdev->flags))
		/* drive has already been failed, just ignore any
		   more fix_read_error() attempts */
		return;
2212

2213 2214 2215 2216 2217
	check_decay_read_errors(mddev, rdev);
	atomic_inc(&rdev->read_errors);
	if (atomic_read(&rdev->read_errors) > max_read_errors) {
		char b[BDEVNAME_SIZE];
		bdevname(rdev->bdev, b);
2218

2219 2220 2221 2222 2223 2224 2225 2226 2227
		printk(KERN_NOTICE
		       "md/raid10:%s: %s: Raid device exceeded "
		       "read_error threshold [cur %d:max %d]\n",
		       mdname(mddev), b,
		       atomic_read(&rdev->read_errors), max_read_errors);
		printk(KERN_NOTICE
		       "md/raid10:%s: %s: Failing raid device\n",
		       mdname(mddev), b);
		md_error(mddev, conf->mirrors[d].rdev);
2228
		r10_bio->devs[r10_bio->read_slot].bio = IO_BLOCKED;
2229
		return;
2230 2231
	}

2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242
	while(sectors) {
		int s = sectors;
		int sl = r10_bio->read_slot;
		int success = 0;
		int start;

		if (s > (PAGE_SIZE>>9))
			s = PAGE_SIZE >> 9;

		rcu_read_lock();
		do {
2243 2244 2245
			sector_t first_bad;
			int bad_sectors;

2246
			d = r10_bio->devs[sl].devnum;
2247 2248
			rdev = rcu_dereference(conf->mirrors[d].rdev);
			if (rdev &&
2249
			    !test_bit(Unmerged, &rdev->flags) &&
2250 2251 2252
			    test_bit(In_sync, &rdev->flags) &&
			    is_badblock(rdev, r10_bio->devs[sl].addr + sect, s,
					&first_bad, &bad_sectors) == 0) {
2253 2254
				atomic_inc(&rdev->nr_pending);
				rcu_read_unlock();
2255
				success = sync_page_io(rdev,
2256
						       r10_bio->devs[sl].addr +
J
Jonathan Brassow 已提交
2257
						       sect,
2258
						       s<<9,
J
Jonathan Brassow 已提交
2259
						       conf->tmppage, READ, false);
2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271
				rdev_dec_pending(rdev, mddev);
				rcu_read_lock();
				if (success)
					break;
			}
			sl++;
			if (sl == conf->copies)
				sl = 0;
		} while (!success && sl != r10_bio->read_slot);
		rcu_read_unlock();

		if (!success) {
2272 2273 2274 2275
			/* Cannot read from anywhere, just mark the block
			 * as bad on the first device to discourage future
			 * reads.
			 */
2276
			int dn = r10_bio->devs[r10_bio->read_slot].devnum;
2277 2278 2279 2280 2281 2282
			rdev = conf->mirrors[dn].rdev;

			if (!rdev_set_badblocks(
				    rdev,
				    r10_bio->devs[r10_bio->read_slot].addr
				    + sect,
2283
				    s, 0)) {
2284
				md_error(mddev, rdev);
2285 2286 2287
				r10_bio->devs[r10_bio->read_slot].bio
					= IO_BLOCKED;
			}
2288 2289 2290 2291 2292 2293 2294
			break;
		}

		start = sl;
		/* write it back and re-read */
		rcu_read_lock();
		while (sl != r10_bio->read_slot) {
2295
			char b[BDEVNAME_SIZE];
2296

2297 2298 2299 2300 2301
			if (sl==0)
				sl = conf->copies;
			sl--;
			d = r10_bio->devs[sl].devnum;
			rdev = rcu_dereference(conf->mirrors[d].rdev);
2302
			if (!rdev ||
2303
			    test_bit(Unmerged, &rdev->flags) ||
2304 2305 2306 2307 2308
			    !test_bit(In_sync, &rdev->flags))
				continue;

			atomic_inc(&rdev->nr_pending);
			rcu_read_unlock();
2309 2310 2311 2312
			if (r10_sync_page_io(rdev,
					     r10_bio->devs[sl].addr +
					     sect,
					     s<<9, conf->tmppage, WRITE)
2313 2314 2315 2316 2317 2318 2319 2320
			    == 0) {
				/* Well, this device is dead */
				printk(KERN_NOTICE
				       "md/raid10:%s: read correction "
				       "write failed"
				       " (%d sectors at %llu on %s)\n",
				       mdname(mddev), s,
				       (unsigned long long)(
2321 2322 2323
					       sect +
					       choose_data_offset(r10_bio,
								  rdev)),
2324 2325 2326 2327 2328
				       bdevname(rdev->bdev, b));
				printk(KERN_NOTICE "md/raid10:%s: %s: failing "
				       "drive\n",
				       mdname(mddev),
				       bdevname(rdev->bdev, b));
2329
			}
2330 2331
			rdev_dec_pending(rdev, mddev);
			rcu_read_lock();
2332 2333 2334
		}
		sl = start;
		while (sl != r10_bio->read_slot) {
2335
			char b[BDEVNAME_SIZE];
2336

2337 2338 2339 2340 2341
			if (sl==0)
				sl = conf->copies;
			sl--;
			d = r10_bio->devs[sl].devnum;
			rdev = rcu_dereference(conf->mirrors[d].rdev);
2342 2343 2344
			if (!rdev ||
			    !test_bit(In_sync, &rdev->flags))
				continue;
2345

2346 2347
			atomic_inc(&rdev->nr_pending);
			rcu_read_unlock();
2348 2349 2350 2351 2352 2353
			switch (r10_sync_page_io(rdev,
					     r10_bio->devs[sl].addr +
					     sect,
					     s<<9, conf->tmppage,
						 READ)) {
			case 0:
2354 2355 2356 2357 2358 2359 2360
				/* Well, this device is dead */
				printk(KERN_NOTICE
				       "md/raid10:%s: unable to read back "
				       "corrected sectors"
				       " (%d sectors at %llu on %s)\n",
				       mdname(mddev), s,
				       (unsigned long long)(
2361 2362
					       sect +
					       choose_data_offset(r10_bio, rdev)),
2363 2364 2365 2366 2367
				       bdevname(rdev->bdev, b));
				printk(KERN_NOTICE "md/raid10:%s: %s: failing "
				       "drive\n",
				       mdname(mddev),
				       bdevname(rdev->bdev, b));
2368 2369
				break;
			case 1:
2370 2371 2372 2373 2374
				printk(KERN_INFO
				       "md/raid10:%s: read error corrected"
				       " (%d sectors at %llu on %s)\n",
				       mdname(mddev), s,
				       (unsigned long long)(
2375 2376
					       sect +
					       choose_data_offset(r10_bio, rdev)),
2377 2378
				       bdevname(rdev->bdev, b));
				atomic_add(s, &rdev->corrected_errors);
2379
			}
2380 2381 2382

			rdev_dec_pending(rdev, mddev);
			rcu_read_lock();
2383 2384 2385 2386 2387 2388 2389 2390
		}
		rcu_read_unlock();

		sectors -= s;
		sect += s;
	}
}

2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409
static void bi_complete(struct bio *bio, int error)
{
	complete((struct completion *)bio->bi_private);
}

static int submit_bio_wait(int rw, struct bio *bio)
{
	struct completion event;
	rw |= REQ_SYNC;

	init_completion(&event);
	bio->bi_private = &event;
	bio->bi_end_io = bi_complete;
	submit_bio(rw, bio);
	wait_for_completion(&event);

	return test_bit(BIO_UPTODATE, &bio->bi_flags);
}

2410
static int narrow_write_error(struct r10bio *r10_bio, int i)
2411 2412
{
	struct bio *bio = r10_bio->master_bio;
2413
	struct mddev *mddev = r10_bio->mddev;
2414
	struct r10conf *conf = mddev->private;
2415
	struct md_rdev *rdev = conf->mirrors[r10_bio->devs[i].devnum].rdev;
2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449
	/* bio has the data to be written to slot 'i' where
	 * we just recently had a write error.
	 * We repeatedly clone the bio and trim down to one block,
	 * then try the write.  Where the write fails we record
	 * a bad block.
	 * It is conceivable that the bio doesn't exactly align with
	 * blocks.  We must handle this.
	 *
	 * We currently own a reference to the rdev.
	 */

	int block_sectors;
	sector_t sector;
	int sectors;
	int sect_to_write = r10_bio->sectors;
	int ok = 1;

	if (rdev->badblocks.shift < 0)
		return 0;

	block_sectors = 1 << rdev->badblocks.shift;
	sector = r10_bio->sector;
	sectors = ((r10_bio->sector + block_sectors)
		   & ~(sector_t)(block_sectors - 1))
		- sector;

	while (sect_to_write) {
		struct bio *wbio;
		if (sectors > sect_to_write)
			sectors = sect_to_write;
		/* Write at 'sector' for 'sectors' */
		wbio = bio_clone_mddev(bio, GFP_NOIO, mddev);
		md_trim_bio(wbio, sector - bio->bi_sector, sectors);
		wbio->bi_sector = (r10_bio->devs[i].addr+
2450
				   choose_data_offset(r10_bio, rdev) +
2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466
				   (sector - r10_bio->sector));
		wbio->bi_bdev = rdev->bdev;
		if (submit_bio_wait(WRITE, wbio) == 0)
			/* Failure! */
			ok = rdev_set_badblocks(rdev, sector,
						sectors, 0)
				&& ok;

		bio_put(wbio);
		sect_to_write -= sectors;
		sector += sectors;
		sectors = block_sectors;
	}
	return ok;
}

2467
static void handle_read_error(struct mddev *mddev, struct r10bio *r10_bio)
2468 2469 2470
{
	int slot = r10_bio->read_slot;
	struct bio *bio;
2471
	struct r10conf *conf = mddev->private;
2472
	struct md_rdev *rdev = r10_bio->devs[slot].rdev;
2473 2474
	char b[BDEVNAME_SIZE];
	unsigned long do_sync;
2475
	int max_sectors;
2476 2477 2478 2479 2480 2481 2482 2483 2484

	/* we got a read error. Maybe the drive is bad.  Maybe just
	 * the block and we can fix it.
	 * We freeze all other IO, and try reading the block from
	 * other devices.  When we find one, we re-write
	 * and check it that fixes the read error.
	 * This is all done synchronously while the array is
	 * frozen.
	 */
2485 2486 2487 2488 2489
	bio = r10_bio->devs[slot].bio;
	bdevname(bio->bi_bdev, b);
	bio_put(bio);
	r10_bio->devs[slot].bio = NULL;

2490 2491 2492 2493
	if (mddev->ro == 0) {
		freeze_array(conf);
		fix_read_error(conf, mddev, r10_bio);
		unfreeze_array(conf);
2494 2495 2496
	} else
		r10_bio->devs[slot].bio = IO_BLOCKED;

2497
	rdev_dec_pending(rdev, mddev);
2498

2499
read_more:
2500 2501
	rdev = read_balance(conf, r10_bio, &max_sectors);
	if (rdev == NULL) {
2502 2503
		printk(KERN_ALERT "md/raid10:%s: %s: unrecoverable I/O"
		       " read error for block %llu\n",
2504
		       mdname(mddev), b,
2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520
		       (unsigned long long)r10_bio->sector);
		raid_end_bio_io(r10_bio);
		return;
	}

	do_sync = (r10_bio->master_bio->bi_rw & REQ_SYNC);
	slot = r10_bio->read_slot;
	printk_ratelimited(
		KERN_ERR
		"md/raid10:%s: %s: redirecting"
		"sector %llu to another mirror\n",
		mdname(mddev),
		bdevname(rdev->bdev, b),
		(unsigned long long)r10_bio->sector);
	bio = bio_clone_mddev(r10_bio->master_bio,
			      GFP_NOIO, mddev);
2521 2522 2523
	md_trim_bio(bio,
		    r10_bio->sector - bio->bi_sector,
		    max_sectors);
2524
	r10_bio->devs[slot].bio = bio;
2525
	r10_bio->devs[slot].rdev = rdev;
2526
	bio->bi_sector = r10_bio->devs[slot].addr
2527
		+ choose_data_offset(r10_bio, rdev);
2528 2529 2530 2531
	bio->bi_bdev = rdev->bdev;
	bio->bi_rw = READ | do_sync;
	bio->bi_private = r10_bio;
	bio->bi_end_io = raid10_end_read_request;
2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561
	if (max_sectors < r10_bio->sectors) {
		/* Drat - have to split this up more */
		struct bio *mbio = r10_bio->master_bio;
		int sectors_handled =
			r10_bio->sector + max_sectors
			- mbio->bi_sector;
		r10_bio->sectors = max_sectors;
		spin_lock_irq(&conf->device_lock);
		if (mbio->bi_phys_segments == 0)
			mbio->bi_phys_segments = 2;
		else
			mbio->bi_phys_segments++;
		spin_unlock_irq(&conf->device_lock);
		generic_make_request(bio);

		r10_bio = mempool_alloc(conf->r10bio_pool,
					GFP_NOIO);
		r10_bio->master_bio = mbio;
		r10_bio->sectors = (mbio->bi_size >> 9)
			- sectors_handled;
		r10_bio->state = 0;
		set_bit(R10BIO_ReadError,
			&r10_bio->state);
		r10_bio->mddev = mddev;
		r10_bio->sector = mbio->bi_sector
			+ sectors_handled;

		goto read_more;
	} else
		generic_make_request(bio);
2562 2563
}

2564
static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio)
2565 2566 2567 2568
{
	/* Some sort of write request has finished and it
	 * succeeded in writing where we thought there was a
	 * bad block.  So forget the bad block.
2569 2570
	 * Or possibly if failed and we need to record
	 * a bad block.
2571 2572
	 */
	int m;
2573
	struct md_rdev *rdev;
2574 2575 2576

	if (test_bit(R10BIO_IsSync, &r10_bio->state) ||
	    test_bit(R10BIO_IsRecover, &r10_bio->state)) {
2577 2578 2579 2580 2581 2582
		for (m = 0; m < conf->copies; m++) {
			int dev = r10_bio->devs[m].devnum;
			rdev = conf->mirrors[dev].rdev;
			if (r10_bio->devs[m].bio == NULL)
				continue;
			if (test_bit(BIO_UPTODATE,
2583 2584 2585 2586
				     &r10_bio->devs[m].bio->bi_flags)) {
				rdev_clear_badblocks(
					rdev,
					r10_bio->devs[m].addr,
2587
					r10_bio->sectors, 0);
2588 2589 2590 2591 2592 2593
			} else {
				if (!rdev_set_badblocks(
					    rdev,
					    r10_bio->devs[m].addr,
					    r10_bio->sectors, 0))
					md_error(conf->mddev, rdev);
2594
			}
2595 2596 2597 2598 2599 2600 2601 2602
			rdev = conf->mirrors[dev].replacement;
			if (r10_bio->devs[m].repl_bio == NULL)
				continue;
			if (test_bit(BIO_UPTODATE,
				     &r10_bio->devs[m].repl_bio->bi_flags)) {
				rdev_clear_badblocks(
					rdev,
					r10_bio->devs[m].addr,
2603
					r10_bio->sectors, 0);
2604 2605 2606 2607 2608 2609 2610
			} else {
				if (!rdev_set_badblocks(
					    rdev,
					    r10_bio->devs[m].addr,
					    r10_bio->sectors, 0))
					md_error(conf->mddev, rdev);
			}
2611
		}
2612 2613
		put_buf(r10_bio);
	} else {
2614 2615 2616 2617 2618
		for (m = 0; m < conf->copies; m++) {
			int dev = r10_bio->devs[m].devnum;
			struct bio *bio = r10_bio->devs[m].bio;
			rdev = conf->mirrors[dev].rdev;
			if (bio == IO_MADE_GOOD) {
2619 2620 2621
				rdev_clear_badblocks(
					rdev,
					r10_bio->devs[m].addr,
2622
					r10_bio->sectors, 0);
2623
				rdev_dec_pending(rdev, conf->mddev);
2624 2625 2626 2627 2628 2629 2630 2631
			} else if (bio != NULL &&
				   !test_bit(BIO_UPTODATE, &bio->bi_flags)) {
				if (!narrow_write_error(r10_bio, m)) {
					md_error(conf->mddev, rdev);
					set_bit(R10BIO_Degraded,
						&r10_bio->state);
				}
				rdev_dec_pending(rdev, conf->mddev);
2632
			}
2633 2634
			bio = r10_bio->devs[m].repl_bio;
			rdev = conf->mirrors[dev].replacement;
2635
			if (rdev && bio == IO_MADE_GOOD) {
2636 2637 2638
				rdev_clear_badblocks(
					rdev,
					r10_bio->devs[m].addr,
2639
					r10_bio->sectors, 0);
2640 2641
				rdev_dec_pending(rdev, conf->mddev);
			}
2642 2643 2644 2645
		}
		if (test_bit(R10BIO_WriteError,
			     &r10_bio->state))
			close_write(r10_bio);
2646 2647 2648 2649
		raid_end_bio_io(r10_bio);
	}
}

2650
static void raid10d(struct mddev *mddev)
L
Linus Torvalds 已提交
2651
{
2652
	struct r10bio *r10_bio;
L
Linus Torvalds 已提交
2653
	unsigned long flags;
2654
	struct r10conf *conf = mddev->private;
L
Linus Torvalds 已提交
2655
	struct list_head *head = &conf->retry_list;
2656
	struct blk_plug plug;
L
Linus Torvalds 已提交
2657 2658 2659

	md_check_recovery(mddev);

2660
	blk_start_plug(&plug);
L
Linus Torvalds 已提交
2661
	for (;;) {
2662

J
Jens Axboe 已提交
2663
		flush_pending_writes(conf);
2664

2665 2666 2667
		spin_lock_irqsave(&conf->device_lock, flags);
		if (list_empty(head)) {
			spin_unlock_irqrestore(&conf->device_lock, flags);
L
Linus Torvalds 已提交
2668
			break;
2669
		}
2670
		r10_bio = list_entry(head->prev, struct r10bio, retry_list);
L
Linus Torvalds 已提交
2671
		list_del(head->prev);
2672
		conf->nr_queued--;
L
Linus Torvalds 已提交
2673 2674 2675
		spin_unlock_irqrestore(&conf->device_lock, flags);

		mddev = r10_bio->mddev;
2676
		conf = mddev->private;
2677 2678
		if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
		    test_bit(R10BIO_WriteError, &r10_bio->state))
2679
			handle_write_completed(conf, r10_bio);
N
NeilBrown 已提交
2680 2681
		else if (test_bit(R10BIO_IsReshape, &r10_bio->state))
			reshape_request_write(mddev, r10_bio);
2682
		else if (test_bit(R10BIO_IsSync, &r10_bio->state))
L
Linus Torvalds 已提交
2683
			sync_request_write(mddev, r10_bio);
J
Jens Axboe 已提交
2684
		else if (test_bit(R10BIO_IsRecover, &r10_bio->state))
L
Linus Torvalds 已提交
2685
			recovery_request_write(mddev, r10_bio);
2686
		else if (test_bit(R10BIO_ReadError, &r10_bio->state))
2687
			handle_read_error(mddev, r10_bio);
2688 2689 2690 2691 2692 2693 2694
		else {
			/* just a partial read to be scheduled from a
			 * separate context
			 */
			int slot = r10_bio->read_slot;
			generic_make_request(r10_bio->devs[slot].bio);
		}
2695

N
NeilBrown 已提交
2696
		cond_resched();
2697 2698
		if (mddev->flags & ~(1<<MD_CHANGE_PENDING))
			md_check_recovery(mddev);
L
Linus Torvalds 已提交
2699
	}
2700
	blk_finish_plug(&plug);
L
Linus Torvalds 已提交
2701 2702 2703
}


2704
static int init_resync(struct r10conf *conf)
L
Linus Torvalds 已提交
2705 2706
{
	int buffs;
2707
	int i;
L
Linus Torvalds 已提交
2708 2709

	buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE;
2710
	BUG_ON(conf->r10buf_pool);
2711
	conf->have_replacement = 0;
2712
	for (i = 0; i < conf->geo.raid_disks; i++)
2713 2714
		if (conf->mirrors[i].replacement)
			conf->have_replacement = 1;
L
Linus Torvalds 已提交
2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753
	conf->r10buf_pool = mempool_create(buffs, r10buf_pool_alloc, r10buf_pool_free, conf);
	if (!conf->r10buf_pool)
		return -ENOMEM;
	conf->next_resync = 0;
	return 0;
}

/*
 * perform a "sync" on one "block"
 *
 * We need to make sure that no normal I/O request - particularly write
 * requests - conflict with active sync requests.
 *
 * This is achieved by tracking pending requests and a 'barrier' concept
 * that can be installed to exclude normal IO requests.
 *
 * Resync and recovery are handled very differently.
 * We differentiate by looking at MD_RECOVERY_SYNC in mddev->recovery.
 *
 * For resync, we iterate over virtual addresses, read all copies,
 * and update if there are differences.  If only one copy is live,
 * skip it.
 * For recovery, we iterate over physical addresses, read a good
 * value for each non-in_sync drive, and over-write.
 *
 * So, for recovery we may have several outstanding complex requests for a
 * given address, one for each out-of-sync device.  We model this by allocating
 * a number of r10_bio structures, one for each out-of-sync device.
 * As we setup these structures, we collect all bio's together into a list
 * which we then process collectively to add pages, and then process again
 * to pass to generic_make_request.
 *
 * The r10_bio structures are linked using a borrowed master_bio pointer.
 * This link is counted in ->remaining.  When the r10_bio that points to NULL
 * has its remaining count decremented to 0, the whole complex operation
 * is complete.
 *
 */

2754
static sector_t sync_request(struct mddev *mddev, sector_t sector_nr,
2755
			     int *skipped, int go_faster)
L
Linus Torvalds 已提交
2756
{
2757
	struct r10conf *conf = mddev->private;
2758
	struct r10bio *r10_bio;
L
Linus Torvalds 已提交
2759 2760 2761
	struct bio *biolist = NULL, *bio;
	sector_t max_sector, nr_sectors;
	int i;
2762
	int max_sync;
N
NeilBrown 已提交
2763
	sector_t sync_blocks;
L
Linus Torvalds 已提交
2764 2765
	sector_t sectors_skipped = 0;
	int chunks_skipped = 0;
2766
	sector_t chunk_mask = conf->geo.chunk_mask;
L
Linus Torvalds 已提交
2767 2768 2769

	if (!conf->r10buf_pool)
		if (init_resync(conf))
2770
			return 0;
L
Linus Torvalds 已提交
2771 2772

 skipped:
A
Andre Noll 已提交
2773
	max_sector = mddev->dev_sectors;
N
NeilBrown 已提交
2774 2775
	if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) ||
	    test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
L
Linus Torvalds 已提交
2776 2777
		max_sector = mddev->resync_max_sectors;
	if (sector_nr >= max_sector) {
2778 2779 2780 2781 2782 2783 2784 2785 2786
		/* If we aborted, we need to abort the
		 * sync on the 'current' bitmap chucks (there can
		 * be several when recovering multiple devices).
		 * as we may have started syncing it but not finished.
		 * We can find the current address in
		 * mddev->curr_resync, but for recovery,
		 * we need to convert that to several
		 * virtual addresses.
		 */
N
NeilBrown 已提交
2787 2788 2789 2790 2791
		if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {
			end_reshape(conf);
			return 0;
		}

2792 2793 2794 2795
		if (mddev->curr_resync < max_sector) { /* aborted */
			if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery))
				bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
						&sync_blocks, 1);
2796
			else for (i = 0; i < conf->geo.raid_disks; i++) {
2797 2798 2799 2800 2801
				sector_t sect =
					raid10_find_virt(conf, mddev->curr_resync, i);
				bitmap_end_sync(mddev->bitmap, sect,
						&sync_blocks, 1);
			}
2802 2803 2804 2805 2806 2807 2808 2809
		} else {
			/* completed sync */
			if ((!mddev->bitmap || conf->fullsync)
			    && conf->have_replacement
			    && test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
				/* Completed a full sync so the replacements
				 * are now fully recovered.
				 */
2810
				for (i = 0; i < conf->geo.raid_disks; i++)
2811 2812 2813 2814 2815
					if (conf->mirrors[i].replacement)
						conf->mirrors[i].replacement
							->recovery_offset
							= MaxSector;
			}
2816
			conf->fullsync = 0;
2817
		}
2818
		bitmap_close_sync(mddev->bitmap);
L
Linus Torvalds 已提交
2819
		close_sync(conf);
2820
		*skipped = 1;
L
Linus Torvalds 已提交
2821 2822
		return sectors_skipped;
	}
N
NeilBrown 已提交
2823 2824 2825 2826

	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
		return reshape_request(mddev, sector_nr, skipped);

2827
	if (chunks_skipped >= conf->geo.raid_disks) {
L
Linus Torvalds 已提交
2828 2829 2830
		/* if there has been nothing to do on any drive,
		 * then there is nothing to do at all..
		 */
2831 2832
		*skipped = 1;
		return (max_sector - sector_nr) + sectors_skipped;
L
Linus Torvalds 已提交
2833 2834
	}

2835 2836 2837
	if (max_sector > mddev->resync_max)
		max_sector = mddev->resync_max; /* Don't do IO beyond here */

L
Linus Torvalds 已提交
2838 2839 2840
	/* make sure whole request will fit in a chunk - if chunks
	 * are meaningful
	 */
2841 2842 2843
	if (conf->geo.near_copies < conf->geo.raid_disks &&
	    max_sector > (sector_nr | chunk_mask))
		max_sector = (sector_nr | chunk_mask) + 1;
L
Linus Torvalds 已提交
2844 2845 2846 2847
	/*
	 * If there is non-resync activity waiting for us then
	 * put in a delay to throttle resync.
	 */
2848
	if (!go_faster && conf->nr_waiting)
L
Linus Torvalds 已提交
2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865
		msleep_interruptible(1000);

	/* Again, very different code for resync and recovery.
	 * Both must result in an r10bio with a list of bios that
	 * have bi_end_io, bi_sector, bi_bdev set,
	 * and bi_private set to the r10bio.
	 * For recovery, we may actually create several r10bios
	 * with 2 bios in each, that correspond to the bios in the main one.
	 * In this case, the subordinate r10bios link back through a
	 * borrowed master_bio pointer, and the counter in the master
	 * includes a ref from each subordinate.
	 */
	/* First, we decide what to do and set ->bi_end_io
	 * To end_sync_read if we want to read, and
	 * end_sync_write if we will want to write.
	 */

2866
	max_sync = RESYNC_PAGES << (PAGE_SHIFT-9);
L
Linus Torvalds 已提交
2867 2868
	if (!test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
		/* recovery... the complicated one */
2869
		int j;
L
Linus Torvalds 已提交
2870 2871
		r10_bio = NULL;

2872
		for (i = 0 ; i < conf->geo.raid_disks; i++) {
2873
			int still_degraded;
2874
			struct r10bio *rb2;
2875 2876
			sector_t sect;
			int must_sync;
2877
			int any_working;
2878 2879 2880 2881 2882 2883 2884 2885
			struct mirror_info *mirror = &conf->mirrors[i];

			if ((mirror->rdev == NULL ||
			     test_bit(In_sync, &mirror->rdev->flags))
			    &&
			    (mirror->replacement == NULL ||
			     test_bit(Faulty,
				      &mirror->replacement->flags)))
2886
				continue;
L
Linus Torvalds 已提交
2887

2888 2889 2890 2891
			still_degraded = 0;
			/* want to reconstruct this device */
			rb2 = r10_bio;
			sect = raid10_find_virt(conf, sector_nr, i);
2892 2893 2894
			/* Unless we are doing a full sync, or a replacement
			 * we only need to recover the block if it is set in
			 * the bitmap
2895 2896 2897 2898 2899 2900
			 */
			must_sync = bitmap_start_sync(mddev->bitmap, sect,
						      &sync_blocks, 1);
			if (sync_blocks < max_sync)
				max_sync = sync_blocks;
			if (!must_sync &&
2901
			    mirror->replacement == NULL &&
2902 2903 2904 2905 2906 2907 2908
			    !conf->fullsync) {
				/* yep, skip the sync_blocks here, but don't assume
				 * that there will never be anything to do here
				 */
				chunks_skipped = -1;
				continue;
			}
2909

2910 2911 2912
			r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
			raise_barrier(conf, rb2 != NULL);
			atomic_set(&r10_bio->remaining, 0);
2913

2914 2915 2916 2917 2918 2919
			r10_bio->master_bio = (struct bio*)rb2;
			if (rb2)
				atomic_inc(&rb2->remaining);
			r10_bio->mddev = mddev;
			set_bit(R10BIO_IsRecover, &r10_bio->state);
			r10_bio->sector = sect;
L
Linus Torvalds 已提交
2920

2921 2922 2923 2924 2925
			raid10_find_phys(conf, r10_bio);

			/* Need to check if the array will still be
			 * degraded
			 */
2926
			for (j = 0; j < conf->geo.raid_disks; j++)
2927 2928 2929
				if (conf->mirrors[j].rdev == NULL ||
				    test_bit(Faulty, &conf->mirrors[j].rdev->flags)) {
					still_degraded = 1;
2930
					break;
L
Linus Torvalds 已提交
2931
				}
2932 2933 2934 2935

			must_sync = bitmap_start_sync(mddev->bitmap, sect,
						      &sync_blocks, still_degraded);

2936
			any_working = 0;
2937
			for (j=0; j<conf->copies;j++) {
2938
				int k;
2939
				int d = r10_bio->devs[j].devnum;
2940
				sector_t from_addr, to_addr;
2941
				struct md_rdev *rdev;
2942 2943
				sector_t sector, first_bad;
				int bad_sectors;
2944 2945 2946 2947
				if (!conf->mirrors[d].rdev ||
				    !test_bit(In_sync, &conf->mirrors[d].rdev->flags))
					continue;
				/* This is where we read from */
2948
				any_working = 1;
2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963
				rdev = conf->mirrors[d].rdev;
				sector = r10_bio->devs[j].addr;

				if (is_badblock(rdev, sector, max_sync,
						&first_bad, &bad_sectors)) {
					if (first_bad > sector)
						max_sync = first_bad - sector;
					else {
						bad_sectors -= (sector
								- first_bad);
						if (max_sync > bad_sectors)
							max_sync = bad_sectors;
						continue;
					}
				}
2964 2965 2966 2967 2968 2969
				bio = r10_bio->devs[0].bio;
				bio->bi_next = biolist;
				biolist = bio;
				bio->bi_private = r10_bio;
				bio->bi_end_io = end_sync_read;
				bio->bi_rw = READ;
2970
				from_addr = r10_bio->devs[j].addr;
2971 2972 2973 2974
				bio->bi_sector = from_addr + rdev->data_offset;
				bio->bi_bdev = rdev->bdev;
				atomic_inc(&rdev->nr_pending);
				/* and we write to 'i' (if not in_sync) */
2975 2976 2977 2978 2979

				for (k=0; k<conf->copies; k++)
					if (r10_bio->devs[k].devnum == i)
						break;
				BUG_ON(k == conf->copies);
2980
				to_addr = r10_bio->devs[k].addr;
2981
				r10_bio->devs[0].devnum = d;
2982
				r10_bio->devs[0].addr = from_addr;
2983
				r10_bio->devs[1].devnum = i;
2984
				r10_bio->devs[1].addr = to_addr;
2985

2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024
				rdev = mirror->rdev;
				if (!test_bit(In_sync, &rdev->flags)) {
					bio = r10_bio->devs[1].bio;
					bio->bi_next = biolist;
					biolist = bio;
					bio->bi_private = r10_bio;
					bio->bi_end_io = end_sync_write;
					bio->bi_rw = WRITE;
					bio->bi_sector = to_addr
						+ rdev->data_offset;
					bio->bi_bdev = rdev->bdev;
					atomic_inc(&r10_bio->remaining);
				} else
					r10_bio->devs[1].bio->bi_end_io = NULL;

				/* and maybe write to replacement */
				bio = r10_bio->devs[1].repl_bio;
				if (bio)
					bio->bi_end_io = NULL;
				rdev = mirror->replacement;
				/* Note: if rdev != NULL, then bio
				 * cannot be NULL as r10buf_pool_alloc will
				 * have allocated it.
				 * So the second test here is pointless.
				 * But it keeps semantic-checkers happy, and
				 * this comment keeps human reviewers
				 * happy.
				 */
				if (rdev == NULL || bio == NULL ||
				    test_bit(Faulty, &rdev->flags))
					break;
				bio->bi_next = biolist;
				biolist = bio;
				bio->bi_private = r10_bio;
				bio->bi_end_io = end_sync_write;
				bio->bi_rw = WRITE;
				bio->bi_sector = to_addr + rdev->data_offset;
				bio->bi_bdev = rdev->bdev;
				atomic_inc(&r10_bio->remaining);
3025 3026 3027
				break;
			}
			if (j == conf->copies) {
3028 3029
				/* Cannot recover, so abort the recovery or
				 * record a bad block */
3030 3031 3032 3033
				put_buf(r10_bio);
				if (rb2)
					atomic_dec(&rb2->remaining);
				r10_bio = rb2;
3034 3035 3036 3037 3038 3039 3040 3041
				if (any_working) {
					/* problem is that there are bad blocks
					 * on other device(s)
					 */
					int k;
					for (k = 0; k < conf->copies; k++)
						if (r10_bio->devs[k].devnum == i)
							break;
3042 3043 3044 3045 3046 3047 3048 3049 3050 3051
					if (!test_bit(In_sync,
						      &mirror->rdev->flags)
					    && !rdev_set_badblocks(
						    mirror->rdev,
						    r10_bio->devs[k].addr,
						    max_sync, 0))
						any_working = 0;
					if (mirror->replacement &&
					    !rdev_set_badblocks(
						    mirror->replacement,
3052 3053 3054 3055 3056 3057 3058 3059 3060 3061
						    r10_bio->devs[k].addr,
						    max_sync, 0))
						any_working = 0;
				}
				if (!any_working)  {
					if (!test_and_set_bit(MD_RECOVERY_INTR,
							      &mddev->recovery))
						printk(KERN_INFO "md/raid10:%s: insufficient "
						       "working devices for recovery.\n",
						       mdname(mddev));
3062
					mirror->recovery_disabled
3063 3064
						= mddev->recovery_disabled;
				}
3065
				break;
L
Linus Torvalds 已提交
3066
			}
3067
		}
L
Linus Torvalds 已提交
3068 3069
		if (biolist == NULL) {
			while (r10_bio) {
3070 3071
				struct r10bio *rb2 = r10_bio;
				r10_bio = (struct r10bio*) rb2->master_bio;
L
Linus Torvalds 已提交
3072 3073 3074 3075 3076 3077 3078 3079
				rb2->master_bio = NULL;
				put_buf(rb2);
			}
			goto giveup;
		}
	} else {
		/* resync. Schedule a read for every block at this virt offset */
		int count = 0;
3080

3081 3082
		bitmap_cond_end_sync(mddev->bitmap, sector_nr);

3083 3084
		if (!bitmap_start_sync(mddev->bitmap, sector_nr,
				       &sync_blocks, mddev->degraded) &&
3085 3086
		    !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED,
						 &mddev->recovery)) {
3087 3088 3089 3090 3091 3092
			/* We can skip this block */
			*skipped = 1;
			return sync_blocks + sectors_skipped;
		}
		if (sync_blocks < max_sync)
			max_sync = sync_blocks;
L
Linus Torvalds 已提交
3093 3094 3095 3096
		r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);

		r10_bio->mddev = mddev;
		atomic_set(&r10_bio->remaining, 0);
3097 3098
		raise_barrier(conf, 0);
		conf->next_resync = sector_nr;
L
Linus Torvalds 已提交
3099 3100 3101 3102 3103

		r10_bio->master_bio = NULL;
		r10_bio->sector = sector_nr;
		set_bit(R10BIO_IsSync, &r10_bio->state);
		raid10_find_phys(conf, r10_bio);
3104
		r10_bio->sectors = (sector_nr | chunk_mask) - sector_nr + 1;
L
Linus Torvalds 已提交
3105

3106
		for (i = 0; i < conf->copies; i++) {
L
Linus Torvalds 已提交
3107
			int d = r10_bio->devs[i].devnum;
3108 3109 3110
			sector_t first_bad, sector;
			int bad_sectors;

3111 3112 3113
			if (r10_bio->devs[i].repl_bio)
				r10_bio->devs[i].repl_bio->bi_end_io = NULL;

L
Linus Torvalds 已提交
3114 3115
			bio = r10_bio->devs[i].bio;
			bio->bi_end_io = NULL;
N
NeilBrown 已提交
3116
			clear_bit(BIO_UPTODATE, &bio->bi_flags);
L
Linus Torvalds 已提交
3117
			if (conf->mirrors[d].rdev == NULL ||
3118
			    test_bit(Faulty, &conf->mirrors[d].rdev->flags))
L
Linus Torvalds 已提交
3119
				continue;
3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132
			sector = r10_bio->devs[i].addr;
			if (is_badblock(conf->mirrors[d].rdev,
					sector, max_sync,
					&first_bad, &bad_sectors)) {
				if (first_bad > sector)
					max_sync = first_bad - sector;
				else {
					bad_sectors -= (sector - first_bad);
					if (max_sync > bad_sectors)
						max_sync = max_sync;
					continue;
				}
			}
L
Linus Torvalds 已提交
3133 3134 3135 3136 3137 3138
			atomic_inc(&conf->mirrors[d].rdev->nr_pending);
			atomic_inc(&r10_bio->remaining);
			bio->bi_next = biolist;
			biolist = bio;
			bio->bi_private = r10_bio;
			bio->bi_end_io = end_sync_read;
3139
			bio->bi_rw = READ;
3140
			bio->bi_sector = sector +
L
Linus Torvalds 已提交
3141 3142 3143
				conf->mirrors[d].rdev->data_offset;
			bio->bi_bdev = conf->mirrors[d].rdev->bdev;
			count++;
3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164

			if (conf->mirrors[d].replacement == NULL ||
			    test_bit(Faulty,
				     &conf->mirrors[d].replacement->flags))
				continue;

			/* Need to set up for writing to the replacement */
			bio = r10_bio->devs[i].repl_bio;
			clear_bit(BIO_UPTODATE, &bio->bi_flags);

			sector = r10_bio->devs[i].addr;
			atomic_inc(&conf->mirrors[d].rdev->nr_pending);
			bio->bi_next = biolist;
			biolist = bio;
			bio->bi_private = r10_bio;
			bio->bi_end_io = end_sync_write;
			bio->bi_rw = WRITE;
			bio->bi_sector = sector +
				conf->mirrors[d].replacement->data_offset;
			bio->bi_bdev = conf->mirrors[d].replacement->bdev;
			count++;
L
Linus Torvalds 已提交
3165 3166 3167 3168 3169 3170
		}

		if (count < 2) {
			for (i=0; i<conf->copies; i++) {
				int d = r10_bio->devs[i].devnum;
				if (r10_bio->devs[i].bio->bi_end_io)
3171 3172
					rdev_dec_pending(conf->mirrors[d].rdev,
							 mddev);
3173 3174 3175 3176 3177
				if (r10_bio->devs[i].repl_bio &&
				    r10_bio->devs[i].repl_bio->bi_end_io)
					rdev_dec_pending(
						conf->mirrors[d].replacement,
						mddev);
L
Linus Torvalds 已提交
3178 3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196
			}
			put_buf(r10_bio);
			biolist = NULL;
			goto giveup;
		}
	}

	for (bio = biolist; bio ; bio=bio->bi_next) {

		bio->bi_flags &= ~(BIO_POOL_MASK - 1);
		if (bio->bi_end_io)
			bio->bi_flags |= 1 << BIO_UPTODATE;
		bio->bi_vcnt = 0;
		bio->bi_idx = 0;
		bio->bi_phys_segments = 0;
		bio->bi_size = 0;
	}

	nr_sectors = 0;
3197 3198
	if (sector_nr + max_sync < max_sector)
		max_sector = sector_nr + max_sync;
L
Linus Torvalds 已提交
3199 3200 3201 3202 3203 3204 3205 3206
	do {
		struct page *page;
		int len = PAGE_SIZE;
		if (sector_nr + (len>>9) > max_sector)
			len = (max_sector - sector_nr) << 9;
		if (len == 0)
			break;
		for (bio= biolist ; bio ; bio=bio->bi_next) {
3207
			struct bio *bio2;
L
Linus Torvalds 已提交
3208
			page = bio->bi_io_vec[bio->bi_vcnt].bv_page;
3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220
			if (bio_add_page(bio, page, len, 0))
				continue;

			/* stop here */
			bio->bi_io_vec[bio->bi_vcnt].bv_page = page;
			for (bio2 = biolist;
			     bio2 && bio2 != bio;
			     bio2 = bio2->bi_next) {
				/* remove last page from this bio */
				bio2->bi_vcnt--;
				bio2->bi_size -= len;
				bio2->bi_flags &= ~(1<< BIO_SEG_VALID);
L
Linus Torvalds 已提交
3221
			}
3222
			goto bio_full;
L
Linus Torvalds 已提交
3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233 3234 3235 3236 3237 3238 3239 3240 3241 3242 3243
		}
		nr_sectors += len>>9;
		sector_nr += len>>9;
	} while (biolist->bi_vcnt < RESYNC_PAGES);
 bio_full:
	r10_bio->sectors = nr_sectors;

	while (biolist) {
		bio = biolist;
		biolist = biolist->bi_next;

		bio->bi_next = NULL;
		r10_bio = bio->bi_private;
		r10_bio->sectors = nr_sectors;

		if (bio->bi_end_io == end_sync_read) {
			md_sync_acct(bio->bi_bdev, nr_sectors);
			generic_make_request(bio);
		}
	}

3244 3245 3246 3247 3248 3249
	if (sectors_skipped)
		/* pretend they weren't skipped, it makes
		 * no important difference in this case
		 */
		md_done_sync(mddev, sectors_skipped, 1);

L
Linus Torvalds 已提交
3250 3251 3252
	return sectors_skipped + nr_sectors;
 giveup:
	/* There is nowhere to write, so all non-sync
3253 3254
	 * drives must be failed or in resync, all drives
	 * have a bad block, so try the next chunk...
L
Linus Torvalds 已提交
3255
	 */
3256 3257 3258 3259
	if (sector_nr + max_sync < max_sector)
		max_sector = sector_nr + max_sync;

	sectors_skipped += (max_sector - sector_nr);
L
Linus Torvalds 已提交
3260 3261 3262 3263 3264
	chunks_skipped ++;
	sector_nr = max_sector;
	goto skipped;
}

3265
static sector_t
3266
raid10_size(struct mddev *mddev, sector_t sectors, int raid_disks)
3267 3268
{
	sector_t size;
3269
	struct r10conf *conf = mddev->private;
3270 3271

	if (!raid_disks)
N
NeilBrown 已提交
3272 3273
		raid_disks = min(conf->geo.raid_disks,
				 conf->prev.raid_disks);
3274
	if (!sectors)
3275
		sectors = conf->dev_sectors;
3276

3277 3278
	size = sectors >> conf->geo.chunk_shift;
	sector_div(size, conf->geo.far_copies);
3279
	size = size * raid_disks;
3280
	sector_div(size, conf->geo.near_copies);
3281

3282
	return size << conf->geo.chunk_shift;
3283 3284
}

3285 3286 3287 3288 3289 3290 3291
static void calc_sectors(struct r10conf *conf, sector_t size)
{
	/* Calculate the number of sectors-per-device that will
	 * actually be used, and set conf->dev_sectors and
	 * conf->stride
	 */

3292 3293 3294 3295
	size = size >> conf->geo.chunk_shift;
	sector_div(size, conf->geo.far_copies);
	size = size * conf->geo.raid_disks;
	sector_div(size, conf->geo.near_copies);
3296 3297 3298 3299 3300 3301 3302
	/* 'size' is now the number of chunks in the array */
	/* calculate "used chunks per device" */
	size = size * conf->copies;

	/* We need to round up when dividing by raid_disks to
	 * get the stride size.
	 */
3303
	size = DIV_ROUND_UP_SECTOR_T(size, conf->geo.raid_disks);
3304

3305
	conf->dev_sectors = size << conf->geo.chunk_shift;
3306

3307 3308
	if (conf->geo.far_offset)
		conf->geo.stride = 1 << conf->geo.chunk_shift;
3309
	else {
3310 3311
		sector_div(size, conf->geo.far_copies);
		conf->geo.stride = size << conf->geo.chunk_shift;
3312 3313
	}
}
3314

3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355
enum geo_type {geo_new, geo_old, geo_start};
static int setup_geo(struct geom *geo, struct mddev *mddev, enum geo_type new)
{
	int nc, fc, fo;
	int layout, chunk, disks;
	switch (new) {
	case geo_old:
		layout = mddev->layout;
		chunk = mddev->chunk_sectors;
		disks = mddev->raid_disks - mddev->delta_disks;
		break;
	case geo_new:
		layout = mddev->new_layout;
		chunk = mddev->new_chunk_sectors;
		disks = mddev->raid_disks;
		break;
	default: /* avoid 'may be unused' warnings */
	case geo_start: /* new when starting reshape - raid_disks not
			 * updated yet. */
		layout = mddev->new_layout;
		chunk = mddev->new_chunk_sectors;
		disks = mddev->raid_disks + mddev->delta_disks;
		break;
	}
	if (layout >> 17)
		return -1;
	if (chunk < (PAGE_SIZE >> 9) ||
	    !is_power_of_2(chunk))
		return -2;
	nc = layout & 255;
	fc = (layout >> 8) & 255;
	fo = layout & (1<<16);
	geo->raid_disks = disks;
	geo->near_copies = nc;
	geo->far_copies = fc;
	geo->far_offset = fo;
	geo->chunk_mask = chunk - 1;
	geo->chunk_shift = ffz(~chunk);
	return nc*fc;
}

3356
static struct r10conf *setup_conf(struct mddev *mddev)
L
Linus Torvalds 已提交
3357
{
3358
	struct r10conf *conf = NULL;
3359
	int err = -EINVAL;
3360 3361 3362 3363
	struct geom geo;
	int copies;

	copies = setup_geo(&geo, mddev, geo_new);
L
Linus Torvalds 已提交
3364

3365
	if (copies == -2) {
N
NeilBrown 已提交
3366 3367 3368
		printk(KERN_ERR "md/raid10:%s: chunk size must be "
		       "at least PAGE_SIZE(%ld) and be a power of 2.\n",
		       mdname(mddev), PAGE_SIZE);
3369
		goto out;
L
Linus Torvalds 已提交
3370
	}
3371

3372
	if (copies < 2 || copies > mddev->raid_disks) {
N
NeilBrown 已提交
3373
		printk(KERN_ERR "md/raid10:%s: unsupported raid10 layout: 0x%8x\n",
3374
		       mdname(mddev), mddev->new_layout);
L
Linus Torvalds 已提交
3375 3376
		goto out;
	}
3377 3378

	err = -ENOMEM;
3379
	conf = kzalloc(sizeof(struct r10conf), GFP_KERNEL);
3380
	if (!conf)
L
Linus Torvalds 已提交
3381
		goto out;
3382

N
NeilBrown 已提交
3383 3384 3385
	/* FIXME calc properly */
	conf->mirrors = kzalloc(sizeof(struct mirror_info)*(mddev->raid_disks +
							    max(0,mddev->delta_disks)),
3386 3387 3388
				GFP_KERNEL);
	if (!conf->mirrors)
		goto out;
3389 3390 3391

	conf->tmppage = alloc_page(GFP_KERNEL);
	if (!conf->tmppage)
3392 3393
		goto out;

3394 3395
	conf->geo = geo;
	conf->copies = copies;
3396 3397 3398 3399 3400
	conf->r10bio_pool = mempool_create(NR_RAID10_BIOS, r10bio_pool_alloc,
					   r10bio_pool_free, conf);
	if (!conf->r10bio_pool)
		goto out;

3401
	calc_sectors(conf, mddev->dev_sectors);
N
NeilBrown 已提交
3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416
	if (mddev->reshape_position == MaxSector) {
		conf->prev = conf->geo;
		conf->reshape_progress = MaxSector;
	} else {
		if (setup_geo(&conf->prev, mddev, geo_old) != conf->copies) {
			err = -EINVAL;
			goto out;
		}
		conf->reshape_progress = mddev->reshape_position;
		if (conf->prev.far_offset)
			conf->prev.stride = 1 << conf->prev.chunk_shift;
		else
			/* far_copies must be 1 */
			conf->prev.stride = conf->dev_sectors;
	}
3417
	spin_lock_init(&conf->device_lock);
3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430
	INIT_LIST_HEAD(&conf->retry_list);

	spin_lock_init(&conf->resync_lock);
	init_waitqueue_head(&conf->wait_barrier);

	conf->thread = md_register_thread(raid10d, mddev, NULL);
	if (!conf->thread)
		goto out;

	conf->mddev = mddev;
	return conf;

 out:
N
NeilBrown 已提交
3431 3432 3433
	if (err == -ENOMEM)
		printk(KERN_ERR "md/raid10:%s: couldn't allocate memory.\n",
		       mdname(mddev));
3434 3435 3436 3437 3438 3439 3440 3441 3442 3443
	if (conf) {
		if (conf->r10bio_pool)
			mempool_destroy(conf->r10bio_pool);
		kfree(conf->mirrors);
		safe_put_page(conf->tmppage);
		kfree(conf);
	}
	return ERR_PTR(err);
}

3444
static int run(struct mddev *mddev)
3445
{
3446
	struct r10conf *conf;
3447
	int i, disk_idx, chunk_size;
3448
	struct mirror_info *disk;
3449
	struct md_rdev *rdev;
3450
	sector_t size;
N
NeilBrown 已提交
3451 3452
	sector_t min_offset_diff = 0;
	int first = 1;
3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466

	if (mddev->private == NULL) {
		conf = setup_conf(mddev);
		if (IS_ERR(conf))
			return PTR_ERR(conf);
		mddev->private = conf;
	}
	conf = mddev->private;
	if (!conf)
		goto out;

	mddev->thread = conf->thread;
	conf->thread = NULL;

3467 3468
	chunk_size = mddev->chunk_sectors << 9;
	blk_queue_io_min(mddev->queue, chunk_size);
3469 3470
	if (conf->geo.raid_disks % conf->geo.near_copies)
		blk_queue_io_opt(mddev->queue, chunk_size * conf->geo.raid_disks);
3471 3472
	else
		blk_queue_io_opt(mddev->queue, chunk_size *
3473
				 (conf->geo.raid_disks / conf->geo.near_copies));
3474

N
NeilBrown 已提交
3475
	rdev_for_each(rdev, mddev) {
N
NeilBrown 已提交
3476
		long long diff;
3477

L
Linus Torvalds 已提交
3478
		disk_idx = rdev->raid_disk;
3479 3480 3481 3482
		if (disk_idx < 0)
			continue;
		if (disk_idx >= conf->geo.raid_disks &&
		    disk_idx >= conf->prev.raid_disks)
L
Linus Torvalds 已提交
3483 3484 3485
			continue;
		disk = conf->mirrors + disk_idx;

3486 3487 3488 3489 3490 3491 3492 3493 3494
		if (test_bit(Replacement, &rdev->flags)) {
			if (disk->replacement)
				goto out_free_conf;
			disk->replacement = rdev;
		} else {
			if (disk->rdev)
				goto out_free_conf;
			disk->rdev = rdev;
		}
N
NeilBrown 已提交
3495 3496 3497 3498 3499 3500 3501
		diff = (rdev->new_data_offset - rdev->data_offset);
		if (!mddev->reshape_backwards)
			diff = -diff;
		if (diff < 0)
			diff = 0;
		if (first || diff < min_offset_diff)
			min_offset_diff = diff;
3502

3503 3504
		disk_stack_limits(mddev->gendisk, rdev->bdev,
				  rdev->data_offset << 9);
L
Linus Torvalds 已提交
3505 3506 3507

		disk->head_position = 0;
	}
N
NeilBrown 已提交
3508

3509
	/* need to check that every block has at least one working mirror */
3510
	if (!enough(conf, -1)) {
N
NeilBrown 已提交
3511
		printk(KERN_ERR "md/raid10:%s: not enough operational mirrors.\n",
3512
		       mdname(mddev));
L
Linus Torvalds 已提交
3513 3514 3515
		goto out_free_conf;
	}

N
NeilBrown 已提交
3516 3517 3518 3519 3520 3521 3522 3523 3524 3525
	if (conf->reshape_progress != MaxSector) {
		/* must ensure that shape change is supported */
		if (conf->geo.far_copies != 1 &&
		    conf->geo.far_offset == 0)
			goto out_free_conf;
		if (conf->prev.far_copies != 1 &&
		    conf->geo.far_offset == 0)
			goto out_free_conf;
	}

L
Linus Torvalds 已提交
3526
	mddev->degraded = 0;
3527 3528 3529 3530
	for (i = 0;
	     i < conf->geo.raid_disks
		     || i < conf->prev.raid_disks;
	     i++) {
L
Linus Torvalds 已提交
3531 3532 3533

		disk = conf->mirrors + i;

3534 3535 3536 3537 3538 3539 3540
		if (!disk->rdev && disk->replacement) {
			/* The replacement is all we have - use it */
			disk->rdev = disk->replacement;
			disk->replacement = NULL;
			clear_bit(Replacement, &disk->rdev->flags);
		}

3541
		if (!disk->rdev ||
3542
		    !test_bit(In_sync, &disk->rdev->flags)) {
L
Linus Torvalds 已提交
3543 3544
			disk->head_position = 0;
			mddev->degraded++;
3545 3546
			if (disk->rdev)
				conf->fullsync = 1;
L
Linus Torvalds 已提交
3547
		}
3548
		disk->recovery_disabled = mddev->recovery_disabled - 1;
L
Linus Torvalds 已提交
3549 3550
	}

3551
	if (mddev->recovery_cp != MaxSector)
N
NeilBrown 已提交
3552
		printk(KERN_NOTICE "md/raid10:%s: not clean"
3553 3554
		       " -- starting background reconstruction\n",
		       mdname(mddev));
L
Linus Torvalds 已提交
3555
	printk(KERN_INFO
N
NeilBrown 已提交
3556
		"md/raid10:%s: active with %d out of %d devices\n",
3557 3558
		mdname(mddev), conf->geo.raid_disks - mddev->degraded,
		conf->geo.raid_disks);
L
Linus Torvalds 已提交
3559 3560 3561
	/*
	 * Ok, everything is just fine now
	 */
3562 3563 3564 3565
	mddev->dev_sectors = conf->dev_sectors;
	size = raid10_size(mddev, 0, 0);
	md_set_array_sectors(mddev, size);
	mddev->resync_max_sectors = size;
L
Linus Torvalds 已提交
3566

3567 3568
	mddev->queue->backing_dev_info.congested_fn = raid10_congested;
	mddev->queue->backing_dev_info.congested_data = mddev;
3569

L
Linus Torvalds 已提交
3570 3571 3572 3573 3574
	/* Calculate max read-ahead size.
	 * We need to readahead at least twice a whole stripe....
	 * maybe...
	 */
	{
3575
		int stripe = conf->geo.raid_disks *
3576
			((mddev->chunk_sectors << 9) / PAGE_SIZE);
3577
		stripe /= conf->geo.near_copies;
N
NeilBrown 已提交
3578 3579
		if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
			mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
L
Linus Torvalds 已提交
3580 3581
	}

3582
	blk_queue_merge_bvec(mddev->queue, raid10_mergeable_bvec);
3583 3584 3585 3586

	if (md_integrity_register(mddev))
		goto out_free_conf;

N
NeilBrown 已提交
3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610
	if (conf->reshape_progress != MaxSector) {
		unsigned long before_length, after_length;

		before_length = ((1 << conf->prev.chunk_shift) *
				 conf->prev.far_copies);
		after_length = ((1 << conf->geo.chunk_shift) *
				conf->geo.far_copies);

		if (max(before_length, after_length) > min_offset_diff) {
			/* This cannot work */
			printk("md/raid10: offset difference not enough to continue reshape\n");
			goto out_free_conf;
		}
		conf->offset_diff = min_offset_diff;

		conf->reshape_safe = conf->reshape_progress;
		clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
		clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
		set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
		set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
		mddev->sync_thread = md_register_thread(md_do_sync, mddev,
							"reshape");
	}

L
Linus Torvalds 已提交
3611 3612 3613
	return 0;

out_free_conf:
3614
	md_unregister_thread(&mddev->thread);
L
Linus Torvalds 已提交
3615 3616
	if (conf->r10bio_pool)
		mempool_destroy(conf->r10bio_pool);
3617
	safe_put_page(conf->tmppage);
3618
	kfree(conf->mirrors);
L
Linus Torvalds 已提交
3619 3620 3621 3622 3623 3624
	kfree(conf);
	mddev->private = NULL;
out:
	return -EIO;
}

3625
static int stop(struct mddev *mddev)
L
Linus Torvalds 已提交
3626
{
3627
	struct r10conf *conf = mddev->private;
L
Linus Torvalds 已提交
3628

3629 3630 3631
	raise_barrier(conf, 0);
	lower_barrier(conf);

3632
	md_unregister_thread(&mddev->thread);
L
Linus Torvalds 已提交
3633 3634 3635
	blk_sync_queue(mddev->queue); /* the unplug fn references 'conf'*/
	if (conf->r10bio_pool)
		mempool_destroy(conf->r10bio_pool);
3636
	kfree(conf->mirrors);
L
Linus Torvalds 已提交
3637 3638 3639 3640 3641
	kfree(conf);
	mddev->private = NULL;
	return 0;
}

3642
static void raid10_quiesce(struct mddev *mddev, int state)
3643
{
3644
	struct r10conf *conf = mddev->private;
3645 3646 3647 3648 3649 3650 3651 3652 3653 3654

	switch(state) {
	case 1:
		raise_barrier(conf, 0);
		break;
	case 0:
		lower_barrier(conf);
		break;
	}
}
L
Linus Torvalds 已提交
3655

3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672
static int raid10_resize(struct mddev *mddev, sector_t sectors)
{
	/* Resize of 'far' arrays is not supported.
	 * For 'near' and 'offset' arrays we can set the
	 * number of sectors used to be an appropriate multiple
	 * of the chunk size.
	 * For 'offset', this is far_copies*chunksize.
	 * For 'near' the multiplier is the LCM of
	 * near_copies and raid_disks.
	 * So if far_copies > 1 && !far_offset, fail.
	 * Else find LCM(raid_disks, near_copy)*far_copies and
	 * multiply by chunk_size.  Then round to this number.
	 * This is mostly done by raid10_size()
	 */
	struct r10conf *conf = mddev->private;
	sector_t oldsize, size;

3673 3674 3675
	if (mddev->reshape_position != MaxSector)
		return -EBUSY;

3676
	if (conf->geo.far_copies > 1 && !conf->geo.far_offset)
3677 3678 3679 3680
		return -EINVAL;

	oldsize = raid10_size(mddev, 0, 0);
	size = raid10_size(mddev, sectors, 0);
3681 3682
	if (mddev->external_size &&
	    mddev->array_sectors > size)
3683
		return -EINVAL;
3684 3685 3686 3687 3688 3689
	if (mddev->bitmap) {
		int ret = bitmap_resize(mddev->bitmap, size, 0, 0);
		if (ret)
			return ret;
	}
	md_set_array_sectors(mddev, size);
3690 3691 3692 3693 3694 3695 3696
	set_capacity(mddev->gendisk, mddev->array_sectors);
	revalidate_disk(mddev->gendisk);
	if (sectors > mddev->dev_sectors &&
	    mddev->recovery_cp > oldsize) {
		mddev->recovery_cp = oldsize;
		set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
	}
3697 3698
	calc_sectors(conf, sectors);
	mddev->dev_sectors = conf->dev_sectors;
3699 3700 3701 3702
	mddev->resync_max_sectors = size;
	return 0;
}

3703
static void *raid10_takeover_raid0(struct mddev *mddev)
3704
{
3705
	struct md_rdev *rdev;
3706
	struct r10conf *conf;
3707 3708

	if (mddev->degraded > 0) {
N
NeilBrown 已提交
3709 3710
		printk(KERN_ERR "md/raid10:%s: Error: degraded raid0!\n",
		       mdname(mddev));
3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724
		return ERR_PTR(-EINVAL);
	}

	/* Set new parameters */
	mddev->new_level = 10;
	/* new layout: far_copies = 1, near_copies = 2 */
	mddev->new_layout = (1<<8) + 2;
	mddev->new_chunk_sectors = mddev->chunk_sectors;
	mddev->delta_disks = mddev->raid_disks;
	mddev->raid_disks *= 2;
	/* make sure it will be not marked as dirty */
	mddev->recovery_cp = MaxSector;

	conf = setup_conf(mddev);
3725
	if (!IS_ERR(conf)) {
N
NeilBrown 已提交
3726
		rdev_for_each(rdev, mddev)
3727 3728
			if (rdev->raid_disk >= 0)
				rdev->new_raid_disk = rdev->raid_disk * 2;
3729 3730 3731
		conf->barrier = 1;
	}

3732 3733 3734
	return conf;
}

3735
static void *raid10_takeover(struct mddev *mddev)
3736
{
3737
	struct r0conf *raid0_conf;
3738 3739 3740 3741 3742 3743

	/* raid10 can take over:
	 *  raid0 - providing it has only two drives
	 */
	if (mddev->level == 0) {
		/* for raid0 takeover only one zone is supported */
3744 3745
		raid0_conf = mddev->private;
		if (raid0_conf->nr_strip_zones > 1) {
N
NeilBrown 已提交
3746 3747 3748
			printk(KERN_ERR "md/raid10:%s: cannot takeover raid 0"
			       " with more than one zone.\n",
			       mdname(mddev));
3749 3750 3751 3752 3753 3754 3755
			return ERR_PTR(-EINVAL);
		}
		return raid10_takeover_raid0(mddev);
	}
	return ERR_PTR(-EINVAL);
}

N
NeilBrown 已提交
3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846 3847 3848 3849 3850 3851 3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902 3903 3904 3905 3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080 4081 4082 4083 4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155 4156 4157 4158 4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348 4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364 4365 4366 4367 4368 4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434 4435 4436 4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484
static int raid10_check_reshape(struct mddev *mddev)
{
	/* Called when there is a request to change
	 * - layout (to ->new_layout)
	 * - chunk size (to ->new_chunk_sectors)
	 * - raid_disks (by delta_disks)
	 * or when trying to restart a reshape that was ongoing.
	 *
	 * We need to validate the request and possibly allocate
	 * space if that might be an issue later.
	 *
	 * Currently we reject any reshape of a 'far' mode array,
	 * allow chunk size to change if new is generally acceptable,
	 * allow raid_disks to increase, and allow
	 * a switch between 'near' mode and 'offset' mode.
	 */
	struct r10conf *conf = mddev->private;
	struct geom geo;

	if (conf->geo.far_copies != 1 && !conf->geo.far_offset)
		return -EINVAL;

	if (setup_geo(&geo, mddev, geo_start) != conf->copies)
		/* mustn't change number of copies */
		return -EINVAL;
	if (geo.far_copies > 1 && !geo.far_offset)
		/* Cannot switch to 'far' mode */
		return -EINVAL;

	if (mddev->array_sectors & geo.chunk_mask)
			/* not factor of array size */
			return -EINVAL;

	if (mddev->bitmap)
		return -EBUSY;
	if (!enough(conf, -1))
		return -EINVAL;

	kfree(conf->mirrors_new);
	conf->mirrors_new = NULL;
	if (mddev->delta_disks > 0) {
		/* allocate new 'mirrors' list */
		conf->mirrors_new = kzalloc(
			sizeof(struct mirror_info)
			*(mddev->raid_disks +
			  mddev->delta_disks),
			GFP_KERNEL);
		if (!conf->mirrors_new)
			return -ENOMEM;
	}
	return 0;
}

/*
 * Need to check if array has failed when deciding whether to:
 *  - start an array
 *  - remove non-faulty devices
 *  - add a spare
 *  - allow a reshape
 * This determination is simple when no reshape is happening.
 * However if there is a reshape, we need to carefully check
 * both the before and after sections.
 * This is because some failed devices may only affect one
 * of the two sections, and some non-in_sync devices may
 * be insync in the section most affected by failed devices.
 */
static int calc_degraded(struct r10conf *conf)
{
	int degraded, degraded2;
	int i;

	rcu_read_lock();
	degraded = 0;
	/* 'prev' section first */
	for (i = 0; i < conf->prev.raid_disks; i++) {
		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
		if (!rdev || test_bit(Faulty, &rdev->flags))
			degraded++;
		else if (!test_bit(In_sync, &rdev->flags))
			/* When we can reduce the number of devices in
			 * an array, this might not contribute to
			 * 'degraded'.  It does now.
			 */
			degraded++;
	}
	rcu_read_unlock();
	if (conf->geo.raid_disks == conf->prev.raid_disks)
		return degraded;
	rcu_read_lock();
	degraded2 = 0;
	for (i = 0; i < conf->geo.raid_disks; i++) {
		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
		if (!rdev || test_bit(Faulty, &rdev->flags))
			degraded2++;
		else if (!test_bit(In_sync, &rdev->flags)) {
			/* If reshape is increasing the number of devices,
			 * this section has already been recovered, so
			 * it doesn't contribute to degraded.
			 * else it does.
			 */
			if (conf->geo.raid_disks <= conf->prev.raid_disks)
				degraded2++;
		}
	}
	rcu_read_unlock();
	if (degraded2 > degraded)
		return degraded2;
	return degraded;
}

static int raid10_start_reshape(struct mddev *mddev)
{
	/* A 'reshape' has been requested. This commits
	 * the various 'new' fields and sets MD_RECOVER_RESHAPE
	 * This also checks if there are enough spares and adds them
	 * to the array.
	 * We currently require enough spares to make the final
	 * array non-degraded.  We also require that the difference
	 * between old and new data_offset - on each device - is
	 * enough that we never risk over-writing.
	 */

	unsigned long before_length, after_length;
	sector_t min_offset_diff = 0;
	int first = 1;
	struct geom new;
	struct r10conf *conf = mddev->private;
	struct md_rdev *rdev;
	int spares = 0;

	if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
		return -EBUSY;

	if (setup_geo(&new, mddev, geo_start) != conf->copies)
		return -EINVAL;

	before_length = ((1 << conf->prev.chunk_shift) *
			 conf->prev.far_copies);
	after_length = ((1 << conf->geo.chunk_shift) *
			conf->geo.far_copies);

	rdev_for_each(rdev, mddev) {
		if (!test_bit(In_sync, &rdev->flags)
		    && !test_bit(Faulty, &rdev->flags))
			spares++;
		if (rdev->raid_disk >= 0) {
			long long diff = (rdev->new_data_offset
					  - rdev->data_offset);
			if (!mddev->reshape_backwards)
				diff = -diff;
			if (diff < 0)
				diff = 0;
			if (first || diff < min_offset_diff)
				min_offset_diff = diff;
		}
	}

	if (max(before_length, after_length) > min_offset_diff)
		return -EINVAL;

	if (spares < mddev->delta_disks)
		return -EINVAL;

	conf->offset_diff = min_offset_diff;
	spin_lock_irq(&conf->device_lock);
	if (conf->mirrors_new) {
		memcpy(conf->mirrors_new, conf->mirrors,
		       sizeof(struct mirror_info)*conf->prev.raid_disks);
		smp_mb();
		kfree(conf->mirrors_old); /* FIXME and elsewhere */
		conf->mirrors_old = conf->mirrors;
		conf->mirrors = conf->mirrors_new;
		conf->mirrors_new = NULL;
	}
	setup_geo(&conf->geo, mddev, geo_start);
	smp_mb();
	if (mddev->reshape_backwards) {
		sector_t size = raid10_size(mddev, 0, 0);
		if (size < mddev->array_sectors) {
			spin_unlock_irq(&conf->device_lock);
			printk(KERN_ERR "md/raid10:%s: array size must be reduce before number of disks\n",
			       mdname(mddev));
			return -EINVAL;
		}
		mddev->resync_max_sectors = size;
		conf->reshape_progress = size;
	} else
		conf->reshape_progress = 0;
	spin_unlock_irq(&conf->device_lock);

	if (mddev->delta_disks > 0) {
		rdev_for_each(rdev, mddev)
			if (rdev->raid_disk < 0 &&
			    !test_bit(Faulty, &rdev->flags)) {
				if (raid10_add_disk(mddev, rdev) == 0) {
					if (rdev->raid_disk >=
					    conf->prev.raid_disks)
						set_bit(In_sync, &rdev->flags);
					else
						rdev->recovery_offset = 0;

					if (sysfs_link_rdev(mddev, rdev))
						/* Failure here  is OK */;
				}
			} else if (rdev->raid_disk >= conf->prev.raid_disks
				   && !test_bit(Faulty, &rdev->flags)) {
				/* This is a spare that was manually added */
				set_bit(In_sync, &rdev->flags);
			}
	}
	/* When a reshape changes the number of devices,
	 * ->degraded is measured against the larger of the
	 * pre and  post numbers.
	 */
	spin_lock_irq(&conf->device_lock);
	mddev->degraded = calc_degraded(conf);
	spin_unlock_irq(&conf->device_lock);
	mddev->raid_disks = conf->geo.raid_disks;
	mddev->reshape_position = conf->reshape_progress;
	set_bit(MD_CHANGE_DEVS, &mddev->flags);

	clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
	clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
	set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
	set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);

	mddev->sync_thread = md_register_thread(md_do_sync, mddev,
						"reshape");
	if (!mddev->sync_thread) {
		mddev->recovery = 0;
		spin_lock_irq(&conf->device_lock);
		conf->geo = conf->prev;
		mddev->raid_disks = conf->geo.raid_disks;
		rdev_for_each(rdev, mddev)
			rdev->new_data_offset = rdev->data_offset;
		smp_wmb();
		conf->reshape_progress = MaxSector;
		mddev->reshape_position = MaxSector;
		spin_unlock_irq(&conf->device_lock);
		return -EAGAIN;
	}
	conf->reshape_checkpoint = jiffies;
	md_wakeup_thread(mddev->sync_thread);
	md_new_event(mddev);
	return 0;
}

/* Calculate the last device-address that could contain
 * any block from the chunk that includes the array-address 's'
 * and report the next address.
 * i.e. the address returned will be chunk-aligned and after
 * any data that is in the chunk containing 's'.
 */
static sector_t last_dev_address(sector_t s, struct geom *geo)
{
	s = (s | geo->chunk_mask) + 1;
	s >>= geo->chunk_shift;
	s *= geo->near_copies;
	s = DIV_ROUND_UP_SECTOR_T(s, geo->raid_disks);
	s *= geo->far_copies;
	s <<= geo->chunk_shift;
	return s;
}

/* Calculate the first device-address that could contain
 * any block from the chunk that includes the array-address 's'.
 * This too will be the start of a chunk
 */
static sector_t first_dev_address(sector_t s, struct geom *geo)
{
	s >>= geo->chunk_shift;
	s *= geo->near_copies;
	sector_div(s, geo->raid_disks);
	s *= geo->far_copies;
	s <<= geo->chunk_shift;
	return s;
}

static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
				int *skipped)
{
	/* We simply copy at most one chunk (smallest of old and new)
	 * at a time, possibly less if that exceeds RESYNC_PAGES,
	 * or we hit a bad block or something.
	 * This might mean we pause for normal IO in the middle of
	 * a chunk, but that is not a problem was mddev->reshape_position
	 * can record any location.
	 *
	 * If we will want to write to a location that isn't
	 * yet recorded as 'safe' (i.e. in metadata on disk) then
	 * we need to flush all reshape requests and update the metadata.
	 *
	 * When reshaping forwards (e.g. to more devices), we interpret
	 * 'safe' as the earliest block which might not have been copied
	 * down yet.  We divide this by previous stripe size and multiply
	 * by previous stripe length to get lowest device offset that we
	 * cannot write to yet.
	 * We interpret 'sector_nr' as an address that we want to write to.
	 * From this we use last_device_address() to find where we might
	 * write to, and first_device_address on the  'safe' position.
	 * If this 'next' write position is after the 'safe' position,
	 * we must update the metadata to increase the 'safe' position.
	 *
	 * When reshaping backwards, we round in the opposite direction
	 * and perform the reverse test:  next write position must not be
	 * less than current safe position.
	 *
	 * In all this the minimum difference in data offsets
	 * (conf->offset_diff - always positive) allows a bit of slack,
	 * so next can be after 'safe', but not by more than offset_disk
	 *
	 * We need to prepare all the bios here before we start any IO
	 * to ensure the size we choose is acceptable to all devices.
	 * The means one for each copy for write-out and an extra one for
	 * read-in.
	 * We store the read-in bio in ->master_bio and the others in
	 * ->devs[x].bio and ->devs[x].repl_bio.
	 */
	struct r10conf *conf = mddev->private;
	struct r10bio *r10_bio;
	sector_t next, safe, last;
	int max_sectors;
	int nr_sectors;
	int s;
	struct md_rdev *rdev;
	int need_flush = 0;
	struct bio *blist;
	struct bio *bio, *read_bio;
	int sectors_done = 0;

	if (sector_nr == 0) {
		/* If restarting in the middle, skip the initial sectors */
		if (mddev->reshape_backwards &&
		    conf->reshape_progress < raid10_size(mddev, 0, 0)) {
			sector_nr = (raid10_size(mddev, 0, 0)
				     - conf->reshape_progress);
		} else if (!mddev->reshape_backwards &&
			   conf->reshape_progress > 0)
			sector_nr = conf->reshape_progress;
		if (sector_nr) {
			mddev->curr_resync_completed = sector_nr;
			sysfs_notify(&mddev->kobj, NULL, "sync_completed");
			*skipped = 1;
			return sector_nr;
		}
	}

	/* We don't use sector_nr to track where we are up to
	 * as that doesn't work well for ->reshape_backwards.
	 * So just use ->reshape_progress.
	 */
	if (mddev->reshape_backwards) {
		/* 'next' is the earliest device address that we might
		 * write to for this chunk in the new layout
		 */
		next = first_dev_address(conf->reshape_progress - 1,
					 &conf->geo);

		/* 'safe' is the last device address that we might read from
		 * in the old layout after a restart
		 */
		safe = last_dev_address(conf->reshape_safe - 1,
					&conf->prev);

		if (next + conf->offset_diff < safe)
			need_flush = 1;

		last = conf->reshape_progress - 1;
		sector_nr = last & ~(sector_t)(conf->geo.chunk_mask
					       & conf->prev.chunk_mask);
		if (sector_nr + RESYNC_BLOCK_SIZE/512 < last)
			sector_nr = last + 1 - RESYNC_BLOCK_SIZE/512;
	} else {
		/* 'next' is after the last device address that we
		 * might write to for this chunk in the new layout
		 */
		next = last_dev_address(conf->reshape_progress, &conf->geo);

		/* 'safe' is the earliest device address that we might
		 * read from in the old layout after a restart
		 */
		safe = first_dev_address(conf->reshape_safe, &conf->prev);

		/* Need to update metadata if 'next' might be beyond 'safe'
		 * as that would possibly corrupt data
		 */
		if (next > safe + conf->offset_diff)
			need_flush = 1;

		sector_nr = conf->reshape_progress;
		last  = sector_nr | (conf->geo.chunk_mask
				     & conf->prev.chunk_mask);

		if (sector_nr + RESYNC_BLOCK_SIZE/512 <= last)
			last = sector_nr + RESYNC_BLOCK_SIZE/512 - 1;
	}

	if (need_flush ||
	    time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
		/* Need to update reshape_position in metadata */
		wait_barrier(conf);
		mddev->reshape_position = conf->reshape_progress;
		if (mddev->reshape_backwards)
			mddev->curr_resync_completed = raid10_size(mddev, 0, 0)
				- conf->reshape_progress;
		else
			mddev->curr_resync_completed = conf->reshape_progress;
		conf->reshape_checkpoint = jiffies;
		set_bit(MD_CHANGE_DEVS, &mddev->flags);
		md_wakeup_thread(mddev->thread);
		wait_event(mddev->sb_wait, mddev->flags == 0 ||
			   kthread_should_stop());
		conf->reshape_safe = mddev->reshape_position;
		allow_barrier(conf);
	}

read_more:
	/* Now schedule reads for blocks from sector_nr to last */
	r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
	raise_barrier(conf, sectors_done != 0);
	atomic_set(&r10_bio->remaining, 0);
	r10_bio->mddev = mddev;
	r10_bio->sector = sector_nr;
	set_bit(R10BIO_IsReshape, &r10_bio->state);
	r10_bio->sectors = last - sector_nr + 1;
	rdev = read_balance(conf, r10_bio, &max_sectors);
	BUG_ON(!test_bit(R10BIO_Previous, &r10_bio->state));

	if (!rdev) {
		/* Cannot read from here, so need to record bad blocks
		 * on all the target devices.
		 */
		// FIXME
		set_bit(MD_RECOVERY_INTR, &mddev->recovery);
		return sectors_done;
	}

	read_bio = bio_alloc_mddev(GFP_KERNEL, RESYNC_PAGES, mddev);

	read_bio->bi_bdev = rdev->bdev;
	read_bio->bi_sector = (r10_bio->devs[r10_bio->read_slot].addr
			       + rdev->data_offset);
	read_bio->bi_private = r10_bio;
	read_bio->bi_end_io = end_sync_read;
	read_bio->bi_rw = READ;
	read_bio->bi_flags &= ~(BIO_POOL_MASK - 1);
	read_bio->bi_flags |= 1 << BIO_UPTODATE;
	read_bio->bi_vcnt = 0;
	read_bio->bi_idx = 0;
	read_bio->bi_size = 0;
	r10_bio->master_bio = read_bio;
	r10_bio->read_slot = r10_bio->devs[r10_bio->read_slot].devnum;

	/* Now find the locations in the new layout */
	__raid10_find_phys(&conf->geo, r10_bio);

	blist = read_bio;
	read_bio->bi_next = NULL;

	for (s = 0; s < conf->copies*2; s++) {
		struct bio *b;
		int d = r10_bio->devs[s/2].devnum;
		struct md_rdev *rdev2;
		if (s&1) {
			rdev2 = conf->mirrors[d].replacement;
			b = r10_bio->devs[s/2].repl_bio;
		} else {
			rdev2 = conf->mirrors[d].rdev;
			b = r10_bio->devs[s/2].bio;
		}
		if (!rdev2 || test_bit(Faulty, &rdev2->flags))
			continue;
		b->bi_bdev = rdev2->bdev;
		b->bi_sector = r10_bio->devs[s/2].addr + rdev2->new_data_offset;
		b->bi_private = r10_bio;
		b->bi_end_io = end_reshape_write;
		b->bi_rw = WRITE;
		b->bi_flags &= ~(BIO_POOL_MASK - 1);
		b->bi_flags |= 1 << BIO_UPTODATE;
		b->bi_next = blist;
		b->bi_vcnt = 0;
		b->bi_idx = 0;
		b->bi_size = 0;
		blist = b;
	}

	/* Now add as many pages as possible to all of these bios. */

	nr_sectors = 0;
	for (s = 0 ; s < max_sectors; s += PAGE_SIZE >> 9) {
		struct page *page = r10_bio->devs[0].bio->bi_io_vec[s/(PAGE_SIZE>>9)].bv_page;
		int len = (max_sectors - s) << 9;
		if (len > PAGE_SIZE)
			len = PAGE_SIZE;
		for (bio = blist; bio ; bio = bio->bi_next) {
			struct bio *bio2;
			if (bio_add_page(bio, page, len, 0))
				continue;

			/* Didn't fit, must stop */
			for (bio2 = blist;
			     bio2 && bio2 != bio;
			     bio2 = bio2->bi_next) {
				/* Remove last page from this bio */
				bio2->bi_vcnt--;
				bio2->bi_size -= len;
				bio2->bi_flags &= ~(1<<BIO_SEG_VALID);
			}
			goto bio_full;
		}
		sector_nr += len >> 9;
		nr_sectors += len >> 9;
	}
bio_full:
	r10_bio->sectors = nr_sectors;

	/* Now submit the read */
	md_sync_acct(read_bio->bi_bdev, r10_bio->sectors);
	atomic_inc(&r10_bio->remaining);
	read_bio->bi_next = NULL;
	generic_make_request(read_bio);
	sector_nr += nr_sectors;
	sectors_done += nr_sectors;
	if (sector_nr <= last)
		goto read_more;

	/* Now that we have done the whole section we can
	 * update reshape_progress
	 */
	if (mddev->reshape_backwards)
		conf->reshape_progress -= sectors_done;
	else
		conf->reshape_progress += sectors_done;

	return sectors_done;
}

static void end_reshape_request(struct r10bio *r10_bio);
static int handle_reshape_read_error(struct mddev *mddev,
				     struct r10bio *r10_bio);
static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio)
{
	/* Reshape read completed.  Hopefully we have a block
	 * to write out.
	 * If we got a read error then we do sync 1-page reads from
	 * elsewhere until we find the data - or give up.
	 */
	struct r10conf *conf = mddev->private;
	int s;

	if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
		if (handle_reshape_read_error(mddev, r10_bio) < 0) {
			/* Reshape has been aborted */
			md_done_sync(mddev, r10_bio->sectors, 0);
			return;
		}

	/* We definitely have the data in the pages, schedule the
	 * writes.
	 */
	atomic_set(&r10_bio->remaining, 1);
	for (s = 0; s < conf->copies*2; s++) {
		struct bio *b;
		int d = r10_bio->devs[s/2].devnum;
		struct md_rdev *rdev;
		if (s&1) {
			rdev = conf->mirrors[d].replacement;
			b = r10_bio->devs[s/2].repl_bio;
		} else {
			rdev = conf->mirrors[d].rdev;
			b = r10_bio->devs[s/2].bio;
		}
		if (!rdev || test_bit(Faulty, &rdev->flags))
			continue;
		atomic_inc(&rdev->nr_pending);
		md_sync_acct(b->bi_bdev, r10_bio->sectors);
		atomic_inc(&r10_bio->remaining);
		b->bi_next = NULL;
		generic_make_request(b);
	}
	end_reshape_request(r10_bio);
}

static void end_reshape(struct r10conf *conf)
{
	if (test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery))
		return;

	spin_lock_irq(&conf->device_lock);
	conf->prev = conf->geo;
	md_finish_reshape(conf->mddev);
	smp_wmb();
	conf->reshape_progress = MaxSector;
	spin_unlock_irq(&conf->device_lock);

	/* read-ahead size must cover two whole stripes, which is
	 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices
	 */
	if (conf->mddev->queue) {
		int stripe = conf->geo.raid_disks *
			((conf->mddev->chunk_sectors << 9) / PAGE_SIZE);
		stripe /= conf->geo.near_copies;
		if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
			conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
	}
	conf->fullsync = 0;
}


static int handle_reshape_read_error(struct mddev *mddev,
				     struct r10bio *r10_bio)
{
	/* Use sync reads to get the blocks from somewhere else */
	int sectors = r10_bio->sectors;
	struct r10bio r10b;
	struct r10conf *conf = mddev->private;
	int slot = 0;
	int idx = 0;
	struct bio_vec *bvec = r10_bio->master_bio->bi_io_vec;

	r10b.sector = r10_bio->sector;
	__raid10_find_phys(&conf->prev, &r10b);

	while (sectors) {
		int s = sectors;
		int success = 0;
		int first_slot = slot;

		if (s > (PAGE_SIZE >> 9))
			s = PAGE_SIZE >> 9;

		while (!success) {
			int d = r10b.devs[slot].devnum;
			struct md_rdev *rdev = conf->mirrors[d].rdev;
			sector_t addr;
			if (rdev == NULL ||
			    test_bit(Faulty, &rdev->flags) ||
			    !test_bit(In_sync, &rdev->flags))
				goto failed;

			addr = r10b.devs[slot].addr + idx * PAGE_SIZE;
			success = sync_page_io(rdev,
					       addr,
					       s << 9,
					       bvec[idx].bv_page,
					       READ, false);
			if (success)
				break;
		failed:
			slot++;
			if (slot >= conf->copies)
				slot = 0;
			if (slot == first_slot)
				break;
		}
		if (!success) {
			/* couldn't read this block, must give up */
			set_bit(MD_RECOVERY_INTR,
				&mddev->recovery);
			return -EIO;
		}
		sectors -= s;
		idx++;
	}
	return 0;
}

static void end_reshape_write(struct bio *bio, int error)
{
	int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
	struct r10bio *r10_bio = bio->bi_private;
	struct mddev *mddev = r10_bio->mddev;
	struct r10conf *conf = mddev->private;
	int d;
	int slot;
	int repl;
	struct md_rdev *rdev = NULL;

	d = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
	if (repl)
		rdev = conf->mirrors[d].replacement;
	if (!rdev) {
		smp_mb();
		rdev = conf->mirrors[d].rdev;
	}

	if (!uptodate) {
		/* FIXME should record badblock */
		md_error(mddev, rdev);
	}

	rdev_dec_pending(rdev, mddev);
	end_reshape_request(r10_bio);
}

static void end_reshape_request(struct r10bio *r10_bio)
{
	if (!atomic_dec_and_test(&r10_bio->remaining))
		return;
	md_done_sync(r10_bio->mddev, r10_bio->sectors, 1);
	bio_put(r10_bio->master_bio);
	put_buf(r10_bio);
}

static void raid10_finish_reshape(struct mddev *mddev)
{
	struct r10conf *conf = mddev->private;

	if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
		return;

	if (mddev->delta_disks > 0) {
		sector_t size = raid10_size(mddev, 0, 0);
		md_set_array_sectors(mddev, size);
		if (mddev->recovery_cp > mddev->resync_max_sectors) {
			mddev->recovery_cp = mddev->resync_max_sectors;
			set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
		}
		mddev->resync_max_sectors = size;
		set_capacity(mddev->gendisk, mddev->array_sectors);
		revalidate_disk(mddev->gendisk);
	}
	mddev->layout = mddev->new_layout;
	mddev->chunk_sectors = 1 << conf->geo.chunk_shift;
	mddev->reshape_position = MaxSector;
	mddev->delta_disks = 0;
	mddev->reshape_backwards = 0;
}

4485
static struct md_personality raid10_personality =
L
Linus Torvalds 已提交
4486 4487
{
	.name		= "raid10",
4488
	.level		= 10,
L
Linus Torvalds 已提交
4489 4490 4491 4492 4493 4494 4495 4496 4497 4498
	.owner		= THIS_MODULE,
	.make_request	= make_request,
	.run		= run,
	.stop		= stop,
	.status		= status,
	.error_handler	= error,
	.hot_add_disk	= raid10_add_disk,
	.hot_remove_disk= raid10_remove_disk,
	.spare_active	= raid10_spare_active,
	.sync_request	= sync_request,
4499
	.quiesce	= raid10_quiesce,
4500
	.size		= raid10_size,
4501
	.resize		= raid10_resize,
4502
	.takeover	= raid10_takeover,
N
NeilBrown 已提交
4503 4504 4505
	.check_reshape	= raid10_check_reshape,
	.start_reshape	= raid10_start_reshape,
	.finish_reshape	= raid10_finish_reshape,
L
Linus Torvalds 已提交
4506 4507 4508 4509
};

static int __init raid_init(void)
{
4510
	return register_md_personality(&raid10_personality);
L
Linus Torvalds 已提交
4511 4512 4513 4514
}

static void raid_exit(void)
{
4515
	unregister_md_personality(&raid10_personality);
L
Linus Torvalds 已提交
4516 4517 4518 4519 4520
}

module_init(raid_init);
module_exit(raid_exit);
MODULE_LICENSE("GPL");
4521
MODULE_DESCRIPTION("RAID10 (striped mirror) personality for MD");
L
Linus Torvalds 已提交
4522
MODULE_ALIAS("md-personality-9"); /* RAID10 */
4523
MODULE_ALIAS("md-raid10");
4524
MODULE_ALIAS("md-level-10");
4525 4526

module_param(max_queued_requests, int, S_IRUGO|S_IWUSR);