raid10.c 129.4 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7
/*
 * raid10.c : Multiple Devices driver for Linux
 *
 * Copyright (C) 2000-2004 Neil Brown
 *
 * RAID-10 support for md.
 *
L
Lucas De Marchi 已提交
8
 * Base on code in raid1.c.  See raid1.c for further copyright information.
L
Linus Torvalds 已提交
9 10 11 12 13 14 15 16 17 18 19 20
 *
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2, or (at your option)
 * any later version.
 *
 * You should have received a copy of the GNU General Public License
 * (for example /usr/src/linux/COPYING); if not, write to the Free
 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

21
#include <linux/slab.h>
22
#include <linux/delay.h>
23
#include <linux/blkdev.h>
24
#include <linux/module.h>
25
#include <linux/seq_file.h>
26
#include <linux/ratelimit.h>
N
NeilBrown 已提交
27
#include <linux/kthread.h>
28
#include "md.h"
29
#include "raid10.h"
30
#include "raid0.h"
31
#include "bitmap.h"
L
Linus Torvalds 已提交
32 33 34 35 36 37 38 39

/*
 * RAID10 provides a combination of RAID0 and RAID1 functionality.
 * The layout of data is defined by
 *    chunk_size
 *    raid_disks
 *    near_copies (stored in low byte of layout)
 *    far_copies (stored in second byte of layout)
40
 *    far_offset (stored in bit 16 of layout )
41
 *    use_far_sets (stored in bit 17 of layout )
L
Linus Torvalds 已提交
42
 *
43 44 45 46 47 48 49 50
 * The data to be stored is divided into chunks using chunksize.  Each device
 * is divided into far_copies sections.   In each section, chunks are laid out
 * in a style similar to raid0, but near_copies copies of each chunk is stored
 * (each on a different drive).  The starting device for each section is offset
 * near_copies from the starting device of the previous section.  Thus there
 * are (near_copies * far_copies) of each chunk, and each is on a different
 * drive.  near_copies and far_copies must be at least one, and their product
 * is at most raid_disks.
51 52
 *
 * If far_offset is true, then the far_copies are handled a bit differently.
53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70
 * The copies are still in different stripes, but instead of being very far
 * apart on disk, there are adjacent stripes.
 *
 * The far and offset algorithms are handled slightly differently if
 * 'use_far_sets' is true.  In this case, the array's devices are grouped into
 * sets that are (near_copies * far_copies) in size.  The far copied stripes
 * are still shifted by 'near_copies' devices, but this shifting stays confined
 * to the set rather than the entire array.  This is done to improve the number
 * of device combinations that can fail without causing the array to fail.
 * Example 'far' algorithm w/o 'use_far_sets' (each letter represents a chunk
 * on a device):
 *    A B C D    A B C D E
 *      ...         ...
 *    D A B C    E A B C D
 * Example 'far' algorithm w/ 'use_far_sets' enabled (sets illustrated w/ []'s):
 *    [A B] [C D]    [A B] [C D E]
 *    |...| |...|    |...| | ... |
 *    [B A] [D C]    [B A] [E C D]
L
Linus Torvalds 已提交
71 72 73 74 75 76 77
 */

/*
 * Number of guaranteed r10bios in case of extreme VM load:
 */
#define	NR_RAID10_BIOS 256

78 79 80 81 82 83 84 85 86 87 88 89 90 91 92
/* when we get a read error on a read-only array, we redirect to another
 * device without failing the first device, or trying to over-write to
 * correct the read error.  To keep track of bad blocks on a per-bio
 * level, we store IO_BLOCKED in the appropriate 'bios' pointer
 */
#define IO_BLOCKED ((struct bio *)1)
/* When we successfully write to a known bad-block, we need to remove the
 * bad-block marking which must be done from process context.  So we record
 * the success by setting devs[n].bio to IO_MADE_GOOD
 */
#define IO_MADE_GOOD ((struct bio *)2)

#define BIO_SPECIAL(bio) ((unsigned long)bio <= 2)

/* When there are this many requests queued to be written by
93 94 95 96 97
 * the raid10 thread, we become 'congested' to provide back-pressure
 * for writeback.
 */
static int max_queued_requests = 1024;

98 99
static void allow_barrier(struct r10conf *conf);
static void lower_barrier(struct r10conf *conf);
100
static int _enough(struct r10conf *conf, int previous, int ignore);
N
NeilBrown 已提交
101 102 103
static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
				int *skipped);
static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio);
104
static void end_reshape_write(struct bio *bio);
N
NeilBrown 已提交
105
static void end_reshape(struct r10conf *conf);
106

A
Al Viro 已提交
107
static void * r10bio_pool_alloc(gfp_t gfp_flags, void *data)
L
Linus Torvalds 已提交
108
{
109
	struct r10conf *conf = data;
110
	int size = offsetof(struct r10bio, devs[conf->copies]);
L
Linus Torvalds 已提交
111

112 113
	/* allocate a r10bio with room for raid_disks entries in the
	 * bios array */
J
Jens Axboe 已提交
114
	return kzalloc(size, gfp_flags);
L
Linus Torvalds 已提交
115 116 117 118 119 120 121
}

static void r10bio_pool_free(void *r10_bio, void *data)
{
	kfree(r10_bio);
}

122
/* Maximum size of each resync request */
L
Linus Torvalds 已提交
123 124
#define RESYNC_BLOCK_SIZE (64*1024)
#define RESYNC_PAGES ((RESYNC_BLOCK_SIZE + PAGE_SIZE-1) / PAGE_SIZE)
125 126 127 128
/* amount of memory to reserve for resync requests */
#define RESYNC_WINDOW (1024*1024)
/* maximum number of concurrent requests, memory permitting */
#define RESYNC_DEPTH (32*1024*1024/RESYNC_BLOCK_SIZE)
L
Linus Torvalds 已提交
129 130 131 132 133 134 135 136

/*
 * When performing a resync, we need to read and compare, so
 * we need as many pages are there are copies.
 * When performing a recovery, we need 2 bios, one for read,
 * one for write (we recover only one drive per r10buf)
 *
 */
A
Al Viro 已提交
137
static void * r10buf_pool_alloc(gfp_t gfp_flags, void *data)
L
Linus Torvalds 已提交
138
{
139
	struct r10conf *conf = data;
L
Linus Torvalds 已提交
140
	struct page *page;
141
	struct r10bio *r10_bio;
L
Linus Torvalds 已提交
142 143 144 145 146
	struct bio *bio;
	int i, j;
	int nalloc;

	r10_bio = r10bio_pool_alloc(gfp_flags, conf);
J
Jens Axboe 已提交
147
	if (!r10_bio)
L
Linus Torvalds 已提交
148 149
		return NULL;

N
NeilBrown 已提交
150 151
	if (test_bit(MD_RECOVERY_SYNC, &conf->mddev->recovery) ||
	    test_bit(MD_RECOVERY_RESHAPE, &conf->mddev->recovery))
L
Linus Torvalds 已提交
152 153 154 155 156 157 158 159
		nalloc = conf->copies; /* resync */
	else
		nalloc = 2; /* recovery */

	/*
	 * Allocate bios.
	 */
	for (j = nalloc ; j-- ; ) {
160
		bio = bio_kmalloc(gfp_flags, RESYNC_PAGES);
L
Linus Torvalds 已提交
161 162 163
		if (!bio)
			goto out_free_bio;
		r10_bio->devs[j].bio = bio;
164 165 166 167 168 169
		if (!conf->have_replacement)
			continue;
		bio = bio_kmalloc(gfp_flags, RESYNC_PAGES);
		if (!bio)
			goto out_free_bio;
		r10_bio->devs[j].repl_bio = bio;
L
Linus Torvalds 已提交
170 171 172 173 174 175
	}
	/*
	 * Allocate RESYNC_PAGES data pages and attach them
	 * where needed.
	 */
	for (j = 0 ; j < nalloc; j++) {
176
		struct bio *rbio = r10_bio->devs[j].repl_bio;
L
Linus Torvalds 已提交
177 178
		bio = r10_bio->devs[j].bio;
		for (i = 0; i < RESYNC_PAGES; i++) {
N
NeilBrown 已提交
179 180 181 182
			if (j > 0 && !test_bit(MD_RECOVERY_SYNC,
					       &conf->mddev->recovery)) {
				/* we can share bv_page's during recovery
				 * and reshape */
183 184 185 186 187
				struct bio *rbio = r10_bio->devs[0].bio;
				page = rbio->bi_io_vec[i].bv_page;
				get_page(page);
			} else
				page = alloc_page(gfp_flags);
L
Linus Torvalds 已提交
188 189 190 191
			if (unlikely(!page))
				goto out_free_pages;

			bio->bi_io_vec[i].bv_page = page;
192 193
			if (rbio)
				rbio->bi_io_vec[i].bv_page = page;
L
Linus Torvalds 已提交
194 195 196 197 198 199 200
		}
	}

	return r10_bio;

out_free_pages:
	for ( ; i > 0 ; i--)
201
		safe_put_page(bio->bi_io_vec[i-1].bv_page);
L
Linus Torvalds 已提交
202 203
	while (j--)
		for (i = 0; i < RESYNC_PAGES ; i++)
204
			safe_put_page(r10_bio->devs[j].bio->bi_io_vec[i].bv_page);
205
	j = 0;
L
Linus Torvalds 已提交
206
out_free_bio:
207 208 209
	for ( ; j < nalloc; j++) {
		if (r10_bio->devs[j].bio)
			bio_put(r10_bio->devs[j].bio);
210 211 212
		if (r10_bio->devs[j].repl_bio)
			bio_put(r10_bio->devs[j].repl_bio);
	}
L
Linus Torvalds 已提交
213 214 215 216 217 218 219
	r10bio_pool_free(r10_bio, conf);
	return NULL;
}

static void r10buf_pool_free(void *__r10_bio, void *data)
{
	int i;
220
	struct r10conf *conf = data;
221
	struct r10bio *r10bio = __r10_bio;
L
Linus Torvalds 已提交
222 223 224 225 226 227
	int j;

	for (j=0; j < conf->copies; j++) {
		struct bio *bio = r10bio->devs[j].bio;
		if (bio) {
			for (i = 0; i < RESYNC_PAGES; i++) {
228
				safe_put_page(bio->bi_io_vec[i].bv_page);
L
Linus Torvalds 已提交
229 230 231 232
				bio->bi_io_vec[i].bv_page = NULL;
			}
			bio_put(bio);
		}
233 234 235
		bio = r10bio->devs[j].repl_bio;
		if (bio)
			bio_put(bio);
L
Linus Torvalds 已提交
236 237 238 239
	}
	r10bio_pool_free(r10bio, conf);
}

240
static void put_all_bios(struct r10conf *conf, struct r10bio *r10_bio)
L
Linus Torvalds 已提交
241 242 243 244 245
{
	int i;

	for (i = 0; i < conf->copies; i++) {
		struct bio **bio = & r10_bio->devs[i].bio;
246
		if (!BIO_SPECIAL(*bio))
L
Linus Torvalds 已提交
247 248
			bio_put(*bio);
		*bio = NULL;
249 250 251 252
		bio = &r10_bio->devs[i].repl_bio;
		if (r10_bio->read_slot < 0 && !BIO_SPECIAL(*bio))
			bio_put(*bio);
		*bio = NULL;
L
Linus Torvalds 已提交
253 254 255
	}
}

256
static void free_r10bio(struct r10bio *r10_bio)
L
Linus Torvalds 已提交
257
{
258
	struct r10conf *conf = r10_bio->mddev->private;
L
Linus Torvalds 已提交
259 260 261 262 263

	put_all_bios(conf, r10_bio);
	mempool_free(r10_bio, conf->r10bio_pool);
}

264
static void put_buf(struct r10bio *r10_bio)
L
Linus Torvalds 已提交
265
{
266
	struct r10conf *conf = r10_bio->mddev->private;
L
Linus Torvalds 已提交
267 268 269

	mempool_free(r10_bio, conf->r10buf_pool);

270
	lower_barrier(conf);
L
Linus Torvalds 已提交
271 272
}

273
static void reschedule_retry(struct r10bio *r10_bio)
L
Linus Torvalds 已提交
274 275
{
	unsigned long flags;
276
	struct mddev *mddev = r10_bio->mddev;
277
	struct r10conf *conf = mddev->private;
L
Linus Torvalds 已提交
278 279 280

	spin_lock_irqsave(&conf->device_lock, flags);
	list_add(&r10_bio->retry_list, &conf->retry_list);
281
	conf->nr_queued ++;
L
Linus Torvalds 已提交
282 283
	spin_unlock_irqrestore(&conf->device_lock, flags);

A
Arthur Jones 已提交
284 285 286
	/* wake up frozen array... */
	wake_up(&conf->wait_barrier);

L
Linus Torvalds 已提交
287 288 289 290 291 292 293 294
	md_wakeup_thread(mddev->thread);
}

/*
 * raid_end_bio_io() is called when we have finished servicing a mirrored
 * operation and are ready to return a success/failure code to the buffer
 * cache layer.
 */
295
static void raid_end_bio_io(struct r10bio *r10_bio)
L
Linus Torvalds 已提交
296 297
{
	struct bio *bio = r10_bio->master_bio;
298
	int done;
299
	struct r10conf *conf = r10_bio->mddev->private;
L
Linus Torvalds 已提交
300

301 302 303 304 305 306 307 308 309
	if (bio->bi_phys_segments) {
		unsigned long flags;
		spin_lock_irqsave(&conf->device_lock, flags);
		bio->bi_phys_segments--;
		done = (bio->bi_phys_segments == 0);
		spin_unlock_irqrestore(&conf->device_lock, flags);
	} else
		done = 1;
	if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
310
		bio->bi_error = -EIO;
311
	if (done) {
312
		bio_endio(bio);
313 314 315 316 317 318
		/*
		 * Wake up any possible resync thread that waits for the device
		 * to go idle.
		 */
		allow_barrier(conf);
	}
L
Linus Torvalds 已提交
319 320 321 322 323 324
	free_r10bio(r10_bio);
}

/*
 * Update disk head position estimator based on IRQ completion info.
 */
325
static inline void update_head_pos(int slot, struct r10bio *r10_bio)
L
Linus Torvalds 已提交
326
{
327
	struct r10conf *conf = r10_bio->mddev->private;
L
Linus Torvalds 已提交
328 329 330 331 332

	conf->mirrors[r10_bio->devs[slot].devnum].head_position =
		r10_bio->devs[slot].addr + (r10_bio->sectors);
}

333 334 335
/*
 * Find the disk number which triggered given bio
 */
336
static int find_bio_disk(struct r10conf *conf, struct r10bio *r10_bio,
337
			 struct bio *bio, int *slotp, int *replp)
338 339
{
	int slot;
340
	int repl = 0;
341

342
	for (slot = 0; slot < conf->copies; slot++) {
343 344
		if (r10_bio->devs[slot].bio == bio)
			break;
345 346 347 348 349
		if (r10_bio->devs[slot].repl_bio == bio) {
			repl = 1;
			break;
		}
	}
350 351 352 353

	BUG_ON(slot == conf->copies);
	update_head_pos(slot, r10_bio);

354 355
	if (slotp)
		*slotp = slot;
356 357
	if (replp)
		*replp = repl;
358 359 360
	return r10_bio->devs[slot].devnum;
}

361
static void raid10_end_read_request(struct bio *bio)
L
Linus Torvalds 已提交
362
{
363
	int uptodate = !bio->bi_error;
364
	struct r10bio *r10_bio = bio->bi_private;
L
Linus Torvalds 已提交
365
	int slot, dev;
366
	struct md_rdev *rdev;
367
	struct r10conf *conf = r10_bio->mddev->private;
L
Linus Torvalds 已提交
368 369 370

	slot = r10_bio->read_slot;
	dev = r10_bio->devs[slot].devnum;
371
	rdev = r10_bio->devs[slot].rdev;
L
Linus Torvalds 已提交
372 373 374
	/*
	 * this branch is our 'one mirror IO has finished' event handler:
	 */
375 376 377
	update_head_pos(slot, r10_bio);

	if (uptodate) {
L
Linus Torvalds 已提交
378 379 380 381 382 383 384 385 386 387
		/*
		 * Set R10BIO_Uptodate in our master bio, so that
		 * we will return a good error code to the higher
		 * levels even if IO on some other mirrored buffer fails.
		 *
		 * The 'master' represents the composite IO operation to
		 * user-side. So if something waits for IO, then it will
		 * wait for the 'master' bio.
		 */
		set_bit(R10BIO_Uptodate, &r10_bio->state);
388 389 390 391 392 393
	} else {
		/* If all other devices that store this block have
		 * failed, we want to return the error upwards rather
		 * than fail the last device.  Here we redefine
		 * "uptodate" to mean "Don't want to retry"
		 */
394 395
		if (!_enough(conf, test_bit(R10BIO_Previous, &r10_bio->state),
			     rdev->raid_disk))
396 397 398
			uptodate = 1;
	}
	if (uptodate) {
L
Linus Torvalds 已提交
399
		raid_end_bio_io(r10_bio);
400
		rdev_dec_pending(rdev, conf->mddev);
401
	} else {
L
Linus Torvalds 已提交
402
		/*
403
		 * oops, read error - keep the refcount on the rdev
L
Linus Torvalds 已提交
404 405
		 */
		char b[BDEVNAME_SIZE];
406 407 408
		printk_ratelimited(KERN_ERR
				   "md/raid10:%s: %s: rescheduling sector %llu\n",
				   mdname(conf->mddev),
409
				   bdevname(rdev->bdev, b),
410
				   (unsigned long long)r10_bio->sector);
411
		set_bit(R10BIO_ReadError, &r10_bio->state);
L
Linus Torvalds 已提交
412 413 414 415
		reschedule_retry(r10_bio);
	}
}

416
static void close_write(struct r10bio *r10_bio)
417 418 419 420 421 422 423 424 425
{
	/* clear the bitmap if all writes complete successfully */
	bitmap_endwrite(r10_bio->mddev->bitmap, r10_bio->sector,
			r10_bio->sectors,
			!test_bit(R10BIO_Degraded, &r10_bio->state),
			0);
	md_write_end(r10_bio->mddev);
}

426
static void one_write_done(struct r10bio *r10_bio)
427 428 429 430 431 432 433 434 435 436 437 438 439 440
{
	if (atomic_dec_and_test(&r10_bio->remaining)) {
		if (test_bit(R10BIO_WriteError, &r10_bio->state))
			reschedule_retry(r10_bio);
		else {
			close_write(r10_bio);
			if (test_bit(R10BIO_MadeGood, &r10_bio->state))
				reschedule_retry(r10_bio);
			else
				raid_end_bio_io(r10_bio);
		}
	}
}

441
static void raid10_end_write_request(struct bio *bio)
L
Linus Torvalds 已提交
442
{
443
	struct r10bio *r10_bio = bio->bi_private;
444
	int dev;
445
	int dec_rdev = 1;
446
	struct r10conf *conf = r10_bio->mddev->private;
447
	int slot, repl;
448
	struct md_rdev *rdev = NULL;
L
Linus Torvalds 已提交
449

450
	dev = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
L
Linus Torvalds 已提交
451

452 453
	if (repl)
		rdev = conf->mirrors[dev].replacement;
454 455 456
	if (!rdev) {
		smp_rmb();
		repl = 0;
457
		rdev = conf->mirrors[dev].rdev;
458
	}
L
Linus Torvalds 已提交
459 460 461
	/*
	 * this branch is our 'one mirror IO has finished' event handler:
	 */
462
	if (bio->bi_error) {
463 464 465 466 467 468 469
		if (repl)
			/* Never record new bad blocks to replacement,
			 * just fail it.
			 */
			md_error(rdev->mddev, rdev);
		else {
			set_bit(WriteErrorSeen,	&rdev->flags);
470 471 472
			if (!test_and_set_bit(WantReplacement, &rdev->flags))
				set_bit(MD_RECOVERY_NEEDED,
					&rdev->mddev->recovery);
473 474 475
			set_bit(R10BIO_WriteError, &r10_bio->state);
			dec_rdev = 0;
		}
476
	} else {
L
Linus Torvalds 已提交
477 478 479 480 481 482 483 484 485
		/*
		 * Set R10BIO_Uptodate in our master bio, so that
		 * we will return a good error code for to the higher
		 * levels even if IO on some other mirrored buffer fails.
		 *
		 * The 'master' represents the composite IO operation to
		 * user-side. So if something waits for IO, then it will
		 * wait for the 'master' bio.
		 */
486 487 488
		sector_t first_bad;
		int bad_sectors;

489 490 491 492 493 494 495 496 497 498 499
		/*
		 * Do not set R10BIO_Uptodate if the current device is
		 * rebuilding or Faulty. This is because we cannot use
		 * such device for properly reading the data back (we could
		 * potentially use it, if the current write would have felt
		 * before rdev->recovery_offset, but for simplicity we don't
		 * check this here.
		 */
		if (test_bit(In_sync, &rdev->flags) &&
		    !test_bit(Faulty, &rdev->flags))
			set_bit(R10BIO_Uptodate, &r10_bio->state);
L
Linus Torvalds 已提交
500

501
		/* Maybe we can clear some bad blocks. */
502
		if (is_badblock(rdev,
503 504 505 506
				r10_bio->devs[slot].addr,
				r10_bio->sectors,
				&first_bad, &bad_sectors)) {
			bio_put(bio);
507 508 509 510
			if (repl)
				r10_bio->devs[slot].repl_bio = IO_MADE_GOOD;
			else
				r10_bio->devs[slot].bio = IO_MADE_GOOD;
511 512 513 514 515
			dec_rdev = 0;
			set_bit(R10BIO_MadeGood, &r10_bio->state);
		}
	}

L
Linus Torvalds 已提交
516 517 518 519 520
	/*
	 *
	 * Let's see if all mirrored write operations have finished
	 * already.
	 */
521
	one_write_done(r10_bio);
522
	if (dec_rdev)
523
		rdev_dec_pending(rdev, conf->mddev);
L
Linus Torvalds 已提交
524 525 526 527
}

/*
 * RAID10 layout manager
L
Lucas De Marchi 已提交
528
 * As well as the chunksize and raid_disks count, there are two
L
Linus Torvalds 已提交
529 530 531 532 533 534
 * parameters: near_copies and far_copies.
 * near_copies * far_copies must be <= raid_disks.
 * Normally one of these will be 1.
 * If both are 1, we get raid0.
 * If near_copies == raid_disks, we get raid1.
 *
L
Lucas De Marchi 已提交
535
 * Chunks are laid out in raid0 style with near_copies copies of the
L
Linus Torvalds 已提交
536 537 538 539 540 541 542 543 544
 * first chunk, followed by near_copies copies of the next chunk and
 * so on.
 * If far_copies > 1, then after 1/far_copies of the array has been assigned
 * as described above, we start again with a device offset of near_copies.
 * So we effectively have another copy of the whole array further down all
 * the drives, but with blocks on different drives.
 * With this layout, and block is never stored twice on the one device.
 *
 * raid10_find_phys finds the sector offset of a given virtual sector
545
 * on each device that it is on.
L
Linus Torvalds 已提交
546 547 548 549 550
 *
 * raid10_find_virt does the reverse mapping, from a device and a
 * sector offset to a virtual address
 */

551
static void __raid10_find_phys(struct geom *geo, struct r10bio *r10bio)
L
Linus Torvalds 已提交
552 553 554 555 556 557 558
{
	int n,f;
	sector_t sector;
	sector_t chunk;
	sector_t stripe;
	int dev;
	int slot = 0;
559 560 561 562 563 564 565
	int last_far_set_start, last_far_set_size;

	last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1;
	last_far_set_start *= geo->far_set_size;

	last_far_set_size = geo->far_set_size;
	last_far_set_size += (geo->raid_disks % geo->far_set_size);
L
Linus Torvalds 已提交
566 567

	/* now calculate first sector/dev */
568 569
	chunk = r10bio->sector >> geo->chunk_shift;
	sector = r10bio->sector & geo->chunk_mask;
L
Linus Torvalds 已提交
570

571
	chunk *= geo->near_copies;
L
Linus Torvalds 已提交
572
	stripe = chunk;
573 574 575
	dev = sector_div(stripe, geo->raid_disks);
	if (geo->far_offset)
		stripe *= geo->far_copies;
L
Linus Torvalds 已提交
576

577
	sector += stripe << geo->chunk_shift;
L
Linus Torvalds 已提交
578 579

	/* and calculate all the others */
580
	for (n = 0; n < geo->near_copies; n++) {
L
Linus Torvalds 已提交
581
		int d = dev;
582
		int set;
L
Linus Torvalds 已提交
583 584
		sector_t s = sector;
		r10bio->devs[slot].devnum = d;
585
		r10bio->devs[slot].addr = s;
L
Linus Torvalds 已提交
586 587
		slot++;

588
		for (f = 1; f < geo->far_copies; f++) {
589
			set = d / geo->far_set_size;
590
			d += geo->near_copies;
591

592 593 594 595 596 597 598 599 600
			if ((geo->raid_disks % geo->far_set_size) &&
			    (d > last_far_set_start)) {
				d -= last_far_set_start;
				d %= last_far_set_size;
				d += last_far_set_start;
			} else {
				d %= geo->far_set_size;
				d += geo->far_set_size * set;
			}
601
			s += geo->stride;
L
Linus Torvalds 已提交
602 603 604 605 606
			r10bio->devs[slot].devnum = d;
			r10bio->devs[slot].addr = s;
			slot++;
		}
		dev++;
607
		if (dev >= geo->raid_disks) {
L
Linus Torvalds 已提交
608
			dev = 0;
609
			sector += (geo->chunk_mask + 1);
L
Linus Torvalds 已提交
610 611
		}
	}
612 613 614 615 616 617 618 619 620 621 622 623 624 625 626
}

static void raid10_find_phys(struct r10conf *conf, struct r10bio *r10bio)
{
	struct geom *geo = &conf->geo;

	if (conf->reshape_progress != MaxSector &&
	    ((r10bio->sector >= conf->reshape_progress) !=
	     conf->mddev->reshape_backwards)) {
		set_bit(R10BIO_Previous, &r10bio->state);
		geo = &conf->prev;
	} else
		clear_bit(R10BIO_Previous, &r10bio->state);

	__raid10_find_phys(geo, r10bio);
L
Linus Torvalds 已提交
627 628
}

629
static sector_t raid10_find_virt(struct r10conf *conf, sector_t sector, int dev)
L
Linus Torvalds 已提交
630 631
{
	sector_t offset, chunk, vchunk;
632 633 634
	/* Never use conf->prev as this is only called during resync
	 * or recovery, so reshape isn't happening
	 */
635
	struct geom *geo = &conf->geo;
636 637
	int far_set_start = (dev / geo->far_set_size) * geo->far_set_size;
	int far_set_size = geo->far_set_size;
638 639 640 641 642 643 644 645 646 647 648 649
	int last_far_set_start;

	if (geo->raid_disks % geo->far_set_size) {
		last_far_set_start = (geo->raid_disks / geo->far_set_size) - 1;
		last_far_set_start *= geo->far_set_size;

		if (dev >= last_far_set_start) {
			far_set_size = geo->far_set_size;
			far_set_size += (geo->raid_disks % geo->far_set_size);
			far_set_start = last_far_set_start;
		}
	}
L
Linus Torvalds 已提交
650

651 652
	offset = sector & geo->chunk_mask;
	if (geo->far_offset) {
653
		int fc;
654 655 656
		chunk = sector >> geo->chunk_shift;
		fc = sector_div(chunk, geo->far_copies);
		dev -= fc * geo->near_copies;
657 658
		if (dev < far_set_start)
			dev += far_set_size;
659
	} else {
660 661
		while (sector >= geo->stride) {
			sector -= geo->stride;
662 663
			if (dev < (geo->near_copies + far_set_start))
				dev += far_set_size - geo->near_copies;
664
			else
665
				dev -= geo->near_copies;
666
		}
667
		chunk = sector >> geo->chunk_shift;
668
	}
669 670 671
	vchunk = chunk * geo->raid_disks + dev;
	sector_div(vchunk, geo->near_copies);
	return (vchunk << geo->chunk_shift) + offset;
L
Linus Torvalds 已提交
672 673 674 675
}

/**
 *	raid10_mergeable_bvec -- tell bio layer if a two requests can be merged
676
 *	@mddev: the md device
677
 *	@bvm: properties of new bio
L
Linus Torvalds 已提交
678 679 680
 *	@biovec: the request that could be merged to it.
 *
 *	Return amount of bytes we can accept at this offset
681 682
 *	This requires checking for end-of-chunk if near_copies != raid_disks,
 *	and for subordinate merge_bvec_fns if merge_check_needed.
L
Linus Torvalds 已提交
683
 */
684
static int raid10_mergeable_bvec(struct mddev *mddev,
685 686
				 struct bvec_merge_data *bvm,
				 struct bio_vec *biovec)
L
Linus Torvalds 已提交
687
{
688
	struct r10conf *conf = mddev->private;
689
	sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev);
L
Linus Torvalds 已提交
690
	int max;
N
NeilBrown 已提交
691
	unsigned int chunk_sectors;
692
	unsigned int bio_sectors = bvm->bi_size >> 9;
693
	struct geom *geo = &conf->geo;
L
Linus Torvalds 已提交
694

N
NeilBrown 已提交
695
	chunk_sectors = (conf->geo.chunk_mask & conf->prev.chunk_mask) + 1;
696 697 698 699 700
	if (conf->reshape_progress != MaxSector &&
	    ((sector >= conf->reshape_progress) !=
	     conf->mddev->reshape_backwards))
		geo = &conf->prev;

701
	if (geo->near_copies < geo->raid_disks) {
702 703 704 705 706 707 708 709 710 711 712
		max = (chunk_sectors - ((sector & (chunk_sectors - 1))
					+ bio_sectors)) << 9;
		if (max < 0)
			/* bio_add cannot handle a negative return */
			max = 0;
		if (max <= biovec->bv_len && bio_sectors == 0)
			return biovec->bv_len;
	} else
		max = biovec->bv_len;

	if (mddev->merge_check_needed) {
713 714 715 716 717
		struct {
			struct r10bio r10_bio;
			struct r10dev devs[conf->copies];
		} on_stack;
		struct r10bio *r10_bio = &on_stack.r10_bio;
718
		int s;
719 720 721 722 723 724
		if (conf->reshape_progress != MaxSector) {
			/* Cannot give any guidance during reshape */
			if (max <= biovec->bv_len && bio_sectors == 0)
				return biovec->bv_len;
			return 0;
		}
725 726
		r10_bio->sector = sector;
		raid10_find_phys(conf, r10_bio);
727 728
		rcu_read_lock();
		for (s = 0; s < conf->copies; s++) {
729
			int disk = r10_bio->devs[s].devnum;
730 731 732 733 734 735
			struct md_rdev *rdev = rcu_dereference(
				conf->mirrors[disk].rdev);
			if (rdev && !test_bit(Faulty, &rdev->flags)) {
				struct request_queue *q =
					bdev_get_queue(rdev->bdev);
				if (q->merge_bvec_fn) {
736
					bvm->bi_sector = r10_bio->devs[s].addr
737 738 739 740 741 742 743 744 745 746 747
						+ rdev->data_offset;
					bvm->bi_bdev = rdev->bdev;
					max = min(max, q->merge_bvec_fn(
							  q, bvm, biovec));
				}
			}
			rdev = rcu_dereference(conf->mirrors[disk].replacement);
			if (rdev && !test_bit(Faulty, &rdev->flags)) {
				struct request_queue *q =
					bdev_get_queue(rdev->bdev);
				if (q->merge_bvec_fn) {
748
					bvm->bi_sector = r10_bio->devs[s].addr
749 750 751 752 753 754 755 756 757 758
						+ rdev->data_offset;
					bvm->bi_bdev = rdev->bdev;
					max = min(max, q->merge_bvec_fn(
							  q, bvm, biovec));
				}
			}
		}
		rcu_read_unlock();
	}
	return max;
L
Linus Torvalds 已提交
759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779
}

/*
 * This routine returns the disk from which the requested read should
 * be done. There is a per-array 'next expected sequential IO' sector
 * number - if this matches on the next IO then we use the last disk.
 * There is also a per-disk 'last know head position' sector that is
 * maintained from IRQ contexts, both the normal and the resync IO
 * completion handlers update this position correctly. If there is no
 * perfect sequential match then we pick the disk whose head is closest.
 *
 * If there are 2 mirrors in the same 2 devices, performance degrades
 * because position is mirror, not device based.
 *
 * The rdev for the device selected will have nr_pending incremented.
 */

/*
 * FIXME: possibly should rethink readbalancing and do it differently
 * depending on near_copies / far_copies geometry.
 */
780 781 782
static struct md_rdev *read_balance(struct r10conf *conf,
				    struct r10bio *r10_bio,
				    int *max_sectors)
L
Linus Torvalds 已提交
783
{
784
	const sector_t this_sector = r10_bio->sector;
N
NeilBrown 已提交
785
	int disk, slot;
786 787
	int sectors = r10_bio->sectors;
	int best_good_sectors;
N
NeilBrown 已提交
788
	sector_t new_distance, best_dist;
789
	struct md_rdev *best_rdev, *rdev = NULL;
N
NeilBrown 已提交
790 791
	int do_balance;
	int best_slot;
792
	struct geom *geo = &conf->geo;
L
Linus Torvalds 已提交
793 794 795

	raid10_find_phys(conf, r10_bio);
	rcu_read_lock();
N
NeilBrown 已提交
796
retry:
797
	sectors = r10_bio->sectors;
N
NeilBrown 已提交
798
	best_slot = -1;
799
	best_rdev = NULL;
N
NeilBrown 已提交
800
	best_dist = MaxSector;
801
	best_good_sectors = 0;
N
NeilBrown 已提交
802
	do_balance = 1;
L
Linus Torvalds 已提交
803 804
	/*
	 * Check if we can balance. We can balance on the whole
805 806 807
	 * device if no resync is going on (recovery is ok), or below
	 * the resync window. We take the first readable disk when
	 * above the resync window.
L
Linus Torvalds 已提交
808 809
	 */
	if (conf->mddev->recovery_cp < MaxSector
N
NeilBrown 已提交
810 811
	    && (this_sector + sectors >= conf->next_resync))
		do_balance = 0;
L
Linus Torvalds 已提交
812

N
NeilBrown 已提交
813
	for (slot = 0; slot < conf->copies ; slot++) {
814 815 816 817
		sector_t first_bad;
		int bad_sectors;
		sector_t dev_sector;

N
NeilBrown 已提交
818 819
		if (r10_bio->devs[slot].bio == IO_BLOCKED)
			continue;
L
Linus Torvalds 已提交
820
		disk = r10_bio->devs[slot].devnum;
821 822
		rdev = rcu_dereference(conf->mirrors[disk].replacement);
		if (rdev == NULL || test_bit(Faulty, &rdev->flags) ||
823
		    test_bit(Unmerged, &rdev->flags) ||
824 825
		    r10_bio->devs[slot].addr + sectors > rdev->recovery_offset)
			rdev = rcu_dereference(conf->mirrors[disk].rdev);
826 827 828
		if (rdev == NULL ||
		    test_bit(Faulty, &rdev->flags) ||
		    test_bit(Unmerged, &rdev->flags))
829 830 831
			continue;
		if (!test_bit(In_sync, &rdev->flags) &&
		    r10_bio->devs[slot].addr + sectors > rdev->recovery_offset)
N
NeilBrown 已提交
832 833
			continue;

834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855
		dev_sector = r10_bio->devs[slot].addr;
		if (is_badblock(rdev, dev_sector, sectors,
				&first_bad, &bad_sectors)) {
			if (best_dist < MaxSector)
				/* Already have a better slot */
				continue;
			if (first_bad <= dev_sector) {
				/* Cannot read here.  If this is the
				 * 'primary' device, then we must not read
				 * beyond 'bad_sectors' from another device.
				 */
				bad_sectors -= (dev_sector - first_bad);
				if (!do_balance && sectors > bad_sectors)
					sectors = bad_sectors;
				if (best_good_sectors > sectors)
					best_good_sectors = sectors;
			} else {
				sector_t good_sectors =
					first_bad - dev_sector;
				if (good_sectors > best_good_sectors) {
					best_good_sectors = good_sectors;
					best_slot = slot;
856
					best_rdev = rdev;
857 858 859 860 861 862 863 864 865
				}
				if (!do_balance)
					/* Must read from here */
					break;
			}
			continue;
		} else
			best_good_sectors = sectors;

N
NeilBrown 已提交
866 867
		if (!do_balance)
			break;
L
Linus Torvalds 已提交
868

869 870 871 872
		/* This optimisation is debatable, and completely destroys
		 * sequential read speed for 'far copies' arrays.  So only
		 * keep it for 'near' arrays, and review those later.
		 */
873
		if (geo->near_copies > 1 && !atomic_read(&rdev->nr_pending))
L
Linus Torvalds 已提交
874
			break;
875 876

		/* for far > 1 always use the lowest address */
877
		if (geo->far_copies > 1)
N
NeilBrown 已提交
878
			new_distance = r10_bio->devs[slot].addr;
879
		else
N
NeilBrown 已提交
880 881 882 883 884
			new_distance = abs(r10_bio->devs[slot].addr -
					   conf->mirrors[disk].head_position);
		if (new_distance < best_dist) {
			best_dist = new_distance;
			best_slot = slot;
885
			best_rdev = rdev;
L
Linus Torvalds 已提交
886 887
		}
	}
888
	if (slot >= conf->copies) {
N
NeilBrown 已提交
889
		slot = best_slot;
890 891
		rdev = best_rdev;
	}
L
Linus Torvalds 已提交
892

N
NeilBrown 已提交
893 894 895 896 897 898 899 900 901 902 903
	if (slot >= 0) {
		atomic_inc(&rdev->nr_pending);
		if (test_bit(Faulty, &rdev->flags)) {
			/* Cannot risk returning a device that failed
			 * before we inc'ed nr_pending
			 */
			rdev_dec_pending(rdev, conf->mddev);
			goto retry;
		}
		r10_bio->read_slot = slot;
	} else
904
		rdev = NULL;
L
Linus Torvalds 已提交
905
	rcu_read_unlock();
906
	*max_sectors = best_good_sectors;
L
Linus Torvalds 已提交
907

908
	return rdev;
L
Linus Torvalds 已提交
909 910
}

911
static int raid10_congested(struct mddev *mddev, int bits)
912
{
913
	struct r10conf *conf = mddev->private;
914 915
	int i, ret = 0;

916
	if ((bits & (1 << WB_async_congested)) &&
917 918 919
	    conf->pending_count >= max_queued_requests)
		return 1;

920
	rcu_read_lock();
921 922 923 924
	for (i = 0;
	     (i < conf->geo.raid_disks || i < conf->prev.raid_disks)
		     && ret == 0;
	     i++) {
925
		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
926
		if (rdev && !test_bit(Faulty, &rdev->flags)) {
927
			struct request_queue *q = bdev_get_queue(rdev->bdev);
928 929 930 931 932 933 934 935

			ret |= bdi_congested(&q->backing_dev_info, bits);
		}
	}
	rcu_read_unlock();
	return ret;
}

936
static void flush_pending_writes(struct r10conf *conf)
937 938 939 940 941 942 943 944 945
{
	/* Any writes that have been queued but are awaiting
	 * bitmap updates get flushed here.
	 */
	spin_lock_irq(&conf->device_lock);

	if (conf->pending_bio_list.head) {
		struct bio *bio;
		bio = bio_list_get(&conf->pending_bio_list);
946
		conf->pending_count = 0;
947 948 949 950
		spin_unlock_irq(&conf->device_lock);
		/* flush any pending bitmap writes to disk
		 * before proceeding w/ I/O */
		bitmap_unplug(conf->mddev->bitmap);
951
		wake_up(&conf->wait_barrier);
952 953 954 955

		while (bio) { /* submit pending writes */
			struct bio *next = bio->bi_next;
			bio->bi_next = NULL;
S
Shaohua Li 已提交
956 957 958
			if (unlikely((bio->bi_rw & REQ_DISCARD) &&
			    !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
				/* Just ignore it */
959
				bio_endio(bio);
S
Shaohua Li 已提交
960 961
			else
				generic_make_request(bio);
962 963 964 965 966
			bio = next;
		}
	} else
		spin_unlock_irq(&conf->device_lock);
}
J
Jens Axboe 已提交
967

968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987
/* Barriers....
 * Sometimes we need to suspend IO while we do something else,
 * either some resync/recovery, or reconfigure the array.
 * To do this we raise a 'barrier'.
 * The 'barrier' is a counter that can be raised multiple times
 * to count how many activities are happening which preclude
 * normal IO.
 * We can only raise the barrier if there is no pending IO.
 * i.e. if nr_pending == 0.
 * We choose only to raise the barrier if no-one is waiting for the
 * barrier to go down.  This means that as soon as an IO request
 * is ready, no other operations which require a barrier will start
 * until the IO request has had a chance.
 *
 * So: regular IO calls 'wait_barrier'.  When that returns there
 *    is no backgroup IO happening,  It must arrange to call
 *    allow_barrier when it has finished its IO.
 * backgroup IO calls must call raise_barrier.  Once that returns
 *    there is no normal IO happeing.  It must arrange to call
 *    lower_barrier when the particular background IO completes.
L
Linus Torvalds 已提交
988 989
 */

990
static void raise_barrier(struct r10conf *conf, int force)
L
Linus Torvalds 已提交
991
{
992
	BUG_ON(force && !conf->barrier);
L
Linus Torvalds 已提交
993
	spin_lock_irq(&conf->resync_lock);
994

995 996
	/* Wait until no block IO is waiting (unless 'force') */
	wait_event_lock_irq(conf->wait_barrier, force || !conf->nr_waiting,
997
			    conf->resync_lock);
998 999 1000 1001

	/* block any new IO from starting */
	conf->barrier++;

N
NeilBrown 已提交
1002
	/* Now wait for all pending IO to complete */
1003 1004
	wait_event_lock_irq(conf->wait_barrier,
			    !conf->nr_pending && conf->barrier < RESYNC_DEPTH,
1005
			    conf->resync_lock);
1006 1007 1008 1009

	spin_unlock_irq(&conf->resync_lock);
}

1010
static void lower_barrier(struct r10conf *conf)
1011 1012 1013 1014 1015 1016 1017 1018
{
	unsigned long flags;
	spin_lock_irqsave(&conf->resync_lock, flags);
	conf->barrier--;
	spin_unlock_irqrestore(&conf->resync_lock, flags);
	wake_up(&conf->wait_barrier);
}

1019
static void wait_barrier(struct r10conf *conf)
1020 1021 1022 1023
{
	spin_lock_irq(&conf->resync_lock);
	if (conf->barrier) {
		conf->nr_waiting++;
1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037
		/* Wait for the barrier to drop.
		 * However if there are already pending
		 * requests (preventing the barrier from
		 * rising completely), and the
		 * pre-process bio queue isn't empty,
		 * then don't wait, as we need to empty
		 * that queue to get the nr_pending
		 * count down.
		 */
		wait_event_lock_irq(conf->wait_barrier,
				    !conf->barrier ||
				    (conf->nr_pending &&
				     current->bio_list &&
				     !bio_list_empty(current->bio_list)),
1038
				    conf->resync_lock);
1039
		conf->nr_waiting--;
L
Linus Torvalds 已提交
1040
	}
1041
	conf->nr_pending++;
L
Linus Torvalds 已提交
1042 1043 1044
	spin_unlock_irq(&conf->resync_lock);
}

1045
static void allow_barrier(struct r10conf *conf)
1046 1047 1048 1049 1050 1051 1052 1053
{
	unsigned long flags;
	spin_lock_irqsave(&conf->resync_lock, flags);
	conf->nr_pending--;
	spin_unlock_irqrestore(&conf->resync_lock, flags);
	wake_up(&conf->wait_barrier);
}

1054
static void freeze_array(struct r10conf *conf, int extra)
1055 1056
{
	/* stop syncio and normal IO and wait for everything to
N
NeilBrown 已提交
1057
	 * go quiet.
1058
	 * We increment barrier and nr_waiting, and then
1059
	 * wait until nr_pending match nr_queued+extra
1060 1061 1062 1063
	 * This is called in the context of one normal IO request
	 * that has failed. Thus any sync request that might be pending
	 * will be blocked by nr_pending, and we need to wait for
	 * pending IO requests to complete or be queued for re-try.
1064
	 * Thus the number queued (nr_queued) plus this request (extra)
1065 1066
	 * must match the number of pending IOs (nr_pending) before
	 * we continue.
1067 1068 1069 1070
	 */
	spin_lock_irq(&conf->resync_lock);
	conf->barrier++;
	conf->nr_waiting++;
1071
	wait_event_lock_irq_cmd(conf->wait_barrier,
1072
				conf->nr_pending == conf->nr_queued+extra,
1073 1074
				conf->resync_lock,
				flush_pending_writes(conf));
N
NeilBrown 已提交
1075

1076 1077 1078
	spin_unlock_irq(&conf->resync_lock);
}

1079
static void unfreeze_array(struct r10conf *conf)
1080 1081 1082 1083 1084 1085 1086 1087 1088
{
	/* reverse the effect of the freeze */
	spin_lock_irq(&conf->resync_lock);
	conf->barrier--;
	conf->nr_waiting--;
	wake_up(&conf->wait_barrier);
	spin_unlock_irq(&conf->resync_lock);
}

1089 1090 1091 1092 1093 1094 1095 1096 1097 1098
static sector_t choose_data_offset(struct r10bio *r10_bio,
				   struct md_rdev *rdev)
{
	if (!test_bit(MD_RECOVERY_RESHAPE, &rdev->mddev->recovery) ||
	    test_bit(R10BIO_Previous, &r10_bio->state))
		return rdev->data_offset;
	else
		return rdev->new_data_offset;
}

1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112
struct raid10_plug_cb {
	struct blk_plug_cb	cb;
	struct bio_list		pending;
	int			pending_cnt;
};

static void raid10_unplug(struct blk_plug_cb *cb, bool from_schedule)
{
	struct raid10_plug_cb *plug = container_of(cb, struct raid10_plug_cb,
						   cb);
	struct mddev *mddev = plug->cb.data;
	struct r10conf *conf = mddev->private;
	struct bio *bio;

1113
	if (from_schedule || current->bio_list) {
1114 1115 1116 1117
		spin_lock_irq(&conf->device_lock);
		bio_list_merge(&conf->pending_bio_list, &plug->pending);
		conf->pending_count += plug->pending_cnt;
		spin_unlock_irq(&conf->device_lock);
1118
		wake_up(&conf->wait_barrier);
1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131
		md_wakeup_thread(mddev->thread);
		kfree(plug);
		return;
	}

	/* we aren't scheduling, so we can do the write-out directly. */
	bio = bio_list_get(&plug->pending);
	bitmap_unplug(mddev->bitmap);
	wake_up(&conf->wait_barrier);

	while (bio) { /* submit pending writes */
		struct bio *next = bio->bi_next;
		bio->bi_next = NULL;
1132 1133 1134
		if (unlikely((bio->bi_rw & REQ_DISCARD) &&
		    !blk_queue_discard(bdev_get_queue(bio->bi_bdev))))
			/* Just ignore it */
1135
			bio_endio(bio);
1136 1137
		else
			generic_make_request(bio);
1138 1139 1140 1141 1142
		bio = next;
	}
	kfree(plug);
}

K
Kent Overstreet 已提交
1143
static void __make_request(struct mddev *mddev, struct bio *bio)
L
Linus Torvalds 已提交
1144
{
1145
	struct r10conf *conf = mddev->private;
1146
	struct r10bio *r10_bio;
L
Linus Torvalds 已提交
1147 1148
	struct bio *read_bio;
	int i;
1149
	const int rw = bio_data_dir(bio);
1150
	const unsigned long do_sync = (bio->bi_rw & REQ_SYNC);
T
Tejun Heo 已提交
1151
	const unsigned long do_fua = (bio->bi_rw & REQ_FUA);
S
Shaohua Li 已提交
1152 1153
	const unsigned long do_discard = (bio->bi_rw
					  & (REQ_DISCARD | REQ_SECURE));
1154
	const unsigned long do_same = (bio->bi_rw & REQ_WRITE_SAME);
1155
	unsigned long flags;
1156
	struct md_rdev *blocked_rdev;
1157 1158
	struct blk_plug_cb *cb;
	struct raid10_plug_cb *plug = NULL;
1159 1160
	int sectors_handled;
	int max_sectors;
N
NeilBrown 已提交
1161
	int sectors;
L
Linus Torvalds 已提交
1162

1163 1164 1165 1166 1167 1168 1169
	/*
	 * Register the new request and wait if the reconstruction
	 * thread has put up a bar for new requests.
	 * Continue immediately if no resync is active currently.
	 */
	wait_barrier(conf);

1170
	sectors = bio_sectors(bio);
N
NeilBrown 已提交
1171
	while (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
1172 1173
	    bio->bi_iter.bi_sector < conf->reshape_progress &&
	    bio->bi_iter.bi_sector + sectors > conf->reshape_progress) {
N
NeilBrown 已提交
1174 1175 1176 1177 1178
		/* IO spans the reshape position.  Need to wait for
		 * reshape to pass
		 */
		allow_barrier(conf);
		wait_event(conf->wait_barrier,
1179 1180 1181
			   conf->reshape_progress <= bio->bi_iter.bi_sector ||
			   conf->reshape_progress >= bio->bi_iter.bi_sector +
			   sectors);
N
NeilBrown 已提交
1182 1183 1184 1185 1186
		wait_barrier(conf);
	}
	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
	    bio_data_dir(bio) == WRITE &&
	    (mddev->reshape_backwards
1187 1188 1189 1190
	     ? (bio->bi_iter.bi_sector < conf->reshape_safe &&
		bio->bi_iter.bi_sector + sectors > conf->reshape_progress)
	     : (bio->bi_iter.bi_sector + sectors > conf->reshape_safe &&
		bio->bi_iter.bi_sector < conf->reshape_progress))) {
N
NeilBrown 已提交
1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201
		/* Need to update reshape_position in metadata */
		mddev->reshape_position = conf->reshape_progress;
		set_bit(MD_CHANGE_DEVS, &mddev->flags);
		set_bit(MD_CHANGE_PENDING, &mddev->flags);
		md_wakeup_thread(mddev->thread);
		wait_event(mddev->sb_wait,
			   !test_bit(MD_CHANGE_PENDING, &mddev->flags));

		conf->reshape_safe = mddev->reshape_position;
	}

L
Linus Torvalds 已提交
1202 1203 1204
	r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);

	r10_bio->master_bio = bio;
N
NeilBrown 已提交
1205
	r10_bio->sectors = sectors;
L
Linus Torvalds 已提交
1206 1207

	r10_bio->mddev = mddev;
1208
	r10_bio->sector = bio->bi_iter.bi_sector;
1209
	r10_bio->state = 0;
L
Linus Torvalds 已提交
1210

1211 1212 1213 1214 1215 1216 1217 1218
	/* We might need to issue multiple reads to different
	 * devices if there are bad blocks around, so we keep
	 * track of the number of reads in bio->bi_phys_segments.
	 * If this is 0, there is only one r10_bio and no locking
	 * will be needed when the request completes.  If it is
	 * non-zero, then it is the number of not-completed requests.
	 */
	bio->bi_phys_segments = 0;
1219
	bio_clear_flag(bio, BIO_SEG_VALID);
1220

1221
	if (rw == READ) {
L
Linus Torvalds 已提交
1222 1223 1224
		/*
		 * read balancing logic:
		 */
1225
		struct md_rdev *rdev;
1226 1227 1228
		int slot;

read_again:
1229 1230
		rdev = read_balance(conf, r10_bio, &max_sectors);
		if (!rdev) {
L
Linus Torvalds 已提交
1231
			raid_end_bio_io(r10_bio);
1232
			return;
L
Linus Torvalds 已提交
1233
		}
1234
		slot = r10_bio->read_slot;
L
Linus Torvalds 已提交
1235

1236
		read_bio = bio_clone_mddev(bio, GFP_NOIO, mddev);
1237
		bio_trim(read_bio, r10_bio->sector - bio->bi_iter.bi_sector,
1238
			 max_sectors);
L
Linus Torvalds 已提交
1239 1240

		r10_bio->devs[slot].bio = read_bio;
1241
		r10_bio->devs[slot].rdev = rdev;
L
Linus Torvalds 已提交
1242

1243
		read_bio->bi_iter.bi_sector = r10_bio->devs[slot].addr +
1244
			choose_data_offset(r10_bio, rdev);
1245
		read_bio->bi_bdev = rdev->bdev;
L
Linus Torvalds 已提交
1246
		read_bio->bi_end_io = raid10_end_read_request;
1247
		read_bio->bi_rw = READ | do_sync;
L
Linus Torvalds 已提交
1248 1249
		read_bio->bi_private = r10_bio;

1250 1251 1252 1253
		if (max_sectors < r10_bio->sectors) {
			/* Could not read all from this device, so we will
			 * need another r10_bio.
			 */
1254
			sectors_handled = (r10_bio->sector + max_sectors
1255
					   - bio->bi_iter.bi_sector);
1256 1257 1258 1259 1260 1261
			r10_bio->sectors = max_sectors;
			spin_lock_irq(&conf->device_lock);
			if (bio->bi_phys_segments == 0)
				bio->bi_phys_segments = 2;
			else
				bio->bi_phys_segments++;
1262
			spin_unlock_irq(&conf->device_lock);
1263 1264 1265 1266 1267 1268 1269 1270 1271 1272
			/* Cannot call generic_make_request directly
			 * as that will be queued in __generic_make_request
			 * and subsequent mempool_alloc might block
			 * waiting for it.  so hand bio over to raid10d.
			 */
			reschedule_retry(r10_bio);

			r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);

			r10_bio->master_bio = bio;
1273
			r10_bio->sectors = bio_sectors(bio) - sectors_handled;
1274 1275
			r10_bio->state = 0;
			r10_bio->mddev = mddev;
1276 1277
			r10_bio->sector = bio->bi_iter.bi_sector +
				sectors_handled;
1278 1279 1280
			goto read_again;
		} else
			generic_make_request(read_bio);
1281
		return;
L
Linus Torvalds 已提交
1282 1283 1284 1285 1286
	}

	/*
	 * WRITE:
	 */
1287 1288 1289 1290 1291
	if (conf->pending_count >= max_queued_requests) {
		md_wakeup_thread(mddev->thread);
		wait_event(conf->wait_barrier,
			   conf->pending_count < max_queued_requests);
	}
1292
	/* first select target devices under rcu_lock and
L
Linus Torvalds 已提交
1293 1294
	 * inc refcount on their rdev.  Record them by setting
	 * bios[x] to bio
1295 1296 1297 1298 1299 1300 1301
	 * If there are known/acknowledged bad blocks on any device
	 * on which we have seen a write error, we want to avoid
	 * writing to those blocks.  This potentially requires several
	 * writes to write around the bad blocks.  Each set of writes
	 * gets its own r10_bio with a set of bios attached.  The number
	 * of r10_bios is recored in bio->bi_phys_segments just as with
	 * the read case.
L
Linus Torvalds 已提交
1302
	 */
N
NeilBrown 已提交
1303

1304
	r10_bio->read_slot = -1; /* make sure repl_bio gets freed */
L
Linus Torvalds 已提交
1305
	raid10_find_phys(conf, r10_bio);
1306
retry_write:
1307
	blocked_rdev = NULL;
L
Linus Torvalds 已提交
1308
	rcu_read_lock();
1309 1310
	max_sectors = r10_bio->sectors;

L
Linus Torvalds 已提交
1311 1312
	for (i = 0;  i < conf->copies; i++) {
		int d = r10_bio->devs[i].devnum;
1313
		struct md_rdev *rdev = rcu_dereference(conf->mirrors[d].rdev);
1314 1315
		struct md_rdev *rrdev = rcu_dereference(
			conf->mirrors[d].replacement);
1316 1317
		if (rdev == rrdev)
			rrdev = NULL;
1318 1319 1320 1321 1322
		if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) {
			atomic_inc(&rdev->nr_pending);
			blocked_rdev = rdev;
			break;
		}
1323 1324 1325 1326 1327
		if (rrdev && unlikely(test_bit(Blocked, &rrdev->flags))) {
			atomic_inc(&rrdev->nr_pending);
			blocked_rdev = rrdev;
			break;
		}
1328 1329 1330
		if (rdev && (test_bit(Faulty, &rdev->flags)
			     || test_bit(Unmerged, &rdev->flags)))
			rdev = NULL;
1331 1332
		if (rrdev && (test_bit(Faulty, &rrdev->flags)
			      || test_bit(Unmerged, &rrdev->flags)))
1333 1334
			rrdev = NULL;

1335
		r10_bio->devs[i].bio = NULL;
1336
		r10_bio->devs[i].repl_bio = NULL;
1337 1338

		if (!rdev && !rrdev) {
1339
			set_bit(R10BIO_Degraded, &r10_bio->state);
1340 1341
			continue;
		}
1342
		if (rdev && test_bit(WriteErrorSeen, &rdev->flags)) {
1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382
			sector_t first_bad;
			sector_t dev_sector = r10_bio->devs[i].addr;
			int bad_sectors;
			int is_bad;

			is_bad = is_badblock(rdev, dev_sector,
					     max_sectors,
					     &first_bad, &bad_sectors);
			if (is_bad < 0) {
				/* Mustn't write here until the bad block
				 * is acknowledged
				 */
				atomic_inc(&rdev->nr_pending);
				set_bit(BlockedBadBlocks, &rdev->flags);
				blocked_rdev = rdev;
				break;
			}
			if (is_bad && first_bad <= dev_sector) {
				/* Cannot write here at all */
				bad_sectors -= (dev_sector - first_bad);
				if (bad_sectors < max_sectors)
					/* Mustn't write more than bad_sectors
					 * to other devices yet
					 */
					max_sectors = bad_sectors;
				/* We don't set R10BIO_Degraded as that
				 * only applies if the disk is missing,
				 * so it might be re-added, and we want to
				 * know to recover this chunk.
				 * In this case the device is here, and the
				 * fact that this chunk is not in-sync is
				 * recorded in the bad block log.
				 */
				continue;
			}
			if (is_bad) {
				int good_sectors = first_bad - dev_sector;
				if (good_sectors < max_sectors)
					max_sectors = good_sectors;
			}
1383
		}
1384 1385 1386 1387
		if (rdev) {
			r10_bio->devs[i].bio = bio;
			atomic_inc(&rdev->nr_pending);
		}
1388 1389 1390 1391
		if (rrdev) {
			r10_bio->devs[i].repl_bio = bio;
			atomic_inc(&rrdev->nr_pending);
		}
L
Linus Torvalds 已提交
1392 1393 1394
	}
	rcu_read_unlock();

1395 1396 1397 1398 1399
	if (unlikely(blocked_rdev)) {
		/* Have to wait for this device to get unblocked, then retry */
		int j;
		int d;

1400
		for (j = 0; j < i; j++) {
1401 1402 1403 1404
			if (r10_bio->devs[j].bio) {
				d = r10_bio->devs[j].devnum;
				rdev_dec_pending(conf->mirrors[d].rdev, mddev);
			}
1405
			if (r10_bio->devs[j].repl_bio) {
1406
				struct md_rdev *rdev;
1407
				d = r10_bio->devs[j].devnum;
1408 1409 1410 1411 1412 1413 1414
				rdev = conf->mirrors[d].replacement;
				if (!rdev) {
					/* Race with remove_disk */
					smp_mb();
					rdev = conf->mirrors[d].rdev;
				}
				rdev_dec_pending(rdev, mddev);
1415 1416
			}
		}
1417 1418 1419 1420 1421 1422
		allow_barrier(conf);
		md_wait_for_blocked_rdev(blocked_rdev, mddev);
		wait_barrier(conf);
		goto retry_write;
	}

1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434
	if (max_sectors < r10_bio->sectors) {
		/* We are splitting this into multiple parts, so
		 * we need to prepare for allocating another r10_bio.
		 */
		r10_bio->sectors = max_sectors;
		spin_lock_irq(&conf->device_lock);
		if (bio->bi_phys_segments == 0)
			bio->bi_phys_segments = 2;
		else
			bio->bi_phys_segments++;
		spin_unlock_irq(&conf->device_lock);
	}
1435 1436
	sectors_handled = r10_bio->sector + max_sectors -
		bio->bi_iter.bi_sector;
1437

1438
	atomic_set(&r10_bio->remaining, 1);
1439
	bitmap_startwrite(mddev->bitmap, r10_bio->sector, r10_bio->sectors, 0);
1440

L
Linus Torvalds 已提交
1441 1442 1443
	for (i = 0; i < conf->copies; i++) {
		struct bio *mbio;
		int d = r10_bio->devs[i].devnum;
1444 1445 1446
		if (r10_bio->devs[i].bio) {
			struct md_rdev *rdev = conf->mirrors[d].rdev;
			mbio = bio_clone_mddev(bio, GFP_NOIO, mddev);
1447
			bio_trim(mbio, r10_bio->sector - bio->bi_iter.bi_sector,
1448
				 max_sectors);
1449 1450
			r10_bio->devs[i].bio = mbio;

1451
			mbio->bi_iter.bi_sector	= (r10_bio->devs[i].addr+
1452 1453 1454 1455
					   choose_data_offset(r10_bio,
							      rdev));
			mbio->bi_bdev = rdev->bdev;
			mbio->bi_end_io	= raid10_end_write_request;
1456 1457
			mbio->bi_rw =
				WRITE | do_sync | do_fua | do_discard | do_same;
1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480
			mbio->bi_private = r10_bio;

			atomic_inc(&r10_bio->remaining);

			cb = blk_check_plugged(raid10_unplug, mddev,
					       sizeof(*plug));
			if (cb)
				plug = container_of(cb, struct raid10_plug_cb,
						    cb);
			else
				plug = NULL;
			spin_lock_irqsave(&conf->device_lock, flags);
			if (plug) {
				bio_list_add(&plug->pending, mbio);
				plug->pending_cnt++;
			} else {
				bio_list_add(&conf->pending_bio_list, mbio);
				conf->pending_count++;
			}
			spin_unlock_irqrestore(&conf->device_lock, flags);
			if (!plug)
				md_wakeup_thread(mddev->thread);
		}
1481

1482 1483 1484 1485 1486 1487 1488 1489
		if (r10_bio->devs[i].repl_bio) {
			struct md_rdev *rdev = conf->mirrors[d].replacement;
			if (rdev == NULL) {
				/* Replacement just got moved to main 'rdev' */
				smp_mb();
				rdev = conf->mirrors[d].rdev;
			}
			mbio = bio_clone_mddev(bio, GFP_NOIO, mddev);
1490
			bio_trim(mbio, r10_bio->sector - bio->bi_iter.bi_sector,
1491
				 max_sectors);
1492 1493
			r10_bio->devs[i].repl_bio = mbio;

1494
			mbio->bi_iter.bi_sector	= (r10_bio->devs[i].addr +
1495 1496 1497 1498
					   choose_data_offset(
						   r10_bio, rdev));
			mbio->bi_bdev = rdev->bdev;
			mbio->bi_end_io	= raid10_end_write_request;
1499 1500
			mbio->bi_rw =
				WRITE | do_sync | do_fua | do_discard | do_same;
1501 1502 1503 1504
			mbio->bi_private = r10_bio;

			atomic_inc(&r10_bio->remaining);
			spin_lock_irqsave(&conf->device_lock, flags);
1505 1506
			bio_list_add(&conf->pending_bio_list, mbio);
			conf->pending_count++;
1507 1508 1509
			spin_unlock_irqrestore(&conf->device_lock, flags);
			if (!mddev_check_plugged(mddev))
				md_wakeup_thread(mddev->thread);
1510
		}
L
Linus Torvalds 已提交
1511 1512
	}

1513 1514 1515
	/* Don't remove the bias on 'remaining' (one_write_done) until
	 * after checking if we need to go around again.
	 */
1516

1517
	if (sectors_handled < bio_sectors(bio)) {
1518
		one_write_done(r10_bio);
1519
		/* We need another r10_bio.  It has already been counted
1520 1521 1522 1523 1524
		 * in bio->bi_phys_segments.
		 */
		r10_bio = mempool_alloc(conf->r10bio_pool, GFP_NOIO);

		r10_bio->master_bio = bio;
1525
		r10_bio->sectors = bio_sectors(bio) - sectors_handled;
1526 1527

		r10_bio->mddev = mddev;
1528
		r10_bio->sector = bio->bi_iter.bi_sector + sectors_handled;
1529 1530 1531
		r10_bio->state = 0;
		goto retry_write;
	}
1532
	one_write_done(r10_bio);
K
Kent Overstreet 已提交
1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571
}

static void make_request(struct mddev *mddev, struct bio *bio)
{
	struct r10conf *conf = mddev->private;
	sector_t chunk_mask = (conf->geo.chunk_mask & conf->prev.chunk_mask);
	int chunk_sects = chunk_mask + 1;

	struct bio *split;

	if (unlikely(bio->bi_rw & REQ_FLUSH)) {
		md_flush_request(mddev, bio);
		return;
	}

	md_write_start(mddev, bio);

	do {

		/*
		 * If this request crosses a chunk boundary, we need to split
		 * it.
		 */
		if (unlikely((bio->bi_iter.bi_sector & chunk_mask) +
			     bio_sectors(bio) > chunk_sects
			     && (conf->geo.near_copies < conf->geo.raid_disks
				 || conf->prev.near_copies <
				 conf->prev.raid_disks))) {
			split = bio_split(bio, chunk_sects -
					  (bio->bi_iter.bi_sector &
					   (chunk_sects - 1)),
					  GFP_NOIO, fs_bio_set);
			bio_chain(split, bio);
		} else {
			split = bio;
		}

		__make_request(mddev, split);
	} while (split != bio);
1572 1573 1574

	/* In case raid10d snuck in to freeze_array */
	wake_up(&conf->wait_barrier);
L
Linus Torvalds 已提交
1575 1576
}

1577
static void status(struct seq_file *seq, struct mddev *mddev)
L
Linus Torvalds 已提交
1578
{
1579
	struct r10conf *conf = mddev->private;
L
Linus Torvalds 已提交
1580 1581
	int i;

1582
	if (conf->geo.near_copies < conf->geo.raid_disks)
1583
		seq_printf(seq, " %dK chunks", mddev->chunk_sectors / 2);
1584 1585 1586 1587 1588
	if (conf->geo.near_copies > 1)
		seq_printf(seq, " %d near-copies", conf->geo.near_copies);
	if (conf->geo.far_copies > 1) {
		if (conf->geo.far_offset)
			seq_printf(seq, " %d offset-copies", conf->geo.far_copies);
1589
		else
1590
			seq_printf(seq, " %d far-copies", conf->geo.far_copies);
1591
	}
1592 1593 1594
	seq_printf(seq, " [%d/%d] [", conf->geo.raid_disks,
					conf->geo.raid_disks - mddev->degraded);
	for (i = 0; i < conf->geo.raid_disks; i++)
L
Linus Torvalds 已提交
1595 1596
		seq_printf(seq, "%s",
			      conf->mirrors[i].rdev &&
1597
			      test_bit(In_sync, &conf->mirrors[i].rdev->flags) ? "U" : "_");
L
Linus Torvalds 已提交
1598 1599 1600
	seq_printf(seq, "]");
}

1601 1602 1603 1604 1605
/* check if there are enough drives for
 * every block to appear on atleast one.
 * Don't consider the device numbered 'ignore'
 * as we might be about to remove it.
 */
1606
static int _enough(struct r10conf *conf, int previous, int ignore)
1607 1608
{
	int first = 0;
1609
	int has_enough = 0;
1610 1611 1612 1613 1614 1615 1616 1617
	int disks, ncopies;
	if (previous) {
		disks = conf->prev.raid_disks;
		ncopies = conf->prev.near_copies;
	} else {
		disks = conf->geo.raid_disks;
		ncopies = conf->geo.near_copies;
	}
1618

1619
	rcu_read_lock();
1620 1621 1622
	do {
		int n = conf->copies;
		int cnt = 0;
1623
		int this = first;
1624
		while (n--) {
1625 1626 1627 1628
			struct md_rdev *rdev;
			if (this != ignore &&
			    (rdev = rcu_dereference(conf->mirrors[this].rdev)) &&
			    test_bit(In_sync, &rdev->flags))
1629
				cnt++;
1630
			this = (this+1) % disks;
1631 1632
		}
		if (cnt == 0)
1633
			goto out;
1634
		first = (first + ncopies) % disks;
1635
	} while (first != 0);
1636 1637 1638 1639
	has_enough = 1;
out:
	rcu_read_unlock();
	return has_enough;
1640 1641
}

1642 1643
static int enough(struct r10conf *conf, int ignore)
{
1644 1645 1646 1647 1648 1649 1650
	/* when calling 'enough', both 'prev' and 'geo' must
	 * be stable.
	 * This is ensured if ->reconfig_mutex or ->device_lock
	 * is held.
	 */
	return _enough(conf, 0, ignore) &&
		_enough(conf, 1, ignore);
1651 1652
}

1653
static void error(struct mddev *mddev, struct md_rdev *rdev)
L
Linus Torvalds 已提交
1654 1655
{
	char b[BDEVNAME_SIZE];
1656
	struct r10conf *conf = mddev->private;
1657
	unsigned long flags;
L
Linus Torvalds 已提交
1658 1659 1660 1661 1662 1663 1664

	/*
	 * If it is not operational, then we have already marked it as dead
	 * else if it is the last working disks, ignore the error, let the
	 * next level up know.
	 * else mark the drive as failed
	 */
1665
	spin_lock_irqsave(&conf->device_lock, flags);
1666
	if (test_bit(In_sync, &rdev->flags)
1667
	    && !enough(conf, rdev->raid_disk)) {
L
Linus Torvalds 已提交
1668 1669 1670
		/*
		 * Don't fail the drive, just return an IO error.
		 */
1671
		spin_unlock_irqrestore(&conf->device_lock, flags);
L
Linus Torvalds 已提交
1672
		return;
1673
	}
1674
	if (test_and_clear_bit(In_sync, &rdev->flags))
L
Linus Torvalds 已提交
1675
		mddev->degraded++;
1676 1677 1678 1679
	/*
	 * If recovery is running, make sure it aborts.
	 */
	set_bit(MD_RECOVERY_INTR, &mddev->recovery);
1680
	set_bit(Blocked, &rdev->flags);
1681
	set_bit(Faulty, &rdev->flags);
1682
	set_bit(MD_CHANGE_DEVS, &mddev->flags);
1683
	spin_unlock_irqrestore(&conf->device_lock, flags);
1684 1685 1686
	printk(KERN_ALERT
	       "md/raid10:%s: Disk failure on %s, disabling device.\n"
	       "md/raid10:%s: Operation continuing on %d devices.\n",
N
NeilBrown 已提交
1687
	       mdname(mddev), bdevname(rdev->bdev, b),
1688
	       mdname(mddev), conf->geo.raid_disks - mddev->degraded);
L
Linus Torvalds 已提交
1689 1690
}

1691
static void print_conf(struct r10conf *conf)
L
Linus Torvalds 已提交
1692 1693
{
	int i;
1694
	struct raid10_info *tmp;
L
Linus Torvalds 已提交
1695

N
NeilBrown 已提交
1696
	printk(KERN_DEBUG "RAID10 conf printout:\n");
L
Linus Torvalds 已提交
1697
	if (!conf) {
N
NeilBrown 已提交
1698
		printk(KERN_DEBUG "(!conf)\n");
L
Linus Torvalds 已提交
1699 1700
		return;
	}
1701 1702
	printk(KERN_DEBUG " --- wd:%d rd:%d\n", conf->geo.raid_disks - conf->mddev->degraded,
		conf->geo.raid_disks);
L
Linus Torvalds 已提交
1703

1704
	for (i = 0; i < conf->geo.raid_disks; i++) {
L
Linus Torvalds 已提交
1705 1706 1707
		char b[BDEVNAME_SIZE];
		tmp = conf->mirrors + i;
		if (tmp->rdev)
N
NeilBrown 已提交
1708
			printk(KERN_DEBUG " disk %d, wo:%d, o:%d, dev:%s\n",
1709 1710
				i, !test_bit(In_sync, &tmp->rdev->flags),
			        !test_bit(Faulty, &tmp->rdev->flags),
L
Linus Torvalds 已提交
1711 1712 1713 1714
				bdevname(tmp->rdev->bdev,b));
	}
}

1715
static void close_sync(struct r10conf *conf)
L
Linus Torvalds 已提交
1716
{
1717 1718
	wait_barrier(conf);
	allow_barrier(conf);
L
Linus Torvalds 已提交
1719 1720 1721 1722 1723

	mempool_destroy(conf->r10buf_pool);
	conf->r10buf_pool = NULL;
}

1724
static int raid10_spare_active(struct mddev *mddev)
L
Linus Torvalds 已提交
1725 1726
{
	int i;
1727
	struct r10conf *conf = mddev->private;
1728
	struct raid10_info *tmp;
1729 1730
	int count = 0;
	unsigned long flags;
L
Linus Torvalds 已提交
1731 1732 1733 1734 1735

	/*
	 * Find all non-in_sync disks within the RAID10 configuration
	 * and mark them in_sync
	 */
1736
	for (i = 0; i < conf->geo.raid_disks; i++) {
L
Linus Torvalds 已提交
1737
		tmp = conf->mirrors + i;
1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756
		if (tmp->replacement
		    && tmp->replacement->recovery_offset == MaxSector
		    && !test_bit(Faulty, &tmp->replacement->flags)
		    && !test_and_set_bit(In_sync, &tmp->replacement->flags)) {
			/* Replacement has just become active */
			if (!tmp->rdev
			    || !test_and_clear_bit(In_sync, &tmp->rdev->flags))
				count++;
			if (tmp->rdev) {
				/* Replaced device not technically faulty,
				 * but we need to be sure it gets removed
				 * and never re-added.
				 */
				set_bit(Faulty, &tmp->rdev->flags);
				sysfs_notify_dirent_safe(
					tmp->rdev->sysfs_state);
			}
			sysfs_notify_dirent_safe(tmp->replacement->sysfs_state);
		} else if (tmp->rdev
1757
			   && tmp->rdev->recovery_offset == MaxSector
1758 1759
			   && !test_bit(Faulty, &tmp->rdev->flags)
			   && !test_and_set_bit(In_sync, &tmp->rdev->flags)) {
1760
			count++;
1761
			sysfs_notify_dirent_safe(tmp->rdev->sysfs_state);
L
Linus Torvalds 已提交
1762 1763
		}
	}
1764 1765 1766
	spin_lock_irqsave(&conf->device_lock, flags);
	mddev->degraded -= count;
	spin_unlock_irqrestore(&conf->device_lock, flags);
L
Linus Torvalds 已提交
1767 1768

	print_conf(conf);
1769
	return count;
L
Linus Torvalds 已提交
1770 1771
}

1772
static int raid10_add_disk(struct mddev *mddev, struct md_rdev *rdev)
L
Linus Torvalds 已提交
1773
{
1774
	struct r10conf *conf = mddev->private;
1775
	int err = -EEXIST;
L
Linus Torvalds 已提交
1776
	int mirror;
1777
	int first = 0;
1778
	int last = conf->geo.raid_disks - 1;
1779
	struct request_queue *q = bdev_get_queue(rdev->bdev);
L
Linus Torvalds 已提交
1780 1781 1782 1783 1784

	if (mddev->recovery_cp < MaxSector)
		/* only hot-add to in-sync arrays, as recovery is
		 * very different from resync
		 */
1785
		return -EBUSY;
1786
	if (rdev->saved_raid_disk < 0 && !_enough(conf, 1, -1))
1787
		return -EINVAL;
L
Linus Torvalds 已提交
1788

N
NeilBrown 已提交
1789
	if (rdev->raid_disk >= 0)
1790
		first = last = rdev->raid_disk;
L
Linus Torvalds 已提交
1791

1792 1793 1794 1795 1796
	if (q->merge_bvec_fn) {
		set_bit(Unmerged, &rdev->flags);
		mddev->merge_check_needed = 1;
	}

1797
	if (rdev->saved_raid_disk >= first &&
1798 1799 1800
	    conf->mirrors[rdev->saved_raid_disk].rdev == NULL)
		mirror = rdev->saved_raid_disk;
	else
1801
		mirror = first;
1802
	for ( ; mirror <= last ; mirror++) {
1803
		struct raid10_info *p = &conf->mirrors[mirror];
1804 1805
		if (p->recovery_disabled == mddev->recovery_disabled)
			continue;
1806 1807 1808 1809 1810 1811 1812 1813
		if (p->rdev) {
			if (!test_bit(WantReplacement, &p->rdev->flags) ||
			    p->replacement != NULL)
				continue;
			clear_bit(In_sync, &rdev->flags);
			set_bit(Replacement, &rdev->flags);
			rdev->raid_disk = mirror;
			err = 0;
1814 1815 1816
			if (mddev->gendisk)
				disk_stack_limits(mddev->gendisk, rdev->bdev,
						  rdev->data_offset << 9);
1817 1818 1819 1820
			conf->fullsync = 1;
			rcu_assign_pointer(p->replacement, rdev);
			break;
		}
L
Linus Torvalds 已提交
1821

1822 1823 1824
		if (mddev->gendisk)
			disk_stack_limits(mddev->gendisk, rdev->bdev,
					  rdev->data_offset << 9);
L
Linus Torvalds 已提交
1825

1826
		p->head_position = 0;
1827
		p->recovery_disabled = mddev->recovery_disabled - 1;
1828 1829 1830 1831 1832 1833 1834
		rdev->raid_disk = mirror;
		err = 0;
		if (rdev->saved_raid_disk != mirror)
			conf->fullsync = 1;
		rcu_assign_pointer(p->rdev, rdev);
		break;
	}
1835 1836 1837 1838 1839 1840 1841 1842 1843
	if (err == 0 && test_bit(Unmerged, &rdev->flags)) {
		/* Some requests might not have seen this new
		 * merge_bvec_fn.  We must wait for them to complete
		 * before merging the device fully.
		 * First we make sure any code which has tested
		 * our function has submitted the request, then
		 * we wait for all outstanding requests to complete.
		 */
		synchronize_sched();
1844 1845
		freeze_array(conf, 0);
		unfreeze_array(conf);
1846 1847
		clear_bit(Unmerged, &rdev->flags);
	}
1848
	md_integrity_add_rdev(rdev, mddev);
1849
	if (mddev->queue && blk_queue_discard(bdev_get_queue(rdev->bdev)))
S
Shaohua Li 已提交
1850 1851
		queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, mddev->queue);

L
Linus Torvalds 已提交
1852
	print_conf(conf);
1853
	return err;
L
Linus Torvalds 已提交
1854 1855
}

1856
static int raid10_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
L
Linus Torvalds 已提交
1857
{
1858
	struct r10conf *conf = mddev->private;
L
Linus Torvalds 已提交
1859
	int err = 0;
1860
	int number = rdev->raid_disk;
1861
	struct md_rdev **rdevp;
1862
	struct raid10_info *p = conf->mirrors + number;
L
Linus Torvalds 已提交
1863 1864

	print_conf(conf);
1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881
	if (rdev == p->rdev)
		rdevp = &p->rdev;
	else if (rdev == p->replacement)
		rdevp = &p->replacement;
	else
		return 0;

	if (test_bit(In_sync, &rdev->flags) ||
	    atomic_read(&rdev->nr_pending)) {
		err = -EBUSY;
		goto abort;
	}
	/* Only remove faulty devices if recovery
	 * is not possible.
	 */
	if (!test_bit(Faulty, &rdev->flags) &&
	    mddev->recovery_disabled != p->recovery_disabled &&
1882
	    (!p->replacement || p->replacement == rdev) &&
1883
	    number < conf->geo.raid_disks &&
1884 1885 1886
	    enough(conf, -1)) {
		err = -EBUSY;
		goto abort;
L
Linus Torvalds 已提交
1887
	}
1888 1889 1890 1891 1892 1893 1894
	*rdevp = NULL;
	synchronize_rcu();
	if (atomic_read(&rdev->nr_pending)) {
		/* lost the race, try later */
		err = -EBUSY;
		*rdevp = rdev;
		goto abort;
1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909
	} else if (p->replacement) {
		/* We must have just cleared 'rdev' */
		p->rdev = p->replacement;
		clear_bit(Replacement, &p->replacement->flags);
		smp_mb(); /* Make sure other CPUs may see both as identical
			   * but will never see neither -- if they are careful.
			   */
		p->replacement = NULL;
		clear_bit(WantReplacement, &rdev->flags);
	} else
		/* We might have just remove the Replacement as faulty
		 * Clear the flag just in case
		 */
		clear_bit(WantReplacement, &rdev->flags);

1910 1911
	err = md_integrity_register(mddev);

L
Linus Torvalds 已提交
1912 1913 1914 1915 1916 1917
abort:

	print_conf(conf);
	return err;
}

1918
static void end_sync_read(struct bio *bio)
L
Linus Torvalds 已提交
1919
{
1920
	struct r10bio *r10_bio = bio->bi_private;
1921
	struct r10conf *conf = r10_bio->mddev->private;
1922
	int d;
L
Linus Torvalds 已提交
1923

N
NeilBrown 已提交
1924 1925 1926 1927 1928
	if (bio == r10_bio->master_bio) {
		/* this is a reshape read */
		d = r10_bio->read_slot; /* really the read dev */
	} else
		d = find_bio_disk(conf, r10_bio, bio, NULL, NULL);
1929

1930
	if (!bio->bi_error)
1931
		set_bit(R10BIO_Uptodate, &r10_bio->state);
1932 1933 1934 1935
	else
		/* The write handler will notice the lack of
		 * R10BIO_Uptodate and record any errors etc
		 */
1936 1937
		atomic_add(r10_bio->sectors,
			   &conf->mirrors[d].rdev->corrected_errors);
L
Linus Torvalds 已提交
1938 1939 1940 1941

	/* for reconstruct, we always reschedule after a read.
	 * for resync, only after all reads
	 */
1942
	rdev_dec_pending(conf->mirrors[d].rdev, conf->mddev);
L
Linus Torvalds 已提交
1943 1944 1945 1946 1947 1948 1949 1950 1951
	if (test_bit(R10BIO_IsRecover, &r10_bio->state) ||
	    atomic_dec_and_test(&r10_bio->remaining)) {
		/* we have read all the blocks,
		 * do the comparison in process context in raid10d
		 */
		reschedule_retry(r10_bio);
	}
}

1952
static void end_sync_request(struct r10bio *r10_bio)
L
Linus Torvalds 已提交
1953
{
1954
	struct mddev *mddev = r10_bio->mddev;
1955

L
Linus Torvalds 已提交
1956 1957 1958
	while (atomic_dec_and_test(&r10_bio->remaining)) {
		if (r10_bio->master_bio == NULL) {
			/* the primary of several recovery bios */
1959
			sector_t s = r10_bio->sectors;
1960 1961
			if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
			    test_bit(R10BIO_WriteError, &r10_bio->state))
1962 1963 1964
				reschedule_retry(r10_bio);
			else
				put_buf(r10_bio);
1965
			md_done_sync(mddev, s, 1);
L
Linus Torvalds 已提交
1966 1967
			break;
		} else {
1968
			struct r10bio *r10_bio2 = (struct r10bio *)r10_bio->master_bio;
1969 1970
			if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
			    test_bit(R10BIO_WriteError, &r10_bio->state))
1971 1972 1973
				reschedule_retry(r10_bio);
			else
				put_buf(r10_bio);
L
Linus Torvalds 已提交
1974 1975 1976 1977 1978
			r10_bio = r10_bio2;
		}
	}
}

1979
static void end_sync_write(struct bio *bio)
1980
{
1981
	struct r10bio *r10_bio = bio->bi_private;
1982
	struct mddev *mddev = r10_bio->mddev;
1983
	struct r10conf *conf = mddev->private;
1984 1985 1986 1987
	int d;
	sector_t first_bad;
	int bad_sectors;
	int slot;
1988
	int repl;
1989
	struct md_rdev *rdev = NULL;
1990

1991 1992 1993
	d = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
	if (repl)
		rdev = conf->mirrors[d].replacement;
1994
	else
1995
		rdev = conf->mirrors[d].rdev;
1996

1997
	if (bio->bi_error) {
1998 1999 2000 2001
		if (repl)
			md_error(mddev, rdev);
		else {
			set_bit(WriteErrorSeen, &rdev->flags);
2002 2003 2004
			if (!test_and_set_bit(WantReplacement, &rdev->flags))
				set_bit(MD_RECOVERY_NEEDED,
					&rdev->mddev->recovery);
2005 2006 2007
			set_bit(R10BIO_WriteError, &r10_bio->state);
		}
	} else if (is_badblock(rdev,
2008 2009 2010 2011 2012
			     r10_bio->devs[slot].addr,
			     r10_bio->sectors,
			     &first_bad, &bad_sectors))
		set_bit(R10BIO_MadeGood, &r10_bio->state);

2013
	rdev_dec_pending(rdev, mddev);
2014 2015 2016 2017

	end_sync_request(r10_bio);
}

L
Linus Torvalds 已提交
2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033
/*
 * Note: sync and recover and handled very differently for raid10
 * This code is for resync.
 * For resync, we read through virtual addresses and read all blocks.
 * If there is any error, we schedule a write.  The lowest numbered
 * drive is authoritative.
 * However requests come for physical address, so we need to map.
 * For every physical address there are raid_disks/copies virtual addresses,
 * which is always are least one, but is not necessarly an integer.
 * This means that a physical address can span multiple chunks, so we may
 * have to submit multiple io requests for a single sync request.
 */
/*
 * We check if all blocks are in-sync and only write to blocks that
 * aren't in sync
 */
2034
static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio)
L
Linus Torvalds 已提交
2035
{
2036
	struct r10conf *conf = mddev->private;
L
Linus Torvalds 已提交
2037 2038
	int i, first;
	struct bio *tbio, *fbio;
2039
	int vcnt;
L
Linus Torvalds 已提交
2040 2041 2042 2043 2044

	atomic_set(&r10_bio->remaining, 1);

	/* find the first device with a block */
	for (i=0; i<conf->copies; i++)
2045
		if (!r10_bio->devs[i].bio->bi_error)
L
Linus Torvalds 已提交
2046 2047 2048 2049 2050 2051 2052 2053
			break;

	if (i == conf->copies)
		goto done;

	first = i;
	fbio = r10_bio->devs[i].bio;

2054
	vcnt = (r10_bio->sectors + (PAGE_SIZE >> 9) - 1) >> (PAGE_SHIFT - 9);
L
Linus Torvalds 已提交
2055
	/* now find blocks with errors */
2056 2057
	for (i=0 ; i < conf->copies ; i++) {
		int  j, d;
L
Linus Torvalds 已提交
2058 2059

		tbio = r10_bio->devs[i].bio;
2060 2061 2062 2063

		if (tbio->bi_end_io != end_sync_read)
			continue;
		if (i == first)
L
Linus Torvalds 已提交
2064
			continue;
2065
		if (!r10_bio->devs[i].bio->bi_error) {
2066 2067 2068 2069
			/* We know that the bi_io_vec layout is the same for
			 * both 'first' and 'i', so we just compare them.
			 * All vec entries are PAGE_SIZE;
			 */
2070 2071 2072 2073 2074
			int sectors = r10_bio->sectors;
			for (j = 0; j < vcnt; j++) {
				int len = PAGE_SIZE;
				if (sectors < (len / 512))
					len = sectors * 512;
2075 2076
				if (memcmp(page_address(fbio->bi_io_vec[j].bv_page),
					   page_address(tbio->bi_io_vec[j].bv_page),
2077
					   len))
2078
					break;
2079 2080
				sectors -= len/512;
			}
2081 2082
			if (j == vcnt)
				continue;
2083
			atomic64_add(r10_bio->sectors, &mddev->resync_mismatches);
2084 2085 2086
			if (test_bit(MD_RECOVERY_CHECK, &mddev->recovery))
				/* Don't fix anything. */
				continue;
2087
		}
2088 2089
		/* Ok, we need to write this bio, either to correct an
		 * inconsistency or to correct an unreadable block.
L
Linus Torvalds 已提交
2090 2091 2092
		 * First we need to fixup bv_offset, bv_len and
		 * bi_vecs, as the read request might have corrupted these
		 */
K
Kent Overstreet 已提交
2093 2094
		bio_reset(tbio);

L
Linus Torvalds 已提交
2095
		tbio->bi_vcnt = vcnt;
2096
		tbio->bi_iter.bi_size = r10_bio->sectors << 9;
L
Linus Torvalds 已提交
2097 2098
		tbio->bi_rw = WRITE;
		tbio->bi_private = r10_bio;
2099
		tbio->bi_iter.bi_sector = r10_bio->devs[i].addr;
L
Linus Torvalds 已提交
2100 2101
		tbio->bi_end_io = end_sync_write;

2102 2103
		bio_copy_data(tbio, fbio);

L
Linus Torvalds 已提交
2104 2105 2106
		d = r10_bio->devs[i].devnum;
		atomic_inc(&conf->mirrors[d].rdev->nr_pending);
		atomic_inc(&r10_bio->remaining);
2107
		md_sync_acct(conf->mirrors[d].rdev->bdev, bio_sectors(tbio));
L
Linus Torvalds 已提交
2108

2109
		tbio->bi_iter.bi_sector += conf->mirrors[d].rdev->data_offset;
L
Linus Torvalds 已提交
2110 2111 2112 2113
		tbio->bi_bdev = conf->mirrors[d].rdev->bdev;
		generic_make_request(tbio);
	}

2114 2115 2116 2117
	/* Now write out to any replacement devices
	 * that are active
	 */
	for (i = 0; i < conf->copies; i++) {
2118
		int d;
2119 2120 2121 2122 2123 2124

		tbio = r10_bio->devs[i].repl_bio;
		if (!tbio || !tbio->bi_end_io)
			continue;
		if (r10_bio->devs[i].bio->bi_end_io != end_sync_write
		    && r10_bio->devs[i].bio != fbio)
2125
			bio_copy_data(tbio, fbio);
2126 2127 2128
		d = r10_bio->devs[i].devnum;
		atomic_inc(&r10_bio->remaining);
		md_sync_acct(conf->mirrors[d].replacement->bdev,
2129
			     bio_sectors(tbio));
2130 2131 2132
		generic_make_request(tbio);
	}

L
Linus Torvalds 已提交
2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149
done:
	if (atomic_dec_and_test(&r10_bio->remaining)) {
		md_done_sync(mddev, r10_bio->sectors, 1);
		put_buf(r10_bio);
	}
}

/*
 * Now for the recovery code.
 * Recovery happens across physical sectors.
 * We recover all non-is_sync drives by finding the virtual address of
 * each, and then choose a working drive that also has that virt address.
 * There is a separate r10_bio for each non-in_sync drive.
 * Only the first two slots are in use. The first for reading,
 * The second for writing.
 *
 */
2150
static void fix_recovery_read_error(struct r10bio *r10_bio)
2151 2152 2153 2154 2155 2156 2157 2158
{
	/* We got a read error during recovery.
	 * We repeat the read in smaller page-sized sections.
	 * If a read succeeds, write it to the new device or record
	 * a bad block if we cannot.
	 * If a read fails, record a bad block on both old and
	 * new devices.
	 */
2159
	struct mddev *mddev = r10_bio->mddev;
2160
	struct r10conf *conf = mddev->private;
2161 2162 2163 2164 2165 2166 2167 2168 2169
	struct bio *bio = r10_bio->devs[0].bio;
	sector_t sect = 0;
	int sectors = r10_bio->sectors;
	int idx = 0;
	int dr = r10_bio->devs[0].devnum;
	int dw = r10_bio->devs[1].devnum;

	while (sectors) {
		int s = sectors;
2170
		struct md_rdev *rdev;
2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191
		sector_t addr;
		int ok;

		if (s > (PAGE_SIZE>>9))
			s = PAGE_SIZE >> 9;

		rdev = conf->mirrors[dr].rdev;
		addr = r10_bio->devs[0].addr + sect,
		ok = sync_page_io(rdev,
				  addr,
				  s << 9,
				  bio->bi_io_vec[idx].bv_page,
				  READ, false);
		if (ok) {
			rdev = conf->mirrors[dw].rdev;
			addr = r10_bio->devs[1].addr + sect;
			ok = sync_page_io(rdev,
					  addr,
					  s << 9,
					  bio->bi_io_vec[idx].bv_page,
					  WRITE, false);
2192
			if (!ok) {
2193
				set_bit(WriteErrorSeen, &rdev->flags);
2194 2195 2196 2197 2198
				if (!test_and_set_bit(WantReplacement,
						      &rdev->flags))
					set_bit(MD_RECOVERY_NEEDED,
						&rdev->mddev->recovery);
			}
2199 2200 2201 2202 2203 2204 2205 2206 2207 2208
		}
		if (!ok) {
			/* We don't worry if we cannot set a bad block -
			 * it really is bad so there is no loss in not
			 * recording it yet
			 */
			rdev_set_badblocks(rdev, addr, s, 0);

			if (rdev != conf->mirrors[dw].rdev) {
				/* need bad block on destination too */
2209
				struct md_rdev *rdev2 = conf->mirrors[dw].rdev;
2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232
				addr = r10_bio->devs[1].addr + sect;
				ok = rdev_set_badblocks(rdev2, addr, s, 0);
				if (!ok) {
					/* just abort the recovery */
					printk(KERN_NOTICE
					       "md/raid10:%s: recovery aborted"
					       " due to read error\n",
					       mdname(mddev));

					conf->mirrors[dw].recovery_disabled
						= mddev->recovery_disabled;
					set_bit(MD_RECOVERY_INTR,
						&mddev->recovery);
					break;
				}
			}
		}

		sectors -= s;
		sect += s;
		idx++;
	}
}
L
Linus Torvalds 已提交
2233

2234
static void recovery_request_write(struct mddev *mddev, struct r10bio *r10_bio)
L
Linus Torvalds 已提交
2235
{
2236
	struct r10conf *conf = mddev->private;
2237
	int d;
2238
	struct bio *wbio, *wbio2;
L
Linus Torvalds 已提交
2239

2240 2241 2242 2243 2244 2245
	if (!test_bit(R10BIO_Uptodate, &r10_bio->state)) {
		fix_recovery_read_error(r10_bio);
		end_sync_request(r10_bio);
		return;
	}

2246 2247
	/*
	 * share the pages with the first bio
L
Linus Torvalds 已提交
2248 2249 2250
	 * and submit the write request
	 */
	d = r10_bio->devs[1].devnum;
2251 2252
	wbio = r10_bio->devs[1].bio;
	wbio2 = r10_bio->devs[1].repl_bio;
2253 2254 2255 2256 2257 2258
	/* Need to test wbio2->bi_end_io before we call
	 * generic_make_request as if the former is NULL,
	 * the latter is free to free wbio2.
	 */
	if (wbio2 && !wbio2->bi_end_io)
		wbio2 = NULL;
2259 2260
	if (wbio->bi_end_io) {
		atomic_inc(&conf->mirrors[d].rdev->nr_pending);
2261
		md_sync_acct(conf->mirrors[d].rdev->bdev, bio_sectors(wbio));
2262 2263
		generic_make_request(wbio);
	}
2264
	if (wbio2) {
2265 2266
		atomic_inc(&conf->mirrors[d].replacement->nr_pending);
		md_sync_acct(conf->mirrors[d].replacement->bdev,
2267
			     bio_sectors(wbio2));
2268 2269
		generic_make_request(wbio2);
	}
L
Linus Torvalds 已提交
2270 2271
}

2272 2273 2274 2275 2276 2277
/*
 * Used by fix_read_error() to decay the per rdev read_errors.
 * We halve the read error count for every hour that has elapsed
 * since the last recorded read error.
 *
 */
2278
static void check_decay_read_errors(struct mddev *mddev, struct md_rdev *rdev)
2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308
{
	struct timespec cur_time_mon;
	unsigned long hours_since_last;
	unsigned int read_errors = atomic_read(&rdev->read_errors);

	ktime_get_ts(&cur_time_mon);

	if (rdev->last_read_error.tv_sec == 0 &&
	    rdev->last_read_error.tv_nsec == 0) {
		/* first time we've seen a read error */
		rdev->last_read_error = cur_time_mon;
		return;
	}

	hours_since_last = (cur_time_mon.tv_sec -
			    rdev->last_read_error.tv_sec) / 3600;

	rdev->last_read_error = cur_time_mon;

	/*
	 * if hours_since_last is > the number of bits in read_errors
	 * just set read errors to 0. We do this to avoid
	 * overflowing the shift of read_errors by hours_since_last.
	 */
	if (hours_since_last >= 8 * sizeof(read_errors))
		atomic_set(&rdev->read_errors, 0);
	else
		atomic_set(&rdev->read_errors, read_errors >> hours_since_last);
}

2309
static int r10_sync_page_io(struct md_rdev *rdev, sector_t sector,
2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320
			    int sectors, struct page *page, int rw)
{
	sector_t first_bad;
	int bad_sectors;

	if (is_badblock(rdev, sector, sectors, &first_bad, &bad_sectors)
	    && (rw == READ || test_bit(WriteErrorSeen, &rdev->flags)))
		return -1;
	if (sync_page_io(rdev, sector, sectors << 9, page, rw, false))
		/* success */
		return 1;
2321
	if (rw == WRITE) {
2322
		set_bit(WriteErrorSeen, &rdev->flags);
2323 2324 2325 2326
		if (!test_and_set_bit(WantReplacement, &rdev->flags))
			set_bit(MD_RECOVERY_NEEDED,
				&rdev->mddev->recovery);
	}
2327 2328 2329 2330 2331 2332
	/* need to record an error - either for the block or the device */
	if (!rdev_set_badblocks(rdev, sector, sectors, 0))
		md_error(rdev->mddev, rdev);
	return 0;
}

L
Linus Torvalds 已提交
2333 2334 2335 2336 2337
/*
 * This is a kernel thread which:
 *
 *	1.	Retries failed read operations on working mirrors.
 *	2.	Updates the raid superblock when problems encounter.
2338
 *	3.	Performs writes following reads for array synchronising.
L
Linus Torvalds 已提交
2339 2340
 */

2341
static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10bio *r10_bio)
2342 2343 2344
{
	int sect = 0; /* Offset from r10_bio->sector */
	int sectors = r10_bio->sectors;
2345
	struct md_rdev*rdev;
2346
	int max_read_errors = atomic_read(&mddev->max_corr_read_errors);
2347
	int d = r10_bio->devs[r10_bio->read_slot].devnum;
2348

2349 2350 2351 2352
	/* still own a reference to this rdev, so it cannot
	 * have been cleared recently.
	 */
	rdev = conf->mirrors[d].rdev;
2353

2354 2355 2356 2357
	if (test_bit(Faulty, &rdev->flags))
		/* drive has already been failed, just ignore any
		   more fix_read_error() attempts */
		return;
2358

2359 2360 2361 2362 2363
	check_decay_read_errors(mddev, rdev);
	atomic_inc(&rdev->read_errors);
	if (atomic_read(&rdev->read_errors) > max_read_errors) {
		char b[BDEVNAME_SIZE];
		bdevname(rdev->bdev, b);
2364

2365 2366 2367 2368 2369 2370 2371 2372 2373
		printk(KERN_NOTICE
		       "md/raid10:%s: %s: Raid device exceeded "
		       "read_error threshold [cur %d:max %d]\n",
		       mdname(mddev), b,
		       atomic_read(&rdev->read_errors), max_read_errors);
		printk(KERN_NOTICE
		       "md/raid10:%s: %s: Failing raid device\n",
		       mdname(mddev), b);
		md_error(mddev, conf->mirrors[d].rdev);
2374
		r10_bio->devs[r10_bio->read_slot].bio = IO_BLOCKED;
2375
		return;
2376 2377
	}

2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388
	while(sectors) {
		int s = sectors;
		int sl = r10_bio->read_slot;
		int success = 0;
		int start;

		if (s > (PAGE_SIZE>>9))
			s = PAGE_SIZE >> 9;

		rcu_read_lock();
		do {
2389 2390 2391
			sector_t first_bad;
			int bad_sectors;

2392
			d = r10_bio->devs[sl].devnum;
2393 2394
			rdev = rcu_dereference(conf->mirrors[d].rdev);
			if (rdev &&
2395
			    !test_bit(Unmerged, &rdev->flags) &&
2396 2397 2398
			    test_bit(In_sync, &rdev->flags) &&
			    is_badblock(rdev, r10_bio->devs[sl].addr + sect, s,
					&first_bad, &bad_sectors) == 0) {
2399 2400
				atomic_inc(&rdev->nr_pending);
				rcu_read_unlock();
2401
				success = sync_page_io(rdev,
2402
						       r10_bio->devs[sl].addr +
J
Jonathan Brassow 已提交
2403
						       sect,
2404
						       s<<9,
J
Jonathan Brassow 已提交
2405
						       conf->tmppage, READ, false);
2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417
				rdev_dec_pending(rdev, mddev);
				rcu_read_lock();
				if (success)
					break;
			}
			sl++;
			if (sl == conf->copies)
				sl = 0;
		} while (!success && sl != r10_bio->read_slot);
		rcu_read_unlock();

		if (!success) {
2418 2419 2420 2421
			/* Cannot read from anywhere, just mark the block
			 * as bad on the first device to discourage future
			 * reads.
			 */
2422
			int dn = r10_bio->devs[r10_bio->read_slot].devnum;
2423 2424 2425 2426 2427 2428
			rdev = conf->mirrors[dn].rdev;

			if (!rdev_set_badblocks(
				    rdev,
				    r10_bio->devs[r10_bio->read_slot].addr
				    + sect,
2429
				    s, 0)) {
2430
				md_error(mddev, rdev);
2431 2432 2433
				r10_bio->devs[r10_bio->read_slot].bio
					= IO_BLOCKED;
			}
2434 2435 2436 2437 2438 2439 2440
			break;
		}

		start = sl;
		/* write it back and re-read */
		rcu_read_lock();
		while (sl != r10_bio->read_slot) {
2441
			char b[BDEVNAME_SIZE];
2442

2443 2444 2445 2446 2447
			if (sl==0)
				sl = conf->copies;
			sl--;
			d = r10_bio->devs[sl].devnum;
			rdev = rcu_dereference(conf->mirrors[d].rdev);
2448
			if (!rdev ||
2449
			    test_bit(Unmerged, &rdev->flags) ||
2450 2451 2452 2453 2454
			    !test_bit(In_sync, &rdev->flags))
				continue;

			atomic_inc(&rdev->nr_pending);
			rcu_read_unlock();
2455 2456 2457
			if (r10_sync_page_io(rdev,
					     r10_bio->devs[sl].addr +
					     sect,
2458
					     s, conf->tmppage, WRITE)
2459 2460 2461 2462 2463 2464 2465 2466
			    == 0) {
				/* Well, this device is dead */
				printk(KERN_NOTICE
				       "md/raid10:%s: read correction "
				       "write failed"
				       " (%d sectors at %llu on %s)\n",
				       mdname(mddev), s,
				       (unsigned long long)(
2467 2468 2469
					       sect +
					       choose_data_offset(r10_bio,
								  rdev)),
2470 2471 2472 2473 2474
				       bdevname(rdev->bdev, b));
				printk(KERN_NOTICE "md/raid10:%s: %s: failing "
				       "drive\n",
				       mdname(mddev),
				       bdevname(rdev->bdev, b));
2475
			}
2476 2477
			rdev_dec_pending(rdev, mddev);
			rcu_read_lock();
2478 2479 2480
		}
		sl = start;
		while (sl != r10_bio->read_slot) {
2481
			char b[BDEVNAME_SIZE];
2482

2483 2484 2485 2486 2487
			if (sl==0)
				sl = conf->copies;
			sl--;
			d = r10_bio->devs[sl].devnum;
			rdev = rcu_dereference(conf->mirrors[d].rdev);
2488 2489 2490
			if (!rdev ||
			    !test_bit(In_sync, &rdev->flags))
				continue;
2491

2492 2493
			atomic_inc(&rdev->nr_pending);
			rcu_read_unlock();
2494 2495 2496
			switch (r10_sync_page_io(rdev,
					     r10_bio->devs[sl].addr +
					     sect,
2497
					     s, conf->tmppage,
2498 2499
						 READ)) {
			case 0:
2500 2501 2502 2503 2504 2505 2506
				/* Well, this device is dead */
				printk(KERN_NOTICE
				       "md/raid10:%s: unable to read back "
				       "corrected sectors"
				       " (%d sectors at %llu on %s)\n",
				       mdname(mddev), s,
				       (unsigned long long)(
2507 2508
					       sect +
					       choose_data_offset(r10_bio, rdev)),
2509 2510 2511 2512 2513
				       bdevname(rdev->bdev, b));
				printk(KERN_NOTICE "md/raid10:%s: %s: failing "
				       "drive\n",
				       mdname(mddev),
				       bdevname(rdev->bdev, b));
2514 2515
				break;
			case 1:
2516 2517 2518 2519 2520
				printk(KERN_INFO
				       "md/raid10:%s: read error corrected"
				       " (%d sectors at %llu on %s)\n",
				       mdname(mddev), s,
				       (unsigned long long)(
2521 2522
					       sect +
					       choose_data_offset(r10_bio, rdev)),
2523 2524
				       bdevname(rdev->bdev, b));
				atomic_add(s, &rdev->corrected_errors);
2525
			}
2526 2527 2528

			rdev_dec_pending(rdev, mddev);
			rcu_read_lock();
2529 2530 2531 2532 2533 2534 2535 2536
		}
		rcu_read_unlock();

		sectors -= s;
		sect += s;
	}
}

2537
static int narrow_write_error(struct r10bio *r10_bio, int i)
2538 2539
{
	struct bio *bio = r10_bio->master_bio;
2540
	struct mddev *mddev = r10_bio->mddev;
2541
	struct r10conf *conf = mddev->private;
2542
	struct md_rdev *rdev = conf->mirrors[r10_bio->devs[i].devnum].rdev;
2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562
	/* bio has the data to be written to slot 'i' where
	 * we just recently had a write error.
	 * We repeatedly clone the bio and trim down to one block,
	 * then try the write.  Where the write fails we record
	 * a bad block.
	 * It is conceivable that the bio doesn't exactly align with
	 * blocks.  We must handle this.
	 *
	 * We currently own a reference to the rdev.
	 */

	int block_sectors;
	sector_t sector;
	int sectors;
	int sect_to_write = r10_bio->sectors;
	int ok = 1;

	if (rdev->badblocks.shift < 0)
		return 0;

2563 2564
	block_sectors = roundup(1 << rdev->badblocks.shift,
				bdev_logical_block_size(rdev->bdev) >> 9);
2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575
	sector = r10_bio->sector;
	sectors = ((r10_bio->sector + block_sectors)
		   & ~(sector_t)(block_sectors - 1))
		- sector;

	while (sect_to_write) {
		struct bio *wbio;
		if (sectors > sect_to_write)
			sectors = sect_to_write;
		/* Write at 'sector' for 'sectors' */
		wbio = bio_clone_mddev(bio, GFP_NOIO, mddev);
2576 2577
		bio_trim(wbio, sector - bio->bi_iter.bi_sector, sectors);
		wbio->bi_iter.bi_sector = (r10_bio->devs[i].addr+
2578
				   choose_data_offset(r10_bio, rdev) +
2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594
				   (sector - r10_bio->sector));
		wbio->bi_bdev = rdev->bdev;
		if (submit_bio_wait(WRITE, wbio) == 0)
			/* Failure! */
			ok = rdev_set_badblocks(rdev, sector,
						sectors, 0)
				&& ok;

		bio_put(wbio);
		sect_to_write -= sectors;
		sector += sectors;
		sectors = block_sectors;
	}
	return ok;
}

2595
static void handle_read_error(struct mddev *mddev, struct r10bio *r10_bio)
2596 2597 2598
{
	int slot = r10_bio->read_slot;
	struct bio *bio;
2599
	struct r10conf *conf = mddev->private;
2600
	struct md_rdev *rdev = r10_bio->devs[slot].rdev;
2601 2602
	char b[BDEVNAME_SIZE];
	unsigned long do_sync;
2603
	int max_sectors;
2604 2605 2606 2607 2608 2609 2610 2611 2612

	/* we got a read error. Maybe the drive is bad.  Maybe just
	 * the block and we can fix it.
	 * We freeze all other IO, and try reading the block from
	 * other devices.  When we find one, we re-write
	 * and check it that fixes the read error.
	 * This is all done synchronously while the array is
	 * frozen.
	 */
2613 2614 2615 2616 2617
	bio = r10_bio->devs[slot].bio;
	bdevname(bio->bi_bdev, b);
	bio_put(bio);
	r10_bio->devs[slot].bio = NULL;

2618
	if (mddev->ro == 0) {
2619
		freeze_array(conf, 1);
2620 2621
		fix_read_error(conf, mddev, r10_bio);
		unfreeze_array(conf);
2622 2623 2624
	} else
		r10_bio->devs[slot].bio = IO_BLOCKED;

2625
	rdev_dec_pending(rdev, mddev);
2626

2627
read_more:
2628 2629
	rdev = read_balance(conf, r10_bio, &max_sectors);
	if (rdev == NULL) {
2630 2631
		printk(KERN_ALERT "md/raid10:%s: %s: unrecoverable I/O"
		       " read error for block %llu\n",
2632
		       mdname(mddev), b,
2633 2634 2635 2636 2637 2638 2639 2640 2641
		       (unsigned long long)r10_bio->sector);
		raid_end_bio_io(r10_bio);
		return;
	}

	do_sync = (r10_bio->master_bio->bi_rw & REQ_SYNC);
	slot = r10_bio->read_slot;
	printk_ratelimited(
		KERN_ERR
2642
		"md/raid10:%s: %s: redirecting "
2643 2644 2645 2646 2647 2648
		"sector %llu to another mirror\n",
		mdname(mddev),
		bdevname(rdev->bdev, b),
		(unsigned long long)r10_bio->sector);
	bio = bio_clone_mddev(r10_bio->master_bio,
			      GFP_NOIO, mddev);
2649
	bio_trim(bio, r10_bio->sector - bio->bi_iter.bi_sector, max_sectors);
2650
	r10_bio->devs[slot].bio = bio;
2651
	r10_bio->devs[slot].rdev = rdev;
2652
	bio->bi_iter.bi_sector = r10_bio->devs[slot].addr
2653
		+ choose_data_offset(r10_bio, rdev);
2654 2655 2656 2657
	bio->bi_bdev = rdev->bdev;
	bio->bi_rw = READ | do_sync;
	bio->bi_private = r10_bio;
	bio->bi_end_io = raid10_end_read_request;
2658 2659 2660 2661 2662
	if (max_sectors < r10_bio->sectors) {
		/* Drat - have to split this up more */
		struct bio *mbio = r10_bio->master_bio;
		int sectors_handled =
			r10_bio->sector + max_sectors
2663
			- mbio->bi_iter.bi_sector;
2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675
		r10_bio->sectors = max_sectors;
		spin_lock_irq(&conf->device_lock);
		if (mbio->bi_phys_segments == 0)
			mbio->bi_phys_segments = 2;
		else
			mbio->bi_phys_segments++;
		spin_unlock_irq(&conf->device_lock);
		generic_make_request(bio);

		r10_bio = mempool_alloc(conf->r10bio_pool,
					GFP_NOIO);
		r10_bio->master_bio = mbio;
2676
		r10_bio->sectors = bio_sectors(mbio) - sectors_handled;
2677 2678 2679 2680
		r10_bio->state = 0;
		set_bit(R10BIO_ReadError,
			&r10_bio->state);
		r10_bio->mddev = mddev;
2681
		r10_bio->sector = mbio->bi_iter.bi_sector
2682 2683 2684 2685 2686
			+ sectors_handled;

		goto read_more;
	} else
		generic_make_request(bio);
2687 2688
}

2689
static void handle_write_completed(struct r10conf *conf, struct r10bio *r10_bio)
2690 2691 2692 2693
{
	/* Some sort of write request has finished and it
	 * succeeded in writing where we thought there was a
	 * bad block.  So forget the bad block.
2694 2695
	 * Or possibly if failed and we need to record
	 * a bad block.
2696 2697
	 */
	int m;
2698
	struct md_rdev *rdev;
2699 2700 2701

	if (test_bit(R10BIO_IsSync, &r10_bio->state) ||
	    test_bit(R10BIO_IsRecover, &r10_bio->state)) {
2702 2703 2704 2705 2706
		for (m = 0; m < conf->copies; m++) {
			int dev = r10_bio->devs[m].devnum;
			rdev = conf->mirrors[dev].rdev;
			if (r10_bio->devs[m].bio == NULL)
				continue;
2707
			if (!r10_bio->devs[m].bio->bi_error) {
2708 2709 2710
				rdev_clear_badblocks(
					rdev,
					r10_bio->devs[m].addr,
2711
					r10_bio->sectors, 0);
2712 2713 2714 2715 2716 2717
			} else {
				if (!rdev_set_badblocks(
					    rdev,
					    r10_bio->devs[m].addr,
					    r10_bio->sectors, 0))
					md_error(conf->mddev, rdev);
2718
			}
2719 2720 2721
			rdev = conf->mirrors[dev].replacement;
			if (r10_bio->devs[m].repl_bio == NULL)
				continue;
2722 2723

			if (!r10_bio->devs[m].repl_bio->bi_error) {
2724 2725 2726
				rdev_clear_badblocks(
					rdev,
					r10_bio->devs[m].addr,
2727
					r10_bio->sectors, 0);
2728 2729 2730 2731 2732 2733 2734
			} else {
				if (!rdev_set_badblocks(
					    rdev,
					    r10_bio->devs[m].addr,
					    r10_bio->sectors, 0))
					md_error(conf->mddev, rdev);
			}
2735
		}
2736 2737
		put_buf(r10_bio);
	} else {
2738 2739 2740 2741 2742
		for (m = 0; m < conf->copies; m++) {
			int dev = r10_bio->devs[m].devnum;
			struct bio *bio = r10_bio->devs[m].bio;
			rdev = conf->mirrors[dev].rdev;
			if (bio == IO_MADE_GOOD) {
2743 2744 2745
				rdev_clear_badblocks(
					rdev,
					r10_bio->devs[m].addr,
2746
					r10_bio->sectors, 0);
2747
				rdev_dec_pending(rdev, conf->mddev);
2748
			} else if (bio != NULL && bio->bi_error) {
2749 2750 2751 2752 2753 2754
				if (!narrow_write_error(r10_bio, m)) {
					md_error(conf->mddev, rdev);
					set_bit(R10BIO_Degraded,
						&r10_bio->state);
				}
				rdev_dec_pending(rdev, conf->mddev);
2755
			}
2756 2757
			bio = r10_bio->devs[m].repl_bio;
			rdev = conf->mirrors[dev].replacement;
2758
			if (rdev && bio == IO_MADE_GOOD) {
2759 2760 2761
				rdev_clear_badblocks(
					rdev,
					r10_bio->devs[m].addr,
2762
					r10_bio->sectors, 0);
2763 2764
				rdev_dec_pending(rdev, conf->mddev);
			}
2765 2766 2767 2768
		}
		if (test_bit(R10BIO_WriteError,
			     &r10_bio->state))
			close_write(r10_bio);
2769 2770 2771 2772
		raid_end_bio_io(r10_bio);
	}
}

S
Shaohua Li 已提交
2773
static void raid10d(struct md_thread *thread)
L
Linus Torvalds 已提交
2774
{
S
Shaohua Li 已提交
2775
	struct mddev *mddev = thread->mddev;
2776
	struct r10bio *r10_bio;
L
Linus Torvalds 已提交
2777
	unsigned long flags;
2778
	struct r10conf *conf = mddev->private;
L
Linus Torvalds 已提交
2779
	struct list_head *head = &conf->retry_list;
2780
	struct blk_plug plug;
L
Linus Torvalds 已提交
2781 2782 2783

	md_check_recovery(mddev);

2784
	blk_start_plug(&plug);
L
Linus Torvalds 已提交
2785
	for (;;) {
2786

2787
		flush_pending_writes(conf);
2788

2789 2790 2791
		spin_lock_irqsave(&conf->device_lock, flags);
		if (list_empty(head)) {
			spin_unlock_irqrestore(&conf->device_lock, flags);
L
Linus Torvalds 已提交
2792
			break;
2793
		}
2794
		r10_bio = list_entry(head->prev, struct r10bio, retry_list);
L
Linus Torvalds 已提交
2795
		list_del(head->prev);
2796
		conf->nr_queued--;
L
Linus Torvalds 已提交
2797 2798 2799
		spin_unlock_irqrestore(&conf->device_lock, flags);

		mddev = r10_bio->mddev;
2800
		conf = mddev->private;
2801 2802
		if (test_bit(R10BIO_MadeGood, &r10_bio->state) ||
		    test_bit(R10BIO_WriteError, &r10_bio->state))
2803
			handle_write_completed(conf, r10_bio);
N
NeilBrown 已提交
2804 2805
		else if (test_bit(R10BIO_IsReshape, &r10_bio->state))
			reshape_request_write(mddev, r10_bio);
2806
		else if (test_bit(R10BIO_IsSync, &r10_bio->state))
L
Linus Torvalds 已提交
2807
			sync_request_write(mddev, r10_bio);
J
Jens Axboe 已提交
2808
		else if (test_bit(R10BIO_IsRecover, &r10_bio->state))
L
Linus Torvalds 已提交
2809
			recovery_request_write(mddev, r10_bio);
2810
		else if (test_bit(R10BIO_ReadError, &r10_bio->state))
2811
			handle_read_error(mddev, r10_bio);
2812 2813 2814 2815 2816 2817 2818
		else {
			/* just a partial read to be scheduled from a
			 * separate context
			 */
			int slot = r10_bio->read_slot;
			generic_make_request(r10_bio->devs[slot].bio);
		}
2819

N
NeilBrown 已提交
2820
		cond_resched();
2821 2822
		if (mddev->flags & ~(1<<MD_CHANGE_PENDING))
			md_check_recovery(mddev);
L
Linus Torvalds 已提交
2823
	}
2824
	blk_finish_plug(&plug);
L
Linus Torvalds 已提交
2825 2826
}

2827
static int init_resync(struct r10conf *conf)
L
Linus Torvalds 已提交
2828 2829
{
	int buffs;
2830
	int i;
L
Linus Torvalds 已提交
2831 2832

	buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE;
2833
	BUG_ON(conf->r10buf_pool);
2834
	conf->have_replacement = 0;
2835
	for (i = 0; i < conf->geo.raid_disks; i++)
2836 2837
		if (conf->mirrors[i].replacement)
			conf->have_replacement = 1;
L
Linus Torvalds 已提交
2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876
	conf->r10buf_pool = mempool_create(buffs, r10buf_pool_alloc, r10buf_pool_free, conf);
	if (!conf->r10buf_pool)
		return -ENOMEM;
	conf->next_resync = 0;
	return 0;
}

/*
 * perform a "sync" on one "block"
 *
 * We need to make sure that no normal I/O request - particularly write
 * requests - conflict with active sync requests.
 *
 * This is achieved by tracking pending requests and a 'barrier' concept
 * that can be installed to exclude normal IO requests.
 *
 * Resync and recovery are handled very differently.
 * We differentiate by looking at MD_RECOVERY_SYNC in mddev->recovery.
 *
 * For resync, we iterate over virtual addresses, read all copies,
 * and update if there are differences.  If only one copy is live,
 * skip it.
 * For recovery, we iterate over physical addresses, read a good
 * value for each non-in_sync drive, and over-write.
 *
 * So, for recovery we may have several outstanding complex requests for a
 * given address, one for each out-of-sync device.  We model this by allocating
 * a number of r10_bio structures, one for each out-of-sync device.
 * As we setup these structures, we collect all bio's together into a list
 * which we then process collectively to add pages, and then process again
 * to pass to generic_make_request.
 *
 * The r10_bio structures are linked using a borrowed master_bio pointer.
 * This link is counted in ->remaining.  When the r10_bio that points to NULL
 * has its remaining count decremented to 0, the whole complex operation
 * is complete.
 *
 */

2877
static sector_t sync_request(struct mddev *mddev, sector_t sector_nr,
2878
			     int *skipped)
L
Linus Torvalds 已提交
2879
{
2880
	struct r10conf *conf = mddev->private;
2881
	struct r10bio *r10_bio;
L
Linus Torvalds 已提交
2882 2883 2884
	struct bio *biolist = NULL, *bio;
	sector_t max_sector, nr_sectors;
	int i;
2885
	int max_sync;
N
NeilBrown 已提交
2886
	sector_t sync_blocks;
L
Linus Torvalds 已提交
2887 2888
	sector_t sectors_skipped = 0;
	int chunks_skipped = 0;
2889
	sector_t chunk_mask = conf->geo.chunk_mask;
L
Linus Torvalds 已提交
2890 2891 2892

	if (!conf->r10buf_pool)
		if (init_resync(conf))
2893
			return 0;
L
Linus Torvalds 已提交
2894

2895 2896 2897 2898 2899 2900
	/*
	 * Allow skipping a full rebuild for incremental assembly
	 * of a clean array, like RAID1 does.
	 */
	if (mddev->bitmap == NULL &&
	    mddev->recovery_cp == MaxSector &&
2901 2902
	    mddev->reshape_position == MaxSector &&
	    !test_bit(MD_RECOVERY_SYNC, &mddev->recovery) &&
2903
	    !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
2904
	    !test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery) &&
2905 2906
	    conf->fullsync == 0) {
		*skipped = 1;
2907
		return mddev->dev_sectors - sector_nr;
2908 2909
	}

L
Linus Torvalds 已提交
2910
 skipped:
A
Andre Noll 已提交
2911
	max_sector = mddev->dev_sectors;
N
NeilBrown 已提交
2912 2913
	if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) ||
	    test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
L
Linus Torvalds 已提交
2914 2915
		max_sector = mddev->resync_max_sectors;
	if (sector_nr >= max_sector) {
2916 2917 2918 2919 2920 2921 2922 2923 2924
		/* If we aborted, we need to abort the
		 * sync on the 'current' bitmap chucks (there can
		 * be several when recovering multiple devices).
		 * as we may have started syncing it but not finished.
		 * We can find the current address in
		 * mddev->curr_resync, but for recovery,
		 * we need to convert that to several
		 * virtual addresses.
		 */
N
NeilBrown 已提交
2925 2926
		if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {
			end_reshape(conf);
2927
			close_sync(conf);
N
NeilBrown 已提交
2928 2929 2930
			return 0;
		}

2931 2932 2933 2934
		if (mddev->curr_resync < max_sector) { /* aborted */
			if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery))
				bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
						&sync_blocks, 1);
2935
			else for (i = 0; i < conf->geo.raid_disks; i++) {
2936 2937 2938 2939 2940
				sector_t sect =
					raid10_find_virt(conf, mddev->curr_resync, i);
				bitmap_end_sync(mddev->bitmap, sect,
						&sync_blocks, 1);
			}
2941 2942 2943 2944 2945 2946 2947 2948
		} else {
			/* completed sync */
			if ((!mddev->bitmap || conf->fullsync)
			    && conf->have_replacement
			    && test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
				/* Completed a full sync so the replacements
				 * are now fully recovered.
				 */
2949
				for (i = 0; i < conf->geo.raid_disks; i++)
2950 2951 2952 2953 2954
					if (conf->mirrors[i].replacement)
						conf->mirrors[i].replacement
							->recovery_offset
							= MaxSector;
			}
2955
			conf->fullsync = 0;
2956
		}
2957
		bitmap_close_sync(mddev->bitmap);
L
Linus Torvalds 已提交
2958
		close_sync(conf);
2959
		*skipped = 1;
L
Linus Torvalds 已提交
2960 2961
		return sectors_skipped;
	}
N
NeilBrown 已提交
2962 2963 2964 2965

	if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
		return reshape_request(mddev, sector_nr, skipped);

2966
	if (chunks_skipped >= conf->geo.raid_disks) {
L
Linus Torvalds 已提交
2967 2968 2969
		/* if there has been nothing to do on any drive,
		 * then there is nothing to do at all..
		 */
2970 2971
		*skipped = 1;
		return (max_sector - sector_nr) + sectors_skipped;
L
Linus Torvalds 已提交
2972 2973
	}

2974 2975 2976
	if (max_sector > mddev->resync_max)
		max_sector = mddev->resync_max; /* Don't do IO beyond here */

L
Linus Torvalds 已提交
2977 2978 2979
	/* make sure whole request will fit in a chunk - if chunks
	 * are meaningful
	 */
2980 2981 2982
	if (conf->geo.near_copies < conf->geo.raid_disks &&
	    max_sector > (sector_nr | chunk_mask))
		max_sector = (sector_nr | chunk_mask) + 1;
L
Linus Torvalds 已提交
2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998

	/* Again, very different code for resync and recovery.
	 * Both must result in an r10bio with a list of bios that
	 * have bi_end_io, bi_sector, bi_bdev set,
	 * and bi_private set to the r10bio.
	 * For recovery, we may actually create several r10bios
	 * with 2 bios in each, that correspond to the bios in the main one.
	 * In this case, the subordinate r10bios link back through a
	 * borrowed master_bio pointer, and the counter in the master
	 * includes a ref from each subordinate.
	 */
	/* First, we decide what to do and set ->bi_end_io
	 * To end_sync_read if we want to read, and
	 * end_sync_write if we will want to write.
	 */

2999
	max_sync = RESYNC_PAGES << (PAGE_SHIFT-9);
L
Linus Torvalds 已提交
3000 3001
	if (!test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
		/* recovery... the complicated one */
3002
		int j;
L
Linus Torvalds 已提交
3003 3004
		r10_bio = NULL;

3005
		for (i = 0 ; i < conf->geo.raid_disks; i++) {
3006
			int still_degraded;
3007
			struct r10bio *rb2;
3008 3009
			sector_t sect;
			int must_sync;
3010
			int any_working;
3011
			struct raid10_info *mirror = &conf->mirrors[i];
3012 3013 3014 3015 3016 3017 3018

			if ((mirror->rdev == NULL ||
			     test_bit(In_sync, &mirror->rdev->flags))
			    &&
			    (mirror->replacement == NULL ||
			     test_bit(Faulty,
				      &mirror->replacement->flags)))
3019
				continue;
L
Linus Torvalds 已提交
3020

3021 3022 3023 3024
			still_degraded = 0;
			/* want to reconstruct this device */
			rb2 = r10_bio;
			sect = raid10_find_virt(conf, sector_nr, i);
3025 3026 3027 3028 3029 3030
			if (sect >= mddev->resync_max_sectors) {
				/* last stripe is not complete - don't
				 * try to recover this sector.
				 */
				continue;
			}
3031 3032 3033
			/* Unless we are doing a full sync, or a replacement
			 * we only need to recover the block if it is set in
			 * the bitmap
3034 3035 3036 3037 3038 3039
			 */
			must_sync = bitmap_start_sync(mddev->bitmap, sect,
						      &sync_blocks, 1);
			if (sync_blocks < max_sync)
				max_sync = sync_blocks;
			if (!must_sync &&
3040
			    mirror->replacement == NULL &&
3041 3042 3043 3044 3045 3046 3047
			    !conf->fullsync) {
				/* yep, skip the sync_blocks here, but don't assume
				 * that there will never be anything to do here
				 */
				chunks_skipped = -1;
				continue;
			}
3048

3049
			r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
3050
			r10_bio->state = 0;
3051 3052
			raise_barrier(conf, rb2 != NULL);
			atomic_set(&r10_bio->remaining, 0);
3053

3054 3055 3056 3057 3058 3059
			r10_bio->master_bio = (struct bio*)rb2;
			if (rb2)
				atomic_inc(&rb2->remaining);
			r10_bio->mddev = mddev;
			set_bit(R10BIO_IsRecover, &r10_bio->state);
			r10_bio->sector = sect;
L
Linus Torvalds 已提交
3060

3061 3062 3063 3064 3065
			raid10_find_phys(conf, r10_bio);

			/* Need to check if the array will still be
			 * degraded
			 */
3066
			for (j = 0; j < conf->geo.raid_disks; j++)
3067 3068 3069
				if (conf->mirrors[j].rdev == NULL ||
				    test_bit(Faulty, &conf->mirrors[j].rdev->flags)) {
					still_degraded = 1;
3070
					break;
L
Linus Torvalds 已提交
3071
				}
3072 3073 3074 3075

			must_sync = bitmap_start_sync(mddev->bitmap, sect,
						      &sync_blocks, still_degraded);

3076
			any_working = 0;
3077
			for (j=0; j<conf->copies;j++) {
3078
				int k;
3079
				int d = r10_bio->devs[j].devnum;
3080
				sector_t from_addr, to_addr;
3081
				struct md_rdev *rdev;
3082 3083
				sector_t sector, first_bad;
				int bad_sectors;
3084 3085 3086 3087
				if (!conf->mirrors[d].rdev ||
				    !test_bit(In_sync, &conf->mirrors[d].rdev->flags))
					continue;
				/* This is where we read from */
3088
				any_working = 1;
3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103
				rdev = conf->mirrors[d].rdev;
				sector = r10_bio->devs[j].addr;

				if (is_badblock(rdev, sector, max_sync,
						&first_bad, &bad_sectors)) {
					if (first_bad > sector)
						max_sync = first_bad - sector;
					else {
						bad_sectors -= (sector
								- first_bad);
						if (max_sync > bad_sectors)
							max_sync = bad_sectors;
						continue;
					}
				}
3104
				bio = r10_bio->devs[0].bio;
K
Kent Overstreet 已提交
3105
				bio_reset(bio);
3106 3107 3108 3109 3110
				bio->bi_next = biolist;
				biolist = bio;
				bio->bi_private = r10_bio;
				bio->bi_end_io = end_sync_read;
				bio->bi_rw = READ;
3111
				from_addr = r10_bio->devs[j].addr;
3112 3113
				bio->bi_iter.bi_sector = from_addr +
					rdev->data_offset;
3114 3115 3116
				bio->bi_bdev = rdev->bdev;
				atomic_inc(&rdev->nr_pending);
				/* and we write to 'i' (if not in_sync) */
3117 3118 3119 3120 3121

				for (k=0; k<conf->copies; k++)
					if (r10_bio->devs[k].devnum == i)
						break;
				BUG_ON(k == conf->copies);
3122
				to_addr = r10_bio->devs[k].addr;
3123
				r10_bio->devs[0].devnum = d;
3124
				r10_bio->devs[0].addr = from_addr;
3125
				r10_bio->devs[1].devnum = i;
3126
				r10_bio->devs[1].addr = to_addr;
3127

3128 3129 3130
				rdev = mirror->rdev;
				if (!test_bit(In_sync, &rdev->flags)) {
					bio = r10_bio->devs[1].bio;
K
Kent Overstreet 已提交
3131
					bio_reset(bio);
3132 3133 3134 3135 3136
					bio->bi_next = biolist;
					biolist = bio;
					bio->bi_private = r10_bio;
					bio->bi_end_io = end_sync_write;
					bio->bi_rw = WRITE;
3137
					bio->bi_iter.bi_sector = to_addr
3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159
						+ rdev->data_offset;
					bio->bi_bdev = rdev->bdev;
					atomic_inc(&r10_bio->remaining);
				} else
					r10_bio->devs[1].bio->bi_end_io = NULL;

				/* and maybe write to replacement */
				bio = r10_bio->devs[1].repl_bio;
				if (bio)
					bio->bi_end_io = NULL;
				rdev = mirror->replacement;
				/* Note: if rdev != NULL, then bio
				 * cannot be NULL as r10buf_pool_alloc will
				 * have allocated it.
				 * So the second test here is pointless.
				 * But it keeps semantic-checkers happy, and
				 * this comment keeps human reviewers
				 * happy.
				 */
				if (rdev == NULL || bio == NULL ||
				    test_bit(Faulty, &rdev->flags))
					break;
K
Kent Overstreet 已提交
3160
				bio_reset(bio);
3161 3162 3163 3164 3165
				bio->bi_next = biolist;
				biolist = bio;
				bio->bi_private = r10_bio;
				bio->bi_end_io = end_sync_write;
				bio->bi_rw = WRITE;
3166 3167
				bio->bi_iter.bi_sector = to_addr +
					rdev->data_offset;
3168 3169
				bio->bi_bdev = rdev->bdev;
				atomic_inc(&r10_bio->remaining);
3170 3171 3172
				break;
			}
			if (j == conf->copies) {
3173 3174 3175 3176 3177 3178 3179 3180 3181 3182
				/* Cannot recover, so abort the recovery or
				 * record a bad block */
				if (any_working) {
					/* problem is that there are bad blocks
					 * on other device(s)
					 */
					int k;
					for (k = 0; k < conf->copies; k++)
						if (r10_bio->devs[k].devnum == i)
							break;
3183 3184 3185 3186 3187 3188 3189 3190 3191 3192
					if (!test_bit(In_sync,
						      &mirror->rdev->flags)
					    && !rdev_set_badblocks(
						    mirror->rdev,
						    r10_bio->devs[k].addr,
						    max_sync, 0))
						any_working = 0;
					if (mirror->replacement &&
					    !rdev_set_badblocks(
						    mirror->replacement,
3193 3194 3195 3196 3197 3198 3199 3200 3201 3202
						    r10_bio->devs[k].addr,
						    max_sync, 0))
						any_working = 0;
				}
				if (!any_working)  {
					if (!test_and_set_bit(MD_RECOVERY_INTR,
							      &mddev->recovery))
						printk(KERN_INFO "md/raid10:%s: insufficient "
						       "working devices for recovery.\n",
						       mdname(mddev));
3203
					mirror->recovery_disabled
3204 3205
						= mddev->recovery_disabled;
				}
3206 3207 3208 3209
				put_buf(r10_bio);
				if (rb2)
					atomic_dec(&rb2->remaining);
				r10_bio = rb2;
3210
				break;
L
Linus Torvalds 已提交
3211
			}
3212
		}
L
Linus Torvalds 已提交
3213 3214
		if (biolist == NULL) {
			while (r10_bio) {
3215 3216
				struct r10bio *rb2 = r10_bio;
				r10_bio = (struct r10bio*) rb2->master_bio;
L
Linus Torvalds 已提交
3217 3218 3219 3220 3221 3222 3223 3224
				rb2->master_bio = NULL;
				put_buf(rb2);
			}
			goto giveup;
		}
	} else {
		/* resync. Schedule a read for every block at this virt offset */
		int count = 0;
3225

3226 3227
		bitmap_cond_end_sync(mddev->bitmap, sector_nr);

3228 3229
		if (!bitmap_start_sync(mddev->bitmap, sector_nr,
				       &sync_blocks, mddev->degraded) &&
3230 3231
		    !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED,
						 &mddev->recovery)) {
3232 3233 3234 3235 3236 3237
			/* We can skip this block */
			*skipped = 1;
			return sync_blocks + sectors_skipped;
		}
		if (sync_blocks < max_sync)
			max_sync = sync_blocks;
L
Linus Torvalds 已提交
3238
		r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
3239
		r10_bio->state = 0;
L
Linus Torvalds 已提交
3240 3241 3242

		r10_bio->mddev = mddev;
		atomic_set(&r10_bio->remaining, 0);
3243 3244
		raise_barrier(conf, 0);
		conf->next_resync = sector_nr;
L
Linus Torvalds 已提交
3245 3246 3247 3248 3249

		r10_bio->master_bio = NULL;
		r10_bio->sector = sector_nr;
		set_bit(R10BIO_IsSync, &r10_bio->state);
		raid10_find_phys(conf, r10_bio);
3250
		r10_bio->sectors = (sector_nr | chunk_mask) - sector_nr + 1;
L
Linus Torvalds 已提交
3251

3252
		for (i = 0; i < conf->copies; i++) {
L
Linus Torvalds 已提交
3253
			int d = r10_bio->devs[i].devnum;
3254 3255 3256
			sector_t first_bad, sector;
			int bad_sectors;

3257 3258 3259
			if (r10_bio->devs[i].repl_bio)
				r10_bio->devs[i].repl_bio->bi_end_io = NULL;

L
Linus Torvalds 已提交
3260
			bio = r10_bio->devs[i].bio;
K
Kent Overstreet 已提交
3261
			bio_reset(bio);
3262
			bio->bi_error = -EIO;
L
Linus Torvalds 已提交
3263
			if (conf->mirrors[d].rdev == NULL ||
3264
			    test_bit(Faulty, &conf->mirrors[d].rdev->flags))
L
Linus Torvalds 已提交
3265
				continue;
3266 3267 3268 3269 3270 3271 3272 3273 3274
			sector = r10_bio->devs[i].addr;
			if (is_badblock(conf->mirrors[d].rdev,
					sector, max_sync,
					&first_bad, &bad_sectors)) {
				if (first_bad > sector)
					max_sync = first_bad - sector;
				else {
					bad_sectors -= (sector - first_bad);
					if (max_sync > bad_sectors)
3275
						max_sync = bad_sectors;
3276 3277 3278
					continue;
				}
			}
L
Linus Torvalds 已提交
3279 3280 3281 3282 3283 3284
			atomic_inc(&conf->mirrors[d].rdev->nr_pending);
			atomic_inc(&r10_bio->remaining);
			bio->bi_next = biolist;
			biolist = bio;
			bio->bi_private = r10_bio;
			bio->bi_end_io = end_sync_read;
3285
			bio->bi_rw = READ;
3286
			bio->bi_iter.bi_sector = sector +
L
Linus Torvalds 已提交
3287 3288 3289
				conf->mirrors[d].rdev->data_offset;
			bio->bi_bdev = conf->mirrors[d].rdev->bdev;
			count++;
3290 3291 3292 3293 3294 3295 3296 3297

			if (conf->mirrors[d].replacement == NULL ||
			    test_bit(Faulty,
				     &conf->mirrors[d].replacement->flags))
				continue;

			/* Need to set up for writing to the replacement */
			bio = r10_bio->devs[i].repl_bio;
K
Kent Overstreet 已提交
3298
			bio_reset(bio);
3299
			bio->bi_error = -EIO;
3300 3301 3302 3303 3304 3305 3306 3307

			sector = r10_bio->devs[i].addr;
			atomic_inc(&conf->mirrors[d].rdev->nr_pending);
			bio->bi_next = biolist;
			biolist = bio;
			bio->bi_private = r10_bio;
			bio->bi_end_io = end_sync_write;
			bio->bi_rw = WRITE;
3308
			bio->bi_iter.bi_sector = sector +
3309 3310 3311
				conf->mirrors[d].replacement->data_offset;
			bio->bi_bdev = conf->mirrors[d].replacement->bdev;
			count++;
L
Linus Torvalds 已提交
3312 3313 3314 3315 3316 3317
		}

		if (count < 2) {
			for (i=0; i<conf->copies; i++) {
				int d = r10_bio->devs[i].devnum;
				if (r10_bio->devs[i].bio->bi_end_io)
3318 3319
					rdev_dec_pending(conf->mirrors[d].rdev,
							 mddev);
3320 3321 3322 3323 3324
				if (r10_bio->devs[i].repl_bio &&
				    r10_bio->devs[i].repl_bio->bi_end_io)
					rdev_dec_pending(
						conf->mirrors[d].replacement,
						mddev);
L
Linus Torvalds 已提交
3325 3326 3327 3328 3329 3330 3331 3332
			}
			put_buf(r10_bio);
			biolist = NULL;
			goto giveup;
		}
	}

	nr_sectors = 0;
3333 3334
	if (sector_nr + max_sync < max_sector)
		max_sector = sector_nr + max_sync;
L
Linus Torvalds 已提交
3335 3336 3337 3338 3339 3340 3341 3342
	do {
		struct page *page;
		int len = PAGE_SIZE;
		if (sector_nr + (len>>9) > max_sector)
			len = (max_sector - sector_nr) << 9;
		if (len == 0)
			break;
		for (bio= biolist ; bio ; bio=bio->bi_next) {
3343
			struct bio *bio2;
L
Linus Torvalds 已提交
3344
			page = bio->bi_io_vec[bio->bi_vcnt].bv_page;
3345 3346 3347 3348 3349 3350 3351 3352 3353 3354
			if (bio_add_page(bio, page, len, 0))
				continue;

			/* stop here */
			bio->bi_io_vec[bio->bi_vcnt].bv_page = page;
			for (bio2 = biolist;
			     bio2 && bio2 != bio;
			     bio2 = bio2->bi_next) {
				/* remove last page from this bio */
				bio2->bi_vcnt--;
3355
				bio2->bi_iter.bi_size -= len;
3356
				bio_clear_flag(bio2, BIO_SEG_VALID);
L
Linus Torvalds 已提交
3357
			}
3358
			goto bio_full;
L
Linus Torvalds 已提交
3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370 3371 3372 3373 3374 3375
		}
		nr_sectors += len>>9;
		sector_nr += len>>9;
	} while (biolist->bi_vcnt < RESYNC_PAGES);
 bio_full:
	r10_bio->sectors = nr_sectors;

	while (biolist) {
		bio = biolist;
		biolist = biolist->bi_next;

		bio->bi_next = NULL;
		r10_bio = bio->bi_private;
		r10_bio->sectors = nr_sectors;

		if (bio->bi_end_io == end_sync_read) {
			md_sync_acct(bio->bi_bdev, nr_sectors);
3376
			bio->bi_error = 0;
L
Linus Torvalds 已提交
3377 3378 3379 3380
			generic_make_request(bio);
		}
	}

3381 3382 3383 3384 3385 3386
	if (sectors_skipped)
		/* pretend they weren't skipped, it makes
		 * no important difference in this case
		 */
		md_done_sync(mddev, sectors_skipped, 1);

L
Linus Torvalds 已提交
3387 3388 3389
	return sectors_skipped + nr_sectors;
 giveup:
	/* There is nowhere to write, so all non-sync
3390 3391
	 * drives must be failed or in resync, all drives
	 * have a bad block, so try the next chunk...
L
Linus Torvalds 已提交
3392
	 */
3393 3394 3395 3396
	if (sector_nr + max_sync < max_sector)
		max_sector = sector_nr + max_sync;

	sectors_skipped += (max_sector - sector_nr);
L
Linus Torvalds 已提交
3397 3398 3399 3400 3401
	chunks_skipped ++;
	sector_nr = max_sector;
	goto skipped;
}

3402
static sector_t
3403
raid10_size(struct mddev *mddev, sector_t sectors, int raid_disks)
3404 3405
{
	sector_t size;
3406
	struct r10conf *conf = mddev->private;
3407 3408

	if (!raid_disks)
N
NeilBrown 已提交
3409 3410
		raid_disks = min(conf->geo.raid_disks,
				 conf->prev.raid_disks);
3411
	if (!sectors)
3412
		sectors = conf->dev_sectors;
3413

3414 3415
	size = sectors >> conf->geo.chunk_shift;
	sector_div(size, conf->geo.far_copies);
3416
	size = size * raid_disks;
3417
	sector_div(size, conf->geo.near_copies);
3418

3419
	return size << conf->geo.chunk_shift;
3420 3421
}

3422 3423 3424 3425 3426 3427 3428
static void calc_sectors(struct r10conf *conf, sector_t size)
{
	/* Calculate the number of sectors-per-device that will
	 * actually be used, and set conf->dev_sectors and
	 * conf->stride
	 */

3429 3430 3431 3432
	size = size >> conf->geo.chunk_shift;
	sector_div(size, conf->geo.far_copies);
	size = size * conf->geo.raid_disks;
	sector_div(size, conf->geo.near_copies);
3433 3434 3435 3436 3437 3438 3439
	/* 'size' is now the number of chunks in the array */
	/* calculate "used chunks per device" */
	size = size * conf->copies;

	/* We need to round up when dividing by raid_disks to
	 * get the stride size.
	 */
3440
	size = DIV_ROUND_UP_SECTOR_T(size, conf->geo.raid_disks);
3441

3442
	conf->dev_sectors = size << conf->geo.chunk_shift;
3443

3444 3445
	if (conf->geo.far_offset)
		conf->geo.stride = 1 << conf->geo.chunk_shift;
3446
	else {
3447 3448
		sector_div(size, conf->geo.far_copies);
		conf->geo.stride = size << conf->geo.chunk_shift;
3449 3450
	}
}
3451

3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475
enum geo_type {geo_new, geo_old, geo_start};
static int setup_geo(struct geom *geo, struct mddev *mddev, enum geo_type new)
{
	int nc, fc, fo;
	int layout, chunk, disks;
	switch (new) {
	case geo_old:
		layout = mddev->layout;
		chunk = mddev->chunk_sectors;
		disks = mddev->raid_disks - mddev->delta_disks;
		break;
	case geo_new:
		layout = mddev->new_layout;
		chunk = mddev->new_chunk_sectors;
		disks = mddev->raid_disks;
		break;
	default: /* avoid 'may be unused' warnings */
	case geo_start: /* new when starting reshape - raid_disks not
			 * updated yet. */
		layout = mddev->new_layout;
		chunk = mddev->new_chunk_sectors;
		disks = mddev->raid_disks + mddev->delta_disks;
		break;
	}
3476
	if (layout >> 18)
3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487
		return -1;
	if (chunk < (PAGE_SIZE >> 9) ||
	    !is_power_of_2(chunk))
		return -2;
	nc = layout & 255;
	fc = (layout >> 8) & 255;
	fo = layout & (1<<16);
	geo->raid_disks = disks;
	geo->near_copies = nc;
	geo->far_copies = fc;
	geo->far_offset = fo;
3488
	geo->far_set_size = (layout & (1<<17)) ? disks / fc : disks;
3489 3490 3491 3492 3493
	geo->chunk_mask = chunk - 1;
	geo->chunk_shift = ffz(~chunk);
	return nc*fc;
}

3494
static struct r10conf *setup_conf(struct mddev *mddev)
L
Linus Torvalds 已提交
3495
{
3496
	struct r10conf *conf = NULL;
3497
	int err = -EINVAL;
3498 3499 3500 3501
	struct geom geo;
	int copies;

	copies = setup_geo(&geo, mddev, geo_new);
L
Linus Torvalds 已提交
3502

3503
	if (copies == -2) {
N
NeilBrown 已提交
3504 3505 3506
		printk(KERN_ERR "md/raid10:%s: chunk size must be "
		       "at least PAGE_SIZE(%ld) and be a power of 2.\n",
		       mdname(mddev), PAGE_SIZE);
3507
		goto out;
L
Linus Torvalds 已提交
3508
	}
3509

3510
	if (copies < 2 || copies > mddev->raid_disks) {
N
NeilBrown 已提交
3511
		printk(KERN_ERR "md/raid10:%s: unsupported raid10 layout: 0x%8x\n",
3512
		       mdname(mddev), mddev->new_layout);
L
Linus Torvalds 已提交
3513 3514
		goto out;
	}
3515 3516

	err = -ENOMEM;
3517
	conf = kzalloc(sizeof(struct r10conf), GFP_KERNEL);
3518
	if (!conf)
L
Linus Torvalds 已提交
3519
		goto out;
3520

N
NeilBrown 已提交
3521
	/* FIXME calc properly */
3522
	conf->mirrors = kzalloc(sizeof(struct raid10_info)*(mddev->raid_disks +
3523
							    max(0,-mddev->delta_disks)),
3524 3525 3526
				GFP_KERNEL);
	if (!conf->mirrors)
		goto out;
3527 3528 3529

	conf->tmppage = alloc_page(GFP_KERNEL);
	if (!conf->tmppage)
3530 3531
		goto out;

3532 3533
	conf->geo = geo;
	conf->copies = copies;
3534 3535 3536 3537 3538
	conf->r10bio_pool = mempool_create(NR_RAID10_BIOS, r10bio_pool_alloc,
					   r10bio_pool_free, conf);
	if (!conf->r10bio_pool)
		goto out;

3539
	calc_sectors(conf, mddev->dev_sectors);
N
NeilBrown 已提交
3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554
	if (mddev->reshape_position == MaxSector) {
		conf->prev = conf->geo;
		conf->reshape_progress = MaxSector;
	} else {
		if (setup_geo(&conf->prev, mddev, geo_old) != conf->copies) {
			err = -EINVAL;
			goto out;
		}
		conf->reshape_progress = mddev->reshape_position;
		if (conf->prev.far_offset)
			conf->prev.stride = 1 << conf->prev.chunk_shift;
		else
			/* far_copies must be 1 */
			conf->prev.stride = conf->dev_sectors;
	}
3555
	spin_lock_init(&conf->device_lock);
3556 3557 3558 3559 3560
	INIT_LIST_HEAD(&conf->retry_list);

	spin_lock_init(&conf->resync_lock);
	init_waitqueue_head(&conf->wait_barrier);

3561
	conf->thread = md_register_thread(raid10d, mddev, "raid10");
3562 3563 3564 3565 3566 3567 3568
	if (!conf->thread)
		goto out;

	conf->mddev = mddev;
	return conf;

 out:
N
NeilBrown 已提交
3569 3570 3571
	if (err == -ENOMEM)
		printk(KERN_ERR "md/raid10:%s: couldn't allocate memory.\n",
		       mdname(mddev));
3572 3573 3574 3575 3576 3577 3578 3579 3580 3581
	if (conf) {
		if (conf->r10bio_pool)
			mempool_destroy(conf->r10bio_pool);
		kfree(conf->mirrors);
		safe_put_page(conf->tmppage);
		kfree(conf);
	}
	return ERR_PTR(err);
}

3582
static int run(struct mddev *mddev)
3583
{
3584
	struct r10conf *conf;
3585
	int i, disk_idx, chunk_size;
3586
	struct raid10_info *disk;
3587
	struct md_rdev *rdev;
3588
	sector_t size;
N
NeilBrown 已提交
3589 3590
	sector_t min_offset_diff = 0;
	int first = 1;
S
Shaohua Li 已提交
3591
	bool discard_supported = false;
3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605

	if (mddev->private == NULL) {
		conf = setup_conf(mddev);
		if (IS_ERR(conf))
			return PTR_ERR(conf);
		mddev->private = conf;
	}
	conf = mddev->private;
	if (!conf)
		goto out;

	mddev->thread = conf->thread;
	conf->thread = NULL;

3606
	chunk_size = mddev->chunk_sectors << 9;
3607
	if (mddev->queue) {
S
Shaohua Li 已提交
3608 3609
		blk_queue_max_discard_sectors(mddev->queue,
					      mddev->chunk_sectors);
3610
		blk_queue_max_write_same_sectors(mddev->queue, 0);
3611 3612 3613 3614 3615 3616 3617
		blk_queue_io_min(mddev->queue, chunk_size);
		if (conf->geo.raid_disks % conf->geo.near_copies)
			blk_queue_io_opt(mddev->queue, chunk_size * conf->geo.raid_disks);
		else
			blk_queue_io_opt(mddev->queue, chunk_size *
					 (conf->geo.raid_disks / conf->geo.near_copies));
	}
3618

N
NeilBrown 已提交
3619
	rdev_for_each(rdev, mddev) {
N
NeilBrown 已提交
3620
		long long diff;
3621
		struct request_queue *q;
3622

L
Linus Torvalds 已提交
3623
		disk_idx = rdev->raid_disk;
3624 3625 3626 3627
		if (disk_idx < 0)
			continue;
		if (disk_idx >= conf->geo.raid_disks &&
		    disk_idx >= conf->prev.raid_disks)
L
Linus Torvalds 已提交
3628 3629 3630
			continue;
		disk = conf->mirrors + disk_idx;

3631 3632 3633 3634 3635 3636 3637 3638 3639
		if (test_bit(Replacement, &rdev->flags)) {
			if (disk->replacement)
				goto out_free_conf;
			disk->replacement = rdev;
		} else {
			if (disk->rdev)
				goto out_free_conf;
			disk->rdev = rdev;
		}
3640 3641 3642
		q = bdev_get_queue(rdev->bdev);
		if (q->merge_bvec_fn)
			mddev->merge_check_needed = 1;
N
NeilBrown 已提交
3643 3644 3645 3646 3647 3648 3649
		diff = (rdev->new_data_offset - rdev->data_offset);
		if (!mddev->reshape_backwards)
			diff = -diff;
		if (diff < 0)
			diff = 0;
		if (first || diff < min_offset_diff)
			min_offset_diff = diff;
3650

3651 3652 3653
		if (mddev->gendisk)
			disk_stack_limits(mddev->gendisk, rdev->bdev,
					  rdev->data_offset << 9);
L
Linus Torvalds 已提交
3654 3655

		disk->head_position = 0;
S
Shaohua Li 已提交
3656 3657 3658

		if (blk_queue_discard(bdev_get_queue(rdev->bdev)))
			discard_supported = true;
L
Linus Torvalds 已提交
3659
	}
N
NeilBrown 已提交
3660

3661 3662 3663 3664 3665 3666 3667 3668
	if (mddev->queue) {
		if (discard_supported)
			queue_flag_set_unlocked(QUEUE_FLAG_DISCARD,
						mddev->queue);
		else
			queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD,
						  mddev->queue);
	}
3669
	/* need to check that every block has at least one working mirror */
3670
	if (!enough(conf, -1)) {
N
NeilBrown 已提交
3671
		printk(KERN_ERR "md/raid10:%s: not enough operational mirrors.\n",
3672
		       mdname(mddev));
L
Linus Torvalds 已提交
3673 3674 3675
		goto out_free_conf;
	}

N
NeilBrown 已提交
3676 3677 3678 3679 3680 3681
	if (conf->reshape_progress != MaxSector) {
		/* must ensure that shape change is supported */
		if (conf->geo.far_copies != 1 &&
		    conf->geo.far_offset == 0)
			goto out_free_conf;
		if (conf->prev.far_copies != 1 &&
3682
		    conf->prev.far_offset == 0)
N
NeilBrown 已提交
3683 3684 3685
			goto out_free_conf;
	}

L
Linus Torvalds 已提交
3686
	mddev->degraded = 0;
3687 3688 3689 3690
	for (i = 0;
	     i < conf->geo.raid_disks
		     || i < conf->prev.raid_disks;
	     i++) {
L
Linus Torvalds 已提交
3691 3692 3693

		disk = conf->mirrors + i;

3694 3695 3696 3697 3698 3699 3700
		if (!disk->rdev && disk->replacement) {
			/* The replacement is all we have - use it */
			disk->rdev = disk->replacement;
			disk->replacement = NULL;
			clear_bit(Replacement, &disk->rdev->flags);
		}

3701
		if (!disk->rdev ||
3702
		    !test_bit(In_sync, &disk->rdev->flags)) {
L
Linus Torvalds 已提交
3703 3704
			disk->head_position = 0;
			mddev->degraded++;
3705 3706
			if (disk->rdev &&
			    disk->rdev->saved_raid_disk < 0)
3707
				conf->fullsync = 1;
L
Linus Torvalds 已提交
3708
		}
3709
		disk->recovery_disabled = mddev->recovery_disabled - 1;
L
Linus Torvalds 已提交
3710 3711
	}

3712
	if (mddev->recovery_cp != MaxSector)
N
NeilBrown 已提交
3713
		printk(KERN_NOTICE "md/raid10:%s: not clean"
3714 3715
		       " -- starting background reconstruction\n",
		       mdname(mddev));
L
Linus Torvalds 已提交
3716
	printk(KERN_INFO
N
NeilBrown 已提交
3717
		"md/raid10:%s: active with %d out of %d devices\n",
3718 3719
		mdname(mddev), conf->geo.raid_disks - mddev->degraded,
		conf->geo.raid_disks);
L
Linus Torvalds 已提交
3720 3721 3722
	/*
	 * Ok, everything is just fine now
	 */
3723 3724 3725 3726
	mddev->dev_sectors = conf->dev_sectors;
	size = raid10_size(mddev, 0, 0);
	md_set_array_sectors(mddev, size);
	mddev->resync_max_sectors = size;
L
Linus Torvalds 已提交
3727

3728
	if (mddev->queue) {
3729
		int stripe = conf->geo.raid_disks *
3730
			((mddev->chunk_sectors << 9) / PAGE_SIZE);
3731 3732 3733 3734 3735

		/* Calculate max read-ahead size.
		 * We need to readahead at least twice a whole stripe....
		 * maybe...
		 */
3736
		stripe /= conf->geo.near_copies;
N
NeilBrown 已提交
3737 3738
		if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
			mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
L
Linus Torvalds 已提交
3739 3740
	}

3741 3742 3743
	if (md_integrity_register(mddev))
		goto out_free_conf;

N
NeilBrown 已提交
3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767
	if (conf->reshape_progress != MaxSector) {
		unsigned long before_length, after_length;

		before_length = ((1 << conf->prev.chunk_shift) *
				 conf->prev.far_copies);
		after_length = ((1 << conf->geo.chunk_shift) *
				conf->geo.far_copies);

		if (max(before_length, after_length) > min_offset_diff) {
			/* This cannot work */
			printk("md/raid10: offset difference not enough to continue reshape\n");
			goto out_free_conf;
		}
		conf->offset_diff = min_offset_diff;

		conf->reshape_safe = conf->reshape_progress;
		clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
		clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
		set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
		set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
		mddev->sync_thread = md_register_thread(md_do_sync, mddev,
							"reshape");
	}

L
Linus Torvalds 已提交
3768 3769 3770
	return 0;

out_free_conf:
3771
	md_unregister_thread(&mddev->thread);
L
Linus Torvalds 已提交
3772 3773
	if (conf->r10bio_pool)
		mempool_destroy(conf->r10bio_pool);
3774
	safe_put_page(conf->tmppage);
3775
	kfree(conf->mirrors);
L
Linus Torvalds 已提交
3776 3777 3778 3779 3780 3781
	kfree(conf);
	mddev->private = NULL;
out:
	return -EIO;
}

N
NeilBrown 已提交
3782
static void raid10_free(struct mddev *mddev, void *priv)
L
Linus Torvalds 已提交
3783
{
N
NeilBrown 已提交
3784
	struct r10conf *conf = priv;
L
Linus Torvalds 已提交
3785 3786 3787

	if (conf->r10bio_pool)
		mempool_destroy(conf->r10bio_pool);
3788
	safe_put_page(conf->tmppage);
3789
	kfree(conf->mirrors);
3790 3791
	kfree(conf->mirrors_old);
	kfree(conf->mirrors_new);
L
Linus Torvalds 已提交
3792 3793 3794
	kfree(conf);
}

3795
static void raid10_quiesce(struct mddev *mddev, int state)
3796
{
3797
	struct r10conf *conf = mddev->private;
3798 3799 3800 3801 3802 3803 3804 3805 3806 3807

	switch(state) {
	case 1:
		raise_barrier(conf, 0);
		break;
	case 0:
		lower_barrier(conf);
		break;
	}
}
L
Linus Torvalds 已提交
3808

3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825
static int raid10_resize(struct mddev *mddev, sector_t sectors)
{
	/* Resize of 'far' arrays is not supported.
	 * For 'near' and 'offset' arrays we can set the
	 * number of sectors used to be an appropriate multiple
	 * of the chunk size.
	 * For 'offset', this is far_copies*chunksize.
	 * For 'near' the multiplier is the LCM of
	 * near_copies and raid_disks.
	 * So if far_copies > 1 && !far_offset, fail.
	 * Else find LCM(raid_disks, near_copy)*far_copies and
	 * multiply by chunk_size.  Then round to this number.
	 * This is mostly done by raid10_size()
	 */
	struct r10conf *conf = mddev->private;
	sector_t oldsize, size;

3826 3827 3828
	if (mddev->reshape_position != MaxSector)
		return -EBUSY;

3829
	if (conf->geo.far_copies > 1 && !conf->geo.far_offset)
3830 3831 3832 3833
		return -EINVAL;

	oldsize = raid10_size(mddev, 0, 0);
	size = raid10_size(mddev, sectors, 0);
3834 3835
	if (mddev->external_size &&
	    mddev->array_sectors > size)
3836
		return -EINVAL;
3837 3838 3839 3840 3841 3842
	if (mddev->bitmap) {
		int ret = bitmap_resize(mddev->bitmap, size, 0, 0);
		if (ret)
			return ret;
	}
	md_set_array_sectors(mddev, size);
3843 3844 3845 3846 3847 3848 3849
	set_capacity(mddev->gendisk, mddev->array_sectors);
	revalidate_disk(mddev->gendisk);
	if (sectors > mddev->dev_sectors &&
	    mddev->recovery_cp > oldsize) {
		mddev->recovery_cp = oldsize;
		set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
	}
3850 3851
	calc_sectors(conf, sectors);
	mddev->dev_sectors = conf->dev_sectors;
3852 3853 3854 3855
	mddev->resync_max_sectors = size;
	return 0;
}

3856
static void *raid10_takeover_raid0(struct mddev *mddev, sector_t size, int devs)
3857
{
3858
	struct md_rdev *rdev;
3859
	struct r10conf *conf;
3860 3861

	if (mddev->degraded > 0) {
N
NeilBrown 已提交
3862 3863
		printk(KERN_ERR "md/raid10:%s: Error: degraded raid0!\n",
		       mdname(mddev));
3864 3865
		return ERR_PTR(-EINVAL);
	}
3866
	sector_div(size, devs);
3867 3868 3869 3870 3871 3872 3873 3874 3875 3876

	/* Set new parameters */
	mddev->new_level = 10;
	/* new layout: far_copies = 1, near_copies = 2 */
	mddev->new_layout = (1<<8) + 2;
	mddev->new_chunk_sectors = mddev->chunk_sectors;
	mddev->delta_disks = mddev->raid_disks;
	mddev->raid_disks *= 2;
	/* make sure it will be not marked as dirty */
	mddev->recovery_cp = MaxSector;
3877
	mddev->dev_sectors = size;
3878 3879

	conf = setup_conf(mddev);
3880
	if (!IS_ERR(conf)) {
N
NeilBrown 已提交
3881
		rdev_for_each(rdev, mddev)
3882
			if (rdev->raid_disk >= 0) {
3883
				rdev->new_raid_disk = rdev->raid_disk * 2;
3884 3885
				rdev->sectors = size;
			}
3886 3887 3888
		conf->barrier = 1;
	}

3889 3890 3891
	return conf;
}

3892
static void *raid10_takeover(struct mddev *mddev)
3893
{
3894
	struct r0conf *raid0_conf;
3895 3896 3897 3898 3899 3900

	/* raid10 can take over:
	 *  raid0 - providing it has only two drives
	 */
	if (mddev->level == 0) {
		/* for raid0 takeover only one zone is supported */
3901 3902
		raid0_conf = mddev->private;
		if (raid0_conf->nr_strip_zones > 1) {
N
NeilBrown 已提交
3903 3904 3905
			printk(KERN_ERR "md/raid10:%s: cannot takeover raid 0"
			       " with more than one zone.\n",
			       mdname(mddev));
3906 3907
			return ERR_PTR(-EINVAL);
		}
3908 3909 3910
		return raid10_takeover_raid0(mddev,
			raid0_conf->strip_zone->zone_end,
			raid0_conf->strip_zone->nb_dev);
3911 3912 3913 3914
	}
	return ERR_PTR(-EINVAL);
}

N
NeilBrown 已提交
3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926 3927 3928 3929 3930 3931 3932 3933 3934 3935 3936 3937 3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955
static int raid10_check_reshape(struct mddev *mddev)
{
	/* Called when there is a request to change
	 * - layout (to ->new_layout)
	 * - chunk size (to ->new_chunk_sectors)
	 * - raid_disks (by delta_disks)
	 * or when trying to restart a reshape that was ongoing.
	 *
	 * We need to validate the request and possibly allocate
	 * space if that might be an issue later.
	 *
	 * Currently we reject any reshape of a 'far' mode array,
	 * allow chunk size to change if new is generally acceptable,
	 * allow raid_disks to increase, and allow
	 * a switch between 'near' mode and 'offset' mode.
	 */
	struct r10conf *conf = mddev->private;
	struct geom geo;

	if (conf->geo.far_copies != 1 && !conf->geo.far_offset)
		return -EINVAL;

	if (setup_geo(&geo, mddev, geo_start) != conf->copies)
		/* mustn't change number of copies */
		return -EINVAL;
	if (geo.far_copies > 1 && !geo.far_offset)
		/* Cannot switch to 'far' mode */
		return -EINVAL;

	if (mddev->array_sectors & geo.chunk_mask)
			/* not factor of array size */
			return -EINVAL;

	if (!enough(conf, -1))
		return -EINVAL;

	kfree(conf->mirrors_new);
	conf->mirrors_new = NULL;
	if (mddev->delta_disks > 0) {
		/* allocate new 'mirrors' list */
		conf->mirrors_new = kzalloc(
3956
			sizeof(struct raid10_info)
N
NeilBrown 已提交
3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996 3997 3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019 4020 4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041
			*(mddev->raid_disks +
			  mddev->delta_disks),
			GFP_KERNEL);
		if (!conf->mirrors_new)
			return -ENOMEM;
	}
	return 0;
}

/*
 * Need to check if array has failed when deciding whether to:
 *  - start an array
 *  - remove non-faulty devices
 *  - add a spare
 *  - allow a reshape
 * This determination is simple when no reshape is happening.
 * However if there is a reshape, we need to carefully check
 * both the before and after sections.
 * This is because some failed devices may only affect one
 * of the two sections, and some non-in_sync devices may
 * be insync in the section most affected by failed devices.
 */
static int calc_degraded(struct r10conf *conf)
{
	int degraded, degraded2;
	int i;

	rcu_read_lock();
	degraded = 0;
	/* 'prev' section first */
	for (i = 0; i < conf->prev.raid_disks; i++) {
		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
		if (!rdev || test_bit(Faulty, &rdev->flags))
			degraded++;
		else if (!test_bit(In_sync, &rdev->flags))
			/* When we can reduce the number of devices in
			 * an array, this might not contribute to
			 * 'degraded'.  It does now.
			 */
			degraded++;
	}
	rcu_read_unlock();
	if (conf->geo.raid_disks == conf->prev.raid_disks)
		return degraded;
	rcu_read_lock();
	degraded2 = 0;
	for (i = 0; i < conf->geo.raid_disks; i++) {
		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
		if (!rdev || test_bit(Faulty, &rdev->flags))
			degraded2++;
		else if (!test_bit(In_sync, &rdev->flags)) {
			/* If reshape is increasing the number of devices,
			 * this section has already been recovered, so
			 * it doesn't contribute to degraded.
			 * else it does.
			 */
			if (conf->geo.raid_disks <= conf->prev.raid_disks)
				degraded2++;
		}
	}
	rcu_read_unlock();
	if (degraded2 > degraded)
		return degraded2;
	return degraded;
}

static int raid10_start_reshape(struct mddev *mddev)
{
	/* A 'reshape' has been requested. This commits
	 * the various 'new' fields and sets MD_RECOVER_RESHAPE
	 * This also checks if there are enough spares and adds them
	 * to the array.
	 * We currently require enough spares to make the final
	 * array non-degraded.  We also require that the difference
	 * between old and new data_offset - on each device - is
	 * enough that we never risk over-writing.
	 */

	unsigned long before_length, after_length;
	sector_t min_offset_diff = 0;
	int first = 1;
	struct geom new;
	struct r10conf *conf = mddev->private;
	struct md_rdev *rdev;
	int spares = 0;
4042
	int ret;
N
NeilBrown 已提交
4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074 4075 4076 4077 4078 4079 4080

	if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
		return -EBUSY;

	if (setup_geo(&new, mddev, geo_start) != conf->copies)
		return -EINVAL;

	before_length = ((1 << conf->prev.chunk_shift) *
			 conf->prev.far_copies);
	after_length = ((1 << conf->geo.chunk_shift) *
			conf->geo.far_copies);

	rdev_for_each(rdev, mddev) {
		if (!test_bit(In_sync, &rdev->flags)
		    && !test_bit(Faulty, &rdev->flags))
			spares++;
		if (rdev->raid_disk >= 0) {
			long long diff = (rdev->new_data_offset
					  - rdev->data_offset);
			if (!mddev->reshape_backwards)
				diff = -diff;
			if (diff < 0)
				diff = 0;
			if (first || diff < min_offset_diff)
				min_offset_diff = diff;
		}
	}

	if (max(before_length, after_length) > min_offset_diff)
		return -EINVAL;

	if (spares < mddev->delta_disks)
		return -EINVAL;

	conf->offset_diff = min_offset_diff;
	spin_lock_irq(&conf->device_lock);
	if (conf->mirrors_new) {
		memcpy(conf->mirrors_new, conf->mirrors,
4081
		       sizeof(struct raid10_info)*conf->prev.raid_disks);
N
NeilBrown 已提交
4082
		smp_mb();
4083
		kfree(conf->mirrors_old);
N
NeilBrown 已提交
4084 4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103
		conf->mirrors_old = conf->mirrors;
		conf->mirrors = conf->mirrors_new;
		conf->mirrors_new = NULL;
	}
	setup_geo(&conf->geo, mddev, geo_start);
	smp_mb();
	if (mddev->reshape_backwards) {
		sector_t size = raid10_size(mddev, 0, 0);
		if (size < mddev->array_sectors) {
			spin_unlock_irq(&conf->device_lock);
			printk(KERN_ERR "md/raid10:%s: array size must be reduce before number of disks\n",
			       mdname(mddev));
			return -EINVAL;
		}
		mddev->resync_max_sectors = size;
		conf->reshape_progress = size;
	} else
		conf->reshape_progress = 0;
	spin_unlock_irq(&conf->device_lock);

4104 4105 4106 4107 4108 4109 4110 4111
	if (mddev->delta_disks && mddev->bitmap) {
		ret = bitmap_resize(mddev->bitmap,
				    raid10_size(mddev, 0,
						conf->geo.raid_disks),
				    0, 0);
		if (ret)
			goto abort;
	}
N
NeilBrown 已提交
4112 4113 4114 4115 4116 4117 4118 4119 4120 4121 4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144
	if (mddev->delta_disks > 0) {
		rdev_for_each(rdev, mddev)
			if (rdev->raid_disk < 0 &&
			    !test_bit(Faulty, &rdev->flags)) {
				if (raid10_add_disk(mddev, rdev) == 0) {
					if (rdev->raid_disk >=
					    conf->prev.raid_disks)
						set_bit(In_sync, &rdev->flags);
					else
						rdev->recovery_offset = 0;

					if (sysfs_link_rdev(mddev, rdev))
						/* Failure here  is OK */;
				}
			} else if (rdev->raid_disk >= conf->prev.raid_disks
				   && !test_bit(Faulty, &rdev->flags)) {
				/* This is a spare that was manually added */
				set_bit(In_sync, &rdev->flags);
			}
	}
	/* When a reshape changes the number of devices,
	 * ->degraded is measured against the larger of the
	 * pre and  post numbers.
	 */
	spin_lock_irq(&conf->device_lock);
	mddev->degraded = calc_degraded(conf);
	spin_unlock_irq(&conf->device_lock);
	mddev->raid_disks = conf->geo.raid_disks;
	mddev->reshape_position = conf->reshape_progress;
	set_bit(MD_CHANGE_DEVS, &mddev->flags);

	clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
	clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
4145
	clear_bit(MD_RECOVERY_DONE, &mddev->recovery);
N
NeilBrown 已提交
4146 4147 4148 4149 4150 4151
	set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
	set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);

	mddev->sync_thread = md_register_thread(md_do_sync, mddev,
						"reshape");
	if (!mddev->sync_thread) {
4152 4153
		ret = -EAGAIN;
		goto abort;
N
NeilBrown 已提交
4154 4155 4156 4157 4158
	}
	conf->reshape_checkpoint = jiffies;
	md_wakeup_thread(mddev->sync_thread);
	md_new_event(mddev);
	return 0;
4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171

abort:
	mddev->recovery = 0;
	spin_lock_irq(&conf->device_lock);
	conf->geo = conf->prev;
	mddev->raid_disks = conf->geo.raid_disks;
	rdev_for_each(rdev, mddev)
		rdev->new_data_offset = rdev->data_offset;
	smp_wmb();
	conf->reshape_progress = MaxSector;
	mddev->reshape_position = MaxSector;
	spin_unlock_irq(&conf->device_lock);
	return ret;
N
NeilBrown 已提交
4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247 4248 4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267 4268 4269 4270 4271 4272 4273 4274 4275 4276 4277 4278 4279 4280 4281 4282 4283 4284 4285 4286 4287 4288 4289 4290 4291 4292 4293 4294 4295 4296 4297 4298 4299 4300 4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316 4317 4318 4319 4320 4321 4322 4323 4324 4325 4326 4327 4328 4329 4330 4331 4332 4333 4334 4335 4336 4337
}

/* Calculate the last device-address that could contain
 * any block from the chunk that includes the array-address 's'
 * and report the next address.
 * i.e. the address returned will be chunk-aligned and after
 * any data that is in the chunk containing 's'.
 */
static sector_t last_dev_address(sector_t s, struct geom *geo)
{
	s = (s | geo->chunk_mask) + 1;
	s >>= geo->chunk_shift;
	s *= geo->near_copies;
	s = DIV_ROUND_UP_SECTOR_T(s, geo->raid_disks);
	s *= geo->far_copies;
	s <<= geo->chunk_shift;
	return s;
}

/* Calculate the first device-address that could contain
 * any block from the chunk that includes the array-address 's'.
 * This too will be the start of a chunk
 */
static sector_t first_dev_address(sector_t s, struct geom *geo)
{
	s >>= geo->chunk_shift;
	s *= geo->near_copies;
	sector_div(s, geo->raid_disks);
	s *= geo->far_copies;
	s <<= geo->chunk_shift;
	return s;
}

static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr,
				int *skipped)
{
	/* We simply copy at most one chunk (smallest of old and new)
	 * at a time, possibly less if that exceeds RESYNC_PAGES,
	 * or we hit a bad block or something.
	 * This might mean we pause for normal IO in the middle of
	 * a chunk, but that is not a problem was mddev->reshape_position
	 * can record any location.
	 *
	 * If we will want to write to a location that isn't
	 * yet recorded as 'safe' (i.e. in metadata on disk) then
	 * we need to flush all reshape requests and update the metadata.
	 *
	 * When reshaping forwards (e.g. to more devices), we interpret
	 * 'safe' as the earliest block which might not have been copied
	 * down yet.  We divide this by previous stripe size and multiply
	 * by previous stripe length to get lowest device offset that we
	 * cannot write to yet.
	 * We interpret 'sector_nr' as an address that we want to write to.
	 * From this we use last_device_address() to find where we might
	 * write to, and first_device_address on the  'safe' position.
	 * If this 'next' write position is after the 'safe' position,
	 * we must update the metadata to increase the 'safe' position.
	 *
	 * When reshaping backwards, we round in the opposite direction
	 * and perform the reverse test:  next write position must not be
	 * less than current safe position.
	 *
	 * In all this the minimum difference in data offsets
	 * (conf->offset_diff - always positive) allows a bit of slack,
	 * so next can be after 'safe', but not by more than offset_disk
	 *
	 * We need to prepare all the bios here before we start any IO
	 * to ensure the size we choose is acceptable to all devices.
	 * The means one for each copy for write-out and an extra one for
	 * read-in.
	 * We store the read-in bio in ->master_bio and the others in
	 * ->devs[x].bio and ->devs[x].repl_bio.
	 */
	struct r10conf *conf = mddev->private;
	struct r10bio *r10_bio;
	sector_t next, safe, last;
	int max_sectors;
	int nr_sectors;
	int s;
	struct md_rdev *rdev;
	int need_flush = 0;
	struct bio *blist;
	struct bio *bio, *read_bio;
	int sectors_done = 0;

	if (sector_nr == 0) {
		/* If restarting in the middle, skip the initial sectors */
		if (mddev->reshape_backwards &&
		    conf->reshape_progress < raid10_size(mddev, 0, 0)) {
			sector_nr = (raid10_size(mddev, 0, 0)
				     - conf->reshape_progress);
		} else if (!mddev->reshape_backwards &&
			   conf->reshape_progress > 0)
			sector_nr = conf->reshape_progress;
		if (sector_nr) {
			mddev->curr_resync_completed = sector_nr;
			sysfs_notify(&mddev->kobj, NULL, "sync_completed");
			*skipped = 1;
			return sector_nr;
		}
	}

	/* We don't use sector_nr to track where we are up to
	 * as that doesn't work well for ->reshape_backwards.
	 * So just use ->reshape_progress.
	 */
	if (mddev->reshape_backwards) {
		/* 'next' is the earliest device address that we might
		 * write to for this chunk in the new layout
		 */
		next = first_dev_address(conf->reshape_progress - 1,
					 &conf->geo);

		/* 'safe' is the last device address that we might read from
		 * in the old layout after a restart
		 */
		safe = last_dev_address(conf->reshape_safe - 1,
					&conf->prev);

		if (next + conf->offset_diff < safe)
			need_flush = 1;

		last = conf->reshape_progress - 1;
		sector_nr = last & ~(sector_t)(conf->geo.chunk_mask
					       & conf->prev.chunk_mask);
		if (sector_nr + RESYNC_BLOCK_SIZE/512 < last)
			sector_nr = last + 1 - RESYNC_BLOCK_SIZE/512;
	} else {
		/* 'next' is after the last device address that we
		 * might write to for this chunk in the new layout
		 */
		next = last_dev_address(conf->reshape_progress, &conf->geo);

		/* 'safe' is the earliest device address that we might
		 * read from in the old layout after a restart
		 */
		safe = first_dev_address(conf->reshape_safe, &conf->prev);

		/* Need to update metadata if 'next' might be beyond 'safe'
		 * as that would possibly corrupt data
		 */
		if (next > safe + conf->offset_diff)
			need_flush = 1;

		sector_nr = conf->reshape_progress;
		last  = sector_nr | (conf->geo.chunk_mask
				     & conf->prev.chunk_mask);

		if (sector_nr + RESYNC_BLOCK_SIZE/512 <= last)
			last = sector_nr + RESYNC_BLOCK_SIZE/512 - 1;
	}

	if (need_flush ||
	    time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
		/* Need to update reshape_position in metadata */
		wait_barrier(conf);
		mddev->reshape_position = conf->reshape_progress;
		if (mddev->reshape_backwards)
			mddev->curr_resync_completed = raid10_size(mddev, 0, 0)
				- conf->reshape_progress;
		else
			mddev->curr_resync_completed = conf->reshape_progress;
		conf->reshape_checkpoint = jiffies;
		set_bit(MD_CHANGE_DEVS, &mddev->flags);
		md_wakeup_thread(mddev->thread);
		wait_event(mddev->sb_wait, mddev->flags == 0 ||
4338 4339 4340 4341 4342
			   test_bit(MD_RECOVERY_INTR, &mddev->recovery));
		if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
			allow_barrier(conf);
			return sectors_done;
		}
N
NeilBrown 已提交
4343 4344 4345 4346 4347 4348 4349
		conf->reshape_safe = mddev->reshape_position;
		allow_barrier(conf);
	}

read_more:
	/* Now schedule reads for blocks from sector_nr to last */
	r10_bio = mempool_alloc(conf->r10buf_pool, GFP_NOIO);
4350
	r10_bio->state = 0;
N
NeilBrown 已提交
4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364
	raise_barrier(conf, sectors_done != 0);
	atomic_set(&r10_bio->remaining, 0);
	r10_bio->mddev = mddev;
	r10_bio->sector = sector_nr;
	set_bit(R10BIO_IsReshape, &r10_bio->state);
	r10_bio->sectors = last - sector_nr + 1;
	rdev = read_balance(conf, r10_bio, &max_sectors);
	BUG_ON(!test_bit(R10BIO_Previous, &r10_bio->state));

	if (!rdev) {
		/* Cannot read from here, so need to record bad blocks
		 * on all the target devices.
		 */
		// FIXME
4365
		mempool_free(r10_bio, conf->r10buf_pool);
N
NeilBrown 已提交
4366 4367 4368 4369 4370 4371 4372
		set_bit(MD_RECOVERY_INTR, &mddev->recovery);
		return sectors_done;
	}

	read_bio = bio_alloc_mddev(GFP_KERNEL, RESYNC_PAGES, mddev);

	read_bio->bi_bdev = rdev->bdev;
4373
	read_bio->bi_iter.bi_sector = (r10_bio->devs[r10_bio->read_slot].addr
N
NeilBrown 已提交
4374 4375 4376 4377
			       + rdev->data_offset);
	read_bio->bi_private = r10_bio;
	read_bio->bi_end_io = end_sync_read;
	read_bio->bi_rw = READ;
4378
	read_bio->bi_flags &= (~0UL << BIO_RESET_BITS);
4379
	read_bio->bi_error = 0;
N
NeilBrown 已提交
4380
	read_bio->bi_vcnt = 0;
4381
	read_bio->bi_iter.bi_size = 0;
N
NeilBrown 已提交
4382 4383 4384 4385 4386 4387 4388 4389 4390 4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401 4402 4403
	r10_bio->master_bio = read_bio;
	r10_bio->read_slot = r10_bio->devs[r10_bio->read_slot].devnum;

	/* Now find the locations in the new layout */
	__raid10_find_phys(&conf->geo, r10_bio);

	blist = read_bio;
	read_bio->bi_next = NULL;

	for (s = 0; s < conf->copies*2; s++) {
		struct bio *b;
		int d = r10_bio->devs[s/2].devnum;
		struct md_rdev *rdev2;
		if (s&1) {
			rdev2 = conf->mirrors[d].replacement;
			b = r10_bio->devs[s/2].repl_bio;
		} else {
			rdev2 = conf->mirrors[d].rdev;
			b = r10_bio->devs[s/2].bio;
		}
		if (!rdev2 || test_bit(Faulty, &rdev2->flags))
			continue;
K
Kent Overstreet 已提交
4404 4405

		bio_reset(b);
N
NeilBrown 已提交
4406
		b->bi_bdev = rdev2->bdev;
4407 4408
		b->bi_iter.bi_sector = r10_bio->devs[s/2].addr +
			rdev2->new_data_offset;
N
NeilBrown 已提交
4409 4410 4411 4412 4413 4414 4415 4416 4417 4418 4419 4420 4421 4422 4423 4424 4425 4426 4427 4428 4429 4430 4431 4432 4433 4434
		b->bi_private = r10_bio;
		b->bi_end_io = end_reshape_write;
		b->bi_rw = WRITE;
		b->bi_next = blist;
		blist = b;
	}

	/* Now add as many pages as possible to all of these bios. */

	nr_sectors = 0;
	for (s = 0 ; s < max_sectors; s += PAGE_SIZE >> 9) {
		struct page *page = r10_bio->devs[0].bio->bi_io_vec[s/(PAGE_SIZE>>9)].bv_page;
		int len = (max_sectors - s) << 9;
		if (len > PAGE_SIZE)
			len = PAGE_SIZE;
		for (bio = blist; bio ; bio = bio->bi_next) {
			struct bio *bio2;
			if (bio_add_page(bio, page, len, 0))
				continue;

			/* Didn't fit, must stop */
			for (bio2 = blist;
			     bio2 && bio2 != bio;
			     bio2 = bio2->bi_next) {
				/* Remove last page from this bio */
				bio2->bi_vcnt--;
4435
				bio2->bi_iter.bi_size -= len;
4436
				bio_clear_flag(bio2, BIO_SEG_VALID);
N
NeilBrown 已提交
4437 4438 4439 4440 4441 4442 4443 4444 4445 4446 4447 4448 4449 4450 4451 4452 4453 4454 4455 4456 4457 4458 4459 4460 4461 4462 4463 4464 4465 4466 4467 4468 4469 4470 4471 4472 4473 4474 4475 4476 4477 4478 4479 4480 4481 4482 4483 4484 4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520 4521 4522 4523 4524 4525 4526 4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543
			}
			goto bio_full;
		}
		sector_nr += len >> 9;
		nr_sectors += len >> 9;
	}
bio_full:
	r10_bio->sectors = nr_sectors;

	/* Now submit the read */
	md_sync_acct(read_bio->bi_bdev, r10_bio->sectors);
	atomic_inc(&r10_bio->remaining);
	read_bio->bi_next = NULL;
	generic_make_request(read_bio);
	sector_nr += nr_sectors;
	sectors_done += nr_sectors;
	if (sector_nr <= last)
		goto read_more;

	/* Now that we have done the whole section we can
	 * update reshape_progress
	 */
	if (mddev->reshape_backwards)
		conf->reshape_progress -= sectors_done;
	else
		conf->reshape_progress += sectors_done;

	return sectors_done;
}

static void end_reshape_request(struct r10bio *r10_bio);
static int handle_reshape_read_error(struct mddev *mddev,
				     struct r10bio *r10_bio);
static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio)
{
	/* Reshape read completed.  Hopefully we have a block
	 * to write out.
	 * If we got a read error then we do sync 1-page reads from
	 * elsewhere until we find the data - or give up.
	 */
	struct r10conf *conf = mddev->private;
	int s;

	if (!test_bit(R10BIO_Uptodate, &r10_bio->state))
		if (handle_reshape_read_error(mddev, r10_bio) < 0) {
			/* Reshape has been aborted */
			md_done_sync(mddev, r10_bio->sectors, 0);
			return;
		}

	/* We definitely have the data in the pages, schedule the
	 * writes.
	 */
	atomic_set(&r10_bio->remaining, 1);
	for (s = 0; s < conf->copies*2; s++) {
		struct bio *b;
		int d = r10_bio->devs[s/2].devnum;
		struct md_rdev *rdev;
		if (s&1) {
			rdev = conf->mirrors[d].replacement;
			b = r10_bio->devs[s/2].repl_bio;
		} else {
			rdev = conf->mirrors[d].rdev;
			b = r10_bio->devs[s/2].bio;
		}
		if (!rdev || test_bit(Faulty, &rdev->flags))
			continue;
		atomic_inc(&rdev->nr_pending);
		md_sync_acct(b->bi_bdev, r10_bio->sectors);
		atomic_inc(&r10_bio->remaining);
		b->bi_next = NULL;
		generic_make_request(b);
	}
	end_reshape_request(r10_bio);
}

static void end_reshape(struct r10conf *conf)
{
	if (test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery))
		return;

	spin_lock_irq(&conf->device_lock);
	conf->prev = conf->geo;
	md_finish_reshape(conf->mddev);
	smp_wmb();
	conf->reshape_progress = MaxSector;
	spin_unlock_irq(&conf->device_lock);

	/* read-ahead size must cover two whole stripes, which is
	 * 2 * (datadisks) * chunksize where 'n' is the number of raid devices
	 */
	if (conf->mddev->queue) {
		int stripe = conf->geo.raid_disks *
			((conf->mddev->chunk_sectors << 9) / PAGE_SIZE);
		stripe /= conf->geo.near_copies;
		if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe)
			conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe;
	}
	conf->fullsync = 0;
}

static int handle_reshape_read_error(struct mddev *mddev,
				     struct r10bio *r10_bio)
{
	/* Use sync reads to get the blocks from somewhere else */
	int sectors = r10_bio->sectors;
	struct r10conf *conf = mddev->private;
4544 4545 4546 4547 4548
	struct {
		struct r10bio r10_bio;
		struct r10dev devs[conf->copies];
	} on_stack;
	struct r10bio *r10b = &on_stack.r10_bio;
N
NeilBrown 已提交
4549 4550 4551 4552
	int slot = 0;
	int idx = 0;
	struct bio_vec *bvec = r10_bio->master_bio->bi_io_vec;

4553 4554
	r10b->sector = r10_bio->sector;
	__raid10_find_phys(&conf->prev, r10b);
N
NeilBrown 已提交
4555 4556 4557 4558 4559 4560 4561 4562 4563 4564

	while (sectors) {
		int s = sectors;
		int success = 0;
		int first_slot = slot;

		if (s > (PAGE_SIZE >> 9))
			s = PAGE_SIZE >> 9;

		while (!success) {
4565
			int d = r10b->devs[slot].devnum;
N
NeilBrown 已提交
4566 4567 4568 4569 4570 4571 4572
			struct md_rdev *rdev = conf->mirrors[d].rdev;
			sector_t addr;
			if (rdev == NULL ||
			    test_bit(Faulty, &rdev->flags) ||
			    !test_bit(In_sync, &rdev->flags))
				goto failed;

4573
			addr = r10b->devs[slot].addr + idx * PAGE_SIZE;
N
NeilBrown 已提交
4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597 4598 4599
			success = sync_page_io(rdev,
					       addr,
					       s << 9,
					       bvec[idx].bv_page,
					       READ, false);
			if (success)
				break;
		failed:
			slot++;
			if (slot >= conf->copies)
				slot = 0;
			if (slot == first_slot)
				break;
		}
		if (!success) {
			/* couldn't read this block, must give up */
			set_bit(MD_RECOVERY_INTR,
				&mddev->recovery);
			return -EIO;
		}
		sectors -= s;
		idx++;
	}
	return 0;
}

4600
static void end_reshape_write(struct bio *bio)
N
NeilBrown 已提交
4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612 4613 4614 4615 4616 4617
{
	struct r10bio *r10_bio = bio->bi_private;
	struct mddev *mddev = r10_bio->mddev;
	struct r10conf *conf = mddev->private;
	int d;
	int slot;
	int repl;
	struct md_rdev *rdev = NULL;

	d = find_bio_disk(conf, r10_bio, bio, &slot, &repl);
	if (repl)
		rdev = conf->mirrors[d].replacement;
	if (!rdev) {
		smp_mb();
		rdev = conf->mirrors[d].rdev;
	}

4618
	if (bio->bi_error) {
N
NeilBrown 已提交
4619 4620 4621 4622 4623 4624 4625 4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649 4650 4651 4652
		/* FIXME should record badblock */
		md_error(mddev, rdev);
	}

	rdev_dec_pending(rdev, mddev);
	end_reshape_request(r10_bio);
}

static void end_reshape_request(struct r10bio *r10_bio)
{
	if (!atomic_dec_and_test(&r10_bio->remaining))
		return;
	md_done_sync(r10_bio->mddev, r10_bio->sectors, 1);
	bio_put(r10_bio->master_bio);
	put_buf(r10_bio);
}

static void raid10_finish_reshape(struct mddev *mddev)
{
	struct r10conf *conf = mddev->private;

	if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
		return;

	if (mddev->delta_disks > 0) {
		sector_t size = raid10_size(mddev, 0, 0);
		md_set_array_sectors(mddev, size);
		if (mddev->recovery_cp > mddev->resync_max_sectors) {
			mddev->recovery_cp = mddev->resync_max_sectors;
			set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
		}
		mddev->resync_max_sectors = size;
		set_capacity(mddev->gendisk, mddev->array_sectors);
		revalidate_disk(mddev->gendisk);
4653 4654 4655 4656 4657 4658 4659 4660 4661 4662 4663 4664
	} else {
		int d;
		for (d = conf->geo.raid_disks ;
		     d < conf->geo.raid_disks - mddev->delta_disks;
		     d++) {
			struct md_rdev *rdev = conf->mirrors[d].rdev;
			if (rdev)
				clear_bit(In_sync, &rdev->flags);
			rdev = conf->mirrors[d].replacement;
			if (rdev)
				clear_bit(In_sync, &rdev->flags);
		}
N
NeilBrown 已提交
4665 4666 4667 4668 4669 4670 4671 4672
	}
	mddev->layout = mddev->new_layout;
	mddev->chunk_sectors = 1 << conf->geo.chunk_shift;
	mddev->reshape_position = MaxSector;
	mddev->delta_disks = 0;
	mddev->reshape_backwards = 0;
}

4673
static struct md_personality raid10_personality =
L
Linus Torvalds 已提交
4674 4675
{
	.name		= "raid10",
4676
	.level		= 10,
L
Linus Torvalds 已提交
4677 4678 4679
	.owner		= THIS_MODULE,
	.make_request	= make_request,
	.run		= run,
N
NeilBrown 已提交
4680
	.free		= raid10_free,
L
Linus Torvalds 已提交
4681 4682 4683 4684 4685 4686
	.status		= status,
	.error_handler	= error,
	.hot_add_disk	= raid10_add_disk,
	.hot_remove_disk= raid10_remove_disk,
	.spare_active	= raid10_spare_active,
	.sync_request	= sync_request,
4687
	.quiesce	= raid10_quiesce,
4688
	.size		= raid10_size,
4689
	.resize		= raid10_resize,
4690
	.takeover	= raid10_takeover,
N
NeilBrown 已提交
4691 4692 4693
	.check_reshape	= raid10_check_reshape,
	.start_reshape	= raid10_start_reshape,
	.finish_reshape	= raid10_finish_reshape,
4694
	.congested	= raid10_congested,
4695
	.mergeable_bvec	= raid10_mergeable_bvec,
L
Linus Torvalds 已提交
4696 4697 4698 4699
};

static int __init raid_init(void)
{
4700
	return register_md_personality(&raid10_personality);
L
Linus Torvalds 已提交
4701 4702 4703 4704
}

static void raid_exit(void)
{
4705
	unregister_md_personality(&raid10_personality);
L
Linus Torvalds 已提交
4706 4707 4708 4709 4710
}

module_init(raid_init);
module_exit(raid_exit);
MODULE_LICENSE("GPL");
4711
MODULE_DESCRIPTION("RAID10 (striped mirror) personality for MD");
L
Linus Torvalds 已提交
4712
MODULE_ALIAS("md-personality-9"); /* RAID10 */
4713
MODULE_ALIAS("md-raid10");
4714
MODULE_ALIAS("md-level-10");
4715 4716

module_param(max_queued_requests, int, S_IRUGO|S_IWUSR);