raid1.c 78.1 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5 6 7 8 9 10 11
/*
 * raid1.c : Multiple Devices driver for Linux
 *
 * Copyright (C) 1999, 2000, 2001 Ingo Molnar, Red Hat
 *
 * Copyright (C) 1996, 1997, 1998 Ingo Molnar, Miguel de Icaza, Gadi Oxman
 *
 * RAID-1 management functions.
 *
 * Better read-balancing code written by Mika Kuoppala <miku@iki.fi>, 2000
 *
12
 * Fixes to reconstruction by Jakob Østergaard" <jakob@ostenfeld.dk>
L
Linus Torvalds 已提交
13 14
 * Various fixes by Neil Brown <neilb@cse.unsw.edu.au>
 *
15 16 17 18 19 20 21 22 23
 * Changes by Peter T. Breuer <ptb@it.uc3m.es> 31/1/2003 to support
 * bitmapped intelligence in resync:
 *
 *      - bitmap marked during normal i/o
 *      - bitmap used to skip nondirty blocks during sync
 *
 * Additions to bitmap code, (C) 2003-2004 Paul Clements, SteelEye Technology:
 * - persistent bitmap code
 *
L
Linus Torvalds 已提交
24 25 26 27 28 29 30 31 32 33
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2, or (at your option)
 * any later version.
 *
 * You should have received a copy of the GNU General Public License
 * (for example /usr/src/linux/COPYING); if not, write to the Free
 * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

34
#include <linux/slab.h>
35
#include <linux/delay.h>
36
#include <linux/blkdev.h>
37
#include <linux/module.h>
38
#include <linux/seq_file.h>
39
#include <linux/ratelimit.h>
40
#include "md.h"
41 42
#include "raid1.h"
#include "bitmap.h"
43

L
Linus Torvalds 已提交
44 45 46 47 48
/*
 * Number of guaranteed r1bios in case of extreme VM load:
 */
#define	NR_RAID1_BIOS 256

49 50 51 52 53
/* When there are this many requests queue to be written by
 * the raid1 thread, we become 'congested' to provide back-pressure
 * for writeback.
 */
static int max_queued_requests = 1024;
L
Linus Torvalds 已提交
54

55 56
static void allow_barrier(struct r1conf *conf);
static void lower_barrier(struct r1conf *conf);
L
Linus Torvalds 已提交
57

A
Al Viro 已提交
58
static void * r1bio_pool_alloc(gfp_t gfp_flags, void *data)
L
Linus Torvalds 已提交
59 60
{
	struct pool_info *pi = data;
61
	int size = offsetof(struct r1bio, bios[pi->raid_disks]);
L
Linus Torvalds 已提交
62 63

	/* allocate a r1bio with room for raid_disks entries in the bios array */
J
Jens Axboe 已提交
64
	return kzalloc(size, gfp_flags);
L
Linus Torvalds 已提交
65 66 67 68 69 70 71 72 73 74 75 76 77
}

static void r1bio_pool_free(void *r1_bio, void *data)
{
	kfree(r1_bio);
}

#define RESYNC_BLOCK_SIZE (64*1024)
//#define RESYNC_BLOCK_SIZE PAGE_SIZE
#define RESYNC_SECTORS (RESYNC_BLOCK_SIZE >> 9)
#define RESYNC_PAGES ((RESYNC_BLOCK_SIZE + PAGE_SIZE-1) / PAGE_SIZE)
#define RESYNC_WINDOW (2048*1024)

A
Al Viro 已提交
78
static void * r1buf_pool_alloc(gfp_t gfp_flags, void *data)
L
Linus Torvalds 已提交
79 80 81
{
	struct pool_info *pi = data;
	struct page *page;
82
	struct r1bio *r1_bio;
L
Linus Torvalds 已提交
83 84 85 86
	struct bio *bio;
	int i, j;

	r1_bio = r1bio_pool_alloc(gfp_flags, pi);
J
Jens Axboe 已提交
87
	if (!r1_bio)
L
Linus Torvalds 已提交
88 89 90 91 92 93
		return NULL;

	/*
	 * Allocate bios : 1 for reading, n-1 for writing
	 */
	for (j = pi->raid_disks ; j-- ; ) {
94
		bio = bio_kmalloc(gfp_flags, RESYNC_PAGES);
L
Linus Torvalds 已提交
95 96 97 98 99 100
		if (!bio)
			goto out_free_bio;
		r1_bio->bios[j] = bio;
	}
	/*
	 * Allocate RESYNC_PAGES data pages and attach them to
101 102 103
	 * the first bio.
	 * If this is a user-requested check/repair, allocate
	 * RESYNC_PAGES for each bio.
L
Linus Torvalds 已提交
104
	 */
105 106 107 108 109 110 111 112 113 114 115 116
	if (test_bit(MD_RECOVERY_REQUESTED, &pi->mddev->recovery))
		j = pi->raid_disks;
	else
		j = 1;
	while(j--) {
		bio = r1_bio->bios[j];
		for (i = 0; i < RESYNC_PAGES; i++) {
			page = alloc_page(gfp_flags);
			if (unlikely(!page))
				goto out_free_pages;

			bio->bi_io_vec[i].bv_page = page;
117
			bio->bi_vcnt = i+1;
118 119 120 121 122 123 124 125
		}
	}
	/* If not user-requests, copy the page pointers to all bios */
	if (!test_bit(MD_RECOVERY_REQUESTED, &pi->mddev->recovery)) {
		for (i=0; i<RESYNC_PAGES ; i++)
			for (j=1; j<pi->raid_disks; j++)
				r1_bio->bios[j]->bi_io_vec[i].bv_page =
					r1_bio->bios[0]->bi_io_vec[i].bv_page;
L
Linus Torvalds 已提交
126 127 128 129 130 131 132
	}

	r1_bio->master_bio = NULL;

	return r1_bio;

out_free_pages:
133 134 135
	for (j=0 ; j < pi->raid_disks; j++)
		for (i=0; i < r1_bio->bios[j]->bi_vcnt ; i++)
			put_page(r1_bio->bios[j]->bi_io_vec[i].bv_page);
136
	j = -1;
L
Linus Torvalds 已提交
137
out_free_bio:
138
	while (++j < pi->raid_disks)
L
Linus Torvalds 已提交
139 140 141 142 143 144 145 146
		bio_put(r1_bio->bios[j]);
	r1bio_pool_free(r1_bio, data);
	return NULL;
}

static void r1buf_pool_free(void *__r1_bio, void *data)
{
	struct pool_info *pi = data;
147
	int i,j;
148
	struct r1bio *r1bio = __r1_bio;
L
Linus Torvalds 已提交
149

150 151 152 153 154
	for (i = 0; i < RESYNC_PAGES; i++)
		for (j = pi->raid_disks; j-- ;) {
			if (j == 0 ||
			    r1bio->bios[j]->bi_io_vec[i].bv_page !=
			    r1bio->bios[0]->bi_io_vec[i].bv_page)
155
				safe_put_page(r1bio->bios[j]->bi_io_vec[i].bv_page);
156
		}
L
Linus Torvalds 已提交
157 158 159 160 161 162
	for (i=0 ; i < pi->raid_disks; i++)
		bio_put(r1bio->bios[i]);

	r1bio_pool_free(r1bio, data);
}

163
static void put_all_bios(struct r1conf *conf, struct r1bio *r1_bio)
L
Linus Torvalds 已提交
164 165 166
{
	int i;

167
	for (i = 0; i < conf->raid_disks * 2; i++) {
L
Linus Torvalds 已提交
168
		struct bio **bio = r1_bio->bios + i;
169
		if (!BIO_SPECIAL(*bio))
L
Linus Torvalds 已提交
170 171 172 173 174
			bio_put(*bio);
		*bio = NULL;
	}
}

175
static void free_r1bio(struct r1bio *r1_bio)
L
Linus Torvalds 已提交
176
{
177
	struct r1conf *conf = r1_bio->mddev->private;
L
Linus Torvalds 已提交
178 179 180 181 182

	put_all_bios(conf, r1_bio);
	mempool_free(r1_bio, conf->r1bio_pool);
}

183
static void put_buf(struct r1bio *r1_bio)
L
Linus Torvalds 已提交
184
{
185
	struct r1conf *conf = r1_bio->mddev->private;
186 187
	int i;

188
	for (i = 0; i < conf->raid_disks * 2; i++) {
189 190 191 192
		struct bio *bio = r1_bio->bios[i];
		if (bio->bi_end_io)
			rdev_dec_pending(conf->mirrors[i].rdev, r1_bio->mddev);
	}
L
Linus Torvalds 已提交
193 194 195

	mempool_free(r1_bio, conf->r1buf_pool);

196
	lower_barrier(conf);
L
Linus Torvalds 已提交
197 198
}

199
static void reschedule_retry(struct r1bio *r1_bio)
L
Linus Torvalds 已提交
200 201
{
	unsigned long flags;
202
	struct mddev *mddev = r1_bio->mddev;
203
	struct r1conf *conf = mddev->private;
L
Linus Torvalds 已提交
204 205 206

	spin_lock_irqsave(&conf->device_lock, flags);
	list_add(&r1_bio->retry_list, &conf->retry_list);
207
	conf->nr_queued ++;
L
Linus Torvalds 已提交
208 209
	spin_unlock_irqrestore(&conf->device_lock, flags);

210
	wake_up(&conf->wait_barrier);
L
Linus Torvalds 已提交
211 212 213 214 215 216 217 218
	md_wakeup_thread(mddev->thread);
}

/*
 * raid_end_bio_io() is called when we have finished servicing a mirrored
 * operation and are ready to return a success/failure code to the buffer
 * cache layer.
 */
219
static void call_bio_endio(struct r1bio *r1_bio)
220 221 222
{
	struct bio *bio = r1_bio->master_bio;
	int done;
223
	struct r1conf *conf = r1_bio->mddev->private;
224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245

	if (bio->bi_phys_segments) {
		unsigned long flags;
		spin_lock_irqsave(&conf->device_lock, flags);
		bio->bi_phys_segments--;
		done = (bio->bi_phys_segments == 0);
		spin_unlock_irqrestore(&conf->device_lock, flags);
	} else
		done = 1;

	if (!test_bit(R1BIO_Uptodate, &r1_bio->state))
		clear_bit(BIO_UPTODATE, &bio->bi_flags);
	if (done) {
		bio_endio(bio, 0);
		/*
		 * Wake up any possible resync thread that waits for the device
		 * to go idle.
		 */
		allow_barrier(conf);
	}
}

246
static void raid_end_bio_io(struct r1bio *r1_bio)
L
Linus Torvalds 已提交
247 248 249
{
	struct bio *bio = r1_bio->master_bio;

250 251
	/* if nobody has done the final endio yet, do it now */
	if (!test_and_set_bit(R1BIO_Returned, &r1_bio->state)) {
252 253 254 255 256
		pr_debug("raid1: sync end %s on sectors %llu-%llu\n",
			 (bio_data_dir(bio) == WRITE) ? "write" : "read",
			 (unsigned long long) bio->bi_sector,
			 (unsigned long long) bio->bi_sector +
			 (bio->bi_size >> 9) - 1);
257

258
		call_bio_endio(r1_bio);
259
	}
L
Linus Torvalds 已提交
260 261 262 263 264 265
	free_r1bio(r1_bio);
}

/*
 * Update disk head position estimator based on IRQ completion info.
 */
266
static inline void update_head_pos(int disk, struct r1bio *r1_bio)
L
Linus Torvalds 已提交
267
{
268
	struct r1conf *conf = r1_bio->mddev->private;
L
Linus Torvalds 已提交
269 270 271 272 273

	conf->mirrors[disk].head_position =
		r1_bio->sector + (r1_bio->sectors);
}

274 275 276
/*
 * Find the disk number which triggered given bio
 */
277
static int find_bio_disk(struct r1bio *r1_bio, struct bio *bio)
278 279
{
	int mirror;
280 281
	struct r1conf *conf = r1_bio->mddev->private;
	int raid_disks = conf->raid_disks;
282

283
	for (mirror = 0; mirror < raid_disks * 2; mirror++)
284 285 286
		if (r1_bio->bios[mirror] == bio)
			break;

287
	BUG_ON(mirror == raid_disks * 2);
288 289 290 291 292
	update_head_pos(mirror, r1_bio);

	return mirror;
}

293
static void raid1_end_read_request(struct bio *bio, int error)
L
Linus Torvalds 已提交
294 295
{
	int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
296
	struct r1bio *r1_bio = bio->bi_private;
L
Linus Torvalds 已提交
297
	int mirror;
298
	struct r1conf *conf = r1_bio->mddev->private;
L
Linus Torvalds 已提交
299 300 301 302 303

	mirror = r1_bio->read_disk;
	/*
	 * this branch is our 'one mirror IO has finished' event handler:
	 */
304 305
	update_head_pos(mirror, r1_bio);

306 307 308 309 310 311
	if (uptodate)
		set_bit(R1BIO_Uptodate, &r1_bio->state);
	else {
		/* If all other devices have failed, we want to return
		 * the error upwards rather than fail the last device.
		 * Here we redefine "uptodate" to mean "Don't want to retry"
L
Linus Torvalds 已提交
312
		 */
313 314 315 316 317 318 319 320
		unsigned long flags;
		spin_lock_irqsave(&conf->device_lock, flags);
		if (r1_bio->mddev->degraded == conf->raid_disks ||
		    (r1_bio->mddev->degraded == conf->raid_disks-1 &&
		     !test_bit(Faulty, &conf->mirrors[mirror].rdev->flags)))
			uptodate = 1;
		spin_unlock_irqrestore(&conf->device_lock, flags);
	}
L
Linus Torvalds 已提交
321

322
	if (uptodate)
L
Linus Torvalds 已提交
323
		raid_end_bio_io(r1_bio);
324
	else {
L
Linus Torvalds 已提交
325 326 327 328
		/*
		 * oops, read error:
		 */
		char b[BDEVNAME_SIZE];
329 330 331 332 333 334 335
		printk_ratelimited(
			KERN_ERR "md/raid1:%s: %s: "
			"rescheduling sector %llu\n",
			mdname(conf->mddev),
			bdevname(conf->mirrors[mirror].rdev->bdev,
				 b),
			(unsigned long long)r1_bio->sector);
336
		set_bit(R1BIO_ReadError, &r1_bio->state);
L
Linus Torvalds 已提交
337 338 339 340 341 342
		reschedule_retry(r1_bio);
	}

	rdev_dec_pending(conf->mirrors[mirror].rdev, conf->mddev);
}

343
static void close_write(struct r1bio *r1_bio)
344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361
{
	/* it really is the end of this request */
	if (test_bit(R1BIO_BehindIO, &r1_bio->state)) {
		/* free extra copy of the data pages */
		int i = r1_bio->behind_page_count;
		while (i--)
			safe_put_page(r1_bio->behind_bvecs[i].bv_page);
		kfree(r1_bio->behind_bvecs);
		r1_bio->behind_bvecs = NULL;
	}
	/* clear the bitmap if all writes complete successfully */
	bitmap_endwrite(r1_bio->mddev->bitmap, r1_bio->sector,
			r1_bio->sectors,
			!test_bit(R1BIO_Degraded, &r1_bio->state),
			test_bit(R1BIO_BehindIO, &r1_bio->state));
	md_write_end(r1_bio->mddev);
}

362
static void r1_bio_write_done(struct r1bio *r1_bio)
363
{
364 365 366 367 368 369 370
	if (!atomic_dec_and_test(&r1_bio->remaining))
		return;

	if (test_bit(R1BIO_WriteError, &r1_bio->state))
		reschedule_retry(r1_bio);
	else {
		close_write(r1_bio);
371 372 373 374
		if (test_bit(R1BIO_MadeGood, &r1_bio->state))
			reschedule_retry(r1_bio);
		else
			raid_end_bio_io(r1_bio);
375 376 377
	}
}

378
static void raid1_end_write_request(struct bio *bio, int error)
L
Linus Torvalds 已提交
379 380
{
	int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
381
	struct r1bio *r1_bio = bio->bi_private;
382
	int mirror, behind = test_bit(R1BIO_BehindIO, &r1_bio->state);
383
	struct r1conf *conf = r1_bio->mddev->private;
384
	struct bio *to_put = NULL;
L
Linus Torvalds 已提交
385

386
	mirror = find_bio_disk(r1_bio, bio);
L
Linus Torvalds 已提交
387

T
Tejun Heo 已提交
388 389 390 391
	/*
	 * 'one mirror IO has finished' event handler:
	 */
	if (!uptodate) {
392 393
		set_bit(WriteErrorSeen,
			&conf->mirrors[mirror].rdev->flags);
394 395 396 397 398
		if (!test_and_set_bit(WantReplacement,
				      &conf->mirrors[mirror].rdev->flags))
			set_bit(MD_RECOVERY_NEEDED, &
				conf->mddev->recovery);

399
		set_bit(R1BIO_WriteError, &r1_bio->state);
400
	} else {
L
Linus Torvalds 已提交
401
		/*
T
Tejun Heo 已提交
402 403 404 405 406 407 408 409
		 * Set R1BIO_Uptodate in our master bio, so that we
		 * will return a good error code for to the higher
		 * levels even if IO on some other mirrored buffer
		 * fails.
		 *
		 * The 'master' represents the composite IO operation
		 * to user-side. So if something waits for IO, then it
		 * will wait for the 'master' bio.
L
Linus Torvalds 已提交
410
		 */
411 412 413
		sector_t first_bad;
		int bad_sectors;

414 415
		r1_bio->bios[mirror] = NULL;
		to_put = bio;
T
Tejun Heo 已提交
416 417
		set_bit(R1BIO_Uptodate, &r1_bio->state);

418 419 420 421 422 423 424 425 426
		/* Maybe we can clear some bad blocks. */
		if (is_badblock(conf->mirrors[mirror].rdev,
				r1_bio->sector, r1_bio->sectors,
				&first_bad, &bad_sectors)) {
			r1_bio->bios[mirror] = IO_MADE_GOOD;
			set_bit(R1BIO_MadeGood, &r1_bio->state);
		}
	}

T
Tejun Heo 已提交
427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442
	if (behind) {
		if (test_bit(WriteMostly, &conf->mirrors[mirror].rdev->flags))
			atomic_dec(&r1_bio->behind_remaining);

		/*
		 * In behind mode, we ACK the master bio once the I/O
		 * has safely reached all non-writemostly
		 * disks. Setting the Returned bit ensures that this
		 * gets done only once -- we don't ever want to return
		 * -EIO here, instead we'll wait
		 */
		if (atomic_read(&r1_bio->behind_remaining) >= (atomic_read(&r1_bio->remaining)-1) &&
		    test_bit(R1BIO_Uptodate, &r1_bio->state)) {
			/* Maybe we can return now */
			if (!test_and_set_bit(R1BIO_Returned, &r1_bio->state)) {
				struct bio *mbio = r1_bio->master_bio;
443 444 445 446 447
				pr_debug("raid1: behind end write sectors"
					 " %llu-%llu\n",
					 (unsigned long long) mbio->bi_sector,
					 (unsigned long long) mbio->bi_sector +
					 (mbio->bi_size >> 9) - 1);
448
				call_bio_endio(r1_bio);
449 450 451
			}
		}
	}
452 453 454
	if (r1_bio->bios[mirror] == NULL)
		rdev_dec_pending(conf->mirrors[mirror].rdev,
				 conf->mddev);
T
Tejun Heo 已提交
455

L
Linus Torvalds 已提交
456 457 458 459
	/*
	 * Let's see if all mirrored write operations have finished
	 * already.
	 */
460
	r1_bio_write_done(r1_bio);
461

462 463
	if (to_put)
		bio_put(to_put);
L
Linus Torvalds 已提交
464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480
}


/*
 * This routine returns the disk from which the requested read should
 * be done. There is a per-array 'next expected sequential IO' sector
 * number - if this matches on the next IO then we use the last disk.
 * There is also a per-disk 'last know head position' sector that is
 * maintained from IRQ contexts, both the normal and the resync IO
 * completion handlers update this position correctly. If there is no
 * perfect sequential match then we pick the disk whose head is closest.
 *
 * If there are 2 mirrors in the same 2 devices, performance degrades
 * because position is mirror, not device based.
 *
 * The rdev for the device selected will have nr_pending incremented.
 */
481
static int read_balance(struct r1conf *conf, struct r1bio *r1_bio, int *max_sectors)
L
Linus Torvalds 已提交
482
{
483
	const sector_t this_sector = r1_bio->sector;
484 485
	int sectors;
	int best_good_sectors;
486
	int start_disk;
N
NeilBrown 已提交
487
	int best_disk;
488
	int i;
N
NeilBrown 已提交
489
	sector_t best_dist;
490
	struct md_rdev *rdev;
491
	int choose_first;
L
Linus Torvalds 已提交
492 493 494

	rcu_read_lock();
	/*
495
	 * Check if we can balance. We can balance on the whole
L
Linus Torvalds 已提交
496 497 498 499
	 * device if no resync is going on, or below the resync window.
	 * We take the first readable disk when above the resync window.
	 */
 retry:
500
	sectors = r1_bio->sectors;
N
NeilBrown 已提交
501 502
	best_disk = -1;
	best_dist = MaxSector;
503 504
	best_good_sectors = 0;

L
Linus Torvalds 已提交
505 506
	if (conf->mddev->recovery_cp < MaxSector &&
	    (this_sector + sectors >= conf->next_resync)) {
507 508 509 510 511
		choose_first = 1;
		start_disk = 0;
	} else {
		choose_first = 0;
		start_disk = conf->last_used;
L
Linus Torvalds 已提交
512 513
	}

514
	for (i = 0 ; i < conf->raid_disks * 2 ; i++) {
N
NeilBrown 已提交
515
		sector_t dist;
516 517 518
		sector_t first_bad;
		int bad_sectors;

519 520 521 522 523 524 525
		int disk = start_disk + i;
		if (disk >= conf->raid_disks)
			disk -= conf->raid_disks;

		rdev = rcu_dereference(conf->mirrors[disk].rdev);
		if (r1_bio->bios[disk] == IO_BLOCKED
		    || rdev == NULL
526
		    || test_bit(Unmerged, &rdev->flags)
N
NeilBrown 已提交
527
		    || test_bit(Faulty, &rdev->flags))
528
			continue;
N
NeilBrown 已提交
529 530
		if (!test_bit(In_sync, &rdev->flags) &&
		    rdev->recovery_offset < this_sector + sectors)
L
Linus Torvalds 已提交
531
			continue;
N
NeilBrown 已提交
532 533 534
		if (test_bit(WriteMostly, &rdev->flags)) {
			/* Don't balance among write-mostly, just
			 * use the first as a last resort */
535 536 537 538 539 540 541 542 543
			if (best_disk < 0) {
				if (is_badblock(rdev, this_sector, sectors,
						&first_bad, &bad_sectors)) {
					if (first_bad < this_sector)
						/* Cannot use this */
						continue;
					best_good_sectors = first_bad - this_sector;
				} else
					best_good_sectors = sectors;
N
NeilBrown 已提交
544
				best_disk = disk;
545
			}
N
NeilBrown 已提交
546 547 548 549 550
			continue;
		}
		/* This is a reasonable device to use.  It might
		 * even be best.
		 */
551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579
		if (is_badblock(rdev, this_sector, sectors,
				&first_bad, &bad_sectors)) {
			if (best_dist < MaxSector)
				/* already have a better device */
				continue;
			if (first_bad <= this_sector) {
				/* cannot read here. If this is the 'primary'
				 * device, then we must not read beyond
				 * bad_sectors from another device..
				 */
				bad_sectors -= (this_sector - first_bad);
				if (choose_first && sectors > bad_sectors)
					sectors = bad_sectors;
				if (best_good_sectors > sectors)
					best_good_sectors = sectors;

			} else {
				sector_t good_sectors = first_bad - this_sector;
				if (good_sectors > best_good_sectors) {
					best_good_sectors = good_sectors;
					best_disk = disk;
				}
				if (choose_first)
					break;
			}
			continue;
		} else
			best_good_sectors = sectors;

N
NeilBrown 已提交
580 581 582 583 584 585 586 587
		dist = abs(this_sector - conf->mirrors[disk].head_position);
		if (choose_first
		    /* Don't change to another disk for sequential reads */
		    || conf->next_seq_sect == this_sector
		    || dist == 0
		    /* If device is idle, use it */
		    || atomic_read(&rdev->nr_pending) == 0) {
			best_disk = disk;
L
Linus Torvalds 已提交
588 589
			break;
		}
N
NeilBrown 已提交
590 591 592
		if (dist < best_dist) {
			best_dist = dist;
			best_disk = disk;
L
Linus Torvalds 已提交
593
		}
594
	}
L
Linus Torvalds 已提交
595

N
NeilBrown 已提交
596 597
	if (best_disk >= 0) {
		rdev = rcu_dereference(conf->mirrors[best_disk].rdev);
598 599 600
		if (!rdev)
			goto retry;
		atomic_inc(&rdev->nr_pending);
N
NeilBrown 已提交
601
		if (test_bit(Faulty, &rdev->flags)) {
L
Linus Torvalds 已提交
602 603 604
			/* cannot risk returning a device that failed
			 * before we inc'ed nr_pending
			 */
605
			rdev_dec_pending(rdev, conf->mddev);
L
Linus Torvalds 已提交
606 607
			goto retry;
		}
608
		sectors = best_good_sectors;
609
		conf->next_seq_sect = this_sector + sectors;
N
NeilBrown 已提交
610
		conf->last_used = best_disk;
L
Linus Torvalds 已提交
611 612
	}
	rcu_read_unlock();
613
	*max_sectors = sectors;
L
Linus Torvalds 已提交
614

N
NeilBrown 已提交
615
	return best_disk;
L
Linus Torvalds 已提交
616 617
}

618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650
static int raid1_mergeable_bvec(struct request_queue *q,
				struct bvec_merge_data *bvm,
				struct bio_vec *biovec)
{
	struct mddev *mddev = q->queuedata;
	struct r1conf *conf = mddev->private;
	sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev);
	int max = biovec->bv_len;

	if (mddev->merge_check_needed) {
		int disk;
		rcu_read_lock();
		for (disk = 0; disk < conf->raid_disks * 2; disk++) {
			struct md_rdev *rdev = rcu_dereference(
				conf->mirrors[disk].rdev);
			if (rdev && !test_bit(Faulty, &rdev->flags)) {
				struct request_queue *q =
					bdev_get_queue(rdev->bdev);
				if (q->merge_bvec_fn) {
					bvm->bi_sector = sector +
						rdev->data_offset;
					bvm->bi_bdev = rdev->bdev;
					max = min(max, q->merge_bvec_fn(
							  q, bvm, biovec));
				}
			}
		}
		rcu_read_unlock();
	}
	return max;

}

651
int md_raid1_congested(struct mddev *mddev, int bits)
652
{
653
	struct r1conf *conf = mddev->private;
654 655
	int i, ret = 0;

656 657 658 659
	if ((bits & (1 << BDI_async_congested)) &&
	    conf->pending_count >= max_queued_requests)
		return 1;

660
	rcu_read_lock();
661
	for (i = 0; i < conf->raid_disks * 2; i++) {
662
		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
663
		if (rdev && !test_bit(Faulty, &rdev->flags)) {
664
			struct request_queue *q = bdev_get_queue(rdev->bdev);
665

666 667
			BUG_ON(!q);

668 669 670
			/* Note the '|| 1' - when read_balance prefers
			 * non-congested targets, it can be removed
			 */
671
			if ((bits & (1<<BDI_async_congested)) || 1)
672 673 674 675 676 677 678 679
				ret |= bdi_congested(&q->backing_dev_info, bits);
			else
				ret &= bdi_congested(&q->backing_dev_info, bits);
		}
	}
	rcu_read_unlock();
	return ret;
}
680
EXPORT_SYMBOL_GPL(md_raid1_congested);
681

682 683
static int raid1_congested(void *data, int bits)
{
684
	struct mddev *mddev = data;
685 686 687 688

	return mddev_congested(mddev, bits) ||
		md_raid1_congested(mddev, bits);
}
689

690
static void flush_pending_writes(struct r1conf *conf)
691 692 693 694 695 696 697 698 699
{
	/* Any writes that have been queued but are awaiting
	 * bitmap updates get flushed here.
	 */
	spin_lock_irq(&conf->device_lock);

	if (conf->pending_bio_list.head) {
		struct bio *bio;
		bio = bio_list_get(&conf->pending_bio_list);
700
		conf->pending_count = 0;
701 702 703 704
		spin_unlock_irq(&conf->device_lock);
		/* flush any pending bitmap writes to
		 * disk before proceeding w/ I/O */
		bitmap_unplug(conf->mddev->bitmap);
705
		wake_up(&conf->wait_barrier);
706 707 708 709 710 711 712 713 714

		while (bio) { /* submit pending writes */
			struct bio *next = bio->bi_next;
			bio->bi_next = NULL;
			generic_make_request(bio);
			bio = next;
		}
	} else
		spin_unlock_irq(&conf->device_lock);
J
Jens Axboe 已提交
715 716
}

717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736
/* Barriers....
 * Sometimes we need to suspend IO while we do something else,
 * either some resync/recovery, or reconfigure the array.
 * To do this we raise a 'barrier'.
 * The 'barrier' is a counter that can be raised multiple times
 * to count how many activities are happening which preclude
 * normal IO.
 * We can only raise the barrier if there is no pending IO.
 * i.e. if nr_pending == 0.
 * We choose only to raise the barrier if no-one is waiting for the
 * barrier to go down.  This means that as soon as an IO request
 * is ready, no other operations which require a barrier will start
 * until the IO request has had a chance.
 *
 * So: regular IO calls 'wait_barrier'.  When that returns there
 *    is no backgroup IO happening,  It must arrange to call
 *    allow_barrier when it has finished its IO.
 * backgroup IO calls must call raise_barrier.  Once that returns
 *    there is no normal IO happeing.  It must arrange to call
 *    lower_barrier when the particular background IO completes.
L
Linus Torvalds 已提交
737 738 739
 */
#define RESYNC_DEPTH 32

740
static void raise_barrier(struct r1conf *conf)
L
Linus Torvalds 已提交
741 742
{
	spin_lock_irq(&conf->resync_lock);
743 744 745

	/* Wait until no block IO is waiting */
	wait_event_lock_irq(conf->wait_barrier, !conf->nr_waiting,
N
NeilBrown 已提交
746
			    conf->resync_lock, );
747 748 749 750

	/* block any new IO from starting */
	conf->barrier++;

N
NeilBrown 已提交
751
	/* Now wait for all pending IO to complete */
752 753
	wait_event_lock_irq(conf->wait_barrier,
			    !conf->nr_pending && conf->barrier < RESYNC_DEPTH,
N
NeilBrown 已提交
754
			    conf->resync_lock, );
755 756 757 758

	spin_unlock_irq(&conf->resync_lock);
}

759
static void lower_barrier(struct r1conf *conf)
760 761
{
	unsigned long flags;
762
	BUG_ON(conf->barrier <= 0);
763 764 765 766 767 768
	spin_lock_irqsave(&conf->resync_lock, flags);
	conf->barrier--;
	spin_unlock_irqrestore(&conf->resync_lock, flags);
	wake_up(&conf->wait_barrier);
}

769
static void wait_barrier(struct r1conf *conf)
770 771 772 773
{
	spin_lock_irq(&conf->resync_lock);
	if (conf->barrier) {
		conf->nr_waiting++;
774 775 776 777 778 779 780 781 782 783 784 785 786 787
		/* Wait for the barrier to drop.
		 * However if there are already pending
		 * requests (preventing the barrier from
		 * rising completely), and the
		 * pre-process bio queue isn't empty,
		 * then don't wait, as we need to empty
		 * that queue to get the nr_pending
		 * count down.
		 */
		wait_event_lock_irq(conf->wait_barrier,
				    !conf->barrier ||
				    (conf->nr_pending &&
				     current->bio_list &&
				     !bio_list_empty(current->bio_list)),
788
				    conf->resync_lock,
789
			);
790
		conf->nr_waiting--;
L
Linus Torvalds 已提交
791
	}
792
	conf->nr_pending++;
L
Linus Torvalds 已提交
793 794 795
	spin_unlock_irq(&conf->resync_lock);
}

796
static void allow_barrier(struct r1conf *conf)
797 798 799 800 801 802 803 804
{
	unsigned long flags;
	spin_lock_irqsave(&conf->resync_lock, flags);
	conf->nr_pending--;
	spin_unlock_irqrestore(&conf->resync_lock, flags);
	wake_up(&conf->wait_barrier);
}

805
static void freeze_array(struct r1conf *conf)
806 807 808 809
{
	/* stop syncio and normal IO and wait for everything to
	 * go quite.
	 * We increment barrier and nr_waiting, and then
810 811 812 813 814 815 816 817
	 * wait until nr_pending match nr_queued+1
	 * This is called in the context of one normal IO request
	 * that has failed. Thus any sync request that might be pending
	 * will be blocked by nr_pending, and we need to wait for
	 * pending IO requests to complete or be queued for re-try.
	 * Thus the number queued (nr_queued) plus this request (1)
	 * must match the number of pending IOs (nr_pending) before
	 * we continue.
818 819 820 821 822
	 */
	spin_lock_irq(&conf->resync_lock);
	conf->barrier++;
	conf->nr_waiting++;
	wait_event_lock_irq(conf->wait_barrier,
823
			    conf->nr_pending == conf->nr_queued+1,
824
			    conf->resync_lock,
N
NeilBrown 已提交
825
			    flush_pending_writes(conf));
826 827
	spin_unlock_irq(&conf->resync_lock);
}
828
static void unfreeze_array(struct r1conf *conf)
829 830 831 832 833 834 835 836 837
{
	/* reverse the effect of the freeze */
	spin_lock_irq(&conf->resync_lock);
	conf->barrier--;
	conf->nr_waiting--;
	wake_up(&conf->wait_barrier);
	spin_unlock_irq(&conf->resync_lock);
}

838

839 840
/* duplicate the data pages for behind I/O 
 */
841
static void alloc_behind_pages(struct bio *bio, struct r1bio *r1_bio)
842 843 844
{
	int i;
	struct bio_vec *bvec;
845
	struct bio_vec *bvecs = kzalloc(bio->bi_vcnt * sizeof(struct bio_vec),
846
					GFP_NOIO);
847
	if (unlikely(!bvecs))
848
		return;
849 850

	bio_for_each_segment(bvec, bio, i) {
851 852 853
		bvecs[i] = *bvec;
		bvecs[i].bv_page = alloc_page(GFP_NOIO);
		if (unlikely(!bvecs[i].bv_page))
854
			goto do_sync_io;
855 856 857
		memcpy(kmap(bvecs[i].bv_page) + bvec->bv_offset,
		       kmap(bvec->bv_page) + bvec->bv_offset, bvec->bv_len);
		kunmap(bvecs[i].bv_page);
858 859
		kunmap(bvec->bv_page);
	}
860
	r1_bio->behind_bvecs = bvecs;
861 862 863
	r1_bio->behind_page_count = bio->bi_vcnt;
	set_bit(R1BIO_BehindIO, &r1_bio->state);
	return;
864 865

do_sync_io:
866
	for (i = 0; i < bio->bi_vcnt; i++)
867 868 869
		if (bvecs[i].bv_page)
			put_page(bvecs[i].bv_page);
	kfree(bvecs);
870
	pr_debug("%dB behind alloc failed, doing sync I/O\n", bio->bi_size);
871 872
}

873
static void make_request(struct mddev *mddev, struct bio * bio)
L
Linus Torvalds 已提交
874
{
875
	struct r1conf *conf = mddev->private;
876
	struct mirror_info *mirror;
877
	struct r1bio *r1_bio;
L
Linus Torvalds 已提交
878
	struct bio *read_bio;
879
	int i, disks;
880
	struct bitmap *bitmap;
881
	unsigned long flags;
882
	const int rw = bio_data_dir(bio);
883
	const unsigned long do_sync = (bio->bi_rw & REQ_SYNC);
T
Tejun Heo 已提交
884
	const unsigned long do_flush_fua = (bio->bi_rw & (REQ_FLUSH | REQ_FUA));
885
	struct md_rdev *blocked_rdev;
N
NeilBrown 已提交
886
	int plugged;
887 888 889
	int first_clone;
	int sectors_handled;
	int max_sectors;
890

L
Linus Torvalds 已提交
891 892 893 894 895
	/*
	 * Register the new request and wait if the reconstruction
	 * thread has put up a bar for new requests.
	 * Continue immediately if no resync is active currently.
	 */
896

897 898
	md_write_start(mddev, bio); /* wait on superblock update early */

899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917
	if (bio_data_dir(bio) == WRITE &&
	    bio->bi_sector + bio->bi_size/512 > mddev->suspend_lo &&
	    bio->bi_sector < mddev->suspend_hi) {
		/* As the suspend_* range is controlled by
		 * userspace, we want an interruptible
		 * wait.
		 */
		DEFINE_WAIT(w);
		for (;;) {
			flush_signals(current);
			prepare_to_wait(&conf->wait_barrier,
					&w, TASK_INTERRUPTIBLE);
			if (bio->bi_sector + bio->bi_size/512 <= mddev->suspend_lo ||
			    bio->bi_sector >= mddev->suspend_hi)
				break;
			schedule();
		}
		finish_wait(&conf->wait_barrier, &w);
	}
918

919
	wait_barrier(conf);
L
Linus Torvalds 已提交
920

921 922
	bitmap = mddev->bitmap;

L
Linus Torvalds 已提交
923 924 925 926 927 928 929 930 931
	/*
	 * make_request() can abort the operation when READA is being
	 * used and no empty request is available.
	 *
	 */
	r1_bio = mempool_alloc(conf->r1bio_pool, GFP_NOIO);

	r1_bio->master_bio = bio;
	r1_bio->sectors = bio->bi_size >> 9;
932
	r1_bio->state = 0;
L
Linus Torvalds 已提交
933 934 935
	r1_bio->mddev = mddev;
	r1_bio->sector = bio->bi_sector;

936 937 938 939 940 941 942 943 944 945
	/* We might need to issue multiple reads to different
	 * devices if there are bad blocks around, so we keep
	 * track of the number of reads in bio->bi_phys_segments.
	 * If this is 0, there is only one r1_bio and no locking
	 * will be needed when requests complete.  If it is
	 * non-zero, then it is the number of not-completed requests.
	 */
	bio->bi_phys_segments = 0;
	clear_bit(BIO_SEG_VALID, &bio->bi_flags);

946
	if (rw == READ) {
L
Linus Torvalds 已提交
947 948 949
		/*
		 * read balancing logic:
		 */
950 951 952 953
		int rdisk;

read_again:
		rdisk = read_balance(conf, r1_bio, &max_sectors);
L
Linus Torvalds 已提交
954 955 956 957

		if (rdisk < 0) {
			/* couldn't find anywhere to read from */
			raid_end_bio_io(r1_bio);
958
			return;
L
Linus Torvalds 已提交
959 960 961
		}
		mirror = conf->mirrors + rdisk;

962 963 964 965 966 967 968 969 970
		if (test_bit(WriteMostly, &mirror->rdev->flags) &&
		    bitmap) {
			/* Reading from a write-mostly device must
			 * take care not to over-take any writes
			 * that are 'behind'
			 */
			wait_event(bitmap->behind_wait,
				   atomic_read(&bitmap->behind_writes) == 0);
		}
L
Linus Torvalds 已提交
971 972
		r1_bio->read_disk = rdisk;

973
		read_bio = bio_clone_mddev(bio, GFP_NOIO, mddev);
974 975
		md_trim_bio(read_bio, r1_bio->sector - bio->bi_sector,
			    max_sectors);
L
Linus Torvalds 已提交
976 977 978 979 980 981

		r1_bio->bios[rdisk] = read_bio;

		read_bio->bi_sector = r1_bio->sector + mirror->rdev->data_offset;
		read_bio->bi_bdev = mirror->rdev->bdev;
		read_bio->bi_end_io = raid1_end_read_request;
982
		read_bio->bi_rw = READ | do_sync;
L
Linus Torvalds 已提交
983 984
		read_bio->bi_private = r1_bio;

985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015
		if (max_sectors < r1_bio->sectors) {
			/* could not read all from this device, so we will
			 * need another r1_bio.
			 */

			sectors_handled = (r1_bio->sector + max_sectors
					   - bio->bi_sector);
			r1_bio->sectors = max_sectors;
			spin_lock_irq(&conf->device_lock);
			if (bio->bi_phys_segments == 0)
				bio->bi_phys_segments = 2;
			else
				bio->bi_phys_segments++;
			spin_unlock_irq(&conf->device_lock);
			/* Cannot call generic_make_request directly
			 * as that will be queued in __make_request
			 * and subsequent mempool_alloc might block waiting
			 * for it.  So hand bio over to raid1d.
			 */
			reschedule_retry(r1_bio);

			r1_bio = mempool_alloc(conf->r1bio_pool, GFP_NOIO);

			r1_bio->master_bio = bio;
			r1_bio->sectors = (bio->bi_size >> 9) - sectors_handled;
			r1_bio->state = 0;
			r1_bio->mddev = mddev;
			r1_bio->sector = bio->bi_sector + sectors_handled;
			goto read_again;
		} else
			generic_make_request(read_bio);
1016
		return;
L
Linus Torvalds 已提交
1017 1018 1019 1020 1021
	}

	/*
	 * WRITE:
	 */
1022 1023 1024 1025 1026
	if (conf->pending_count >= max_queued_requests) {
		md_wakeup_thread(mddev->thread);
		wait_event(conf->wait_barrier,
			   conf->pending_count < max_queued_requests);
	}
1027
	/* first select target devices under rcu_lock and
L
Linus Torvalds 已提交
1028 1029
	 * inc refcount on their rdev.  Record them by setting
	 * bios[x] to bio
1030 1031 1032 1033 1034 1035
	 * If there are known/acknowledged bad blocks on any device on
	 * which we have seen a write error, we want to avoid writing those
	 * blocks.
	 * This potentially requires several writes to write around
	 * the bad blocks.  Each set of writes gets it's own r1bio
	 * with a set of bios attached.
L
Linus Torvalds 已提交
1036
	 */
N
NeilBrown 已提交
1037 1038
	plugged = mddev_check_plugged(mddev);

1039
	disks = conf->raid_disks * 2;
1040 1041
 retry_write:
	blocked_rdev = NULL;
L
Linus Torvalds 已提交
1042
	rcu_read_lock();
1043
	max_sectors = r1_bio->sectors;
L
Linus Torvalds 已提交
1044
	for (i = 0;  i < disks; i++) {
1045
		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
1046 1047 1048 1049 1050
		if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) {
			atomic_inc(&rdev->nr_pending);
			blocked_rdev = rdev;
			break;
		}
1051
		r1_bio->bios[i] = NULL;
1052 1053
		if (!rdev || test_bit(Faulty, &rdev->flags)
		    || test_bit(Unmerged, &rdev->flags)) {
1054 1055
			if (i < conf->raid_disks)
				set_bit(R1BIO_Degraded, &r1_bio->state);
1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082
			continue;
		}

		atomic_inc(&rdev->nr_pending);
		if (test_bit(WriteErrorSeen, &rdev->flags)) {
			sector_t first_bad;
			int bad_sectors;
			int is_bad;

			is_bad = is_badblock(rdev, r1_bio->sector,
					     max_sectors,
					     &first_bad, &bad_sectors);
			if (is_bad < 0) {
				/* mustn't write here until the bad block is
				 * acknowledged*/
				set_bit(BlockedBadBlocks, &rdev->flags);
				blocked_rdev = rdev;
				break;
			}
			if (is_bad && first_bad <= r1_bio->sector) {
				/* Cannot write here at all */
				bad_sectors -= (r1_bio->sector - first_bad);
				if (bad_sectors < max_sectors)
					/* mustn't write more than bad_sectors
					 * to other devices yet
					 */
					max_sectors = bad_sectors;
1083
				rdev_dec_pending(rdev, mddev);
1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094
				/* We don't set R1BIO_Degraded as that
				 * only applies if the disk is
				 * missing, so it might be re-added,
				 * and we want to know to recover this
				 * chunk.
				 * In this case the device is here,
				 * and the fact that this chunk is not
				 * in-sync is recorded in the bad
				 * block log
				 */
				continue;
1095
			}
1096 1097 1098 1099 1100 1101 1102
			if (is_bad) {
				int good_sectors = first_bad - r1_bio->sector;
				if (good_sectors < max_sectors)
					max_sectors = good_sectors;
			}
		}
		r1_bio->bios[i] = bio;
L
Linus Torvalds 已提交
1103 1104 1105
	}
	rcu_read_unlock();

1106 1107 1108 1109 1110 1111 1112
	if (unlikely(blocked_rdev)) {
		/* Wait for this device to become unblocked */
		int j;

		for (j = 0; j < i; j++)
			if (r1_bio->bios[j])
				rdev_dec_pending(conf->mirrors[j].rdev, mddev);
1113
		r1_bio->state = 0;
1114 1115 1116 1117 1118 1119
		allow_barrier(conf);
		md_wait_for_blocked_rdev(blocked_rdev, mddev);
		wait_barrier(conf);
		goto retry_write;
	}

1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130
	if (max_sectors < r1_bio->sectors) {
		/* We are splitting this write into multiple parts, so
		 * we need to prepare for allocating another r1_bio.
		 */
		r1_bio->sectors = max_sectors;
		spin_lock_irq(&conf->device_lock);
		if (bio->bi_phys_segments == 0)
			bio->bi_phys_segments = 2;
		else
			bio->bi_phys_segments++;
		spin_unlock_irq(&conf->device_lock);
1131
	}
1132
	sectors_handled = r1_bio->sector + max_sectors - bio->bi_sector;
1133

1134
	atomic_set(&r1_bio->remaining, 1);
1135
	atomic_set(&r1_bio->behind_remaining, 0);
1136

1137
	first_clone = 1;
L
Linus Torvalds 已提交
1138 1139 1140 1141 1142
	for (i = 0; i < disks; i++) {
		struct bio *mbio;
		if (!r1_bio->bios[i])
			continue;

1143
		mbio = bio_clone_mddev(bio, GFP_NOIO, mddev);
1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162
		md_trim_bio(mbio, r1_bio->sector - bio->bi_sector, max_sectors);

		if (first_clone) {
			/* do behind I/O ?
			 * Not if there are too many, or cannot
			 * allocate memory, or a reader on WriteMostly
			 * is waiting for behind writes to flush */
			if (bitmap &&
			    (atomic_read(&bitmap->behind_writes)
			     < mddev->bitmap_info.max_write_behind) &&
			    !waitqueue_active(&bitmap->behind_wait))
				alloc_behind_pages(mbio, r1_bio);

			bitmap_startwrite(bitmap, r1_bio->sector,
					  r1_bio->sectors,
					  test_bit(R1BIO_BehindIO,
						   &r1_bio->state));
			first_clone = 0;
		}
1163
		if (r1_bio->behind_bvecs) {
1164 1165 1166 1167 1168 1169 1170
			struct bio_vec *bvec;
			int j;

			/* Yes, I really want the '__' version so that
			 * we clear any unused pointer in the io_vec, rather
			 * than leave them unchanged.  This is important
			 * because when we come to free the pages, we won't
N
NeilBrown 已提交
1171
			 * know the original bi_idx, so we just free
1172 1173 1174
			 * them all
			 */
			__bio_for_each_segment(bvec, mbio, j, 0)
1175
				bvec->bv_page = r1_bio->behind_bvecs[j].bv_page;
1176 1177 1178 1179
			if (test_bit(WriteMostly, &conf->mirrors[i].rdev->flags))
				atomic_inc(&r1_bio->behind_remaining);
		}

1180 1181 1182 1183 1184 1185 1186 1187 1188
		r1_bio->bios[i] = mbio;

		mbio->bi_sector	= (r1_bio->sector +
				   conf->mirrors[i].rdev->data_offset);
		mbio->bi_bdev = conf->mirrors[i].rdev->bdev;
		mbio->bi_end_io	= raid1_end_write_request;
		mbio->bi_rw = WRITE | do_flush_fua | do_sync;
		mbio->bi_private = r1_bio;

L
Linus Torvalds 已提交
1189
		atomic_inc(&r1_bio->remaining);
1190 1191
		spin_lock_irqsave(&conf->device_lock, flags);
		bio_list_add(&conf->pending_bio_list, mbio);
1192
		conf->pending_count++;
1193
		spin_unlock_irqrestore(&conf->device_lock, flags);
L
Linus Torvalds 已提交
1194
	}
1195 1196 1197
	/* Mustn't call r1_bio_write_done before this next test,
	 * as it could result in the bio being freed.
	 */
1198
	if (sectors_handled < (bio->bi_size >> 9)) {
1199
		r1_bio_write_done(r1_bio);
1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211
		/* We need another r1_bio.  It has already been counted
		 * in bio->bi_phys_segments
		 */
		r1_bio = mempool_alloc(conf->r1bio_pool, GFP_NOIO);
		r1_bio->master_bio = bio;
		r1_bio->sectors = (bio->bi_size >> 9) - sectors_handled;
		r1_bio->state = 0;
		r1_bio->mddev = mddev;
		r1_bio->sector = bio->bi_sector + sectors_handled;
		goto retry_write;
	}

1212 1213 1214 1215 1216
	r1_bio_write_done(r1_bio);

	/* In case raid1d snuck in to freeze_array */
	wake_up(&conf->wait_barrier);

N
NeilBrown 已提交
1217
	if (do_sync || !bitmap || !plugged)
1218
		md_wakeup_thread(mddev->thread);
L
Linus Torvalds 已提交
1219 1220
}

1221
static void status(struct seq_file *seq, struct mddev *mddev)
L
Linus Torvalds 已提交
1222
{
1223
	struct r1conf *conf = mddev->private;
L
Linus Torvalds 已提交
1224 1225 1226
	int i;

	seq_printf(seq, " [%d/%d] [", conf->raid_disks,
1227
		   conf->raid_disks - mddev->degraded);
1228 1229
	rcu_read_lock();
	for (i = 0; i < conf->raid_disks; i++) {
1230
		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
L
Linus Torvalds 已提交
1231
		seq_printf(seq, "%s",
1232 1233 1234
			   rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_");
	}
	rcu_read_unlock();
L
Linus Torvalds 已提交
1235 1236 1237 1238
	seq_printf(seq, "]");
}


1239
static void error(struct mddev *mddev, struct md_rdev *rdev)
L
Linus Torvalds 已提交
1240 1241
{
	char b[BDEVNAME_SIZE];
1242
	struct r1conf *conf = mddev->private;
L
Linus Torvalds 已提交
1243 1244 1245 1246 1247 1248 1249

	/*
	 * If it is not operational, then we have already marked it as dead
	 * else if it is the last working disks, ignore the error, let the
	 * next level up know.
	 * else mark the drive as failed
	 */
1250
	if (test_bit(In_sync, &rdev->flags)
1251
	    && (conf->raid_disks - mddev->degraded) == 1) {
L
Linus Torvalds 已提交
1252 1253
		/*
		 * Don't fail the drive, act as though we were just a
1254 1255 1256
		 * normal single drive.
		 * However don't try a recovery from this drive as
		 * it is very likely to fail.
L
Linus Torvalds 已提交
1257
		 */
1258
		conf->recovery_disabled = mddev->recovery_disabled;
L
Linus Torvalds 已提交
1259
		return;
1260
	}
1261
	set_bit(Blocked, &rdev->flags);
1262 1263 1264
	if (test_and_clear_bit(In_sync, &rdev->flags)) {
		unsigned long flags;
		spin_lock_irqsave(&conf->device_lock, flags);
L
Linus Torvalds 已提交
1265
		mddev->degraded++;
1266
		set_bit(Faulty, &rdev->flags);
1267
		spin_unlock_irqrestore(&conf->device_lock, flags);
L
Linus Torvalds 已提交
1268 1269 1270
		/*
		 * if recovery is running, make sure it aborts.
		 */
1271
		set_bit(MD_RECOVERY_INTR, &mddev->recovery);
1272 1273
	} else
		set_bit(Faulty, &rdev->flags);
1274
	set_bit(MD_CHANGE_DEVS, &mddev->flags);
1275 1276 1277
	printk(KERN_ALERT
	       "md/raid1:%s: Disk failure on %s, disabling device.\n"
	       "md/raid1:%s: Operation continuing on %d devices.\n",
N
NeilBrown 已提交
1278 1279
	       mdname(mddev), bdevname(rdev->bdev, b),
	       mdname(mddev), conf->raid_disks - mddev->degraded);
L
Linus Torvalds 已提交
1280 1281
}

1282
static void print_conf(struct r1conf *conf)
L
Linus Torvalds 已提交
1283 1284 1285
{
	int i;

N
NeilBrown 已提交
1286
	printk(KERN_DEBUG "RAID1 conf printout:\n");
L
Linus Torvalds 已提交
1287
	if (!conf) {
N
NeilBrown 已提交
1288
		printk(KERN_DEBUG "(!conf)\n");
L
Linus Torvalds 已提交
1289 1290
		return;
	}
N
NeilBrown 已提交
1291
	printk(KERN_DEBUG " --- wd:%d rd:%d\n", conf->raid_disks - conf->mddev->degraded,
L
Linus Torvalds 已提交
1292 1293
		conf->raid_disks);

1294
	rcu_read_lock();
L
Linus Torvalds 已提交
1295 1296
	for (i = 0; i < conf->raid_disks; i++) {
		char b[BDEVNAME_SIZE];
1297
		struct md_rdev *rdev = rcu_dereference(conf->mirrors[i].rdev);
1298
		if (rdev)
N
NeilBrown 已提交
1299
			printk(KERN_DEBUG " disk %d, wo:%d, o:%d, dev:%s\n",
1300 1301 1302
			       i, !test_bit(In_sync, &rdev->flags),
			       !test_bit(Faulty, &rdev->flags),
			       bdevname(rdev->bdev,b));
L
Linus Torvalds 已提交
1303
	}
1304
	rcu_read_unlock();
L
Linus Torvalds 已提交
1305 1306
}

1307
static void close_sync(struct r1conf *conf)
L
Linus Torvalds 已提交
1308
{
1309 1310
	wait_barrier(conf);
	allow_barrier(conf);
L
Linus Torvalds 已提交
1311 1312 1313 1314 1315

	mempool_destroy(conf->r1buf_pool);
	conf->r1buf_pool = NULL;
}

1316
static int raid1_spare_active(struct mddev *mddev)
L
Linus Torvalds 已提交
1317 1318
{
	int i;
1319
	struct r1conf *conf = mddev->private;
1320 1321
	int count = 0;
	unsigned long flags;
L
Linus Torvalds 已提交
1322 1323 1324

	/*
	 * Find all failed disks within the RAID1 configuration 
1325 1326
	 * and mark them readable.
	 * Called under mddev lock, so rcu protection not needed.
L
Linus Torvalds 已提交
1327 1328
	 */
	for (i = 0; i < conf->raid_disks; i++) {
1329
		struct md_rdev *rdev = conf->mirrors[i].rdev;
1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348
		struct md_rdev *repl = conf->mirrors[conf->raid_disks + i].rdev;
		if (repl
		    && repl->recovery_offset == MaxSector
		    && !test_bit(Faulty, &repl->flags)
		    && !test_and_set_bit(In_sync, &repl->flags)) {
			/* replacement has just become active */
			if (!rdev ||
			    !test_and_clear_bit(In_sync, &rdev->flags))
				count++;
			if (rdev) {
				/* Replaced device not technically
				 * faulty, but we need to be sure
				 * it gets removed and never re-added
				 */
				set_bit(Faulty, &rdev->flags);
				sysfs_notify_dirent_safe(
					rdev->sysfs_state);
			}
		}
1349 1350
		if (rdev
		    && !test_bit(Faulty, &rdev->flags)
1351
		    && !test_and_set_bit(In_sync, &rdev->flags)) {
1352
			count++;
1353
			sysfs_notify_dirent_safe(rdev->sysfs_state);
L
Linus Torvalds 已提交
1354 1355
		}
	}
1356 1357 1358
	spin_lock_irqsave(&conf->device_lock, flags);
	mddev->degraded -= count;
	spin_unlock_irqrestore(&conf->device_lock, flags);
L
Linus Torvalds 已提交
1359 1360

	print_conf(conf);
1361
	return count;
L
Linus Torvalds 已提交
1362 1363 1364
}


1365
static int raid1_add_disk(struct mddev *mddev, struct md_rdev *rdev)
L
Linus Torvalds 已提交
1366
{
1367
	struct r1conf *conf = mddev->private;
1368
	int err = -EEXIST;
1369
	int mirror = 0;
1370
	struct mirror_info *p;
1371
	int first = 0;
1372
	int last = conf->raid_disks - 1;
1373
	struct request_queue *q = bdev_get_queue(rdev->bdev);
L
Linus Torvalds 已提交
1374

1375 1376 1377
	if (mddev->recovery_disabled == conf->recovery_disabled)
		return -EBUSY;

1378 1379 1380
	if (rdev->raid_disk >= 0)
		first = last = rdev->raid_disk;

1381 1382 1383 1384 1385
	if (q->merge_bvec_fn) {
		set_bit(Unmerged, &rdev->flags);
		mddev->merge_check_needed = 1;
	}

1386 1387 1388
	for (mirror = first; mirror <= last; mirror++) {
		p = conf->mirrors+mirror;
		if (!p->rdev) {
L
Linus Torvalds 已提交
1389

1390 1391
			disk_stack_limits(mddev->gendisk, rdev->bdev,
					  rdev->data_offset << 9);
L
Linus Torvalds 已提交
1392 1393 1394

			p->head_position = 0;
			rdev->raid_disk = mirror;
1395
			err = 0;
1396 1397 1398 1399
			/* As all devices are equivalent, we don't need a full recovery
			 * if this was recently any drive of the array
			 */
			if (rdev->saved_raid_disk < 0)
1400
				conf->fullsync = 1;
1401
			rcu_assign_pointer(p->rdev, rdev);
L
Linus Torvalds 已提交
1402 1403
			break;
		}
1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415
		if (test_bit(WantReplacement, &p->rdev->flags) &&
		    p[conf->raid_disks].rdev == NULL) {
			/* Add this device as a replacement */
			clear_bit(In_sync, &rdev->flags);
			set_bit(Replacement, &rdev->flags);
			rdev->raid_disk = mirror;
			err = 0;
			conf->fullsync = 1;
			rcu_assign_pointer(p[conf->raid_disks].rdev, rdev);
			break;
		}
	}
1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428
	if (err == 0 && test_bit(Unmerged, &rdev->flags)) {
		/* Some requests might not have seen this new
		 * merge_bvec_fn.  We must wait for them to complete
		 * before merging the device fully.
		 * First we make sure any code which has tested
		 * our function has submitted the request, then
		 * we wait for all outstanding requests to complete.
		 */
		synchronize_sched();
		raise_barrier(conf);
		lower_barrier(conf);
		clear_bit(Unmerged, &rdev->flags);
	}
1429
	md_integrity_add_rdev(rdev, mddev);
L
Linus Torvalds 已提交
1430
	print_conf(conf);
1431
	return err;
L
Linus Torvalds 已提交
1432 1433
}

1434
static int raid1_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
L
Linus Torvalds 已提交
1435
{
1436
	struct r1conf *conf = mddev->private;
L
Linus Torvalds 已提交
1437
	int err = 0;
1438
	int number = rdev->raid_disk;
1439
	struct mirror_info *p = conf->mirrors+ number;
L
Linus Torvalds 已提交
1440

1441 1442 1443
	if (rdev != p->rdev)
		p = conf->mirrors + conf->raid_disks + number;

L
Linus Torvalds 已提交
1444
	print_conf(conf);
1445
	if (rdev == p->rdev) {
1446
		if (test_bit(In_sync, &rdev->flags) ||
L
Linus Torvalds 已提交
1447 1448 1449 1450
		    atomic_read(&rdev->nr_pending)) {
			err = -EBUSY;
			goto abort;
		}
N
NeilBrown 已提交
1451
		/* Only remove non-faulty devices if recovery
1452 1453 1454
		 * is not possible.
		 */
		if (!test_bit(Faulty, &rdev->flags) &&
1455
		    mddev->recovery_disabled != conf->recovery_disabled &&
1456 1457 1458 1459
		    mddev->degraded < conf->raid_disks) {
			err = -EBUSY;
			goto abort;
		}
L
Linus Torvalds 已提交
1460
		p->rdev = NULL;
1461
		synchronize_rcu();
L
Linus Torvalds 已提交
1462 1463 1464 1465
		if (atomic_read(&rdev->nr_pending)) {
			/* lost the race, try later */
			err = -EBUSY;
			p->rdev = rdev;
1466
			goto abort;
1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480
		} else if (conf->mirrors[conf->raid_disks + number].rdev) {
			/* We just removed a device that is being replaced.
			 * Move down the replacement.  We drain all IO before
			 * doing this to avoid confusion.
			 */
			struct md_rdev *repl =
				conf->mirrors[conf->raid_disks + number].rdev;
			raise_barrier(conf);
			clear_bit(Replacement, &repl->flags);
			p->rdev = repl;
			conf->mirrors[conf->raid_disks + number].rdev = NULL;
			lower_barrier(conf);
			clear_bit(WantReplacement, &rdev->flags);
		} else
1481
			clear_bit(WantReplacement, &rdev->flags);
1482
		err = md_integrity_register(mddev);
L
Linus Torvalds 已提交
1483 1484 1485 1486 1487 1488 1489 1490
	}
abort:

	print_conf(conf);
	return err;
}


1491
static void end_sync_read(struct bio *bio, int error)
L
Linus Torvalds 已提交
1492
{
1493
	struct r1bio *r1_bio = bio->bi_private;
L
Linus Torvalds 已提交
1494

1495
	update_head_pos(r1_bio->read_disk, r1_bio);
1496

L
Linus Torvalds 已提交
1497 1498 1499 1500 1501
	/*
	 * we have read a block, now it needs to be re-written,
	 * or re-read if the read failed.
	 * We don't do much here, just schedule handling by raid1d
	 */
1502
	if (test_bit(BIO_UPTODATE, &bio->bi_flags))
L
Linus Torvalds 已提交
1503
		set_bit(R1BIO_Uptodate, &r1_bio->state);
1504 1505 1506

	if (atomic_dec_and_test(&r1_bio->remaining))
		reschedule_retry(r1_bio);
L
Linus Torvalds 已提交
1507 1508
}

1509
static void end_sync_write(struct bio *bio, int error)
L
Linus Torvalds 已提交
1510 1511
{
	int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
1512
	struct r1bio *r1_bio = bio->bi_private;
1513
	struct mddev *mddev = r1_bio->mddev;
1514
	struct r1conf *conf = mddev->private;
L
Linus Torvalds 已提交
1515
	int mirror=0;
1516 1517
	sector_t first_bad;
	int bad_sectors;
L
Linus Torvalds 已提交
1518

1519 1520
	mirror = find_bio_disk(r1_bio, bio);

1521
	if (!uptodate) {
N
NeilBrown 已提交
1522
		sector_t sync_blocks = 0;
1523 1524 1525 1526
		sector_t s = r1_bio->sector;
		long sectors_to_go = r1_bio->sectors;
		/* make sure these bits doesn't get cleared. */
		do {
1527
			bitmap_end_sync(mddev->bitmap, s,
1528 1529 1530 1531
					&sync_blocks, 1);
			s += sync_blocks;
			sectors_to_go -= sync_blocks;
		} while (sectors_to_go > 0);
1532 1533
		set_bit(WriteErrorSeen,
			&conf->mirrors[mirror].rdev->flags);
1534 1535 1536 1537
		if (!test_and_set_bit(WantReplacement,
				      &conf->mirrors[mirror].rdev->flags))
			set_bit(MD_RECOVERY_NEEDED, &
				mddev->recovery);
1538
		set_bit(R1BIO_WriteError, &r1_bio->state);
1539 1540 1541
	} else if (is_badblock(conf->mirrors[mirror].rdev,
			       r1_bio->sector,
			       r1_bio->sectors,
1542 1543 1544 1545 1546 1547
			       &first_bad, &bad_sectors) &&
		   !is_badblock(conf->mirrors[r1_bio->read_disk].rdev,
				r1_bio->sector,
				r1_bio->sectors,
				&first_bad, &bad_sectors)
		)
1548
		set_bit(R1BIO_MadeGood, &r1_bio->state);
1549

L
Linus Torvalds 已提交
1550
	if (atomic_dec_and_test(&r1_bio->remaining)) {
1551
		int s = r1_bio->sectors;
1552 1553
		if (test_bit(R1BIO_MadeGood, &r1_bio->state) ||
		    test_bit(R1BIO_WriteError, &r1_bio->state))
1554 1555 1556 1557 1558
			reschedule_retry(r1_bio);
		else {
			put_buf(r1_bio);
			md_done_sync(mddev, s, uptodate);
		}
L
Linus Torvalds 已提交
1559 1560 1561
	}
}

1562
static int r1_sync_page_io(struct md_rdev *rdev, sector_t sector,
1563 1564 1565 1566 1567
			    int sectors, struct page *page, int rw)
{
	if (sync_page_io(rdev, sector, sectors << 9, page, rw, false))
		/* success */
		return 1;
1568
	if (rw == WRITE) {
1569
		set_bit(WriteErrorSeen, &rdev->flags);
1570 1571 1572 1573 1574
		if (!test_and_set_bit(WantReplacement,
				      &rdev->flags))
			set_bit(MD_RECOVERY_NEEDED, &
				rdev->mddev->recovery);
	}
1575 1576 1577 1578 1579 1580
	/* need to record an error - either for the block or the device */
	if (!rdev_set_badblocks(rdev, sector, sectors, 0))
		md_error(rdev->mddev, rdev);
	return 0;
}

1581
static int fix_sync_read_error(struct r1bio *r1_bio)
L
Linus Torvalds 已提交
1582
{
1583 1584 1585 1586 1587 1588 1589
	/* Try some synchronous reads of other devices to get
	 * good data, much like with normal read errors.  Only
	 * read into the pages we already have so we don't
	 * need to re-issue the read request.
	 * We don't need to freeze the array, because being in an
	 * active sync request, there is no normal IO, and
	 * no overlapping syncs.
1590 1591 1592
	 * We don't need to check is_badblock() again as we
	 * made sure that anything with a bad block in range
	 * will have bi_end_io clear.
1593
	 */
1594
	struct mddev *mddev = r1_bio->mddev;
1595
	struct r1conf *conf = mddev->private;
1596 1597 1598 1599 1600 1601 1602 1603 1604
	struct bio *bio = r1_bio->bios[r1_bio->read_disk];
	sector_t sect = r1_bio->sector;
	int sectors = r1_bio->sectors;
	int idx = 0;

	while(sectors) {
		int s = sectors;
		int d = r1_bio->read_disk;
		int success = 0;
1605
		struct md_rdev *rdev;
1606
		int start;
1607 1608 1609 1610 1611 1612 1613 1614 1615 1616

		if (s > (PAGE_SIZE>>9))
			s = PAGE_SIZE >> 9;
		do {
			if (r1_bio->bios[d]->bi_end_io == end_sync_read) {
				/* No rcu protection needed here devices
				 * can only be removed when no resync is
				 * active, and resync is currently active
				 */
				rdev = conf->mirrors[d].rdev;
1617
				if (sync_page_io(rdev, sect, s<<9,
1618 1619 1620 1621 1622 1623 1624
						 bio->bi_io_vec[idx].bv_page,
						 READ, false)) {
					success = 1;
					break;
				}
			}
			d++;
1625
			if (d == conf->raid_disks * 2)
1626 1627 1628
				d = 0;
		} while (!success && d != r1_bio->read_disk);

1629
		if (!success) {
1630
			char b[BDEVNAME_SIZE];
1631 1632 1633 1634 1635 1636
			int abort = 0;
			/* Cannot read from anywhere, this block is lost.
			 * Record a bad block on each device.  If that doesn't
			 * work just disable and interrupt the recovery.
			 * Don't fail devices as that won't really help.
			 */
1637 1638 1639 1640 1641
			printk(KERN_ALERT "md/raid1:%s: %s: unrecoverable I/O read error"
			       " for block %llu\n",
			       mdname(mddev),
			       bdevname(bio->bi_bdev, b),
			       (unsigned long long)r1_bio->sector);
1642
			for (d = 0; d < conf->raid_disks * 2; d++) {
1643 1644 1645 1646 1647 1648 1649
				rdev = conf->mirrors[d].rdev;
				if (!rdev || test_bit(Faulty, &rdev->flags))
					continue;
				if (!rdev_set_badblocks(rdev, sect, s, 0))
					abort = 1;
			}
			if (abort) {
1650 1651
				conf->recovery_disabled =
					mddev->recovery_disabled;
1652 1653 1654 1655 1656 1657 1658 1659 1660 1661
				set_bit(MD_RECOVERY_INTR, &mddev->recovery);
				md_done_sync(mddev, r1_bio->sectors, 0);
				put_buf(r1_bio);
				return 0;
			}
			/* Try next page */
			sectors -= s;
			sect += s;
			idx++;
			continue;
1662
		}
1663 1664 1665 1666 1667

		start = d;
		/* write it back and re-read */
		while (d != r1_bio->read_disk) {
			if (d == 0)
1668
				d = conf->raid_disks * 2;
1669 1670 1671 1672
			d--;
			if (r1_bio->bios[d]->bi_end_io != end_sync_read)
				continue;
			rdev = conf->mirrors[d].rdev;
1673 1674 1675
			if (r1_sync_page_io(rdev, sect, s,
					    bio->bi_io_vec[idx].bv_page,
					    WRITE) == 0) {
1676 1677
				r1_bio->bios[d]->bi_end_io = NULL;
				rdev_dec_pending(rdev, mddev);
1678
			}
1679 1680 1681 1682
		}
		d = start;
		while (d != r1_bio->read_disk) {
			if (d == 0)
1683
				d = conf->raid_disks * 2;
1684 1685 1686 1687
			d--;
			if (r1_bio->bios[d]->bi_end_io != end_sync_read)
				continue;
			rdev = conf->mirrors[d].rdev;
1688 1689 1690
			if (r1_sync_page_io(rdev, sect, s,
					    bio->bi_io_vec[idx].bv_page,
					    READ) != 0)
1691
				atomic_add(s, &rdev->corrected_errors);
1692
		}
1693 1694 1695 1696
		sectors -= s;
		sect += s;
		idx ++;
	}
1697
	set_bit(R1BIO_Uptodate, &r1_bio->state);
1698
	set_bit(BIO_UPTODATE, &bio->bi_flags);
1699 1700 1701
	return 1;
}

1702
static int process_checks(struct r1bio *r1_bio)
1703 1704 1705 1706 1707 1708 1709 1710
{
	/* We have read all readable devices.  If we haven't
	 * got the block, then there is no hope left.
	 * If we have, then we want to do a comparison
	 * and skip the write if everything is the same.
	 * If any blocks failed to read, then we need to
	 * attempt an over-write
	 */
1711
	struct mddev *mddev = r1_bio->mddev;
1712
	struct r1conf *conf = mddev->private;
1713 1714 1715
	int primary;
	int i;

1716
	for (primary = 0; primary < conf->raid_disks * 2; primary++)
1717 1718 1719 1720 1721 1722 1723
		if (r1_bio->bios[primary]->bi_end_io == end_sync_read &&
		    test_bit(BIO_UPTODATE, &r1_bio->bios[primary]->bi_flags)) {
			r1_bio->bios[primary]->bi_end_io = NULL;
			rdev_dec_pending(conf->mirrors[primary].rdev, mddev);
			break;
		}
	r1_bio->read_disk = primary;
1724
	for (i = 0; i < conf->raid_disks * 2; i++) {
1725 1726 1727 1728 1729
		int j;
		int vcnt = r1_bio->sectors >> (PAGE_SHIFT- 9);
		struct bio *pbio = r1_bio->bios[primary];
		struct bio *sbio = r1_bio->bios[i];
		int size;
1730

1731 1732 1733 1734 1735 1736 1737 1738 1739 1740
		if (r1_bio->bios[i]->bi_end_io != end_sync_read)
			continue;

		if (test_bit(BIO_UPTODATE, &sbio->bi_flags)) {
			for (j = vcnt; j-- ; ) {
				struct page *p, *s;
				p = pbio->bi_io_vec[j].bv_page;
				s = sbio->bi_io_vec[j].bv_page;
				if (memcmp(page_address(p),
					   page_address(s),
1741
					   sbio->bi_io_vec[j].bv_len))
1742
					break;
1743
			}
1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778
		} else
			j = 0;
		if (j >= 0)
			mddev->resync_mismatches += r1_bio->sectors;
		if (j < 0 || (test_bit(MD_RECOVERY_CHECK, &mddev->recovery)
			      && test_bit(BIO_UPTODATE, &sbio->bi_flags))) {
			/* No need to write to this device. */
			sbio->bi_end_io = NULL;
			rdev_dec_pending(conf->mirrors[i].rdev, mddev);
			continue;
		}
		/* fixup the bio for reuse */
		sbio->bi_vcnt = vcnt;
		sbio->bi_size = r1_bio->sectors << 9;
		sbio->bi_idx = 0;
		sbio->bi_phys_segments = 0;
		sbio->bi_flags &= ~(BIO_POOL_MASK - 1);
		sbio->bi_flags |= 1 << BIO_UPTODATE;
		sbio->bi_next = NULL;
		sbio->bi_sector = r1_bio->sector +
			conf->mirrors[i].rdev->data_offset;
		sbio->bi_bdev = conf->mirrors[i].rdev->bdev;
		size = sbio->bi_size;
		for (j = 0; j < vcnt ; j++) {
			struct bio_vec *bi;
			bi = &sbio->bi_io_vec[j];
			bi->bv_offset = 0;
			if (size > PAGE_SIZE)
				bi->bv_len = PAGE_SIZE;
			else
				bi->bv_len = size;
			size -= PAGE_SIZE;
			memcpy(page_address(bi->bv_page),
			       page_address(pbio->bi_io_vec[j].bv_page),
			       PAGE_SIZE);
1779
		}
1780
	}
1781 1782 1783
	return 0;
}

1784
static void sync_request_write(struct mddev *mddev, struct r1bio *r1_bio)
1785
{
1786
	struct r1conf *conf = mddev->private;
1787
	int i;
1788
	int disks = conf->raid_disks * 2;
1789 1790 1791 1792 1793 1794 1795 1796
	struct bio *bio, *wbio;

	bio = r1_bio->bios[r1_bio->read_disk];

	if (!test_bit(R1BIO_Uptodate, &r1_bio->state))
		/* ouch - failed to read all of that. */
		if (!fix_sync_read_error(r1_bio))
			return;
1797 1798 1799 1800

	if (test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery))
		if (process_checks(r1_bio) < 0)
			return;
1801 1802 1803
	/*
	 * schedule writes
	 */
L
Linus Torvalds 已提交
1804 1805 1806
	atomic_set(&r1_bio->remaining, 1);
	for (i = 0; i < disks ; i++) {
		wbio = r1_bio->bios[i];
1807 1808 1809 1810
		if (wbio->bi_end_io == NULL ||
		    (wbio->bi_end_io == end_sync_read &&
		     (i == r1_bio->read_disk ||
		      !test_bit(MD_RECOVERY_SYNC, &mddev->recovery))))
L
Linus Torvalds 已提交
1811 1812
			continue;

1813 1814
		wbio->bi_rw = WRITE;
		wbio->bi_end_io = end_sync_write;
L
Linus Torvalds 已提交
1815 1816
		atomic_inc(&r1_bio->remaining);
		md_sync_acct(conf->mirrors[i].rdev->bdev, wbio->bi_size >> 9);
1817

L
Linus Torvalds 已提交
1818 1819 1820 1821
		generic_make_request(wbio);
	}

	if (atomic_dec_and_test(&r1_bio->remaining)) {
1822
		/* if we're here, all write(s) have completed, so clean up */
L
Linus Torvalds 已提交
1823 1824 1825 1826 1827 1828 1829 1830 1831 1832
		md_done_sync(mddev, r1_bio->sectors, 1);
		put_buf(r1_bio);
	}
}

/*
 * This is a kernel thread which:
 *
 *	1.	Retries failed read operations on working mirrors.
 *	2.	Updates the raid superblock when problems encounter.
1833
 *	3.	Performs writes following reads for array synchronising.
L
Linus Torvalds 已提交
1834 1835
 */

1836
static void fix_read_error(struct r1conf *conf, int read_disk,
1837 1838
			   sector_t sect, int sectors)
{
1839
	struct mddev *mddev = conf->mddev;
1840 1841 1842 1843 1844
	while(sectors) {
		int s = sectors;
		int d = read_disk;
		int success = 0;
		int start;
1845
		struct md_rdev *rdev;
1846 1847 1848 1849 1850 1851 1852 1853 1854 1855

		if (s > (PAGE_SIZE>>9))
			s = PAGE_SIZE >> 9;

		do {
			/* Note: no rcu protection needed here
			 * as this is synchronous in the raid1d thread
			 * which is the thread that might remove
			 * a device.  If raid1d ever becomes multi-threaded....
			 */
1856 1857 1858
			sector_t first_bad;
			int bad_sectors;

1859 1860 1861
			rdev = conf->mirrors[d].rdev;
			if (rdev &&
			    test_bit(In_sync, &rdev->flags) &&
1862 1863
			    is_badblock(rdev, sect, s,
					&first_bad, &bad_sectors) == 0 &&
J
Jonathan Brassow 已提交
1864 1865
			    sync_page_io(rdev, sect, s<<9,
					 conf->tmppage, READ, false))
1866 1867 1868
				success = 1;
			else {
				d++;
1869
				if (d == conf->raid_disks * 2)
1870 1871 1872 1873 1874
					d = 0;
			}
		} while (!success && d != read_disk);

		if (!success) {
1875
			/* Cannot read from anywhere - mark it bad */
1876
			struct md_rdev *rdev = conf->mirrors[read_disk].rdev;
1877 1878
			if (!rdev_set_badblocks(rdev, sect, s, 0))
				md_error(mddev, rdev);
1879 1880 1881 1882 1883 1884
			break;
		}
		/* write it back and re-read */
		start = d;
		while (d != read_disk) {
			if (d==0)
1885
				d = conf->raid_disks * 2;
1886 1887 1888
			d--;
			rdev = conf->mirrors[d].rdev;
			if (rdev &&
1889 1890 1891
			    test_bit(In_sync, &rdev->flags))
				r1_sync_page_io(rdev, sect, s,
						conf->tmppage, WRITE);
1892 1893 1894 1895 1896
		}
		d = start;
		while (d != read_disk) {
			char b[BDEVNAME_SIZE];
			if (d==0)
1897
				d = conf->raid_disks * 2;
1898 1899 1900 1901
			d--;
			rdev = conf->mirrors[d].rdev;
			if (rdev &&
			    test_bit(In_sync, &rdev->flags)) {
1902 1903
				if (r1_sync_page_io(rdev, sect, s,
						    conf->tmppage, READ)) {
1904 1905
					atomic_add(s, &rdev->corrected_errors);
					printk(KERN_INFO
N
NeilBrown 已提交
1906
					       "md/raid1:%s: read error corrected "
1907 1908
					       "(%d sectors at %llu on %s)\n",
					       mdname(mddev), s,
1909 1910
					       (unsigned long long)(sect +
					           rdev->data_offset),
1911 1912 1913 1914 1915 1916 1917 1918 1919
					       bdevname(rdev->bdev, b));
				}
			}
		}
		sectors -= s;
		sect += s;
	}
}

1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938
static void bi_complete(struct bio *bio, int error)
{
	complete((struct completion *)bio->bi_private);
}

static int submit_bio_wait(int rw, struct bio *bio)
{
	struct completion event;
	rw |= REQ_SYNC;

	init_completion(&event);
	bio->bi_private = &event;
	bio->bi_end_io = bi_complete;
	submit_bio(rw, bio);
	wait_for_completion(&event);

	return test_bit(BIO_UPTODATE, &bio->bi_flags);
}

1939
static int narrow_write_error(struct r1bio *r1_bio, int i)
1940
{
1941
	struct mddev *mddev = r1_bio->mddev;
1942
	struct r1conf *conf = mddev->private;
1943
	struct md_rdev *rdev = conf->mirrors[i].rdev;
1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014
	int vcnt, idx;
	struct bio_vec *vec;

	/* bio has the data to be written to device 'i' where
	 * we just recently had a write error.
	 * We repeatedly clone the bio and trim down to one block,
	 * then try the write.  Where the write fails we record
	 * a bad block.
	 * It is conceivable that the bio doesn't exactly align with
	 * blocks.  We must handle this somehow.
	 *
	 * We currently own a reference on the rdev.
	 */

	int block_sectors;
	sector_t sector;
	int sectors;
	int sect_to_write = r1_bio->sectors;
	int ok = 1;

	if (rdev->badblocks.shift < 0)
		return 0;

	block_sectors = 1 << rdev->badblocks.shift;
	sector = r1_bio->sector;
	sectors = ((sector + block_sectors)
		   & ~(sector_t)(block_sectors - 1))
		- sector;

	if (test_bit(R1BIO_BehindIO, &r1_bio->state)) {
		vcnt = r1_bio->behind_page_count;
		vec = r1_bio->behind_bvecs;
		idx = 0;
		while (vec[idx].bv_page == NULL)
			idx++;
	} else {
		vcnt = r1_bio->master_bio->bi_vcnt;
		vec = r1_bio->master_bio->bi_io_vec;
		idx = r1_bio->master_bio->bi_idx;
	}
	while (sect_to_write) {
		struct bio *wbio;
		if (sectors > sect_to_write)
			sectors = sect_to_write;
		/* Write at 'sector' for 'sectors'*/

		wbio = bio_alloc_mddev(GFP_NOIO, vcnt, mddev);
		memcpy(wbio->bi_io_vec, vec, vcnt * sizeof(struct bio_vec));
		wbio->bi_sector = r1_bio->sector;
		wbio->bi_rw = WRITE;
		wbio->bi_vcnt = vcnt;
		wbio->bi_size = r1_bio->sectors << 9;
		wbio->bi_idx = idx;

		md_trim_bio(wbio, sector - r1_bio->sector, sectors);
		wbio->bi_sector += rdev->data_offset;
		wbio->bi_bdev = rdev->bdev;
		if (submit_bio_wait(WRITE, wbio) == 0)
			/* failure! */
			ok = rdev_set_badblocks(rdev, sector,
						sectors, 0)
				&& ok;

		bio_put(wbio);
		sect_to_write -= sectors;
		sector += sectors;
		sectors = block_sectors;
	}
	return ok;
}

2015
static void handle_sync_write_finished(struct r1conf *conf, struct r1bio *r1_bio)
2016 2017 2018
{
	int m;
	int s = r1_bio->sectors;
2019
	for (m = 0; m < conf->raid_disks * 2 ; m++) {
2020
		struct md_rdev *rdev = conf->mirrors[m].rdev;
2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037
		struct bio *bio = r1_bio->bios[m];
		if (bio->bi_end_io == NULL)
			continue;
		if (test_bit(BIO_UPTODATE, &bio->bi_flags) &&
		    test_bit(R1BIO_MadeGood, &r1_bio->state)) {
			rdev_clear_badblocks(rdev, r1_bio->sector, s);
		}
		if (!test_bit(BIO_UPTODATE, &bio->bi_flags) &&
		    test_bit(R1BIO_WriteError, &r1_bio->state)) {
			if (!rdev_set_badblocks(rdev, r1_bio->sector, s, 0))
				md_error(conf->mddev, rdev);
		}
	}
	put_buf(r1_bio);
	md_done_sync(conf->mddev, s, 1);
}

2038
static void handle_write_finished(struct r1conf *conf, struct r1bio *r1_bio)
2039 2040
{
	int m;
2041
	for (m = 0; m < conf->raid_disks * 2 ; m++)
2042
		if (r1_bio->bios[m] == IO_MADE_GOOD) {
2043
			struct md_rdev *rdev = conf->mirrors[m].rdev;
2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066
			rdev_clear_badblocks(rdev,
					     r1_bio->sector,
					     r1_bio->sectors);
			rdev_dec_pending(rdev, conf->mddev);
		} else if (r1_bio->bios[m] != NULL) {
			/* This drive got a write error.  We need to
			 * narrow down and record precise write
			 * errors.
			 */
			if (!narrow_write_error(r1_bio, m)) {
				md_error(conf->mddev,
					 conf->mirrors[m].rdev);
				/* an I/O failed, we can't clear the bitmap */
				set_bit(R1BIO_Degraded, &r1_bio->state);
			}
			rdev_dec_pending(conf->mirrors[m].rdev,
					 conf->mddev);
		}
	if (test_bit(R1BIO_WriteError, &r1_bio->state))
		close_write(r1_bio);
	raid_end_bio_io(r1_bio);
}

2067
static void handle_read_error(struct r1conf *conf, struct r1bio *r1_bio)
2068 2069 2070
{
	int disk;
	int max_sectors;
2071
	struct mddev *mddev = conf->mddev;
2072 2073
	struct bio *bio;
	char b[BDEVNAME_SIZE];
2074
	struct md_rdev *rdev;
2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156

	clear_bit(R1BIO_ReadError, &r1_bio->state);
	/* we got a read error. Maybe the drive is bad.  Maybe just
	 * the block and we can fix it.
	 * We freeze all other IO, and try reading the block from
	 * other devices.  When we find one, we re-write
	 * and check it that fixes the read error.
	 * This is all done synchronously while the array is
	 * frozen
	 */
	if (mddev->ro == 0) {
		freeze_array(conf);
		fix_read_error(conf, r1_bio->read_disk,
			       r1_bio->sector, r1_bio->sectors);
		unfreeze_array(conf);
	} else
		md_error(mddev, conf->mirrors[r1_bio->read_disk].rdev);

	bio = r1_bio->bios[r1_bio->read_disk];
	bdevname(bio->bi_bdev, b);
read_more:
	disk = read_balance(conf, r1_bio, &max_sectors);
	if (disk == -1) {
		printk(KERN_ALERT "md/raid1:%s: %s: unrecoverable I/O"
		       " read error for block %llu\n",
		       mdname(mddev), b, (unsigned long long)r1_bio->sector);
		raid_end_bio_io(r1_bio);
	} else {
		const unsigned long do_sync
			= r1_bio->master_bio->bi_rw & REQ_SYNC;
		if (bio) {
			r1_bio->bios[r1_bio->read_disk] =
				mddev->ro ? IO_BLOCKED : NULL;
			bio_put(bio);
		}
		r1_bio->read_disk = disk;
		bio = bio_clone_mddev(r1_bio->master_bio, GFP_NOIO, mddev);
		md_trim_bio(bio, r1_bio->sector - bio->bi_sector, max_sectors);
		r1_bio->bios[r1_bio->read_disk] = bio;
		rdev = conf->mirrors[disk].rdev;
		printk_ratelimited(KERN_ERR
				   "md/raid1:%s: redirecting sector %llu"
				   " to other mirror: %s\n",
				   mdname(mddev),
				   (unsigned long long)r1_bio->sector,
				   bdevname(rdev->bdev, b));
		bio->bi_sector = r1_bio->sector + rdev->data_offset;
		bio->bi_bdev = rdev->bdev;
		bio->bi_end_io = raid1_end_read_request;
		bio->bi_rw = READ | do_sync;
		bio->bi_private = r1_bio;
		if (max_sectors < r1_bio->sectors) {
			/* Drat - have to split this up more */
			struct bio *mbio = r1_bio->master_bio;
			int sectors_handled = (r1_bio->sector + max_sectors
					       - mbio->bi_sector);
			r1_bio->sectors = max_sectors;
			spin_lock_irq(&conf->device_lock);
			if (mbio->bi_phys_segments == 0)
				mbio->bi_phys_segments = 2;
			else
				mbio->bi_phys_segments++;
			spin_unlock_irq(&conf->device_lock);
			generic_make_request(bio);
			bio = NULL;

			r1_bio = mempool_alloc(conf->r1bio_pool, GFP_NOIO);

			r1_bio->master_bio = mbio;
			r1_bio->sectors = (mbio->bi_size >> 9)
					  - sectors_handled;
			r1_bio->state = 0;
			set_bit(R1BIO_ReadError, &r1_bio->state);
			r1_bio->mddev = mddev;
			r1_bio->sector = mbio->bi_sector + sectors_handled;

			goto read_more;
		} else
			generic_make_request(bio);
	}
}

2157
static void raid1d(struct mddev *mddev)
L
Linus Torvalds 已提交
2158
{
2159
	struct r1bio *r1_bio;
L
Linus Torvalds 已提交
2160
	unsigned long flags;
2161
	struct r1conf *conf = mddev->private;
L
Linus Torvalds 已提交
2162
	struct list_head *head = &conf->retry_list;
2163
	struct blk_plug plug;
L
Linus Torvalds 已提交
2164 2165

	md_check_recovery(mddev);
2166 2167

	blk_start_plug(&plug);
L
Linus Torvalds 已提交
2168
	for (;;) {
2169

N
NeilBrown 已提交
2170 2171
		if (atomic_read(&mddev->plug_cnt) == 0)
			flush_pending_writes(conf);
2172

2173 2174 2175
		spin_lock_irqsave(&conf->device_lock, flags);
		if (list_empty(head)) {
			spin_unlock_irqrestore(&conf->device_lock, flags);
L
Linus Torvalds 已提交
2176
			break;
2177
		}
2178
		r1_bio = list_entry(head->prev, struct r1bio, retry_list);
L
Linus Torvalds 已提交
2179
		list_del(head->prev);
2180
		conf->nr_queued--;
L
Linus Torvalds 已提交
2181 2182 2183
		spin_unlock_irqrestore(&conf->device_lock, flags);

		mddev = r1_bio->mddev;
2184
		conf = mddev->private;
2185
		if (test_bit(R1BIO_IsSync, &r1_bio->state)) {
2186
			if (test_bit(R1BIO_MadeGood, &r1_bio->state) ||
2187 2188 2189
			    test_bit(R1BIO_WriteError, &r1_bio->state))
				handle_sync_write_finished(conf, r1_bio);
			else
2190
				sync_request_write(mddev, r1_bio);
2191
		} else if (test_bit(R1BIO_MadeGood, &r1_bio->state) ||
2192 2193 2194 2195 2196
			   test_bit(R1BIO_WriteError, &r1_bio->state))
			handle_write_finished(conf, r1_bio);
		else if (test_bit(R1BIO_ReadError, &r1_bio->state))
			handle_read_error(conf, r1_bio);
		else
2197 2198 2199 2200
			/* just a partial read to be scheduled from separate
			 * context
			 */
			generic_make_request(r1_bio->bios[r1_bio->read_disk]);
2201

N
NeilBrown 已提交
2202
		cond_resched();
2203 2204
		if (mddev->flags & ~(1<<MD_CHANGE_PENDING))
			md_check_recovery(mddev);
L
Linus Torvalds 已提交
2205
	}
2206
	blk_finish_plug(&plug);
L
Linus Torvalds 已提交
2207 2208 2209
}


2210
static int init_resync(struct r1conf *conf)
L
Linus Torvalds 已提交
2211 2212 2213 2214
{
	int buffs;

	buffs = RESYNC_WINDOW / RESYNC_BLOCK_SIZE;
2215
	BUG_ON(conf->r1buf_pool);
L
Linus Torvalds 已提交
2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233
	conf->r1buf_pool = mempool_create(buffs, r1buf_pool_alloc, r1buf_pool_free,
					  conf->poolinfo);
	if (!conf->r1buf_pool)
		return -ENOMEM;
	conf->next_resync = 0;
	return 0;
}

/*
 * perform a "sync" on one "block"
 *
 * We need to make sure that no normal I/O request - particularly write
 * requests - conflict with active sync requests.
 *
 * This is achieved by tracking pending requests and a 'barrier' concept
 * that can be installed to exclude normal IO requests.
 */

2234
static sector_t sync_request(struct mddev *mddev, sector_t sector_nr, int *skipped, int go_faster)
L
Linus Torvalds 已提交
2235
{
2236
	struct r1conf *conf = mddev->private;
2237
	struct r1bio *r1_bio;
L
Linus Torvalds 已提交
2238 2239
	struct bio *bio;
	sector_t max_sector, nr_sectors;
2240
	int disk = -1;
L
Linus Torvalds 已提交
2241
	int i;
2242 2243
	int wonly = -1;
	int write_targets = 0, read_targets = 0;
N
NeilBrown 已提交
2244
	sector_t sync_blocks;
2245
	int still_degraded = 0;
2246 2247
	int good_sectors = RESYNC_SECTORS;
	int min_bad = 0; /* number of sectors that are bad in all devices */
L
Linus Torvalds 已提交
2248 2249 2250

	if (!conf->r1buf_pool)
		if (init_resync(conf))
2251
			return 0;
L
Linus Torvalds 已提交
2252

A
Andre Noll 已提交
2253
	max_sector = mddev->dev_sectors;
L
Linus Torvalds 已提交
2254
	if (sector_nr >= max_sector) {
2255 2256 2257 2258 2259
		/* If we aborted, we need to abort the
		 * sync on the 'current' bitmap chunk (there will
		 * only be one in raid1 resync.
		 * We can find the current addess in mddev->curr_resync
		 */
2260 2261
		if (mddev->curr_resync < max_sector) /* aborted */
			bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
2262
						&sync_blocks, 1);
2263
		else /* completed sync */
2264
			conf->fullsync = 0;
2265 2266

		bitmap_close_sync(mddev->bitmap);
L
Linus Torvalds 已提交
2267 2268 2269 2270
		close_sync(conf);
		return 0;
	}

2271 2272
	if (mddev->bitmap == NULL &&
	    mddev->recovery_cp == MaxSector &&
2273
	    !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
2274 2275 2276 2277
	    conf->fullsync == 0) {
		*skipped = 1;
		return max_sector - sector_nr;
	}
2278 2279 2280
	/* before building a request, check if we can skip these blocks..
	 * This call the bitmap_start_sync doesn't actually record anything
	 */
2281
	if (!bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) &&
2282
	    !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) {
2283 2284 2285 2286
		/* We can skip this block, and probably several more */
		*skipped = 1;
		return sync_blocks;
	}
L
Linus Torvalds 已提交
2287
	/*
2288 2289 2290
	 * If there is non-resync activity waiting for a turn,
	 * and resync is going fast enough,
	 * then let it though before starting on this new sync request.
L
Linus Torvalds 已提交
2291
	 */
2292
	if (!go_faster && conf->nr_waiting)
L
Linus Torvalds 已提交
2293
		msleep_interruptible(1000);
2294

N
NeilBrown 已提交
2295
	bitmap_cond_end_sync(mddev->bitmap, sector_nr);
2296
	r1_bio = mempool_alloc(conf->r1buf_pool, GFP_NOIO);
2297 2298 2299
	raise_barrier(conf);

	conf->next_resync = sector_nr;
L
Linus Torvalds 已提交
2300

2301
	rcu_read_lock();
L
Linus Torvalds 已提交
2302
	/*
2303 2304 2305 2306 2307 2308
	 * If we get a correctably read error during resync or recovery,
	 * we might want to read from a different device.  So we
	 * flag all drives that could conceivably be read from for READ,
	 * and any others (which will be non-In_sync devices) for WRITE.
	 * If a read fails, we try reading from something else for which READ
	 * is OK.
L
Linus Torvalds 已提交
2309 2310 2311 2312
	 */

	r1_bio->mddev = mddev;
	r1_bio->sector = sector_nr;
2313
	r1_bio->state = 0;
L
Linus Torvalds 已提交
2314 2315
	set_bit(R1BIO_IsSync, &r1_bio->state);

2316
	for (i = 0; i < conf->raid_disks * 2; i++) {
2317
		struct md_rdev *rdev;
L
Linus Torvalds 已提交
2318 2319 2320 2321
		bio = r1_bio->bios[i];

		/* take from bio_init */
		bio->bi_next = NULL;
2322
		bio->bi_flags &= ~(BIO_POOL_MASK-1);
L
Linus Torvalds 已提交
2323
		bio->bi_flags |= 1 << BIO_UPTODATE;
2324
		bio->bi_rw = READ;
L
Linus Torvalds 已提交
2325 2326 2327 2328 2329 2330 2331
		bio->bi_vcnt = 0;
		bio->bi_idx = 0;
		bio->bi_phys_segments = 0;
		bio->bi_size = 0;
		bio->bi_end_io = NULL;
		bio->bi_private = NULL;

2332 2333
		rdev = rcu_dereference(conf->mirrors[i].rdev);
		if (rdev == NULL ||
2334
		    test_bit(Faulty, &rdev->flags)) {
2335 2336
			if (i < conf->raid_disks)
				still_degraded = 1;
2337
		} else if (!test_bit(In_sync, &rdev->flags)) {
L
Linus Torvalds 已提交
2338 2339 2340
			bio->bi_rw = WRITE;
			bio->bi_end_io = end_sync_write;
			write_targets ++;
2341 2342
		} else {
			/* may need to read from here */
2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367
			sector_t first_bad = MaxSector;
			int bad_sectors;

			if (is_badblock(rdev, sector_nr, good_sectors,
					&first_bad, &bad_sectors)) {
				if (first_bad > sector_nr)
					good_sectors = first_bad - sector_nr;
				else {
					bad_sectors -= (sector_nr - first_bad);
					if (min_bad == 0 ||
					    min_bad > bad_sectors)
						min_bad = bad_sectors;
				}
			}
			if (sector_nr < first_bad) {
				if (test_bit(WriteMostly, &rdev->flags)) {
					if (wonly < 0)
						wonly = i;
				} else {
					if (disk < 0)
						disk = i;
				}
				bio->bi_rw = READ;
				bio->bi_end_io = end_sync_read;
				read_targets++;
2368 2369
			}
		}
2370 2371 2372 2373 2374 2375
		if (bio->bi_end_io) {
			atomic_inc(&rdev->nr_pending);
			bio->bi_sector = sector_nr + rdev->data_offset;
			bio->bi_bdev = rdev->bdev;
			bio->bi_private = r1_bio;
		}
L
Linus Torvalds 已提交
2376
	}
2377 2378 2379 2380
	rcu_read_unlock();
	if (disk < 0)
		disk = wonly;
	r1_bio->read_disk = disk;
2381

2382 2383 2384 2385 2386
	if (read_targets == 0 && min_bad > 0) {
		/* These sectors are bad on all InSync devices, so we
		 * need to mark them bad on all write targets
		 */
		int ok = 1;
2387
		for (i = 0 ; i < conf->raid_disks * 2 ; i++)
2388
			if (r1_bio->bios[i]->bi_end_io == end_sync_write) {
2389
				struct md_rdev *rdev = conf->mirrors[i].rdev;
2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416
				ok = rdev_set_badblocks(rdev, sector_nr,
							min_bad, 0
					) && ok;
			}
		set_bit(MD_CHANGE_DEVS, &mddev->flags);
		*skipped = 1;
		put_buf(r1_bio);

		if (!ok) {
			/* Cannot record the badblocks, so need to
			 * abort the resync.
			 * If there are multiple read targets, could just
			 * fail the really bad ones ???
			 */
			conf->recovery_disabled = mddev->recovery_disabled;
			set_bit(MD_RECOVERY_INTR, &mddev->recovery);
			return 0;
		} else
			return min_bad;

	}
	if (min_bad > 0 && min_bad < good_sectors) {
		/* only resync enough to reach the next bad->good
		 * transition */
		good_sectors = min_bad;
	}

2417 2418 2419 2420 2421
	if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) && read_targets > 0)
		/* extra read targets are also write targets */
		write_targets += read_targets-1;

	if (write_targets == 0 || read_targets == 0) {
L
Linus Torvalds 已提交
2422 2423 2424
		/* There is nowhere to write, so all non-sync
		 * drives must be failed - so we are finished
		 */
2425 2426
		sector_t rv = max_sector - sector_nr;
		*skipped = 1;
L
Linus Torvalds 已提交
2427 2428 2429 2430
		put_buf(r1_bio);
		return rv;
	}

2431 2432
	if (max_sector > mddev->resync_max)
		max_sector = mddev->resync_max; /* Don't do IO beyond here */
2433 2434
	if (max_sector > sector_nr + good_sectors)
		max_sector = sector_nr + good_sectors;
L
Linus Torvalds 已提交
2435
	nr_sectors = 0;
2436
	sync_blocks = 0;
L
Linus Torvalds 已提交
2437 2438 2439 2440 2441 2442 2443
	do {
		struct page *page;
		int len = PAGE_SIZE;
		if (sector_nr + (len>>9) > max_sector)
			len = (max_sector - sector_nr) << 9;
		if (len == 0)
			break;
2444 2445
		if (sync_blocks == 0) {
			if (!bitmap_start_sync(mddev->bitmap, sector_nr,
2446 2447 2448
					       &sync_blocks, still_degraded) &&
			    !conf->fullsync &&
			    !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery))
2449
				break;
2450
			BUG_ON(sync_blocks < (PAGE_SIZE>>9));
2451
			if ((len >> 9) > sync_blocks)
2452
				len = sync_blocks<<9;
2453
		}
2454

2455
		for (i = 0 ; i < conf->raid_disks * 2; i++) {
L
Linus Torvalds 已提交
2456 2457
			bio = r1_bio->bios[i];
			if (bio->bi_end_io) {
2458
				page = bio->bi_io_vec[bio->bi_vcnt].bv_page;
L
Linus Torvalds 已提交
2459 2460
				if (bio_add_page(bio, page, len, 0) == 0) {
					/* stop here */
2461
					bio->bi_io_vec[bio->bi_vcnt].bv_page = page;
L
Linus Torvalds 已提交
2462 2463 2464
					while (i > 0) {
						i--;
						bio = r1_bio->bios[i];
2465 2466
						if (bio->bi_end_io==NULL)
							continue;
L
Linus Torvalds 已提交
2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477
						/* remove last page from this bio */
						bio->bi_vcnt--;
						bio->bi_size -= len;
						bio->bi_flags &= ~(1<< BIO_SEG_VALID);
					}
					goto bio_full;
				}
			}
		}
		nr_sectors += len>>9;
		sector_nr += len>>9;
2478
		sync_blocks -= (len>>9);
L
Linus Torvalds 已提交
2479 2480 2481 2482
	} while (r1_bio->bios[disk]->bi_vcnt < RESYNC_PAGES);
 bio_full:
	r1_bio->sectors = nr_sectors;

2483 2484 2485 2486 2487
	/* For a user-requested sync, we read all readable devices and do a
	 * compare
	 */
	if (test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) {
		atomic_set(&r1_bio->remaining, read_targets);
2488
		for (i = 0; i < conf->raid_disks * 2; i++) {
2489 2490
			bio = r1_bio->bios[i];
			if (bio->bi_end_io == end_sync_read) {
2491
				md_sync_acct(bio->bi_bdev, nr_sectors);
2492 2493 2494 2495 2496 2497
				generic_make_request(bio);
			}
		}
	} else {
		atomic_set(&r1_bio->remaining, 1);
		bio = r1_bio->bios[r1_bio->read_disk];
2498
		md_sync_acct(bio->bi_bdev, nr_sectors);
2499
		generic_make_request(bio);
L
Linus Torvalds 已提交
2500

2501
	}
L
Linus Torvalds 已提交
2502 2503 2504
	return nr_sectors;
}

2505
static sector_t raid1_size(struct mddev *mddev, sector_t sectors, int raid_disks)
2506 2507 2508 2509 2510 2511 2512
{
	if (sectors)
		return sectors;

	return mddev->dev_sectors;
}

2513
static struct r1conf *setup_conf(struct mddev *mddev)
L
Linus Torvalds 已提交
2514
{
2515
	struct r1conf *conf;
2516
	int i;
2517
	struct mirror_info *disk;
2518
	struct md_rdev *rdev;
2519
	int err = -ENOMEM;
L
Linus Torvalds 已提交
2520

2521
	conf = kzalloc(sizeof(struct r1conf), GFP_KERNEL);
L
Linus Torvalds 已提交
2522
	if (!conf)
2523
		goto abort;
L
Linus Torvalds 已提交
2524

2525 2526
	conf->mirrors = kzalloc(sizeof(struct mirror_info)
				* mddev->raid_disks * 2,
L
Linus Torvalds 已提交
2527 2528
				 GFP_KERNEL);
	if (!conf->mirrors)
2529
		goto abort;
L
Linus Torvalds 已提交
2530

2531 2532
	conf->tmppage = alloc_page(GFP_KERNEL);
	if (!conf->tmppage)
2533
		goto abort;
2534

2535
	conf->poolinfo = kzalloc(sizeof(*conf->poolinfo), GFP_KERNEL);
L
Linus Torvalds 已提交
2536
	if (!conf->poolinfo)
2537
		goto abort;
2538
	conf->poolinfo->raid_disks = mddev->raid_disks * 2;
L
Linus Torvalds 已提交
2539 2540 2541 2542
	conf->r1bio_pool = mempool_create(NR_RAID1_BIOS, r1bio_pool_alloc,
					  r1bio_pool_free,
					  conf->poolinfo);
	if (!conf->r1bio_pool)
2543 2544
		goto abort;

2545
	conf->poolinfo->mddev = mddev;
L
Linus Torvalds 已提交
2546

2547
	err = -EINVAL;
2548
	spin_lock_init(&conf->device_lock);
N
NeilBrown 已提交
2549
	rdev_for_each(rdev, mddev) {
2550
		int disk_idx = rdev->raid_disk;
L
Linus Torvalds 已提交
2551 2552 2553
		if (disk_idx >= mddev->raid_disks
		    || disk_idx < 0)
			continue;
2554 2555 2556 2557
		if (test_bit(Replacement, &rdev->flags))
			disk = conf->mirrors + conf->raid_disks + disk_idx;
		else
			disk = conf->mirrors + disk_idx;
L
Linus Torvalds 已提交
2558

2559 2560
		if (disk->rdev)
			goto abort;
L
Linus Torvalds 已提交
2561 2562 2563 2564 2565 2566 2567 2568 2569
		disk->rdev = rdev;

		disk->head_position = 0;
	}
	conf->raid_disks = mddev->raid_disks;
	conf->mddev = mddev;
	INIT_LIST_HEAD(&conf->retry_list);

	spin_lock_init(&conf->resync_lock);
2570
	init_waitqueue_head(&conf->wait_barrier);
L
Linus Torvalds 已提交
2571

2572
	bio_list_init(&conf->pending_bio_list);
2573
	conf->pending_count = 0;
2574
	conf->recovery_disabled = mddev->recovery_disabled - 1;
2575

2576
	err = -EIO;
2577
	conf->last_used = -1;
2578
	for (i = 0; i < conf->raid_disks * 2; i++) {
L
Linus Torvalds 已提交
2579 2580 2581

		disk = conf->mirrors + i;

2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596
		if (i < conf->raid_disks &&
		    disk[conf->raid_disks].rdev) {
			/* This slot has a replacement. */
			if (!disk->rdev) {
				/* No original, just make the replacement
				 * a recovering spare
				 */
				disk->rdev =
					disk[conf->raid_disks].rdev;
				disk[conf->raid_disks].rdev = NULL;
			} else if (!test_bit(In_sync, &disk->rdev->flags))
				/* Original is not in_sync - bad */
				goto abort;
		}

2597 2598
		if (!disk->rdev ||
		    !test_bit(In_sync, &disk->rdev->flags)) {
L
Linus Torvalds 已提交
2599
			disk->head_position = 0;
2600 2601
			if (disk->rdev)
				conf->fullsync = 1;
2602 2603 2604 2605 2606 2607
		} else if (conf->last_used < 0)
			/*
			 * The first working device is used as a
			 * starting point to read balancing.
			 */
			conf->last_used = i;
L
Linus Torvalds 已提交
2608
	}
2609 2610

	if (conf->last_used < 0) {
N
NeilBrown 已提交
2611
		printk(KERN_ERR "md/raid1:%s: no operational mirrors\n",
2612 2613 2614 2615 2616 2617 2618
		       mdname(mddev));
		goto abort;
	}
	err = -ENOMEM;
	conf->thread = md_register_thread(raid1d, mddev, NULL);
	if (!conf->thread) {
		printk(KERN_ERR
N
NeilBrown 已提交
2619
		       "md/raid1:%s: couldn't allocate thread\n",
2620 2621
		       mdname(mddev));
		goto abort;
2622
	}
L
Linus Torvalds 已提交
2623

2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637
	return conf;

 abort:
	if (conf) {
		if (conf->r1bio_pool)
			mempool_destroy(conf->r1bio_pool);
		kfree(conf->mirrors);
		safe_put_page(conf->tmppage);
		kfree(conf->poolinfo);
		kfree(conf);
	}
	return ERR_PTR(err);
}

2638
static int stop(struct mddev *mddev);
2639
static int run(struct mddev *mddev)
2640
{
2641
	struct r1conf *conf;
2642
	int i;
2643
	struct md_rdev *rdev;
2644
	int ret;
2645 2646

	if (mddev->level != 1) {
N
NeilBrown 已提交
2647
		printk(KERN_ERR "md/raid1:%s: raid level not set to mirroring (%d)\n",
2648 2649 2650 2651
		       mdname(mddev), mddev->level);
		return -EIO;
	}
	if (mddev->reshape_position != MaxSector) {
N
NeilBrown 已提交
2652
		printk(KERN_ERR "md/raid1:%s: reshape_position set but not supported\n",
2653 2654 2655
		       mdname(mddev));
		return -EIO;
	}
L
Linus Torvalds 已提交
2656
	/*
2657 2658 2659
	 * copy the already verified devices into our private RAID1
	 * bookkeeping area. [whatever we allocate in run(),
	 * should be freed in stop()]
L
Linus Torvalds 已提交
2660
	 */
2661 2662 2663 2664
	if (mddev->private == NULL)
		conf = setup_conf(mddev);
	else
		conf = mddev->private;
L
Linus Torvalds 已提交
2665

2666 2667
	if (IS_ERR(conf))
		return PTR_ERR(conf);
L
Linus Torvalds 已提交
2668

N
NeilBrown 已提交
2669
	rdev_for_each(rdev, mddev) {
2670 2671
		if (!mddev->gendisk)
			continue;
2672 2673
		disk_stack_limits(mddev->gendisk, rdev->bdev,
				  rdev->data_offset << 9);
L
Linus Torvalds 已提交
2674
	}
2675

2676 2677 2678 2679 2680 2681 2682 2683 2684 2685
	mddev->degraded = 0;
	for (i=0; i < conf->raid_disks; i++)
		if (conf->mirrors[i].rdev == NULL ||
		    !test_bit(In_sync, &conf->mirrors[i].rdev->flags) ||
		    test_bit(Faulty, &conf->mirrors[i].rdev->flags))
			mddev->degraded++;

	if (conf->raid_disks - mddev->degraded == 1)
		mddev->recovery_cp = MaxSector;

2686
	if (mddev->recovery_cp != MaxSector)
N
NeilBrown 已提交
2687
		printk(KERN_NOTICE "md/raid1:%s: not clean"
2688 2689
		       " -- starting background reconstruction\n",
		       mdname(mddev));
L
Linus Torvalds 已提交
2690
	printk(KERN_INFO 
N
NeilBrown 已提交
2691
		"md/raid1:%s: active with %d out of %d mirrors\n",
L
Linus Torvalds 已提交
2692 2693
		mdname(mddev), mddev->raid_disks - mddev->degraded, 
		mddev->raid_disks);
2694

L
Linus Torvalds 已提交
2695 2696 2697
	/*
	 * Ok, everything is just fine now
	 */
2698 2699 2700 2701
	mddev->thread = conf->thread;
	conf->thread = NULL;
	mddev->private = conf;

2702
	md_set_array_sectors(mddev, raid1_size(mddev, 0, 0));
L
Linus Torvalds 已提交
2703

2704 2705 2706
	if (mddev->queue) {
		mddev->queue->backing_dev_info.congested_fn = raid1_congested;
		mddev->queue->backing_dev_info.congested_data = mddev;
2707
		blk_queue_merge_bvec(mddev->queue, raid1_mergeable_bvec);
2708
	}
2709 2710 2711 2712 2713

	ret =  md_integrity_register(mddev);
	if (ret)
		stop(mddev);
	return ret;
L
Linus Torvalds 已提交
2714 2715
}

2716
static int stop(struct mddev *mddev)
L
Linus Torvalds 已提交
2717
{
2718
	struct r1conf *conf = mddev->private;
2719 2720 2721
	struct bitmap *bitmap = mddev->bitmap;

	/* wait for behind writes to complete */
2722
	if (bitmap && atomic_read(&bitmap->behind_writes) > 0) {
N
NeilBrown 已提交
2723 2724
		printk(KERN_INFO "md/raid1:%s: behind writes in progress - waiting to stop.\n",
		       mdname(mddev));
2725
		/* need to kick something here to make sure I/O goes? */
2726 2727
		wait_event(bitmap->behind_wait,
			   atomic_read(&bitmap->behind_writes) == 0);
2728
	}
L
Linus Torvalds 已提交
2729

2730 2731 2732
	raise_barrier(conf);
	lower_barrier(conf);

2733
	md_unregister_thread(&mddev->thread);
L
Linus Torvalds 已提交
2734 2735
	if (conf->r1bio_pool)
		mempool_destroy(conf->r1bio_pool);
2736 2737
	kfree(conf->mirrors);
	kfree(conf->poolinfo);
L
Linus Torvalds 已提交
2738 2739 2740 2741 2742
	kfree(conf);
	mddev->private = NULL;
	return 0;
}

2743
static int raid1_resize(struct mddev *mddev, sector_t sectors)
L
Linus Torvalds 已提交
2744 2745 2746 2747 2748 2749 2750 2751
{
	/* no resync is happening, and there is enough space
	 * on all devices, so we can resize.
	 * We need to make sure resync covers any new space.
	 * If the array is shrinking we should possibly wait until
	 * any io in the removed space completes, but it hardly seems
	 * worth it.
	 */
2752
	md_set_array_sectors(mddev, raid1_size(mddev, sectors, 0));
D
Dan Williams 已提交
2753 2754
	if (mddev->array_sectors > raid1_size(mddev, sectors, 0))
		return -EINVAL;
2755
	set_capacity(mddev->gendisk, mddev->array_sectors);
2756
	revalidate_disk(mddev->gendisk);
D
Dan Williams 已提交
2757
	if (sectors > mddev->dev_sectors &&
2758
	    mddev->recovery_cp > mddev->dev_sectors) {
A
Andre Noll 已提交
2759
		mddev->recovery_cp = mddev->dev_sectors;
L
Linus Torvalds 已提交
2760 2761
		set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
	}
D
Dan Williams 已提交
2762
	mddev->dev_sectors = sectors;
2763
	mddev->resync_max_sectors = sectors;
L
Linus Torvalds 已提交
2764 2765 2766
	return 0;
}

2767
static int raid1_reshape(struct mddev *mddev)
L
Linus Torvalds 已提交
2768 2769 2770 2771 2772 2773 2774 2775
{
	/* We need to:
	 * 1/ resize the r1bio_pool
	 * 2/ resize conf->mirrors
	 *
	 * We allocate a new r1bio_pool if we can.
	 * Then raise a device barrier and wait until all IO stops.
	 * Then resize conf->mirrors and swap in the new r1bio pool.
2776 2777 2778
	 *
	 * At the same time, we "pack" the devices so that all the missing
	 * devices have the higher raid_disk numbers.
L
Linus Torvalds 已提交
2779 2780 2781
	 */
	mempool_t *newpool, *oldpool;
	struct pool_info *newpoolinfo;
2782
	struct mirror_info *newmirrors;
2783
	struct r1conf *conf = mddev->private;
2784
	int cnt, raid_disks;
2785
	unsigned long flags;
2786
	int d, d2, err;
L
Linus Torvalds 已提交
2787

2788
	/* Cannot change chunk_size, layout, or level */
2789
	if (mddev->chunk_sectors != mddev->new_chunk_sectors ||
2790 2791
	    mddev->layout != mddev->new_layout ||
	    mddev->level != mddev->new_level) {
2792
		mddev->new_chunk_sectors = mddev->chunk_sectors;
2793 2794 2795 2796 2797
		mddev->new_layout = mddev->layout;
		mddev->new_level = mddev->level;
		return -EINVAL;
	}

2798 2799 2800
	err = md_allow_write(mddev);
	if (err)
		return err;
2801

2802 2803
	raid_disks = mddev->raid_disks + mddev->delta_disks;

2804 2805 2806 2807 2808 2809
	if (raid_disks < conf->raid_disks) {
		cnt=0;
		for (d= 0; d < conf->raid_disks; d++)
			if (conf->mirrors[d].rdev)
				cnt++;
		if (cnt > raid_disks)
L
Linus Torvalds 已提交
2810
			return -EBUSY;
2811
	}
L
Linus Torvalds 已提交
2812 2813 2814 2815 2816

	newpoolinfo = kmalloc(sizeof(*newpoolinfo), GFP_KERNEL);
	if (!newpoolinfo)
		return -ENOMEM;
	newpoolinfo->mddev = mddev;
2817
	newpoolinfo->raid_disks = raid_disks * 2;
L
Linus Torvalds 已提交
2818 2819 2820 2821 2822 2823 2824

	newpool = mempool_create(NR_RAID1_BIOS, r1bio_pool_alloc,
				 r1bio_pool_free, newpoolinfo);
	if (!newpool) {
		kfree(newpoolinfo);
		return -ENOMEM;
	}
2825 2826
	newmirrors = kzalloc(sizeof(struct mirror_info) * raid_disks * 2,
			     GFP_KERNEL);
L
Linus Torvalds 已提交
2827 2828 2829 2830 2831 2832
	if (!newmirrors) {
		kfree(newpoolinfo);
		mempool_destroy(newpool);
		return -ENOMEM;
	}

2833
	raise_barrier(conf);
L
Linus Torvalds 已提交
2834 2835 2836 2837

	/* ok, everything is stopped */
	oldpool = conf->r1bio_pool;
	conf->r1bio_pool = newpool;
2838

2839
	for (d = d2 = 0; d < conf->raid_disks; d++) {
2840
		struct md_rdev *rdev = conf->mirrors[d].rdev;
2841
		if (rdev && rdev->raid_disk != d2) {
2842
			sysfs_unlink_rdev(mddev, rdev);
2843
			rdev->raid_disk = d2;
2844 2845
			sysfs_unlink_rdev(mddev, rdev);
			if (sysfs_link_rdev(mddev, rdev))
2846
				printk(KERN_WARNING
2847 2848
				       "md/raid1:%s: cannot register rd%d\n",
				       mdname(mddev), rdev->raid_disk);
2849
		}
2850 2851 2852
		if (rdev)
			newmirrors[d2++].rdev = rdev;
	}
L
Linus Torvalds 已提交
2853 2854 2855 2856 2857
	kfree(conf->mirrors);
	conf->mirrors = newmirrors;
	kfree(conf->poolinfo);
	conf->poolinfo = newpoolinfo;

2858
	spin_lock_irqsave(&conf->device_lock, flags);
L
Linus Torvalds 已提交
2859
	mddev->degraded += (raid_disks - conf->raid_disks);
2860
	spin_unlock_irqrestore(&conf->device_lock, flags);
L
Linus Torvalds 已提交
2861
	conf->raid_disks = mddev->raid_disks = raid_disks;
2862
	mddev->delta_disks = 0;
L
Linus Torvalds 已提交
2863

2864
	conf->last_used = 0; /* just make sure it is in-range */
2865
	lower_barrier(conf);
L
Linus Torvalds 已提交
2866 2867 2868 2869 2870 2871 2872 2873

	set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
	md_wakeup_thread(mddev->thread);

	mempool_destroy(oldpool);
	return 0;
}

2874
static void raid1_quiesce(struct mddev *mddev, int state)
2875
{
2876
	struct r1conf *conf = mddev->private;
2877 2878

	switch(state) {
2879 2880 2881
	case 2: /* wake for suspend */
		wake_up(&conf->wait_barrier);
		break;
2882
	case 1:
2883
		raise_barrier(conf);
2884
		break;
2885
	case 0:
2886
		lower_barrier(conf);
2887 2888 2889 2890
		break;
	}
}

2891
static void *raid1_takeover(struct mddev *mddev)
2892 2893 2894 2895 2896
{
	/* raid1 can take over:
	 *  raid5 with 2 devices, any layout or chunk size
	 */
	if (mddev->level == 5 && mddev->raid_disks == 2) {
2897
		struct r1conf *conf;
2898 2899 2900 2901 2902 2903 2904 2905 2906 2907
		mddev->new_level = 1;
		mddev->new_layout = 0;
		mddev->new_chunk_sectors = 0;
		conf = setup_conf(mddev);
		if (!IS_ERR(conf))
			conf->barrier = 1;
		return conf;
	}
	return ERR_PTR(-EINVAL);
}
L
Linus Torvalds 已提交
2908

2909
static struct md_personality raid1_personality =
L
Linus Torvalds 已提交
2910 2911
{
	.name		= "raid1",
2912
	.level		= 1,
L
Linus Torvalds 已提交
2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923
	.owner		= THIS_MODULE,
	.make_request	= make_request,
	.run		= run,
	.stop		= stop,
	.status		= status,
	.error_handler	= error,
	.hot_add_disk	= raid1_add_disk,
	.hot_remove_disk= raid1_remove_disk,
	.spare_active	= raid1_spare_active,
	.sync_request	= sync_request,
	.resize		= raid1_resize,
2924
	.size		= raid1_size,
2925
	.check_reshape	= raid1_reshape,
2926
	.quiesce	= raid1_quiesce,
2927
	.takeover	= raid1_takeover,
L
Linus Torvalds 已提交
2928 2929 2930 2931
};

static int __init raid_init(void)
{
2932
	return register_md_personality(&raid1_personality);
L
Linus Torvalds 已提交
2933 2934 2935 2936
}

static void raid_exit(void)
{
2937
	unregister_md_personality(&raid1_personality);
L
Linus Torvalds 已提交
2938 2939 2940 2941 2942
}

module_init(raid_init);
module_exit(raid_exit);
MODULE_LICENSE("GPL");
2943
MODULE_DESCRIPTION("RAID1 (mirroring) personality for MD");
L
Linus Torvalds 已提交
2944
MODULE_ALIAS("md-personality-3"); /* RAID1 */
2945
MODULE_ALIAS("md-raid1");
2946
MODULE_ALIAS("md-level-1");
2947 2948

module_param(max_queued_requests, int, S_IRUGO|S_IWUSR);