bitmap.c 59.9 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
/*
 * bitmap.c two-level bitmap (C) Peter T. Breuer (ptb@ot.uc3m.es) 2003
 *
 * bitmap_create  - sets up the bitmap structure
 * bitmap_destroy - destroys the bitmap structure
 *
 * additions, Copyright (C) 2003-2004, Paul Clements, SteelEye Technology, Inc.:
 * - added disk storage for bitmap
 * - changes to allow various bitmap chunk sizes
 */

/*
 * Still to do:
 *
 * flush after percent set rather than just time based. (maybe both).
 */

18
#include <linux/blkdev.h>
19 20 21 22 23 24 25 26 27 28
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/timer.h>
#include <linux/sched.h>
#include <linux/list.h>
#include <linux/file.h>
#include <linux/mount.h>
#include <linux/buffer_head.h>
29
#include <linux/seq_file.h>
30
#include "md.h"
31
#include "bitmap.h"
32

33
static inline char *bmname(struct bitmap *bitmap)
34 35 36 37 38 39 40 41 42 43 44 45 46 47
{
	return bitmap->mddev ? mdname(bitmap->mddev) : "mdX";
}

/*
 * check a page and, if necessary, allocate it (or hijack it if the alloc fails)
 *
 * 1) check to see if this page is allocated, if it's not then try to alloc
 * 2) if the alloc fails, set the page's hijacked flag so we'll use the
 *    page pointer directly as a counter
 *
 * if we find our page, we increment the page's refcount so that it stays
 * allocated while we're using it
 */
48
static int bitmap_checkpage(struct bitmap_counts *bitmap,
49
			    unsigned long page, int create)
50 51
__releases(bitmap->lock)
__acquires(bitmap->lock)
52 53 54 55
{
	unsigned char *mappage;

	if (page >= bitmap->pages) {
56 57 58 59
		/* This can happen if bitmap_start_sync goes beyond
		 * End-of-device while looking for a whole page.
		 * It is harmless.
		 */
60 61 62 63 64 65 66 67 68 69 70 71 72 73
		return -EINVAL;
	}

	if (bitmap->bp[page].hijacked) /* it's hijacked, don't try to alloc */
		return 0;

	if (bitmap->bp[page].map) /* page is already allocated, just return */
		return 0;

	if (!create)
		return -ENOENT;

	/* this page has not been allocated yet */

74
	spin_unlock_irq(&bitmap->lock);
75 76 77 78 79 80 81 82 83 84 85 86 87
	/* It is possible that this is being called inside a
	 * prepare_to_wait/finish_wait loop from raid5c:make_request().
	 * In general it is not permitted to sleep in that context as it
	 * can cause the loop to spin freely.
	 * That doesn't apply here as we can only reach this point
	 * once with any loop.
	 * When this function completes, either bp[page].map or
	 * bp[page].hijacked.  In either case, this function will
	 * abort before getting to this point again.  So there is
	 * no risk of a free-spin, and so it is safe to assert
	 * that sleeping here is allowed.
	 */
	sched_annotate_sleep();
88
	mappage = kzalloc(PAGE_SIZE, GFP_NOIO);
89 90 91
	spin_lock_irq(&bitmap->lock);

	if (mappage == NULL) {
92
		pr_debug("md/bitmap: map page allocation failed, hijacking\n");
93 94 95 96
		/* failed - set the hijacked flag so that we can use the
		 * pointer as a counter */
		if (!bitmap->bp[page].map)
			bitmap->bp[page].hijacked = 1;
97 98
	} else if (bitmap->bp[page].map ||
		   bitmap->bp[page].hijacked) {
99
		/* somebody beat us to getting the page */
100
		kfree(mappage);
101
		return 0;
102
	} else {
103

104
		/* no page was in place and we have one, so install it */
105

106 107 108
		bitmap->bp[page].map = mappage;
		bitmap->missing_pages--;
	}
109 110 111 112 113 114
	return 0;
}

/* if page is completely empty, put it back on the free list, or dealloc it */
/* if page was hijacked, unmark the flag so it might get alloced next time */
/* Note: lock should be held when calling this */
115
static void bitmap_checkfree(struct bitmap_counts *bitmap, unsigned long page)
116 117 118 119 120 121 122 123 124 125 126
{
	char *ptr;

	if (bitmap->bp[page].count) /* page is still busy */
		return;

	/* page is no longer in use, it can be released */

	if (bitmap->bp[page].hijacked) { /* page was hijacked, undo this now */
		bitmap->bp[page].hijacked = 0;
		bitmap->bp[page].map = NULL;
127 128 129 130 131
	} else {
		/* normal case, free the page */
		ptr = bitmap->bp[page].map;
		bitmap->bp[page].map = NULL;
		bitmap->missing_pages++;
132
		kfree(ptr);
133 134 135 136 137 138 139 140 141 142 143
	}
}

/*
 * bitmap file handling - read and write the bitmap file and its superblock
 */

/*
 * basic page I/O operations
 */

144
/* IO operations when bitmap is stored near all superblocks */
145 146 147
static int read_sb_page(struct mddev *mddev, loff_t offset,
			struct page *page,
			unsigned long index, int size)
148 149 150
{
	/* choose a good rdev and read the page from there */

151
	struct md_rdev *rdev;
152 153
	sector_t target;

N
NeilBrown 已提交
154
	rdev_for_each(rdev, mddev) {
155 156
		if (! test_bit(In_sync, &rdev->flags)
		    || test_bit(Faulty, &rdev->flags))
157 158
			continue;

J
Jonathan Brassow 已提交
159
		target = offset + index * (PAGE_SIZE/512);
160

161
		if (sync_page_io(rdev, target,
162
				 roundup(size, bdev_logical_block_size(rdev->bdev)),
J
Jonathan Brassow 已提交
163
				 page, READ, true)) {
164
			page->index = index;
165
			return 0;
166 167
		}
	}
168
	return -EIO;
169 170
}

171
static struct md_rdev *next_active_rdev(struct md_rdev *rdev, struct mddev *mddev)
172 173 174 175 176 177 178
{
	/* Iterate the disks of an mddev, using rcu to protect access to the
	 * linked list, and raising the refcount of devices we return to ensure
	 * they don't disappear while in use.
	 * As devices are only added or removed when raid_disk is < 0 and
	 * nr_pending is 0 and In_sync is clear, the entries we return will
	 * still be in the same position on the list when we re-enter
179
	 * list_for_each_entry_continue_rcu.
180 181 182 183
	 */
	rcu_read_lock();
	if (rdev == NULL)
		/* start at the beginning */
184
		rdev = list_entry_rcu(&mddev->disks, struct md_rdev, same_set);
185 186 187 188
	else {
		/* release the previous rdev and start from there. */
		rdev_dec_pending(rdev, mddev);
	}
189
	list_for_each_entry_continue_rcu(rdev, &mddev->disks, same_set) {
190 191 192 193 194 195 196 197 198 199 200 201
		if (rdev->raid_disk >= 0 &&
		    !test_bit(Faulty, &rdev->flags)) {
			/* this is a usable devices */
			atomic_inc(&rdev->nr_pending);
			rcu_read_unlock();
			return rdev;
		}
	}
	rcu_read_unlock();
	return NULL;
}

202
static int write_sb_page(struct bitmap *bitmap, struct page *page, int wait)
203
{
204
	struct md_rdev *rdev = NULL;
205
	struct block_device *bdev;
206
	struct mddev *mddev = bitmap->mddev;
207
	struct bitmap_storage *store = &bitmap->storage;
208

209
	while ((rdev = next_active_rdev(rdev, mddev)) != NULL) {
210 211
		int size = PAGE_SIZE;
		loff_t offset = mddev->bitmap_info.offset;
212 213 214

		bdev = (rdev->meta_bdev) ? rdev->meta_bdev : rdev->bdev;

215 216 217 218 219
		if (page->index == store->file_pages-1) {
			int last_page_size = store->bytes & (PAGE_SIZE-1);
			if (last_page_size == 0)
				last_page_size = PAGE_SIZE;
			size = roundup(last_page_size,
220
				       bdev_logical_block_size(bdev));
221
		}
222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261
		/* Just make sure we aren't corrupting data or
		 * metadata
		 */
		if (mddev->external) {
			/* Bitmap could be anywhere. */
			if (rdev->sb_start + offset + (page->index
						       * (PAGE_SIZE/512))
			    > rdev->data_offset
			    &&
			    rdev->sb_start + offset
			    < (rdev->data_offset + mddev->dev_sectors
			     + (PAGE_SIZE/512)))
				goto bad_alignment;
		} else if (offset < 0) {
			/* DATA  BITMAP METADATA  */
			if (offset
			    + (long)(page->index * (PAGE_SIZE/512))
			    + size/512 > 0)
				/* bitmap runs in to metadata */
				goto bad_alignment;
			if (rdev->data_offset + mddev->dev_sectors
			    > rdev->sb_start + offset)
				/* data runs in to bitmap */
				goto bad_alignment;
		} else if (rdev->sb_start < rdev->data_offset) {
			/* METADATA BITMAP DATA */
			if (rdev->sb_start
			    + offset
			    + page->index*(PAGE_SIZE/512) + size/512
			    > rdev->data_offset)
				/* bitmap runs in to data */
				goto bad_alignment;
		} else {
			/* DATA METADATA BITMAP - no problems */
		}
		md_super_write(mddev, rdev,
			       rdev->sb_start + offset
			       + page->index * (PAGE_SIZE/512),
			       size,
			       page);
262
	}
263 264

	if (wait)
265
		md_super_wait(mddev);
266
	return 0;
267 268 269

 bad_alignment:
	return -EINVAL;
270 271
}

272
static void bitmap_file_kick(struct bitmap *bitmap);
273
/*
274
 * write out a page to a file
275
 */
276
static void write_page(struct bitmap *bitmap, struct page *page, int wait)
277
{
278
	struct buffer_head *bh;
279

280
	if (bitmap->storage.file == NULL) {
281 282
		switch (write_sb_page(bitmap, page, wait)) {
		case -EINVAL:
283
			set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);
284
		}
285
	} else {
286

287
		bh = page_buffers(page);
288

289 290 291 292
		while (bh && bh->b_blocknr) {
			atomic_inc(&bitmap->pending_writes);
			set_buffer_locked(bh);
			set_buffer_mapped(bh);
J
Jens Axboe 已提交
293
			submit_bh(WRITE | REQ_SYNC, bh);
294 295
			bh = bh->b_this_page;
		}
296

297
		if (wait)
298 299
			wait_event(bitmap->write_wait,
				   atomic_read(&bitmap->pending_writes)==0);
300
	}
301
	if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
302
		bitmap_file_kick(bitmap);
303 304 305 306 307
}

static void end_bitmap_write(struct buffer_head *bh, int uptodate)
{
	struct bitmap *bitmap = bh->b_private;
308

309 310
	if (!uptodate)
		set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);
311 312 313
	if (atomic_dec_and_test(&bitmap->pending_writes))
		wake_up(&bitmap->write_wait);
}
314

315 316 317 318 319 320 321 322 323 324
/* copied from buffer.c */
static void
__clear_page_buffers(struct page *page)
{
	ClearPagePrivate(page);
	set_page_private(page, 0);
	page_cache_release(page);
}
static void free_buffers(struct page *page)
{
325
	struct buffer_head *bh;
326

327 328 329 330
	if (!PagePrivate(page))
		return;

	bh = page_buffers(page);
331 332 333 334
	while (bh) {
		struct buffer_head *next = bh->b_this_page;
		free_buffer_head(bh);
		bh = next;
335
	}
336 337
	__clear_page_buffers(page);
	put_page(page);
338 339
}

340 341 342 343 344 345 346
/* read a page from a file.
 * We both read the page, and attach buffers to the page to record the
 * address of each block (using bmap).  These addresses will be used
 * to write the block later, completely bypassing the filesystem.
 * This usage is similar to how swap files are handled, and allows us
 * to write to a file with no concerns of memory allocation failing.
 */
347 348 349 350
static int read_page(struct file *file, unsigned long index,
		     struct bitmap *bitmap,
		     unsigned long count,
		     struct page *page)
351
{
352
	int ret = 0;
A
Al Viro 已提交
353
	struct inode *inode = file_inode(file);
354 355
	struct buffer_head *bh;
	sector_t block;
356

357 358
	pr_debug("read bitmap file (%dB @ %llu)\n", (int)PAGE_SIZE,
		 (unsigned long long)index << PAGE_SHIFT);
359

360 361
	bh = alloc_page_buffers(page, 1<<inode->i_blkbits, 0);
	if (!bh) {
362
		ret = -ENOMEM;
363 364
		goto out;
	}
365 366 367 368 369 370 371 372 373
	attach_page_buffers(page, bh);
	block = index << (PAGE_SHIFT - inode->i_blkbits);
	while (bh) {
		if (count == 0)
			bh->b_blocknr = 0;
		else {
			bh->b_blocknr = bmap(inode, block);
			if (bh->b_blocknr == 0) {
				/* Cannot use this file! */
374
				ret = -EINVAL;
375 376 377 378 379 380 381 382 383 384
				goto out;
			}
			bh->b_bdev = inode->i_sb->s_bdev;
			if (count < (1<<inode->i_blkbits))
				count = 0;
			else
				count -= (1<<inode->i_blkbits);

			bh->b_end_io = end_bitmap_write;
			bh->b_private = bitmap;
385 386 387 388
			atomic_inc(&bitmap->pending_writes);
			set_buffer_locked(bh);
			set_buffer_mapped(bh);
			submit_bh(READ, bh);
389 390 391 392 393
		}
		block++;
		bh = bh->b_this_page;
	}
	page->index = index;
394 395 396

	wait_event(bitmap->write_wait,
		   atomic_read(&bitmap->pending_writes)==0);
397
	if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
398
		ret = -EIO;
399
out:
400 401
	if (ret)
		printk(KERN_ALERT "md: bitmap read error: (%dB @ %llu): %d\n",
402 403
			(int)PAGE_SIZE,
			(unsigned long long)index << PAGE_SHIFT,
404 405
			ret);
	return ret;
406 407 408 409 410 411 412
}

/*
 * bitmap file superblock operations
 */

/* update the event counter and sync the superblock to disk */
413
void bitmap_update_sb(struct bitmap *bitmap)
414 415 416 417
{
	bitmap_super_t *sb;

	if (!bitmap || !bitmap->mddev) /* no bitmap for this array */
418
		return;
419 420
	if (bitmap->mddev->bitmap_info.external)
		return;
421
	if (!bitmap->storage.sb_page) /* no superblock */
422
		return;
423
	sb = kmap_atomic(bitmap->storage.sb_page);
424
	sb->events = cpu_to_le64(bitmap->mddev->events);
425
	if (bitmap->mddev->events < bitmap->events_cleared)
426 427
		/* rocking back to read-only */
		bitmap->events_cleared = bitmap->mddev->events;
428 429
	sb->events_cleared = cpu_to_le64(bitmap->events_cleared);
	sb->state = cpu_to_le32(bitmap->flags);
430 431 432
	/* Just in case these have been changed via sysfs: */
	sb->daemon_sleep = cpu_to_le32(bitmap->mddev->bitmap_info.daemon_sleep/HZ);
	sb->write_behind = cpu_to_le32(bitmap->mddev->bitmap_info.max_write_behind);
433 434 435
	/* This might have been changed by a reshape */
	sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
	sb->chunksize = cpu_to_le32(bitmap->mddev->bitmap_info.chunksize);
436 437
	sb->sectors_reserved = cpu_to_le32(bitmap->mddev->
					   bitmap_info.space);
438
	kunmap_atomic(sb);
439
	write_page(bitmap, bitmap->storage.sb_page, 1);
440 441 442 443 444 445 446
}

/* print out the bitmap file superblock */
void bitmap_print_sb(struct bitmap *bitmap)
{
	bitmap_super_t *sb;

447
	if (!bitmap || !bitmap->storage.sb_page)
448
		return;
449
	sb = kmap_atomic(bitmap->storage.sb_page);
450
	printk(KERN_DEBUG "%s: bitmap file superblock:\n", bmname(bitmap));
451 452 453
	printk(KERN_DEBUG "         magic: %08x\n", le32_to_cpu(sb->magic));
	printk(KERN_DEBUG "       version: %d\n", le32_to_cpu(sb->version));
	printk(KERN_DEBUG "          uuid: %08x.%08x.%08x.%08x\n",
454 455 456 457
					*(__u32 *)(sb->uuid+0),
					*(__u32 *)(sb->uuid+4),
					*(__u32 *)(sb->uuid+8),
					*(__u32 *)(sb->uuid+12));
458
	printk(KERN_DEBUG "        events: %llu\n",
459
			(unsigned long long) le64_to_cpu(sb->events));
460
	printk(KERN_DEBUG "events cleared: %llu\n",
461
			(unsigned long long) le64_to_cpu(sb->events_cleared));
462 463 464 465 466
	printk(KERN_DEBUG "         state: %08x\n", le32_to_cpu(sb->state));
	printk(KERN_DEBUG "     chunksize: %d B\n", le32_to_cpu(sb->chunksize));
	printk(KERN_DEBUG "  daemon sleep: %ds\n", le32_to_cpu(sb->daemon_sleep));
	printk(KERN_DEBUG "     sync size: %llu KB\n",
			(unsigned long long)le64_to_cpu(sb->sync_size)/2);
467
	printk(KERN_DEBUG "max write behind: %d\n", le32_to_cpu(sb->write_behind));
468
	kunmap_atomic(sb);
469 470
}

471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486
/*
 * bitmap_new_disk_sb
 * @bitmap
 *
 * This function is somewhat the reverse of bitmap_read_sb.  bitmap_read_sb
 * reads and verifies the on-disk bitmap superblock and populates bitmap_info.
 * This function verifies 'bitmap_info' and populates the on-disk bitmap
 * structure, which is to be written to disk.
 *
 * Returns: 0 on success, -Exxx on error
 */
static int bitmap_new_disk_sb(struct bitmap *bitmap)
{
	bitmap_super_t *sb;
	unsigned long chunksize, daemon_sleep, write_behind;

487
	bitmap->storage.sb_page = alloc_page(GFP_KERNEL);
488 489
	if (bitmap->storage.sb_page == NULL)
		return -ENOMEM;
490
	bitmap->storage.sb_page->index = 0;
491

492
	sb = kmap_atomic(bitmap->storage.sb_page);
493 494 495 496 497 498 499

	sb->magic = cpu_to_le32(BITMAP_MAGIC);
	sb->version = cpu_to_le32(BITMAP_MAJOR_HI);

	chunksize = bitmap->mddev->bitmap_info.chunksize;
	BUG_ON(!chunksize);
	if (!is_power_of_2(chunksize)) {
500
		kunmap_atomic(sb);
501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529
		printk(KERN_ERR "bitmap chunksize not a power of 2\n");
		return -EINVAL;
	}
	sb->chunksize = cpu_to_le32(chunksize);

	daemon_sleep = bitmap->mddev->bitmap_info.daemon_sleep;
	if (!daemon_sleep ||
	    (daemon_sleep < 1) || (daemon_sleep > MAX_SCHEDULE_TIMEOUT)) {
		printk(KERN_INFO "Choosing daemon_sleep default (5 sec)\n");
		daemon_sleep = 5 * HZ;
	}
	sb->daemon_sleep = cpu_to_le32(daemon_sleep);
	bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;

	/*
	 * FIXME: write_behind for RAID1.  If not specified, what
	 * is a good choice?  We choose COUNTER_MAX / 2 arbitrarily.
	 */
	write_behind = bitmap->mddev->bitmap_info.max_write_behind;
	if (write_behind > COUNTER_MAX)
		write_behind = COUNTER_MAX / 2;
	sb->write_behind = cpu_to_le32(write_behind);
	bitmap->mddev->bitmap_info.max_write_behind = write_behind;

	/* keep the array size field of the bitmap superblock up to date */
	sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);

	memcpy(sb->uuid, bitmap->mddev->uuid, 16);

530
	set_bit(BITMAP_STALE, &bitmap->flags);
531
	sb->state = cpu_to_le32(bitmap->flags);
532 533 534
	bitmap->events_cleared = bitmap->mddev->events;
	sb->events_cleared = cpu_to_le64(bitmap->mddev->events);

535
	kunmap_atomic(sb);
536 537 538 539

	return 0;
}

540 541 542 543 544
/* read the superblock from the bitmap file and initialize some bitmap fields */
static int bitmap_read_sb(struct bitmap *bitmap)
{
	char *reason = NULL;
	bitmap_super_t *sb;
545
	unsigned long chunksize, daemon_sleep, write_behind;
546
	unsigned long long events;
547
	unsigned long sectors_reserved = 0;
548
	int err = -EINVAL;
549
	struct page *sb_page;
550

551
	if (!bitmap->storage.file && !bitmap->mddev->bitmap_info.offset) {
552 553 554
		chunksize = 128 * 1024 * 1024;
		daemon_sleep = 5 * HZ;
		write_behind = 0;
555
		set_bit(BITMAP_STALE, &bitmap->flags);
556 557 558
		err = 0;
		goto out_no_sb;
	}
559
	/* page 0 is the superblock, read it... */
560 561 562
	sb_page = alloc_page(GFP_KERNEL);
	if (!sb_page)
		return -ENOMEM;
563
	bitmap->storage.sb_page = sb_page;
564

565 566
	if (bitmap->storage.file) {
		loff_t isize = i_size_read(bitmap->storage.file->f_mapping->host);
567 568
		int bytes = isize > PAGE_SIZE ? PAGE_SIZE : isize;

569
		err = read_page(bitmap->storage.file, 0,
570
				bitmap, bytes, sb_page);
571
	} else {
572 573 574 575
		err = read_sb_page(bitmap->mddev,
				   bitmap->mddev->bitmap_info.offset,
				   sb_page,
				   0, sizeof(bitmap_super_t));
576
	}
577
	if (err)
578 579
		return err;

580
	sb = kmap_atomic(sb_page);
581 582

	chunksize = le32_to_cpu(sb->chunksize);
583
	daemon_sleep = le32_to_cpu(sb->daemon_sleep) * HZ;
584
	write_behind = le32_to_cpu(sb->write_behind);
585
	sectors_reserved = le32_to_cpu(sb->sectors_reserved);
586 587 588 589

	/* verify that the bitmap-specific fields are valid */
	if (sb->magic != cpu_to_le32(BITMAP_MAGIC))
		reason = "bad magic";
590 591
	else if (le32_to_cpu(sb->version) < BITMAP_MAJOR_LO ||
		 le32_to_cpu(sb->version) > BITMAP_MAJOR_HI)
592
		reason = "unrecognized superblock version";
593
	else if (chunksize < 512)
594
		reason = "bitmap chunksize too small";
J
Jonathan Brassow 已提交
595
	else if (!is_power_of_2(chunksize))
596
		reason = "bitmap chunksize not a power of 2";
597
	else if (daemon_sleep < 1 || daemon_sleep > MAX_SCHEDULE_TIMEOUT)
598
		reason = "daemon sleep period out of range";
599 600
	else if (write_behind > COUNTER_MAX)
		reason = "write-behind limit out of range (0 - 16383)";
601 602 603 604 605 606 607 608 609
	if (reason) {
		printk(KERN_INFO "%s: invalid bitmap file superblock: %s\n",
			bmname(bitmap), reason);
		goto out;
	}

	/* keep the array size field of the bitmap superblock up to date */
	sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);

610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627
	if (bitmap->mddev->persistent) {
		/*
		 * We have a persistent array superblock, so compare the
		 * bitmap's UUID and event counter to the mddev's
		 */
		if (memcmp(sb->uuid, bitmap->mddev->uuid, 16)) {
			printk(KERN_INFO
			       "%s: bitmap superblock UUID mismatch\n",
			       bmname(bitmap));
			goto out;
		}
		events = le64_to_cpu(sb->events);
		if (events < bitmap->mddev->events) {
			printk(KERN_INFO
			       "%s: bitmap file is out of date (%llu < %llu) "
			       "-- forcing full recovery\n",
			       bmname(bitmap), events,
			       (unsigned long long) bitmap->mddev->events);
628
			set_bit(BITMAP_STALE, &bitmap->flags);
629
		}
630
	}
631

632
	/* assign fields using values from superblock */
633
	bitmap->flags |= le32_to_cpu(sb->state);
634
	if (le32_to_cpu(sb->version) == BITMAP_MAJOR_HOSTENDIAN)
635
		set_bit(BITMAP_HOSTENDIAN, &bitmap->flags);
636 637 638
	bitmap->events_cleared = le64_to_cpu(sb->events_cleared);
	err = 0;
out:
639
	kunmap_atomic(sb);
640
out_no_sb:
641
	if (test_bit(BITMAP_STALE, &bitmap->flags))
642 643 644 645
		bitmap->events_cleared = bitmap->mddev->events;
	bitmap->mddev->bitmap_info.chunksize = chunksize;
	bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
	bitmap->mddev->bitmap_info.max_write_behind = write_behind;
646 647 648
	if (bitmap->mddev->bitmap_info.space == 0 ||
	    bitmap->mddev->bitmap_info.space > sectors_reserved)
		bitmap->mddev->bitmap_info.space = sectors_reserved;
649 650 651 652 653 654 655 656 657
	if (err)
		bitmap_print_sb(bitmap);
	return err;
}

/*
 * general bitmap file operations
 */

658 659 660 661 662 663
/*
 * on-disk bitmap:
 *
 * Use one bit per "chunk" (block set). We do the disk I/O on the bitmap
 * file a page at a time. There's a superblock at the start of the file.
 */
664
/* calculate the index of the page that contains this bit */
665 666
static inline unsigned long file_page_index(struct bitmap_storage *store,
					    unsigned long chunk)
667
{
668
	if (store->sb_page)
669 670
		chunk += sizeof(bitmap_super_t) << 3;
	return chunk >> PAGE_BIT_SHIFT;
671 672 673
}

/* calculate the (bit) offset of this bit within a page */
674 675
static inline unsigned long file_page_offset(struct bitmap_storage *store,
					     unsigned long chunk)
676
{
677
	if (store->sb_page)
678 679
		chunk += sizeof(bitmap_super_t) << 3;
	return chunk & (PAGE_BITS - 1);
680 681 682 683 684 685
}

/*
 * return a pointer to the page in the filemap that contains the given bit
 *
 */
686
static inline struct page *filemap_get_page(struct bitmap_storage *store,
687
					    unsigned long chunk)
688
{
689
	if (file_page_index(store, chunk) >= store->file_pages)
690
		return NULL;
691
	return store->filemap[file_page_index(store, chunk)];
692 693
}

694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712
static int bitmap_storage_alloc(struct bitmap_storage *store,
				unsigned long chunks, int with_super)
{
	int pnum;
	unsigned long num_pages;
	unsigned long bytes;

	bytes = DIV_ROUND_UP(chunks, 8);
	if (with_super)
		bytes += sizeof(bitmap_super_t);

	num_pages = DIV_ROUND_UP(bytes, PAGE_SIZE);

	store->filemap = kmalloc(sizeof(struct page *)
				 * num_pages, GFP_KERNEL);
	if (!store->filemap)
		return -ENOMEM;

	if (with_super && !store->sb_page) {
713
		store->sb_page = alloc_page(GFP_KERNEL|__GFP_ZERO);
714 715 716 717 718 719 720 721 722 723
		if (store->sb_page == NULL)
			return -ENOMEM;
		store->sb_page->index = 0;
	}
	pnum = 0;
	if (store->sb_page) {
		store->filemap[0] = store->sb_page;
		pnum = 1;
	}
	for ( ; pnum < num_pages; pnum++) {
724
		store->filemap[pnum] = alloc_page(GFP_KERNEL|__GFP_ZERO);
725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745
		if (!store->filemap[pnum]) {
			store->file_pages = pnum;
			return -ENOMEM;
		}
		store->filemap[pnum]->index = pnum;
	}
	store->file_pages = pnum;

	/* We need 4 bits per page, rounded up to a multiple
	 * of sizeof(unsigned long) */
	store->filemap_attr = kzalloc(
		roundup(DIV_ROUND_UP(num_pages*4, 8), sizeof(unsigned long)),
		GFP_KERNEL);
	if (!store->filemap_attr)
		return -ENOMEM;

	store->bytes = bytes;

	return 0;
}

746
static void bitmap_file_unmap(struct bitmap_storage *store)
747 748 749
{
	struct page **map, *sb_page;
	int pages;
750
	struct file *file;
751

752
	file = store->file;
753 754 755
	map = store->filemap;
	pages = store->file_pages;
	sb_page = store->sb_page;
756 757

	while (pages--)
758
		if (map[pages] != sb_page) /* 0 is sb_page, release it below */
759
			free_buffers(map[pages]);
760
	kfree(map);
761
	kfree(store->filemap_attr);
762

763 764
	if (sb_page)
		free_buffers(sb_page);
765

766
	if (file) {
A
Al Viro 已提交
767
		struct inode *inode = file_inode(file);
768
		invalidate_mapping_pages(inode->i_mapping, 0, -1);
769
		fput(file);
770
	}
771 772 773 774 775 776 777 778 779 780 781
}

/*
 * bitmap_file_kick - if an error occurs while manipulating the bitmap file
 * then it is no longer reliable, so we stop using it and we mark the file
 * as failed in the superblock
 */
static void bitmap_file_kick(struct bitmap *bitmap)
{
	char *path, *ptr = NULL;

782
	if (!test_and_set_bit(BITMAP_STALE, &bitmap->flags)) {
783
		bitmap_update_sb(bitmap);
784

785
		if (bitmap->storage.file) {
786 787
			path = kmalloc(PAGE_SIZE, GFP_KERNEL);
			if (path)
788 789
				ptr = d_path(&bitmap->storage.file->f_path,
					     path, PAGE_SIZE);
C
Christoph Hellwig 已提交
790

791 792
			printk(KERN_ALERT
			      "%s: kicking failed bitmap file %s from array!\n",
C
Christoph Hellwig 已提交
793
			      bmname(bitmap), IS_ERR(ptr) ? "" : ptr);
794

795 796 797 798 799
			kfree(path);
		} else
			printk(KERN_ALERT
			       "%s: disabling internal bitmap due to errors\n",
			       bmname(bitmap));
800
	}
801 802 803
}

enum bitmap_page_attr {
804
	BITMAP_PAGE_DIRTY = 0,     /* there are set bits that need to be synced */
805 806
	BITMAP_PAGE_PENDING = 1,   /* there are bits that are being cleaned.
				    * i.e. counter is 1 or 2. */
807
	BITMAP_PAGE_NEEDWRITE = 2, /* there are cleared bits that need to be synced */
808 809
};

810 811
static inline void set_page_attr(struct bitmap *bitmap, int pnum,
				 enum bitmap_page_attr attr)
812
{
813
	set_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
814 815
}

816 817
static inline void clear_page_attr(struct bitmap *bitmap, int pnum,
				   enum bitmap_page_attr attr)
818
{
819
	clear_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
820 821
}

822 823
static inline int test_page_attr(struct bitmap *bitmap, int pnum,
				 enum bitmap_page_attr attr)
824
{
825
	return test_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
826 827
}

828 829 830 831 832 833
static inline int test_and_clear_page_attr(struct bitmap *bitmap, int pnum,
					   enum bitmap_page_attr attr)
{
	return test_and_clear_bit((pnum<<2) + attr,
				  bitmap->storage.filemap_attr);
}
834 835 836 837 838 839 840 841 842 843
/*
 * bitmap_file_set_bit -- called before performing a write to the md device
 * to set (and eventually sync) a particular bit in the bitmap file
 *
 * we set the bit immediately, then we record the page number so that
 * when an unplug occurs, we can flush the dirty pages out to disk
 */
static void bitmap_file_set_bit(struct bitmap *bitmap, sector_t block)
{
	unsigned long bit;
844
	struct page *page;
845
	void *kaddr;
846
	unsigned long chunk = block >> bitmap->counts.chunkshift;
847

848
	page = filemap_get_page(&bitmap->storage, chunk);
849 850
	if (!page)
		return;
851
	bit = file_page_offset(&bitmap->storage, chunk);
852

853
	/* set the bit */
854
	kaddr = kmap_atomic(page);
855
	if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
856 857
		set_bit(bit, kaddr);
	else
858
		set_bit_le(bit, kaddr);
859
	kunmap_atomic(kaddr);
860
	pr_debug("set file bit %lu page %lu\n", bit, page->index);
861
	/* record page number so it gets flushed to disk when unplug occurs */
862
	set_page_attr(bitmap, page->index, BITMAP_PAGE_DIRTY);
863 864
}

865 866 867 868 869
static void bitmap_file_clear_bit(struct bitmap *bitmap, sector_t block)
{
	unsigned long bit;
	struct page *page;
	void *paddr;
870
	unsigned long chunk = block >> bitmap->counts.chunkshift;
871

872
	page = filemap_get_page(&bitmap->storage, chunk);
873 874
	if (!page)
		return;
875
	bit = file_page_offset(&bitmap->storage, chunk);
876
	paddr = kmap_atomic(page);
877
	if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
878 879
		clear_bit(bit, paddr);
	else
880
		clear_bit_le(bit, paddr);
881
	kunmap_atomic(paddr);
882 883
	if (!test_page_attr(bitmap, page->index, BITMAP_PAGE_NEEDWRITE)) {
		set_page_attr(bitmap, page->index, BITMAP_PAGE_PENDING);
884 885 886 887
		bitmap->allclean = 0;
	}
}

888 889 890
/* this gets called when the md device is ready to unplug its underlying
 * (slave) device queues -- before we let any writes go down, we need to
 * sync the dirty pages of the bitmap file to disk */
891
void bitmap_unplug(struct bitmap *bitmap)
892
{
893
	unsigned long i;
894
	int dirty, need_write;
895

896 897
	if (!bitmap || !bitmap->storage.filemap ||
	    test_bit(BITMAP_STALE, &bitmap->flags))
898
		return;
899 900 901

	/* look at each page to see if there are any set bits that need to be
	 * flushed out to disk */
902
	for (i = 0; i < bitmap->storage.file_pages; i++) {
903
		if (!bitmap->storage.filemap)
904
			return;
905 906 907 908
		dirty = test_and_clear_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
		need_write = test_and_clear_page_attr(bitmap, i,
						      BITMAP_PAGE_NEEDWRITE);
		if (dirty || need_write) {
909
			clear_page_attr(bitmap, i, BITMAP_PAGE_PENDING);
910 911
			write_page(bitmap, bitmap->storage.filemap[i], 0);
		}
912
	}
913 914 915 916 917 918
	if (bitmap->storage.file)
		wait_event(bitmap->write_wait,
			   atomic_read(&bitmap->pending_writes)==0);
	else
		md_super_wait(bitmap->mddev);

919
	if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
920
		bitmap_file_kick(bitmap);
921
}
922
EXPORT_SYMBOL(bitmap_unplug);
923

924
static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed);
925 926 927 928 929 930 931
/* * bitmap_init_from_disk -- called at bitmap_create time to initialize
 * the in-memory bitmap from the on-disk bitmap -- also, sets up the
 * memory mapping of the bitmap file
 * Special cases:
 *   if there's no bitmap file, or if the bitmap file had been
 *   previously kicked from the array, we mark all the bits as
 *   1's in order to cause a full resync.
932 933 934
 *
 * We ignore all bits for sectors that end earlier than 'start'.
 * This is used when reading an out-of-date bitmap...
935
 */
936
static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
937 938
{
	unsigned long i, chunks, index, oldindex, bit;
939
	struct page *page = NULL;
940
	unsigned long bit_cnt = 0;
941
	struct file *file;
942
	unsigned long offset;
943 944
	int outofdate;
	int ret = -ENOSPC;
945
	void *paddr;
946
	struct bitmap_storage *store = &bitmap->storage;
947

948
	chunks = bitmap->counts.chunks;
949
	file = store->file;
950

951 952
	if (!file && !bitmap->mddev->bitmap_info.offset) {
		/* No permanent bitmap - fill with '1s'. */
953 954
		store->filemap = NULL;
		store->file_pages = 0;
955 956
		for (i = 0; i < chunks ; i++) {
			/* if the disk bit is set, set the memory bit */
957
			int needed = ((sector_t)(i+1) << (bitmap->counts.chunkshift)
958 959
				      >= start);
			bitmap_set_memory_bits(bitmap,
960
					       (sector_t)i << bitmap->counts.chunkshift,
961 962 963 964
					       needed);
		}
		return 0;
	}
965

966
	outofdate = test_bit(BITMAP_STALE, &bitmap->flags);
967 968 969 970
	if (outofdate)
		printk(KERN_INFO "%s: bitmap file is out of date, doing full "
			"recovery\n", bmname(bitmap));

971
	if (file && i_size_read(file->f_mapping->host) < store->bytes) {
972
		printk(KERN_INFO "%s: bitmap file too short %lu < %lu\n",
973 974 975
		       bmname(bitmap),
		       (unsigned long) i_size_read(file->f_mapping->host),
		       store->bytes);
976
		goto err;
977
	}
978

979
	oldindex = ~0L;
980
	offset = 0;
981
	if (!bitmap->mddev->bitmap_info.external)
982
		offset = sizeof(bitmap_super_t);
983 984

	for (i = 0; i < chunks; i++) {
985
		int b;
986 987
		index = file_page_index(&bitmap->storage, i);
		bit = file_page_offset(&bitmap->storage, i);
988
		if (index != oldindex) { /* this is a new page, read it in */
989
			int count;
990
			/* unmap the old page, we're done with it */
991 992
			if (index == store->file_pages-1)
				count = store->bytes - index * PAGE_SIZE;
993 994
			else
				count = PAGE_SIZE;
995
			page = store->filemap[index];
996 997 998 999 1000 1001 1002 1003 1004 1005 1006
			if (file)
				ret = read_page(file, index, bitmap,
						count, page);
			else
				ret = read_sb_page(
					bitmap->mddev,
					bitmap->mddev->bitmap_info.offset,
					page,
					index, count);

			if (ret)
1007
				goto err;
1008

1009 1010 1011 1012 1013
			oldindex = index;

			if (outofdate) {
				/*
				 * if bitmap is out of date, dirty the
1014
				 * whole page and write it out
1015
				 */
1016
				paddr = kmap_atomic(page);
1017
				memset(paddr + offset, 0xff,
1018
				       PAGE_SIZE - offset);
1019
				kunmap_atomic(paddr);
1020 1021 1022
				write_page(bitmap, page, 1);

				ret = -EIO;
1023 1024
				if (test_bit(BITMAP_WRITE_ERROR,
					     &bitmap->flags))
1025
					goto err;
1026 1027
			}
		}
1028
		paddr = kmap_atomic(page);
1029
		if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
1030
			b = test_bit(bit, paddr);
1031
		else
A
Akinobu Mita 已提交
1032
			b = test_bit_le(bit, paddr);
1033
		kunmap_atomic(paddr);
1034
		if (b) {
1035
			/* if the disk bit is set, set the memory bit */
1036
			int needed = ((sector_t)(i+1) << bitmap->counts.chunkshift
1037 1038
				      >= start);
			bitmap_set_memory_bits(bitmap,
1039
					       (sector_t)i << bitmap->counts.chunkshift,
1040
					       needed);
1041 1042
			bit_cnt++;
		}
1043
		offset = 0;
1044 1045 1046
	}

	printk(KERN_INFO "%s: bitmap initialized from disk: "
1047
	       "read %lu pages, set %lu of %lu bits\n",
1048
	       bmname(bitmap), store->file_pages,
1049
	       bit_cnt, chunks);
1050 1051

	return 0;
1052

1053 1054 1055
 err:
	printk(KERN_INFO "%s: bitmap initialisation failed: %d\n",
	       bmname(bitmap), ret);
1056 1057 1058
	return ret;
}

1059 1060 1061 1062 1063
void bitmap_write_all(struct bitmap *bitmap)
{
	/* We don't actually write all bitmap blocks here,
	 * just flag them as needing to be written
	 */
1064
	int i;
1065

1066
	if (!bitmap || !bitmap->storage.filemap)
1067
		return;
1068
	if (bitmap->storage.file)
1069 1070 1071
		/* Only one copy, so nothing needed */
		return;

1072
	for (i = 0; i < bitmap->storage.file_pages; i++)
1073
		set_page_attr(bitmap, i,
1074
			      BITMAP_PAGE_NEEDWRITE);
1075
	bitmap->allclean = 0;
1076 1077
}

1078 1079
static void bitmap_count_page(struct bitmap_counts *bitmap,
			      sector_t offset, int inc)
1080
{
1081
	sector_t chunk = offset >> bitmap->chunkshift;
1082 1083 1084 1085
	unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
	bitmap->bp[page].count += inc;
	bitmap_checkfree(bitmap, page);
}
1086

1087
static void bitmap_set_pending(struct bitmap_counts *bitmap, sector_t offset)
1088 1089 1090 1091 1092 1093 1094 1095 1096
{
	sector_t chunk = offset >> bitmap->chunkshift;
	unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
	struct bitmap_page *bp = &bitmap->bp[page];

	if (!bp->pending)
		bp->pending = 1;
}

1097
static bitmap_counter_t *bitmap_get_counter(struct bitmap_counts *bitmap,
N
NeilBrown 已提交
1098
					    sector_t offset, sector_t *blocks,
1099 1100 1101 1102 1103 1104 1105
					    int create);

/*
 * bitmap daemon -- periodically wakes up to clean bits and flush pages
 *			out to disk
 */

1106
void bitmap_daemon_work(struct mddev *mddev)
1107
{
1108
	struct bitmap *bitmap;
1109
	unsigned long j;
1110
	unsigned long nextpage;
N
NeilBrown 已提交
1111
	sector_t blocks;
1112
	struct bitmap_counts *counts;
1113

1114 1115 1116
	/* Use a mutex to guard daemon_work against
	 * bitmap_destroy.
	 */
1117
	mutex_lock(&mddev->bitmap_info.mutex);
1118 1119
	bitmap = mddev->bitmap;
	if (bitmap == NULL) {
1120
		mutex_unlock(&mddev->bitmap_info.mutex);
1121
		return;
1122
	}
1123
	if (time_before(jiffies, bitmap->daemon_lastrun
N
NeilBrown 已提交
1124
			+ mddev->bitmap_info.daemon_sleep))
1125 1126
		goto done;

1127
	bitmap->daemon_lastrun = jiffies;
1128
	if (bitmap->allclean) {
N
NeilBrown 已提交
1129
		mddev->thread->timeout = MAX_SCHEDULE_TIMEOUT;
1130
		goto done;
1131 1132
	}
	bitmap->allclean = 1;
1133

1134 1135 1136 1137
	/* Any file-page which is PENDING now needs to be written.
	 * So set NEEDWRITE now, then after we make any last-minute changes
	 * we will write it.
	 */
1138
	for (j = 0; j < bitmap->storage.file_pages; j++)
1139 1140
		if (test_and_clear_page_attr(bitmap, j,
					     BITMAP_PAGE_PENDING))
1141
			set_page_attr(bitmap, j,
1142 1143 1144 1145 1146 1147 1148 1149
				      BITMAP_PAGE_NEEDWRITE);

	if (bitmap->need_sync &&
	    mddev->bitmap_info.external == 0) {
		/* Arrange for superblock update as well as
		 * other changes */
		bitmap_super_t *sb;
		bitmap->need_sync = 0;
1150 1151
		if (bitmap->storage.filemap) {
			sb = kmap_atomic(bitmap->storage.sb_page);
1152 1153 1154
			sb->events_cleared =
				cpu_to_le64(bitmap->events_cleared);
			kunmap_atomic(sb);
1155
			set_page_attr(bitmap, 0,
1156 1157
				      BITMAP_PAGE_NEEDWRITE);
		}
1158 1159 1160 1161
	}
	/* Now look at the bitmap counters and if any are '2' or '1',
	 * decrement and handle accordingly.
	 */
1162 1163
	counts = &bitmap->counts;
	spin_lock_irq(&counts->lock);
1164
	nextpage = 0;
1165
	for (j = 0; j < counts->chunks; j++) {
1166
		bitmap_counter_t *bmc;
1167
		sector_t  block = (sector_t)j << counts->chunkshift;
1168

1169 1170
		if (j == nextpage) {
			nextpage += PAGE_COUNTER_RATIO;
1171
			if (!counts->bp[j >> PAGE_COUNTER_SHIFT].pending) {
1172
				j |= PAGE_COUNTER_MASK;
1173 1174
				continue;
			}
1175
			counts->bp[j >> PAGE_COUNTER_SHIFT].pending = 0;
1176
		}
1177
		bmc = bitmap_get_counter(counts,
1178
					 block,
1179
					 &blocks, 0);
1180 1181

		if (!bmc) {
1182
			j |= PAGE_COUNTER_MASK;
1183 1184 1185 1186 1187
			continue;
		}
		if (*bmc == 1 && !bitmap->need_sync) {
			/* We can clear the bit */
			*bmc = 0;
1188
			bitmap_count_page(counts, block, -1);
1189
			bitmap_file_clear_bit(bitmap, block);
1190 1191
		} else if (*bmc && *bmc <= 2) {
			*bmc = 1;
1192
			bitmap_set_pending(counts, block);
1193
			bitmap->allclean = 0;
1194
		}
1195
	}
1196
	spin_unlock_irq(&counts->lock);
1197

1198 1199 1200 1201 1202 1203 1204 1205
	/* Now start writeout on any page in NEEDWRITE that isn't DIRTY.
	 * DIRTY pages need to be written by bitmap_unplug so it can wait
	 * for them.
	 * If we find any DIRTY page we stop there and let bitmap_unplug
	 * handle all the rest.  This is important in the case where
	 * the first blocking holds the superblock and it has been updated.
	 * We mustn't write any other blocks before the superblock.
	 */
1206 1207 1208 1209
	for (j = 0;
	     j < bitmap->storage.file_pages
		     && !test_bit(BITMAP_STALE, &bitmap->flags);
	     j++) {
1210

1211
		if (test_page_attr(bitmap, j,
1212 1213 1214
				   BITMAP_PAGE_DIRTY))
			/* bitmap_unplug will handle the rest */
			break;
1215 1216
		if (test_and_clear_page_attr(bitmap, j,
					     BITMAP_PAGE_NEEDWRITE)) {
1217
			write_page(bitmap, bitmap->storage.filemap[j], 0);
1218 1219 1220
		}
	}

1221
 done:
1222
	if (bitmap->allclean == 0)
N
NeilBrown 已提交
1223 1224
		mddev->thread->timeout =
			mddev->bitmap_info.daemon_sleep;
1225
	mutex_unlock(&mddev->bitmap_info.mutex);
1226 1227
}

1228
static bitmap_counter_t *bitmap_get_counter(struct bitmap_counts *bitmap,
N
NeilBrown 已提交
1229
					    sector_t offset, sector_t *blocks,
1230
					    int create)
1231 1232
__releases(bitmap->lock)
__acquires(bitmap->lock)
1233 1234 1235 1236 1237
{
	/* If 'create', we might release the lock and reclaim it.
	 * The lock must have been taken with interrupts enabled.
	 * If !create, we don't release the lock.
	 */
1238
	sector_t chunk = offset >> bitmap->chunkshift;
1239 1240 1241
	unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
	unsigned long pageoff = (chunk & PAGE_COUNTER_MASK) << COUNTER_BYTE_SHIFT;
	sector_t csize;
1242
	int err;
1243

1244 1245 1246 1247
	err = bitmap_checkpage(bitmap, page, create);

	if (bitmap->bp[page].hijacked ||
	    bitmap->bp[page].map == NULL)
1248
		csize = ((sector_t)1) << (bitmap->chunkshift +
1249 1250
					  PAGE_COUNTER_SHIFT - 1);
	else
1251
		csize = ((sector_t)1) << bitmap->chunkshift;
1252 1253 1254
	*blocks = csize - (offset & (csize - 1));

	if (err < 0)
1255
		return NULL;
1256

1257 1258 1259 1260 1261 1262 1263 1264
	/* now locked ... */

	if (bitmap->bp[page].hijacked) { /* hijacked pointer */
		/* should we use the first or second counter field
		 * of the hijacked pointer? */
		int hi = (pageoff > PAGE_COUNTER_MASK);
		return  &((bitmap_counter_t *)
			  &bitmap->bp[page].map)[hi];
1265
	} else /* page is allocated */
1266 1267 1268 1269
		return (bitmap_counter_t *)
			&(bitmap->bp[page].map[pageoff]);
}

1270
int bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors, int behind)
1271
{
1272 1273
	if (!bitmap)
		return 0;
1274 1275

	if (behind) {
1276
		int bw;
1277
		atomic_inc(&bitmap->behind_writes);
1278 1279 1280 1281
		bw = atomic_read(&bitmap->behind_writes);
		if (bw > bitmap->behind_writes_used)
			bitmap->behind_writes_used = bw;

1282 1283
		pr_debug("inc write-behind count %d/%lu\n",
			 bw, bitmap->mddev->bitmap_info.max_write_behind);
1284 1285
	}

1286
	while (sectors) {
N
NeilBrown 已提交
1287
		sector_t blocks;
1288 1289
		bitmap_counter_t *bmc;

1290 1291
		spin_lock_irq(&bitmap->counts.lock);
		bmc = bitmap_get_counter(&bitmap->counts, offset, &blocks, 1);
1292
		if (!bmc) {
1293
			spin_unlock_irq(&bitmap->counts.lock);
1294 1295 1296
			return 0;
		}

1297
		if (unlikely(COUNTER(*bmc) == COUNTER_MAX)) {
1298 1299 1300 1301 1302 1303 1304
			DEFINE_WAIT(__wait);
			/* note that it is safe to do the prepare_to_wait
			 * after the test as long as we do it before dropping
			 * the spinlock.
			 */
			prepare_to_wait(&bitmap->overflow_wait, &__wait,
					TASK_UNINTERRUPTIBLE);
1305
			spin_unlock_irq(&bitmap->counts.lock);
1306
			schedule();
1307 1308 1309 1310
			finish_wait(&bitmap->overflow_wait, &__wait);
			continue;
		}

1311
		switch (*bmc) {
1312 1313
		case 0:
			bitmap_file_set_bit(bitmap, offset);
1314
			bitmap_count_page(&bitmap->counts, offset, 1);
1315 1316 1317 1318
			/* fall through */
		case 1:
			*bmc = 2;
		}
1319

1320 1321
		(*bmc)++;

1322
		spin_unlock_irq(&bitmap->counts.lock);
1323 1324 1325 1326

		offset += blocks;
		if (sectors > blocks)
			sectors -= blocks;
1327 1328
		else
			sectors = 0;
1329 1330 1331
	}
	return 0;
}
1332
EXPORT_SYMBOL(bitmap_startwrite);
1333 1334

void bitmap_endwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors,
1335
		     int success, int behind)
1336
{
1337 1338
	if (!bitmap)
		return;
1339
	if (behind) {
1340 1341
		if (atomic_dec_and_test(&bitmap->behind_writes))
			wake_up(&bitmap->behind_wait);
1342 1343 1344
		pr_debug("dec write-behind count %d/%lu\n",
			 atomic_read(&bitmap->behind_writes),
			 bitmap->mddev->bitmap_info.max_write_behind);
1345 1346
	}

1347
	while (sectors) {
N
NeilBrown 已提交
1348
		sector_t blocks;
1349 1350 1351
		unsigned long flags;
		bitmap_counter_t *bmc;

1352 1353
		spin_lock_irqsave(&bitmap->counts.lock, flags);
		bmc = bitmap_get_counter(&bitmap->counts, offset, &blocks, 0);
1354
		if (!bmc) {
1355
			spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1356 1357 1358
			return;
		}

1359
		if (success && !bitmap->mddev->degraded &&
1360 1361 1362
		    bitmap->events_cleared < bitmap->mddev->events) {
			bitmap->events_cleared = bitmap->mddev->events;
			bitmap->need_sync = 1;
1363
			sysfs_notify_dirent_safe(bitmap->sysfs_can_clear);
1364 1365
		}

1366
		if (!success && !NEEDED(*bmc))
1367 1368
			*bmc |= NEEDED_MASK;

1369
		if (COUNTER(*bmc) == COUNTER_MAX)
1370 1371
			wake_up(&bitmap->overflow_wait);

1372
		(*bmc)--;
1373
		if (*bmc <= 2) {
1374
			bitmap_set_pending(&bitmap->counts, offset);
1375 1376
			bitmap->allclean = 0;
		}
1377
		spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1378 1379 1380
		offset += blocks;
		if (sectors > blocks)
			sectors -= blocks;
1381 1382
		else
			sectors = 0;
1383 1384
	}
}
1385
EXPORT_SYMBOL(bitmap_endwrite);
1386

N
NeilBrown 已提交
1387
static int __bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
1388
			       int degraded)
1389 1390 1391 1392 1393 1394 1395
{
	bitmap_counter_t *bmc;
	int rv;
	if (bitmap == NULL) {/* FIXME or bitmap set as 'failed' */
		*blocks = 1024;
		return 1; /* always resync if no bitmap */
	}
1396 1397
	spin_lock_irq(&bitmap->counts.lock);
	bmc = bitmap_get_counter(&bitmap->counts, offset, blocks, 0);
1398 1399 1400 1401 1402 1403 1404
	rv = 0;
	if (bmc) {
		/* locked */
		if (RESYNC(*bmc))
			rv = 1;
		else if (NEEDED(*bmc)) {
			rv = 1;
1405 1406 1407 1408
			if (!degraded) { /* don't set/clear bits if degraded */
				*bmc |= RESYNC_MASK;
				*bmc &= ~NEEDED_MASK;
			}
1409 1410
		}
	}
1411
	spin_unlock_irq(&bitmap->counts.lock);
1412 1413 1414
	return rv;
}

N
NeilBrown 已提交
1415
int bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
1416 1417 1418 1419 1420 1421 1422 1423 1424 1425
		      int degraded)
{
	/* bitmap_start_sync must always report on multiples of whole
	 * pages, otherwise resync (which is very PAGE_SIZE based) will
	 * get confused.
	 * So call __bitmap_start_sync repeatedly (if needed) until
	 * At least PAGE_SIZE>>9 blocks are covered.
	 * Return the 'or' of the result.
	 */
	int rv = 0;
N
NeilBrown 已提交
1426
	sector_t blocks1;
1427 1428 1429 1430 1431 1432 1433 1434 1435 1436

	*blocks = 0;
	while (*blocks < (PAGE_SIZE>>9)) {
		rv |= __bitmap_start_sync(bitmap, offset,
					  &blocks1, degraded);
		offset += blocks1;
		*blocks += blocks1;
	}
	return rv;
}
1437
EXPORT_SYMBOL(bitmap_start_sync);
1438

N
NeilBrown 已提交
1439
void bitmap_end_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks, int aborted)
1440 1441 1442
{
	bitmap_counter_t *bmc;
	unsigned long flags;
1443 1444

	if (bitmap == NULL) {
1445 1446 1447
		*blocks = 1024;
		return;
	}
1448 1449
	spin_lock_irqsave(&bitmap->counts.lock, flags);
	bmc = bitmap_get_counter(&bitmap->counts, offset, blocks, 0);
1450 1451 1452 1453 1454 1455 1456 1457 1458
	if (bmc == NULL)
		goto unlock;
	/* locked */
	if (RESYNC(*bmc)) {
		*bmc &= ~RESYNC_MASK;

		if (!NEEDED(*bmc) && aborted)
			*bmc |= NEEDED_MASK;
		else {
1459
			if (*bmc <= 2) {
1460
				bitmap_set_pending(&bitmap->counts, offset);
1461 1462
				bitmap->allclean = 0;
			}
1463 1464 1465
		}
	}
 unlock:
1466
	spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1467
}
1468
EXPORT_SYMBOL(bitmap_end_sync);
1469 1470 1471 1472 1473 1474 1475 1476

void bitmap_close_sync(struct bitmap *bitmap)
{
	/* Sync has finished, and any bitmap chunks that weren't synced
	 * properly have been aborted.  It remains to us to clear the
	 * RESYNC bit wherever it is still on
	 */
	sector_t sector = 0;
N
NeilBrown 已提交
1477
	sector_t blocks;
N
NeilBrown 已提交
1478 1479
	if (!bitmap)
		return;
1480 1481
	while (sector < bitmap->mddev->resync_max_sectors) {
		bitmap_end_sync(bitmap, sector, &blocks, 0);
N
NeilBrown 已提交
1482 1483 1484
		sector += blocks;
	}
}
1485
EXPORT_SYMBOL(bitmap_close_sync);
N
NeilBrown 已提交
1486 1487 1488 1489

void bitmap_cond_end_sync(struct bitmap *bitmap, sector_t sector)
{
	sector_t s = 0;
N
NeilBrown 已提交
1490
	sector_t blocks;
N
NeilBrown 已提交
1491 1492 1493 1494 1495 1496 1497 1498

	if (!bitmap)
		return;
	if (sector == 0) {
		bitmap->last_end_sync = jiffies;
		return;
	}
	if (time_before(jiffies, (bitmap->last_end_sync
1499
				  + bitmap->mddev->bitmap_info.daemon_sleep)))
N
NeilBrown 已提交
1500 1501 1502 1503
		return;
	wait_event(bitmap->mddev->recovery_wait,
		   atomic_read(&bitmap->mddev->recovery_active) == 0);

1504
	bitmap->mddev->curr_resync_completed = sector;
1505
	set_bit(MD_CHANGE_CLEAN, &bitmap->mddev->flags);
1506
	sector &= ~((1ULL << bitmap->counts.chunkshift) - 1);
N
NeilBrown 已提交
1507 1508 1509 1510
	s = 0;
	while (s < sector && s < bitmap->mddev->resync_max_sectors) {
		bitmap_end_sync(bitmap, s, &blocks, 0);
		s += blocks;
1511
	}
N
NeilBrown 已提交
1512
	bitmap->last_end_sync = jiffies;
1513
	sysfs_notify(&bitmap->mddev->kobj, NULL, "sync_completed");
1514
}
1515
EXPORT_SYMBOL(bitmap_cond_end_sync);
1516

1517
static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed)
1518 1519
{
	/* For each chunk covered by any of these sectors, set the
1520
	 * counter to 2 and possibly set resync_needed.  They should all
1521 1522
	 * be 0 at this point
	 */
1523

N
NeilBrown 已提交
1524
	sector_t secs;
1525
	bitmap_counter_t *bmc;
1526 1527
	spin_lock_irq(&bitmap->counts.lock);
	bmc = bitmap_get_counter(&bitmap->counts, offset, &secs, 1);
1528
	if (!bmc) {
1529
		spin_unlock_irq(&bitmap->counts.lock);
1530
		return;
1531
	}
1532
	if (!*bmc) {
1533
		*bmc = 2 | (needed ? NEEDED_MASK : 0);
1534 1535
		bitmap_count_page(&bitmap->counts, offset, 1);
		bitmap_set_pending(&bitmap->counts, offset);
1536
		bitmap->allclean = 0;
1537
	}
1538
	spin_unlock_irq(&bitmap->counts.lock);
1539 1540
}

1541 1542 1543 1544 1545 1546
/* dirty the memory and file bits for bitmap chunks "s" to "e" */
void bitmap_dirty_bits(struct bitmap *bitmap, unsigned long s, unsigned long e)
{
	unsigned long chunk;

	for (chunk = s; chunk <= e; chunk++) {
1547
		sector_t sec = (sector_t)chunk << bitmap->counts.chunkshift;
1548 1549
		bitmap_set_memory_bits(bitmap, sec, 1);
		bitmap_file_set_bit(bitmap, sec);
1550 1551 1552 1553 1554 1555
		if (sec < bitmap->mddev->recovery_cp)
			/* We are asserting that the array is dirty,
			 * so move the recovery_cp address back so
			 * that it is obvious that it is dirty
			 */
			bitmap->mddev->recovery_cp = sec;
1556 1557 1558
	}
}

1559 1560 1561
/*
 * flush out any pending updates
 */
1562
void bitmap_flush(struct mddev *mddev)
1563 1564
{
	struct bitmap *bitmap = mddev->bitmap;
1565
	long sleep;
1566 1567 1568 1569 1570 1571 1572

	if (!bitmap) /* there was no bitmap */
		return;

	/* run the daemon_work three time to ensure everything is flushed
	 * that can be
	 */
1573
	sleep = mddev->bitmap_info.daemon_sleep * 2;
1574
	bitmap->daemon_lastrun -= sleep;
1575
	bitmap_daemon_work(mddev);
1576
	bitmap->daemon_lastrun -= sleep;
1577
	bitmap_daemon_work(mddev);
1578
	bitmap->daemon_lastrun -= sleep;
1579
	bitmap_daemon_work(mddev);
1580 1581 1582
	bitmap_update_sb(bitmap);
}

1583 1584 1585
/*
 * free memory that was allocated
 */
1586
static void bitmap_free(struct bitmap *bitmap)
1587 1588 1589 1590 1591 1592 1593
{
	unsigned long k, pages;
	struct bitmap_page *bp;

	if (!bitmap) /* there was no bitmap */
		return;

1594 1595 1596 1597 1598 1599
	/* Shouldn't be needed - but just in case.... */
	wait_event(bitmap->write_wait,
		   atomic_read(&bitmap->pending_writes) == 0);

	/* release the bitmap file  */
	bitmap_file_unmap(&bitmap->storage);
1600

1601 1602
	bp = bitmap->counts.bp;
	pages = bitmap->counts.pages;
1603 1604 1605 1606 1607 1608 1609 1610 1611 1612

	/* free all allocated memory */

	if (bp) /* deallocate the page memory */
		for (k = 0; k < pages; k++)
			if (bp[k].map && !bp[k].hijacked)
				kfree(bp[k].map);
	kfree(bp);
	kfree(bitmap);
}
1613

1614
void bitmap_destroy(struct mddev *mddev)
1615 1616 1617 1618 1619 1620
{
	struct bitmap *bitmap = mddev->bitmap;

	if (!bitmap) /* there was no bitmap */
		return;

1621
	mutex_lock(&mddev->bitmap_info.mutex);
1622
	spin_lock(&mddev->lock);
1623
	mddev->bitmap = NULL; /* disconnect from the md device */
1624
	spin_unlock(&mddev->lock);
1625
	mutex_unlock(&mddev->bitmap_info.mutex);
1626 1627
	if (mddev->thread)
		mddev->thread->timeout = MAX_SCHEDULE_TIMEOUT;
1628

1629 1630 1631
	if (bitmap->sysfs_can_clear)
		sysfs_put(bitmap->sysfs_can_clear);

1632 1633
	bitmap_free(bitmap);
}
1634 1635 1636 1637 1638

/*
 * initialize the bitmap structure
 * if this returns an error, bitmap_destroy must be called to do clean up
 */
1639
int bitmap_create(struct mddev *mddev)
1640 1641
{
	struct bitmap *bitmap;
1642
	sector_t blocks = mddev->resync_max_sectors;
1643
	struct file *file = mddev->bitmap_info.file;
1644
	int err;
1645
	struct kernfs_node *bm = NULL;
1646

A
Alexey Dobriyan 已提交
1647
	BUILD_BUG_ON(sizeof(bitmap_super_t) != 256);
1648

1649
	BUG_ON(file && mddev->bitmap_info.offset);
1650

1651
	bitmap = kzalloc(sizeof(*bitmap), GFP_KERNEL);
1652 1653 1654
	if (!bitmap)
		return -ENOMEM;

1655
	spin_lock_init(&bitmap->counts.lock);
1656 1657
	atomic_set(&bitmap->pending_writes, 0);
	init_waitqueue_head(&bitmap->write_wait);
1658
	init_waitqueue_head(&bitmap->overflow_wait);
1659
	init_waitqueue_head(&bitmap->behind_wait);
1660

1661 1662
	bitmap->mddev = mddev;

1663
	if (mddev->kobj.sd)
T
Tejun Heo 已提交
1664
		bm = sysfs_get_dirent(mddev->kobj.sd, "bitmap");
1665
	if (bm) {
T
Tejun Heo 已提交
1666
		bitmap->sysfs_can_clear = sysfs_get_dirent(bm, "can_clear");
1667 1668 1669 1670
		sysfs_put(bm);
	} else
		bitmap->sysfs_can_clear = NULL;

1671
	bitmap->storage.file = file;
1672 1673
	if (file) {
		get_file(file);
1674 1675 1676 1677
		/* As future accesses to this file will use bmap,
		 * and bypass the page cache, we must sync the file
		 * first.
		 */
1678
		vfs_fsync(file, 1);
1679
	}
1680
	/* read superblock from bitmap file (this sets mddev->bitmap_info.chunksize) */
1681 1682 1683 1684 1685 1686 1687 1688 1689 1690
	if (!mddev->bitmap_info.external) {
		/*
		 * If 'MD_ARRAY_FIRST_USE' is set, then device-mapper is
		 * instructing us to create a new on-disk bitmap instance.
		 */
		if (test_and_clear_bit(MD_ARRAY_FIRST_USE, &mddev->flags))
			err = bitmap_new_disk_sb(bitmap);
		else
			err = bitmap_read_sb(bitmap);
	} else {
1691 1692 1693 1694 1695 1696 1697
		err = 0;
		if (mddev->bitmap_info.chunksize == 0 ||
		    mddev->bitmap_info.daemon_sleep == 0)
			/* chunksize and time_base need to be
			 * set first. */
			err = -EINVAL;
	}
1698
	if (err)
1699
		goto error;
1700

1701
	bitmap->daemon_lastrun = jiffies;
1702 1703
	err = bitmap_resize(bitmap, blocks, mddev->bitmap_info.chunksize, 1);
	if (err)
1704
		goto error;
1705

1706
	printk(KERN_INFO "created bitmap (%lu pages) for device %s\n",
1707
	       bitmap->counts.pages, bmname(bitmap));
1708 1709

	mddev->bitmap = bitmap;
1710
	return test_bit(BITMAP_WRITE_ERROR, &bitmap->flags) ? -EIO : 0;
1711 1712 1713 1714 1715 1716

 error:
	bitmap_free(bitmap);
	return err;
}

1717
int bitmap_load(struct mddev *mddev)
1718 1719
{
	int err = 0;
1720
	sector_t start = 0;
1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732
	sector_t sector = 0;
	struct bitmap *bitmap = mddev->bitmap;

	if (!bitmap)
		goto out;

	/* Clear out old bitmap info first:  Either there is none, or we
	 * are resuming after someone else has possibly changed things,
	 * so we should forget old cached info.
	 * All chunks should be clean, but some might need_sync.
	 */
	while (sector < mddev->resync_max_sectors) {
N
NeilBrown 已提交
1733
		sector_t blocks;
1734 1735 1736 1737 1738
		bitmap_start_sync(bitmap, sector, &blocks, 0);
		sector += blocks;
	}
	bitmap_close_sync(bitmap);

1739 1740 1741 1742 1743 1744
	if (mddev->degraded == 0
	    || bitmap->events_cleared == mddev->events)
		/* no need to keep dirty bits to optimise a
		 * re-add of a missing device */
		start = mddev->recovery_cp;

1745
	mutex_lock(&mddev->bitmap_info.mutex);
1746
	err = bitmap_init_from_disk(bitmap, start);
1747
	mutex_unlock(&mddev->bitmap_info.mutex);
1748

1749
	if (err)
1750
		goto out;
1751
	clear_bit(BITMAP_STALE, &bitmap->flags);
1752 1753 1754

	/* Kick recovery in case any bits were set */
	set_bit(MD_RECOVERY_NEEDED, &bitmap->mddev->recovery);
1755

1756
	mddev->thread->timeout = mddev->bitmap_info.daemon_sleep;
1757
	md_wakeup_thread(mddev->thread);
1758

1759 1760
	bitmap_update_sb(bitmap);

1761
	if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
1762 1763
		err = -EIO;
out:
1764
	return err;
1765
}
1766
EXPORT_SYMBOL_GPL(bitmap_load);
1767

1768 1769 1770
void bitmap_status(struct seq_file *seq, struct bitmap *bitmap)
{
	unsigned long chunk_kb;
1771
	struct bitmap_counts *counts;
1772 1773 1774 1775

	if (!bitmap)
		return;

1776 1777
	counts = &bitmap->counts;

1778 1779 1780
	chunk_kb = bitmap->mddev->bitmap_info.chunksize >> 10;
	seq_printf(seq, "bitmap: %lu/%lu pages [%luKB], "
		   "%lu%s chunk",
1781 1782 1783
		   counts->pages - counts->missing_pages,
		   counts->pages,
		   (counts->pages - counts->missing_pages)
1784 1785 1786
		   << (PAGE_SHIFT - 10),
		   chunk_kb ? chunk_kb : bitmap->mddev->bitmap_info.chunksize,
		   chunk_kb ? "KB" : "B");
1787
	if (bitmap->storage.file) {
1788
		seq_printf(seq, ", file: ");
1789
		seq_path(seq, &bitmap->storage.file->f_path, " \t\n");
1790 1791 1792 1793 1794
	}

	seq_printf(seq, "\n");
}

1795 1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958
int bitmap_resize(struct bitmap *bitmap, sector_t blocks,
		  int chunksize, int init)
{
	/* If chunk_size is 0, choose an appropriate chunk size.
	 * Then possibly allocate new storage space.
	 * Then quiesce, copy bits, replace bitmap, and re-start
	 *
	 * This function is called both to set up the initial bitmap
	 * and to resize the bitmap while the array is active.
	 * If this happens as a result of the array being resized,
	 * chunksize will be zero, and we need to choose a suitable
	 * chunksize, otherwise we use what we are given.
	 */
	struct bitmap_storage store;
	struct bitmap_counts old_counts;
	unsigned long chunks;
	sector_t block;
	sector_t old_blocks, new_blocks;
	int chunkshift;
	int ret = 0;
	long pages;
	struct bitmap_page *new_bp;

	if (chunksize == 0) {
		/* If there is enough space, leave the chunk size unchanged,
		 * else increase by factor of two until there is enough space.
		 */
		long bytes;
		long space = bitmap->mddev->bitmap_info.space;

		if (space == 0) {
			/* We don't know how much space there is, so limit
			 * to current size - in sectors.
			 */
			bytes = DIV_ROUND_UP(bitmap->counts.chunks, 8);
			if (!bitmap->mddev->bitmap_info.external)
				bytes += sizeof(bitmap_super_t);
			space = DIV_ROUND_UP(bytes, 512);
			bitmap->mddev->bitmap_info.space = space;
		}
		chunkshift = bitmap->counts.chunkshift;
		chunkshift--;
		do {
			/* 'chunkshift' is shift from block size to chunk size */
			chunkshift++;
			chunks = DIV_ROUND_UP_SECTOR_T(blocks, 1 << chunkshift);
			bytes = DIV_ROUND_UP(chunks, 8);
			if (!bitmap->mddev->bitmap_info.external)
				bytes += sizeof(bitmap_super_t);
		} while (bytes > (space << 9));
	} else
		chunkshift = ffz(~chunksize) - BITMAP_BLOCK_SHIFT;

	chunks = DIV_ROUND_UP_SECTOR_T(blocks, 1 << chunkshift);
	memset(&store, 0, sizeof(store));
	if (bitmap->mddev->bitmap_info.offset || bitmap->mddev->bitmap_info.file)
		ret = bitmap_storage_alloc(&store, chunks,
					   !bitmap->mddev->bitmap_info.external);
	if (ret)
		goto err;

	pages = DIV_ROUND_UP(chunks, PAGE_COUNTER_RATIO);

	new_bp = kzalloc(pages * sizeof(*new_bp), GFP_KERNEL);
	ret = -ENOMEM;
	if (!new_bp) {
		bitmap_file_unmap(&store);
		goto err;
	}

	if (!init)
		bitmap->mddev->pers->quiesce(bitmap->mddev, 1);

	store.file = bitmap->storage.file;
	bitmap->storage.file = NULL;

	if (store.sb_page && bitmap->storage.sb_page)
		memcpy(page_address(store.sb_page),
		       page_address(bitmap->storage.sb_page),
		       sizeof(bitmap_super_t));
	bitmap_file_unmap(&bitmap->storage);
	bitmap->storage = store;

	old_counts = bitmap->counts;
	bitmap->counts.bp = new_bp;
	bitmap->counts.pages = pages;
	bitmap->counts.missing_pages = pages;
	bitmap->counts.chunkshift = chunkshift;
	bitmap->counts.chunks = chunks;
	bitmap->mddev->bitmap_info.chunksize = 1 << (chunkshift +
						     BITMAP_BLOCK_SHIFT);

	blocks = min(old_counts.chunks << old_counts.chunkshift,
		     chunks << chunkshift);

	spin_lock_irq(&bitmap->counts.lock);
	for (block = 0; block < blocks; ) {
		bitmap_counter_t *bmc_old, *bmc_new;
		int set;

		bmc_old = bitmap_get_counter(&old_counts, block,
					     &old_blocks, 0);
		set = bmc_old && NEEDED(*bmc_old);

		if (set) {
			bmc_new = bitmap_get_counter(&bitmap->counts, block,
						     &new_blocks, 1);
			if (*bmc_new == 0) {
				/* need to set on-disk bits too. */
				sector_t end = block + new_blocks;
				sector_t start = block >> chunkshift;
				start <<= chunkshift;
				while (start < end) {
					bitmap_file_set_bit(bitmap, block);
					start += 1 << chunkshift;
				}
				*bmc_new = 2;
				bitmap_count_page(&bitmap->counts,
						  block, 1);
				bitmap_set_pending(&bitmap->counts,
						   block);
			}
			*bmc_new |= NEEDED_MASK;
			if (new_blocks < old_blocks)
				old_blocks = new_blocks;
		}
		block += old_blocks;
	}

	if (!init) {
		int i;
		while (block < (chunks << chunkshift)) {
			bitmap_counter_t *bmc;
			bmc = bitmap_get_counter(&bitmap->counts, block,
						 &new_blocks, 1);
			if (bmc) {
				/* new space.  It needs to be resynced, so
				 * we set NEEDED_MASK.
				 */
				if (*bmc == 0) {
					*bmc = NEEDED_MASK | 2;
					bitmap_count_page(&bitmap->counts,
							  block, 1);
					bitmap_set_pending(&bitmap->counts,
							   block);
				}
			}
			block += new_blocks;
		}
		for (i = 0; i < bitmap->storage.file_pages; i++)
			set_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
	}
	spin_unlock_irq(&bitmap->counts.lock);

	if (!init) {
		bitmap_unplug(bitmap);
		bitmap->mddev->pers->quiesce(bitmap->mddev, 0);
	}
	ret = 0;
err:
	return ret;
}
EXPORT_SYMBOL_GPL(bitmap_resize);

1959
static ssize_t
1960
location_show(struct mddev *mddev, char *page)
1961 1962
{
	ssize_t len;
1963
	if (mddev->bitmap_info.file)
1964
		len = sprintf(page, "file");
1965
	else if (mddev->bitmap_info.offset)
1966
		len = sprintf(page, "%+lld", (long long)mddev->bitmap_info.offset);
1967
	else
1968 1969 1970 1971 1972 1973
		len = sprintf(page, "none");
	len += sprintf(page+len, "\n");
	return len;
}

static ssize_t
1974
location_store(struct mddev *mddev, const char *buf, size_t len)
1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010
{

	if (mddev->pers) {
		if (!mddev->pers->quiesce)
			return -EBUSY;
		if (mddev->recovery || mddev->sync_thread)
			return -EBUSY;
	}

	if (mddev->bitmap || mddev->bitmap_info.file ||
	    mddev->bitmap_info.offset) {
		/* bitmap already configured.  Only option is to clear it */
		if (strncmp(buf, "none", 4) != 0)
			return -EBUSY;
		if (mddev->pers) {
			mddev->pers->quiesce(mddev, 1);
			bitmap_destroy(mddev);
			mddev->pers->quiesce(mddev, 0);
		}
		mddev->bitmap_info.offset = 0;
		if (mddev->bitmap_info.file) {
			struct file *f = mddev->bitmap_info.file;
			mddev->bitmap_info.file = NULL;
			fput(f);
		}
	} else {
		/* No bitmap, OK to set a location */
		long long offset;
		if (strncmp(buf, "none", 4) == 0)
			/* nothing to be done */;
		else if (strncmp(buf, "file:", 5) == 0) {
			/* Not supported yet */
			return -EINVAL;
		} else {
			int rv;
			if (buf[0] == '+')
2011
				rv = kstrtoll(buf+1, 10, &offset);
2012
			else
2013
				rv = kstrtoll(buf, 10, &offset);
2014 2015 2016 2017
			if (rv)
				return rv;
			if (offset == 0)
				return -EINVAL;
2018 2019
			if (mddev->bitmap_info.external == 0 &&
			    mddev->major_version == 0 &&
2020 2021 2022 2023 2024 2025
			    offset != mddev->bitmap_info.default_offset)
				return -EINVAL;
			mddev->bitmap_info.offset = offset;
			if (mddev->pers) {
				mddev->pers->quiesce(mddev, 1);
				rv = bitmap_create(mddev);
2026 2027
				if (!rv)
					rv = bitmap_load(mddev);
2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050
				if (rv) {
					bitmap_destroy(mddev);
					mddev->bitmap_info.offset = 0;
				}
				mddev->pers->quiesce(mddev, 0);
				if (rv)
					return rv;
			}
		}
	}
	if (!mddev->external) {
		/* Ensure new bitmap info is stored in
		 * metadata promptly.
		 */
		set_bit(MD_CHANGE_DEVS, &mddev->flags);
		md_wakeup_thread(mddev->thread);
	}
	return len;
}

static struct md_sysfs_entry bitmap_location =
__ATTR(location, S_IRUGO|S_IWUSR, location_show, location_store);

2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074
/* 'bitmap/space' is the space available at 'location' for the
 * bitmap.  This allows the kernel to know when it is safe to
 * resize the bitmap to match a resized array.
 */
static ssize_t
space_show(struct mddev *mddev, char *page)
{
	return sprintf(page, "%lu\n", mddev->bitmap_info.space);
}

static ssize_t
space_store(struct mddev *mddev, const char *buf, size_t len)
{
	unsigned long sectors;
	int rv;

	rv = kstrtoul(buf, 10, &sectors);
	if (rv)
		return rv;

	if (sectors == 0)
		return -EINVAL;

	if (mddev->bitmap &&
2075
	    sectors < (mddev->bitmap->storage.bytes + 511) >> 9)
2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087
		return -EFBIG; /* Bitmap is too big for this small space */

	/* could make sure it isn't too big, but that isn't really
	 * needed - user-space should be careful.
	 */
	mddev->bitmap_info.space = sectors;
	return len;
}

static struct md_sysfs_entry bitmap_space =
__ATTR(space, S_IRUGO|S_IWUSR, space_show, space_store);

2088
static ssize_t
2089
timeout_show(struct mddev *mddev, char *page)
2090 2091 2092 2093
{
	ssize_t len;
	unsigned long secs = mddev->bitmap_info.daemon_sleep / HZ;
	unsigned long jifs = mddev->bitmap_info.daemon_sleep % HZ;
2094

2095 2096 2097 2098 2099 2100 2101 2102
	len = sprintf(page, "%lu", secs);
	if (jifs)
		len += sprintf(page+len, ".%03u", jiffies_to_msecs(jifs));
	len += sprintf(page+len, "\n");
	return len;
}

static ssize_t
2103
timeout_store(struct mddev *mddev, const char *buf, size_t len)
2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138
{
	/* timeout can be set at any time */
	unsigned long timeout;
	int rv = strict_strtoul_scaled(buf, &timeout, 4);
	if (rv)
		return rv;

	/* just to make sure we don't overflow... */
	if (timeout >= LONG_MAX / HZ)
		return -EINVAL;

	timeout = timeout * HZ / 10000;

	if (timeout >= MAX_SCHEDULE_TIMEOUT)
		timeout = MAX_SCHEDULE_TIMEOUT-1;
	if (timeout < 1)
		timeout = 1;
	mddev->bitmap_info.daemon_sleep = timeout;
	if (mddev->thread) {
		/* if thread->timeout is MAX_SCHEDULE_TIMEOUT, then
		 * the bitmap is all clean and we don't need to
		 * adjust the timeout right now
		 */
		if (mddev->thread->timeout < MAX_SCHEDULE_TIMEOUT) {
			mddev->thread->timeout = timeout;
			md_wakeup_thread(mddev->thread);
		}
	}
	return len;
}

static struct md_sysfs_entry bitmap_timeout =
__ATTR(time_base, S_IRUGO|S_IWUSR, timeout_show, timeout_store);

static ssize_t
2139
backlog_show(struct mddev *mddev, char *page)
2140 2141 2142 2143 2144
{
	return sprintf(page, "%lu\n", mddev->bitmap_info.max_write_behind);
}

static ssize_t
2145
backlog_store(struct mddev *mddev, const char *buf, size_t len)
2146 2147
{
	unsigned long backlog;
2148
	int rv = kstrtoul(buf, 10, &backlog);
2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160
	if (rv)
		return rv;
	if (backlog > COUNTER_MAX)
		return -EINVAL;
	mddev->bitmap_info.max_write_behind = backlog;
	return len;
}

static struct md_sysfs_entry bitmap_backlog =
__ATTR(backlog, S_IRUGO|S_IWUSR, backlog_show, backlog_store);

static ssize_t
2161
chunksize_show(struct mddev *mddev, char *page)
2162 2163 2164 2165 2166
{
	return sprintf(page, "%lu\n", mddev->bitmap_info.chunksize);
}

static ssize_t
2167
chunksize_store(struct mddev *mddev, const char *buf, size_t len)
2168 2169 2170 2171 2172 2173
{
	/* Can only be changed when no bitmap is active */
	int rv;
	unsigned long csize;
	if (mddev->bitmap)
		return -EBUSY;
2174
	rv = kstrtoul(buf, 10, &csize);
2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186
	if (rv)
		return rv;
	if (csize < 512 ||
	    !is_power_of_2(csize))
		return -EINVAL;
	mddev->bitmap_info.chunksize = csize;
	return len;
}

static struct md_sysfs_entry bitmap_chunksize =
__ATTR(chunksize, S_IRUGO|S_IWUSR, chunksize_show, chunksize_store);

2187
static ssize_t metadata_show(struct mddev *mddev, char *page)
2188 2189 2190 2191 2192
{
	return sprintf(page, "%s\n", (mddev->bitmap_info.external
				      ? "external" : "internal"));
}

2193
static ssize_t metadata_store(struct mddev *mddev, const char *buf, size_t len)
2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210
{
	if (mddev->bitmap ||
	    mddev->bitmap_info.file ||
	    mddev->bitmap_info.offset)
		return -EBUSY;
	if (strncmp(buf, "external", 8) == 0)
		mddev->bitmap_info.external = 1;
	else if (strncmp(buf, "internal", 8) == 0)
		mddev->bitmap_info.external = 0;
	else
		return -EINVAL;
	return len;
}

static struct md_sysfs_entry bitmap_metadata =
__ATTR(metadata, S_IRUGO|S_IWUSR, metadata_show, metadata_store);

2211
static ssize_t can_clear_show(struct mddev *mddev, char *page)
2212 2213
{
	int len;
2214
	spin_lock(&mddev->lock);
2215 2216 2217 2218 2219
	if (mddev->bitmap)
		len = sprintf(page, "%s\n", (mddev->bitmap->need_sync ?
					     "false" : "true"));
	else
		len = sprintf(page, "\n");
2220
	spin_unlock(&mddev->lock);
2221 2222 2223
	return len;
}

2224
static ssize_t can_clear_store(struct mddev *mddev, const char *buf, size_t len)
2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241
{
	if (mddev->bitmap == NULL)
		return -ENOENT;
	if (strncmp(buf, "false", 5) == 0)
		mddev->bitmap->need_sync = 1;
	else if (strncmp(buf, "true", 4) == 0) {
		if (mddev->degraded)
			return -EBUSY;
		mddev->bitmap->need_sync = 0;
	} else
		return -EINVAL;
	return len;
}

static struct md_sysfs_entry bitmap_can_clear =
__ATTR(can_clear, S_IRUGO|S_IWUSR, can_clear_show, can_clear_store);

2242
static ssize_t
2243
behind_writes_used_show(struct mddev *mddev, char *page)
2244
{
2245 2246
	ssize_t ret;
	spin_lock(&mddev->lock);
2247
	if (mddev->bitmap == NULL)
2248 2249 2250 2251 2252 2253
		ret = sprintf(page, "0\n");
	else
		ret = sprintf(page, "%lu\n",
			      mddev->bitmap->behind_writes_used);
	spin_unlock(&mddev->lock);
	return ret;
2254 2255 2256
}

static ssize_t
2257
behind_writes_used_reset(struct mddev *mddev, const char *buf, size_t len)
2258 2259 2260 2261 2262 2263 2264 2265 2266 2267
{
	if (mddev->bitmap)
		mddev->bitmap->behind_writes_used = 0;
	return len;
}

static struct md_sysfs_entry max_backlog_used =
__ATTR(max_backlog_used, S_IRUGO | S_IWUSR,
       behind_writes_used_show, behind_writes_used_reset);

2268 2269
static struct attribute *md_bitmap_attrs[] = {
	&bitmap_location.attr,
2270
	&bitmap_space.attr,
2271 2272 2273
	&bitmap_timeout.attr,
	&bitmap_backlog.attr,
	&bitmap_chunksize.attr,
2274 2275
	&bitmap_metadata.attr,
	&bitmap_can_clear.attr,
2276
	&max_backlog_used.attr,
2277 2278 2279 2280 2281 2282 2283
	NULL
};
struct attribute_group md_bitmap_group = {
	.name = "bitmap",
	.attrs = md_bitmap_attrs,
};