bitmap.c 56.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
/*
 * bitmap.c two-level bitmap (C) Peter T. Breuer (ptb@ot.uc3m.es) 2003
 *
 * bitmap_create  - sets up the bitmap structure
 * bitmap_destroy - destroys the bitmap structure
 *
 * additions, Copyright (C) 2003-2004, Paul Clements, SteelEye Technology, Inc.:
 * - added disk storage for bitmap
 * - changes to allow various bitmap chunk sizes
 */

/*
 * Still to do:
 *
 * flush after percent set rather than just time based. (maybe both).
 */

18
#include <linux/blkdev.h>
19 20 21 22 23 24 25 26 27 28
#include <linux/module.h>
#include <linux/errno.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/timer.h>
#include <linux/sched.h>
#include <linux/list.h>
#include <linux/file.h>
#include <linux/mount.h>
#include <linux/buffer_head.h>
29
#include <linux/seq_file.h>
30
#include "md.h"
31
#include "bitmap.h"
32

33
static inline char *bmname(struct bitmap *bitmap)
34 35 36 37 38 39 40 41 42 43 44 45 46 47
{
	return bitmap->mddev ? mdname(bitmap->mddev) : "mdX";
}

/*
 * check a page and, if necessary, allocate it (or hijack it if the alloc fails)
 *
 * 1) check to see if this page is allocated, if it's not then try to alloc
 * 2) if the alloc fails, set the page's hijacked flag so we'll use the
 *    page pointer directly as a counter
 *
 * if we find our page, we increment the page's refcount so that it stays
 * allocated while we're using it
 */
48 49
static int bitmap_checkpage(struct bitmap *bitmap,
			    unsigned long page, int create)
50 51
__releases(bitmap->lock)
__acquires(bitmap->lock)
52 53 54 55
{
	unsigned char *mappage;

	if (page >= bitmap->pages) {
56 57 58 59
		/* This can happen if bitmap_start_sync goes beyond
		 * End-of-device while looking for a whole page.
		 * It is harmless.
		 */
60 61 62 63 64 65 66 67 68 69 70 71 72 73
		return -EINVAL;
	}

	if (bitmap->bp[page].hijacked) /* it's hijacked, don't try to alloc */
		return 0;

	if (bitmap->bp[page].map) /* page is already allocated, just return */
		return 0;

	if (!create)
		return -ENOENT;

	/* this page has not been allocated yet */

74
	spin_unlock_irq(&bitmap->lock);
75
	mappage = kzalloc(PAGE_SIZE, GFP_NOIO);
76 77 78
	spin_lock_irq(&bitmap->lock);

	if (mappage == NULL) {
79 80
		pr_debug("%s: bitmap map page allocation failed, hijacking\n",
			 bmname(bitmap));
81 82 83 84
		/* failed - set the hijacked flag so that we can use the
		 * pointer as a counter */
		if (!bitmap->bp[page].map)
			bitmap->bp[page].hijacked = 1;
85 86
	} else if (bitmap->bp[page].map ||
		   bitmap->bp[page].hijacked) {
87
		/* somebody beat us to getting the page */
88
		kfree(mappage);
89
		return 0;
90
	} else {
91

92
		/* no page was in place and we have one, so install it */
93

94 95 96
		bitmap->bp[page].map = mappage;
		bitmap->missing_pages--;
	}
97 98 99 100 101 102
	return 0;
}

/* if page is completely empty, put it back on the free list, or dealloc it */
/* if page was hijacked, unmark the flag so it might get alloced next time */
/* Note: lock should be held when calling this */
103
static void bitmap_checkfree(struct bitmap *bitmap, unsigned long page)
104 105 106 107 108 109 110 111 112 113 114
{
	char *ptr;

	if (bitmap->bp[page].count) /* page is still busy */
		return;

	/* page is no longer in use, it can be released */

	if (bitmap->bp[page].hijacked) { /* page was hijacked, undo this now */
		bitmap->bp[page].hijacked = 0;
		bitmap->bp[page].map = NULL;
115 116 117 118 119
	} else {
		/* normal case, free the page */
		ptr = bitmap->bp[page].map;
		bitmap->bp[page].map = NULL;
		bitmap->missing_pages++;
120
		kfree(ptr);
121 122 123 124 125 126 127 128 129 130 131
	}
}

/*
 * bitmap file handling - read and write the bitmap file and its superblock
 */

/*
 * basic page I/O operations
 */

132
/* IO operations when bitmap is stored near all superblocks */
133 134 135
static int read_sb_page(struct mddev *mddev, loff_t offset,
			struct page *page,
			unsigned long index, int size)
136 137 138
{
	/* choose a good rdev and read the page from there */

139
	struct md_rdev *rdev;
140 141
	sector_t target;

N
NeilBrown 已提交
142
	rdev_for_each(rdev, mddev) {
143 144
		if (! test_bit(In_sync, &rdev->flags)
		    || test_bit(Faulty, &rdev->flags))
145 146
			continue;

J
Jonathan Brassow 已提交
147
		target = offset + index * (PAGE_SIZE/512);
148

149
		if (sync_page_io(rdev, target,
150
				 roundup(size, bdev_logical_block_size(rdev->bdev)),
J
Jonathan Brassow 已提交
151
				 page, READ, true)) {
152
			page->index = index;
153
			return 0;
154 155
		}
	}
156
	return -EIO;
157 158
}

159
static struct md_rdev *next_active_rdev(struct md_rdev *rdev, struct mddev *mddev)
160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179
{
	/* Iterate the disks of an mddev, using rcu to protect access to the
	 * linked list, and raising the refcount of devices we return to ensure
	 * they don't disappear while in use.
	 * As devices are only added or removed when raid_disk is < 0 and
	 * nr_pending is 0 and In_sync is clear, the entries we return will
	 * still be in the same position on the list when we re-enter
	 * list_for_each_continue_rcu.
	 */
	struct list_head *pos;
	rcu_read_lock();
	if (rdev == NULL)
		/* start at the beginning */
		pos = &mddev->disks;
	else {
		/* release the previous rdev and start from there. */
		rdev_dec_pending(rdev, mddev);
		pos = &rdev->same_set;
	}
	list_for_each_continue_rcu(pos, &mddev->disks) {
180
		rdev = list_entry(pos, struct md_rdev, same_set);
181 182 183 184 185 186 187 188 189 190 191 192
		if (rdev->raid_disk >= 0 &&
		    !test_bit(Faulty, &rdev->flags)) {
			/* this is a usable devices */
			atomic_inc(&rdev->nr_pending);
			rcu_read_unlock();
			return rdev;
		}
	}
	rcu_read_unlock();
	return NULL;
}

193
static int write_sb_page(struct bitmap *bitmap, struct page *page, int wait)
194
{
195
	struct md_rdev *rdev = NULL;
196
	struct block_device *bdev;
197
	struct mddev *mddev = bitmap->mddev;
198
	struct bitmap_storage *store = &bitmap->storage;
199

200
	while ((rdev = next_active_rdev(rdev, mddev)) != NULL) {
201 202
		int size = PAGE_SIZE;
		loff_t offset = mddev->bitmap_info.offset;
203 204 205

		bdev = (rdev->meta_bdev) ? rdev->meta_bdev : rdev->bdev;

206 207 208 209 210
		if (page->index == store->file_pages-1) {
			int last_page_size = store->bytes & (PAGE_SIZE-1);
			if (last_page_size == 0)
				last_page_size = PAGE_SIZE;
			size = roundup(last_page_size,
211
				       bdev_logical_block_size(bdev));
212
		}
213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252
		/* Just make sure we aren't corrupting data or
		 * metadata
		 */
		if (mddev->external) {
			/* Bitmap could be anywhere. */
			if (rdev->sb_start + offset + (page->index
						       * (PAGE_SIZE/512))
			    > rdev->data_offset
			    &&
			    rdev->sb_start + offset
			    < (rdev->data_offset + mddev->dev_sectors
			     + (PAGE_SIZE/512)))
				goto bad_alignment;
		} else if (offset < 0) {
			/* DATA  BITMAP METADATA  */
			if (offset
			    + (long)(page->index * (PAGE_SIZE/512))
			    + size/512 > 0)
				/* bitmap runs in to metadata */
				goto bad_alignment;
			if (rdev->data_offset + mddev->dev_sectors
			    > rdev->sb_start + offset)
				/* data runs in to bitmap */
				goto bad_alignment;
		} else if (rdev->sb_start < rdev->data_offset) {
			/* METADATA BITMAP DATA */
			if (rdev->sb_start
			    + offset
			    + page->index*(PAGE_SIZE/512) + size/512
			    > rdev->data_offset)
				/* bitmap runs in to data */
				goto bad_alignment;
		} else {
			/* DATA METADATA BITMAP - no problems */
		}
		md_super_write(mddev, rdev,
			       rdev->sb_start + offset
			       + page->index * (PAGE_SIZE/512),
			       size,
			       page);
253
	}
254 255

	if (wait)
256
		md_super_wait(mddev);
257
	return 0;
258 259 260

 bad_alignment:
	return -EINVAL;
261 262
}

263
static void bitmap_file_kick(struct bitmap *bitmap);
264
/*
265
 * write out a page to a file
266
 */
267
static void write_page(struct bitmap *bitmap, struct page *page, int wait)
268
{
269
	struct buffer_head *bh;
270

271
	if (bitmap->storage.file == NULL) {
272 273 274 275
		switch (write_sb_page(bitmap, page, wait)) {
		case -EINVAL:
			bitmap->flags |= BITMAP_WRITE_ERROR;
		}
276
	} else {
277

278
		bh = page_buffers(page);
279

280 281 282 283
		while (bh && bh->b_blocknr) {
			atomic_inc(&bitmap->pending_writes);
			set_buffer_locked(bh);
			set_buffer_mapped(bh);
J
Jens Axboe 已提交
284
			submit_bh(WRITE | REQ_SYNC, bh);
285 286
			bh = bh->b_this_page;
		}
287

288
		if (wait)
289 290
			wait_event(bitmap->write_wait,
				   atomic_read(&bitmap->pending_writes)==0);
291
	}
292 293
	if (bitmap->flags & BITMAP_WRITE_ERROR)
		bitmap_file_kick(bitmap);
294 295 296 297 298 299
}

static void end_bitmap_write(struct buffer_head *bh, int uptodate)
{
	struct bitmap *bitmap = bh->b_private;
	unsigned long flags;
300

301 302 303 304
	if (!uptodate) {
		spin_lock_irqsave(&bitmap->lock, flags);
		bitmap->flags |= BITMAP_WRITE_ERROR;
		spin_unlock_irqrestore(&bitmap->lock, flags);
305
	}
306 307 308
	if (atomic_dec_and_test(&bitmap->pending_writes))
		wake_up(&bitmap->write_wait);
}
309

310 311 312 313 314 315 316 317 318 319
/* copied from buffer.c */
static void
__clear_page_buffers(struct page *page)
{
	ClearPagePrivate(page);
	set_page_private(page, 0);
	page_cache_release(page);
}
static void free_buffers(struct page *page)
{
320
	struct buffer_head *bh;
321

322 323 324 325
	if (!PagePrivate(page))
		return;

	bh = page_buffers(page);
326 327 328 329
	while (bh) {
		struct buffer_head *next = bh->b_this_page;
		free_buffer_head(bh);
		bh = next;
330
	}
331 332
	__clear_page_buffers(page);
	put_page(page);
333 334
}

335 336 337 338 339 340 341
/* read a page from a file.
 * We both read the page, and attach buffers to the page to record the
 * address of each block (using bmap).  These addresses will be used
 * to write the block later, completely bypassing the filesystem.
 * This usage is similar to how swap files are handled, and allows us
 * to write to a file with no concerns of memory allocation failing.
 */
342 343 344 345
static int read_page(struct file *file, unsigned long index,
		     struct bitmap *bitmap,
		     unsigned long count,
		     struct page *page)
346
{
347
	int ret = 0;
J
Josef Sipek 已提交
348
	struct inode *inode = file->f_path.dentry->d_inode;
349 350
	struct buffer_head *bh;
	sector_t block;
351

352 353
	pr_debug("read bitmap file (%dB @ %llu)\n", (int)PAGE_SIZE,
		 (unsigned long long)index << PAGE_SHIFT);
354

355 356
	bh = alloc_page_buffers(page, 1<<inode->i_blkbits, 0);
	if (!bh) {
357
		ret = -ENOMEM;
358 359
		goto out;
	}
360 361 362 363 364 365 366 367 368
	attach_page_buffers(page, bh);
	block = index << (PAGE_SHIFT - inode->i_blkbits);
	while (bh) {
		if (count == 0)
			bh->b_blocknr = 0;
		else {
			bh->b_blocknr = bmap(inode, block);
			if (bh->b_blocknr == 0) {
				/* Cannot use this file! */
369
				ret = -EINVAL;
370 371 372 373 374 375 376 377 378 379
				goto out;
			}
			bh->b_bdev = inode->i_sb->s_bdev;
			if (count < (1<<inode->i_blkbits))
				count = 0;
			else
				count -= (1<<inode->i_blkbits);

			bh->b_end_io = end_bitmap_write;
			bh->b_private = bitmap;
380 381 382 383
			atomic_inc(&bitmap->pending_writes);
			set_buffer_locked(bh);
			set_buffer_mapped(bh);
			submit_bh(READ, bh);
384 385 386 387 388
		}
		block++;
		bh = bh->b_this_page;
	}
	page->index = index;
389 390 391

	wait_event(bitmap->write_wait,
		   atomic_read(&bitmap->pending_writes)==0);
392 393
	if (bitmap->flags & BITMAP_WRITE_ERROR)
		ret = -EIO;
394
out:
395 396
	if (ret)
		printk(KERN_ALERT "md: bitmap read error: (%dB @ %llu): %d\n",
397 398
			(int)PAGE_SIZE,
			(unsigned long long)index << PAGE_SHIFT,
399 400
			ret);
	return ret;
401 402 403 404 405 406 407
}

/*
 * bitmap file superblock operations
 */

/* update the event counter and sync the superblock to disk */
408
void bitmap_update_sb(struct bitmap *bitmap)
409 410 411 412
{
	bitmap_super_t *sb;

	if (!bitmap || !bitmap->mddev) /* no bitmap for this array */
413
		return;
414 415
	if (bitmap->mddev->bitmap_info.external)
		return;
416
	if (!bitmap->storage.sb_page) /* no superblock */
417
		return;
418
	sb = kmap_atomic(bitmap->storage.sb_page);
419
	sb->events = cpu_to_le64(bitmap->mddev->events);
420
	if (bitmap->mddev->events < bitmap->events_cleared)
421 422
		/* rocking back to read-only */
		bitmap->events_cleared = bitmap->mddev->events;
423 424
	sb->events_cleared = cpu_to_le64(bitmap->events_cleared);
	sb->state = cpu_to_le32(bitmap->flags);
425 426 427
	/* Just in case these have been changed via sysfs: */
	sb->daemon_sleep = cpu_to_le32(bitmap->mddev->bitmap_info.daemon_sleep/HZ);
	sb->write_behind = cpu_to_le32(bitmap->mddev->bitmap_info.max_write_behind);
428
	kunmap_atomic(sb);
429
	write_page(bitmap, bitmap->storage.sb_page, 1);
430 431 432 433 434 435 436
}

/* print out the bitmap file superblock */
void bitmap_print_sb(struct bitmap *bitmap)
{
	bitmap_super_t *sb;

437
	if (!bitmap || !bitmap->storage.sb_page)
438
		return;
439
	sb = kmap_atomic(bitmap->storage.sb_page);
440
	printk(KERN_DEBUG "%s: bitmap file superblock:\n", bmname(bitmap));
441 442 443
	printk(KERN_DEBUG "         magic: %08x\n", le32_to_cpu(sb->magic));
	printk(KERN_DEBUG "       version: %d\n", le32_to_cpu(sb->version));
	printk(KERN_DEBUG "          uuid: %08x.%08x.%08x.%08x\n",
444 445 446 447
					*(__u32 *)(sb->uuid+0),
					*(__u32 *)(sb->uuid+4),
					*(__u32 *)(sb->uuid+8),
					*(__u32 *)(sb->uuid+12));
448
	printk(KERN_DEBUG "        events: %llu\n",
449
			(unsigned long long) le64_to_cpu(sb->events));
450
	printk(KERN_DEBUG "events cleared: %llu\n",
451
			(unsigned long long) le64_to_cpu(sb->events_cleared));
452 453 454 455 456
	printk(KERN_DEBUG "         state: %08x\n", le32_to_cpu(sb->state));
	printk(KERN_DEBUG "     chunksize: %d B\n", le32_to_cpu(sb->chunksize));
	printk(KERN_DEBUG "  daemon sleep: %ds\n", le32_to_cpu(sb->daemon_sleep));
	printk(KERN_DEBUG "     sync size: %llu KB\n",
			(unsigned long long)le64_to_cpu(sb->sync_size)/2);
457
	printk(KERN_DEBUG "max write behind: %d\n", le32_to_cpu(sb->write_behind));
458
	kunmap_atomic(sb);
459 460
}

461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477
/*
 * bitmap_new_disk_sb
 * @bitmap
 *
 * This function is somewhat the reverse of bitmap_read_sb.  bitmap_read_sb
 * reads and verifies the on-disk bitmap superblock and populates bitmap_info.
 * This function verifies 'bitmap_info' and populates the on-disk bitmap
 * structure, which is to be written to disk.
 *
 * Returns: 0 on success, -Exxx on error
 */
static int bitmap_new_disk_sb(struct bitmap *bitmap)
{
	bitmap_super_t *sb;
	unsigned long chunksize, daemon_sleep, write_behind;
	int err = -EINVAL;

478 479 480 481
	bitmap->storage.sb_page = alloc_page(GFP_KERNEL);
	if (IS_ERR(bitmap->storage.sb_page)) {
		err = PTR_ERR(bitmap->storage.sb_page);
		bitmap->storage.sb_page = NULL;
482 483
		return err;
	}
484
	bitmap->storage.sb_page->index = 0;
485

486
	sb = kmap_atomic(bitmap->storage.sb_page);
487 488 489 490 491 492 493

	sb->magic = cpu_to_le32(BITMAP_MAGIC);
	sb->version = cpu_to_le32(BITMAP_MAJOR_HI);

	chunksize = bitmap->mddev->bitmap_info.chunksize;
	BUG_ON(!chunksize);
	if (!is_power_of_2(chunksize)) {
494
		kunmap_atomic(sb);
495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528
		printk(KERN_ERR "bitmap chunksize not a power of 2\n");
		return -EINVAL;
	}
	sb->chunksize = cpu_to_le32(chunksize);

	daemon_sleep = bitmap->mddev->bitmap_info.daemon_sleep;
	if (!daemon_sleep ||
	    (daemon_sleep < 1) || (daemon_sleep > MAX_SCHEDULE_TIMEOUT)) {
		printk(KERN_INFO "Choosing daemon_sleep default (5 sec)\n");
		daemon_sleep = 5 * HZ;
	}
	sb->daemon_sleep = cpu_to_le32(daemon_sleep);
	bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;

	/*
	 * FIXME: write_behind for RAID1.  If not specified, what
	 * is a good choice?  We choose COUNTER_MAX / 2 arbitrarily.
	 */
	write_behind = bitmap->mddev->bitmap_info.max_write_behind;
	if (write_behind > COUNTER_MAX)
		write_behind = COUNTER_MAX / 2;
	sb->write_behind = cpu_to_le32(write_behind);
	bitmap->mddev->bitmap_info.max_write_behind = write_behind;

	/* keep the array size field of the bitmap superblock up to date */
	sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);

	memcpy(sb->uuid, bitmap->mddev->uuid, 16);

	bitmap->flags |= BITMAP_STALE;
	sb->state |= cpu_to_le32(BITMAP_STALE);
	bitmap->events_cleared = bitmap->mddev->events;
	sb->events_cleared = cpu_to_le64(bitmap->mddev->events);

529
	kunmap_atomic(sb);
530 531 532 533

	return 0;
}

534 535 536 537 538
/* read the superblock from the bitmap file and initialize some bitmap fields */
static int bitmap_read_sb(struct bitmap *bitmap)
{
	char *reason = NULL;
	bitmap_super_t *sb;
539
	unsigned long chunksize, daemon_sleep, write_behind;
540 541
	unsigned long long events;
	int err = -EINVAL;
542
	struct page *sb_page;
543

544
	if (!bitmap->storage.file && !bitmap->mddev->bitmap_info.offset) {
545 546 547 548 549 550 551
		chunksize = 128 * 1024 * 1024;
		daemon_sleep = 5 * HZ;
		write_behind = 0;
		bitmap->flags = BITMAP_STALE;
		err = 0;
		goto out_no_sb;
	}
552
	/* page 0 is the superblock, read it... */
553 554 555
	sb_page = alloc_page(GFP_KERNEL);
	if (!sb_page)
		return -ENOMEM;
556
	bitmap->storage.sb_page = sb_page;
557

558 559
	if (bitmap->storage.file) {
		loff_t isize = i_size_read(bitmap->storage.file->f_mapping->host);
560 561
		int bytes = isize > PAGE_SIZE ? PAGE_SIZE : isize;

562
		err = read_page(bitmap->storage.file, 0,
563
				bitmap, bytes, sb_page);
564
	} else {
565 566 567 568
		err = read_sb_page(bitmap->mddev,
				   bitmap->mddev->bitmap_info.offset,
				   sb_page,
				   0, sizeof(bitmap_super_t));
569
	}
570
	if (err)
571 572
		return err;

573
	sb = kmap_atomic(sb_page);
574 575

	chunksize = le32_to_cpu(sb->chunksize);
576
	daemon_sleep = le32_to_cpu(sb->daemon_sleep) * HZ;
577
	write_behind = le32_to_cpu(sb->write_behind);
578 579 580 581

	/* verify that the bitmap-specific fields are valid */
	if (sb->magic != cpu_to_le32(BITMAP_MAGIC))
		reason = "bad magic";
582 583
	else if (le32_to_cpu(sb->version) < BITMAP_MAJOR_LO ||
		 le32_to_cpu(sb->version) > BITMAP_MAJOR_HI)
584
		reason = "unrecognized superblock version";
585
	else if (chunksize < 512)
586
		reason = "bitmap chunksize too small";
J
Jonathan Brassow 已提交
587
	else if (!is_power_of_2(chunksize))
588
		reason = "bitmap chunksize not a power of 2";
589
	else if (daemon_sleep < 1 || daemon_sleep > MAX_SCHEDULE_TIMEOUT)
590
		reason = "daemon sleep period out of range";
591 592
	else if (write_behind > COUNTER_MAX)
		reason = "write-behind limit out of range (0 - 16383)";
593 594 595 596 597 598 599 600 601
	if (reason) {
		printk(KERN_INFO "%s: invalid bitmap file superblock: %s\n",
			bmname(bitmap), reason);
		goto out;
	}

	/* keep the array size field of the bitmap superblock up to date */
	sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);

602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621
	if (bitmap->mddev->persistent) {
		/*
		 * We have a persistent array superblock, so compare the
		 * bitmap's UUID and event counter to the mddev's
		 */
		if (memcmp(sb->uuid, bitmap->mddev->uuid, 16)) {
			printk(KERN_INFO
			       "%s: bitmap superblock UUID mismatch\n",
			       bmname(bitmap));
			goto out;
		}
		events = le64_to_cpu(sb->events);
		if (events < bitmap->mddev->events) {
			printk(KERN_INFO
			       "%s: bitmap file is out of date (%llu < %llu) "
			       "-- forcing full recovery\n",
			       bmname(bitmap), events,
			       (unsigned long long) bitmap->mddev->events);
			sb->state |= cpu_to_le32(BITMAP_STALE);
		}
622
	}
623

624
	/* assign fields using values from superblock */
625
	bitmap->flags |= le32_to_cpu(sb->state);
626 627
	if (le32_to_cpu(sb->version) == BITMAP_MAJOR_HOSTENDIAN)
		bitmap->flags |= BITMAP_HOSTENDIAN;
628 629 630
	bitmap->events_cleared = le64_to_cpu(sb->events_cleared);
	err = 0;
out:
631
	kunmap_atomic(sb);
632 633 634 635 636 637
out_no_sb:
	if (bitmap->flags & BITMAP_STALE)
		bitmap->events_cleared = bitmap->mddev->events;
	bitmap->mddev->bitmap_info.chunksize = chunksize;
	bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
	bitmap->mddev->bitmap_info.max_write_behind = write_behind;
638 639 640 641 642 643 644 645 646 647
	if (err)
		bitmap_print_sb(bitmap);
	return err;
}

enum bitmap_mask_op {
	MASK_SET,
	MASK_UNSET
};

648 649 650
/* record the state of the bitmap in the superblock.  Return the old value */
static int bitmap_mask_state(struct bitmap *bitmap, enum bitmap_state bits,
			     enum bitmap_mask_op op)
651 652
{
	bitmap_super_t *sb;
653
	int old;
654

655
	if (!bitmap->storage.sb_page) /* can't set the state */
656
		return 0;
657
	sb = kmap_atomic(bitmap->storage.sb_page);
658
	old = le32_to_cpu(sb->state) & bits;
659
	switch (op) {
660 661
	case MASK_SET:
		sb->state |= cpu_to_le32(bits);
662
		bitmap->flags |= bits;
663 664 665
		break;
	case MASK_UNSET:
		sb->state &= cpu_to_le32(~bits);
666
		bitmap->flags &= ~bits;
667 668 669
		break;
	default:
		BUG();
670
	}
671
	kunmap_atomic(sb);
672
	return old;
673 674 675 676 677 678
}

/*
 * general bitmap file operations
 */

679 680 681 682 683 684
/*
 * on-disk bitmap:
 *
 * Use one bit per "chunk" (block set). We do the disk I/O on the bitmap
 * file a page at a time. There's a superblock at the start of the file.
 */
685
/* calculate the index of the page that contains this bit */
686 687
static inline unsigned long file_page_index(struct bitmap_storage *store,
					    unsigned long chunk)
688
{
689
	if (store->sb_page)
690 691
		chunk += sizeof(bitmap_super_t) << 3;
	return chunk >> PAGE_BIT_SHIFT;
692 693 694
}

/* calculate the (bit) offset of this bit within a page */
695 696
static inline unsigned long file_page_offset(struct bitmap_storage *store,
					     unsigned long chunk)
697
{
698
	if (store->sb_page)
699 700
		chunk += sizeof(bitmap_super_t) << 3;
	return chunk & (PAGE_BITS - 1);
701 702 703 704 705 706 707 708 709
}

/*
 * return a pointer to the page in the filemap that contains the given bit
 *
 * this lookup is complicated by the fact that the bitmap sb might be exactly
 * 1 page (e.g., x86) or less than 1 page -- so the bitmap might start on page
 * 0 or page 1
 */
710
static inline struct page *filemap_get_page(struct bitmap_storage *store,
711
					    unsigned long chunk)
712
{
713
	if (file_page_index(store, chunk) >= store->file_pages)
714
		return NULL;
715 716
	return store->filemap[file_page_index(store, chunk)
			      - file_page_index(store, 0)];
717 718
}

719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770
static int bitmap_storage_alloc(struct bitmap_storage *store,
				unsigned long chunks, int with_super)
{
	int pnum;
	unsigned long num_pages;
	unsigned long bytes;

	bytes = DIV_ROUND_UP(chunks, 8);
	if (with_super)
		bytes += sizeof(bitmap_super_t);

	num_pages = DIV_ROUND_UP(bytes, PAGE_SIZE);

	store->filemap = kmalloc(sizeof(struct page *)
				 * num_pages, GFP_KERNEL);
	if (!store->filemap)
		return -ENOMEM;

	if (with_super && !store->sb_page) {
		store->sb_page = alloc_page(GFP_KERNEL);
		if (store->sb_page == NULL)
			return -ENOMEM;
		store->sb_page->index = 0;
	}
	pnum = 0;
	if (store->sb_page) {
		store->filemap[0] = store->sb_page;
		pnum = 1;
	}
	for ( ; pnum < num_pages; pnum++) {
		store->filemap[pnum] = alloc_page(GFP_KERNEL);
		if (!store->filemap[pnum]) {
			store->file_pages = pnum;
			return -ENOMEM;
		}
		store->filemap[pnum]->index = pnum;
	}
	store->file_pages = pnum;

	/* We need 4 bits per page, rounded up to a multiple
	 * of sizeof(unsigned long) */
	store->filemap_attr = kzalloc(
		roundup(DIV_ROUND_UP(num_pages*4, 8), sizeof(unsigned long)),
		GFP_KERNEL);
	if (!store->filemap_attr)
		return -ENOMEM;

	store->bytes = bytes;

	return 0;
}

771 772 773 774 775 776
static void bitmap_file_unmap(struct bitmap *bitmap)
{
	struct page **map, *sb_page;
	unsigned long *attr;
	int pages;
	unsigned long flags;
777
	struct bitmap_storage *store = &bitmap->storage;
778 779

	spin_lock_irqsave(&bitmap->lock, flags);
780 781 782 783 784 785 786 787
	map = store->filemap;
	store->filemap = NULL;
	attr = store->filemap_attr;
	store->filemap_attr = NULL;
	pages = store->file_pages;
	store->file_pages = 0;
	sb_page = store->sb_page;
	store->sb_page = NULL;
788 789 790
	spin_unlock_irqrestore(&bitmap->lock, flags);

	while (pages--)
791
		if (map[pages] != sb_page) /* 0 is sb_page, release it below */
792
			free_buffers(map[pages]);
793 794 795
	kfree(map);
	kfree(attr);

796 797
	if (sb_page)
		free_buffers(sb_page);
798 799 800 801 802 803 804 805
}

static void bitmap_file_put(struct bitmap *bitmap)
{
	struct file *file;
	unsigned long flags;

	spin_lock_irqsave(&bitmap->lock, flags);
806 807
	file = bitmap->storage.file;
	bitmap->storage.file = NULL;
808 809
	spin_unlock_irqrestore(&bitmap->lock, flags);

810 811 812
	if (file)
		wait_event(bitmap->write_wait,
			   atomic_read(&bitmap->pending_writes)==0);
813 814
	bitmap_file_unmap(bitmap);

815
	if (file) {
J
Josef Sipek 已提交
816
		struct inode *inode = file->f_path.dentry->d_inode;
817
		invalidate_mapping_pages(inode->i_mapping, 0, -1);
818
		fput(file);
819
	}
820 821 822 823 824 825 826 827 828 829 830
}

/*
 * bitmap_file_kick - if an error occurs while manipulating the bitmap file
 * then it is no longer reliable, so we stop using it and we mark the file
 * as failed in the superblock
 */
static void bitmap_file_kick(struct bitmap *bitmap)
{
	char *path, *ptr = NULL;

831 832
	if (bitmap_mask_state(bitmap, BITMAP_STALE, MASK_SET) == 0) {
		bitmap_update_sb(bitmap);
833

834
		if (bitmap->storage.file) {
835 836
			path = kmalloc(PAGE_SIZE, GFP_KERNEL);
			if (path)
837 838
				ptr = d_path(&bitmap->storage.file->f_path,
					     path, PAGE_SIZE);
C
Christoph Hellwig 已提交
839

840 841
			printk(KERN_ALERT
			      "%s: kicking failed bitmap file %s from array!\n",
C
Christoph Hellwig 已提交
842
			      bmname(bitmap), IS_ERR(ptr) ? "" : ptr);
843

844 845 846 847 848
			kfree(path);
		} else
			printk(KERN_ALERT
			       "%s: disabling internal bitmap due to errors\n",
			       bmname(bitmap));
849
	}
850 851 852 853 854 855 856

	bitmap_file_put(bitmap);

	return;
}

enum bitmap_page_attr {
857
	BITMAP_PAGE_DIRTY = 0,     /* there are set bits that need to be synced */
858 859
	BITMAP_PAGE_PENDING = 1,   /* there are bits that are being cleaned.
				    * i.e. counter is 1 or 2. */
860
	BITMAP_PAGE_NEEDWRITE = 2, /* there are cleared bits that need to be synced */
861 862
};

863 864
static inline void set_page_attr(struct bitmap *bitmap, int pnum,
				 enum bitmap_page_attr attr)
865
{
866
	__set_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
867 868
}

869 870
static inline void clear_page_attr(struct bitmap *bitmap, int pnum,
				   enum bitmap_page_attr attr)
871
{
872
	__clear_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
873 874
}

875
static inline unsigned long test_page_attr(struct bitmap *bitmap, int pnum,
876
					   enum bitmap_page_attr attr)
877
{
878
	return test_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
879 880 881 882 883 884 885 886 887 888 889 890
}

/*
 * bitmap_file_set_bit -- called before performing a write to the md device
 * to set (and eventually sync) a particular bit in the bitmap file
 *
 * we set the bit immediately, then we record the page number so that
 * when an unplug occurs, we can flush the dirty pages out to disk
 */
static void bitmap_file_set_bit(struct bitmap *bitmap, sector_t block)
{
	unsigned long bit;
891
	struct page *page;
892
	void *kaddr;
893
	unsigned long chunk = block >> bitmap->chunkshift;
894

895
	page = filemap_get_page(&bitmap->storage, chunk);
896 897
	if (!page)
		return;
898
	bit = file_page_offset(&bitmap->storage, chunk);
899

900
	/* set the bit */
901
	kaddr = kmap_atomic(page);
902 903 904 905
	if (bitmap->flags & BITMAP_HOSTENDIAN)
		set_bit(bit, kaddr);
	else
		__set_bit_le(bit, kaddr);
906
	kunmap_atomic(kaddr);
907
	pr_debug("set file bit %lu page %lu\n", bit, page->index);
908
	/* record page number so it gets flushed to disk when unplug occurs */
909
	set_page_attr(bitmap, page->index, BITMAP_PAGE_DIRTY);
910 911
}

912 913 914 915 916 917 918
static void bitmap_file_clear_bit(struct bitmap *bitmap, sector_t block)
{
	unsigned long bit;
	struct page *page;
	void *paddr;
	unsigned long chunk = block >> bitmap->chunkshift;

919
	page = filemap_get_page(&bitmap->storage, chunk);
920 921
	if (!page)
		return;
922
	bit = file_page_offset(&bitmap->storage, chunk);
923 924 925 926 927 928
	paddr = kmap_atomic(page);
	if (bitmap->flags & BITMAP_HOSTENDIAN)
		clear_bit(bit, paddr);
	else
		__clear_bit_le(bit, paddr);
	kunmap_atomic(paddr);
929 930
	if (!test_page_attr(bitmap, page->index, BITMAP_PAGE_NEEDWRITE)) {
		set_page_attr(bitmap, page->index, BITMAP_PAGE_PENDING);
931 932 933 934
		bitmap->allclean = 0;
	}
}

935 936 937
/* this gets called when the md device is ready to unplug its underlying
 * (slave) device queues -- before we let any writes go down, we need to
 * sync the dirty pages of the bitmap file to disk */
938
void bitmap_unplug(struct bitmap *bitmap)
939
{
940 941
	unsigned long i, flags;
	int dirty, need_write;
942 943
	int wait = 0;

944
	if (!bitmap || !bitmap->storage.filemap)
945
		return;
946 947 948

	/* look at each page to see if there are any set bits that need to be
	 * flushed out to disk */
949
	for (i = 0; i < bitmap->storage.file_pages; i++) {
950
		spin_lock_irqsave(&bitmap->lock, flags);
951
		if (!bitmap->storage.filemap) {
952
			spin_unlock_irqrestore(&bitmap->lock, flags);
953
			return;
954
		}
955 956 957 958
		dirty = test_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
		need_write = test_page_attr(bitmap, i, BITMAP_PAGE_NEEDWRITE);
		clear_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
		clear_page_attr(bitmap, i, BITMAP_PAGE_NEEDWRITE);
959
		if (dirty || need_write)
960
			clear_page_attr(bitmap, i, BITMAP_PAGE_PENDING);
961
		if (dirty)
962 963 964
			wait = 1;
		spin_unlock_irqrestore(&bitmap->lock, flags);

965
		if (dirty || need_write)
966
			write_page(bitmap, bitmap->storage.filemap[i], 0);
967 968
	}
	if (wait) { /* if any writes were performed, we need to wait on them */
969
		if (bitmap->storage.file)
970 971
			wait_event(bitmap->write_wait,
				   atomic_read(&bitmap->pending_writes)==0);
972
		else
973
			md_super_wait(bitmap->mddev);
974
	}
975 976
	if (bitmap->flags & BITMAP_WRITE_ERROR)
		bitmap_file_kick(bitmap);
977
}
978
EXPORT_SYMBOL(bitmap_unplug);
979

980
static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed);
981 982 983 984 985 986 987
/* * bitmap_init_from_disk -- called at bitmap_create time to initialize
 * the in-memory bitmap from the on-disk bitmap -- also, sets up the
 * memory mapping of the bitmap file
 * Special cases:
 *   if there's no bitmap file, or if the bitmap file had been
 *   previously kicked from the array, we mark all the bits as
 *   1's in order to cause a full resync.
988 989 990
 *
 * We ignore all bits for sectors that end earlier than 'start'.
 * This is used when reading an out-of-date bitmap...
991
 */
992
static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
993 994
{
	unsigned long i, chunks, index, oldindex, bit;
995
	struct page *page = NULL;
996
	unsigned long bit_cnt = 0;
997
	struct file *file;
998
	unsigned long offset;
999 1000
	int outofdate;
	int ret = -ENOSPC;
1001
	void *paddr;
1002
	struct bitmap_storage *store = &bitmap->storage;
1003 1004

	chunks = bitmap->chunks;
1005
	file = store->file;
1006

1007 1008
	if (!file && !bitmap->mddev->bitmap_info.offset) {
		/* No permanent bitmap - fill with '1s'. */
1009 1010
		store->filemap = NULL;
		store->file_pages = 0;
1011 1012 1013 1014 1015 1016 1017 1018 1019 1020
		for (i = 0; i < chunks ; i++) {
			/* if the disk bit is set, set the memory bit */
			int needed = ((sector_t)(i+1) << (bitmap->chunkshift)
				      >= start);
			bitmap_set_memory_bits(bitmap,
					       (sector_t)i << bitmap->chunkshift,
					       needed);
		}
		return 0;
	}
1021 1022 1023 1024 1025 1026

	outofdate = bitmap->flags & BITMAP_STALE;
	if (outofdate)
		printk(KERN_INFO "%s: bitmap file is out of date, doing full "
			"recovery\n", bmname(bitmap));

1027
	if (file && i_size_read(file->f_mapping->host) < store->bytes) {
1028
		printk(KERN_INFO "%s: bitmap file too short %lu < %lu\n",
1029 1030 1031
		       bmname(bitmap),
		       (unsigned long) i_size_read(file->f_mapping->host),
		       store->bytes);
1032
		goto err;
1033
	}
1034

1035 1036 1037
	ret = bitmap_storage_alloc(&bitmap->storage, bitmap->chunks,
				   !bitmap->mddev->bitmap_info.external);
	if (ret)
1038
		goto err;
1039

1040
	oldindex = ~0L;
1041
	offset = 0;
1042
	if (!bitmap->mddev->bitmap_info.external)
1043
		offset = sizeof(bitmap_super_t);
1044 1045

	for (i = 0; i < chunks; i++) {
1046
		int b;
1047 1048
		index = file_page_index(&bitmap->storage, i);
		bit = file_page_offset(&bitmap->storage, i);
1049
		if (index != oldindex) { /* this is a new page, read it in */
1050
			int count;
1051
			/* unmap the old page, we're done with it */
1052 1053
			if (index == store->file_pages-1)
				count = store->bytes - index * PAGE_SIZE;
1054 1055
			else
				count = PAGE_SIZE;
1056
			page = store->filemap[index];
1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067
			if (file)
				ret = read_page(file, index, bitmap,
						count, page);
			else
				ret = read_sb_page(
					bitmap->mddev,
					bitmap->mddev->bitmap_info.offset,
					page,
					index, count);

			if (ret)
1068
				goto err;
1069

1070 1071 1072 1073 1074
			oldindex = index;

			if (outofdate) {
				/*
				 * if bitmap is out of date, dirty the
1075
				 * whole page and write it out
1076
				 */
1077
				paddr = kmap_atomic(page);
1078
				memset(paddr + offset, 0xff,
1079
				       PAGE_SIZE - offset);
1080
				kunmap_atomic(paddr);
1081 1082 1083
				write_page(bitmap, page, 1);

				ret = -EIO;
1084
				if (bitmap->flags & BITMAP_WRITE_ERROR)
1085
					goto err;
1086 1087
			}
		}
1088
		paddr = kmap_atomic(page);
1089
		if (bitmap->flags & BITMAP_HOSTENDIAN)
1090
			b = test_bit(bit, paddr);
1091
		else
A
Akinobu Mita 已提交
1092
			b = test_bit_le(bit, paddr);
1093
		kunmap_atomic(paddr);
1094
		if (b) {
1095
			/* if the disk bit is set, set the memory bit */
1096
			int needed = ((sector_t)(i+1) << bitmap->chunkshift
1097 1098
				      >= start);
			bitmap_set_memory_bits(bitmap,
1099
					       (sector_t)i << bitmap->chunkshift,
1100
					       needed);
1101 1102
			bit_cnt++;
		}
1103
		offset = 0;
1104 1105 1106
	}

	printk(KERN_INFO "%s: bitmap initialized from disk: "
1107
	       "read %lu pages, set %lu of %lu bits\n",
1108
	       bmname(bitmap), store->file_pages,
1109
	       bit_cnt, chunks);
1110 1111

	return 0;
1112

1113 1114 1115
 err:
	printk(KERN_INFO "%s: bitmap initialisation failed: %d\n",
	       bmname(bitmap), ret);
1116 1117 1118
	return ret;
}

1119 1120 1121 1122 1123
void bitmap_write_all(struct bitmap *bitmap)
{
	/* We don't actually write all bitmap blocks here,
	 * just flag them as needing to be written
	 */
1124
	int i;
1125

1126
	if (!bitmap || !bitmap->storage.filemap)
1127
		return;
1128
	if (bitmap->storage.file)
1129 1130 1131
		/* Only one copy, so nothing needed */
		return;

1132
	spin_lock_irq(&bitmap->lock);
1133
	for (i = 0; i < bitmap->storage.file_pages; i++)
1134
		set_page_attr(bitmap, i,
1135
			      BITMAP_PAGE_NEEDWRITE);
1136
	bitmap->allclean = 0;
1137
	spin_unlock_irq(&bitmap->lock);
1138 1139
}

1140 1141
static void bitmap_count_page(struct bitmap *bitmap, sector_t offset, int inc)
{
1142
	sector_t chunk = offset >> bitmap->chunkshift;
1143 1144 1145 1146
	unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
	bitmap->bp[page].count += inc;
	bitmap_checkfree(bitmap, page);
}
1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157

static void bitmap_set_pending(struct bitmap *bitmap, sector_t offset)
{
	sector_t chunk = offset >> bitmap->chunkshift;
	unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
	struct bitmap_page *bp = &bitmap->bp[page];

	if (!bp->pending)
		bp->pending = 1;
}

1158
static bitmap_counter_t *bitmap_get_counter(struct bitmap *bitmap,
N
NeilBrown 已提交
1159
					    sector_t offset, sector_t *blocks,
1160 1161 1162 1163 1164 1165 1166
					    int create);

/*
 * bitmap daemon -- periodically wakes up to clean bits and flush pages
 *			out to disk
 */

1167
void bitmap_daemon_work(struct mddev *mddev)
1168
{
1169
	struct bitmap *bitmap;
1170
	unsigned long j;
1171
	unsigned long nextpage;
1172
	unsigned long flags;
N
NeilBrown 已提交
1173
	sector_t blocks;
1174

1175 1176 1177
	/* Use a mutex to guard daemon_work against
	 * bitmap_destroy.
	 */
1178
	mutex_lock(&mddev->bitmap_info.mutex);
1179 1180
	bitmap = mddev->bitmap;
	if (bitmap == NULL) {
1181
		mutex_unlock(&mddev->bitmap_info.mutex);
1182
		return;
1183
	}
1184
	if (time_before(jiffies, bitmap->daemon_lastrun
N
NeilBrown 已提交
1185
			+ mddev->bitmap_info.daemon_sleep))
1186 1187
		goto done;

1188
	bitmap->daemon_lastrun = jiffies;
1189
	if (bitmap->allclean) {
N
NeilBrown 已提交
1190
		mddev->thread->timeout = MAX_SCHEDULE_TIMEOUT;
1191
		goto done;
1192 1193
	}
	bitmap->allclean = 1;
1194

1195 1196 1197 1198
	/* Any file-page which is PENDING now needs to be written.
	 * So set NEEDWRITE now, then after we make any last-minute changes
	 * we will write it.
	 */
1199
	spin_lock_irqsave(&bitmap->lock, flags);
1200
	for (j = 0; j < bitmap->storage.file_pages; j++)
1201
		if (test_page_attr(bitmap, j,
1202
				   BITMAP_PAGE_PENDING)) {
1203
			set_page_attr(bitmap, j,
1204
				      BITMAP_PAGE_NEEDWRITE);
1205
			clear_page_attr(bitmap, j,
1206 1207 1208 1209 1210 1211 1212 1213 1214
					BITMAP_PAGE_PENDING);
		}

	if (bitmap->need_sync &&
	    mddev->bitmap_info.external == 0) {
		/* Arrange for superblock update as well as
		 * other changes */
		bitmap_super_t *sb;
		bitmap->need_sync = 0;
1215 1216
		if (bitmap->storage.filemap) {
			sb = kmap_atomic(bitmap->storage.sb_page);
1217 1218 1219
			sb->events_cleared =
				cpu_to_le64(bitmap->events_cleared);
			kunmap_atomic(sb);
1220
			set_page_attr(bitmap, 0,
1221 1222
				      BITMAP_PAGE_NEEDWRITE);
		}
1223 1224 1225 1226 1227
	}
	/* Now look at the bitmap counters and if any are '2' or '1',
	 * decrement and handle accordingly.
	 */
	nextpage = 0;
1228 1229
	for (j = 0; j < bitmap->chunks; j++) {
		bitmap_counter_t *bmc;
1230
		sector_t  block = (sector_t)j << bitmap->chunkshift;
1231

1232 1233 1234 1235
		if (j == nextpage) {
			nextpage += PAGE_COUNTER_RATIO;
			if (!bitmap->bp[j >> PAGE_COUNTER_SHIFT].pending) {
				j |= PAGE_COUNTER_MASK;
1236 1237
				continue;
			}
1238
			bitmap->bp[j >> PAGE_COUNTER_SHIFT].pending = 0;
1239
		}
1240
		bmc = bitmap_get_counter(bitmap,
1241
					 block,
1242
					 &blocks, 0);
1243 1244

		if (!bmc) {
1245
			j |= PAGE_COUNTER_MASK;
1246 1247 1248 1249 1250
			continue;
		}
		if (*bmc == 1 && !bitmap->need_sync) {
			/* We can clear the bit */
			*bmc = 0;
1251 1252
			bitmap_count_page(bitmap, block, -1);
			bitmap_file_clear_bit(bitmap, block);
1253 1254
		} else if (*bmc && *bmc <= 2) {
			*bmc = 1;
1255
			bitmap_set_pending(bitmap, block);
1256
			bitmap->allclean = 0;
1257
		}
1258 1259
	}

1260 1261 1262 1263 1264 1265 1266 1267
	/* Now start writeout on any page in NEEDWRITE that isn't DIRTY.
	 * DIRTY pages need to be written by bitmap_unplug so it can wait
	 * for them.
	 * If we find any DIRTY page we stop there and let bitmap_unplug
	 * handle all the rest.  This is important in the case where
	 * the first blocking holds the superblock and it has been updated.
	 * We mustn't write any other blocks before the superblock.
	 */
1268
	for (j = 0; j < bitmap->storage.file_pages; j++) {
1269

1270
		if (test_page_attr(bitmap, j,
1271 1272 1273
				   BITMAP_PAGE_DIRTY))
			/* bitmap_unplug will handle the rest */
			break;
1274
		if (test_page_attr(bitmap, j,
1275
				   BITMAP_PAGE_NEEDWRITE)) {
1276
			clear_page_attr(bitmap, j,
1277
					BITMAP_PAGE_NEEDWRITE);
1278
			spin_unlock_irqrestore(&bitmap->lock, flags);
1279
			write_page(bitmap, bitmap->storage.filemap[j], 0);
1280
			spin_lock_irqsave(&bitmap->lock, flags);
1281
			if (!bitmap->storage.filemap)
1282
				break;
1283 1284
		}
	}
1285
	spin_unlock_irqrestore(&bitmap->lock, flags);
1286

1287
 done:
1288
	if (bitmap->allclean == 0)
N
NeilBrown 已提交
1289 1290
		mddev->thread->timeout =
			mddev->bitmap_info.daemon_sleep;
1291
	mutex_unlock(&mddev->bitmap_info.mutex);
1292 1293 1294
}

static bitmap_counter_t *bitmap_get_counter(struct bitmap *bitmap,
N
NeilBrown 已提交
1295
					    sector_t offset, sector_t *blocks,
1296
					    int create)
1297 1298
__releases(bitmap->lock)
__acquires(bitmap->lock)
1299 1300 1301 1302 1303
{
	/* If 'create', we might release the lock and reclaim it.
	 * The lock must have been taken with interrupts enabled.
	 * If !create, we don't release the lock.
	 */
1304
	sector_t chunk = offset >> bitmap->chunkshift;
1305 1306 1307
	unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
	unsigned long pageoff = (chunk & PAGE_COUNTER_MASK) << COUNTER_BYTE_SHIFT;
	sector_t csize;
1308
	int err;
1309

1310 1311 1312 1313
	err = bitmap_checkpage(bitmap, page, create);

	if (bitmap->bp[page].hijacked ||
	    bitmap->bp[page].map == NULL)
1314
		csize = ((sector_t)1) << (bitmap->chunkshift +
1315 1316
					  PAGE_COUNTER_SHIFT - 1);
	else
1317
		csize = ((sector_t)1) << bitmap->chunkshift;
1318 1319 1320
	*blocks = csize - (offset & (csize - 1));

	if (err < 0)
1321
		return NULL;
1322

1323 1324 1325 1326 1327 1328 1329 1330
	/* now locked ... */

	if (bitmap->bp[page].hijacked) { /* hijacked pointer */
		/* should we use the first or second counter field
		 * of the hijacked pointer? */
		int hi = (pageoff > PAGE_COUNTER_MASK);
		return  &((bitmap_counter_t *)
			  &bitmap->bp[page].map)[hi];
1331
	} else /* page is allocated */
1332 1333 1334 1335
		return (bitmap_counter_t *)
			&(bitmap->bp[page].map[pageoff]);
}

1336
int bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors, int behind)
1337
{
1338 1339
	if (!bitmap)
		return 0;
1340 1341

	if (behind) {
1342
		int bw;
1343
		atomic_inc(&bitmap->behind_writes);
1344 1345 1346 1347
		bw = atomic_read(&bitmap->behind_writes);
		if (bw > bitmap->behind_writes_used)
			bitmap->behind_writes_used = bw;

1348 1349
		pr_debug("inc write-behind count %d/%lu\n",
			 bw, bitmap->mddev->bitmap_info.max_write_behind);
1350 1351
	}

1352
	while (sectors) {
N
NeilBrown 已提交
1353
		sector_t blocks;
1354 1355 1356 1357 1358 1359 1360 1361 1362
		bitmap_counter_t *bmc;

		spin_lock_irq(&bitmap->lock);
		bmc = bitmap_get_counter(bitmap, offset, &blocks, 1);
		if (!bmc) {
			spin_unlock_irq(&bitmap->lock);
			return 0;
		}

1363
		if (unlikely(COUNTER(*bmc) == COUNTER_MAX)) {
1364 1365 1366 1367 1368 1369 1370 1371
			DEFINE_WAIT(__wait);
			/* note that it is safe to do the prepare_to_wait
			 * after the test as long as we do it before dropping
			 * the spinlock.
			 */
			prepare_to_wait(&bitmap->overflow_wait, &__wait,
					TASK_UNINTERRUPTIBLE);
			spin_unlock_irq(&bitmap->lock);
J
Jens Axboe 已提交
1372
			io_schedule();
1373 1374 1375 1376
			finish_wait(&bitmap->overflow_wait, &__wait);
			continue;
		}

1377
		switch (*bmc) {
1378 1379
		case 0:
			bitmap_file_set_bit(bitmap, offset);
1380
			bitmap_count_page(bitmap, offset, 1);
1381 1382 1383 1384
			/* fall through */
		case 1:
			*bmc = 2;
		}
1385

1386 1387 1388 1389 1390 1391 1392
		(*bmc)++;

		spin_unlock_irq(&bitmap->lock);

		offset += blocks;
		if (sectors > blocks)
			sectors -= blocks;
1393 1394
		else
			sectors = 0;
1395 1396 1397
	}
	return 0;
}
1398
EXPORT_SYMBOL(bitmap_startwrite);
1399 1400

void bitmap_endwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors,
1401
		     int success, int behind)
1402
{
1403 1404
	if (!bitmap)
		return;
1405
	if (behind) {
1406 1407
		if (atomic_dec_and_test(&bitmap->behind_writes))
			wake_up(&bitmap->behind_wait);
1408 1409 1410
		pr_debug("dec write-behind count %d/%lu\n",
			 atomic_read(&bitmap->behind_writes),
			 bitmap->mddev->bitmap_info.max_write_behind);
1411 1412
	}

1413
	while (sectors) {
N
NeilBrown 已提交
1414
		sector_t blocks;
1415 1416 1417 1418 1419 1420 1421 1422 1423 1424
		unsigned long flags;
		bitmap_counter_t *bmc;

		spin_lock_irqsave(&bitmap->lock, flags);
		bmc = bitmap_get_counter(bitmap, offset, &blocks, 0);
		if (!bmc) {
			spin_unlock_irqrestore(&bitmap->lock, flags);
			return;
		}

1425
		if (success && !bitmap->mddev->degraded &&
1426 1427 1428
		    bitmap->events_cleared < bitmap->mddev->events) {
			bitmap->events_cleared = bitmap->mddev->events;
			bitmap->need_sync = 1;
1429
			sysfs_notify_dirent_safe(bitmap->sysfs_can_clear);
1430 1431
		}

1432
		if (!success && !NEEDED(*bmc))
1433 1434
			*bmc |= NEEDED_MASK;

1435
		if (COUNTER(*bmc) == COUNTER_MAX)
1436 1437
			wake_up(&bitmap->overflow_wait);

1438
		(*bmc)--;
1439
		if (*bmc <= 2) {
1440
			bitmap_set_pending(bitmap, offset);
1441 1442
			bitmap->allclean = 0;
		}
1443 1444 1445 1446
		spin_unlock_irqrestore(&bitmap->lock, flags);
		offset += blocks;
		if (sectors > blocks)
			sectors -= blocks;
1447 1448
		else
			sectors = 0;
1449 1450
	}
}
1451
EXPORT_SYMBOL(bitmap_endwrite);
1452

N
NeilBrown 已提交
1453
static int __bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
1454
			       int degraded)
1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470
{
	bitmap_counter_t *bmc;
	int rv;
	if (bitmap == NULL) {/* FIXME or bitmap set as 'failed' */
		*blocks = 1024;
		return 1; /* always resync if no bitmap */
	}
	spin_lock_irq(&bitmap->lock);
	bmc = bitmap_get_counter(bitmap, offset, blocks, 0);
	rv = 0;
	if (bmc) {
		/* locked */
		if (RESYNC(*bmc))
			rv = 1;
		else if (NEEDED(*bmc)) {
			rv = 1;
1471 1472 1473 1474
			if (!degraded) { /* don't set/clear bits if degraded */
				*bmc |= RESYNC_MASK;
				*bmc &= ~NEEDED_MASK;
			}
1475 1476 1477 1478 1479 1480
		}
	}
	spin_unlock_irq(&bitmap->lock);
	return rv;
}

N
NeilBrown 已提交
1481
int bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
1482 1483 1484 1485 1486 1487 1488 1489 1490 1491
		      int degraded)
{
	/* bitmap_start_sync must always report on multiples of whole
	 * pages, otherwise resync (which is very PAGE_SIZE based) will
	 * get confused.
	 * So call __bitmap_start_sync repeatedly (if needed) until
	 * At least PAGE_SIZE>>9 blocks are covered.
	 * Return the 'or' of the result.
	 */
	int rv = 0;
N
NeilBrown 已提交
1492
	sector_t blocks1;
1493 1494 1495 1496 1497 1498 1499 1500 1501 1502

	*blocks = 0;
	while (*blocks < (PAGE_SIZE>>9)) {
		rv |= __bitmap_start_sync(bitmap, offset,
					  &blocks1, degraded);
		offset += blocks1;
		*blocks += blocks1;
	}
	return rv;
}
1503
EXPORT_SYMBOL(bitmap_start_sync);
1504

N
NeilBrown 已提交
1505
void bitmap_end_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks, int aborted)
1506 1507 1508
{
	bitmap_counter_t *bmc;
	unsigned long flags;
1509 1510

	if (bitmap == NULL) {
1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524
		*blocks = 1024;
		return;
	}
	spin_lock_irqsave(&bitmap->lock, flags);
	bmc = bitmap_get_counter(bitmap, offset, blocks, 0);
	if (bmc == NULL)
		goto unlock;
	/* locked */
	if (RESYNC(*bmc)) {
		*bmc &= ~RESYNC_MASK;

		if (!NEEDED(*bmc) && aborted)
			*bmc |= NEEDED_MASK;
		else {
1525
			if (*bmc <= 2) {
1526
				bitmap_set_pending(bitmap, offset);
1527 1528
				bitmap->allclean = 0;
			}
1529 1530 1531 1532 1533
		}
	}
 unlock:
	spin_unlock_irqrestore(&bitmap->lock, flags);
}
1534
EXPORT_SYMBOL(bitmap_end_sync);
1535 1536 1537 1538 1539 1540 1541 1542

void bitmap_close_sync(struct bitmap *bitmap)
{
	/* Sync has finished, and any bitmap chunks that weren't synced
	 * properly have been aborted.  It remains to us to clear the
	 * RESYNC bit wherever it is still on
	 */
	sector_t sector = 0;
N
NeilBrown 已提交
1543
	sector_t blocks;
N
NeilBrown 已提交
1544 1545
	if (!bitmap)
		return;
1546 1547
	while (sector < bitmap->mddev->resync_max_sectors) {
		bitmap_end_sync(bitmap, sector, &blocks, 0);
N
NeilBrown 已提交
1548 1549 1550
		sector += blocks;
	}
}
1551
EXPORT_SYMBOL(bitmap_close_sync);
N
NeilBrown 已提交
1552 1553 1554 1555

void bitmap_cond_end_sync(struct bitmap *bitmap, sector_t sector)
{
	sector_t s = 0;
N
NeilBrown 已提交
1556
	sector_t blocks;
N
NeilBrown 已提交
1557 1558 1559 1560 1561 1562 1563 1564

	if (!bitmap)
		return;
	if (sector == 0) {
		bitmap->last_end_sync = jiffies;
		return;
	}
	if (time_before(jiffies, (bitmap->last_end_sync
1565
				  + bitmap->mddev->bitmap_info.daemon_sleep)))
N
NeilBrown 已提交
1566 1567 1568 1569
		return;
	wait_event(bitmap->mddev->recovery_wait,
		   atomic_read(&bitmap->mddev->recovery_active) == 0);

1570
	bitmap->mddev->curr_resync_completed = sector;
1571
	set_bit(MD_CHANGE_CLEAN, &bitmap->mddev->flags);
1572
	sector &= ~((1ULL << bitmap->chunkshift) - 1);
N
NeilBrown 已提交
1573 1574 1575 1576
	s = 0;
	while (s < sector && s < bitmap->mddev->resync_max_sectors) {
		bitmap_end_sync(bitmap, s, &blocks, 0);
		s += blocks;
1577
	}
N
NeilBrown 已提交
1578
	bitmap->last_end_sync = jiffies;
1579
	sysfs_notify(&bitmap->mddev->kobj, NULL, "sync_completed");
1580
}
1581
EXPORT_SYMBOL(bitmap_cond_end_sync);
1582

1583
static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed)
1584 1585
{
	/* For each chunk covered by any of these sectors, set the
1586
	 * counter to 2 and possibly set resync_needed.  They should all
1587 1588
	 * be 0 at this point
	 */
1589

N
NeilBrown 已提交
1590
	sector_t secs;
1591 1592 1593 1594
	bitmap_counter_t *bmc;
	spin_lock_irq(&bitmap->lock);
	bmc = bitmap_get_counter(bitmap, offset, &secs, 1);
	if (!bmc) {
1595
		spin_unlock_irq(&bitmap->lock);
1596
		return;
1597
	}
1598
	if (!*bmc) {
1599
		*bmc = 2 | (needed ? NEEDED_MASK : 0);
1600
		bitmap_count_page(bitmap, offset, 1);
1601
		bitmap_set_pending(bitmap, offset);
1602
		bitmap->allclean = 0;
1603 1604
	}
	spin_unlock_irq(&bitmap->lock);
1605 1606
}

1607 1608 1609 1610 1611 1612
/* dirty the memory and file bits for bitmap chunks "s" to "e" */
void bitmap_dirty_bits(struct bitmap *bitmap, unsigned long s, unsigned long e)
{
	unsigned long chunk;

	for (chunk = s; chunk <= e; chunk++) {
1613
		sector_t sec = (sector_t)chunk << bitmap->chunkshift;
1614
		bitmap_set_memory_bits(bitmap, sec, 1);
1615
		spin_lock_irq(&bitmap->lock);
1616
		bitmap_file_set_bit(bitmap, sec);
1617
		spin_unlock_irq(&bitmap->lock);
1618 1619 1620 1621 1622 1623
		if (sec < bitmap->mddev->recovery_cp)
			/* We are asserting that the array is dirty,
			 * so move the recovery_cp address back so
			 * that it is obvious that it is dirty
			 */
			bitmap->mddev->recovery_cp = sec;
1624 1625 1626
	}
}

1627 1628 1629
/*
 * flush out any pending updates
 */
1630
void bitmap_flush(struct mddev *mddev)
1631 1632
{
	struct bitmap *bitmap = mddev->bitmap;
1633
	long sleep;
1634 1635 1636 1637 1638 1639 1640

	if (!bitmap) /* there was no bitmap */
		return;

	/* run the daemon_work three time to ensure everything is flushed
	 * that can be
	 */
1641
	sleep = mddev->bitmap_info.daemon_sleep * 2;
1642
	bitmap->daemon_lastrun -= sleep;
1643
	bitmap_daemon_work(mddev);
1644
	bitmap->daemon_lastrun -= sleep;
1645
	bitmap_daemon_work(mddev);
1646
	bitmap->daemon_lastrun -= sleep;
1647
	bitmap_daemon_work(mddev);
1648 1649 1650
	bitmap_update_sb(bitmap);
}

1651 1652 1653
/*
 * free memory that was allocated
 */
1654
static void bitmap_free(struct bitmap *bitmap)
1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676
{
	unsigned long k, pages;
	struct bitmap_page *bp;

	if (!bitmap) /* there was no bitmap */
		return;

	/* release the bitmap file and kill the daemon */
	bitmap_file_put(bitmap);

	bp = bitmap->bp;
	pages = bitmap->pages;

	/* free all allocated memory */

	if (bp) /* deallocate the page memory */
		for (k = 0; k < pages; k++)
			if (bp[k].map && !bp[k].hijacked)
				kfree(bp[k].map);
	kfree(bp);
	kfree(bitmap);
}
1677

1678
void bitmap_destroy(struct mddev *mddev)
1679 1680 1681 1682 1683 1684
{
	struct bitmap *bitmap = mddev->bitmap;

	if (!bitmap) /* there was no bitmap */
		return;

1685
	mutex_lock(&mddev->bitmap_info.mutex);
1686
	mddev->bitmap = NULL; /* disconnect from the md device */
1687
	mutex_unlock(&mddev->bitmap_info.mutex);
1688 1689
	if (mddev->thread)
		mddev->thread->timeout = MAX_SCHEDULE_TIMEOUT;
1690

1691 1692 1693
	if (bitmap->sysfs_can_clear)
		sysfs_put(bitmap->sysfs_can_clear);

1694 1695
	bitmap_free(bitmap);
}
1696 1697 1698 1699 1700

/*
 * initialize the bitmap structure
 * if this returns an error, bitmap_destroy must be called to do clean up
 */
1701
int bitmap_create(struct mddev *mddev)
1702 1703
{
	struct bitmap *bitmap;
1704
	sector_t blocks = mddev->resync_max_sectors;
1705 1706
	unsigned long chunks;
	unsigned long pages;
1707
	struct file *file = mddev->bitmap_info.file;
1708
	int err;
1709
	struct sysfs_dirent *bm = NULL;
1710

A
Alexey Dobriyan 已提交
1711
	BUILD_BUG_ON(sizeof(bitmap_super_t) != 256);
1712

1713
	BUG_ON(file && mddev->bitmap_info.offset);
1714

1715
	bitmap = kzalloc(sizeof(*bitmap), GFP_KERNEL);
1716 1717 1718 1719
	if (!bitmap)
		return -ENOMEM;

	spin_lock_init(&bitmap->lock);
1720 1721
	atomic_set(&bitmap->pending_writes, 0);
	init_waitqueue_head(&bitmap->write_wait);
1722
	init_waitqueue_head(&bitmap->overflow_wait);
1723
	init_waitqueue_head(&bitmap->behind_wait);
1724

1725 1726
	bitmap->mddev = mddev;

1727 1728
	if (mddev->kobj.sd)
		bm = sysfs_get_dirent(mddev->kobj.sd, NULL, "bitmap");
1729
	if (bm) {
1730
		bitmap->sysfs_can_clear = sysfs_get_dirent(bm, NULL, "can_clear");
1731 1732 1733 1734
		sysfs_put(bm);
	} else
		bitmap->sysfs_can_clear = NULL;

1735
	bitmap->storage.file = file;
1736 1737
	if (file) {
		get_file(file);
1738 1739 1740 1741
		/* As future accesses to this file will use bmap,
		 * and bypass the page cache, we must sync the file
		 * first.
		 */
1742
		vfs_fsync(file, 1);
1743
	}
1744
	/* read superblock from bitmap file (this sets mddev->bitmap_info.chunksize) */
1745 1746 1747 1748 1749 1750 1751 1752 1753 1754
	if (!mddev->bitmap_info.external) {
		/*
		 * If 'MD_ARRAY_FIRST_USE' is set, then device-mapper is
		 * instructing us to create a new on-disk bitmap instance.
		 */
		if (test_and_clear_bit(MD_ARRAY_FIRST_USE, &mddev->flags))
			err = bitmap_new_disk_sb(bitmap);
		else
			err = bitmap_read_sb(bitmap);
	} else {
1755 1756 1757 1758 1759 1760 1761
		err = 0;
		if (mddev->bitmap_info.chunksize == 0 ||
		    mddev->bitmap_info.daemon_sleep == 0)
			/* chunksize and time_base need to be
			 * set first. */
			err = -EINVAL;
	}
1762
	if (err)
1763
		goto error;
1764

1765
	bitmap->daemon_lastrun = jiffies;
1766 1767
	bitmap->chunkshift = (ffz(~mddev->bitmap_info.chunksize)
			      - BITMAP_BLOCK_SHIFT);
1768

1769
	chunks = (blocks + (1 << bitmap->chunkshift) - 1) >>
1770
			bitmap->chunkshift;
1771
	pages = (chunks + PAGE_COUNTER_RATIO - 1) / PAGE_COUNTER_RATIO;
1772 1773 1774 1775 1776 1777 1778

	BUG_ON(!pages);

	bitmap->chunks = chunks;
	bitmap->pages = pages;
	bitmap->missing_pages = pages;

1779
	bitmap->bp = kzalloc(pages * sizeof(*bitmap->bp), GFP_KERNEL);
1780

1781
	err = -ENOMEM;
1782
	if (!bitmap->bp)
1783
		goto error;
1784

1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797
	printk(KERN_INFO "created bitmap (%lu pages) for device %s\n",
		pages, bmname(bitmap));

	mddev->bitmap = bitmap;


	return (bitmap->flags & BITMAP_WRITE_ERROR) ? -EIO : 0;

 error:
	bitmap_free(bitmap);
	return err;
}

1798
int bitmap_load(struct mddev *mddev)
1799 1800
{
	int err = 0;
1801
	sector_t start = 0;
1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813
	sector_t sector = 0;
	struct bitmap *bitmap = mddev->bitmap;

	if (!bitmap)
		goto out;

	/* Clear out old bitmap info first:  Either there is none, or we
	 * are resuming after someone else has possibly changed things,
	 * so we should forget old cached info.
	 * All chunks should be clean, but some might need_sync.
	 */
	while (sector < mddev->resync_max_sectors) {
N
NeilBrown 已提交
1814
		sector_t blocks;
1815 1816 1817 1818 1819
		bitmap_start_sync(bitmap, sector, &blocks, 0);
		sector += blocks;
	}
	bitmap_close_sync(bitmap);

1820 1821 1822 1823 1824 1825
	if (mddev->degraded == 0
	    || bitmap->events_cleared == mddev->events)
		/* no need to keep dirty bits to optimise a
		 * re-add of a missing device */
		start = mddev->recovery_cp;

1826
	mutex_lock(&mddev->bitmap_info.mutex);
1827
	err = bitmap_init_from_disk(bitmap, start);
1828
	mutex_unlock(&mddev->bitmap_info.mutex);
1829

1830
	if (err)
1831
		goto out;
1832 1833 1834 1835
	bitmap_mask_state(bitmap, BITMAP_STALE, MASK_UNSET);

	/* Kick recovery in case any bits were set */
	set_bit(MD_RECOVERY_NEEDED, &bitmap->mddev->recovery);
1836

1837
	mddev->thread->timeout = mddev->bitmap_info.daemon_sleep;
1838
	md_wakeup_thread(mddev->thread);
1839

1840 1841
	bitmap_update_sb(bitmap);

1842 1843 1844
	if (bitmap->flags & BITMAP_WRITE_ERROR)
		err = -EIO;
out:
1845
	return err;
1846
}
1847
EXPORT_SYMBOL_GPL(bitmap_load);
1848

1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866
void bitmap_status(struct seq_file *seq, struct bitmap *bitmap)
{
	unsigned long chunk_kb;
	unsigned long flags;

	if (!bitmap)
		return;

	spin_lock_irqsave(&bitmap->lock, flags);
	chunk_kb = bitmap->mddev->bitmap_info.chunksize >> 10;
	seq_printf(seq, "bitmap: %lu/%lu pages [%luKB], "
		   "%lu%s chunk",
		   bitmap->pages - bitmap->missing_pages,
		   bitmap->pages,
		   (bitmap->pages - bitmap->missing_pages)
		   << (PAGE_SHIFT - 10),
		   chunk_kb ? chunk_kb : bitmap->mddev->bitmap_info.chunksize,
		   chunk_kb ? "KB" : "B");
1867
	if (bitmap->storage.file) {
1868
		seq_printf(seq, ", file: ");
1869
		seq_path(seq, &bitmap->storage.file->f_path, " \t\n");
1870 1871 1872 1873 1874 1875
	}

	seq_printf(seq, "\n");
	spin_unlock_irqrestore(&bitmap->lock, flags);
}

1876
static ssize_t
1877
location_show(struct mddev *mddev, char *page)
1878 1879
{
	ssize_t len;
1880
	if (mddev->bitmap_info.file)
1881
		len = sprintf(page, "file");
1882
	else if (mddev->bitmap_info.offset)
1883
		len = sprintf(page, "%+lld", (long long)mddev->bitmap_info.offset);
1884
	else
1885 1886 1887 1888 1889 1890
		len = sprintf(page, "none");
	len += sprintf(page+len, "\n");
	return len;
}

static ssize_t
1891
location_store(struct mddev *mddev, const char *buf, size_t len)
1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935
{

	if (mddev->pers) {
		if (!mddev->pers->quiesce)
			return -EBUSY;
		if (mddev->recovery || mddev->sync_thread)
			return -EBUSY;
	}

	if (mddev->bitmap || mddev->bitmap_info.file ||
	    mddev->bitmap_info.offset) {
		/* bitmap already configured.  Only option is to clear it */
		if (strncmp(buf, "none", 4) != 0)
			return -EBUSY;
		if (mddev->pers) {
			mddev->pers->quiesce(mddev, 1);
			bitmap_destroy(mddev);
			mddev->pers->quiesce(mddev, 0);
		}
		mddev->bitmap_info.offset = 0;
		if (mddev->bitmap_info.file) {
			struct file *f = mddev->bitmap_info.file;
			mddev->bitmap_info.file = NULL;
			restore_bitmap_write_access(f);
			fput(f);
		}
	} else {
		/* No bitmap, OK to set a location */
		long long offset;
		if (strncmp(buf, "none", 4) == 0)
			/* nothing to be done */;
		else if (strncmp(buf, "file:", 5) == 0) {
			/* Not supported yet */
			return -EINVAL;
		} else {
			int rv;
			if (buf[0] == '+')
				rv = strict_strtoll(buf+1, 10, &offset);
			else
				rv = strict_strtoll(buf, 10, &offset);
			if (rv)
				return rv;
			if (offset == 0)
				return -EINVAL;
1936 1937
			if (mddev->bitmap_info.external == 0 &&
			    mddev->major_version == 0 &&
1938 1939 1940 1941 1942 1943
			    offset != mddev->bitmap_info.default_offset)
				return -EINVAL;
			mddev->bitmap_info.offset = offset;
			if (mddev->pers) {
				mddev->pers->quiesce(mddev, 1);
				rv = bitmap_create(mddev);
1944 1945
				if (!rv)
					rv = bitmap_load(mddev);
1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968
				if (rv) {
					bitmap_destroy(mddev);
					mddev->bitmap_info.offset = 0;
				}
				mddev->pers->quiesce(mddev, 0);
				if (rv)
					return rv;
			}
		}
	}
	if (!mddev->external) {
		/* Ensure new bitmap info is stored in
		 * metadata promptly.
		 */
		set_bit(MD_CHANGE_DEVS, &mddev->flags);
		md_wakeup_thread(mddev->thread);
	}
	return len;
}

static struct md_sysfs_entry bitmap_location =
__ATTR(location, S_IRUGO|S_IWUSR, location_show, location_store);

1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992
/* 'bitmap/space' is the space available at 'location' for the
 * bitmap.  This allows the kernel to know when it is safe to
 * resize the bitmap to match a resized array.
 */
static ssize_t
space_show(struct mddev *mddev, char *page)
{
	return sprintf(page, "%lu\n", mddev->bitmap_info.space);
}

static ssize_t
space_store(struct mddev *mddev, const char *buf, size_t len)
{
	unsigned long sectors;
	int rv;

	rv = kstrtoul(buf, 10, &sectors);
	if (rv)
		return rv;

	if (sectors == 0)
		return -EINVAL;

	if (mddev->bitmap &&
1993
	    sectors < (mddev->bitmap->storage.bytes + 511) >> 9)
1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005
		return -EFBIG; /* Bitmap is too big for this small space */

	/* could make sure it isn't too big, but that isn't really
	 * needed - user-space should be careful.
	 */
	mddev->bitmap_info.space = sectors;
	return len;
}

static struct md_sysfs_entry bitmap_space =
__ATTR(space, S_IRUGO|S_IWUSR, space_show, space_store);

2006
static ssize_t
2007
timeout_show(struct mddev *mddev, char *page)
2008 2009 2010 2011
{
	ssize_t len;
	unsigned long secs = mddev->bitmap_info.daemon_sleep / HZ;
	unsigned long jifs = mddev->bitmap_info.daemon_sleep % HZ;
2012

2013 2014 2015 2016 2017 2018 2019 2020
	len = sprintf(page, "%lu", secs);
	if (jifs)
		len += sprintf(page+len, ".%03u", jiffies_to_msecs(jifs));
	len += sprintf(page+len, "\n");
	return len;
}

static ssize_t
2021
timeout_store(struct mddev *mddev, const char *buf, size_t len)
2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056
{
	/* timeout can be set at any time */
	unsigned long timeout;
	int rv = strict_strtoul_scaled(buf, &timeout, 4);
	if (rv)
		return rv;

	/* just to make sure we don't overflow... */
	if (timeout >= LONG_MAX / HZ)
		return -EINVAL;

	timeout = timeout * HZ / 10000;

	if (timeout >= MAX_SCHEDULE_TIMEOUT)
		timeout = MAX_SCHEDULE_TIMEOUT-1;
	if (timeout < 1)
		timeout = 1;
	mddev->bitmap_info.daemon_sleep = timeout;
	if (mddev->thread) {
		/* if thread->timeout is MAX_SCHEDULE_TIMEOUT, then
		 * the bitmap is all clean and we don't need to
		 * adjust the timeout right now
		 */
		if (mddev->thread->timeout < MAX_SCHEDULE_TIMEOUT) {
			mddev->thread->timeout = timeout;
			md_wakeup_thread(mddev->thread);
		}
	}
	return len;
}

static struct md_sysfs_entry bitmap_timeout =
__ATTR(time_base, S_IRUGO|S_IWUSR, timeout_show, timeout_store);

static ssize_t
2057
backlog_show(struct mddev *mddev, char *page)
2058 2059 2060 2061 2062
{
	return sprintf(page, "%lu\n", mddev->bitmap_info.max_write_behind);
}

static ssize_t
2063
backlog_store(struct mddev *mddev, const char *buf, size_t len)
2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078
{
	unsigned long backlog;
	int rv = strict_strtoul(buf, 10, &backlog);
	if (rv)
		return rv;
	if (backlog > COUNTER_MAX)
		return -EINVAL;
	mddev->bitmap_info.max_write_behind = backlog;
	return len;
}

static struct md_sysfs_entry bitmap_backlog =
__ATTR(backlog, S_IRUGO|S_IWUSR, backlog_show, backlog_store);

static ssize_t
2079
chunksize_show(struct mddev *mddev, char *page)
2080 2081 2082 2083 2084
{
	return sprintf(page, "%lu\n", mddev->bitmap_info.chunksize);
}

static ssize_t
2085
chunksize_store(struct mddev *mddev, const char *buf, size_t len)
2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104
{
	/* Can only be changed when no bitmap is active */
	int rv;
	unsigned long csize;
	if (mddev->bitmap)
		return -EBUSY;
	rv = strict_strtoul(buf, 10, &csize);
	if (rv)
		return rv;
	if (csize < 512 ||
	    !is_power_of_2(csize))
		return -EINVAL;
	mddev->bitmap_info.chunksize = csize;
	return len;
}

static struct md_sysfs_entry bitmap_chunksize =
__ATTR(chunksize, S_IRUGO|S_IWUSR, chunksize_show, chunksize_store);

2105
static ssize_t metadata_show(struct mddev *mddev, char *page)
2106 2107 2108 2109 2110
{
	return sprintf(page, "%s\n", (mddev->bitmap_info.external
				      ? "external" : "internal"));
}

2111
static ssize_t metadata_store(struct mddev *mddev, const char *buf, size_t len)
2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128
{
	if (mddev->bitmap ||
	    mddev->bitmap_info.file ||
	    mddev->bitmap_info.offset)
		return -EBUSY;
	if (strncmp(buf, "external", 8) == 0)
		mddev->bitmap_info.external = 1;
	else if (strncmp(buf, "internal", 8) == 0)
		mddev->bitmap_info.external = 0;
	else
		return -EINVAL;
	return len;
}

static struct md_sysfs_entry bitmap_metadata =
__ATTR(metadata, S_IRUGO|S_IWUSR, metadata_show, metadata_store);

2129
static ssize_t can_clear_show(struct mddev *mddev, char *page)
2130 2131 2132 2133 2134 2135 2136 2137 2138 2139
{
	int len;
	if (mddev->bitmap)
		len = sprintf(page, "%s\n", (mddev->bitmap->need_sync ?
					     "false" : "true"));
	else
		len = sprintf(page, "\n");
	return len;
}

2140
static ssize_t can_clear_store(struct mddev *mddev, const char *buf, size_t len)
2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156 2157
{
	if (mddev->bitmap == NULL)
		return -ENOENT;
	if (strncmp(buf, "false", 5) == 0)
		mddev->bitmap->need_sync = 1;
	else if (strncmp(buf, "true", 4) == 0) {
		if (mddev->degraded)
			return -EBUSY;
		mddev->bitmap->need_sync = 0;
	} else
		return -EINVAL;
	return len;
}

static struct md_sysfs_entry bitmap_can_clear =
__ATTR(can_clear, S_IRUGO|S_IWUSR, can_clear_show, can_clear_store);

2158
static ssize_t
2159
behind_writes_used_show(struct mddev *mddev, char *page)
2160 2161 2162 2163 2164 2165 2166 2167
{
	if (mddev->bitmap == NULL)
		return sprintf(page, "0\n");
	return sprintf(page, "%lu\n",
		       mddev->bitmap->behind_writes_used);
}

static ssize_t
2168
behind_writes_used_reset(struct mddev *mddev, const char *buf, size_t len)
2169 2170 2171 2172 2173 2174 2175 2176 2177 2178
{
	if (mddev->bitmap)
		mddev->bitmap->behind_writes_used = 0;
	return len;
}

static struct md_sysfs_entry max_backlog_used =
__ATTR(max_backlog_used, S_IRUGO | S_IWUSR,
       behind_writes_used_show, behind_writes_used_reset);

2179 2180
static struct attribute *md_bitmap_attrs[] = {
	&bitmap_location.attr,
2181
	&bitmap_space.attr,
2182 2183 2184
	&bitmap_timeout.attr,
	&bitmap_backlog.attr,
	&bitmap_chunksize.attr,
2185 2186
	&bitmap_metadata.attr,
	&bitmap_can_clear.attr,
2187
	&max_backlog_used.attr,
2188 2189 2190 2191 2192 2193 2194
	NULL
};
struct attribute_group md_bitmap_group = {
	.name = "bitmap",
	.attrs = md_bitmap_attrs,
};