super.c 50.1 KB
Newer Older
K
Kent Overstreet 已提交
1 2 3 4 5 6 7 8 9 10 11
/*
 * bcache setup/teardown code, and some metadata io - read a superblock and
 * figure out what to do with it.
 *
 * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
 * Copyright 2012 Google, Inc.
 */

#include "bcache.h"
#include "btree.h"
#include "debug.h"
12
#include "extents.h"
K
Kent Overstreet 已提交
13
#include "request.h"
14
#include "writeback.h"
K
Kent Overstreet 已提交
15

K
Kent Overstreet 已提交
16
#include <linux/blkdev.h>
K
Kent Overstreet 已提交
17 18 19
#include <linux/buffer_head.h>
#include <linux/debugfs.h>
#include <linux/genhd.h>
20
#include <linux/idr.h>
21
#include <linux/kthread.h>
K
Kent Overstreet 已提交
22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54
#include <linux/module.h>
#include <linux/random.h>
#include <linux/reboot.h>
#include <linux/sysfs.h>

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Kent Overstreet <kent.overstreet@gmail.com>");

static const char bcache_magic[] = {
	0xc6, 0x85, 0x73, 0xf6, 0x4e, 0x1a, 0x45, 0xca,
	0x82, 0x65, 0xf5, 0x7f, 0x48, 0xba, 0x6d, 0x81
};

static const char invalid_uuid[] = {
	0xa0, 0x3e, 0xf8, 0xed, 0x3e, 0xe1, 0xb8, 0x78,
	0xc8, 0x50, 0xfc, 0x5e, 0xcb, 0x16, 0xcd, 0x99
};

/* Default is -1; we skip past it for struct cached_dev's cache mode */
const char * const bch_cache_modes[] = {
	"default",
	"writethrough",
	"writeback",
	"writearound",
	"none",
	NULL
};

static struct kobject *bcache_kobj;
struct mutex bch_register_lock;
LIST_HEAD(bch_cache_sets);
static LIST_HEAD(uncached_devices);

55 56
static int bcache_major;
static DEFINE_IDA(bcache_minor);
K
Kent Overstreet 已提交
57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112
static wait_queue_head_t unregister_wait;
struct workqueue_struct *bcache_wq;

#define BTREE_MAX_PAGES		(256 * 1024 / PAGE_SIZE)

/* Superblock */

static const char *read_super(struct cache_sb *sb, struct block_device *bdev,
			      struct page **res)
{
	const char *err;
	struct cache_sb *s;
	struct buffer_head *bh = __bread(bdev, 1, SB_SIZE);
	unsigned i;

	if (!bh)
		return "IO error";

	s = (struct cache_sb *) bh->b_data;

	sb->offset		= le64_to_cpu(s->offset);
	sb->version		= le64_to_cpu(s->version);

	memcpy(sb->magic,	s->magic, 16);
	memcpy(sb->uuid,	s->uuid, 16);
	memcpy(sb->set_uuid,	s->set_uuid, 16);
	memcpy(sb->label,	s->label, SB_LABEL_SIZE);

	sb->flags		= le64_to_cpu(s->flags);
	sb->seq			= le64_to_cpu(s->seq);
	sb->last_mount		= le32_to_cpu(s->last_mount);
	sb->first_bucket	= le16_to_cpu(s->first_bucket);
	sb->keys		= le16_to_cpu(s->keys);

	for (i = 0; i < SB_JOURNAL_BUCKETS; i++)
		sb->d[i] = le64_to_cpu(s->d[i]);

	pr_debug("read sb version %llu, flags %llu, seq %llu, journal size %u",
		 sb->version, sb->flags, sb->seq, sb->keys);

	err = "Not a bcache superblock";
	if (sb->offset != SB_SECTOR)
		goto err;

	if (memcmp(sb->magic, bcache_magic, 16))
		goto err;

	err = "Too many journal buckets";
	if (sb->keys > SB_JOURNAL_BUCKETS)
		goto err;

	err = "Bad checksum";
	if (s->csum != csum_set(s))
		goto err;

	err = "Bad UUID";
113
	if (bch_is_zero(sb->uuid, 16))
K
Kent Overstreet 已提交
114 115
		goto err;

116 117 118 119 120 121
	sb->block_size	= le16_to_cpu(s->block_size);

	err = "Superblock block size smaller than device block size";
	if (sb->block_size << 9 < bdev_logical_block_size(bdev))
		goto err;

122 123 124 125 126 127 128 129 130 131
	switch (sb->version) {
	case BCACHE_SB_VERSION_BDEV:
		sb->data_offset	= BDEV_DATA_START_DEFAULT;
		break;
	case BCACHE_SB_VERSION_BDEV_WITH_OFFSET:
		sb->data_offset	= le64_to_cpu(s->data_offset);

		err = "Bad data offset";
		if (sb->data_offset < BDEV_DATA_START_DEFAULT)
			goto err;
K
Kent Overstreet 已提交
132

133 134 135 136 137
		break;
	case BCACHE_SB_VERSION_CDEV:
	case BCACHE_SB_VERSION_CDEV_WITH_UUID:
		sb->nbuckets	= le64_to_cpu(s->nbuckets);
		sb->bucket_size	= le16_to_cpu(s->bucket_size);
K
Kent Overstreet 已提交
138

139 140
		sb->nr_in_set	= le16_to_cpu(s->nr_in_set);
		sb->nr_this_dev	= le16_to_cpu(s->nr_this_dev);
K
Kent Overstreet 已提交
141

142 143 144
		err = "Too many buckets";
		if (sb->nbuckets > LONG_MAX)
			goto err;
K
Kent Overstreet 已提交
145

146 147 148
		err = "Not enough buckets";
		if (sb->nbuckets < 1 << 7)
			goto err;
K
Kent Overstreet 已提交
149

150 151 152 153 154 155
		err = "Bad block/bucket size";
		if (!is_power_of_2(sb->block_size) ||
		    sb->block_size > PAGE_SECTORS ||
		    !is_power_of_2(sb->bucket_size) ||
		    sb->bucket_size < PAGE_SECTORS)
			goto err;
K
Kent Overstreet 已提交
156

157 158 159
		err = "Invalid superblock: device too small";
		if (get_capacity(bdev->bd_disk) < sb->bucket_size * sb->nbuckets)
			goto err;
K
Kent Overstreet 已提交
160

161 162 163
		err = "Bad UUID";
		if (bch_is_zero(sb->set_uuid, 16))
			goto err;
K
Kent Overstreet 已提交
164

165 166 167 168
		err = "Bad cache device number in set";
		if (!sb->nr_in_set ||
		    sb->nr_in_set <= sb->nr_this_dev ||
		    sb->nr_in_set > MAX_CACHES_PER_SET)
K
Kent Overstreet 已提交
169 170
			goto err;

171 172 173 174
		err = "Journal buckets not sequential";
		for (i = 0; i < sb->keys; i++)
			if (sb->d[i] != sb->first_bucket + i)
				goto err;
K
Kent Overstreet 已提交
175

176 177 178 179 180 181 182 183 184 185 186
		err = "Too many journal buckets";
		if (sb->first_bucket + sb->keys > sb->nbuckets)
			goto err;

		err = "Invalid superblock: first bucket comes before end of super";
		if (sb->first_bucket * sb->bucket_size < 16)
			goto err;

		break;
	default:
		err = "Unsupported superblock version";
K
Kent Overstreet 已提交
187
		goto err;
188 189
	}

K
Kent Overstreet 已提交
190 191 192 193 194 195 196 197 198 199
	sb->last_mount = get_seconds();
	err = NULL;

	get_page(bh->b_page);
	*res = bh->b_page;
err:
	put_bh(bh);
	return err;
}

200
static void write_bdev_super_endio(struct bio *bio)
K
Kent Overstreet 已提交
201 202 203 204
{
	struct cached_dev *dc = bio->bi_private;
	/* XXX: error checking */

205
	closure_put(&dc->sb_write);
K
Kent Overstreet 已提交
206 207 208 209 210 211 212
}

static void __write_super(struct cache_sb *sb, struct bio *bio)
{
	struct cache_sb *out = page_address(bio->bi_io_vec[0].bv_page);
	unsigned i;

213 214
	bio->bi_iter.bi_sector	= SB_SECTOR;
	bio->bi_iter.bi_size	= SB_SIZE;
M
Mike Christie 已提交
215
	bio_set_op_attrs(bio, REQ_OP_WRITE, REQ_SYNC|REQ_META);
216
	bch_bio_map(bio, NULL);
K
Kent Overstreet 已提交
217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239

	out->offset		= cpu_to_le64(sb->offset);
	out->version		= cpu_to_le64(sb->version);

	memcpy(out->uuid,	sb->uuid, 16);
	memcpy(out->set_uuid,	sb->set_uuid, 16);
	memcpy(out->label,	sb->label, SB_LABEL_SIZE);

	out->flags		= cpu_to_le64(sb->flags);
	out->seq		= cpu_to_le64(sb->seq);

	out->last_mount		= cpu_to_le32(sb->last_mount);
	out->first_bucket	= cpu_to_le16(sb->first_bucket);
	out->keys		= cpu_to_le16(sb->keys);

	for (i = 0; i < sb->keys; i++)
		out->d[i] = cpu_to_le64(sb->d[i]);

	out->csum = csum_set(out);

	pr_debug("ver %llu, flags %llu, seq %llu",
		 sb->version, sb->flags, sb->seq);

240
	submit_bio(bio);
K
Kent Overstreet 已提交
241 242
}

243 244 245 246 247 248 249
static void bch_write_bdev_super_unlock(struct closure *cl)
{
	struct cached_dev *dc = container_of(cl, struct cached_dev, sb_write);

	up(&dc->sb_write_mutex);
}

K
Kent Overstreet 已提交
250 251
void bch_write_bdev_super(struct cached_dev *dc, struct closure *parent)
{
252
	struct closure *cl = &dc->sb_write;
K
Kent Overstreet 已提交
253 254
	struct bio *bio = &dc->sb_bio;

255 256
	down(&dc->sb_write_mutex);
	closure_init(cl, parent);
K
Kent Overstreet 已提交
257 258 259 260 261 262 263 264 265

	bio_reset(bio);
	bio->bi_bdev	= dc->bdev;
	bio->bi_end_io	= write_bdev_super_endio;
	bio->bi_private = dc;

	closure_get(cl);
	__write_super(&dc->sb, bio);

266
	closure_return_with_destructor(cl, bch_write_bdev_super_unlock);
K
Kent Overstreet 已提交
267 268
}

269
static void write_super_endio(struct bio *bio)
K
Kent Overstreet 已提交
270 271 272
{
	struct cache *ca = bio->bi_private;

273
	bch_count_io_errors(ca, bio->bi_error, "writing superblock");
274 275 276 277 278 279 280 281
	closure_put(&ca->set->sb_write);
}

static void bcache_write_super_unlock(struct closure *cl)
{
	struct cache_set *c = container_of(cl, struct cache_set, sb_write);

	up(&c->sb_write_mutex);
K
Kent Overstreet 已提交
282 283 284 285
}

void bcache_write_super(struct cache_set *c)
{
286
	struct closure *cl = &c->sb_write;
K
Kent Overstreet 已提交
287 288 289
	struct cache *ca;
	unsigned i;

290 291
	down(&c->sb_write_mutex);
	closure_init(cl, &c->cl);
K
Kent Overstreet 已提交
292 293 294 295 296 297

	c->sb.seq++;

	for_each_cache(ca, c, i) {
		struct bio *bio = &ca->sb_bio;

298
		ca->sb.version		= BCACHE_SB_VERSION_CDEV_WITH_UUID;
K
Kent Overstreet 已提交
299 300 301 302 303 304 305 306 307 308 309 310 311 312
		ca->sb.seq		= c->sb.seq;
		ca->sb.last_mount	= c->sb.last_mount;

		SET_CACHE_SYNC(&ca->sb, CACHE_SYNC(&c->sb));

		bio_reset(bio);
		bio->bi_bdev	= ca->bdev;
		bio->bi_end_io	= write_super_endio;
		bio->bi_private = ca;

		closure_get(cl);
		__write_super(&ca->sb, bio);
	}

313
	closure_return_with_destructor(cl, bcache_write_super_unlock);
K
Kent Overstreet 已提交
314 315 316 317
}

/* UUID io */

318
static void uuid_endio(struct bio *bio)
K
Kent Overstreet 已提交
319 320
{
	struct closure *cl = bio->bi_private;
321
	struct cache_set *c = container_of(cl, struct cache_set, uuid_write);
K
Kent Overstreet 已提交
322

323
	cache_set_err_on(bio->bi_error, c, "accessing uuids");
K
Kent Overstreet 已提交
324 325 326 327
	bch_bbio_free(bio, c);
	closure_put(cl);
}

328 329 330 331 332 333 334
static void uuid_io_unlock(struct closure *cl)
{
	struct cache_set *c = container_of(cl, struct cache_set, uuid_write);

	up(&c->uuid_write_mutex);
}

M
Mike Christie 已提交
335
static void uuid_io(struct cache_set *c, int op, unsigned long op_flags,
K
Kent Overstreet 已提交
336 337
		    struct bkey *k, struct closure *parent)
{
338
	struct closure *cl = &c->uuid_write;
K
Kent Overstreet 已提交
339 340
	struct uuid_entry *u;
	unsigned i;
341
	char buf[80];
K
Kent Overstreet 已提交
342 343

	BUG_ON(!parent);
344 345
	down(&c->uuid_write_mutex);
	closure_init(cl, parent);
K
Kent Overstreet 已提交
346 347 348 349

	for (i = 0; i < KEY_PTRS(k); i++) {
		struct bio *bio = bch_bbio_alloc(c);

M
Mike Christie 已提交
350
		bio->bi_rw	= REQ_SYNC|REQ_META|op_flags;
351
		bio->bi_iter.bi_size = KEY_SIZE(k) << 9;
K
Kent Overstreet 已提交
352 353 354

		bio->bi_end_io	= uuid_endio;
		bio->bi_private = cl;
M
Mike Christie 已提交
355
		bio_set_op_attrs(bio, op, REQ_SYNC|REQ_META|op_flags);
356
		bch_bio_map(bio, c->uuids);
K
Kent Overstreet 已提交
357 358 359

		bch_submit_bbio(bio, c, k, i);

M
Mike Christie 已提交
360
		if (op != REQ_OP_WRITE)
K
Kent Overstreet 已提交
361 362 363
			break;
	}

364
	bch_extent_to_text(buf, sizeof(buf), k);
M
Mike Christie 已提交
365
	pr_debug("%s UUIDs at %s", op == REQ_OP_WRITE ? "wrote" : "read", buf);
K
Kent Overstreet 已提交
366 367

	for (u = c->uuids; u < c->uuids + c->nr_uuids; u++)
368
		if (!bch_is_zero(u->uuid, 16))
K
Kent Overstreet 已提交
369 370 371 372
			pr_debug("Slot %zi: %pU: %s: 1st: %u last: %u inv: %u",
				 u - c->uuids, u->uuid, u->label,
				 u->first_reg, u->last_reg, u->invalidated);

373
	closure_return_with_destructor(cl, uuid_io_unlock);
K
Kent Overstreet 已提交
374 375 376 377 378 379
}

static char *uuid_read(struct cache_set *c, struct jset *j, struct closure *cl)
{
	struct bkey *k = &j->uuid_bucket;

380
	if (__bch_btree_ptr_invalid(c, k))
K
Kent Overstreet 已提交
381 382 383
		return "bad uuid pointer";

	bkey_copy(&c->uuid_bucket, k);
M
Mike Christie 已提交
384
	uuid_io(c, REQ_OP_READ, READ_SYNC, k, cl);
K
Kent Overstreet 已提交
385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424

	if (j->version < BCACHE_JSET_VERSION_UUIDv1) {
		struct uuid_entry_v0	*u0 = (void *) c->uuids;
		struct uuid_entry	*u1 = (void *) c->uuids;
		int i;

		closure_sync(cl);

		/*
		 * Since the new uuid entry is bigger than the old, we have to
		 * convert starting at the highest memory address and work down
		 * in order to do it in place
		 */

		for (i = c->nr_uuids - 1;
		     i >= 0;
		     --i) {
			memcpy(u1[i].uuid,	u0[i].uuid, 16);
			memcpy(u1[i].label,	u0[i].label, 32);

			u1[i].first_reg		= u0[i].first_reg;
			u1[i].last_reg		= u0[i].last_reg;
			u1[i].invalidated	= u0[i].invalidated;

			u1[i].flags	= 0;
			u1[i].sectors	= 0;
		}
	}

	return NULL;
}

static int __uuid_write(struct cache_set *c)
{
	BKEY_PADDED(key) k;
	struct closure cl;
	closure_init_stack(&cl);

	lockdep_assert_held(&bch_register_lock);

425
	if (bch_bucket_alloc_set(c, RESERVE_BTREE, &k.key, 1, true))
K
Kent Overstreet 已提交
426 427 428
		return 1;

	SET_KEY_SIZE(&k.key, c->sb.bucket_size);
M
Mike Christie 已提交
429
	uuid_io(c, REQ_OP_WRITE, 0, &k.key, &cl);
K
Kent Overstreet 已提交
430 431 432
	closure_sync(&cl);

	bkey_copy(&c->uuid_bucket, &k.key);
433
	bkey_put(c, &k.key);
K
Kent Overstreet 已提交
434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491
	return 0;
}

int bch_uuid_write(struct cache_set *c)
{
	int ret = __uuid_write(c);

	if (!ret)
		bch_journal_meta(c, NULL);

	return ret;
}

static struct uuid_entry *uuid_find(struct cache_set *c, const char *uuid)
{
	struct uuid_entry *u;

	for (u = c->uuids;
	     u < c->uuids + c->nr_uuids; u++)
		if (!memcmp(u->uuid, uuid, 16))
			return u;

	return NULL;
}

static struct uuid_entry *uuid_find_empty(struct cache_set *c)
{
	static const char zero_uuid[16] = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0";
	return uuid_find(c, zero_uuid);
}

/*
 * Bucket priorities/gens:
 *
 * For each bucket, we store on disk its
   * 8 bit gen
   * 16 bit priority
 *
 * See alloc.c for an explanation of the gen. The priority is used to implement
 * lru (and in the future other) cache replacement policies; for most purposes
 * it's just an opaque integer.
 *
 * The gens and the priorities don't have a whole lot to do with each other, and
 * it's actually the gens that must be written out at specific times - it's no
 * big deal if the priorities don't get written, if we lose them we just reuse
 * buckets in suboptimal order.
 *
 * On disk they're stored in a packed array, and in as many buckets are required
 * to fit them all. The buckets we use to store them form a list; the journal
 * header points to the first bucket, the first bucket points to the second
 * bucket, et cetera.
 *
 * This code is used by the allocation code; periodically (whenever it runs out
 * of buckets to allocate from) the allocation code will invalidate some
 * buckets, but it can't use those buckets until their new gens are safely on
 * disk.
 */

492
static void prio_endio(struct bio *bio)
K
Kent Overstreet 已提交
493 494 495
{
	struct cache *ca = bio->bi_private;

496
	cache_set_err_on(bio->bi_error, ca->set, "accessing priorities");
K
Kent Overstreet 已提交
497 498 499 500
	bch_bbio_free(bio, ca->set);
	closure_put(&ca->prio);
}

M
Mike Christie 已提交
501 502
static void prio_io(struct cache *ca, uint64_t bucket, int op,
		    unsigned long op_flags)
K
Kent Overstreet 已提交
503 504 505 506 507 508
{
	struct closure *cl = &ca->prio;
	struct bio *bio = bch_bbio_alloc(ca->set);

	closure_init_stack(cl);

509 510 511
	bio->bi_iter.bi_sector	= bucket * ca->sb.bucket_size;
	bio->bi_bdev		= ca->bdev;
	bio->bi_iter.bi_size	= bucket_bytes(ca);
K
Kent Overstreet 已提交
512 513 514

	bio->bi_end_io	= prio_endio;
	bio->bi_private = ca;
M
Mike Christie 已提交
515
	bio_set_op_attrs(bio, op, REQ_SYNC|REQ_META|op_flags);
516
	bch_bio_map(bio, ca->disk_buckets);
K
Kent Overstreet 已提交
517

518
	closure_bio_submit(bio, &ca->prio);
K
Kent Overstreet 已提交
519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536
	closure_sync(cl);
}

void bch_prio_write(struct cache *ca)
{
	int i;
	struct bucket *b;
	struct closure cl;

	closure_init_stack(&cl);

	lockdep_assert_held(&ca->set->bucket_lock);

	ca->disk_buckets->seq++;

	atomic_long_add(ca->sb.bucket_size * prio_buckets(ca),
			&ca->meta_sectors_written);

537 538
	//pr_debug("free %zu, free_inc %zu, unused %zu", fifo_used(&ca->free),
	//	 fifo_used(&ca->free_inc), fifo_used(&ca->unused));
K
Kent Overstreet 已提交
539 540 541 542

	for (i = prio_buckets(ca) - 1; i >= 0; --i) {
		long bucket;
		struct prio_set *p = ca->disk_buckets;
K
Kent Overstreet 已提交
543 544
		struct bucket_disk *d = p->data;
		struct bucket_disk *end = d + prios_per_bucket(ca);
K
Kent Overstreet 已提交
545 546 547 548 549 550 551 552 553

		for (b = ca->buckets + i * prios_per_bucket(ca);
		     b < ca->buckets + ca->sb.nbuckets && d < end;
		     b++, d++) {
			d->prio = cpu_to_le16(b->prio);
			d->gen = b->gen;
		}

		p->next_bucket	= ca->prio_buckets[i + 1];
554
		p->magic	= pset_magic(&ca->sb);
555
		p->csum		= bch_crc64(&p->magic, bucket_bytes(ca) - 8);
K
Kent Overstreet 已提交
556

557
		bucket = bch_bucket_alloc(ca, RESERVE_PRIO, true);
K
Kent Overstreet 已提交
558 559 560
		BUG_ON(bucket == -1);

		mutex_unlock(&ca->set->bucket_lock);
M
Mike Christie 已提交
561
		prio_io(ca, bucket, REQ_OP_WRITE, 0);
K
Kent Overstreet 已提交
562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578
		mutex_lock(&ca->set->bucket_lock);

		ca->prio_buckets[i] = bucket;
		atomic_dec_bug(&ca->buckets[bucket].pin);
	}

	mutex_unlock(&ca->set->bucket_lock);

	bch_journal_meta(ca->set, &cl);
	closure_sync(&cl);

	mutex_lock(&ca->set->bucket_lock);

	/*
	 * Don't want the old priorities to get garbage collected until after we
	 * finish writing the new ones, and they're journalled
	 */
K
Kent Overstreet 已提交
579 580 581 582 583
	for (i = 0; i < prio_buckets(ca); i++) {
		if (ca->prio_last_buckets[i])
			__bch_bucket_free(ca,
				&ca->buckets[ca->prio_last_buckets[i]]);

K
Kent Overstreet 已提交
584
		ca->prio_last_buckets[i] = ca->prio_buckets[i];
K
Kent Overstreet 已提交
585
	}
K
Kent Overstreet 已提交
586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602
}

static void prio_read(struct cache *ca, uint64_t bucket)
{
	struct prio_set *p = ca->disk_buckets;
	struct bucket_disk *d = p->data + prios_per_bucket(ca), *end = d;
	struct bucket *b;
	unsigned bucket_nr = 0;

	for (b = ca->buckets;
	     b < ca->buckets + ca->sb.nbuckets;
	     b++, d++) {
		if (d == end) {
			ca->prio_buckets[bucket_nr] = bucket;
			ca->prio_last_buckets[bucket_nr] = bucket;
			bucket_nr++;

M
Mike Christie 已提交
603
			prio_io(ca, bucket, REQ_OP_READ, READ_SYNC);
K
Kent Overstreet 已提交
604

605
			if (p->csum != bch_crc64(&p->magic, bucket_bytes(ca) - 8))
K
Kent Overstreet 已提交
606 607
				pr_warn("bad csum reading priorities");

608
			if (p->magic != pset_magic(&ca->sb))
K
Kent Overstreet 已提交
609 610 611 612 613 614 615
				pr_warn("bad magic reading priorities");

			bucket = p->next_bucket;
			d = p->data;
		}

		b->prio = le16_to_cpu(d->prio);
K
Kent Overstreet 已提交
616
		b->gen = b->last_gc = d->gen;
K
Kent Overstreet 已提交
617 618 619 620 621 622 623 624
	}
}

/* Bcache device */

static int open_dev(struct block_device *b, fmode_t mode)
{
	struct bcache_device *d = b->bd_disk->private_data;
625
	if (test_bit(BCACHE_DEV_CLOSING, &d->flags))
K
Kent Overstreet 已提交
626 627 628 629 630 631
		return -ENXIO;

	closure_get(&d->cl);
	return 0;
}

632
static void release_dev(struct gendisk *b, fmode_t mode)
K
Kent Overstreet 已提交
633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653
{
	struct bcache_device *d = b->private_data;
	closure_put(&d->cl);
}

static int ioctl_dev(struct block_device *b, fmode_t mode,
		     unsigned int cmd, unsigned long arg)
{
	struct bcache_device *d = b->bd_disk->private_data;
	return d->ioctl(d, mode, cmd, arg);
}

static const struct block_device_operations bcache_ops = {
	.open		= open_dev,
	.release	= release_dev,
	.ioctl		= ioctl_dev,
	.owner		= THIS_MODULE,
};

void bcache_device_stop(struct bcache_device *d)
{
654
	if (!test_and_set_bit(BCACHE_DEV_CLOSING, &d->flags))
K
Kent Overstreet 已提交
655 656 657
		closure_queue(&d->cl);
}

658 659
static void bcache_device_unlink(struct bcache_device *d)
{
660
	lockdep_assert_held(&bch_register_lock);
661

662 663 664
	if (d->c && !test_and_set_bit(BCACHE_DEV_UNLINK_DONE, &d->flags)) {
		unsigned i;
		struct cache *ca;
665

666 667 668 669 670 671
		sysfs_remove_link(&d->c->kobj, d->name);
		sysfs_remove_link(&d->kobj, "cache");

		for_each_cache(ca, d->c, i)
			bd_unlink_disk_holder(ca->bdev, d->disk);
	}
672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688
}

static void bcache_device_link(struct bcache_device *d, struct cache_set *c,
			       const char *name)
{
	unsigned i;
	struct cache *ca;

	for_each_cache(ca, d->c, i)
		bd_link_disk_holder(ca->bdev, d->disk);

	snprintf(d->name, BCACHEDEVNAME_SIZE,
		 "%s%u", name, d->id);

	WARN(sysfs_create_link(&d->kobj, &c->kobj, "cache") ||
	     sysfs_create_link(&c->kobj, &d->kobj, d->name),
	     "Couldn't create device <-> cache set symlinks");
689 690

	clear_bit(BCACHE_DEV_UNLINK_DONE, &d->flags);
691 692
}

K
Kent Overstreet 已提交
693 694 695 696
static void bcache_device_detach(struct bcache_device *d)
{
	lockdep_assert_held(&bch_register_lock);

697
	if (test_bit(BCACHE_DEV_DETACHING, &d->flags)) {
K
Kent Overstreet 已提交
698 699 700 701 702 703 704 705
		struct uuid_entry *u = d->c->uuids + d->id;

		SET_UUID_FLASH_ONLY(u, 0);
		memcpy(u->uuid, invalid_uuid, 16);
		u->invalidated = cpu_to_le32(get_seconds());
		bch_uuid_write(d->c);
	}

706
	bcache_device_unlink(d);
707

K
Kent Overstreet 已提交
708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730
	d->c->devices[d->id] = NULL;
	closure_put(&d->c->caching);
	d->c = NULL;
}

static void bcache_device_attach(struct bcache_device *d, struct cache_set *c,
				 unsigned id)
{
	d->id = id;
	d->c = c;
	c->devices[id] = d;

	closure_get(&c->caching);
}

static void bcache_device_free(struct bcache_device *d)
{
	lockdep_assert_held(&bch_register_lock);

	pr_info("%s stopped", d->disk->disk_name);

	if (d->c)
		bcache_device_detach(d);
731
	if (d->disk && d->disk->flags & GENHD_FL_UP)
K
Kent Overstreet 已提交
732 733 734
		del_gendisk(d->disk);
	if (d->disk && d->disk->queue)
		blk_cleanup_queue(d->disk->queue);
735 736
	if (d->disk) {
		ida_simple_remove(&bcache_minor, d->disk->first_minor);
K
Kent Overstreet 已提交
737
		put_disk(d->disk);
738
	}
K
Kent Overstreet 已提交
739 740 741

	if (d->bio_split)
		bioset_free(d->bio_split);
742 743
	kvfree(d->full_dirty_stripes);
	kvfree(d->stripe_sectors_dirty);
K
Kent Overstreet 已提交
744 745 746 747

	closure_debug_destroy(&d->cl);
}

748 749
static int bcache_device_init(struct bcache_device *d, unsigned block_size,
			      sector_t sectors)
K
Kent Overstreet 已提交
750 751
{
	struct request_queue *q;
752
	size_t n;
753
	int minor;
754

755 756
	if (!d->stripe_size)
		d->stripe_size = 1 << 31;
757

758
	d->nr_stripes = DIV_ROUND_UP_ULL(sectors, d->stripe_size);
759

760 761 762 763
	if (!d->nr_stripes ||
	    d->nr_stripes > INT_MAX ||
	    d->nr_stripes > SIZE_MAX / sizeof(atomic_t)) {
		pr_err("nr_stripes too large");
764
		return -ENOMEM;
765
	}
766 767 768 769 770 771 772

	n = d->nr_stripes * sizeof(atomic_t);
	d->stripe_sectors_dirty = n < PAGE_SIZE << 6
		? kzalloc(n, GFP_KERNEL)
		: vzalloc(n);
	if (!d->stripe_sectors_dirty)
		return -ENOMEM;
K
Kent Overstreet 已提交
773

774 775 776 777 778 779 780
	n = BITS_TO_LONGS(d->nr_stripes) * sizeof(unsigned long);
	d->full_dirty_stripes = n < PAGE_SIZE << 6
		? kzalloc(n, GFP_KERNEL)
		: vzalloc(n);
	if (!d->full_dirty_stripes)
		return -ENOMEM;

781 782 783 784
	minor = ida_simple_get(&bcache_minor, 0, MINORMASK + 1, GFP_KERNEL);
	if (minor < 0)
		return minor;

K
Kent Overstreet 已提交
785
	if (!(d->bio_split = bioset_create(4, offsetof(struct bbio, bio))) ||
786 787
	    !(d->disk = alloc_disk(1))) {
		ida_simple_remove(&bcache_minor, minor);
K
Kent Overstreet 已提交
788
		return -ENOMEM;
789
	}
K
Kent Overstreet 已提交
790

791
	set_capacity(d->disk, sectors);
792
	snprintf(d->disk->disk_name, DISK_NAME_LEN, "bcache%i", minor);
K
Kent Overstreet 已提交
793 794

	d->disk->major		= bcache_major;
795
	d->disk->first_minor	= minor;
K
Kent Overstreet 已提交
796 797 798
	d->disk->fops		= &bcache_ops;
	d->disk->private_data	= d;

799 800 801 802
	q = blk_alloc_queue(GFP_KERNEL);
	if (!q)
		return -ENOMEM;

K
Kent Overstreet 已提交
803 804 805 806 807 808 809 810
	blk_queue_make_request(q, NULL);
	d->disk->queue			= q;
	q->queuedata			= d;
	q->backing_dev_info.congested_data = d;
	q->limits.max_hw_sectors	= UINT_MAX;
	q->limits.max_sectors		= UINT_MAX;
	q->limits.max_segment_size	= UINT_MAX;
	q->limits.max_segments		= BIO_MAX_PAGES;
811
	blk_queue_max_discard_sectors(q, UINT_MAX);
812
	q->limits.discard_granularity	= 512;
K
Kent Overstreet 已提交
813 814 815 816
	q->limits.io_min		= block_size;
	q->limits.logical_block_size	= block_size;
	q->limits.physical_block_size	= block_size;
	set_bit(QUEUE_FLAG_NONROT,	&d->disk->queue->queue_flags);
817
	clear_bit(QUEUE_FLAG_ADD_RANDOM, &d->disk->queue->queue_flags);
K
Kent Overstreet 已提交
818 819
	set_bit(QUEUE_FLAG_DISCARD,	&d->disk->queue->queue_flags);

820
	blk_queue_write_cache(q, true, true);
821

K
Kent Overstreet 已提交
822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840
	return 0;
}

/* Cached device */

static void calc_cached_dev_sectors(struct cache_set *c)
{
	uint64_t sectors = 0;
	struct cached_dev *dc;

	list_for_each_entry(dc, &c->cached_devs, list)
		sectors += bdev_sectors(dc->bdev);

	c->cached_dev_sectors = sectors;
}

void bch_cached_dev_run(struct cached_dev *dc)
{
	struct bcache_device *d = &dc->disk;
G
Gabriel de Perthuis 已提交
841
	char buf[SB_LABEL_SIZE + 1];
842 843 844
	char *env[] = {
		"DRIVER=bcache",
		kasprintf(GFP_KERNEL, "CACHED_UUID=%pU", dc->sb.uuid),
G
Gabriel de Perthuis 已提交
845 846
		NULL,
		NULL,
847
	};
K
Kent Overstreet 已提交
848

G
Gabriel de Perthuis 已提交
849 850 851 852
	memcpy(buf, dc->sb.label, SB_LABEL_SIZE);
	buf[SB_LABEL_SIZE] = '\0';
	env[2] = kasprintf(GFP_KERNEL, "CACHED_LABEL=%s", buf);

853 854 855
	if (atomic_xchg(&dc->running, 1)) {
		kfree(env[1]);
		kfree(env[2]);
K
Kent Overstreet 已提交
856
		return;
857
	}
K
Kent Overstreet 已提交
858 859 860 861 862 863 864 865 866 867 868 869

	if (!d->c &&
	    BDEV_STATE(&dc->sb) != BDEV_STATE_NONE) {
		struct closure cl;
		closure_init_stack(&cl);

		SET_BDEV_STATE(&dc->sb, BDEV_STATE_STALE);
		bch_write_bdev_super(dc, &cl);
		closure_sync(&cl);
	}

	add_disk(d->disk);
870
	bd_link_disk_holder(dc->bdev, dc->disk.disk);
871 872
	/* won't show up in the uevent file, use udevadm monitor -e instead
	 * only class / kset properties are persistent */
K
Kent Overstreet 已提交
873
	kobject_uevent_env(&disk_to_dev(d->disk)->kobj, KOBJ_CHANGE, env);
874
	kfree(env[1]);
G
Gabriel de Perthuis 已提交
875
	kfree(env[2]);
876

K
Kent Overstreet 已提交
877 878 879 880 881 882 883 884 885 886 887 888
	if (sysfs_create_link(&d->kobj, &disk_to_dev(d->disk)->kobj, "dev") ||
	    sysfs_create_link(&disk_to_dev(d->disk)->kobj, &d->kobj, "bcache"))
		pr_debug("error creating sysfs link");
}

static void cached_dev_detach_finish(struct work_struct *w)
{
	struct cached_dev *dc = container_of(w, struct cached_dev, detach);
	char buf[BDEVNAME_SIZE];
	struct closure cl;
	closure_init_stack(&cl);

889
	BUG_ON(!test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags));
K
Kent Overstreet 已提交
890 891 892 893 894 895 896 897 898 899 900 901 902
	BUG_ON(atomic_read(&dc->count));

	mutex_lock(&bch_register_lock);

	memset(&dc->sb.set_uuid, 0, 16);
	SET_BDEV_STATE(&dc->sb, BDEV_STATE_NONE);

	bch_write_bdev_super(dc, &cl);
	closure_sync(&cl);

	bcache_device_detach(&dc->disk);
	list_move(&dc->list, &uncached_devices);

903
	clear_bit(BCACHE_DEV_DETACHING, &dc->disk.flags);
904
	clear_bit(BCACHE_DEV_UNLINK_DONE, &dc->disk.flags);
905

K
Kent Overstreet 已提交
906 907 908 909 910 911 912 913 914 915 916 917
	mutex_unlock(&bch_register_lock);

	pr_info("Caching disabled for %s", bdevname(dc->bdev, buf));

	/* Drop ref we took in cached_dev_detach() */
	closure_put(&dc->disk.cl);
}

void bch_cached_dev_detach(struct cached_dev *dc)
{
	lockdep_assert_held(&bch_register_lock);

918
	if (test_bit(BCACHE_DEV_CLOSING, &dc->disk.flags))
K
Kent Overstreet 已提交
919 920
		return;

921
	if (test_and_set_bit(BCACHE_DEV_DETACHING, &dc->disk.flags))
K
Kent Overstreet 已提交
922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956
		return;

	/*
	 * Block the device from being closed and freed until we're finished
	 * detaching
	 */
	closure_get(&dc->disk.cl);

	bch_writeback_queue(dc);
	cached_dev_put(dc);
}

int bch_cached_dev_attach(struct cached_dev *dc, struct cache_set *c)
{
	uint32_t rtime = cpu_to_le32(get_seconds());
	struct uuid_entry *u;
	char buf[BDEVNAME_SIZE];

	bdevname(dc->bdev, buf);

	if (memcmp(dc->sb.set_uuid, c->sb.set_uuid, 16))
		return -ENOENT;

	if (dc->disk.c) {
		pr_err("Can't attach %s: already attached", buf);
		return -EINVAL;
	}

	if (test_bit(CACHE_SET_STOPPING, &c->flags)) {
		pr_err("Can't attach %s: shutting down", buf);
		return -EINVAL;
	}

	if (dc->sb.block_size < c->sb.block_size) {
		/* Will die */
K
Kent Overstreet 已提交
957 958
		pr_err("Couldn't attach %s: block size less than set's block size",
		       buf);
K
Kent Overstreet 已提交
959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988
		return -EINVAL;
	}

	u = uuid_find(c, dc->sb.uuid);

	if (u &&
	    (BDEV_STATE(&dc->sb) == BDEV_STATE_STALE ||
	     BDEV_STATE(&dc->sb) == BDEV_STATE_NONE)) {
		memcpy(u->uuid, invalid_uuid, 16);
		u->invalidated = cpu_to_le32(get_seconds());
		u = NULL;
	}

	if (!u) {
		if (BDEV_STATE(&dc->sb) == BDEV_STATE_DIRTY) {
			pr_err("Couldn't find uuid for %s in set", buf);
			return -ENOENT;
		}

		u = uuid_find_empty(c);
		if (!u) {
			pr_err("Not caching %s, no room for UUID", buf);
			return -EINVAL;
		}
	}

	/* Deadlocks since we're called via sysfs...
	sysfs_remove_file(&dc->kobj, &sysfs_attach);
	 */

989
	if (bch_is_zero(u->uuid, 16)) {
K
Kent Overstreet 已提交
990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018
		struct closure cl;
		closure_init_stack(&cl);

		memcpy(u->uuid, dc->sb.uuid, 16);
		memcpy(u->label, dc->sb.label, SB_LABEL_SIZE);
		u->first_reg = u->last_reg = rtime;
		bch_uuid_write(c);

		memcpy(dc->sb.set_uuid, c->sb.set_uuid, 16);
		SET_BDEV_STATE(&dc->sb, BDEV_STATE_CLEAN);

		bch_write_bdev_super(dc, &cl);
		closure_sync(&cl);
	} else {
		u->last_reg = rtime;
		bch_uuid_write(c);
	}

	bcache_device_attach(&dc->disk, c, u - c->uuids);
	list_move(&dc->list, &c->cached_devs);
	calc_cached_dev_sectors(c);

	smp_wmb();
	/*
	 * dc->c must be set before dc->count != 0 - paired with the mb in
	 * cached_dev_get()
	 */
	atomic_set(&dc->count, 1);

1019 1020 1021 1022
	/* Block writeback thread, but spawn it */
	down_write(&dc->writeback_lock);
	if (bch_cached_dev_writeback_start(dc)) {
		up_write(&dc->writeback_lock);
1023
		return -ENOMEM;
1024
	}
1025

K
Kent Overstreet 已提交
1026
	if (BDEV_STATE(&dc->sb) == BDEV_STATE_DIRTY) {
1027
		bch_sectors_dirty_init(dc);
K
Kent Overstreet 已提交
1028 1029 1030 1031 1032 1033
		atomic_set(&dc->has_dirty, 1);
		atomic_inc(&dc->count);
		bch_writeback_queue(dc);
	}

	bch_cached_dev_run(dc);
1034
	bcache_device_link(&dc->disk, c, "bdev");
K
Kent Overstreet 已提交
1035

1036 1037 1038
	/* Allow the writeback thread to proceed */
	up_write(&dc->writeback_lock);

K
Kent Overstreet 已提交
1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057
	pr_info("Caching %s as %s on set %pU",
		bdevname(dc->bdev, buf), dc->disk.disk->disk_name,
		dc->disk.c->sb.set_uuid);
	return 0;
}

void bch_cached_dev_release(struct kobject *kobj)
{
	struct cached_dev *dc = container_of(kobj, struct cached_dev,
					     disk.kobj);
	kfree(dc);
	module_put(THIS_MODULE);
}

static void cached_dev_free(struct closure *cl)
{
	struct cached_dev *dc = container_of(cl, struct cached_dev, disk.cl);

	cancel_delayed_work_sync(&dc->writeback_rate_update);
1058 1059
	if (!IS_ERR_OR_NULL(dc->writeback_thread))
		kthread_stop(dc->writeback_thread);
K
Kent Overstreet 已提交
1060 1061 1062

	mutex_lock(&bch_register_lock);

1063 1064
	if (atomic_read(&dc->running))
		bd_unlink_disk_holder(dc->bdev, dc->disk.disk);
K
Kent Overstreet 已提交
1065 1066 1067 1068 1069
	bcache_device_free(&dc->disk);
	list_del(&dc->list);

	mutex_unlock(&bch_register_lock);

1070
	if (!IS_ERR_OR_NULL(dc->bdev))
K
Kent Overstreet 已提交
1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082
		blkdev_put(dc->bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL);

	wake_up(&unregister_wait);

	kobject_put(&dc->disk.kobj);
}

static void cached_dev_flush(struct closure *cl)
{
	struct cached_dev *dc = container_of(cl, struct cached_dev, disk.cl);
	struct bcache_device *d = &dc->disk;

1083
	mutex_lock(&bch_register_lock);
1084
	bcache_device_unlink(d);
1085 1086
	mutex_unlock(&bch_register_lock);

K
Kent Overstreet 已提交
1087 1088 1089 1090 1091 1092 1093 1094
	bch_cache_accounting_destroy(&dc->accounting);
	kobject_del(&d->kobj);

	continue_at(cl, cached_dev_free, system_wq);
}

static int cached_dev_init(struct cached_dev *dc, unsigned block_size)
{
1095
	int ret;
K
Kent Overstreet 已提交
1096
	struct io *io;
1097
	struct request_queue *q = bdev_get_queue(dc->bdev);
K
Kent Overstreet 已提交
1098 1099 1100

	__module_get(THIS_MODULE);
	INIT_LIST_HEAD(&dc->list);
1101 1102
	closure_init(&dc->disk.cl, NULL);
	set_closure_fn(&dc->disk.cl, cached_dev_flush, system_wq);
K
Kent Overstreet 已提交
1103 1104
	kobject_init(&dc->disk.kobj, &bch_cached_dev_ktype);
	INIT_WORK(&dc->detach, cached_dev_detach_finish);
1105
	sema_init(&dc->sb_write_mutex, 1);
1106 1107 1108
	INIT_LIST_HEAD(&dc->io_lru);
	spin_lock_init(&dc->io_lock);
	bch_cache_accounting_init(&dc->accounting, &dc->disk.cl);
K
Kent Overstreet 已提交
1109 1110 1111 1112 1113 1114 1115 1116

	dc->sequential_cutoff		= 4 << 20;

	for (io = dc->io; io < dc->io + RECENT_IO; io++) {
		list_add(&io->lru, &dc->io_lru);
		hlist_add_head(&io->hash, dc->io_hash + RECENT_IO);
	}

1117 1118 1119 1120 1121 1122
	dc->disk.stripe_size = q->limits.io_opt >> 9;

	if (dc->disk.stripe_size)
		dc->partial_stripes_expensive =
			q->limits.raid_partial_stripes_expensive;

1123 1124
	ret = bcache_device_init(&dc->disk, block_size,
			 dc->bdev->bd_part->nr_sects - dc->sb.data_offset);
1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136
	if (ret)
		return ret;

	set_capacity(dc->disk.disk,
		     dc->bdev->bd_part->nr_sects - dc->sb.data_offset);

	dc->disk.disk->queue->backing_dev_info.ra_pages =
		max(dc->disk.disk->queue->backing_dev_info.ra_pages,
		    q->backing_dev_info.ra_pages);

	bch_cached_dev_request_init(dc);
	bch_cached_dev_writeback_init(dc);
K
Kent Overstreet 已提交
1137 1138 1139 1140 1141
	return 0;
}

/* Cached device - bcache superblock */

1142
static void register_bdev(struct cache_sb *sb, struct page *sb_page,
K
Kent Overstreet 已提交
1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153
				 struct block_device *bdev,
				 struct cached_dev *dc)
{
	char name[BDEVNAME_SIZE];
	const char *err = "cannot allocate memory";
	struct cache_set *c;

	memcpy(&dc->sb, sb, sizeof(struct cache_sb));
	dc->bdev = bdev;
	dc->bdev->bd_holder = dc;

1154 1155 1156 1157 1158
	bio_init(&dc->sb_bio);
	dc->sb_bio.bi_max_vecs	= 1;
	dc->sb_bio.bi_io_vec	= dc->sb_bio.bi_inline_vecs;
	dc->sb_bio.bi_io_vec[0].bv_page = sb_page;
	get_page(sb_page);
1159

1160 1161
	if (cached_dev_init(dc, sb->block_size << 9))
		goto err;
K
Kent Overstreet 已提交
1162 1163 1164 1165 1166 1167 1168 1169

	err = "error creating kobject";
	if (kobject_add(&dc->disk.kobj, &part_to_dev(bdev->bd_part)->kobj,
			"bcache"))
		goto err;
	if (bch_cache_accounting_add_kobjs(&dc->accounting, &dc->disk.kobj))
		goto err;

1170 1171
	pr_info("registered backing device %s", bdevname(bdev, name));

K
Kent Overstreet 已提交
1172 1173 1174 1175 1176 1177 1178 1179
	list_add(&dc->list, &uncached_devices);
	list_for_each_entry(c, &bch_cache_sets, list)
		bch_cached_dev_attach(dc, c);

	if (BDEV_STATE(&dc->sb) == BDEV_STATE_NONE ||
	    BDEV_STATE(&dc->sb) == BDEV_STATE_STALE)
		bch_cached_dev_run(dc);

1180
	return;
K
Kent Overstreet 已提交
1181 1182
err:
	pr_notice("error opening %s: %s", bdevname(bdev, name), err);
1183
	bcache_device_stop(&dc->disk);
K
Kent Overstreet 已提交
1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197
}

/* Flash only volumes */

void bch_flash_dev_release(struct kobject *kobj)
{
	struct bcache_device *d = container_of(kobj, struct bcache_device,
					       kobj);
	kfree(d);
}

static void flash_dev_free(struct closure *cl)
{
	struct bcache_device *d = container_of(cl, struct bcache_device, cl);
1198
	mutex_lock(&bch_register_lock);
K
Kent Overstreet 已提交
1199
	bcache_device_free(d);
1200
	mutex_unlock(&bch_register_lock);
K
Kent Overstreet 已提交
1201 1202 1203 1204 1205 1206 1207
	kobject_put(&d->kobj);
}

static void flash_dev_flush(struct closure *cl)
{
	struct bcache_device *d = container_of(cl, struct bcache_device, cl);

1208
	mutex_lock(&bch_register_lock);
1209
	bcache_device_unlink(d);
1210
	mutex_unlock(&bch_register_lock);
K
Kent Overstreet 已提交
1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226
	kobject_del(&d->kobj);
	continue_at(cl, flash_dev_free, system_wq);
}

static int flash_dev_run(struct cache_set *c, struct uuid_entry *u)
{
	struct bcache_device *d = kzalloc(sizeof(struct bcache_device),
					  GFP_KERNEL);
	if (!d)
		return -ENOMEM;

	closure_init(&d->cl, NULL);
	set_closure_fn(&d->cl, flash_dev_flush, system_wq);

	kobject_init(&d->kobj, &bch_flash_dev_ktype);

1227
	if (bcache_device_init(d, block_bytes(c), u->sectors))
K
Kent Overstreet 已提交
1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265
		goto err;

	bcache_device_attach(d, c, u - c->uuids);
	bch_flash_dev_request_init(d);
	add_disk(d->disk);

	if (kobject_add(&d->kobj, &disk_to_dev(d->disk)->kobj, "bcache"))
		goto err;

	bcache_device_link(d, c, "volume");

	return 0;
err:
	kobject_put(&d->kobj);
	return -ENOMEM;
}

static int flash_devs_run(struct cache_set *c)
{
	int ret = 0;
	struct uuid_entry *u;

	for (u = c->uuids;
	     u < c->uuids + c->nr_uuids && !ret;
	     u++)
		if (UUID_FLASH_ONLY(u))
			ret = flash_dev_run(c, u);

	return ret;
}

int bch_flash_dev_create(struct cache_set *c, uint64_t size)
{
	struct uuid_entry *u;

	if (test_bit(CACHE_SET_STOPPING, &c->flags))
		return -EINTR;

1266 1267 1268
	if (!test_bit(CACHE_SET_RUNNING, &c->flags))
		return -EPERM;

K
Kent Overstreet 已提交
1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293
	u = uuid_find_empty(c);
	if (!u) {
		pr_err("Can't create volume, no room for UUID");
		return -EINVAL;
	}

	get_random_bytes(u->uuid, 16);
	memset(u->label, 0, 32);
	u->first_reg = u->last_reg = cpu_to_le32(get_seconds());

	SET_UUID_FLASH_ONLY(u, 1);
	u->sectors = size >> 9;

	bch_uuid_write(c);

	return flash_dev_run(c, u);
}

/* Cache set */

__printf(2, 3)
bool bch_cache_set_error(struct cache_set *c, const char *fmt, ...)
{
	va_list args;

1294 1295
	if (c->on_error != ON_ERROR_PANIC &&
	    test_bit(CACHE_SET_STOPPING, &c->flags))
K
Kent Overstreet 已提交
1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309
		return false;

	/* XXX: we can be called from atomic context
	acquire_console_sem();
	*/

	printk(KERN_ERR "bcache: error on %pU: ", c->sb.set_uuid);

	va_start(args, fmt);
	vprintk(fmt, args);
	va_end(args);

	printk(", disabling caching\n");

1310 1311 1312
	if (c->on_error == ON_ERROR_PANIC)
		panic("panic forced after error\n");

K
Kent Overstreet 已提交
1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337
	bch_cache_set_unregister(c);
	return true;
}

void bch_cache_set_release(struct kobject *kobj)
{
	struct cache_set *c = container_of(kobj, struct cache_set, kobj);
	kfree(c);
	module_put(THIS_MODULE);
}

static void cache_set_free(struct closure *cl)
{
	struct cache_set *c = container_of(cl, struct cache_set, cl);
	struct cache *ca;
	unsigned i;

	if (!IS_ERR_OR_NULL(c->debug))
		debugfs_remove(c->debug);

	bch_open_buckets_free(c);
	bch_btree_cache_free(c);
	bch_journal_free(c);

	for_each_cache(ca, c, i)
1338 1339 1340
		if (ca) {
			ca->set = NULL;
			c->cache[ca->sb.nr_this_dev] = NULL;
K
Kent Overstreet 已提交
1341
			kobject_put(&ca->kobj);
1342
		}
K
Kent Overstreet 已提交
1343

1344
	bch_bset_sort_state_free(&c->sort);
K
Kent Overstreet 已提交
1345 1346
	free_pages((unsigned long) c->uuids, ilog2(bucket_pages(c)));

1347 1348
	if (c->moving_gc_wq)
		destroy_workqueue(c->moving_gc_wq);
K
Kent Overstreet 已提交
1349 1350
	if (c->bio_split)
		bioset_free(c->bio_split);
K
Kent Overstreet 已提交
1351 1352
	if (c->fill_iter)
		mempool_destroy(c->fill_iter);
K
Kent Overstreet 已提交
1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372
	if (c->bio_meta)
		mempool_destroy(c->bio_meta);
	if (c->search)
		mempool_destroy(c->search);
	kfree(c->devices);

	mutex_lock(&bch_register_lock);
	list_del(&c->list);
	mutex_unlock(&bch_register_lock);

	pr_info("Cache set %pU unregistered", c->sb.set_uuid);
	wake_up(&unregister_wait);

	closure_debug_destroy(&c->cl);
	kobject_put(&c->kobj);
}

static void cache_set_flush(struct closure *cl)
{
	struct cache_set *c = container_of(cl, struct cache_set, caching);
1373
	struct cache *ca;
K
Kent Overstreet 已提交
1374
	struct btree *b;
1375
	unsigned i;
K
Kent Overstreet 已提交
1376

1377 1378 1379
	if (!c)
		closure_return(cl);

K
Kent Overstreet 已提交
1380 1381 1382 1383 1384
	bch_cache_accounting_destroy(&c->accounting);

	kobject_put(&c->internal);
	kobject_del(&c->kobj);

K
Kent Overstreet 已提交
1385 1386 1387
	if (c->gc_thread)
		kthread_stop(c->gc_thread);

K
Kent Overstreet 已提交
1388 1389 1390 1391
	if (!IS_ERR_OR_NULL(c->root))
		list_add(&c->root->list, &c->btree_cache);

	/* Should skip this if we're unregistering because of an error */
K
Kent Overstreet 已提交
1392 1393
	list_for_each_entry(b, &c->btree_cache, list) {
		mutex_lock(&b->write_lock);
K
Kent Overstreet 已提交
1394
		if (btree_node_dirty(b))
K
Kent Overstreet 已提交
1395 1396 1397
			__bch_btree_node_write(b, NULL);
		mutex_unlock(&b->write_lock);
	}
K
Kent Overstreet 已提交
1398

1399 1400 1401 1402
	for_each_cache(ca, c, i)
		if (ca->alloc_thread)
			kthread_stop(ca->alloc_thread);

1403 1404 1405 1406 1407
	if (c->journal.cur) {
		cancel_delayed_work_sync(&c->journal.work);
		/* flush last journal entry if needed */
		c->journal.work.work.func(&c->journal.work.work);
	}
K
Kent Overstreet 已提交
1408

K
Kent Overstreet 已提交
1409 1410 1411 1412 1413 1414
	closure_return(cl);
}

static void __cache_set_unregister(struct closure *cl)
{
	struct cache_set *c = container_of(cl, struct cache_set, caching);
K
Kent Overstreet 已提交
1415
	struct cached_dev *dc;
K
Kent Overstreet 已提交
1416 1417 1418 1419 1420
	size_t i;

	mutex_lock(&bch_register_lock);

	for (i = 0; i < c->nr_uuids; i++)
K
Kent Overstreet 已提交
1421 1422 1423 1424 1425 1426 1427 1428 1429 1430
		if (c->devices[i]) {
			if (!UUID_FLASH_ONLY(&c->uuids[i]) &&
			    test_bit(CACHE_SET_UNREGISTERING, &c->flags)) {
				dc = container_of(c->devices[i],
						  struct cached_dev, disk);
				bch_cached_dev_detach(dc);
			} else {
				bcache_device_stop(c->devices[i]);
			}
		}
K
Kent Overstreet 已提交
1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483

	mutex_unlock(&bch_register_lock);

	continue_at(cl, cache_set_flush, system_wq);
}

void bch_cache_set_stop(struct cache_set *c)
{
	if (!test_and_set_bit(CACHE_SET_STOPPING, &c->flags))
		closure_queue(&c->caching);
}

void bch_cache_set_unregister(struct cache_set *c)
{
	set_bit(CACHE_SET_UNREGISTERING, &c->flags);
	bch_cache_set_stop(c);
}

#define alloc_bucket_pages(gfp, c)			\
	((void *) __get_free_pages(__GFP_ZERO|gfp, ilog2(bucket_pages(c))))

struct cache_set *bch_cache_set_alloc(struct cache_sb *sb)
{
	int iter_size;
	struct cache_set *c = kzalloc(sizeof(struct cache_set), GFP_KERNEL);
	if (!c)
		return NULL;

	__module_get(THIS_MODULE);
	closure_init(&c->cl, NULL);
	set_closure_fn(&c->cl, cache_set_free, system_wq);

	closure_init(&c->caching, &c->cl);
	set_closure_fn(&c->caching, __cache_set_unregister, system_wq);

	/* Maybe create continue_at_noreturn() and use it here? */
	closure_set_stopped(&c->cl);
	closure_put(&c->cl);

	kobject_init(&c->kobj, &bch_cache_set_ktype);
	kobject_init(&c->internal, &bch_cache_set_internal_ktype);

	bch_cache_accounting_init(&c->accounting, &c->cl);

	memcpy(c->sb.set_uuid, sb->set_uuid, 16);
	c->sb.block_size	= sb->block_size;
	c->sb.bucket_size	= sb->bucket_size;
	c->sb.nr_in_set		= sb->nr_in_set;
	c->sb.last_mount	= sb->last_mount;
	c->bucket_bits		= ilog2(sb->bucket_size);
	c->block_bits		= ilog2(sb->block_size);
	c->nr_uuids		= bucket_bytes(c) / sizeof(struct uuid_entry);

1484
	c->btree_pages		= bucket_pages(c);
K
Kent Overstreet 已提交
1485 1486 1487 1488
	if (c->btree_pages > BTREE_MAX_PAGES)
		c->btree_pages = max_t(int, c->btree_pages / 4,
				       BTREE_MAX_PAGES);

1489
	sema_init(&c->sb_write_mutex, 1);
1490
	mutex_init(&c->bucket_lock);
1491
	init_waitqueue_head(&c->btree_cache_wait);
1492
	init_waitqueue_head(&c->bucket_wait);
1493
	sema_init(&c->uuid_write_mutex, 1);
1494 1495 1496 1497

	spin_lock_init(&c->btree_gc_time.lock);
	spin_lock_init(&c->btree_split_time.lock);
	spin_lock_init(&c->btree_read_time.lock);
1498

K
Kent Overstreet 已提交
1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518
	bch_moving_init_cache_set(c);

	INIT_LIST_HEAD(&c->list);
	INIT_LIST_HEAD(&c->cached_devs);
	INIT_LIST_HEAD(&c->btree_cache);
	INIT_LIST_HEAD(&c->btree_cache_freeable);
	INIT_LIST_HEAD(&c->btree_cache_freed);
	INIT_LIST_HEAD(&c->data_buckets);

	c->search = mempool_create_slab_pool(32, bch_search_cache);
	if (!c->search)
		goto err;

	iter_size = (sb->bucket_size / sb->block_size + 1) *
		sizeof(struct btree_iter_set);

	if (!(c->devices = kzalloc(c->nr_uuids * sizeof(void *), GFP_KERNEL)) ||
	    !(c->bio_meta = mempool_create_kmalloc_pool(2,
				sizeof(struct bbio) + sizeof(struct bio_vec) *
				bucket_pages(c))) ||
K
Kent Overstreet 已提交
1519
	    !(c->fill_iter = mempool_create_kmalloc_pool(1, iter_size)) ||
K
Kent Overstreet 已提交
1520 1521
	    !(c->bio_split = bioset_create(4, offsetof(struct bbio, bio))) ||
	    !(c->uuids = alloc_bucket_pages(GFP_KERNEL, c)) ||
1522 1523
	    !(c->moving_gc_wq = alloc_workqueue("bcache_gc",
						WQ_MEM_RECLAIM, 0)) ||
K
Kent Overstreet 已提交
1524 1525
	    bch_journal_alloc(c) ||
	    bch_btree_cache_alloc(c) ||
1526 1527
	    bch_open_buckets_alloc(c) ||
	    bch_bset_sort_state_init(&c->sort, ilog2(c->btree_pages)))
K
Kent Overstreet 已提交
1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544
		goto err;

	c->congested_read_threshold_us	= 2000;
	c->congested_write_threshold_us	= 20000;
	c->error_limit	= 8 << IO_ERROR_SHIFT;

	return c;
err:
	bch_cache_set_unregister(c);
	return NULL;
}

static void run_cache_set(struct cache_set *c)
{
	const char *err = "cannot allocate memory";
	struct cached_dev *dc, *t;
	struct cache *ca;
K
Kent Overstreet 已提交
1545
	struct closure cl;
K
Kent Overstreet 已提交
1546 1547
	unsigned i;

K
Kent Overstreet 已提交
1548
	closure_init_stack(&cl);
K
Kent Overstreet 已提交
1549 1550 1551 1552 1553 1554 1555 1556 1557 1558

	for_each_cache(ca, c, i)
		c->nbuckets += ca->sb.nbuckets;

	if (CACHE_SYNC(&c->sb)) {
		LIST_HEAD(journal);
		struct bkey *k;
		struct jset *j;

		err = "cannot allocate memory for journal";
K
Kent Overstreet 已提交
1559
		if (bch_journal_read(c, &journal))
K
Kent Overstreet 已提交
1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582
			goto err;

		pr_debug("btree_journal_read() done");

		err = "no journal entries found";
		if (list_empty(&journal))
			goto err;

		j = &list_entry(journal.prev, struct journal_replay, list)->j;

		err = "IO error reading priorities";
		for_each_cache(ca, c, i)
			prio_read(ca, j->prio_bucket[ca->sb.nr_this_dev]);

		/*
		 * If prio_read() fails it'll call cache_set_error and we'll
		 * tear everything down right away, but if we perhaps checked
		 * sooner we could avoid journal replay.
		 */

		k = &j->btree_root;

		err = "bad btree root";
1583
		if (__bch_btree_ptr_invalid(c, k))
K
Kent Overstreet 已提交
1584 1585 1586
			goto err;

		err = "error reading btree root";
1587
		c->root = bch_btree_node_get(c, NULL, k, j->btree_level, true, NULL);
K
Kent Overstreet 已提交
1588 1589 1590 1591 1592 1593
		if (IS_ERR_OR_NULL(c->root))
			goto err;

		list_del_init(&c->root->list);
		rw_unlock(true, c->root);

K
Kent Overstreet 已提交
1594
		err = uuid_read(c, j, &cl);
K
Kent Overstreet 已提交
1595 1596 1597 1598
		if (err)
			goto err;

		err = "error in recovery";
K
Kent Overstreet 已提交
1599
		if (bch_btree_check(c))
K
Kent Overstreet 已提交
1600 1601 1602
			goto err;

		bch_journal_mark(c, &journal);
K
Kent Overstreet 已提交
1603
		bch_initial_gc_finish(c);
K
Kent Overstreet 已提交
1604 1605 1606 1607 1608 1609 1610 1611 1612
		pr_debug("btree_check() done");

		/*
		 * bcache_journal_next() can't happen sooner, or
		 * btree_gc_finish() will give spurious errors about last_gc >
		 * gc_gen - this is a hack but oh well.
		 */
		bch_journal_next(&c->journal);

1613
		err = "error starting allocator thread";
K
Kent Overstreet 已提交
1614
		for_each_cache(ca, c, i)
1615 1616
			if (bch_cache_allocator_start(ca))
				goto err;
K
Kent Overstreet 已提交
1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630

		/*
		 * First place it's safe to allocate: btree_check() and
		 * btree_gc_finish() have to run before we have buckets to
		 * allocate, and bch_bucket_alloc_set() might cause a journal
		 * entry to be written so bcache_journal_next() has to be called
		 * first.
		 *
		 * If the uuids were in the old format we have to rewrite them
		 * before the next journal entry is written:
		 */
		if (j->version < BCACHE_JSET_VERSION_UUID)
			__uuid_write(c);

K
Kent Overstreet 已提交
1631
		bch_journal_replay(c, &journal);
K
Kent Overstreet 已提交
1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644
	} else {
		pr_notice("invalidating existing data");

		for_each_cache(ca, c, i) {
			unsigned j;

			ca->sb.keys = clamp_t(int, ca->sb.nbuckets >> 7,
					      2, SB_JOURNAL_BUCKETS);

			for (j = 0; j < ca->sb.keys; j++)
				ca->sb.d[j] = ca->sb.first_bucket + j;
		}

K
Kent Overstreet 已提交
1645
		bch_initial_gc_finish(c);
K
Kent Overstreet 已提交
1646

1647
		err = "error starting allocator thread";
K
Kent Overstreet 已提交
1648
		for_each_cache(ca, c, i)
1649 1650
			if (bch_cache_allocator_start(ca))
				goto err;
K
Kent Overstreet 已提交
1651 1652 1653 1654 1655 1656 1657 1658

		mutex_lock(&c->bucket_lock);
		for_each_cache(ca, c, i)
			bch_prio_write(ca);
		mutex_unlock(&c->bucket_lock);

		err = "cannot allocate new UUID bucket";
		if (__uuid_write(c))
K
Kent Overstreet 已提交
1659
			goto err;
K
Kent Overstreet 已提交
1660 1661

		err = "cannot allocate new btree root";
1662
		c->root = __bch_btree_node_alloc(c, NULL, 0, true, NULL);
K
Kent Overstreet 已提交
1663
		if (IS_ERR_OR_NULL(c->root))
K
Kent Overstreet 已提交
1664
			goto err;
K
Kent Overstreet 已提交
1665

K
Kent Overstreet 已提交
1666
		mutex_lock(&c->root->write_lock);
K
Kent Overstreet 已提交
1667
		bkey_copy_key(&c->root->key, &MAX_KEY);
K
Kent Overstreet 已提交
1668
		bch_btree_node_write(c->root, &cl);
K
Kent Overstreet 已提交
1669
		mutex_unlock(&c->root->write_lock);
K
Kent Overstreet 已提交
1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681

		bch_btree_set_root(c->root);
		rw_unlock(true, c->root);

		/*
		 * We don't want to write the first journal entry until
		 * everything is set up - fortunately journal entries won't be
		 * written until the SET_CACHE_SYNC() here:
		 */
		SET_CACHE_SYNC(&c->sb, true);

		bch_journal_next(&c->journal);
K
Kent Overstreet 已提交
1682
		bch_journal_meta(c, &cl);
K
Kent Overstreet 已提交
1683 1684
	}

K
Kent Overstreet 已提交
1685 1686 1687 1688
	err = "error starting gc thread";
	if (bch_gc_thread_start(c))
		goto err;

K
Kent Overstreet 已提交
1689
	closure_sync(&cl);
K
Kent Overstreet 已提交
1690 1691 1692 1693 1694 1695 1696 1697
	c->sb.last_mount = get_seconds();
	bcache_write_super(c);

	list_for_each_entry_safe(dc, t, &uncached_devices, list)
		bch_cached_dev_attach(dc, c);

	flash_devs_run(c);

1698
	set_bit(CACHE_SET_RUNNING, &c->flags);
K
Kent Overstreet 已提交
1699 1700
	return;
err:
K
Kent Overstreet 已提交
1701
	closure_sync(&cl);
K
Kent Overstreet 已提交
1702
	/* XXX: test this, it's broken */
1703
	bch_cache_set_error(c, "%s", err);
K
Kent Overstreet 已提交
1704 1705 1706 1707 1708
}

static bool can_attach_cache(struct cache *ca, struct cache_set *c)
{
	return ca->sb.block_size	== c->sb.block_size &&
1709
		ca->sb.bucket_size	== c->sb.bucket_size &&
K
Kent Overstreet 已提交
1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761
		ca->sb.nr_in_set	== c->sb.nr_in_set;
}

static const char *register_cache_set(struct cache *ca)
{
	char buf[12];
	const char *err = "cannot allocate memory";
	struct cache_set *c;

	list_for_each_entry(c, &bch_cache_sets, list)
		if (!memcmp(c->sb.set_uuid, ca->sb.set_uuid, 16)) {
			if (c->cache[ca->sb.nr_this_dev])
				return "duplicate cache set member";

			if (!can_attach_cache(ca, c))
				return "cache sb does not match set";

			if (!CACHE_SYNC(&ca->sb))
				SET_CACHE_SYNC(&c->sb, false);

			goto found;
		}

	c = bch_cache_set_alloc(&ca->sb);
	if (!c)
		return err;

	err = "error creating kobject";
	if (kobject_add(&c->kobj, bcache_kobj, "%pU", c->sb.set_uuid) ||
	    kobject_add(&c->internal, &c->kobj, "internal"))
		goto err;

	if (bch_cache_accounting_add_kobjs(&c->accounting, &c->kobj))
		goto err;

	bch_debug_init_cache_set(c);

	list_add(&c->list, &bch_cache_sets);
found:
	sprintf(buf, "cache%i", ca->sb.nr_this_dev);
	if (sysfs_create_link(&ca->kobj, &c->kobj, "set") ||
	    sysfs_create_link(&c->kobj, &ca->kobj, buf))
		goto err;

	if (ca->sb.seq > c->sb.seq) {
		c->sb.version		= ca->sb.version;
		memcpy(c->sb.set_uuid, ca->sb.set_uuid, 16);
		c->sb.flags             = ca->sb.flags;
		c->sb.seq		= ca->sb.seq;
		pr_debug("set version = %llu", c->sb.version);
	}

1762
	kobject_get(&ca->kobj);
K
Kent Overstreet 已提交
1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780
	ca->set = c;
	ca->set->cache[ca->sb.nr_this_dev] = ca;
	c->cache_by_alloc[c->caches_loaded++] = ca;

	if (c->caches_loaded == c->sb.nr_in_set)
		run_cache_set(c);

	return NULL;
err:
	bch_cache_set_unregister(c);
	return err;
}

/* Cache device */

void bch_cache_release(struct kobject *kobj)
{
	struct cache *ca = container_of(kobj, struct cache, kobj);
1781
	unsigned i;
K
Kent Overstreet 已提交
1782

1783 1784
	if (ca->set) {
		BUG_ON(ca->set->cache[ca->sb.nr_this_dev] != ca);
K
Kent Overstreet 已提交
1785
		ca->set->cache[ca->sb.nr_this_dev] = NULL;
1786
	}
K
Kent Overstreet 已提交
1787 1788 1789 1790 1791 1792 1793

	free_pages((unsigned long) ca->disk_buckets, ilog2(bucket_pages(ca)));
	kfree(ca->prio_buckets);
	vfree(ca->buckets);

	free_heap(&ca->heap);
	free_fifo(&ca->free_inc);
1794 1795 1796

	for (i = 0; i < RESERVE_NR; i++)
		free_fifo(&ca->free[i]);
K
Kent Overstreet 已提交
1797 1798 1799 1800

	if (ca->sb_bio.bi_inline_vecs[0].bv_page)
		put_page(ca->sb_bio.bi_io_vec[0].bv_page);

1801
	if (!IS_ERR_OR_NULL(ca->bdev))
K
Kent Overstreet 已提交
1802 1803 1804 1805 1806 1807
		blkdev_put(ca->bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL);

	kfree(ca);
	module_put(THIS_MODULE);
}

1808
static int cache_alloc(struct cache *ca)
K
Kent Overstreet 已提交
1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819
{
	size_t free;
	struct bucket *b;

	__module_get(THIS_MODULE);
	kobject_init(&ca->kobj, &bch_cache_ktype);

	bio_init(&ca->journal.bio);
	ca->journal.bio.bi_max_vecs = 8;
	ca->journal.bio.bi_io_vec = ca->journal.bio.bi_inline_vecs;

1820
	free = roundup_pow_of_two(ca->sb.nbuckets) >> 10;
K
Kent Overstreet 已提交
1821

1822 1823 1824 1825
	if (!init_fifo(&ca->free[RESERVE_BTREE], 8, GFP_KERNEL) ||
	    !init_fifo(&ca->free[RESERVE_PRIO], prio_buckets(ca), GFP_KERNEL) ||
	    !init_fifo(&ca->free[RESERVE_MOVINGGC], free, GFP_KERNEL) ||
	    !init_fifo(&ca->free[RESERVE_NONE], free, GFP_KERNEL) ||
K
Kent Overstreet 已提交
1826 1827
	    !init_fifo(&ca->free_inc,	free << 2, GFP_KERNEL) ||
	    !init_heap(&ca->heap,	free << 3, GFP_KERNEL) ||
1828
	    !(ca->buckets	= vzalloc(sizeof(struct bucket) *
K
Kent Overstreet 已提交
1829 1830 1831
					  ca->sb.nbuckets)) ||
	    !(ca->prio_buckets	= kzalloc(sizeof(uint64_t) * prio_buckets(ca) *
					  2, GFP_KERNEL)) ||
1832
	    !(ca->disk_buckets	= alloc_bucket_pages(GFP_KERNEL, ca)))
1833
		return -ENOMEM;
K
Kent Overstreet 已提交
1834 1835 1836 1837 1838 1839 1840 1841 1842

	ca->prio_last_buckets = ca->prio_buckets + prio_buckets(ca);

	for_each_bucket(b, ca)
		atomic_set(&b->pin, 0);

	return 0;
}

1843
static int register_cache(struct cache_sb *sb, struct page *sb_page,
1844
				struct block_device *bdev, struct cache *ca)
K
Kent Overstreet 已提交
1845 1846
{
	char name[BDEVNAME_SIZE];
1847 1848
	const char *err = NULL;
	int ret = 0;
K
Kent Overstreet 已提交
1849

1850
	memcpy(&ca->sb, sb, sizeof(struct cache_sb));
K
Kent Overstreet 已提交
1851 1852 1853
	ca->bdev = bdev;
	ca->bdev->bd_holder = ca;

1854 1855 1856 1857 1858 1859
	bio_init(&ca->sb_bio);
	ca->sb_bio.bi_max_vecs	= 1;
	ca->sb_bio.bi_io_vec	= ca->sb_bio.bi_inline_vecs;
	ca->sb_bio.bi_io_vec[0].bv_page = sb_page;
	get_page(sb_page);

K
Kent Overstreet 已提交
1860 1861 1862
	if (blk_queue_discard(bdev_get_queue(ca->bdev)))
		ca->discard = CACHE_DISCARD(&ca->sb);

1863
	ret = cache_alloc(ca);
1864
	if (ret != 0)
1865 1866
		goto err;

1867 1868 1869 1870 1871
	if (kobject_add(&ca->kobj, &part_to_dev(bdev->bd_part)->kobj, "bcache")) {
		err = "error calling kobject_add";
		ret = -ENOMEM;
		goto out;
	}
K
Kent Overstreet 已提交
1872

1873
	mutex_lock(&bch_register_lock);
K
Kent Overstreet 已提交
1874
	err = register_cache_set(ca);
1875 1876
	mutex_unlock(&bch_register_lock);

1877 1878 1879 1880
	if (err) {
		ret = -ENODEV;
		goto out;
	}
K
Kent Overstreet 已提交
1881 1882

	pr_info("registered cache device %s", bdevname(bdev, name));
1883

1884 1885
out:
	kobject_put(&ca->kobj);
1886

K
Kent Overstreet 已提交
1887
err:
1888 1889 1890 1891
	if (err)
		pr_notice("error opening %s: %s", bdevname(bdev, name), err);

	return ret;
K
Kent Overstreet 已提交
1892 1893 1894 1895 1896 1897 1898 1899 1900 1901
}

/* Global interfaces/init */

static ssize_t register_bcache(struct kobject *, struct kobj_attribute *,
			       const char *, size_t);

kobj_attribute_write(register,		register_bcache);
kobj_attribute_write(register_quiet,	register_bcache);

1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931
static bool bch_is_open_backing(struct block_device *bdev) {
	struct cache_set *c, *tc;
	struct cached_dev *dc, *t;

	list_for_each_entry_safe(c, tc, &bch_cache_sets, list)
		list_for_each_entry_safe(dc, t, &c->cached_devs, list)
			if (dc->bdev == bdev)
				return true;
	list_for_each_entry_safe(dc, t, &uncached_devices, list)
		if (dc->bdev == bdev)
			return true;
	return false;
}

static bool bch_is_open_cache(struct block_device *bdev) {
	struct cache_set *c, *tc;
	struct cache *ca;
	unsigned i;

	list_for_each_entry_safe(c, tc, &bch_cache_sets, list)
		for_each_cache(ca, c, i)
			if (ca->bdev == bdev)
				return true;
	return false;
}

static bool bch_is_open(struct block_device *bdev) {
	return bch_is_open_cache(bdev) || bch_is_open_backing(bdev);
}

K
Kent Overstreet 已提交
1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952
static ssize_t register_bcache(struct kobject *k, struct kobj_attribute *attr,
			       const char *buffer, size_t size)
{
	ssize_t ret = size;
	const char *err = "cannot allocate memory";
	char *path = NULL;
	struct cache_sb *sb = NULL;
	struct block_device *bdev = NULL;
	struct page *sb_page = NULL;

	if (!try_module_get(THIS_MODULE))
		return -EBUSY;

	if (!(path = kstrndup(buffer, size, GFP_KERNEL)) ||
	    !(sb = kmalloc(sizeof(struct cache_sb), GFP_KERNEL)))
		goto err;

	err = "failed to open device";
	bdev = blkdev_get_by_path(strim(path),
				  FMODE_READ|FMODE_WRITE|FMODE_EXCL,
				  sb);
1953
	if (IS_ERR(bdev)) {
1954 1955
		if (bdev == ERR_PTR(-EBUSY)) {
			bdev = lookup_bdev(strim(path));
1956
			mutex_lock(&bch_register_lock);
1957 1958 1959 1960
			if (!IS_ERR(bdev) && bch_is_open(bdev))
				err = "device already registered";
			else
				err = "device busy";
1961
			mutex_unlock(&bch_register_lock);
1962 1963
			if (attr == &ksysfs_register_quiet)
				goto out;
1964
		}
K
Kent Overstreet 已提交
1965
		goto err;
1966 1967 1968 1969 1970
	}

	err = "failed to set blocksize";
	if (set_blocksize(bdev, 4096))
		goto err_close;
K
Kent Overstreet 已提交
1971 1972 1973 1974 1975

	err = read_super(sb, bdev, &sb_page);
	if (err)
		goto err_close;

1976
	if (SB_IS_BDEV(sb)) {
K
Kent Overstreet 已提交
1977
		struct cached_dev *dc = kzalloc(sizeof(*dc), GFP_KERNEL);
1978 1979
		if (!dc)
			goto err_close;
K
Kent Overstreet 已提交
1980

1981
		mutex_lock(&bch_register_lock);
1982
		register_bdev(sb, sb_page, bdev, dc);
1983
		mutex_unlock(&bch_register_lock);
K
Kent Overstreet 已提交
1984 1985
	} else {
		struct cache *ca = kzalloc(sizeof(*ca), GFP_KERNEL);
1986 1987
		if (!ca)
			goto err_close;
K
Kent Overstreet 已提交
1988

1989 1990
		if (register_cache(sb, sb_page, bdev, ca) != 0)
			goto err_close;
K
Kent Overstreet 已提交
1991
	}
1992 1993
out:
	if (sb_page)
K
Kent Overstreet 已提交
1994 1995 1996 1997 1998
		put_page(sb_page);
	kfree(sb);
	kfree(path);
	module_put(THIS_MODULE);
	return ret;
1999 2000 2001 2002

err_close:
	blkdev_put(bdev, FMODE_READ|FMODE_WRITE|FMODE_EXCL);
err:
2003
	pr_info("error opening %s: %s", path, err);
2004 2005
	ret = -EINVAL;
	goto out;
K
Kent Overstreet 已提交
2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077
}

static int bcache_reboot(struct notifier_block *n, unsigned long code, void *x)
{
	if (code == SYS_DOWN ||
	    code == SYS_HALT ||
	    code == SYS_POWER_OFF) {
		DEFINE_WAIT(wait);
		unsigned long start = jiffies;
		bool stopped = false;

		struct cache_set *c, *tc;
		struct cached_dev *dc, *tdc;

		mutex_lock(&bch_register_lock);

		if (list_empty(&bch_cache_sets) &&
		    list_empty(&uncached_devices))
			goto out;

		pr_info("Stopping all devices:");

		list_for_each_entry_safe(c, tc, &bch_cache_sets, list)
			bch_cache_set_stop(c);

		list_for_each_entry_safe(dc, tdc, &uncached_devices, list)
			bcache_device_stop(&dc->disk);

		/* What's a condition variable? */
		while (1) {
			long timeout = start + 2 * HZ - jiffies;

			stopped = list_empty(&bch_cache_sets) &&
				list_empty(&uncached_devices);

			if (timeout < 0 || stopped)
				break;

			prepare_to_wait(&unregister_wait, &wait,
					TASK_UNINTERRUPTIBLE);

			mutex_unlock(&bch_register_lock);
			schedule_timeout(timeout);
			mutex_lock(&bch_register_lock);
		}

		finish_wait(&unregister_wait, &wait);

		if (stopped)
			pr_info("All devices stopped");
		else
			pr_notice("Timeout waiting for devices to be closed");
out:
		mutex_unlock(&bch_register_lock);
	}

	return NOTIFY_DONE;
}

static struct notifier_block reboot = {
	.notifier_call	= bcache_reboot,
	.priority	= INT_MAX, /* before any real devices */
};

static void bcache_exit(void)
{
	bch_debug_exit();
	bch_request_exit();
	if (bcache_kobj)
		kobject_put(bcache_kobj);
	if (bcache_wq)
		destroy_workqueue(bcache_wq);
2078 2079
	if (bcache_major)
		unregister_blkdev(bcache_major, "bcache");
K
Kent Overstreet 已提交
2080 2081 2082 2083 2084 2085 2086 2087 2088 2089 2090 2091 2092 2093
	unregister_reboot_notifier(&reboot);
}

static int __init bcache_init(void)
{
	static const struct attribute *files[] = {
		&ksysfs_register.attr,
		&ksysfs_register_quiet.attr,
		NULL
	};

	mutex_init(&bch_register_lock);
	init_waitqueue_head(&unregister_wait);
	register_reboot_notifier(&reboot);
2094
	closure_debug_init();
K
Kent Overstreet 已提交
2095 2096

	bcache_major = register_blkdev(0, "bcache");
2097 2098
	if (bcache_major < 0) {
		unregister_reboot_notifier(&reboot);
K
Kent Overstreet 已提交
2099
		return bcache_major;
2100
	}
K
Kent Overstreet 已提交
2101

2102
	if (!(bcache_wq = alloc_workqueue("bcache", WQ_MEM_RECLAIM, 0)) ||
K
Kent Overstreet 已提交
2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116
	    !(bcache_kobj = kobject_create_and_add("bcache", fs_kobj)) ||
	    sysfs_create_files(bcache_kobj, files) ||
	    bch_request_init() ||
	    bch_debug_init(bcache_kobj))
		goto err;

	return 0;
err:
	bcache_exit();
	return -ENOMEM;
}

module_exit(bcache_exit);
module_init(bcache_init);