pmem.c 11.6 KB
Newer Older
1 2 3
/*
 * Persistent Memory Driver
 *
4
 * Copyright (c) 2014-2015, Intel Corporation.
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 * Copyright (c) 2015, Christoph Hellwig <hch@lst.de>.
 * Copyright (c) 2015, Boaz Harrosh <boaz@plexistor.com>.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 */

#include <asm/cacheflush.h>
#include <linux/blkdev.h>
#include <linux/hdreg.h>
#include <linux/init.h>
#include <linux/platform_device.h>
#include <linux/module.h>
24
#include <linux/memory_hotplug.h>
25
#include <linux/moduleparam.h>
26
#include <linux/vmalloc.h>
27
#include <linux/slab.h>
28
#include <linux/pmem.h>
29
#include <linux/nd.h>
30
#include "pfn.h"
31
#include "nd.h"
32 33 34 35

struct pmem_device {
	struct request_queue	*pmem_queue;
	struct gendisk		*pmem_disk;
36
	struct nd_namespace_common *ndns;
37 38 39

	/* One contiguous memory region per device */
	phys_addr_t		phys_addr;
40 41
	/* when non-zero this device is hosting a 'pfn' instance */
	phys_addr_t		data_offset;
42
	void __pmem		*virt_addr;
43 44 45 46 47 48 49 50 51 52
	size_t			size;
};

static int pmem_major;

static void pmem_do_bvec(struct pmem_device *pmem, struct page *page,
			unsigned int len, unsigned int off, int rw,
			sector_t sector)
{
	void *mem = kmap_atomic(page);
53
	phys_addr_t pmem_off = sector * 512 + pmem->data_offset;
54
	void __pmem *pmem_addr = pmem->virt_addr + pmem_off;
55 56

	if (rw == READ) {
57
		memcpy_from_pmem(mem + off, pmem_addr, len);
58 59 60
		flush_dcache_page(page);
	} else {
		flush_dcache_page(page);
61
		memcpy_to_pmem(pmem_addr, mem + off, len);
62 63 64 65 66 67 68
	}

	kunmap_atomic(mem);
}

static void pmem_make_request(struct request_queue *q, struct bio *bio)
{
D
Dan Williams 已提交
69 70
	bool do_acct;
	unsigned long start;
71 72
	struct bio_vec bvec;
	struct bvec_iter iter;
D
Dan Williams 已提交
73 74
	struct block_device *bdev = bio->bi_bdev;
	struct pmem_device *pmem = bdev->bd_disk->private_data;
75

D
Dan Williams 已提交
76
	do_acct = nd_iostat_start(bio, &start);
D
Dan Williams 已提交
77
	bio_for_each_segment(bvec, bio, iter)
78
		pmem_do_bvec(pmem, bvec.bv_page, bvec.bv_len, bvec.bv_offset,
D
Dan Williams 已提交
79
				bio_data_dir(bio), iter.bi_sector);
D
Dan Williams 已提交
80 81
	if (do_acct)
		nd_iostat_end(bio, start);
82 83 84 85

	if (bio_data_dir(bio))
		wmb_pmem();

86
	bio_endio(bio);
87 88 89 90 91 92 93 94
}

static int pmem_rw_page(struct block_device *bdev, sector_t sector,
		       struct page *page, int rw)
{
	struct pmem_device *pmem = bdev->bd_disk->private_data;

	pmem_do_bvec(pmem, page, PAGE_CACHE_SIZE, 0, rw, sector);
95 96
	if (rw & WRITE)
		wmb_pmem();
97 98 99 100 101 102
	page_endio(page, rw & WRITE, 0);

	return 0;
}

static long pmem_direct_access(struct block_device *bdev, sector_t sector,
103
		      void __pmem **kaddr, unsigned long *pfn)
104 105
{
	struct pmem_device *pmem = bdev->bd_disk->private_data;
106 107 108 109 110 111 112 113 114 115 116
	resource_size_t offset = sector * 512 + pmem->data_offset;
	resource_size_t size;

	if (pmem->data_offset) {
		/*
		 * Limit the direct_access() size to what is covered by
		 * the memmap
		 */
		size = (pmem->size - offset) & ~ND_PFN_MASK;
	} else
		size = pmem->size - offset;
117

118
	/* FIXME convert DAX to comprehend that this mapping has a lifetime */
119
	*kaddr = pmem->virt_addr + offset;
120 121
	*pfn = (pmem->phys_addr + offset) >> PAGE_SHIFT;

122
	return size;
123 124 125 126 127 128
}

static const struct block_device_operations pmem_fops = {
	.owner =		THIS_MODULE,
	.rw_page =		pmem_rw_page,
	.direct_access =	pmem_direct_access,
129
	.revalidate_disk =	nvdimm_revalidate_disk,
130 131
};

132 133
static struct pmem_device *pmem_alloc(struct device *dev,
		struct resource *res, int id)
134 135 136
{
	struct pmem_device *pmem;

137
	pmem = devm_kzalloc(dev, sizeof(*pmem), GFP_KERNEL);
138
	if (!pmem)
139
		return ERR_PTR(-ENOMEM);
140 141 142

	pmem->phys_addr = res->start;
	pmem->size = resource_size(res);
143
	if (!arch_has_wmb_pmem())
144
		dev_warn(dev, "unable to guarantee persistence of writes\n");
145

146 147
	if (!devm_request_mem_region(dev, pmem->phys_addr, pmem->size,
			dev_name(dev))) {
148 149
		dev_warn(dev, "could not reserve region [0x%pa:0x%zx]\n",
				&pmem->phys_addr, pmem->size);
150
		return ERR_PTR(-EBUSY);
151 152
	}

153 154 155 156 157 158 159
	if (pmem_should_map_pages(dev)) {
		void *addr = devm_memremap_pages(dev, res);

		if (IS_ERR(addr))
			return addr;
		pmem->virt_addr = (void __pmem *) addr;
	} else {
D
Dan Williams 已提交
160 161 162
		pmem->virt_addr = (void __pmem *) devm_memremap(dev,
				pmem->phys_addr, pmem->size,
				ARCH_MEMREMAP_PMEM);
163 164
		if (!pmem->virt_addr)
			return ERR_PTR(-ENXIO);
165 166 167 168 169 170 171
	}

	return pmem;
}

static void pmem_detach_disk(struct pmem_device *pmem)
{
172 173 174
	if (!pmem->pmem_disk)
		return;

175 176 177 178 179
	del_gendisk(pmem->pmem_disk);
	put_disk(pmem->pmem_disk);
	blk_cleanup_queue(pmem->pmem_queue);
}

180 181
static int pmem_attach_disk(struct device *dev,
		struct nd_namespace_common *ndns, struct pmem_device *pmem)
182 183
{
	struct gendisk *disk;
184 185 186

	pmem->pmem_queue = blk_alloc_queue(GFP_KERNEL);
	if (!pmem->pmem_queue)
187
		return -ENOMEM;
188 189

	blk_queue_make_request(pmem->pmem_queue, pmem_make_request);
190
	blk_queue_physical_block_size(pmem->pmem_queue, PAGE_SIZE);
191
	blk_queue_max_hw_sectors(pmem->pmem_queue, UINT_MAX);
192
	blk_queue_bounce_limit(pmem->pmem_queue, BLK_BOUNCE_ANY);
193
	queue_flag_set_unlocked(QUEUE_FLAG_NONROT, pmem->pmem_queue);
194

195
	disk = alloc_disk(0);
196 197 198 199
	if (!disk) {
		blk_cleanup_queue(pmem->pmem_queue);
		return -ENOMEM;
	}
200 201

	disk->major		= pmem_major;
202
	disk->first_minor	= 0;
203 204 205 206
	disk->fops		= &pmem_fops;
	disk->private_data	= pmem;
	disk->queue		= pmem->pmem_queue;
	disk->flags		= GENHD_FL_EXT_DEVT;
V
Vishal Verma 已提交
207
	nvdimm_namespace_disk_name(ndns, disk->disk_name);
208 209
	disk->driverfs_dev = dev;
	set_capacity(disk, (pmem->size - pmem->data_offset) / 512);
210 211 212
	pmem->pmem_disk = disk;

	add_disk(disk);
213
	revalidate_disk(disk);
214

215 216
	return 0;
}
217

218 219 220 221 222 223 224 225 226 227 228
static int pmem_rw_bytes(struct nd_namespace_common *ndns,
		resource_size_t offset, void *buf, size_t size, int rw)
{
	struct pmem_device *pmem = dev_get_drvdata(ndns->claim);

	if (unlikely(offset + size > pmem->size)) {
		dev_WARN_ONCE(&ndns->dev, 1, "request out of range\n");
		return -EFAULT;
	}

	if (rw == READ)
229 230 231 232 233
		memcpy_from_pmem(buf, pmem->virt_addr + offset, size);
	else {
		memcpy_to_pmem(pmem->virt_addr + offset, buf, size);
		wmb_pmem();
	}
234 235 236 237

	return 0;
}

238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322
static int nd_pfn_init(struct nd_pfn *nd_pfn)
{
	struct nd_pfn_sb *pfn_sb = kzalloc(sizeof(*pfn_sb), GFP_KERNEL);
	struct pmem_device *pmem = dev_get_drvdata(&nd_pfn->dev);
	struct nd_namespace_common *ndns = nd_pfn->ndns;
	struct nd_region *nd_region;
	unsigned long npfns;
	phys_addr_t offset;
	u64 checksum;
	int rc;

	if (!pfn_sb)
		return -ENOMEM;

	nd_pfn->pfn_sb = pfn_sb;
	rc = nd_pfn_validate(nd_pfn);
	if (rc == 0 || rc == -EBUSY)
		return rc;

	/* section alignment for simple hotplug */
	if (nvdimm_namespace_capacity(ndns) < ND_PFN_ALIGN
			|| pmem->phys_addr & ND_PFN_MASK)
		return -ENODEV;

	nd_region = to_nd_region(nd_pfn->dev.parent);
	if (nd_region->ro) {
		dev_info(&nd_pfn->dev,
				"%s is read-only, unable to init metadata\n",
				dev_name(&nd_region->dev));
		goto err;
	}

	memset(pfn_sb, 0, sizeof(*pfn_sb));
	npfns = (pmem->size - SZ_8K) / SZ_4K;
	/*
	 * Note, we use 64 here for the standard size of struct page,
	 * debugging options may cause it to be larger in which case the
	 * implementation will limit the pfns advertised through
	 * ->direct_access() to those that are included in the memmap.
	 */
	if (nd_pfn->mode == PFN_MODE_PMEM)
		offset = ALIGN(SZ_8K + 64 * npfns, PMD_SIZE);
	else if (nd_pfn->mode == PFN_MODE_RAM)
		offset = SZ_8K;
	else
		goto err;

	npfns = (pmem->size - offset) / SZ_4K;
	pfn_sb->mode = cpu_to_le32(nd_pfn->mode);
	pfn_sb->dataoff = cpu_to_le64(offset);
	pfn_sb->npfns = cpu_to_le64(npfns);
	memcpy(pfn_sb->signature, PFN_SIG, PFN_SIG_LEN);
	memcpy(pfn_sb->uuid, nd_pfn->uuid, 16);
	pfn_sb->version_major = cpu_to_le16(1);
	checksum = nd_sb_checksum((struct nd_gen_sb *) pfn_sb);
	pfn_sb->checksum = cpu_to_le64(checksum);

	rc = nvdimm_write_bytes(ndns, SZ_4K, pfn_sb, sizeof(*pfn_sb));
	if (rc)
		goto err;

	return 0;
 err:
	nd_pfn->pfn_sb = NULL;
	kfree(pfn_sb);
	return -ENXIO;
}

static int nvdimm_namespace_detach_pfn(struct nd_namespace_common *ndns)
{
	struct nd_pfn *nd_pfn = to_nd_pfn(ndns->claim);
	struct pmem_device *pmem;

	/* free pmem disk */
	pmem = dev_get_drvdata(&nd_pfn->dev);
	pmem_detach_disk(pmem);

	/* release nd_pfn resources */
	kfree(nd_pfn->pfn_sb);
	nd_pfn->pfn_sb = NULL;

	return 0;
}

static int nvdimm_namespace_attach_pfn(struct nd_namespace_common *ndns)
323
{
324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366
	struct nd_namespace_io *nsio = to_nd_namespace_io(&ndns->dev);
	struct nd_pfn *nd_pfn = to_nd_pfn(ndns->claim);
	struct device *dev = &nd_pfn->dev;
	struct vmem_altmap *altmap;
	struct nd_region *nd_region;
	struct nd_pfn_sb *pfn_sb;
	struct pmem_device *pmem;
	phys_addr_t offset;
	int rc;

	if (!nd_pfn->uuid || !nd_pfn->ndns)
		return -ENODEV;

	nd_region = to_nd_region(dev->parent);
	rc = nd_pfn_init(nd_pfn);
	if (rc)
		return rc;

	if (PAGE_SIZE != SZ_4K) {
		dev_err(dev, "only supported on systems with 4K PAGE_SIZE\n");
		return -ENXIO;
	}
	if (nsio->res.start & ND_PFN_MASK) {
		dev_err(dev, "%s not memory hotplug section aligned\n",
				dev_name(&ndns->dev));
		return -ENXIO;
	}

	pfn_sb = nd_pfn->pfn_sb;
	offset = le64_to_cpu(pfn_sb->dataoff);
	nd_pfn->mode = le32_to_cpu(nd_pfn->pfn_sb->mode);
	if (nd_pfn->mode == PFN_MODE_RAM) {
		if (offset != SZ_8K)
			return -EINVAL;
		nd_pfn->npfns = le64_to_cpu(pfn_sb->npfns);
		altmap = NULL;
	} else {
		rc = -ENXIO;
		goto err;
	}

	/* establish pfn range for lookup, and switch to direct map */
	pmem = dev_get_drvdata(dev);
D
Dan Williams 已提交
367 368
	devm_memunmap(dev, (void __force *) pmem->virt_addr);
	pmem->virt_addr = (void __pmem *) devm_memremap_pages(dev, &nsio->res);
369 370 371 372 373 374 375 376 377 378 379 380 381 382 383
	if (IS_ERR(pmem->virt_addr)) {
		rc = PTR_ERR(pmem->virt_addr);
		goto err;
	}

	/* attach pmem disk in "pfn-mode" */
	pmem->data_offset = offset;
	rc = pmem_attach_disk(dev, ndns, pmem);
	if (rc)
		goto err;

	return rc;
 err:
	nvdimm_namespace_detach_pfn(ndns);
	return rc;
384 385
}

386
static int nd_pmem_probe(struct device *dev)
387
{
388
	struct nd_region *nd_region = to_nd_region(dev->parent);
389 390
	struct nd_namespace_common *ndns;
	struct nd_namespace_io *nsio;
391 392
	struct pmem_device *pmem;

393 394 395
	ndns = nvdimm_namespace_common_probe(dev);
	if (IS_ERR(ndns))
		return PTR_ERR(ndns);
396

397
	nsio = to_nd_namespace_io(&ndns->dev);
398
	pmem = pmem_alloc(dev, &nsio->res, nd_region->id);
399 400 401
	if (IS_ERR(pmem))
		return PTR_ERR(pmem);

402
	pmem->ndns = ndns;
403
	dev_set_drvdata(dev, pmem);
404
	ndns->rw_bytes = pmem_rw_bytes;
405

406
	if (is_nd_btt(dev))
407 408
		return nvdimm_namespace_attach_btt(ndns);

409 410 411 412
	if (is_nd_pfn(dev))
		return nvdimm_namespace_attach_pfn(ndns);

	if (nd_btt_probe(ndns, pmem) == 0) {
413
		/* we'll come back as btt-pmem */
414
		return -ENXIO;
415 416 417 418 419 420 421 422
	}

	if (nd_pfn_probe(ndns, pmem) == 0) {
		/* we'll come back as pfn-pmem */
		return -ENXIO;
	}

	return pmem_attach_disk(dev, ndns, pmem);
423 424
}

425
static int nd_pmem_remove(struct device *dev)
426
{
427
	struct pmem_device *pmem = dev_get_drvdata(dev);
428

429
	if (is_nd_btt(dev))
430 431 432
		nvdimm_namespace_detach_btt(pmem->ndns);
	else if (is_nd_pfn(dev))
		nvdimm_namespace_detach_pfn(pmem->ndns);
433 434 435
	else
		pmem_detach_disk(pmem);

436 437 438
	return 0;
}

439 440
MODULE_ALIAS("pmem");
MODULE_ALIAS_ND_DEVICE(ND_DEVICE_NAMESPACE_IO);
441
MODULE_ALIAS_ND_DEVICE(ND_DEVICE_NAMESPACE_PMEM);
442 443 444 445 446
static struct nd_device_driver nd_pmem_driver = {
	.probe = nd_pmem_probe,
	.remove = nd_pmem_remove,
	.drv = {
		.name = "nd_pmem",
447
	},
448
	.type = ND_DRIVER_NAMESPACE_IO | ND_DRIVER_NAMESPACE_PMEM,
449 450 451 452 453 454 455 456 457 458
};

static int __init pmem_init(void)
{
	int error;

	pmem_major = register_blkdev(0, "pmem");
	if (pmem_major < 0)
		return pmem_major;

459 460
	error = nd_driver_register(&nd_pmem_driver);
	if (error) {
461
		unregister_blkdev(pmem_major, "pmem");
462 463 464 465
		return error;
	}

	return 0;
466 467 468 469 470
}
module_init(pmem_init);

static void pmem_exit(void)
{
471
	driver_unregister(&nd_pmem_driver.drv);
472 473 474 475 476 477
	unregister_blkdev(pmem_major, "pmem");
}
module_exit(pmem_exit);

MODULE_AUTHOR("Ross Zwisler <ross.zwisler@linux.intel.com>");
MODULE_LICENSE("GPL v2");