pmem.c 16.5 KB
Newer Older
1 2 3
/*
 * Persistent Memory Driver
 *
4
 * Copyright (c) 2014-2015, Intel Corporation.
5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24
 * Copyright (c) 2015, Christoph Hellwig <hch@lst.de>.
 * Copyright (c) 2015, Boaz Harrosh <boaz@plexistor.com>.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 */

#include <asm/cacheflush.h>
#include <linux/blkdev.h>
#include <linux/hdreg.h>
#include <linux/init.h>
#include <linux/platform_device.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
25
#include <linux/badblocks.h>
D
Dan Williams 已提交
26
#include <linux/memremap.h>
27
#include <linux/vmalloc.h>
D
Dan Williams 已提交
28
#include <linux/pfn_t.h>
29
#include <linux/slab.h>
30
#include <linux/pmem.h>
31
#include <linux/nd.h>
32
#include "pfn.h"
33
#include "nd.h"
34 35 36 37

struct pmem_device {
	struct request_queue	*pmem_queue;
	struct gendisk		*pmem_disk;
38
	struct nd_namespace_common *ndns;
39 40 41

	/* One contiguous memory region per device */
	phys_addr_t		phys_addr;
42 43
	/* when non-zero this device is hosting a 'pfn' instance */
	phys_addr_t		data_offset;
A
Arnd Bergmann 已提交
44
	u64			pfn_flags;
45
	void __pmem		*virt_addr;
46
	/* immutable base size of the namespace */
47
	size_t			size;
48 49
	/* trim size when namespace capacity has been section aligned */
	u32			pfn_pad;
50
	struct badblocks	bb;
51 52
};

53 54 55 56 57 58 59 60 61 62 63 64 65
static bool is_bad_pmem(struct badblocks *bb, sector_t sector, unsigned int len)
{
	if (bb->count) {
		sector_t first_bad;
		int num_bad;

		return !!badblocks_check(bb, sector, len / 512, &first_bad,
				&num_bad);
	}

	return false;
}

66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84
static void pmem_clear_poison(struct pmem_device *pmem, phys_addr_t offset,
		unsigned int len)
{
	struct device *dev = disk_to_dev(pmem->pmem_disk);
	sector_t sector;
	long cleared;

	sector = (offset - pmem->data_offset) / 512;
	cleared = nvdimm_clear_poison(dev, pmem->phys_addr + offset, len);

	if (cleared > 0 && cleared / 512) {
		dev_dbg(dev, "%s: %llx clear %ld sector%s\n",
				__func__, (unsigned long long) sector,
				cleared / 512, cleared / 512 > 1 ? "s" : "");
		badblocks_clear(&pmem->bb, sector, cleared / 512);
	}
	invalidate_pmem(pmem->virt_addr + offset, len);
}

85
static int pmem_do_bvec(struct pmem_device *pmem, struct page *page,
86 87 88
			unsigned int len, unsigned int off, int rw,
			sector_t sector)
{
89
	int rc = 0;
90
	bool bad_pmem = false;
91
	void *mem = kmap_atomic(page);
92
	phys_addr_t pmem_off = sector * 512 + pmem->data_offset;
93
	void __pmem *pmem_addr = pmem->virt_addr + pmem_off;
94

95 96 97
	if (unlikely(is_bad_pmem(&pmem->bb, sector, len)))
		bad_pmem = true;

98
	if (rw == READ) {
99
		if (unlikely(bad_pmem))
100 101 102 103 104
			rc = -EIO;
		else {
			memcpy_from_pmem(mem + off, pmem_addr, len);
			flush_dcache_page(page);
		}
105 106
	} else {
		flush_dcache_page(page);
107
		memcpy_to_pmem(pmem_addr, mem + off, len);
108 109 110 111
		if (unlikely(bad_pmem)) {
			pmem_clear_poison(pmem, pmem_off, len);
			memcpy_to_pmem(pmem_addr, mem + off, len);
		}
112 113 114
	}

	kunmap_atomic(mem);
115
	return rc;
116 117
}

118
static blk_qc_t pmem_make_request(struct request_queue *q, struct bio *bio)
119
{
120
	int rc = 0;
D
Dan Williams 已提交
121 122
	bool do_acct;
	unsigned long start;
123 124
	struct bio_vec bvec;
	struct bvec_iter iter;
D
Dan Williams 已提交
125 126
	struct block_device *bdev = bio->bi_bdev;
	struct pmem_device *pmem = bdev->bd_disk->private_data;
127

D
Dan Williams 已提交
128
	do_acct = nd_iostat_start(bio, &start);
129 130 131 132 133 134 135 136 137
	bio_for_each_segment(bvec, bio, iter) {
		rc = pmem_do_bvec(pmem, bvec.bv_page, bvec.bv_len,
				bvec.bv_offset, bio_data_dir(bio),
				iter.bi_sector);
		if (rc) {
			bio->bi_error = rc;
			break;
		}
	}
D
Dan Williams 已提交
138 139
	if (do_acct)
		nd_iostat_end(bio, start);
140 141 142 143

	if (bio_data_dir(bio))
		wmb_pmem();

144
	bio_endio(bio);
145
	return BLK_QC_T_NONE;
146 147 148 149 150 151
}

static int pmem_rw_page(struct block_device *bdev, sector_t sector,
		       struct page *page, int rw)
{
	struct pmem_device *pmem = bdev->bd_disk->private_data;
152
	int rc;
153

154
	rc = pmem_do_bvec(pmem, page, PAGE_CACHE_SIZE, 0, rw, sector);
155 156
	if (rw & WRITE)
		wmb_pmem();
157

158 159 160 161 162 163 164 165 166 167
	/*
	 * The ->rw_page interface is subtle and tricky.  The core
	 * retries on any error, so we can only invoke page_endio() in
	 * the successful completion case.  Otherwise, we'll see crashes
	 * caused by double completion.
	 */
	if (rc == 0)
		page_endio(page, rw & WRITE, 0);

	return rc;
168 169 170
}

static long pmem_direct_access(struct block_device *bdev, sector_t sector,
D
Dan Williams 已提交
171
		      void __pmem **kaddr, pfn_t *pfn)
172 173
{
	struct pmem_device *pmem = bdev->bd_disk->private_data;
174
	resource_size_t offset = sector * 512 + pmem->data_offset;
175

176
	*kaddr = pmem->virt_addr + offset;
D
Dan Williams 已提交
177
	*pfn = phys_to_pfn_t(pmem->phys_addr + offset, pmem->pfn_flags);
178

179
	return pmem->size - pmem->pfn_pad - offset;
180 181 182 183 184 185
}

static const struct block_device_operations pmem_fops = {
	.owner =		THIS_MODULE,
	.rw_page =		pmem_rw_page,
	.direct_access =	pmem_direct_access,
186
	.revalidate_disk =	nvdimm_revalidate_disk,
187 188
};

189 190
static struct pmem_device *pmem_alloc(struct device *dev,
		struct resource *res, int id)
191 192
{
	struct pmem_device *pmem;
193
	struct request_queue *q;
194

195
	pmem = devm_kzalloc(dev, sizeof(*pmem), GFP_KERNEL);
196
	if (!pmem)
197
		return ERR_PTR(-ENOMEM);
198 199 200

	pmem->phys_addr = res->start;
	pmem->size = resource_size(res);
201
	if (!arch_has_wmb_pmem())
202
		dev_warn(dev, "unable to guarantee persistence of writes\n");
203

204 205
	if (!devm_request_mem_region(dev, pmem->phys_addr, pmem->size,
			dev_name(dev))) {
206 207
		dev_warn(dev, "could not reserve region [0x%pa:0x%zx]\n",
				&pmem->phys_addr, pmem->size);
208
		return ERR_PTR(-EBUSY);
209 210
	}

211 212 213 214
	q = blk_alloc_queue_node(GFP_KERNEL, dev_to_node(dev));
	if (!q)
		return ERR_PTR(-ENOMEM);

D
Dan Williams 已提交
215 216
	pmem->pfn_flags = PFN_DEV;
	if (pmem_should_map_pages(dev)) {
217
		pmem->virt_addr = (void __pmem *) devm_memremap_pages(dev, res,
218
				&q->q_usage_counter, NULL);
D
Dan Williams 已提交
219 220
		pmem->pfn_flags |= PFN_MAP;
	} else
D
Dan Williams 已提交
221 222 223
		pmem->virt_addr = (void __pmem *) devm_memremap(dev,
				pmem->phys_addr, pmem->size,
				ARCH_MEMREMAP_PMEM);
224

225 226
	if (IS_ERR(pmem->virt_addr)) {
		blk_cleanup_queue(q);
227
		return (void __force *) pmem->virt_addr;
228
	}
229

230
	pmem->pmem_queue = q;
231 232 233 234 235
	return pmem;
}

static void pmem_detach_disk(struct pmem_device *pmem)
{
236 237 238
	if (!pmem->pmem_disk)
		return;

239 240 241 242 243
	del_gendisk(pmem->pmem_disk);
	put_disk(pmem->pmem_disk);
	blk_cleanup_queue(pmem->pmem_queue);
}

244 245
static int pmem_attach_disk(struct device *dev,
		struct nd_namespace_common *ndns, struct pmem_device *pmem)
246
{
247
	int nid = dev_to_node(dev);
248
	struct gendisk *disk;
249 250

	blk_queue_make_request(pmem->pmem_queue, pmem_make_request);
251
	blk_queue_physical_block_size(pmem->pmem_queue, PAGE_SIZE);
252
	blk_queue_max_hw_sectors(pmem->pmem_queue, UINT_MAX);
253
	blk_queue_bounce_limit(pmem->pmem_queue, BLK_BOUNCE_ANY);
254
	queue_flag_set_unlocked(QUEUE_FLAG_NONROT, pmem->pmem_queue);
255

256
	disk = alloc_disk_node(0, nid);
257 258 259 260
	if (!disk) {
		blk_cleanup_queue(pmem->pmem_queue);
		return -ENOMEM;
	}
261 262 263 264 265

	disk->fops		= &pmem_fops;
	disk->private_data	= pmem;
	disk->queue		= pmem->pmem_queue;
	disk->flags		= GENHD_FL_EXT_DEVT;
V
Vishal Verma 已提交
266
	nvdimm_namespace_disk_name(ndns, disk->disk_name);
267
	disk->driverfs_dev = dev;
268 269
	set_capacity(disk, (pmem->size - pmem->pfn_pad - pmem->data_offset)
			/ 512);
270
	pmem->pmem_disk = disk;
271
	devm_exit_badblocks(dev, &pmem->bb);
272 273 274
	if (devm_init_badblocks(dev, &pmem->bb))
		return -ENOMEM;
	nvdimm_namespace_add_poison(ndns, &pmem->bb, pmem->data_offset);
275

276
	disk->bb = &pmem->bb;
277
	add_disk(disk);
278
	revalidate_disk(disk);
279

280 281
	return 0;
}
282

283 284 285 286 287 288 289 290 291 292
static int pmem_rw_bytes(struct nd_namespace_common *ndns,
		resource_size_t offset, void *buf, size_t size, int rw)
{
	struct pmem_device *pmem = dev_get_drvdata(ndns->claim);

	if (unlikely(offset + size > pmem->size)) {
		dev_WARN_ONCE(&ndns->dev, 1, "request out of range\n");
		return -EFAULT;
	}

293 294 295 296 297
	if (rw == READ) {
		unsigned int sz_align = ALIGN(size + (offset & (512 - 1)), 512);

		if (unlikely(is_bad_pmem(&pmem->bb, offset / 512, sz_align)))
			return -EIO;
298
		memcpy_from_pmem(buf, pmem->virt_addr + offset, size);
299
	} else {
300 301 302
		memcpy_to_pmem(pmem->virt_addr + offset, buf, size);
		wmb_pmem();
	}
303 304 305 306

	return 0;
}

307 308 309 310 311
static int nd_pfn_init(struct nd_pfn *nd_pfn)
{
	struct nd_pfn_sb *pfn_sb = kzalloc(sizeof(*pfn_sb), GFP_KERNEL);
	struct pmem_device *pmem = dev_get_drvdata(&nd_pfn->dev);
	struct nd_namespace_common *ndns = nd_pfn->ndns;
312 313 314
	u32 start_pad = 0, end_trunc = 0;
	resource_size_t start, size;
	struct nd_namespace_io *nsio;
315 316 317 318 319 320 321 322 323 324 325
	struct nd_region *nd_region;
	unsigned long npfns;
	phys_addr_t offset;
	u64 checksum;
	int rc;

	if (!pfn_sb)
		return -ENOMEM;

	nd_pfn->pfn_sb = pfn_sb;
	rc = nd_pfn_validate(nd_pfn);
326 327 328
	if (rc == -ENODEV)
		/* no info block, do init */;
	else
329 330 331 332 333 334 335 336 337 338 339
		return rc;

	nd_region = to_nd_region(nd_pfn->dev.parent);
	if (nd_region->ro) {
		dev_info(&nd_pfn->dev,
				"%s is read-only, unable to init metadata\n",
				dev_name(&nd_region->dev));
		goto err;
	}

	memset(pfn_sb, 0, sizeof(*pfn_sb));
340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366

	/*
	 * Check if pmem collides with 'System RAM' when section aligned and
	 * trim it accordingly
	 */
	nsio = to_nd_namespace_io(&ndns->dev);
	start = PHYS_SECTION_ALIGN_DOWN(nsio->res.start);
	size = resource_size(&nsio->res);
	if (region_intersects(start, size, IORESOURCE_SYSTEM_RAM,
				IORES_DESC_NONE) == REGION_MIXED) {

		start = nsio->res.start;
		start_pad = PHYS_SECTION_ALIGN_UP(start) - start;
	}

	start = nsio->res.start;
	size = PHYS_SECTION_ALIGN_UP(start + size) - start;
	if (region_intersects(start, size, IORESOURCE_SYSTEM_RAM,
				IORES_DESC_NONE) == REGION_MIXED) {
		size = resource_size(&nsio->res);
		end_trunc = start + size - PHYS_SECTION_ALIGN_DOWN(start + size);
	}

	if (start_pad + end_trunc)
		dev_info(&nd_pfn->dev, "%s section collision, truncate %d bytes\n",
				dev_name(&ndns->dev), start_pad + end_trunc);

367 368 369 370 371 372
	/*
	 * Note, we use 64 here for the standard size of struct page,
	 * debugging options may cause it to be larger in which case the
	 * implementation will limit the pfns advertised through
	 * ->direct_access() to those that are included in the memmap.
	 */
373 374
	start += start_pad;
	npfns = (pmem->size - start_pad - end_trunc - SZ_8K) / SZ_4K;
375
	if (nd_pfn->mode == PFN_MODE_PMEM)
376 377
		offset = ALIGN(start + SZ_8K + 64 * npfns, nd_pfn->align)
			- start;
378
	else if (nd_pfn->mode == PFN_MODE_RAM)
379
		offset = ALIGN(start + SZ_8K, nd_pfn->align) - start;
380 381 382
	else
		goto err;

383 384 385 386 387 388 389
	if (offset + start_pad + end_trunc >= pmem->size) {
		dev_err(&nd_pfn->dev, "%s unable to satisfy requested alignment\n",
				dev_name(&ndns->dev));
		goto err;
	}

	npfns = (pmem->size - offset - start_pad - end_trunc) / SZ_4K;
390 391 392 393 394
	pfn_sb->mode = cpu_to_le32(nd_pfn->mode);
	pfn_sb->dataoff = cpu_to_le64(offset);
	pfn_sb->npfns = cpu_to_le64(npfns);
	memcpy(pfn_sb->signature, PFN_SIG, PFN_SIG_LEN);
	memcpy(pfn_sb->uuid, nd_pfn->uuid, 16);
395
	memcpy(pfn_sb->parent_uuid, nd_dev_to_uuid(&ndns->dev), 16);
396
	pfn_sb->version_major = cpu_to_le16(1);
397 398 399
	pfn_sb->version_minor = cpu_to_le16(1);
	pfn_sb->start_pad = cpu_to_le32(start_pad);
	pfn_sb->end_trunc = cpu_to_le32(end_trunc);
400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429
	checksum = nd_sb_checksum((struct nd_gen_sb *) pfn_sb);
	pfn_sb->checksum = cpu_to_le64(checksum);

	rc = nvdimm_write_bytes(ndns, SZ_4K, pfn_sb, sizeof(*pfn_sb));
	if (rc)
		goto err;

	return 0;
 err:
	nd_pfn->pfn_sb = NULL;
	kfree(pfn_sb);
	return -ENXIO;
}

static int nvdimm_namespace_detach_pfn(struct nd_namespace_common *ndns)
{
	struct nd_pfn *nd_pfn = to_nd_pfn(ndns->claim);
	struct pmem_device *pmem;

	/* free pmem disk */
	pmem = dev_get_drvdata(&nd_pfn->dev);
	pmem_detach_disk(pmem);

	/* release nd_pfn resources */
	kfree(nd_pfn->pfn_sb);
	nd_pfn->pfn_sb = NULL;

	return 0;
}

430 431 432 433 434 435
/*
 * We hotplug memory at section granularity, pad the reserved area from
 * the previous section base to the namespace base address.
 */
static unsigned long init_altmap_base(resource_size_t base)
{
436
	unsigned long base_pfn = PHYS_PFN(base);
437 438 439 440 441 442

	return PFN_SECTION_ALIGN_DOWN(base_pfn);
}

static unsigned long init_altmap_reserve(resource_size_t base)
{
443 444
	unsigned long reserve = PHYS_PFN(SZ_8K);
	unsigned long base_pfn = PHYS_PFN(base);
445 446 447 448 449

	reserve += base_pfn - PFN_SECTION_ALIGN_DOWN(base_pfn);
	return reserve;
}

450
static int __nvdimm_namespace_attach_pfn(struct nd_pfn *nd_pfn)
451
{
452
	int rc;
453 454 455 456 457 458 459 460 461 462 463
	struct resource res;
	struct request_queue *q;
	struct pmem_device *pmem;
	struct vmem_altmap *altmap;
	struct device *dev = &nd_pfn->dev;
	struct nd_pfn_sb *pfn_sb = nd_pfn->pfn_sb;
	struct nd_namespace_common *ndns = nd_pfn->ndns;
	u32 start_pad = __le32_to_cpu(pfn_sb->start_pad);
	u32 end_trunc = __le32_to_cpu(pfn_sb->end_trunc);
	struct nd_namespace_io *nsio = to_nd_namespace_io(&ndns->dev);
	resource_size_t base = nsio->res.start + start_pad;
464
	struct vmem_altmap __altmap = {
465 466
		.base_pfn = init_altmap_base(base),
		.reserve = init_altmap_reserve(base),
467
	};
468

469 470 471
	pmem = dev_get_drvdata(dev);
	pmem->data_offset = le64_to_cpu(pfn_sb->dataoff);
	pmem->pfn_pad = start_pad + end_trunc;
472 473
	nd_pfn->mode = le32_to_cpu(nd_pfn->pfn_sb->mode);
	if (nd_pfn->mode == PFN_MODE_RAM) {
474
		if (pmem->data_offset < SZ_8K)
475 476 477
			return -EINVAL;
		nd_pfn->npfns = le64_to_cpu(pfn_sb->npfns);
		altmap = NULL;
478
	} else if (nd_pfn->mode == PFN_MODE_PMEM) {
479
		nd_pfn->npfns = (pmem->size - pmem->pfn_pad - pmem->data_offset)
480 481 482 483 484 485 486
			/ PAGE_SIZE;
		if (le64_to_cpu(nd_pfn->pfn_sb->npfns) > nd_pfn->npfns)
			dev_info(&nd_pfn->dev,
					"number of pfns truncated from %lld to %ld\n",
					le64_to_cpu(nd_pfn->pfn_sb->npfns),
					nd_pfn->npfns);
		altmap = & __altmap;
487
		altmap->free = PHYS_PFN(pmem->data_offset - SZ_8K);
488
		altmap->alloc = 0;
489 490 491 492 493 494
	} else {
		rc = -ENXIO;
		goto err;
	}

	/* establish pfn range for lookup, and switch to direct map */
495
	q = pmem->pmem_queue;
496 497 498
	memcpy(&res, &nsio->res, sizeof(res));
	res.start += start_pad;
	res.end -= end_trunc;
D
Dan Williams 已提交
499
	devm_memunmap(dev, (void __force *) pmem->virt_addr);
500
	pmem->virt_addr = (void __pmem *) devm_memremap_pages(dev, &res,
501
			&q->q_usage_counter, altmap);
D
Dan Williams 已提交
502
	pmem->pfn_flags |= PFN_MAP;
503 504 505 506 507 508 509 510 511 512 513 514 515 516
	if (IS_ERR(pmem->virt_addr)) {
		rc = PTR_ERR(pmem->virt_addr);
		goto err;
	}

	/* attach pmem disk in "pfn-mode" */
	rc = pmem_attach_disk(dev, ndns, pmem);
	if (rc)
		goto err;

	return rc;
 err:
	nvdimm_namespace_detach_pfn(ndns);
	return rc;
517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532

}

static int nvdimm_namespace_attach_pfn(struct nd_namespace_common *ndns)
{
	struct nd_pfn *nd_pfn = to_nd_pfn(ndns->claim);
	int rc;

	if (!nd_pfn->uuid || !nd_pfn->ndns)
		return -ENODEV;

	rc = nd_pfn_init(nd_pfn);
	if (rc)
		return rc;
	/* we need a valid pfn_sb before we can init a vmem_altmap */
	return __nvdimm_namespace_attach_pfn(nd_pfn);
533 534
}

535
static int nd_pmem_probe(struct device *dev)
536
{
537
	struct nd_region *nd_region = to_nd_region(dev->parent);
538 539
	struct nd_namespace_common *ndns;
	struct nd_namespace_io *nsio;
540 541
	struct pmem_device *pmem;

542 543 544
	ndns = nvdimm_namespace_common_probe(dev);
	if (IS_ERR(ndns))
		return PTR_ERR(ndns);
545

546
	nsio = to_nd_namespace_io(&ndns->dev);
547
	pmem = pmem_alloc(dev, &nsio->res, nd_region->id);
548 549 550
	if (IS_ERR(pmem))
		return PTR_ERR(pmem);

551
	pmem->ndns = ndns;
552
	dev_set_drvdata(dev, pmem);
553
	ndns->rw_bytes = pmem_rw_bytes;
554 555 556
	if (devm_init_badblocks(dev, &pmem->bb))
		return -ENOMEM;
	nvdimm_namespace_add_poison(ndns, &pmem->bb, 0);
557

558 559 560 561
	if (is_nd_btt(dev)) {
		/* btt allocates its own request_queue */
		blk_cleanup_queue(pmem->pmem_queue);
		pmem->pmem_queue = NULL;
562
		return nvdimm_namespace_attach_btt(ndns);
563
	}
564

565 566 567
	if (is_nd_pfn(dev))
		return nvdimm_namespace_attach_pfn(ndns);

568 569 570 571 572 573
	if (nd_btt_probe(ndns, pmem) == 0 || nd_pfn_probe(ndns, pmem) == 0) {
		/*
		 * We'll come back as either btt-pmem, or pfn-pmem, so
		 * drop the queue allocation for now.
		 */
		blk_cleanup_queue(pmem->pmem_queue);
574 575 576 577
		return -ENXIO;
	}

	return pmem_attach_disk(dev, ndns, pmem);
578 579
}

580
static int nd_pmem_remove(struct device *dev)
581
{
582
	struct pmem_device *pmem = dev_get_drvdata(dev);
583

584
	if (is_nd_btt(dev))
585 586 587
		nvdimm_namespace_detach_btt(pmem->ndns);
	else if (is_nd_pfn(dev))
		nvdimm_namespace_detach_pfn(pmem->ndns);
588 589 590
	else
		pmem_detach_disk(pmem);

591 592 593
	return 0;
}

594 595 596 597 598 599 600 601 602 603 604 605 606 607
static void nd_pmem_notify(struct device *dev, enum nvdimm_event event)
{
	struct pmem_device *pmem = dev_get_drvdata(dev);
	struct nd_namespace_common *ndns = pmem->ndns;

	if (event != NVDIMM_REVALIDATE_POISON)
		return;

	if (is_nd_btt(dev))
		nvdimm_namespace_add_poison(ndns, &pmem->bb, 0);
	else
		nvdimm_namespace_add_poison(ndns, &pmem->bb, pmem->data_offset);
}

608 609
MODULE_ALIAS("pmem");
MODULE_ALIAS_ND_DEVICE(ND_DEVICE_NAMESPACE_IO);
610
MODULE_ALIAS_ND_DEVICE(ND_DEVICE_NAMESPACE_PMEM);
611 612 613
static struct nd_device_driver nd_pmem_driver = {
	.probe = nd_pmem_probe,
	.remove = nd_pmem_remove,
614
	.notify = nd_pmem_notify,
615 616
	.drv = {
		.name = "nd_pmem",
617
	},
618
	.type = ND_DRIVER_NAMESPACE_IO | ND_DRIVER_NAMESPACE_PMEM,
619 620 621 622
};

static int __init pmem_init(void)
{
623
	return nd_driver_register(&nd_pmem_driver);
624 625 626 627 628
}
module_init(pmem_init);

static void pmem_exit(void)
{
629
	driver_unregister(&nd_pmem_driver.drv);
630 631 632 633 634
}
module_exit(pmem_exit);

MODULE_AUTHOR("Ross Zwisler <ross.zwisler@linux.intel.com>");
MODULE_LICENSE("GPL v2");