pat.c 13.9 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13
/*
 * Handle caching attributes in page tables (PAT)
 *
 * Authors: Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>
 *          Suresh B Siddha <suresh.b.siddha@intel.com>
 *
 * Loosely based on earlier PAT patchset from Eric Biederman and Andi Kleen.
 */

#include <linux/mm.h>
#include <linux/kernel.h>
#include <linux/gfp.h>
#include <linux/fs.h>
14
#include <linux/bootmem.h>
15 16 17 18

#include <asm/msr.h>
#include <asm/tlbflush.h>
#include <asm/processor.h>
19
#include <asm/page.h>
20 21 22 23 24 25
#include <asm/pgtable.h>
#include <asm/pat.h>
#include <asm/e820.h>
#include <asm/cacheflush.h>
#include <asm/fcntl.h>
#include <asm/mtrr.h>
26
#include <asm/io.h>
27

28
#ifdef CONFIG_X86_PAT
29
int __read_mostly pat_enabled = 1;
30

31
void __cpuinit pat_disable(char *reason)
32
{
33
	pat_enabled = 0;
34
	printk(KERN_INFO "%s\n", reason);
35 36
}

A
Andrew Morton 已提交
37
static int __init nopat(char *str)
38
{
39
	pat_disable("PAT support disabled.");
40 41
	return 0;
}
42 43 44
early_param("nopat", nopat);
#endif

45 46 47 48 49 50 51 52 53 54 55 56 57

static int debug_enable;
static int __init pat_debug_setup(char *str)
{
	debug_enable = 1;
	return 0;
}
__setup("debugpat", pat_debug_setup);

#define dprintk(fmt, arg...) \
	do { if (debug_enable) printk(KERN_INFO fmt, ##arg); } while (0)


58
static u64 __read_mostly boot_pat_state;
59 60 61 62 63 64 65 66 67 68

enum {
	PAT_UC = 0,		/* uncached */
	PAT_WC = 1,		/* Write combining */
	PAT_WT = 4,		/* Write Through */
	PAT_WP = 5,		/* Write Protected */
	PAT_WB = 6,		/* Write Back (default) */
	PAT_UC_MINUS = 7,	/* UC, but can be overriden by MTRR */
};

69
#define PAT(x, y)	((u64)PAT_ ## y << ((x)*8))
70 71 72 73 74

void pat_init(void)
{
	u64 pat;

75
	if (!pat_enabled)
76 77
		return;

78
	/* Paranoia check. */
79
	if (!cpu_has_pat && boot_pat_state) {
80
		/*
81
		 * If this happens we are on a secondary CPU, but
82 83
		 * switched to PAT on the boot CPU. We have no way to
		 * undo PAT.
84 85 86 87
		 */
		printk(KERN_ERR "PAT enabled, "
		       "but not supported by secondary CPU\n");
		BUG();
88
	}
89 90 91 92 93 94 95 96 97 98 99 100 101 102

	/* Set PWT to Write-Combining. All other bits stay the same */
	/*
	 * PTE encoding used in Linux:
	 *      PAT
	 *      |PCD
	 *      ||PWT
	 *      |||
	 *      000 WB		_PAGE_CACHE_WB
	 *      001 WC		_PAGE_CACHE_WC
	 *      010 UC-		_PAGE_CACHE_UC_MINUS
	 *      011 UC		_PAGE_CACHE_UC
	 * PAT bit unused
	 */
103 104
	pat = PAT(0, WB) | PAT(1, WC) | PAT(2, UC_MINUS) | PAT(3, UC) |
	      PAT(4, WB) | PAT(5, WC) | PAT(6, UC_MINUS) | PAT(7, UC);
105 106

	/* Boot CPU check */
107
	if (!boot_pat_state)
108 109 110 111 112 113 114 115 116 117 118 119
		rdmsrl(MSR_IA32_CR_PAT, boot_pat_state);

	wrmsrl(MSR_IA32_CR_PAT, pat);
	printk(KERN_INFO "x86 PAT enabled: cpu %d, old 0x%Lx, new 0x%Lx\n",
	       smp_processor_id(), boot_pat_state, pat);
}

#undef PAT

static char *cattr_name(unsigned long flags)
{
	switch (flags & _PAGE_CACHE_MASK) {
120 121 122 123 124
	case _PAGE_CACHE_UC:		return "uncached";
	case _PAGE_CACHE_UC_MINUS:	return "uncached-minus";
	case _PAGE_CACHE_WB:		return "write-back";
	case _PAGE_CACHE_WC:		return "write-combining";
	default:			return "broken";
125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161
	}
}

/*
 * The global memtype list keeps track of memory type for specific
 * physical memory areas. Conflicting memory types in different
 * mappings can cause CPU cache corruption. To avoid this we keep track.
 *
 * The list is sorted based on starting address and can contain multiple
 * entries for each address (this allows reference counting for overlapping
 * areas). All the aliases have the same cache attributes of course.
 * Zero attributes are represented as holes.
 *
 * Currently the data structure is a list because the number of mappings
 * are expected to be relatively small. If this should be a problem
 * it could be changed to a rbtree or similar.
 *
 * memtype_lock protects the whole list.
 */

struct memtype {
	u64 start;
	u64 end;
	unsigned long type;
	struct list_head nd;
};

static LIST_HEAD(memtype_list);
static DEFINE_SPINLOCK(memtype_lock); 	/* protects memtype list */

/*
 * Does intersection of PAT memory type and MTRR memory type and returns
 * the resulting memory type as PAT understands it.
 * (Type in pat and mtrr will not have same value)
 * The intersection is based on "Effective Memory Type" tables in IA-32
 * SDM vol 3a
 */
162
static unsigned long pat_x_mtrr_type(u64 start, u64 end, unsigned long req_type)
163
{
164 165 166 167
	/*
	 * Look for MTRR hint to get the effective type in case where PAT
	 * request is for WB.
	 */
168 169 170 171 172 173 174 175 176 177 178
	if (req_type == _PAGE_CACHE_WB) {
		u8 mtrr_type;

		mtrr_type = mtrr_type_lookup(start, end);
		if (mtrr_type == MTRR_TYPE_UNCACHABLE)
			return _PAGE_CACHE_UC;
		if (mtrr_type == MTRR_TYPE_WRCOMB)
			return _PAGE_CACHE_WC;
	}

	return req_type;
179 180
}

181 182 183 184 185 186 187 188 189 190
/*
 * req_type typically has one of the:
 * - _PAGE_CACHE_WB
 * - _PAGE_CACHE_WC
 * - _PAGE_CACHE_UC_MINUS
 * - _PAGE_CACHE_UC
 *
 * req_type will have a special case value '-1', when requester want to inherit
 * the memory type from mtrr (if WB), existing PAT, defaulting to UC_MINUS.
 *
191 192 193
 * If new_type is NULL, function will return an error if it cannot reserve the
 * region with req_type. If new_type is non-NULL, function will return
 * available type in new_type in case of no error. In case of any error
194 195
 * it will return a negative return value.
 */
196
int reserve_memtype(u64 start, u64 end, unsigned long req_type,
197
			unsigned long *new_type)
198
{
199
	struct memtype *new, *entry;
200 201 202
	unsigned long actual_type;
	int err = 0;

203 204
	/* Only track when pat_enabled */
	if (!pat_enabled) {
205
		/* This is identical to page table setting without PAT */
206 207 208 209 210
		if (new_type) {
			if (req_type == -1)
				*new_type = _PAGE_CACHE_WB;
			else
				*new_type = req_type & _PAGE_CACHE_MASK;
211
		}
212 213 214 215
		return 0;
	}

	/* Low ISA region is always mapped WB in page table. No need to track */
216
	if (is_ISA_range(start, end - 1)) {
217 218
		if (new_type)
			*new_type = _PAGE_CACHE_WB;
219 220 221
		return 0;
	}

222 223
	if (req_type == -1) {
		/*
224 225 226 227
		 * Call mtrr_lookup to get the type hint. This is an
		 * optimization for /dev/mem mmap'ers into WB memory (BIOS
		 * tools and ACPI tools). Use WB request for WB memory and use
		 * UC_MINUS otherwise.
228 229 230 231 232 233 234 235 236 237 238 239
		 */
		u8 mtrr_type = mtrr_type_lookup(start, end);

		if (mtrr_type == MTRR_TYPE_WRBACK) {
			req_type = _PAGE_CACHE_WB;
			actual_type = _PAGE_CACHE_WB;
		} else {
			req_type = _PAGE_CACHE_UC_MINUS;
			actual_type = _PAGE_CACHE_UC_MINUS;
		}
	} else {
		req_type &= _PAGE_CACHE_MASK;
240
		actual_type = pat_x_mtrr_type(start, end, req_type);
241 242
	}

243 244
	new  = kmalloc(sizeof(struct memtype), GFP_KERNEL);
	if (!new)
245 246
		return -ENOMEM;

247 248 249
	new->start = start;
	new->end = end;
	new->type = actual_type;
250

251 252
	if (new_type)
		*new_type = actual_type;
253 254 255 256

	spin_lock(&memtype_lock);

	/* Search for existing mapping that overlaps the current range */
257
	list_for_each_entry(entry, &memtype_list, nd) {
258 259
		struct memtype *saved_ptr;

260
		if (entry->start >= end) {
261
			dprintk("New Entry\n");
262 263
			list_add(&new->nd, entry->nd.prev);
			new = NULL;
264 265 266
			break;
		}

267 268 269 270 271
		if (start <= entry->start && end >= entry->start) {
			if (actual_type != entry->type && new_type) {
				actual_type = entry->type;
				*new_type = actual_type;
				new->type = actual_type;
272 273
			}

274
			if (actual_type != entry->type) {
275 276 277 278 279
				printk(
		KERN_INFO "%s:%d conflicting memory types %Lx-%Lx %s<->%s\n",
					current->comm, current->pid,
					start, end,
					cattr_name(actual_type),
280
					cattr_name(entry->type));
281 282 283 284
				err = -EBUSY;
				break;
			}

285
			saved_ptr = entry;
286 287 288 289
			/*
			 * Check to see whether the request overlaps more
			 * than one entry in the list
			 */
290 291
			list_for_each_entry_continue(entry, &memtype_list, nd) {
				if (end <= entry->start) {
292 293 294
					break;
				}

295
				if (actual_type != entry->type) {
296 297 298 299 300
					printk(
		KERN_INFO "%s:%d conflicting memory types %Lx-%Lx %s<->%s\n",
						current->comm, current->pid,
						start, end,
						cattr_name(actual_type),
301
						cattr_name(entry->type));
302 303 304 305 306 307 308 309 310
					err = -EBUSY;
					break;
				}
			}

			if (err) {
				break;
			}

311
			dprintk("Overlap at 0x%Lx-0x%Lx\n",
312
			       saved_ptr->start, saved_ptr->end);
313
			/* No conflict. Go ahead and add this new entry */
314 315
			list_add(&new->nd, saved_ptr->nd.prev);
			new = NULL;
316 317 318
			break;
		}

319 320 321 322 323
		if (start < entry->end) {
			if (actual_type != entry->type && new_type) {
				actual_type = entry->type;
				*new_type = actual_type;
				new->type = actual_type;
324 325
			}

326
			if (actual_type != entry->type) {
327 328 329 330 331
				printk(
		KERN_INFO "%s:%d conflicting memory types %Lx-%Lx %s<->%s\n",
					current->comm, current->pid,
					start, end,
					cattr_name(actual_type),
332
					cattr_name(entry->type));
333 334 335 336
				err = -EBUSY;
				break;
			}

337
			saved_ptr = entry;
338 339 340 341
			/*
			 * Check to see whether the request overlaps more
			 * than one entry in the list
			 */
342 343
			list_for_each_entry_continue(entry, &memtype_list, nd) {
				if (end <= entry->start) {
344 345 346
					break;
				}

347
				if (actual_type != entry->type) {
348 349 350 351 352
					printk(
		KERN_INFO "%s:%d conflicting memory types %Lx-%Lx %s<->%s\n",
						current->comm, current->pid,
						start, end,
						cattr_name(actual_type),
353
						cattr_name(entry->type));
354 355 356 357 358 359 360 361 362
					err = -EBUSY;
					break;
				}
			}

			if (err) {
				break;
			}

363
			dprintk("Overlap at 0x%Lx-0x%Lx\n",
364
				 saved_ptr->start, saved_ptr->end);
365
			/* No conflict. Go ahead and add this new entry */
366 367
			list_add(&new->nd, &saved_ptr->nd);
			new = NULL;
368 369 370 371 372
			break;
		}
	}

	if (err) {
I
Ingo Molnar 已提交
373
		printk(KERN_INFO
374
	"reserve_memtype failed 0x%Lx-0x%Lx, track %s, req %s\n",
375
			start, end, cattr_name(new->type),
376
			cattr_name(req_type));
377
		kfree(new);
378 379 380 381
		spin_unlock(&memtype_lock);
		return err;
	}

382
	if (new) {
383
		/* No conflict. Not yet added to the list. Add to the tail */
384
		list_add_tail(&new->nd, &memtype_list);
385
		dprintk("New Entry\n");
I
Ingo Molnar 已提交
386
	}
387

388
	if (new_type) {
389
		dprintk(
390 391
	"reserve_memtype added 0x%Lx-0x%Lx, track %s, req %s, ret %s\n",
			start, end, cattr_name(actual_type),
392
			cattr_name(req_type), cattr_name(*new_type));
393
	} else {
394
		dprintk(
395 396 397
	"reserve_memtype added 0x%Lx-0x%Lx, track %s, req %s\n",
			start, end, cattr_name(actual_type),
			cattr_name(req_type));
398 399 400 401 402 403 404 405
	}

	spin_unlock(&memtype_lock);
	return err;
}

int free_memtype(u64 start, u64 end)
{
406
	struct memtype *entry;
407 408
	int err = -EINVAL;

409 410
	/* Only track when pat_enabled */
	if (!pat_enabled) {
411 412 413 414
		return 0;
	}

	/* Low ISA region is always mapped WB. No need to track */
415
	if (is_ISA_range(start, end - 1))
416 417 418
		return 0;

	spin_lock(&memtype_lock);
419 420 421 422
	list_for_each_entry(entry, &memtype_list, nd) {
		if (entry->start == start && entry->end == end) {
			list_del(&entry->nd);
			kfree(entry);
423 424 425 426 427 428 429
			err = 0;
			break;
		}
	}
	spin_unlock(&memtype_lock);

	if (err) {
I
Ingo Molnar 已提交
430
		printk(KERN_INFO "%s:%d freeing invalid memtype %Lx-%Lx\n",
431 432
			current->comm, current->pid, start, end);
	}
433

434
	dprintk("free_memtype request 0x%Lx-0x%Lx\n", start, end);
435 436 437
	return err;
}

438

439 440 441 442 443 444 445 446
/*
 * /dev/mem mmap interface. The memtype used for mapping varies:
 * - Use UC for mappings with O_SYNC flag
 * - Without O_SYNC flag, if there is any conflict in reserve_memtype,
 *   inherit the memtype from existing mapping.
 * - Else use UC_MINUS memtype (for backward compatibility with existing
 *   X drivers.
 */
447 448 449 450 451 452
pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
				unsigned long size, pgprot_t vma_prot)
{
	return vma_prot;
}

453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479
#ifdef CONFIG_NONPROMISC_DEVMEM
/* This check is done in drivers/char/mem.c in case of NONPROMISC_DEVMEM*/
static inline int range_is_allowed(unsigned long pfn, unsigned long size)
{
	return 1;
}
#else
static inline int range_is_allowed(unsigned long pfn, unsigned long size)
{
	u64 from = ((u64)pfn) << PAGE_SHIFT;
	u64 to = from + size;
	u64 cursor = from;

	while (cursor < to) {
		if (!devmem_is_allowed(pfn)) {
			printk(KERN_INFO
		"Program %s tried to access /dev/mem between %Lx->%Lx.\n",
				current->comm, from, to);
			return 0;
		}
		cursor += PAGE_SIZE;
		pfn++;
	}
	return 1;
}
#endif /* CONFIG_NONPROMISC_DEVMEM */

480 481 482
int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn,
				unsigned long size, pgprot_t *vma_prot)
{
483 484 485
	u64 offset = ((u64) pfn) << PAGE_SHIFT;
	unsigned long flags = _PAGE_CACHE_UC_MINUS;
	int retval;
486

487 488 489
	if (!range_is_allowed(pfn, size))
		return 0;

490
	if (file->f_flags & O_SYNC) {
491
		flags = _PAGE_CACHE_UC;
492 493 494 495 496 497 498 499 500 501 502
	}

#ifdef CONFIG_X86_32
	/*
	 * On the PPro and successors, the MTRRs are used to set
	 * memory types for physical addresses outside main memory,
	 * so blindly setting UC or PWT on those pages is wrong.
	 * For Pentiums and earlier, the surround logic should disable
	 * caching for the high addresses through the KEN pin, but
	 * we maintain the tradition of paranoia in this code.
	 */
503
	if (!pat_enabled &&
504 505 506 507 508
	    !(boot_cpu_has(X86_FEATURE_MTRR) ||
	      boot_cpu_has(X86_FEATURE_K6_MTRR) ||
	      boot_cpu_has(X86_FEATURE_CYRIX_ARR) ||
	      boot_cpu_has(X86_FEATURE_CENTAUR_MCR)) &&
	    (pfn << PAGE_SHIFT) >= __pa(high_memory)) {
509
		flags = _PAGE_CACHE_UC;
510 511 512
	}
#endif

513 514 515 516 517 518 519 520 521 522
	/*
	 * With O_SYNC, we can only take UC mapping. Fail if we cannot.
	 * Without O_SYNC, we want to get
	 * - WB for WB-able memory and no other conflicting mappings
	 * - UC_MINUS for non-WB-able memory with no other conflicting mappings
	 * - Inherit from confliting mappings otherwise
	 */
	if (flags != _PAGE_CACHE_UC_MINUS) {
		retval = reserve_memtype(offset, offset + size, flags, NULL);
	} else {
I
Ingo Molnar 已提交
523
		retval = reserve_memtype(offset, offset + size, -1, &flags);
524 525 526 527 528 529
	}

	if (retval < 0)
		return 0;

	if (pfn <= max_pfn_mapped &&
530
	    ioremap_change_attr((unsigned long)__va(offset), size, flags) < 0) {
531
		free_memtype(offset, offset + size);
I
Ingo Molnar 已提交
532
		printk(KERN_INFO
533 534 535
		"%s:%d /dev/mem ioremap_change_attr failed %s for %Lx-%Lx\n",
			current->comm, current->pid,
			cattr_name(flags),
536
			offset, (unsigned long long)(offset + size));
537 538 539 540 541
		return 0;
	}

	*vma_prot = __pgprot((pgprot_val(*vma_prot) & ~_PAGE_CACHE_MASK) |
			     flags);
542 543
	return 1;
}
544 545 546 547 548 549 550 551 552

void map_devmem(unsigned long pfn, unsigned long size, pgprot_t vma_prot)
{
	u64 addr = (u64)pfn << PAGE_SHIFT;
	unsigned long flags;
	unsigned long want_flags = (pgprot_val(vma_prot) & _PAGE_CACHE_MASK);

	reserve_memtype(addr, addr + size, want_flags, &flags);
	if (flags != want_flags) {
I
Ingo Molnar 已提交
553
		printk(KERN_INFO
554 555 556
		"%s:%d /dev/mem expected mapping type %s for %Lx-%Lx, got %s\n",
			current->comm, current->pid,
			cattr_name(want_flags),
557
			addr, (unsigned long long)(addr + size),
558 559 560 561 562 563 564 565 566 567
			cattr_name(flags));
	}
}

void unmap_devmem(unsigned long pfn, unsigned long size, pgprot_t vma_prot)
{
	u64 addr = (u64)pfn << PAGE_SHIFT;

	free_memtype(addr, addr + size);
}