generic.c 19.0 KB
Newer Older
1 2 3 4 5 6 7
/*
 * This only handles 32bit MTRR on 32bit hosts. This is strictly wrong
 * because MTRRs can span upto 40 bits (36bits on most modern x86)
 */
#define DEBUG

#include <linux/module.h>
L
Linus Torvalds 已提交
8
#include <linux/init.h>
9
#include <linux/io.h>
L
Linus Torvalds 已提交
10
#include <linux/mm.h>
11

D
Dave Jones 已提交
12
#include <asm/processor-flags.h>
13
#include <asm/cpufeature.h>
L
Linus Torvalds 已提交
14
#include <asm/tlbflush.h>
15 16 17
#include <asm/system.h>
#include <asm/mtrr.h>
#include <asm/msr.h>
18
#include <asm/pat.h>
19

L
Linus Torvalds 已提交
20 21
#include "mtrr.h"

22
struct fixed_range_block {
23 24
	int base_msr;		/* start address of an MTRR block */
	int ranges;		/* number of MTRRs in this block  */
25 26 27
};

static struct fixed_range_block fixed_range_blocks[] = {
28 29 30
	{ MSR_MTRRfix64K_00000, 1 }, /* one   64k MTRR  */
	{ MSR_MTRRfix16K_80000, 2 }, /* two   16k MTRRs */
	{ MSR_MTRRfix4K_C0000,  8 }, /* eight  4k MTRRs */
31 32 33
	{}
};

L
Linus Torvalds 已提交
34
static unsigned long smp_changes_mask;
35
static int mtrr_state_set;
36
u64 mtrr_tom2;
L
Linus Torvalds 已提交
37

38
struct mtrr_state_type mtrr_state;
S
Sheng Yang 已提交
39 40
EXPORT_SYMBOL_GPL(mtrr_state);

41
/*
42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66
 * BIOS is expected to clear MtrrFixDramModEn bit, see for example
 * "BIOS and Kernel Developer's Guide for the AMD Athlon 64 and AMD
 * Opteron Processors" (26094 Rev. 3.30 February 2006), section
 * "13.2.1.2 SYSCFG Register": "The MtrrFixDramModEn bit should be set
 * to 1 during BIOS initalization of the fixed MTRRs, then cleared to
 * 0 for operation."
 */
static inline void k8_check_syscfg_dram_mod_en(void)
{
	u32 lo, hi;

	if (!((boot_cpu_data.x86_vendor == X86_VENDOR_AMD) &&
	      (boot_cpu_data.x86 >= 0x0f)))
		return;

	rdmsr(MSR_K8_SYSCFG, lo, hi);
	if (lo & K8_MTRRFIXRANGE_DRAM_MODIFY) {
		printk(KERN_ERR FW_WARN "MTRR: CPU %u: SYSCFG[MtrrFixDramModEn]"
		       " not cleared by BIOS, clearing this bit\n",
		       smp_processor_id());
		lo &= ~K8_MTRRFIXRANGE_DRAM_MODIFY;
		mtrr_wrmsr(MSR_K8_SYSCFG, lo, hi);
	}
}

67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111
/*
 * Returns the effective MTRR type for the region
 * Error returns:
 * - 0xFE - when the range is "not entirely covered" by _any_ var range MTRR
 * - 0xFF - when MTRR is not enabled
 */
u8 mtrr_type_lookup(u64 start, u64 end)
{
	int i;
	u64 base, mask;
	u8 prev_match, curr_match;

	if (!mtrr_state_set)
		return 0xFF;

	if (!mtrr_state.enabled)
		return 0xFF;

	/* Make end inclusive end, instead of exclusive */
	end--;

	/* Look in fixed ranges. Just return the type as per start */
	if (mtrr_state.have_fixed && (start < 0x100000)) {
		int idx;

		if (start < 0x80000) {
			idx = 0;
			idx += (start >> 16);
			return mtrr_state.fixed_ranges[idx];
		} else if (start < 0xC0000) {
			idx = 1 * 8;
			idx += ((start - 0x80000) >> 14);
			return mtrr_state.fixed_ranges[idx];
		} else if (start < 0x1000000) {
			idx = 3 * 8;
			idx += ((start - 0xC0000) >> 12);
			return mtrr_state.fixed_ranges[idx];
		}
	}

	/*
	 * Look in variable ranges
	 * Look of multiple ranges matching this address and pick type
	 * as per MTRR precedence
	 */
112
	if (!(mtrr_state.enabled & 2))
113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131
		return mtrr_state.def_type;

	prev_match = 0xFF;
	for (i = 0; i < num_var_ranges; ++i) {
		unsigned short start_state, end_state;

		if (!(mtrr_state.var_ranges[i].mask_lo & (1 << 11)))
			continue;

		base = (((u64)mtrr_state.var_ranges[i].base_hi) << 32) +
		       (mtrr_state.var_ranges[i].base_lo & PAGE_MASK);
		mask = (((u64)mtrr_state.var_ranges[i].mask_hi) << 32) +
		       (mtrr_state.var_ranges[i].mask_lo & PAGE_MASK);

		start_state = ((start & mask) == (base & mask));
		end_state = ((end & mask) == (base & mask));
		if (start_state != end_state)
			return 0xFE;

132
		if ((start & mask) != (base & mask))
133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153
			continue;

		curr_match = mtrr_state.var_ranges[i].base_lo & 0xff;
		if (prev_match == 0xFF) {
			prev_match = curr_match;
			continue;
		}

		if (prev_match == MTRR_TYPE_UNCACHABLE ||
		    curr_match == MTRR_TYPE_UNCACHABLE) {
			return MTRR_TYPE_UNCACHABLE;
		}

		if ((prev_match == MTRR_TYPE_WRBACK &&
		     curr_match == MTRR_TYPE_WRTHROUGH) ||
		    (prev_match == MTRR_TYPE_WRTHROUGH &&
		     curr_match == MTRR_TYPE_WRBACK)) {
			prev_match = MTRR_TYPE_WRTHROUGH;
			curr_match = MTRR_TYPE_WRTHROUGH;
		}

154
		if (prev_match != curr_match)
155 156 157
			return MTRR_TYPE_UNCACHABLE;
	}

158 159
	if (mtrr_tom2) {
		if (start >= (1ULL<<32) && (end < mtrr_tom2))
160 161 162
			return MTRR_TYPE_WRBACK;
	}

163 164 165 166 167 168
	if (prev_match != 0xFF)
		return prev_match;

	return mtrr_state.def_type;
}

169
/* Get the MSR pair relating to a var range */
170
static void
L
Linus Torvalds 已提交
171 172 173 174 175 176
get_mtrr_var_range(unsigned int index, struct mtrr_var_range *vr)
{
	rdmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi);
	rdmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi);
}

177
/* Fill the MSR pair relating to a var range */
178 179 180 181 182 183 184 185 186 187 188 189 190
void fill_mtrr_var_range(unsigned int index,
		u32 base_lo, u32 base_hi, u32 mask_lo, u32 mask_hi)
{
	struct mtrr_var_range *vr;

	vr = mtrr_state.var_ranges;

	vr[index].base_lo = base_lo;
	vr[index].base_hi = base_hi;
	vr[index].mask_lo = mask_lo;
	vr[index].mask_hi = mask_hi;
}

191
static void get_fixed_ranges(mtrr_type *frs)
L
Linus Torvalds 已提交
192
{
193
	unsigned int *p = (unsigned int *)frs;
L
Linus Torvalds 已提交
194 195
	int i;

196 197
	k8_check_syscfg_dram_mod_en();

198
	rdmsr(MSR_MTRRfix64K_00000, p[0], p[1]);
L
Linus Torvalds 已提交
199 200

	for (i = 0; i < 2; i++)
201
		rdmsr(MSR_MTRRfix16K_80000 + i, p[2 + i * 2], p[3 + i * 2]);
L
Linus Torvalds 已提交
202
	for (i = 0; i < 8; i++)
203
		rdmsr(MSR_MTRRfix4K_C0000 + i, p[6 + i * 2], p[7 + i * 2]);
L
Linus Torvalds 已提交
204 205
}

206 207
void mtrr_save_fixed_ranges(void *info)
{
A
Andrew Morton 已提交
208 209
	if (cpu_has_mtrr)
		get_fixed_ranges(mtrr_state.fixed_ranges);
210 211
}

212 213 214 215 216 217 218 219 220
static unsigned __initdata last_fixed_start;
static unsigned __initdata last_fixed_end;
static mtrr_type __initdata last_fixed_type;

static void __init print_fixed_last(void)
{
	if (!last_fixed_end)
		return;

221 222
	pr_debug("  %05X-%05X %s\n", last_fixed_start,
		 last_fixed_end - 1, mtrr_attrib_to_str(last_fixed_type));
223 224 225 226 227

	last_fixed_end = 0;
}

static void __init update_fixed_last(unsigned base, unsigned end,
228
				     mtrr_type type)
229 230 231 232 233 234
{
	last_fixed_start = base;
	last_fixed_end = end;
	last_fixed_type = type;
}

235 236
static void __init
print_fixed(unsigned base, unsigned step, const mtrr_type *types)
J
Jan Beulich 已提交
237 238 239
{
	unsigned i;

240 241 242 243 244 245 246 247 248 249 250 251 252
	for (i = 0; i < 8; ++i, ++types, base += step) {
		if (last_fixed_end == 0) {
			update_fixed_last(base, base + step, *types);
			continue;
		}
		if (last_fixed_end == base && last_fixed_type == *types) {
			last_fixed_end = base + step;
			continue;
		}
		/* new segments: gap or different type */
		print_fixed_last();
		update_fixed_last(base, base + step, *types);
	}
J
Jan Beulich 已提交
253 254
}

255 256 257
static void prepare_set(void);
static void post_set(void);

Y
Yinghai Lu 已提交
258 259 260 261 262
static void __init print_mtrr_state(void)
{
	unsigned int i;
	int high_width;

263 264
	pr_debug("MTRR default type: %s\n",
		 mtrr_attrib_to_str(mtrr_state.def_type));
Y
Yinghai Lu 已提交
265
	if (mtrr_state.have_fixed) {
266 267
		pr_debug("MTRR fixed ranges %sabled:\n",
			 mtrr_state.enabled & 1 ? "en" : "dis");
Y
Yinghai Lu 已提交
268 269
		print_fixed(0x00000, 0x10000, mtrr_state.fixed_ranges + 0);
		for (i = 0; i < 2; ++i)
270 271
			print_fixed(0x80000 + i * 0x20000, 0x04000,
				    mtrr_state.fixed_ranges + (i + 1) * 8);
Y
Yinghai Lu 已提交
272
		for (i = 0; i < 8; ++i)
273 274
			print_fixed(0xC0000 + i * 0x08000, 0x01000,
				    mtrr_state.fixed_ranges + (i + 3) * 8);
275 276 277

		/* tail */
		print_fixed_last();
Y
Yinghai Lu 已提交
278
	}
279 280
	pr_debug("MTRR variable ranges %sabled:\n",
		 mtrr_state.enabled & 2 ? "en" : "dis");
281 282 283 284 285
	if (size_or_mask & 0xffffffffUL)
		high_width = ffs(size_or_mask & 0xffffffffUL) - 1;
	else
		high_width = ffs(size_or_mask>>32) + 32 - 1;
	high_width = (high_width - (32 - PAGE_SHIFT) + 3) / 4;
286

Y
Yinghai Lu 已提交
287 288
	for (i = 0; i < num_var_ranges; ++i) {
		if (mtrr_state.var_ranges[i].mask_lo & (1 << 11))
289 290 291 292 293 294 295 296 297
			pr_debug("  %u base %0*X%05X000 mask %0*X%05X000 %s\n",
				 i,
				 high_width,
				 mtrr_state.var_ranges[i].base_hi,
				 mtrr_state.var_ranges[i].base_lo >> 12,
				 high_width,
				 mtrr_state.var_ranges[i].mask_hi,
				 mtrr_state.var_ranges[i].mask_lo >> 12,
				 mtrr_attrib_to_str(mtrr_state.var_ranges[i].base_lo & 0xff));
Y
Yinghai Lu 已提交
298
		else
299
			pr_debug("  %u disabled\n", i);
Y
Yinghai Lu 已提交
300
	}
301 302
	if (mtrr_tom2)
		pr_debug("TOM2: %016llx aka %lldM\n", mtrr_tom2, mtrr_tom2>>20);
Y
Yinghai Lu 已提交
303 304
}

305
/* Grab all of the MTRR state for this CPU into *state */
306
void __init get_mtrr_state(void)
L
Linus Torvalds 已提交
307 308
{
	struct mtrr_var_range *vrs;
309
	unsigned long flags;
310 311
	unsigned lo, dummy;
	unsigned int i;
L
Linus Torvalds 已提交
312 313 314

	vrs = mtrr_state.var_ranges;

315
	rdmsr(MSR_MTRRcap, lo, dummy);
J
Jan Beulich 已提交
316 317
	mtrr_state.have_fixed = (lo >> 8) & 1;

L
Linus Torvalds 已提交
318 319
	for (i = 0; i < num_var_ranges; i++)
		get_mtrr_var_range(i, &vrs[i]);
J
Jan Beulich 已提交
320 321
	if (mtrr_state.have_fixed)
		get_fixed_ranges(mtrr_state.fixed_ranges);
L
Linus Torvalds 已提交
322

323
	rdmsr(MSR_MTRRdefType, lo, dummy);
L
Linus Torvalds 已提交
324 325
	mtrr_state.def_type = (lo & 0xff);
	mtrr_state.enabled = (lo & 0xc00) >> 10;
J
Jan Beulich 已提交
326

327
	if (amd_special_default_mtrr()) {
328
		unsigned low, high;
329

330
		/* TOP_MEM2 */
331
		rdmsr(MSR_K8_TOP_MEM2, low, high);
332 333 334
		mtrr_tom2 = high;
		mtrr_tom2 <<= 32;
		mtrr_tom2 |= low;
Y
Yinghai Lu 已提交
335
		mtrr_tom2 &= 0xffffff800000ULL;
336
	}
Y
Yinghai Lu 已提交
337 338 339

	print_mtrr_state();

340 341 342 343 344 345 346 347 348 349
	mtrr_state_set = 1;

	/* PAT setup for BP. We need to go through sync steps here */
	local_irq_save(flags);
	prepare_set();

	pat_init();

	post_set();
	local_irq_restore(flags);
L
Linus Torvalds 已提交
350 351
}

352
/* Some BIOS's are messed up and don't set all MTRRs the same! */
L
Linus Torvalds 已提交
353 354 355 356 357 358 359
void __init mtrr_state_warn(void)
{
	unsigned long mask = smp_changes_mask;

	if (!mask)
		return;
	if (mask & MTRR_CHANGE_MASK_FIXED)
360
		pr_warning("mtrr: your CPUs had inconsistent fixed MTRR settings\n");
L
Linus Torvalds 已提交
361
	if (mask & MTRR_CHANGE_MASK_VARIABLE)
362
		pr_warning("mtrr: your CPUs had inconsistent variable MTRR settings\n");
L
Linus Torvalds 已提交
363
	if (mask & MTRR_CHANGE_MASK_DEFTYPE)
364 365
		pr_warning("mtrr: your CPUs had inconsistent MTRRdefType settings\n");

L
Linus Torvalds 已提交
366 367 368 369
	printk(KERN_INFO "mtrr: probably your BIOS does not setup all CPUs.\n");
	printk(KERN_INFO "mtrr: corrected configuration.\n");
}

370 371 372 373 374
/*
 * Doesn't attempt to pass an error out to MTRR users
 * because it's quite complicated in some cases and probably not
 * worth it because the best error handling is to ignore it.
 */
L
Linus Torvalds 已提交
375 376
void mtrr_wrmsr(unsigned msr, unsigned a, unsigned b)
{
377
	if (wrmsr_safe(msr, a, b) < 0) {
L
Linus Torvalds 已提交
378 379 380
		printk(KERN_ERR
			"MTRR: CPU %u: Writing MSR %x to %x:%x failed\n",
			smp_processor_id(), msr, a, b);
381
	}
L
Linus Torvalds 已提交
382 383
}

384
/**
385 386
 * set_fixed_range - checks & updates a fixed-range MTRR if it
 *		     differs from the value it should have
387 388 389
 * @msr: MSR address of the MTTR which should be checked and updated
 * @changed: pointer which indicates whether the MTRR needed to be changed
 * @msrwords: pointer to the MSR values which the MSR should have
390
 */
391
static void set_fixed_range(int msr, bool *changed, unsigned int *msrwords)
392 393 394 395 396 397 398
{
	unsigned lo, hi;

	rdmsr(msr, lo, hi);

	if (lo != msrwords[0] || hi != msrwords[1]) {
		mtrr_wrmsr(msr, msrwords[0], msrwords[1]);
399
		*changed = true;
400 401 402
	}
}

403 404 405 406 407 408 409 410
/**
 * generic_get_free_region - Get a free MTRR.
 * @base: The starting (base) address of the region.
 * @size: The size (in bytes) of the region.
 * @replace_reg: mtrr index to be replaced; set to invalid value if none.
 *
 * Returns: The index of the region on success, else negative on error.
 */
411 412
int
generic_get_free_region(unsigned long base, unsigned long size, int replace_reg)
L
Linus Torvalds 已提交
413
{
J
Jan Beulich 已提交
414
	unsigned long lbase, lsize;
415 416
	mtrr_type ltype;
	int i, max;
L
Linus Torvalds 已提交
417 418

	max = num_var_ranges;
J
Jan Beulich 已提交
419 420
	if (replace_reg >= 0 && replace_reg < max)
		return replace_reg;
421

L
Linus Torvalds 已提交
422 423 424 425 426
	for (i = 0; i < max; ++i) {
		mtrr_if->get(i, &lbase, &lsize, &ltype);
		if (lsize == 0)
			return i;
	}
427

L
Linus Torvalds 已提交
428 429 430
	return -ENOSPC;
}

A
Adrian Bunk 已提交
431
static void generic_get_mtrr(unsigned int reg, unsigned long *base,
J
Jan Beulich 已提交
432
			     unsigned long *size, mtrr_type *type)
L
Linus Torvalds 已提交
433 434
{
	unsigned int mask_lo, mask_hi, base_lo, base_hi;
Y
Yinghai Lu 已提交
435
	unsigned int tmp, hi;
436
	int cpu;
L
Linus Torvalds 已提交
437

Y
Yinghai Lu 已提交
438 439 440 441
	/*
	 * get_mtrr doesn't need to update mtrr_state, also it could be called
	 * from any cpu, so try to print it out directly.
	 */
442 443
	cpu = get_cpu();

L
Linus Torvalds 已提交
444
	rdmsr(MTRRphysMask_MSR(reg), mask_lo, mask_hi);
Y
Yinghai Lu 已提交
445

L
Linus Torvalds 已提交
446
	if ((mask_lo & 0x800) == 0) {
447
		/*  Invalid (i.e. free) range */
L
Linus Torvalds 已提交
448 449 450
		*base = 0;
		*size = 0;
		*type = 0;
451
		goto out_put_cpu;
L
Linus Torvalds 已提交
452 453 454 455
	}

	rdmsr(MTRRphysBase_MSR(reg), base_lo, base_hi);

456
	/* Work out the shifted address mask: */
Y
Yinghai Lu 已提交
457 458
	tmp = mask_hi << (32 - PAGE_SHIFT) | mask_lo >> PAGE_SHIFT;
	mask_lo = size_or_mask | tmp;
459 460

	/* Expand tmp with high bits to all 1s: */
Y
Yinghai Lu 已提交
461 462 463 464 465
	hi = fls(tmp);
	if (hi > 0) {
		tmp |= ~((1<<(hi - 1)) - 1);

		if (tmp != mask_lo) {
A
Alan Cox 已提交
466
			printk(KERN_WARNING "mtrr: your BIOS has configured an incorrect mask, fixing it.\n");
Y
Yinghai Lu 已提交
467 468 469
			mask_lo = tmp;
		}
	}
L
Linus Torvalds 已提交
470

471 472 473 474
	/*
	 * This works correctly if size is a power of two, i.e. a
	 * contiguous range:
	 */
L
Linus Torvalds 已提交
475 476 477
	*size = -mask_lo;
	*base = base_hi << (32 - PAGE_SHIFT) | base_lo >> PAGE_SHIFT;
	*type = base_lo & 0xff;
Y
Yinghai Lu 已提交
478

479 480
out_put_cpu:
	put_cpu();
L
Linus Torvalds 已提交
481 482
}

483
/**
484 485
 * set_fixed_ranges - checks & updates the fixed-range MTRRs if they
 *		      differ from the saved set
486
 * @frs: pointer to fixed-range MTRR values, saved by get_fixed_ranges()
487
 */
488
static int set_fixed_ranges(mtrr_type *frs)
L
Linus Torvalds 已提交
489
{
490
	unsigned long long *saved = (unsigned long long *)frs;
491
	bool changed = false;
492
	int block = -1, range;
L
Linus Torvalds 已提交
493

494 495
	k8_check_syscfg_dram_mod_en();

496 497 498 499 500
	while (fixed_range_blocks[++block].ranges) {
		for (range = 0; range < fixed_range_blocks[block].ranges; range++)
			set_fixed_range(fixed_range_blocks[block].base_msr + range,
					&changed, (unsigned int *)saved++);
	}
L
Linus Torvalds 已提交
501 502 503 504

	return changed;
}

505 506 507 508
/*
 * Set the MSR pair relating to a var range.
 * Returns true if changes are made.
 */
509
static bool set_mtrr_var_ranges(unsigned int index, struct mtrr_var_range *vr)
L
Linus Torvalds 已提交
510 511
{
	unsigned int lo, hi;
512
	bool changed = false;
L
Linus Torvalds 已提交
513 514 515

	rdmsr(MTRRphysBase_MSR(index), lo, hi);
	if ((vr->base_lo & 0xfffff0ffUL) != (lo & 0xfffff0ffUL)
516 517
	    || (vr->base_hi & (size_and_mask >> (32 - PAGE_SHIFT))) !=
		(hi & (size_and_mask >> (32 - PAGE_SHIFT)))) {
518

L
Linus Torvalds 已提交
519
		mtrr_wrmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi);
520
		changed = true;
L
Linus Torvalds 已提交
521 522 523 524 525
	}

	rdmsr(MTRRphysMask_MSR(index), lo, hi);

	if ((vr->mask_lo & 0xfffff800UL) != (lo & 0xfffff800UL)
526 527
	    || (vr->mask_hi & (size_and_mask >> (32 - PAGE_SHIFT))) !=
		(hi & (size_and_mask >> (32 - PAGE_SHIFT)))) {
L
Linus Torvalds 已提交
528
		mtrr_wrmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi);
529
		changed = true;
L
Linus Torvalds 已提交
530 531 532 533
	}
	return changed;
}

J
Jan Beulich 已提交
534 535
static u32 deftype_lo, deftype_hi;

536 537 538 539 540 541
/**
 * set_mtrr_state - Set the MTRR state for this CPU.
 *
 * NOTE: The CPU must already be in a safe state for MTRR changes.
 * RETURNS: 0 if no changes made, else a mask indicating what was changed.
 */
J
Jan Beulich 已提交
542
static unsigned long set_mtrr_state(void)
L
Linus Torvalds 已提交
543 544
{
	unsigned long change_mask = 0;
545
	unsigned int i;
L
Linus Torvalds 已提交
546

547
	for (i = 0; i < num_var_ranges; i++) {
L
Linus Torvalds 已提交
548 549
		if (set_mtrr_var_ranges(i, &mtrr_state.var_ranges[i]))
			change_mask |= MTRR_CHANGE_MASK_VARIABLE;
550
	}
L
Linus Torvalds 已提交
551

J
Jan Beulich 已提交
552
	if (mtrr_state.have_fixed && set_fixed_ranges(mtrr_state.fixed_ranges))
L
Linus Torvalds 已提交
553 554
		change_mask |= MTRR_CHANGE_MASK_FIXED;

555 556 557 558
	/*
	 * Set_mtrr_restore restores the old value of MTRRdefType,
	 * so to set it we fiddle with the saved value:
	 */
L
Linus Torvalds 已提交
559 560
	if ((deftype_lo & 0xff) != mtrr_state.def_type
	    || ((deftype_lo & 0xc00) >> 10) != mtrr_state.enabled) {
561 562 563

		deftype_lo = (deftype_lo & ~0xcff) | mtrr_state.def_type |
			     (mtrr_state.enabled << 10);
L
Linus Torvalds 已提交
564 565 566 567 568 569 570
		change_mask |= MTRR_CHANGE_MASK_DEFTYPE;
	}

	return change_mask;
}


571
static unsigned long cr4;
572
static DEFINE_RAW_SPINLOCK(set_atomicity_lock);
L
Linus Torvalds 已提交
573 574

/*
575 576 577 578 579
 * Since we are disabling the cache don't allow any interrupts,
 * they would run extremely slow and would only increase the pain.
 *
 * The caller must ensure that local interrupts are disabled and
 * are reenabled after post_set() has been called.
L
Linus Torvalds 已提交
580
 */
581
static void prepare_set(void) __acquires(set_atomicity_lock)
L
Linus Torvalds 已提交
582 583 584
{
	unsigned long cr0;

585 586 587 588 589 590
	/*
	 * Note that this is not ideal
	 * since the cache is only flushed/disabled for this CPU while the
	 * MTRRs are changed, but changing this requires more invasive
	 * changes to the way the kernel boots
	 */
L
Linus Torvalds 已提交
591

592
	raw_spin_lock(&set_atomicity_lock);
L
Linus Torvalds 已提交
593

594
	/* Enter the no-fill (CD=1, NW=0) cache mode and flush caches. */
D
Dave Jones 已提交
595
	cr0 = read_cr0() | X86_CR0_CD;
L
Linus Torvalds 已提交
596 597 598
	write_cr0(cr0);
	wbinvd();

599 600
	/* Save value of CR4 and clear Page Global Enable (bit 7) */
	if (cpu_has_pge) {
L
Linus Torvalds 已提交
601 602 603 604 605 606 607
		cr4 = read_cr4();
		write_cr4(cr4 & ~X86_CR4_PGE);
	}

	/* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */
	__flush_tlb();

608
	/* Save MTRR state */
609
	rdmsr(MSR_MTRRdefType, deftype_lo, deftype_hi);
L
Linus Torvalds 已提交
610

611
	/* Disable MTRRs, and set the default type to uncached */
612
	mtrr_wrmsr(MSR_MTRRdefType, deftype_lo & ~0xcff, deftype_hi);
L
Linus Torvalds 已提交
613 614
}

615
static void post_set(void) __releases(set_atomicity_lock)
L
Linus Torvalds 已提交
616
{
617
	/* Flush TLBs (no need to flush caches - they are disabled) */
L
Linus Torvalds 已提交
618 619 620
	__flush_tlb();

	/* Intel (P6) standard MTRRs */
621
	mtrr_wrmsr(MSR_MTRRdefType, deftype_lo, deftype_hi);
622 623

	/* Enable caches */
L
Linus Torvalds 已提交
624 625
	write_cr0(read_cr0() & 0xbfffffff);

626 627
	/* Restore value of CR4 */
	if (cpu_has_pge)
L
Linus Torvalds 已提交
628
		write_cr4(cr4);
629
	raw_spin_unlock(&set_atomicity_lock);
L
Linus Torvalds 已提交
630 631 632 633 634 635 636 637 638 639 640
}

static void generic_set_all(void)
{
	unsigned long mask, count;
	unsigned long flags;

	local_irq_save(flags);
	prepare_set();

	/* Actually set the state */
J
Jan Beulich 已提交
641
	mask = set_mtrr_state();
L
Linus Torvalds 已提交
642

643 644 645
	/* also set PAT */
	pat_init();

L
Linus Torvalds 已提交
646 647 648
	post_set();
	local_irq_restore(flags);

649
	/* Use the atomic bitops to update the global mask */
L
Linus Torvalds 已提交
650 651 652 653 654
	for (count = 0; count < sizeof mask * 8; ++count) {
		if (mask & 0x01)
			set_bit(count, &smp_changes_mask);
		mask >>= 1;
	}
655

L
Linus Torvalds 已提交
656 657
}

658 659 660 661 662 663 664 665 666 667
/**
 * generic_set_mtrr - set variable MTRR register on the local CPU.
 *
 * @reg: The register to set.
 * @base: The base address of the region.
 * @size: The size of the region. If this is 0 the region is disabled.
 * @type: The type of the region.
 *
 * Returns nothing.
 */
L
Linus Torvalds 已提交
668 669 670 671
static void generic_set_mtrr(unsigned int reg, unsigned long base,
			     unsigned long size, mtrr_type type)
{
	unsigned long flags;
S
Shaohua Li 已提交
672 673 674
	struct mtrr_var_range *vr;

	vr = &mtrr_state.var_ranges[reg];
L
Linus Torvalds 已提交
675 676 677 678 679

	local_irq_save(flags);
	prepare_set();

	if (size == 0) {
680 681 682 683
		/*
		 * The invalid bit is kept in the mask, so we simply
		 * clear the relevant mask register to disable a range.
		 */
L
Linus Torvalds 已提交
684
		mtrr_wrmsr(MTRRphysMask_MSR(reg), 0, 0);
S
Shaohua Li 已提交
685
		memset(vr, 0, sizeof(struct mtrr_var_range));
L
Linus Torvalds 已提交
686
	} else {
S
Shaohua Li 已提交
687 688 689 690 691 692 693
		vr->base_lo = base << PAGE_SHIFT | type;
		vr->base_hi = (base & size_and_mask) >> (32 - PAGE_SHIFT);
		vr->mask_lo = -size << PAGE_SHIFT | 0x800;
		vr->mask_hi = (-size & size_and_mask) >> (32 - PAGE_SHIFT);

		mtrr_wrmsr(MTRRphysBase_MSR(reg), vr->base_lo, vr->base_hi);
		mtrr_wrmsr(MTRRphysMask_MSR(reg), vr->mask_lo, vr->mask_hi);
L
Linus Torvalds 已提交
694 695 696 697 698 699
	}

	post_set();
	local_irq_restore(flags);
}

700 701
int generic_validate_add_page(unsigned long base, unsigned long size,
			      unsigned int type)
L
Linus Torvalds 已提交
702 703 704
{
	unsigned long lbase, last;

705 706 707 708
	/*
	 * For Intel PPro stepping <= 7
	 * must be 4 MiB aligned and not touch 0x70000000 -> 0x7003FFFF
	 */
L
Linus Torvalds 已提交
709 710 711 712
	if (is_cpu(INTEL) && boot_cpu_data.x86 == 6 &&
	    boot_cpu_data.x86_model == 1 &&
	    boot_cpu_data.x86_mask <= 7) {
		if (base & ((1 << (22 - PAGE_SHIFT)) - 1)) {
713
			pr_warning("mtrr: base(0x%lx000) is not 4 MiB aligned\n", base);
L
Linus Torvalds 已提交
714 715
			return -EINVAL;
		}
716
		if (!(base + size < 0x70000 || base > 0x7003F) &&
L
Linus Torvalds 已提交
717 718
		    (type == MTRR_TYPE_WRCOMB
		     || type == MTRR_TYPE_WRBACK)) {
719
			pr_warning("mtrr: writable mtrr between 0x70000000 and 0x7003FFFF may hang the CPU.\n");
L
Linus Torvalds 已提交
720 721 722 723
			return -EINVAL;
		}
	}

724 725 726 727
	/*
	 * Check upper bits of base and last are equal and lower bits are 0
	 * for base and 1 for last
	 */
L
Linus Torvalds 已提交
728 729
	last = base + size - 1;
	for (lbase = base; !(lbase & 1) && (last & 1);
730 731
	     lbase = lbase >> 1, last = last >> 1)
		;
L
Linus Torvalds 已提交
732
	if (lbase != last) {
733
		pr_warning("mtrr: base(0x%lx000) is not aligned on a size(0x%lx000) boundary\n", base, size);
L
Linus Torvalds 已提交
734 735 736 737 738 739 740 741
		return -EINVAL;
	}
	return 0;
}

static int generic_have_wrcomb(void)
{
	unsigned long config, dummy;
742
	rdmsr(MSR_MTRRcap, config, dummy);
743
	return config & (1 << 10);
L
Linus Torvalds 已提交
744 745 746 747 748 749 750
}

int positive_have_wrcomb(void)
{
	return 1;
}

751 752
/*
 * Generic structure...
L
Linus Torvalds 已提交
753
 */
754
const struct mtrr_ops generic_mtrr_ops = {
755 756 757 758 759 760 761
	.use_intel_if		= 1,
	.set_all		= generic_set_all,
	.get			= generic_get_mtrr,
	.get_free_region	= generic_get_free_region,
	.set			= generic_set_mtrr,
	.validate_add_page	= generic_validate_add_page,
	.have_wrcomb		= generic_have_wrcomb,
L
Linus Torvalds 已提交
762
};