generic.c 19.3 KB
Newer Older
1 2 3 4 5 6 7
/*
 * This only handles 32bit MTRR on 32bit hosts. This is strictly wrong
 * because MTRRs can span upto 40 bits (36bits on most modern x86)
 */
#define DEBUG

#include <linux/module.h>
L
Linus Torvalds 已提交
8
#include <linux/init.h>
9
#include <linux/io.h>
L
Linus Torvalds 已提交
10
#include <linux/mm.h>
11

D
Dave Jones 已提交
12
#include <asm/processor-flags.h>
13
#include <asm/cpufeature.h>
L
Linus Torvalds 已提交
14
#include <asm/tlbflush.h>
15 16 17
#include <asm/system.h>
#include <asm/mtrr.h>
#include <asm/msr.h>
18
#include <asm/pat.h>
19

L
Linus Torvalds 已提交
20 21
#include "mtrr.h"

22
struct fixed_range_block {
23 24
	int base_msr;		/* start address of an MTRR block */
	int ranges;		/* number of MTRRs in this block  */
25 26 27
};

static struct fixed_range_block fixed_range_blocks[] = {
28 29 30
	{ MSR_MTRRfix64K_00000, 1 }, /* one   64k MTRR  */
	{ MSR_MTRRfix16K_80000, 2 }, /* two   16k MTRRs */
	{ MSR_MTRRfix4K_C0000,  8 }, /* eight  4k MTRRs */
31 32 33
	{}
};

L
Linus Torvalds 已提交
34
static unsigned long smp_changes_mask;
35
static int mtrr_state_set;
36
u64 mtrr_tom2;
L
Linus Torvalds 已提交
37

38
struct mtrr_state_type mtrr_state;
S
Sheng Yang 已提交
39 40
EXPORT_SYMBOL_GPL(mtrr_state);

41
/*
42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66
 * BIOS is expected to clear MtrrFixDramModEn bit, see for example
 * "BIOS and Kernel Developer's Guide for the AMD Athlon 64 and AMD
 * Opteron Processors" (26094 Rev. 3.30 February 2006), section
 * "13.2.1.2 SYSCFG Register": "The MtrrFixDramModEn bit should be set
 * to 1 during BIOS initalization of the fixed MTRRs, then cleared to
 * 0 for operation."
 */
static inline void k8_check_syscfg_dram_mod_en(void)
{
	u32 lo, hi;

	if (!((boot_cpu_data.x86_vendor == X86_VENDOR_AMD) &&
	      (boot_cpu_data.x86 >= 0x0f)))
		return;

	rdmsr(MSR_K8_SYSCFG, lo, hi);
	if (lo & K8_MTRRFIXRANGE_DRAM_MODIFY) {
		printk(KERN_ERR FW_WARN "MTRR: CPU %u: SYSCFG[MtrrFixDramModEn]"
		       " not cleared by BIOS, clearing this bit\n",
		       smp_processor_id());
		lo &= ~K8_MTRRFIXRANGE_DRAM_MODIFY;
		mtrr_wrmsr(MSR_K8_SYSCFG, lo, hi);
	}
}

67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93
/*
 * Check and return the effective type for MTRR-MTRR type overlap.
 * Returns 1 if the effective type is UNCACHEABLE, else returns 0
 */
static int check_type_overlap(u8 *prev, u8 *curr)
{
	if (*prev == MTRR_TYPE_UNCACHABLE || *curr == MTRR_TYPE_UNCACHABLE) {
		*prev = MTRR_TYPE_UNCACHABLE;
		*curr = MTRR_TYPE_UNCACHABLE;
		return 1;
	}

	if ((*prev == MTRR_TYPE_WRBACK && *curr == MTRR_TYPE_WRTHROUGH) ||
	    (*prev == MTRR_TYPE_WRTHROUGH && *curr == MTRR_TYPE_WRBACK)) {
		*prev = MTRR_TYPE_WRTHROUGH;
		*curr = MTRR_TYPE_WRTHROUGH;
	}

	if (*prev != *curr) {
		*prev = MTRR_TYPE_UNCACHABLE;
		*curr = MTRR_TYPE_UNCACHABLE;
		return 1;
	}

	return 0;
}

94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138
/*
 * Returns the effective MTRR type for the region
 * Error returns:
 * - 0xFE - when the range is "not entirely covered" by _any_ var range MTRR
 * - 0xFF - when MTRR is not enabled
 */
u8 mtrr_type_lookup(u64 start, u64 end)
{
	int i;
	u64 base, mask;
	u8 prev_match, curr_match;

	if (!mtrr_state_set)
		return 0xFF;

	if (!mtrr_state.enabled)
		return 0xFF;

	/* Make end inclusive end, instead of exclusive */
	end--;

	/* Look in fixed ranges. Just return the type as per start */
	if (mtrr_state.have_fixed && (start < 0x100000)) {
		int idx;

		if (start < 0x80000) {
			idx = 0;
			idx += (start >> 16);
			return mtrr_state.fixed_ranges[idx];
		} else if (start < 0xC0000) {
			idx = 1 * 8;
			idx += ((start - 0x80000) >> 14);
			return mtrr_state.fixed_ranges[idx];
		} else if (start < 0x1000000) {
			idx = 3 * 8;
			idx += ((start - 0xC0000) >> 12);
			return mtrr_state.fixed_ranges[idx];
		}
	}

	/*
	 * Look in variable ranges
	 * Look of multiple ranges matching this address and pick type
	 * as per MTRR precedence
	 */
139
	if (!(mtrr_state.enabled & 2))
140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158
		return mtrr_state.def_type;

	prev_match = 0xFF;
	for (i = 0; i < num_var_ranges; ++i) {
		unsigned short start_state, end_state;

		if (!(mtrr_state.var_ranges[i].mask_lo & (1 << 11)))
			continue;

		base = (((u64)mtrr_state.var_ranges[i].base_hi) << 32) +
		       (mtrr_state.var_ranges[i].base_lo & PAGE_MASK);
		mask = (((u64)mtrr_state.var_ranges[i].mask_hi) << 32) +
		       (mtrr_state.var_ranges[i].mask_lo & PAGE_MASK);

		start_state = ((start & mask) == (base & mask));
		end_state = ((end & mask) == (base & mask));
		if (start_state != end_state)
			return 0xFE;

159
		if ((start & mask) != (base & mask))
160 161 162 163 164 165 166 167
			continue;

		curr_match = mtrr_state.var_ranges[i].base_lo & 0xff;
		if (prev_match == 0xFF) {
			prev_match = curr_match;
			continue;
		}

168 169
		if (check_type_overlap(&prev_match, &curr_match))
			return curr_match;
170 171
	}

172 173
	if (mtrr_tom2) {
		if (start >= (1ULL<<32) && (end < mtrr_tom2))
174 175 176
			return MTRR_TYPE_WRBACK;
	}

177 178 179 180 181 182
	if (prev_match != 0xFF)
		return prev_match;

	return mtrr_state.def_type;
}

183
/* Get the MSR pair relating to a var range */
184
static void
L
Linus Torvalds 已提交
185 186 187 188 189 190
get_mtrr_var_range(unsigned int index, struct mtrr_var_range *vr)
{
	rdmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi);
	rdmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi);
}

191
/* Fill the MSR pair relating to a var range */
192 193 194 195 196 197 198 199 200 201 202 203 204
void fill_mtrr_var_range(unsigned int index,
		u32 base_lo, u32 base_hi, u32 mask_lo, u32 mask_hi)
{
	struct mtrr_var_range *vr;

	vr = mtrr_state.var_ranges;

	vr[index].base_lo = base_lo;
	vr[index].base_hi = base_hi;
	vr[index].mask_lo = mask_lo;
	vr[index].mask_hi = mask_hi;
}

205
static void get_fixed_ranges(mtrr_type *frs)
L
Linus Torvalds 已提交
206
{
207
	unsigned int *p = (unsigned int *)frs;
L
Linus Torvalds 已提交
208 209
	int i;

210 211
	k8_check_syscfg_dram_mod_en();

212
	rdmsr(MSR_MTRRfix64K_00000, p[0], p[1]);
L
Linus Torvalds 已提交
213 214

	for (i = 0; i < 2; i++)
215
		rdmsr(MSR_MTRRfix16K_80000 + i, p[2 + i * 2], p[3 + i * 2]);
L
Linus Torvalds 已提交
216
	for (i = 0; i < 8; i++)
217
		rdmsr(MSR_MTRRfix4K_C0000 + i, p[6 + i * 2], p[7 + i * 2]);
L
Linus Torvalds 已提交
218 219
}

220 221
void mtrr_save_fixed_ranges(void *info)
{
A
Andrew Morton 已提交
222 223
	if (cpu_has_mtrr)
		get_fixed_ranges(mtrr_state.fixed_ranges);
224 225
}

226 227 228 229 230 231 232 233 234
static unsigned __initdata last_fixed_start;
static unsigned __initdata last_fixed_end;
static mtrr_type __initdata last_fixed_type;

static void __init print_fixed_last(void)
{
	if (!last_fixed_end)
		return;

235 236
	pr_debug("  %05X-%05X %s\n", last_fixed_start,
		 last_fixed_end - 1, mtrr_attrib_to_str(last_fixed_type));
237 238 239 240 241

	last_fixed_end = 0;
}

static void __init update_fixed_last(unsigned base, unsigned end,
242
				     mtrr_type type)
243 244 245 246 247 248
{
	last_fixed_start = base;
	last_fixed_end = end;
	last_fixed_type = type;
}

249 250
static void __init
print_fixed(unsigned base, unsigned step, const mtrr_type *types)
J
Jan Beulich 已提交
251 252 253
{
	unsigned i;

254 255 256 257 258 259 260 261 262 263 264 265 266
	for (i = 0; i < 8; ++i, ++types, base += step) {
		if (last_fixed_end == 0) {
			update_fixed_last(base, base + step, *types);
			continue;
		}
		if (last_fixed_end == base && last_fixed_type == *types) {
			last_fixed_end = base + step;
			continue;
		}
		/* new segments: gap or different type */
		print_fixed_last();
		update_fixed_last(base, base + step, *types);
	}
J
Jan Beulich 已提交
267 268
}

269 270 271
static void prepare_set(void);
static void post_set(void);

Y
Yinghai Lu 已提交
272 273 274 275 276
static void __init print_mtrr_state(void)
{
	unsigned int i;
	int high_width;

277 278
	pr_debug("MTRR default type: %s\n",
		 mtrr_attrib_to_str(mtrr_state.def_type));
Y
Yinghai Lu 已提交
279
	if (mtrr_state.have_fixed) {
280 281
		pr_debug("MTRR fixed ranges %sabled:\n",
			 mtrr_state.enabled & 1 ? "en" : "dis");
Y
Yinghai Lu 已提交
282 283
		print_fixed(0x00000, 0x10000, mtrr_state.fixed_ranges + 0);
		for (i = 0; i < 2; ++i)
284 285
			print_fixed(0x80000 + i * 0x20000, 0x04000,
				    mtrr_state.fixed_ranges + (i + 1) * 8);
Y
Yinghai Lu 已提交
286
		for (i = 0; i < 8; ++i)
287 288
			print_fixed(0xC0000 + i * 0x08000, 0x01000,
				    mtrr_state.fixed_ranges + (i + 3) * 8);
289 290 291

		/* tail */
		print_fixed_last();
Y
Yinghai Lu 已提交
292
	}
293 294
	pr_debug("MTRR variable ranges %sabled:\n",
		 mtrr_state.enabled & 2 ? "en" : "dis");
295 296 297 298 299
	if (size_or_mask & 0xffffffffUL)
		high_width = ffs(size_or_mask & 0xffffffffUL) - 1;
	else
		high_width = ffs(size_or_mask>>32) + 32 - 1;
	high_width = (high_width - (32 - PAGE_SHIFT) + 3) / 4;
300

Y
Yinghai Lu 已提交
301 302
	for (i = 0; i < num_var_ranges; ++i) {
		if (mtrr_state.var_ranges[i].mask_lo & (1 << 11))
303 304 305 306 307 308 309 310 311
			pr_debug("  %u base %0*X%05X000 mask %0*X%05X000 %s\n",
				 i,
				 high_width,
				 mtrr_state.var_ranges[i].base_hi,
				 mtrr_state.var_ranges[i].base_lo >> 12,
				 high_width,
				 mtrr_state.var_ranges[i].mask_hi,
				 mtrr_state.var_ranges[i].mask_lo >> 12,
				 mtrr_attrib_to_str(mtrr_state.var_ranges[i].base_lo & 0xff));
Y
Yinghai Lu 已提交
312
		else
313
			pr_debug("  %u disabled\n", i);
Y
Yinghai Lu 已提交
314
	}
315 316
	if (mtrr_tom2)
		pr_debug("TOM2: %016llx aka %lldM\n", mtrr_tom2, mtrr_tom2>>20);
Y
Yinghai Lu 已提交
317 318
}

319
/* Grab all of the MTRR state for this CPU into *state */
320
void __init get_mtrr_state(void)
L
Linus Torvalds 已提交
321 322
{
	struct mtrr_var_range *vrs;
323
	unsigned long flags;
324 325
	unsigned lo, dummy;
	unsigned int i;
L
Linus Torvalds 已提交
326 327 328

	vrs = mtrr_state.var_ranges;

329
	rdmsr(MSR_MTRRcap, lo, dummy);
J
Jan Beulich 已提交
330 331
	mtrr_state.have_fixed = (lo >> 8) & 1;

L
Linus Torvalds 已提交
332 333
	for (i = 0; i < num_var_ranges; i++)
		get_mtrr_var_range(i, &vrs[i]);
J
Jan Beulich 已提交
334 335
	if (mtrr_state.have_fixed)
		get_fixed_ranges(mtrr_state.fixed_ranges);
L
Linus Torvalds 已提交
336

337
	rdmsr(MSR_MTRRdefType, lo, dummy);
L
Linus Torvalds 已提交
338 339
	mtrr_state.def_type = (lo & 0xff);
	mtrr_state.enabled = (lo & 0xc00) >> 10;
J
Jan Beulich 已提交
340

341
	if (amd_special_default_mtrr()) {
342
		unsigned low, high;
343

344
		/* TOP_MEM2 */
345
		rdmsr(MSR_K8_TOP_MEM2, low, high);
346 347 348
		mtrr_tom2 = high;
		mtrr_tom2 <<= 32;
		mtrr_tom2 |= low;
Y
Yinghai Lu 已提交
349
		mtrr_tom2 &= 0xffffff800000ULL;
350
	}
Y
Yinghai Lu 已提交
351 352 353

	print_mtrr_state();

354 355 356 357 358 359 360 361 362 363
	mtrr_state_set = 1;

	/* PAT setup for BP. We need to go through sync steps here */
	local_irq_save(flags);
	prepare_set();

	pat_init();

	post_set();
	local_irq_restore(flags);
L
Linus Torvalds 已提交
364 365
}

366
/* Some BIOS's are messed up and don't set all MTRRs the same! */
L
Linus Torvalds 已提交
367 368 369 370 371 372 373
void __init mtrr_state_warn(void)
{
	unsigned long mask = smp_changes_mask;

	if (!mask)
		return;
	if (mask & MTRR_CHANGE_MASK_FIXED)
374
		pr_warning("mtrr: your CPUs had inconsistent fixed MTRR settings\n");
L
Linus Torvalds 已提交
375
	if (mask & MTRR_CHANGE_MASK_VARIABLE)
376
		pr_warning("mtrr: your CPUs had inconsistent variable MTRR settings\n");
L
Linus Torvalds 已提交
377
	if (mask & MTRR_CHANGE_MASK_DEFTYPE)
378 379
		pr_warning("mtrr: your CPUs had inconsistent MTRRdefType settings\n");

L
Linus Torvalds 已提交
380 381 382 383
	printk(KERN_INFO "mtrr: probably your BIOS does not setup all CPUs.\n");
	printk(KERN_INFO "mtrr: corrected configuration.\n");
}

384 385 386 387 388
/*
 * Doesn't attempt to pass an error out to MTRR users
 * because it's quite complicated in some cases and probably not
 * worth it because the best error handling is to ignore it.
 */
L
Linus Torvalds 已提交
389 390
void mtrr_wrmsr(unsigned msr, unsigned a, unsigned b)
{
391
	if (wrmsr_safe(msr, a, b) < 0) {
L
Linus Torvalds 已提交
392 393 394
		printk(KERN_ERR
			"MTRR: CPU %u: Writing MSR %x to %x:%x failed\n",
			smp_processor_id(), msr, a, b);
395
	}
L
Linus Torvalds 已提交
396 397
}

398
/**
399 400
 * set_fixed_range - checks & updates a fixed-range MTRR if it
 *		     differs from the value it should have
401 402 403
 * @msr: MSR address of the MTTR which should be checked and updated
 * @changed: pointer which indicates whether the MTRR needed to be changed
 * @msrwords: pointer to the MSR values which the MSR should have
404
 */
405
static void set_fixed_range(int msr, bool *changed, unsigned int *msrwords)
406 407 408 409 410 411 412
{
	unsigned lo, hi;

	rdmsr(msr, lo, hi);

	if (lo != msrwords[0] || hi != msrwords[1]) {
		mtrr_wrmsr(msr, msrwords[0], msrwords[1]);
413
		*changed = true;
414 415 416
	}
}

417 418 419 420 421 422 423 424
/**
 * generic_get_free_region - Get a free MTRR.
 * @base: The starting (base) address of the region.
 * @size: The size (in bytes) of the region.
 * @replace_reg: mtrr index to be replaced; set to invalid value if none.
 *
 * Returns: The index of the region on success, else negative on error.
 */
425 426
int
generic_get_free_region(unsigned long base, unsigned long size, int replace_reg)
L
Linus Torvalds 已提交
427
{
J
Jan Beulich 已提交
428
	unsigned long lbase, lsize;
429 430
	mtrr_type ltype;
	int i, max;
L
Linus Torvalds 已提交
431 432

	max = num_var_ranges;
J
Jan Beulich 已提交
433 434
	if (replace_reg >= 0 && replace_reg < max)
		return replace_reg;
435

L
Linus Torvalds 已提交
436 437 438 439 440
	for (i = 0; i < max; ++i) {
		mtrr_if->get(i, &lbase, &lsize, &ltype);
		if (lsize == 0)
			return i;
	}
441

L
Linus Torvalds 已提交
442 443 444
	return -ENOSPC;
}

A
Adrian Bunk 已提交
445
static void generic_get_mtrr(unsigned int reg, unsigned long *base,
J
Jan Beulich 已提交
446
			     unsigned long *size, mtrr_type *type)
L
Linus Torvalds 已提交
447 448
{
	unsigned int mask_lo, mask_hi, base_lo, base_hi;
Y
Yinghai Lu 已提交
449
	unsigned int tmp, hi;
L
Linus Torvalds 已提交
450

Y
Yinghai Lu 已提交
451 452 453 454
	/*
	 * get_mtrr doesn't need to update mtrr_state, also it could be called
	 * from any cpu, so try to print it out directly.
	 */
455
	get_cpu();
456

L
Linus Torvalds 已提交
457
	rdmsr(MTRRphysMask_MSR(reg), mask_lo, mask_hi);
Y
Yinghai Lu 已提交
458

L
Linus Torvalds 已提交
459
	if ((mask_lo & 0x800) == 0) {
460
		/*  Invalid (i.e. free) range */
L
Linus Torvalds 已提交
461 462 463
		*base = 0;
		*size = 0;
		*type = 0;
464
		goto out_put_cpu;
L
Linus Torvalds 已提交
465 466 467 468
	}

	rdmsr(MTRRphysBase_MSR(reg), base_lo, base_hi);

469
	/* Work out the shifted address mask: */
Y
Yinghai Lu 已提交
470 471
	tmp = mask_hi << (32 - PAGE_SHIFT) | mask_lo >> PAGE_SHIFT;
	mask_lo = size_or_mask | tmp;
472 473

	/* Expand tmp with high bits to all 1s: */
Y
Yinghai Lu 已提交
474 475 476 477 478
	hi = fls(tmp);
	if (hi > 0) {
		tmp |= ~((1<<(hi - 1)) - 1);

		if (tmp != mask_lo) {
A
Alan Cox 已提交
479
			printk(KERN_WARNING "mtrr: your BIOS has configured an incorrect mask, fixing it.\n");
Y
Yinghai Lu 已提交
480 481 482
			mask_lo = tmp;
		}
	}
L
Linus Torvalds 已提交
483

484 485 486 487
	/*
	 * This works correctly if size is a power of two, i.e. a
	 * contiguous range:
	 */
L
Linus Torvalds 已提交
488 489 490
	*size = -mask_lo;
	*base = base_hi << (32 - PAGE_SHIFT) | base_lo >> PAGE_SHIFT;
	*type = base_lo & 0xff;
Y
Yinghai Lu 已提交
491

492 493
out_put_cpu:
	put_cpu();
L
Linus Torvalds 已提交
494 495
}

496
/**
497 498
 * set_fixed_ranges - checks & updates the fixed-range MTRRs if they
 *		      differ from the saved set
499
 * @frs: pointer to fixed-range MTRR values, saved by get_fixed_ranges()
500
 */
501
static int set_fixed_ranges(mtrr_type *frs)
L
Linus Torvalds 已提交
502
{
503
	unsigned long long *saved = (unsigned long long *)frs;
504
	bool changed = false;
505
	int block = -1, range;
L
Linus Torvalds 已提交
506

507 508
	k8_check_syscfg_dram_mod_en();

509 510 511 512 513
	while (fixed_range_blocks[++block].ranges) {
		for (range = 0; range < fixed_range_blocks[block].ranges; range++)
			set_fixed_range(fixed_range_blocks[block].base_msr + range,
					&changed, (unsigned int *)saved++);
	}
L
Linus Torvalds 已提交
514 515 516 517

	return changed;
}

518 519 520 521
/*
 * Set the MSR pair relating to a var range.
 * Returns true if changes are made.
 */
522
static bool set_mtrr_var_ranges(unsigned int index, struct mtrr_var_range *vr)
L
Linus Torvalds 已提交
523 524
{
	unsigned int lo, hi;
525
	bool changed = false;
L
Linus Torvalds 已提交
526 527 528

	rdmsr(MTRRphysBase_MSR(index), lo, hi);
	if ((vr->base_lo & 0xfffff0ffUL) != (lo & 0xfffff0ffUL)
529 530
	    || (vr->base_hi & (size_and_mask >> (32 - PAGE_SHIFT))) !=
		(hi & (size_and_mask >> (32 - PAGE_SHIFT)))) {
531

L
Linus Torvalds 已提交
532
		mtrr_wrmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi);
533
		changed = true;
L
Linus Torvalds 已提交
534 535 536 537 538
	}

	rdmsr(MTRRphysMask_MSR(index), lo, hi);

	if ((vr->mask_lo & 0xfffff800UL) != (lo & 0xfffff800UL)
539 540
	    || (vr->mask_hi & (size_and_mask >> (32 - PAGE_SHIFT))) !=
		(hi & (size_and_mask >> (32 - PAGE_SHIFT)))) {
L
Linus Torvalds 已提交
541
		mtrr_wrmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi);
542
		changed = true;
L
Linus Torvalds 已提交
543 544 545 546
	}
	return changed;
}

J
Jan Beulich 已提交
547 548
static u32 deftype_lo, deftype_hi;

549 550 551 552 553 554
/**
 * set_mtrr_state - Set the MTRR state for this CPU.
 *
 * NOTE: The CPU must already be in a safe state for MTRR changes.
 * RETURNS: 0 if no changes made, else a mask indicating what was changed.
 */
J
Jan Beulich 已提交
555
static unsigned long set_mtrr_state(void)
L
Linus Torvalds 已提交
556 557
{
	unsigned long change_mask = 0;
558
	unsigned int i;
L
Linus Torvalds 已提交
559

560
	for (i = 0; i < num_var_ranges; i++) {
L
Linus Torvalds 已提交
561 562
		if (set_mtrr_var_ranges(i, &mtrr_state.var_ranges[i]))
			change_mask |= MTRR_CHANGE_MASK_VARIABLE;
563
	}
L
Linus Torvalds 已提交
564

J
Jan Beulich 已提交
565
	if (mtrr_state.have_fixed && set_fixed_ranges(mtrr_state.fixed_ranges))
L
Linus Torvalds 已提交
566 567
		change_mask |= MTRR_CHANGE_MASK_FIXED;

568 569 570 571
	/*
	 * Set_mtrr_restore restores the old value of MTRRdefType,
	 * so to set it we fiddle with the saved value:
	 */
L
Linus Torvalds 已提交
572 573
	if ((deftype_lo & 0xff) != mtrr_state.def_type
	    || ((deftype_lo & 0xc00) >> 10) != mtrr_state.enabled) {
574 575 576

		deftype_lo = (deftype_lo & ~0xcff) | mtrr_state.def_type |
			     (mtrr_state.enabled << 10);
L
Linus Torvalds 已提交
577 578 579 580 581 582 583
		change_mask |= MTRR_CHANGE_MASK_DEFTYPE;
	}

	return change_mask;
}


584
static unsigned long cr4;
585
static DEFINE_RAW_SPINLOCK(set_atomicity_lock);
L
Linus Torvalds 已提交
586 587

/*
588 589 590 591 592
 * Since we are disabling the cache don't allow any interrupts,
 * they would run extremely slow and would only increase the pain.
 *
 * The caller must ensure that local interrupts are disabled and
 * are reenabled after post_set() has been called.
L
Linus Torvalds 已提交
593
 */
594
static void prepare_set(void) __acquires(set_atomicity_lock)
L
Linus Torvalds 已提交
595 596 597
{
	unsigned long cr0;

598 599 600 601 602 603
	/*
	 * Note that this is not ideal
	 * since the cache is only flushed/disabled for this CPU while the
	 * MTRRs are changed, but changing this requires more invasive
	 * changes to the way the kernel boots
	 */
L
Linus Torvalds 已提交
604

605
	raw_spin_lock(&set_atomicity_lock);
L
Linus Torvalds 已提交
606

607
	/* Enter the no-fill (CD=1, NW=0) cache mode and flush caches. */
D
Dave Jones 已提交
608
	cr0 = read_cr0() | X86_CR0_CD;
L
Linus Torvalds 已提交
609 610 611
	write_cr0(cr0);
	wbinvd();

612 613
	/* Save value of CR4 and clear Page Global Enable (bit 7) */
	if (cpu_has_pge) {
L
Linus Torvalds 已提交
614 615 616 617 618 619 620
		cr4 = read_cr4();
		write_cr4(cr4 & ~X86_CR4_PGE);
	}

	/* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */
	__flush_tlb();

621
	/* Save MTRR state */
622
	rdmsr(MSR_MTRRdefType, deftype_lo, deftype_hi);
L
Linus Torvalds 已提交
623

624
	/* Disable MTRRs, and set the default type to uncached */
625
	mtrr_wrmsr(MSR_MTRRdefType, deftype_lo & ~0xcff, deftype_hi);
L
Linus Torvalds 已提交
626 627
}

628
static void post_set(void) __releases(set_atomicity_lock)
L
Linus Torvalds 已提交
629
{
630
	/* Flush TLBs (no need to flush caches - they are disabled) */
L
Linus Torvalds 已提交
631 632 633
	__flush_tlb();

	/* Intel (P6) standard MTRRs */
634
	mtrr_wrmsr(MSR_MTRRdefType, deftype_lo, deftype_hi);
635 636

	/* Enable caches */
L
Linus Torvalds 已提交
637 638
	write_cr0(read_cr0() & 0xbfffffff);

639 640
	/* Restore value of CR4 */
	if (cpu_has_pge)
L
Linus Torvalds 已提交
641
		write_cr4(cr4);
642
	raw_spin_unlock(&set_atomicity_lock);
L
Linus Torvalds 已提交
643 644 645 646 647 648 649 650 651 652 653
}

static void generic_set_all(void)
{
	unsigned long mask, count;
	unsigned long flags;

	local_irq_save(flags);
	prepare_set();

	/* Actually set the state */
J
Jan Beulich 已提交
654
	mask = set_mtrr_state();
L
Linus Torvalds 已提交
655

656 657 658
	/* also set PAT */
	pat_init();

L
Linus Torvalds 已提交
659 660 661
	post_set();
	local_irq_restore(flags);

662
	/* Use the atomic bitops to update the global mask */
L
Linus Torvalds 已提交
663 664 665 666 667
	for (count = 0; count < sizeof mask * 8; ++count) {
		if (mask & 0x01)
			set_bit(count, &smp_changes_mask);
		mask >>= 1;
	}
668

L
Linus Torvalds 已提交
669 670
}

671 672 673 674 675 676 677 678 679 680
/**
 * generic_set_mtrr - set variable MTRR register on the local CPU.
 *
 * @reg: The register to set.
 * @base: The base address of the region.
 * @size: The size of the region. If this is 0 the region is disabled.
 * @type: The type of the region.
 *
 * Returns nothing.
 */
L
Linus Torvalds 已提交
681 682 683 684
static void generic_set_mtrr(unsigned int reg, unsigned long base,
			     unsigned long size, mtrr_type type)
{
	unsigned long flags;
S
Shaohua Li 已提交
685 686 687
	struct mtrr_var_range *vr;

	vr = &mtrr_state.var_ranges[reg];
L
Linus Torvalds 已提交
688 689 690 691 692

	local_irq_save(flags);
	prepare_set();

	if (size == 0) {
693 694 695 696
		/*
		 * The invalid bit is kept in the mask, so we simply
		 * clear the relevant mask register to disable a range.
		 */
L
Linus Torvalds 已提交
697
		mtrr_wrmsr(MTRRphysMask_MSR(reg), 0, 0);
S
Shaohua Li 已提交
698
		memset(vr, 0, sizeof(struct mtrr_var_range));
L
Linus Torvalds 已提交
699
	} else {
S
Shaohua Li 已提交
700 701 702 703 704 705 706
		vr->base_lo = base << PAGE_SHIFT | type;
		vr->base_hi = (base & size_and_mask) >> (32 - PAGE_SHIFT);
		vr->mask_lo = -size << PAGE_SHIFT | 0x800;
		vr->mask_hi = (-size & size_and_mask) >> (32 - PAGE_SHIFT);

		mtrr_wrmsr(MTRRphysBase_MSR(reg), vr->base_lo, vr->base_hi);
		mtrr_wrmsr(MTRRphysMask_MSR(reg), vr->mask_lo, vr->mask_hi);
L
Linus Torvalds 已提交
707 708 709 710 711 712
	}

	post_set();
	local_irq_restore(flags);
}

713 714
int generic_validate_add_page(unsigned long base, unsigned long size,
			      unsigned int type)
L
Linus Torvalds 已提交
715 716 717
{
	unsigned long lbase, last;

718 719 720 721
	/*
	 * For Intel PPro stepping <= 7
	 * must be 4 MiB aligned and not touch 0x70000000 -> 0x7003FFFF
	 */
L
Linus Torvalds 已提交
722 723 724 725
	if (is_cpu(INTEL) && boot_cpu_data.x86 == 6 &&
	    boot_cpu_data.x86_model == 1 &&
	    boot_cpu_data.x86_mask <= 7) {
		if (base & ((1 << (22 - PAGE_SHIFT)) - 1)) {
726
			pr_warning("mtrr: base(0x%lx000) is not 4 MiB aligned\n", base);
L
Linus Torvalds 已提交
727 728
			return -EINVAL;
		}
729
		if (!(base + size < 0x70000 || base > 0x7003F) &&
L
Linus Torvalds 已提交
730 731
		    (type == MTRR_TYPE_WRCOMB
		     || type == MTRR_TYPE_WRBACK)) {
732
			pr_warning("mtrr: writable mtrr between 0x70000000 and 0x7003FFFF may hang the CPU.\n");
L
Linus Torvalds 已提交
733 734 735 736
			return -EINVAL;
		}
	}

737 738 739 740
	/*
	 * Check upper bits of base and last are equal and lower bits are 0
	 * for base and 1 for last
	 */
L
Linus Torvalds 已提交
741 742
	last = base + size - 1;
	for (lbase = base; !(lbase & 1) && (last & 1);
743 744
	     lbase = lbase >> 1, last = last >> 1)
		;
L
Linus Torvalds 已提交
745
	if (lbase != last) {
746
		pr_warning("mtrr: base(0x%lx000) is not aligned on a size(0x%lx000) boundary\n", base, size);
L
Linus Torvalds 已提交
747 748 749 750 751 752 753 754
		return -EINVAL;
	}
	return 0;
}

static int generic_have_wrcomb(void)
{
	unsigned long config, dummy;
755
	rdmsr(MSR_MTRRcap, config, dummy);
756
	return config & (1 << 10);
L
Linus Torvalds 已提交
757 758 759 760 761 762 763
}

int positive_have_wrcomb(void)
{
	return 1;
}

764 765
/*
 * Generic structure...
L
Linus Torvalds 已提交
766
 */
767
const struct mtrr_ops generic_mtrr_ops = {
768 769 770 771 772 773 774
	.use_intel_if		= 1,
	.set_all		= generic_set_all,
	.get			= generic_get_mtrr,
	.get_free_region	= generic_get_free_region,
	.set			= generic_set_mtrr,
	.validate_add_page	= generic_validate_add_page,
	.have_wrcomb		= generic_have_wrcomb,
L
Linus Torvalds 已提交
775
};