generic.c 19.1 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4 5
/* This only handles 32bit MTRR on 32bit hosts. This is strictly wrong
   because MTRRs can span upto 40 bits (36bits on most modern x86) */ 
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/mm.h>
J
Jan Beulich 已提交
6
#include <linux/module.h>
L
Linus Torvalds 已提交
7 8 9 10 11
#include <asm/io.h>
#include <asm/mtrr.h>
#include <asm/msr.h>
#include <asm/system.h>
#include <asm/cpufeature.h>
D
Dave Jones 已提交
12
#include <asm/processor-flags.h>
L
Linus Torvalds 已提交
13
#include <asm/tlbflush.h>
14
#include <asm/pat.h>
L
Linus Torvalds 已提交
15 16
#include "mtrr.h"

17 18 19 20 21 22
struct fixed_range_block {
	int base_msr; /* start address of an MTRR block */
	int ranges;   /* number of MTRRs in this block  */
};

static struct fixed_range_block fixed_range_blocks[] = {
23
	{ MSR_MTRRfix64K_00000, 1 }, /* one  64k MTRR  */
24
	{ MSR_MTRRfix16K_80000, 2 }, /* two  16k MTRRs */
25 26 27 28
	{ MTRRfix4K_C0000_MSR,  8 }, /* eight 4k MTRRs */
	{}
};

L
Linus Torvalds 已提交
29
static unsigned long smp_changes_mask;
30
static int mtrr_state_set;
31
u64 mtrr_tom2;
L
Linus Torvalds 已提交
32

S
Sheng Yang 已提交
33 34 35
struct mtrr_state_type mtrr_state = {};
EXPORT_SYMBOL_GPL(mtrr_state);

36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61
/**
 * BIOS is expected to clear MtrrFixDramModEn bit, see for example
 * "BIOS and Kernel Developer's Guide for the AMD Athlon 64 and AMD
 * Opteron Processors" (26094 Rev. 3.30 February 2006), section
 * "13.2.1.2 SYSCFG Register": "The MtrrFixDramModEn bit should be set
 * to 1 during BIOS initalization of the fixed MTRRs, then cleared to
 * 0 for operation."
 */
static inline void k8_check_syscfg_dram_mod_en(void)
{
	u32 lo, hi;

	if (!((boot_cpu_data.x86_vendor == X86_VENDOR_AMD) &&
	      (boot_cpu_data.x86 >= 0x0f)))
		return;

	rdmsr(MSR_K8_SYSCFG, lo, hi);
	if (lo & K8_MTRRFIXRANGE_DRAM_MODIFY) {
		printk(KERN_ERR FW_WARN "MTRR: CPU %u: SYSCFG[MtrrFixDramModEn]"
		       " not cleared by BIOS, clearing this bit\n",
		       smp_processor_id());
		lo &= ~K8_MTRRFIXRANGE_DRAM_MODIFY;
		mtrr_wrmsr(MSR_K8_SYSCFG, lo, hi);
	}
}

62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106
/*
 * Returns the effective MTRR type for the region
 * Error returns:
 * - 0xFE - when the range is "not entirely covered" by _any_ var range MTRR
 * - 0xFF - when MTRR is not enabled
 */
u8 mtrr_type_lookup(u64 start, u64 end)
{
	int i;
	u64 base, mask;
	u8 prev_match, curr_match;

	if (!mtrr_state_set)
		return 0xFF;

	if (!mtrr_state.enabled)
		return 0xFF;

	/* Make end inclusive end, instead of exclusive */
	end--;

	/* Look in fixed ranges. Just return the type as per start */
	if (mtrr_state.have_fixed && (start < 0x100000)) {
		int idx;

		if (start < 0x80000) {
			idx = 0;
			idx += (start >> 16);
			return mtrr_state.fixed_ranges[idx];
		} else if (start < 0xC0000) {
			idx = 1 * 8;
			idx += ((start - 0x80000) >> 14);
			return mtrr_state.fixed_ranges[idx];
		} else if (start < 0x1000000) {
			idx = 3 * 8;
			idx += ((start - 0xC0000) >> 12);
			return mtrr_state.fixed_ranges[idx];
		}
	}

	/*
	 * Look in variable ranges
	 * Look of multiple ranges matching this address and pick type
	 * as per MTRR precedence
	 */
H
Harvey Harrison 已提交
107
	if (!(mtrr_state.enabled & 2)) {
108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155
		return mtrr_state.def_type;
	}

	prev_match = 0xFF;
	for (i = 0; i < num_var_ranges; ++i) {
		unsigned short start_state, end_state;

		if (!(mtrr_state.var_ranges[i].mask_lo & (1 << 11)))
			continue;

		base = (((u64)mtrr_state.var_ranges[i].base_hi) << 32) +
		       (mtrr_state.var_ranges[i].base_lo & PAGE_MASK);
		mask = (((u64)mtrr_state.var_ranges[i].mask_hi) << 32) +
		       (mtrr_state.var_ranges[i].mask_lo & PAGE_MASK);

		start_state = ((start & mask) == (base & mask));
		end_state = ((end & mask) == (base & mask));
		if (start_state != end_state)
			return 0xFE;

		if ((start & mask) != (base & mask)) {
			continue;
		}

		curr_match = mtrr_state.var_ranges[i].base_lo & 0xff;
		if (prev_match == 0xFF) {
			prev_match = curr_match;
			continue;
		}

		if (prev_match == MTRR_TYPE_UNCACHABLE ||
		    curr_match == MTRR_TYPE_UNCACHABLE) {
			return MTRR_TYPE_UNCACHABLE;
		}

		if ((prev_match == MTRR_TYPE_WRBACK &&
		     curr_match == MTRR_TYPE_WRTHROUGH) ||
		    (prev_match == MTRR_TYPE_WRTHROUGH &&
		     curr_match == MTRR_TYPE_WRBACK)) {
			prev_match = MTRR_TYPE_WRTHROUGH;
			curr_match = MTRR_TYPE_WRTHROUGH;
		}

		if (prev_match != curr_match) {
			return MTRR_TYPE_UNCACHABLE;
		}
	}

156 157
	if (mtrr_tom2) {
		if (start >= (1ULL<<32) && (end < mtrr_tom2))
158 159 160
			return MTRR_TYPE_WRBACK;
	}

161 162 163 164 165 166
	if (prev_match != 0xFF)
		return prev_match;

	return mtrr_state.def_type;
}

L
Linus Torvalds 已提交
167
/*  Get the MSR pair relating to a var range  */
168
static void
L
Linus Torvalds 已提交
169 170 171 172 173 174
get_mtrr_var_range(unsigned int index, struct mtrr_var_range *vr)
{
	rdmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi);
	rdmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi);
}

175 176 177 178 179 180 181 182 183 184 185 186 187 188
/*  fill the MSR pair relating to a var range  */
void fill_mtrr_var_range(unsigned int index,
		u32 base_lo, u32 base_hi, u32 mask_lo, u32 mask_hi)
{
	struct mtrr_var_range *vr;

	vr = mtrr_state.var_ranges;

	vr[index].base_lo = base_lo;
	vr[index].base_hi = base_hi;
	vr[index].mask_lo = mask_lo;
	vr[index].mask_hi = mask_hi;
}

189
static void
L
Linus Torvalds 已提交
190 191 192 193 194
get_fixed_ranges(mtrr_type * frs)
{
	unsigned int *p = (unsigned int *) frs;
	int i;

195 196
	k8_check_syscfg_dram_mod_en();

197
	rdmsr(MSR_MTRRfix64K_00000, p[0], p[1]);
L
Linus Torvalds 已提交
198 199

	for (i = 0; i < 2; i++)
200
		rdmsr(MSR_MTRRfix16K_80000 + i, p[2 + i * 2], p[3 + i * 2]);
L
Linus Torvalds 已提交
201 202 203 204
	for (i = 0; i < 8; i++)
		rdmsr(MTRRfix4K_C0000_MSR + i, p[6 + i * 2], p[7 + i * 2]);
}

205 206
void mtrr_save_fixed_ranges(void *info)
{
A
Andrew Morton 已提交
207 208
	if (cpu_has_mtrr)
		get_fixed_ranges(mtrr_state.fixed_ranges);
209 210
}

211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235
static unsigned __initdata last_fixed_start;
static unsigned __initdata last_fixed_end;
static mtrr_type __initdata last_fixed_type;

static void __init print_fixed_last(void)
{
	if (!last_fixed_end)
		return;

	printk(KERN_DEBUG "  %05X-%05X %s\n", last_fixed_start,
		last_fixed_end - 1, mtrr_attrib_to_str(last_fixed_type));

	last_fixed_end = 0;
}

static void __init update_fixed_last(unsigned base, unsigned end,
				       mtrr_type type)
{
	last_fixed_start = base;
	last_fixed_end = end;
	last_fixed_type = type;
}

static void __init print_fixed(unsigned base, unsigned step,
			       const mtrr_type *types)
J
Jan Beulich 已提交
236 237 238
{
	unsigned i;

239 240 241 242 243 244 245 246 247 248 249 250 251
	for (i = 0; i < 8; ++i, ++types, base += step) {
		if (last_fixed_end == 0) {
			update_fixed_last(base, base + step, *types);
			continue;
		}
		if (last_fixed_end == base && last_fixed_type == *types) {
			last_fixed_end = base + step;
			continue;
		}
		/* new segments: gap or different type */
		print_fixed_last();
		update_fixed_last(base, base + step, *types);
	}
J
Jan Beulich 已提交
252 253
}

254 255 256
static void prepare_set(void);
static void post_set(void);

Y
Yinghai Lu 已提交
257 258 259 260 261
static void __init print_mtrr_state(void)
{
	unsigned int i;
	int high_width;

262 263
	printk(KERN_DEBUG "MTRR default type: %s\n",
			 mtrr_attrib_to_str(mtrr_state.def_type));
Y
Yinghai Lu 已提交
264
	if (mtrr_state.have_fixed) {
265
		printk(KERN_DEBUG "MTRR fixed ranges %sabled:\n",
Y
Yinghai Lu 已提交
266 267 268 269 270 271
		       mtrr_state.enabled & 1 ? "en" : "dis");
		print_fixed(0x00000, 0x10000, mtrr_state.fixed_ranges + 0);
		for (i = 0; i < 2; ++i)
			print_fixed(0x80000 + i * 0x20000, 0x04000, mtrr_state.fixed_ranges + (i + 1) * 8);
		for (i = 0; i < 8; ++i)
			print_fixed(0xC0000 + i * 0x08000, 0x01000, mtrr_state.fixed_ranges + (i + 3) * 8);
272 273 274

		/* tail */
		print_fixed_last();
Y
Yinghai Lu 已提交
275
	}
276
	printk(KERN_DEBUG "MTRR variable ranges %sabled:\n",
Y
Yinghai Lu 已提交
277 278 279 280
	       mtrr_state.enabled & 2 ? "en" : "dis");
	high_width = ((size_or_mask ? ffs(size_or_mask) - 1 : 32) - (32 - PAGE_SHIFT) + 3) / 4;
	for (i = 0; i < num_var_ranges; ++i) {
		if (mtrr_state.var_ranges[i].mask_lo & (1 << 11))
281
			printk(KERN_DEBUG "  %u base %0*X%05X000 mask %0*X%05X000 %s\n",
Y
Yinghai Lu 已提交
282 283 284 285 286 287 288 289 290
			       i,
			       high_width,
			       mtrr_state.var_ranges[i].base_hi,
			       mtrr_state.var_ranges[i].base_lo >> 12,
			       high_width,
			       mtrr_state.var_ranges[i].mask_hi,
			       mtrr_state.var_ranges[i].mask_lo >> 12,
			       mtrr_attrib_to_str(mtrr_state.var_ranges[i].base_lo & 0xff));
		else
291
			printk(KERN_DEBUG "  %u disabled\n", i);
Y
Yinghai Lu 已提交
292 293
	}
	if (mtrr_tom2) {
294
		printk(KERN_DEBUG "TOM2: %016llx aka %lldM\n",
Y
Yinghai Lu 已提交
295 296 297 298
				  mtrr_tom2, mtrr_tom2>>20);
	}
}

L
Linus Torvalds 已提交
299
/*  Grab all of the MTRR state for this CPU into *state  */
300
void __init get_mtrr_state(void)
L
Linus Torvalds 已提交
301 302 303 304
{
	unsigned int i;
	struct mtrr_var_range *vrs;
	unsigned lo, dummy;
305
	unsigned long flags;
L
Linus Torvalds 已提交
306 307 308

	vrs = mtrr_state.var_ranges;

309
	rdmsr(MSR_MTRRcap, lo, dummy);
J
Jan Beulich 已提交
310 311
	mtrr_state.have_fixed = (lo >> 8) & 1;

L
Linus Torvalds 已提交
312 313
	for (i = 0; i < num_var_ranges; i++)
		get_mtrr_var_range(i, &vrs[i]);
J
Jan Beulich 已提交
314 315
	if (mtrr_state.have_fixed)
		get_fixed_ranges(mtrr_state.fixed_ranges);
L
Linus Torvalds 已提交
316 317 318 319

	rdmsr(MTRRdefType_MSR, lo, dummy);
	mtrr_state.def_type = (lo & 0xff);
	mtrr_state.enabled = (lo & 0xc00) >> 10;
J
Jan Beulich 已提交
320

321
	if (amd_special_default_mtrr()) {
322
		unsigned low, high;
323
		/* TOP_MEM2 */
324
		rdmsr(MSR_K8_TOP_MEM2, low, high);
325 326 327
		mtrr_tom2 = high;
		mtrr_tom2 <<= 32;
		mtrr_tom2 |= low;
Y
Yinghai Lu 已提交
328
		mtrr_tom2 &= 0xffffff800000ULL;
329
	}
Y
Yinghai Lu 已提交
330 331 332

	print_mtrr_state();

333 334 335 336 337 338 339 340 341 342 343
	mtrr_state_set = 1;

	/* PAT setup for BP. We need to go through sync steps here */
	local_irq_save(flags);
	prepare_set();

	pat_init();

	post_set();
	local_irq_restore(flags);

L
Linus Torvalds 已提交
344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373
}

/*  Some BIOS's are fucked and don't set all MTRRs the same!  */
void __init mtrr_state_warn(void)
{
	unsigned long mask = smp_changes_mask;

	if (!mask)
		return;
	if (mask & MTRR_CHANGE_MASK_FIXED)
		printk(KERN_WARNING "mtrr: your CPUs had inconsistent fixed MTRR settings\n");
	if (mask & MTRR_CHANGE_MASK_VARIABLE)
		printk(KERN_WARNING "mtrr: your CPUs had inconsistent variable MTRR settings\n");
	if (mask & MTRR_CHANGE_MASK_DEFTYPE)
		printk(KERN_WARNING "mtrr: your CPUs had inconsistent MTRRdefType settings\n");
	printk(KERN_INFO "mtrr: probably your BIOS does not setup all CPUs.\n");
	printk(KERN_INFO "mtrr: corrected configuration.\n");
}

/* Doesn't attempt to pass an error out to MTRR users
   because it's quite complicated in some cases and probably not
   worth it because the best error handling is to ignore it. */
void mtrr_wrmsr(unsigned msr, unsigned a, unsigned b)
{
	if (wrmsr_safe(msr, a, b) < 0)
		printk(KERN_ERR
			"MTRR: CPU %u: Writing MSR %x to %x:%x failed\n",
			smp_processor_id(), msr, a, b);
}

374
/**
375 376 377 378
 * set_fixed_range - checks & updates a fixed-range MTRR if it differs from the value it should have
 * @msr: MSR address of the MTTR which should be checked and updated
 * @changed: pointer which indicates whether the MTRR needed to be changed
 * @msrwords: pointer to the MSR values which the MSR should have
379
 */
380
static void set_fixed_range(int msr, bool *changed, unsigned int *msrwords)
381 382 383 384 385 386 387
{
	unsigned lo, hi;

	rdmsr(msr, lo, hi);

	if (lo != msrwords[0] || hi != msrwords[1]) {
		mtrr_wrmsr(msr, msrwords[0], msrwords[1]);
388
		*changed = true;
389 390 391
	}
}

392 393 394 395 396 397 398 399
/**
 * generic_get_free_region - Get a free MTRR.
 * @base: The starting (base) address of the region.
 * @size: The size (in bytes) of the region.
 * @replace_reg: mtrr index to be replaced; set to invalid value if none.
 *
 * Returns: The index of the region on success, else negative on error.
 */
J
Jan Beulich 已提交
400
int generic_get_free_region(unsigned long base, unsigned long size, int replace_reg)
L
Linus Torvalds 已提交
401 402 403
{
	int i, max;
	mtrr_type ltype;
J
Jan Beulich 已提交
404
	unsigned long lbase, lsize;
L
Linus Torvalds 已提交
405 406

	max = num_var_ranges;
J
Jan Beulich 已提交
407 408
	if (replace_reg >= 0 && replace_reg < max)
		return replace_reg;
L
Linus Torvalds 已提交
409 410 411 412 413 414 415 416
	for (i = 0; i < max; ++i) {
		mtrr_if->get(i, &lbase, &lsize, &ltype);
		if (lsize == 0)
			return i;
	}
	return -ENOSPC;
}

A
Adrian Bunk 已提交
417
static void generic_get_mtrr(unsigned int reg, unsigned long *base,
J
Jan Beulich 已提交
418
			     unsigned long *size, mtrr_type *type)
L
Linus Torvalds 已提交
419 420
{
	unsigned int mask_lo, mask_hi, base_lo, base_hi;
Y
Yinghai Lu 已提交
421
	unsigned int tmp, hi;
422
	int cpu;
L
Linus Torvalds 已提交
423

Y
Yinghai Lu 已提交
424 425 426 427
	/*
	 * get_mtrr doesn't need to update mtrr_state, also it could be called
	 * from any cpu, so try to print it out directly.
	 */
428 429
	cpu = get_cpu();

L
Linus Torvalds 已提交
430
	rdmsr(MTRRphysMask_MSR(reg), mask_lo, mask_hi);
Y
Yinghai Lu 已提交
431

L
Linus Torvalds 已提交
432 433 434 435 436
	if ((mask_lo & 0x800) == 0) {
		/*  Invalid (i.e. free) range  */
		*base = 0;
		*size = 0;
		*type = 0;
437
		goto out_put_cpu;
L
Linus Torvalds 已提交
438 439 440 441
	}

	rdmsr(MTRRphysBase_MSR(reg), base_lo, base_hi);

442
	/* Work out the shifted address mask: */
Y
Yinghai Lu 已提交
443 444
	tmp = mask_hi << (32 - PAGE_SHIFT) | mask_lo >> PAGE_SHIFT;
	mask_lo = size_or_mask | tmp;
445 446

	/* Expand tmp with high bits to all 1s: */
Y
Yinghai Lu 已提交
447 448 449 450 451
	hi = fls(tmp);
	if (hi > 0) {
		tmp |= ~((1<<(hi - 1)) - 1);

		if (tmp != mask_lo) {
452
			WARN_ONCE(1, KERN_INFO "mtrr: your BIOS has set up an incorrect mask, fixing it up.\n");
Y
Yinghai Lu 已提交
453 454 455
			mask_lo = tmp;
		}
	}
L
Linus Torvalds 已提交
456

457 458 459 460
	/*
	 * This works correctly if size is a power of two, i.e. a
	 * contiguous range:
	 */
L
Linus Torvalds 已提交
461 462 463
	*size = -mask_lo;
	*base = base_hi << (32 - PAGE_SHIFT) | base_lo >> PAGE_SHIFT;
	*type = base_lo & 0xff;
Y
Yinghai Lu 已提交
464

465 466
out_put_cpu:
	put_cpu();
L
Linus Torvalds 已提交
467 468
}

469
/**
470 471
 * set_fixed_ranges - checks & updates the fixed-range MTRRs if they differ from the saved set
 * @frs: pointer to fixed-range MTRR values, saved by get_fixed_ranges()
472
 */
L
Linus Torvalds 已提交
473 474
static int set_fixed_ranges(mtrr_type * frs)
{
475
	unsigned long long *saved = (unsigned long long *) frs;
476
	bool changed = false;
477
	int block=-1, range;
L
Linus Torvalds 已提交
478

479 480
	k8_check_syscfg_dram_mod_en();

481 482 483 484
	while (fixed_range_blocks[++block].ranges)
	    for (range=0; range < fixed_range_blocks[block].ranges; range++)
		set_fixed_range(fixed_range_blocks[block].base_msr + range,
		    &changed, (unsigned int *) saved++);
L
Linus Torvalds 已提交
485 486 487 488 489 490

	return changed;
}

/*  Set the MSR pair relating to a var range. Returns TRUE if
    changes are made  */
491
static bool set_mtrr_var_ranges(unsigned int index, struct mtrr_var_range *vr)
L
Linus Torvalds 已提交
492 493
{
	unsigned int lo, hi;
494
	bool changed = false;
L
Linus Torvalds 已提交
495 496 497

	rdmsr(MTRRphysBase_MSR(index), lo, hi);
	if ((vr->base_lo & 0xfffff0ffUL) != (lo & 0xfffff0ffUL)
498 499
	    || (vr->base_hi & (size_and_mask >> (32 - PAGE_SHIFT))) !=
		(hi & (size_and_mask >> (32 - PAGE_SHIFT)))) {
L
Linus Torvalds 已提交
500
		mtrr_wrmsr(MTRRphysBase_MSR(index), vr->base_lo, vr->base_hi);
501
		changed = true;
L
Linus Torvalds 已提交
502 503 504 505 506
	}

	rdmsr(MTRRphysMask_MSR(index), lo, hi);

	if ((vr->mask_lo & 0xfffff800UL) != (lo & 0xfffff800UL)
507 508
	    || (vr->mask_hi & (size_and_mask >> (32 - PAGE_SHIFT))) !=
		(hi & (size_and_mask >> (32 - PAGE_SHIFT)))) {
L
Linus Torvalds 已提交
509
		mtrr_wrmsr(MTRRphysMask_MSR(index), vr->mask_lo, vr->mask_hi);
510
		changed = true;
L
Linus Torvalds 已提交
511 512 513 514
	}
	return changed;
}

J
Jan Beulich 已提交
515 516
static u32 deftype_lo, deftype_hi;

517 518 519 520 521 522
/**
 * set_mtrr_state - Set the MTRR state for this CPU.
 *
 * NOTE: The CPU must already be in a safe state for MTRR changes.
 * RETURNS: 0 if no changes made, else a mask indicating what was changed.
 */
J
Jan Beulich 已提交
523
static unsigned long set_mtrr_state(void)
L
Linus Torvalds 已提交
524 525 526 527 528 529 530 531
{
	unsigned int i;
	unsigned long change_mask = 0;

	for (i = 0; i < num_var_ranges; i++)
		if (set_mtrr_var_ranges(i, &mtrr_state.var_ranges[i]))
			change_mask |= MTRR_CHANGE_MASK_VARIABLE;

J
Jan Beulich 已提交
532
	if (mtrr_state.have_fixed && set_fixed_ranges(mtrr_state.fixed_ranges))
L
Linus Torvalds 已提交
533 534 535 536 537 538
		change_mask |= MTRR_CHANGE_MASK_FIXED;

	/*  Set_mtrr_restore restores the old value of MTRRdefType,
	   so to set it we fiddle with the saved value  */
	if ((deftype_lo & 0xff) != mtrr_state.def_type
	    || ((deftype_lo & 0xc00) >> 10) != mtrr_state.enabled) {
J
Jan Beulich 已提交
539
		deftype_lo = (deftype_lo & ~0xcff) | mtrr_state.def_type | (mtrr_state.enabled << 10);
L
Linus Torvalds 已提交
540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556
		change_mask |= MTRR_CHANGE_MASK_DEFTYPE;
	}

	return change_mask;
}


static unsigned long cr4 = 0;
static DEFINE_SPINLOCK(set_atomicity_lock);

/*
 * Since we are disabling the cache don't allow any interrupts - they
 * would run extremely slow and would only increase the pain.  The caller must
 * ensure that local interrupts are disabled and are reenabled after post_set()
 * has been called.
 */

557
static void prepare_set(void) __acquires(set_atomicity_lock)
L
Linus Torvalds 已提交
558 559 560 561 562 563 564 565 566 567
{
	unsigned long cr0;

	/*  Note that this is not ideal, since the cache is only flushed/disabled
	   for this CPU while the MTRRs are changed, but changing this requires
	   more invasive changes to the way the kernel boots  */

	spin_lock(&set_atomicity_lock);

	/*  Enter the no-fill (CD=1, NW=0) cache mode and flush caches. */
D
Dave Jones 已提交
568
	cr0 = read_cr0() | X86_CR0_CD;
L
Linus Torvalds 已提交
569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584
	write_cr0(cr0);
	wbinvd();

	/*  Save value of CR4 and clear Page Global Enable (bit 7)  */
	if ( cpu_has_pge ) {
		cr4 = read_cr4();
		write_cr4(cr4 & ~X86_CR4_PGE);
	}

	/* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */
	__flush_tlb();

	/*  Save MTRR state */
	rdmsr(MTRRdefType_MSR, deftype_lo, deftype_hi);

	/*  Disable MTRRs, and set the default type to uncached  */
J
Jan Beulich 已提交
585
	mtrr_wrmsr(MTRRdefType_MSR, deftype_lo & ~0xcff, deftype_hi);
L
Linus Torvalds 已提交
586 587
}

588
static void post_set(void) __releases(set_atomicity_lock)
L
Linus Torvalds 已提交
589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613
{
	/*  Flush TLBs (no need to flush caches - they are disabled)  */
	__flush_tlb();

	/* Intel (P6) standard MTRRs */
	mtrr_wrmsr(MTRRdefType_MSR, deftype_lo, deftype_hi);
		
	/*  Enable caches  */
	write_cr0(read_cr0() & 0xbfffffff);

	/*  Restore value of CR4  */
	if ( cpu_has_pge )
		write_cr4(cr4);
	spin_unlock(&set_atomicity_lock);
}

static void generic_set_all(void)
{
	unsigned long mask, count;
	unsigned long flags;

	local_irq_save(flags);
	prepare_set();

	/* Actually set the state */
J
Jan Beulich 已提交
614
	mask = set_mtrr_state();
L
Linus Torvalds 已提交
615

616 617 618
	/* also set PAT */
	pat_init();

L
Linus Torvalds 已提交
619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641
	post_set();
	local_irq_restore(flags);

	/*  Use the atomic bitops to update the global mask  */
	for (count = 0; count < sizeof mask * 8; ++count) {
		if (mask & 0x01)
			set_bit(count, &smp_changes_mask);
		mask >>= 1;
	}
	
}

static void generic_set_mtrr(unsigned int reg, unsigned long base,
			     unsigned long size, mtrr_type type)
/*  [SUMMARY] Set variable MTRR register on the local CPU.
    <reg> The register to set.
    <base> The base address of the region.
    <size> The size of the region. If this is 0 the region is disabled.
    <type> The type of the region.
    [RETURNS] Nothing.
*/
{
	unsigned long flags;
S
Shaohua Li 已提交
642 643 644
	struct mtrr_var_range *vr;

	vr = &mtrr_state.var_ranges[reg];
L
Linus Torvalds 已提交
645 646 647 648 649 650 651 652

	local_irq_save(flags);
	prepare_set();

	if (size == 0) {
		/* The invalid bit is kept in the mask, so we simply clear the
		   relevant mask register to disable a range. */
		mtrr_wrmsr(MTRRphysMask_MSR(reg), 0, 0);
S
Shaohua Li 已提交
653
		memset(vr, 0, sizeof(struct mtrr_var_range));
L
Linus Torvalds 已提交
654
	} else {
S
Shaohua Li 已提交
655 656 657 658 659 660 661
		vr->base_lo = base << PAGE_SHIFT | type;
		vr->base_hi = (base & size_and_mask) >> (32 - PAGE_SHIFT);
		vr->mask_lo = -size << PAGE_SHIFT | 0x800;
		vr->mask_hi = (-size & size_and_mask) >> (32 - PAGE_SHIFT);

		mtrr_wrmsr(MTRRphysBase_MSR(reg), vr->base_lo, vr->base_hi);
		mtrr_wrmsr(MTRRphysMask_MSR(reg), vr->mask_lo, vr->mask_hi);
L
Linus Torvalds 已提交
662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680
	}

	post_set();
	local_irq_restore(flags);
}

int generic_validate_add_page(unsigned long base, unsigned long size, unsigned int type)
{
	unsigned long lbase, last;

	/*  For Intel PPro stepping <= 7, must be 4 MiB aligned 
	    and not touch 0x70000000->0x7003FFFF */
	if (is_cpu(INTEL) && boot_cpu_data.x86 == 6 &&
	    boot_cpu_data.x86_model == 1 &&
	    boot_cpu_data.x86_mask <= 7) {
		if (base & ((1 << (22 - PAGE_SHIFT)) - 1)) {
			printk(KERN_WARNING "mtrr: base(0x%lx000) is not 4 MiB aligned\n", base);
			return -EINVAL;
		}
681
		if (!(base + size < 0x70000 || base > 0x7003F) &&
L
Linus Torvalds 已提交
682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705
		    (type == MTRR_TYPE_WRCOMB
		     || type == MTRR_TYPE_WRBACK)) {
			printk(KERN_WARNING "mtrr: writable mtrr between 0x70000000 and 0x7003FFFF may hang the CPU.\n");
			return -EINVAL;
		}
	}

	/*  Check upper bits of base and last are equal and lower bits are 0
	    for base and 1 for last  */
	last = base + size - 1;
	for (lbase = base; !(lbase & 1) && (last & 1);
	     lbase = lbase >> 1, last = last >> 1) ;
	if (lbase != last) {
		printk(KERN_WARNING "mtrr: base(0x%lx000) is not aligned on a size(0x%lx000) boundary\n",
		       base, size);
		return -EINVAL;
	}
	return 0;
}


static int generic_have_wrcomb(void)
{
	unsigned long config, dummy;
706
	rdmsr(MSR_MTRRcap, config, dummy);
L
Linus Torvalds 已提交
707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725
	return (config & (1 << 10));
}

int positive_have_wrcomb(void)
{
	return 1;
}

/* generic structure...
 */
struct mtrr_ops generic_mtrr_ops = {
	.use_intel_if      = 1,
	.set_all	   = generic_set_all,
	.get               = generic_get_mtrr,
	.get_free_region   = generic_get_free_region,
	.set               = generic_set_mtrr,
	.validate_add_page = generic_validate_add_page,
	.have_wrcomb       = generic_have_wrcomb,
};