mce_64.c 26.7 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3
/*
 * Machine check handler.
 * K8 parts Copyright 2002,2003 Andi Kleen, SuSE Labs.
4 5
 * Rest from unknown author(s).
 * 2004 Andi Kleen. Rewrote most of it.
6 7
 * Copyright 2008 Intel Corporation
 * Author: Andi Kleen
L
Linus Torvalds 已提交
8 9 10 11 12 13
 */

#include <linux/init.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/sched.h>
A
Arnd Bergmann 已提交
14
#include <linux/smp_lock.h>
L
Linus Torvalds 已提交
15 16 17 18 19 20
#include <linux/string.h>
#include <linux/rcupdate.h>
#include <linux/kallsyms.h>
#include <linux/sysdev.h>
#include <linux/miscdevice.h>
#include <linux/fs.h>
21
#include <linux/capability.h>
22 23
#include <linux/cpu.h>
#include <linux/percpu.h>
24 25
#include <linux/poll.h>
#include <linux/thread_info.h>
26
#include <linux/ctype.h>
27
#include <linux/kmod.h>
28
#include <linux/kdebug.h>
29 30
#include <linux/kobject.h>
#include <linux/sysfs.h>
31
#include <asm/processor.h>
L
Linus Torvalds 已提交
32 33 34
#include <asm/msr.h>
#include <asm/mce.h>
#include <asm/uaccess.h>
35
#include <asm/smp.h>
36
#include <asm/idle.h>
L
Linus Torvalds 已提交
37 38

#define MISC_MCELOG_MINOR 227
39 40 41 42 43 44

/*
 * To support more than 128 would need to escape the predefined
 * Linux defined extended banks first.
 */
#define MAX_NR_BANKS (MCE_EXTENDED_BANK - 1)
L
Linus Torvalds 已提交
45

46 47
atomic_t mce_entry;

L
Linus Torvalds 已提交
48 49
static int mce_dont_init;

50 51 52 53 54 55 56
/*
 * Tolerant levels:
 *   0: always panic on uncorrected errors, log corrected errors
 *   1: panic or SIGBUS on uncorrected errors, log corrected errors
 *   2: SIGBUS or log uncorrected errors (if possible), log corrected errors
 *   3: never panic or SIGBUS, log all errors (for testing only)
 */
L
Linus Torvalds 已提交
57 58
static int tolerant = 1;
static int banks;
59
static u64 *bank;
60
static unsigned long notify_user;
61
static int rip_msr;
62
static int mce_bootlog = -1;
63 64 65 66
static atomic_t mce_events;

static char trigger[128];
static char *trigger_argv[2] = { trigger, NULL };
L
Linus Torvalds 已提交
67

68 69
static DECLARE_WAIT_QUEUE_HEAD(mce_wait);

70 71 72 73 74 75 76 77
/* Do initial initialization of a struct mce */
void mce_setup(struct mce *m)
{
	memset(m, 0, sizeof(struct mce));
	m->cpu = smp_processor_id();
	rdtscll(m->tsc);
}

L
Linus Torvalds 已提交
78 79 80 81 82 83
/*
 * Lockless MCE logging infrastructure.
 * This avoids deadlocks on printk locks without having to break locks. Also
 * separate MCEs from kernel messages to avoid bogus bug reports.
 */

84
static struct mce_log mcelog = {
L
Linus Torvalds 已提交
85 86
	MCE_LOG_SIGNATURE,
	MCE_LOG_LEN,
87
};
L
Linus Torvalds 已提交
88 89 90 91

void mce_log(struct mce *mce)
{
	unsigned next, entry;
92
	atomic_inc(&mce_events);
L
Linus Torvalds 已提交
93
	mce->finished = 0;
M
Mike Waychison 已提交
94
	wmb();
L
Linus Torvalds 已提交
95 96
	for (;;) {
		entry = rcu_dereference(mcelog.next);
97 98 99 100
		for (;;) {
			/* When the buffer fills up discard new entries. Assume
			   that the earlier errors are the more interesting. */
			if (entry >= MCE_LOG_LEN) {
101
				set_bit(MCE_OVERFLOW, (unsigned long *)&mcelog.flags);
102 103 104 105 106 107 108
				return;
			}
			/* Old left over entry. Skip. */
			if (mcelog.entry[entry].finished) {
				entry++;
				continue;
			}
M
Mike Waychison 已提交
109
			break;
L
Linus Torvalds 已提交
110 111 112 113 114 115 116
		}
		smp_rmb();
		next = entry + 1;
		if (cmpxchg(&mcelog.next, entry, next) == entry)
			break;
	}
	memcpy(mcelog.entry + entry, mce, sizeof(struct mce));
M
Mike Waychison 已提交
117
	wmb();
L
Linus Torvalds 已提交
118
	mcelog.entry[entry].finished = 1;
M
Mike Waychison 已提交
119
	wmb();
L
Linus Torvalds 已提交
120

121
	set_bit(0, &notify_user);
L
Linus Torvalds 已提交
122 123 124 125 126
}

static void print_mce(struct mce *m)
{
	printk(KERN_EMERG "\n"
127
	       KERN_EMERG "HARDWARE ERROR\n"
L
Linus Torvalds 已提交
128 129 130
	       KERN_EMERG
	       "CPU %d: Machine Check Exception: %16Lx Bank %d: %016Lx\n",
	       m->cpu, m->mcgstatus, m->bank, m->status);
131
	if (m->ip) {
132
		printk(KERN_EMERG "RIP%s %02x:<%016Lx> ",
L
Linus Torvalds 已提交
133
		       !(m->mcgstatus & MCG_STATUS_EIPV) ? " !INEXACT!" : "",
134
		       m->cs, m->ip);
L
Linus Torvalds 已提交
135
		if (m->cs == __KERNEL_CS)
136
			print_symbol("{%s}", m->ip);
L
Linus Torvalds 已提交
137 138
		printk("\n");
	}
139
	printk(KERN_EMERG "TSC %Lx ", m->tsc);
L
Linus Torvalds 已提交
140 141 142
	if (m->addr)
		printk("ADDR %Lx ", m->addr);
	if (m->misc)
143
		printk("MISC %Lx ", m->misc);
L
Linus Torvalds 已提交
144
	printk("\n");
145
	printk(KERN_EMERG "This is not a software problem!\n");
146 147
	printk(KERN_EMERG "Run through mcelog --ascii to decode "
	       "and contact your hardware vendor\n");
L
Linus Torvalds 已提交
148 149 150
}

static void mce_panic(char *msg, struct mce *backup, unsigned long start)
151
{
L
Linus Torvalds 已提交
152
	int i;
153

L
Linus Torvalds 已提交
154 155 156
	oops_begin();
	for (i = 0; i < MCE_LOG_LEN; i++) {
		unsigned long tsc = mcelog.entry[i].tsc;
157

L
Linus Torvalds 已提交
158 159
		if (time_before(tsc, start))
			continue;
160
		print_mce(&mcelog.entry[i]);
L
Linus Torvalds 已提交
161 162 163 164 165
		if (backup && mcelog.entry[i].tsc == backup->tsc)
			backup = NULL;
	}
	if (backup)
		print_mce(backup);
166
	panic(msg);
167
}
L
Linus Torvalds 已提交
168 169 170

static int mce_available(struct cpuinfo_x86 *c)
{
171 172
	if (mce_dont_init)
		return 0;
173
	return cpu_has(c, X86_FEATURE_MCE) && cpu_has(c, X86_FEATURE_MCA);
L
Linus Torvalds 已提交
174 175
}

176 177 178
static inline void mce_get_rip(struct mce *m, struct pt_regs *regs)
{
	if (regs && (m->mcgstatus & MCG_STATUS_RIPV)) {
179
		m->ip = regs->ip;
180 181
		m->cs = regs->cs;
	} else {
182
		m->ip = 0;
183 184 185 186 187
		m->cs = 0;
	}
	if (rip_msr) {
		/* Assume the RIP in the MSR is exact. Is this true? */
		m->mcgstatus |= MCG_STATUS_EIPV;
188
		rdmsrl(rip_msr, m->ip);
189 190 191 192
		m->cs = 0;
	}
}

193
/*
194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264
 * Poll for corrected events or events that happened before reset.
 * Those are just logged through /dev/mcelog.
 *
 * This is executed in standard interrupt context.
 */
void machine_check_poll(enum mcp_flags flags)
{
	struct mce m;
	int i;

	mce_setup(&m);

	rdmsrl(MSR_IA32_MCG_STATUS, m.mcgstatus);
	for (i = 0; i < banks; i++) {
		if (!bank[i])
			continue;

		m.misc = 0;
		m.addr = 0;
		m.bank = i;
		m.tsc = 0;

		barrier();
		rdmsrl(MSR_IA32_MC0_STATUS + i*4, m.status);
		if (!(m.status & MCI_STATUS_VAL))
			continue;

		/*
		 * Uncorrected events are handled by the exception handler
		 * when it is enabled. But when the exception is disabled log
		 * everything.
		 *
		 * TBD do the same check for MCI_STATUS_EN here?
		 */
		if ((m.status & MCI_STATUS_UC) && !(flags & MCP_UC))
			continue;

		if (m.status & MCI_STATUS_MISCV)
			rdmsrl(MSR_IA32_MC0_MISC + i*4, m.misc);
		if (m.status & MCI_STATUS_ADDRV)
			rdmsrl(MSR_IA32_MC0_ADDR + i*4, m.addr);

		if (!(flags & MCP_TIMESTAMP))
			m.tsc = 0;
		/*
		 * Don't get the IP here because it's unlikely to
		 * have anything to do with the actual error location.
		 */

		mce_log(&m);
		add_taint(TAINT_MACHINE_CHECK);

		/*
		 * Clear state for this bank.
		 */
		wrmsrl(MSR_IA32_MC0_STATUS+4*i, 0);
	}

	/*
	 * Don't clear MCG_STATUS here because it's only defined for
	 * exceptions.
	 */
}

/*
 * The actual machine check handler. This only handles real
 * exceptions when something got corrupted coming in through int 18.
 *
 * This is executed in NMI context not subject to normal locking rules. This
 * implies that most kernel services cannot be safely used. Don't even
 * think about putting a printk in there!
L
Linus Torvalds 已提交
265 266 267 268 269 270 271
 */
void do_machine_check(struct pt_regs * regs, long error_code)
{
	struct mce m, panicm;
	u64 mcestart = 0;
	int i;
	int panicm_found = 0;
272 273 274 275 276 277 278 279 280 281
	/*
	 * If no_way_out gets set, there is no safe way to recover from this
	 * MCE.  If tolerant is cranked up, we'll try anyway.
	 */
	int no_way_out = 0;
	/*
	 * If kill_it gets set, there might be a way to recover from this
	 * error.
	 */
	int kill_it = 0;
282
	DECLARE_BITMAP(toclear, MAX_NR_BANKS);
L
Linus Torvalds 已提交
283

284 285
	atomic_inc(&mce_entry);

286
	if (notify_die(DIE_NMI, "machine check", regs, error_code,
287
			   18, SIGKILL) == NOTIFY_STOP)
288 289
		goto out2;
	if (!banks)
290
		goto out2;
L
Linus Torvalds 已提交
291

292 293
	mce_setup(&m);

L
Linus Torvalds 已提交
294
	rdmsrl(MSR_IA32_MCG_STATUS, m.mcgstatus);
295
	/* if the restart IP is not valid, we're done for */
L
Linus Torvalds 已提交
296
	if (!(m.mcgstatus & MCG_STATUS_RIPV))
297
		no_way_out = 1;
298

L
Linus Torvalds 已提交
299 300 301 302
	rdtscll(mcestart);
	barrier();

	for (i = 0; i < banks; i++) {
303
		__clear_bit(i, toclear);
304
		if (!bank[i])
L
Linus Torvalds 已提交
305
			continue;
306 307

		m.misc = 0;
L
Linus Torvalds 已提交
308 309 310 311 312 313 314
		m.addr = 0;
		m.bank = i;

		rdmsrl(MSR_IA32_MC0_STATUS + i*4, m.status);
		if ((m.status & MCI_STATUS_VAL) == 0)
			continue;

315 316 317 318 319 320 321 322 323 324 325 326 327 328
		/*
		 * Non uncorrected errors are handled by machine_check_poll
		 * Leave them alone.
		 */
		if ((m.status & MCI_STATUS_UC) == 0)
			continue;

		/*
		 * Set taint even when machine check was not enabled.
		 */
		add_taint(TAINT_MACHINE_CHECK);

		__set_bit(i, toclear);

L
Linus Torvalds 已提交
329
		if (m.status & MCI_STATUS_EN) {
330 331 332 333 334 335 336 337 338 339 340 341
			/* if PCC was set, there's no way out */
			no_way_out |= !!(m.status & MCI_STATUS_PCC);
			/*
			 * If this error was uncorrectable and there was
			 * an overflow, we're in trouble.  If no overflow,
			 * we might get away with just killing a task.
			 */
			if (m.status & MCI_STATUS_UC) {
				if (tolerant < 1 || m.status & MCI_STATUS_OVER)
					no_way_out = 1;
				kill_it = 1;
			}
342 343 344 345 346 347
		} else {
			/*
			 * Machine check event was not enabled. Clear, but
			 * ignore.
			 */
			continue;
L
Linus Torvalds 已提交
348 349 350 351 352 353 354
		}

		if (m.status & MCI_STATUS_MISCV)
			rdmsrl(MSR_IA32_MC0_MISC + i*4, m.misc);
		if (m.status & MCI_STATUS_ADDRV)
			rdmsrl(MSR_IA32_MC0_ADDR + i*4, m.addr);

355
		mce_get_rip(&m, regs);
356
		mce_log(&m);
L
Linus Torvalds 已提交
357 358 359 360 361 362 363 364 365 366 367 368 369 370

		/* Did this bank cause the exception? */
		/* Assume that the bank with uncorrectable errors did it,
		   and that there is only a single one. */
		if ((m.status & MCI_STATUS_UC) && (m.status & MCI_STATUS_EN)) {
			panicm = m;
			panicm_found = 1;
		}
	}

	/* If we didn't find an uncorrectable error, pick
	   the last one (shouldn't happen, just being safe). */
	if (!panicm_found)
		panicm = m;
371 372 373 374 375 376

	/*
	 * If we have decided that we just CAN'T continue, and the user
	 *  has not set tolerant to an insane level, give up and die.
	 */
	if (no_way_out && tolerant < 3)
L
Linus Torvalds 已提交
377
		mce_panic("Machine check", &panicm, mcestart);
378 379 380 381 382 383 384 385

	/*
	 * If the error seems to be unrecoverable, something should be
	 * done.  Try to kill as little as possible.  If we can kill just
	 * one task, do that.  If the user has set the tolerance very
	 * high, don't try to do anything at all.
	 */
	if (kill_it && tolerant < 3) {
L
Linus Torvalds 已提交
386 387
		int user_space = 0;

388 389 390 391 392
		/*
		 * If the EIPV bit is set, it means the saved IP is the
		 * instruction which caused the MCE.
		 */
		if (m.mcgstatus & MCG_STATUS_EIPV)
393
			user_space = panicm.ip && (panicm.cs & 3);
394 395 396 397 398

		/*
		 * If we know that the error was in user space, send a
		 * SIGBUS.  Otherwise, panic if tolerance is low.
		 *
399
		 * force_sig() takes an awful lot of locks and has a slight
400 401 402
		 * risk of deadlocking.
		 */
		if (user_space) {
403
			force_sig(SIGBUS, current);
404 405 406 407
		} else if (panic_on_oops || tolerant < 2) {
			mce_panic("Uncorrected machine check",
				&panicm, mcestart);
		}
L
Linus Torvalds 已提交
408 409
	}

410 411 412
	/* notify userspace ASAP */
	set_thread_flag(TIF_MCE_NOTIFY);

413
	/* the last thing we do is clear state */
414 415 416 417
	for (i = 0; i < banks; i++) {
		if (test_bit(i, toclear))
			wrmsrl(MSR_IA32_MC0_STATUS+4*i, 0);
	}
L
Linus Torvalds 已提交
418
	wrmsrl(MSR_IA32_MCG_STATUS, 0);
419 420
 out2:
	atomic_dec(&mce_entry);
L
Linus Torvalds 已提交
421 422
}

423 424 425
#ifdef CONFIG_X86_MCE_INTEL
/***
 * mce_log_therm_throt_event - Logs the thermal throttling event to mcelog
S
Simon Arlott 已提交
426
 * @cpu: The CPU on which the event occurred.
427 428 429 430 431 432 433 434 435 436
 * @status: Event status information
 *
 * This function should be called by the thermal interrupt after the
 * event has been processed and the decision was made to log the event
 * further.
 *
 * The status parameter will be saved to the 'status' field of 'struct mce'
 * and historically has been the register value of the
 * MSR_IA32_THERMAL_STATUS (Intel) msr.
 */
437
void mce_log_therm_throt_event(__u64 status)
438 439 440
{
	struct mce m;

441
	mce_setup(&m);
442 443 444 445 446 447
	m.bank = MCE_THERMAL_BANK;
	m.status = status;
	mce_log(&m);
}
#endif /* CONFIG_X86_MCE_INTEL */

L
Linus Torvalds 已提交
448
/*
449 450 451
 * Periodic polling timer for "silent" machine check errors.  If the
 * poller finds an MCE, poll 2x faster.  When the poller finds no more
 * errors, poll 2x slower (up to check_interval seconds).
L
Linus Torvalds 已提交
452 453 454
 */

static int check_interval = 5 * 60; /* 5 minutes */
455
static int next_interval; /* in jiffies */
456 457
static void mcheck_timer(unsigned long);
static DEFINE_PER_CPU(struct timer_list, mce_timer);
L
Linus Torvalds 已提交
458

459
static void mcheck_timer(unsigned long data)
L
Linus Torvalds 已提交
460
{
461 462 463 464
	struct timer_list *t = &per_cpu(mce_timer, data);

	WARN_ON(smp_processor_id() != data);

L
Linus Torvalds 已提交
465
	if (mce_available(&current_cpu_data))
466
		machine_check_poll(MCP_TIMESTAMP);
L
Linus Torvalds 已提交
467 468

	/*
469 470
	 * Alert userspace if needed.  If we logged an MCE, reduce the
	 * polling interval, otherwise increase the polling interval.
L
Linus Torvalds 已提交
471
	 */
472 473 474
	if (mce_notify_user()) {
		next_interval = max(next_interval/2, HZ/100);
	} else {
475
		next_interval = min(next_interval * 2,
476
				(int)round_jiffies_relative(check_interval*HZ));
477 478
	}

479 480
	t->expires = jiffies + next_interval;
	add_timer(t);
481 482
}

483 484 485 486 487 488 489
static void mce_do_trigger(struct work_struct *work)
{
	call_usermodehelper(trigger, trigger_argv, NULL, UMH_NO_WAIT);
}

static DECLARE_WORK(mce_trigger_work, mce_do_trigger);

490
/*
491 492 493
 * Notify the user(s) about new machine check events.
 * Can be called from interrupt context, but not from machine check/NMI
 * context.
494 495 496 497 498
 */
int mce_notify_user(void)
{
	clear_thread_flag(TIF_MCE_NOTIFY);
	if (test_and_clear_bit(0, &notify_user)) {
499 500 501
		static unsigned long last_print;
		unsigned long now = jiffies;

502
		wake_up_interruptible(&mce_wait);
503 504 505 506 507 508 509 510

		/*
		 * There is no risk of missing notifications because
		 * work_pending is always cleared before the function is
		 * executed.
		 */
		if (trigger[0] && !work_pending(&mce_trigger_work))
			schedule_work(&mce_trigger_work);
511

512 513 514 515
		if (time_after_eq(now, last_print + (check_interval*HZ))) {
			last_print = now;
			printk(KERN_INFO "Machine check events logged\n");
		}
516 517

		return 1;
L
Linus Torvalds 已提交
518
	}
519 520
	return 0;
}
521

522 523 524 525 526 527 528 529 530
/* see if the idle task needs to notify userspace */
static int
mce_idle_callback(struct notifier_block *nfb, unsigned long action, void *junk)
{
	/* IDLE_END should be safe - interrupts are back on */
	if (action == IDLE_END && test_thread_flag(TIF_MCE_NOTIFY))
		mce_notify_user();

	return NOTIFY_OK;
L
Linus Torvalds 已提交
531 532
}

533 534 535
static struct notifier_block mce_idle_notifier = {
	.notifier_call = mce_idle_callback,
};
L
Linus Torvalds 已提交
536 537

static __init int periodic_mcheck_init(void)
538
{
539 540
       idle_notifier_register(&mce_idle_notifier);
       return 0;
541
}
L
Linus Torvalds 已提交
542 543
__initcall(periodic_mcheck_init);

544
/*
L
Linus Torvalds 已提交
545 546
 * Initialize Machine Checks for a CPU.
 */
547
static int mce_cap_init(void)
L
Linus Torvalds 已提交
548 549
{
	u64 cap;
550
	unsigned b;
L
Linus Torvalds 已提交
551 552

	rdmsrl(MSR_IA32_MCG_CAP, cap);
553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568
	b = cap & 0xff;
	if (b > MAX_NR_BANKS) {
		printk(KERN_WARNING
		       "MCE: Using only %u machine check banks out of %u\n",
			MAX_NR_BANKS, b);
		b = MAX_NR_BANKS;
	}

	/* Don't support asymmetric configurations today */
	WARN_ON(banks != 0 && b != banks);
	banks = b;
	if (!bank) {
		bank = kmalloc(banks * sizeof(u64), GFP_KERNEL);
		if (!bank)
			return -ENOMEM;
		memset(bank, 0xff, banks * sizeof(u64));
L
Linus Torvalds 已提交
569
	}
570

571 572 573
	/* Use accurate RIP reporting if available. */
	if ((cap & (1<<9)) && ((cap >> 16) & 0xff) >= 9)
		rip_msr = MSR_IA32_MCG_EIP;
L
Linus Torvalds 已提交
574

575 576 577 578 579 580 581 582
	return 0;
}

static void mce_init(void *dummy)
{
	u64 cap;
	int i;

583 584 585 586
	/*
	 * Log the machine checks left over from the previous reset.
	 */
	machine_check_poll(MCP_UC);
L
Linus Torvalds 已提交
587 588 589

	set_in_cr4(X86_CR4_MCE);

590
	rdmsrl(MSR_IA32_MCG_CAP, cap);
L
Linus Torvalds 已提交
591 592 593 594
	if (cap & MCG_CTL_P)
		wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff);

	for (i = 0; i < banks; i++) {
595
		wrmsrl(MSR_IA32_MC0_CTL+4*i, bank[i]);
L
Linus Torvalds 已提交
596
		wrmsrl(MSR_IA32_MC0_STATUS+4*i, 0);
597
	}
L
Linus Torvalds 已提交
598 599 600
}

/* Add per CPU specific workarounds here */
601
static void __cpuinit mce_cpu_quirks(struct cpuinfo_x86 *c)
602
{
L
Linus Torvalds 已提交
603
	/* This should be disabled by the BIOS, but isn't always */
604
	if (c->x86_vendor == X86_VENDOR_AMD) {
605
		if (c->x86 == 15 && banks > 4)
606 607
			/* disable GART TBL walk error reporting, which trips off
			   incorrectly with the IOMMU & 3ware & Cerberus. */
608
			clear_bit(10, (unsigned long *)&bank[4]);
609 610 611 612
		if(c->x86 <= 17 && mce_bootlog < 0)
			/* Lots of broken BIOS around that don't clear them
			   by default and leave crap in there. Don't log. */
			mce_bootlog = 0;
L
Linus Torvalds 已提交
613
	}
614

615
}
L
Linus Torvalds 已提交
616

617
static void __cpuinit mce_cpu_features(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
618 619 620 621 622
{
	switch (c->x86_vendor) {
	case X86_VENDOR_INTEL:
		mce_intel_feature_init(c);
		break;
623 624 625
	case X86_VENDOR_AMD:
		mce_amd_feature_init(c);
		break;
L
Linus Torvalds 已提交
626 627 628 629 630
	default:
		break;
	}
}

631 632 633 634 635 636 637 638 639 640 641 642 643 644
static void mce_init_timer(void)
{
	struct timer_list *t = &__get_cpu_var(mce_timer);

	/* data race harmless because everyone sets to the same value */
	if (!next_interval)
		next_interval = check_interval * HZ;
	if (!next_interval)
		return;
	setup_timer(t, mcheck_timer, smp_processor_id());
	t->expires = round_jiffies_relative(jiffies + next_interval);
	add_timer(t);
}

645
/*
L
Linus Torvalds 已提交
646
 * Called for each booted CPU to set up machine checks.
647
 * Must be called with preempt off.
L
Linus Torvalds 已提交
648
 */
649
void __cpuinit mcheck_init(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
650
{
651
	if (!mce_available(c))
L
Linus Torvalds 已提交
652 653
		return;

654 655 656 657 658 659
	if (mce_cap_init() < 0) {
		mce_dont_init = 1;
		return;
	}
	mce_cpu_quirks(c);

L
Linus Torvalds 已提交
660 661
	mce_init(NULL);
	mce_cpu_features(c);
662
	mce_init_timer();
L
Linus Torvalds 已提交
663 664 665 666 667 668
}

/*
 * Character device to read and clear the MCE log.
 */

T
Tim Hockin 已提交
669 670 671 672 673 674
static DEFINE_SPINLOCK(mce_state_lock);
static int open_count;	/* #times opened */
static int open_exclu;	/* already open exclusive? */

static int mce_open(struct inode *inode, struct file *file)
{
A
Arnd Bergmann 已提交
675
	lock_kernel();
T
Tim Hockin 已提交
676 677 678 679
	spin_lock(&mce_state_lock);

	if (open_exclu || (open_count && (file->f_flags & O_EXCL))) {
		spin_unlock(&mce_state_lock);
A
Arnd Bergmann 已提交
680
		unlock_kernel();
T
Tim Hockin 已提交
681 682 683 684 685 686 687 688
		return -EBUSY;
	}

	if (file->f_flags & O_EXCL)
		open_exclu = 1;
	open_count++;

	spin_unlock(&mce_state_lock);
A
Arnd Bergmann 已提交
689
	unlock_kernel();
T
Tim Hockin 已提交
690

691
	return nonseekable_open(inode, file);
T
Tim Hockin 已提交
692 693 694 695 696 697 698 699 700 701 702 703 704 705
}

static int mce_release(struct inode *inode, struct file *file)
{
	spin_lock(&mce_state_lock);

	open_count--;
	open_exclu = 0;

	spin_unlock(&mce_state_lock);

	return 0;
}

706 707
static void collect_tscs(void *data)
{
L
Linus Torvalds 已提交
708
	unsigned long *cpu_tsc = (unsigned long *)data;
709

L
Linus Torvalds 已提交
710
	rdtscll(cpu_tsc[smp_processor_id()]);
711
}
L
Linus Torvalds 已提交
712

713 714
static ssize_t mce_read(struct file *filp, char __user *ubuf, size_t usize,
			loff_t *off)
L
Linus Torvalds 已提交
715
{
716
	unsigned long *cpu_tsc;
717
	static DEFINE_MUTEX(mce_read_mutex);
718
	unsigned prev, next;
L
Linus Torvalds 已提交
719 720 721
	char __user *buf = ubuf;
	int i, err;

722
	cpu_tsc = kmalloc(nr_cpu_ids * sizeof(long), GFP_KERNEL);
723 724 725
	if (!cpu_tsc)
		return -ENOMEM;

726
	mutex_lock(&mce_read_mutex);
L
Linus Torvalds 已提交
727 728 729
	next = rcu_dereference(mcelog.next);

	/* Only supports full reads right now */
730
	if (*off != 0 || usize < MCE_LOG_LEN*sizeof(struct mce)) {
731
		mutex_unlock(&mce_read_mutex);
732
		kfree(cpu_tsc);
L
Linus Torvalds 已提交
733 734 735 736
		return -EINVAL;
	}

	err = 0;
737 738 739 740 741 742 743 744 745 746 747 748
	prev = 0;
	do {
		for (i = prev; i < next; i++) {
			unsigned long start = jiffies;

			while (!mcelog.entry[i].finished) {
				if (time_after_eq(jiffies, start + 2)) {
					memset(mcelog.entry + i, 0,
					       sizeof(struct mce));
					goto timeout;
				}
				cpu_relax();
749
			}
750 751 752 753 754 755
			smp_rmb();
			err |= copy_to_user(buf, mcelog.entry + i,
					    sizeof(struct mce));
			buf += sizeof(struct mce);
timeout:
			;
756
		}
L
Linus Torvalds 已提交
757

758 759 760 761 762
		memset(mcelog.entry + prev, 0,
		       (next - prev) * sizeof(struct mce));
		prev = next;
		next = cmpxchg(&mcelog.next, prev, 0);
	} while (next != prev);
L
Linus Torvalds 已提交
763

764
	synchronize_sched();
L
Linus Torvalds 已提交
765

766 767 768 769
	/*
	 * Collect entries that were still getting written before the
	 * synchronize.
	 */
770
	on_each_cpu(collect_tscs, cpu_tsc, 1);
771 772 773 774 775
	for (i = next; i < MCE_LOG_LEN; i++) {
		if (mcelog.entry[i].finished &&
		    mcelog.entry[i].tsc < cpu_tsc[mcelog.entry[i].cpu]) {
			err |= copy_to_user(buf, mcelog.entry+i,
					    sizeof(struct mce));
L
Linus Torvalds 已提交
776 777 778 779
			smp_rmb();
			buf += sizeof(struct mce);
			memset(&mcelog.entry[i], 0, sizeof(struct mce));
		}
780
	}
781
	mutex_unlock(&mce_read_mutex);
782
	kfree(cpu_tsc);
783
	return err ? -EFAULT : buf - ubuf;
L
Linus Torvalds 已提交
784 785
}

786 787 788 789 790 791 792 793
static unsigned int mce_poll(struct file *file, poll_table *wait)
{
	poll_wait(file, &mce_wait, wait);
	if (rcu_dereference(mcelog.next))
		return POLLIN | POLLRDNORM;
	return 0;
}

794
static long mce_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
L
Linus Torvalds 已提交
795 796
{
	int __user *p = (int __user *)arg;
797

L
Linus Torvalds 已提交
798
	if (!capable(CAP_SYS_ADMIN))
799
		return -EPERM;
L
Linus Torvalds 已提交
800
	switch (cmd) {
801
	case MCE_GET_RECORD_LEN:
L
Linus Torvalds 已提交
802 803
		return put_user(sizeof(struct mce), p);
	case MCE_GET_LOG_LEN:
804
		return put_user(MCE_LOG_LEN, p);
L
Linus Torvalds 已提交
805 806
	case MCE_GETCLEAR_FLAGS: {
		unsigned flags;
807 808

		do {
L
Linus Torvalds 已提交
809
			flags = mcelog.flags;
810 811
		} while (cmpxchg(&mcelog.flags, flags, 0) != flags);
		return put_user(flags, p);
L
Linus Torvalds 已提交
812 813
	}
	default:
814 815
		return -ENOTTY;
	}
L
Linus Torvalds 已提交
816 817
}

818
static const struct file_operations mce_chrdev_ops = {
T
Tim Hockin 已提交
819 820
	.open = mce_open,
	.release = mce_release,
L
Linus Torvalds 已提交
821
	.read = mce_read,
822
	.poll = mce_poll,
823
	.unlocked_ioctl = mce_ioctl,
L
Linus Torvalds 已提交
824 825 826 827 828 829 830 831
};

static struct miscdevice mce_log_device = {
	MISC_MCELOG_MINOR,
	"mcelog",
	&mce_chrdev_ops,
};

832 833
/*
 * Old style boot options parsing. Only for compatibility.
L
Linus Torvalds 已提交
834 835 836 837
 */
static int __init mcheck_disable(char *str)
{
	mce_dont_init = 1;
838
	return 1;
L
Linus Torvalds 已提交
839 840
}

841
/* mce=off disables machine check.
842
   mce=TOLERANCELEVEL (number, see above)
843 844
   mce=bootlog Log MCEs from before booting. Disabled by default on AMD.
   mce=nobootlog Don't log MCEs from before booting. */
L
Linus Torvalds 已提交
845 846 847 848
static int __init mcheck_enable(char *str)
{
	if (!strcmp(str, "off"))
		mce_dont_init = 1;
849 850
	else if (!strcmp(str, "bootlog") || !strcmp(str,"nobootlog"))
		mce_bootlog = str[0] == 'b';
851 852
	else if (isdigit(str[0]))
		get_option(&str, &tolerant);
L
Linus Torvalds 已提交
853
	else
854
		printk("mce= argument %s ignored. Please use /sys", str);
855
	return 1;
L
Linus Torvalds 已提交
856 857 858
}

__setup("nomce", mcheck_disable);
859
__setup("mce=", mcheck_enable);
L
Linus Torvalds 已提交
860

861
/*
L
Linus Torvalds 已提交
862
 * Sysfs support
863
 */
L
Linus Torvalds 已提交
864

865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887
/*
 * Disable machine checks on suspend and shutdown. We can't really handle
 * them later.
 */
static int mce_disable(void)
{
	int i;

	for (i = 0; i < banks; i++)
		wrmsrl(MSR_IA32_MC0_CTL + i*4, 0);
	return 0;
}

static int mce_suspend(struct sys_device *dev, pm_message_t state)
{
	return mce_disable();
}

static int mce_shutdown(struct sys_device *dev)
{
	return mce_disable();
}

888 889 890
/* On resume clear all MCE state. Don't want to see leftovers from the BIOS.
   Only one CPU is active at this time, the others get readded later using
   CPU hotplug. */
L
Linus Torvalds 已提交
891 892
static int mce_resume(struct sys_device *dev)
{
893
	mce_init(NULL);
894
	mce_cpu_features(&current_cpu_data);
L
Linus Torvalds 已提交
895 896 897
	return 0;
}

898 899 900 901 902 903 904 905
static void mce_cpu_restart(void *data)
{
	del_timer_sync(&__get_cpu_var(mce_timer));
	if (mce_available(&current_cpu_data))
		mce_init(NULL);
	mce_init_timer();
}

L
Linus Torvalds 已提交
906
/* Reinit MCEs after user configuration changes */
907 908
static void mce_restart(void)
{
909
	next_interval = check_interval * HZ;
910
	on_each_cpu(mce_cpu_restart, NULL, 1);
L
Linus Torvalds 已提交
911 912 913
}

static struct sysdev_class mce_sysclass = {
914 915
	.suspend = mce_suspend,
	.shutdown = mce_shutdown,
L
Linus Torvalds 已提交
916
	.resume = mce_resume,
917
	.name = "machinecheck",
L
Linus Torvalds 已提交
918 919
};

920
DEFINE_PER_CPU(struct sys_device, device_mce);
921
void (*threshold_cpu_callback)(unsigned long action, unsigned int cpu) __cpuinitdata;
L
Linus Torvalds 已提交
922 923 924

/* Why are there no generic functions for this? */
#define ACCESSOR(name, var, start) \
925 926 927
	static ssize_t show_ ## name(struct sys_device *s,		\
				     struct sysdev_attribute *attr,	\
				     char *buf) {			\
928 929
		return sprintf(buf, "%lx\n", (unsigned long)var);	\
	}								\
930 931 932
	static ssize_t set_ ## name(struct sys_device *s,		\
				    struct sysdev_attribute *attr,	\
				    const char *buf, size_t siz) {	\
933 934 935 936 937 938 939
		char *end;						\
		unsigned long new = simple_strtoul(buf, &end, 0);	\
		if (end == buf) return -EINVAL;				\
		var = new;						\
		start;							\
		return end-buf;						\
	}								\
L
Linus Torvalds 已提交
940 941
	static SYSDEV_ATTR(name, 0644, show_ ## name, set_ ## name);

942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961
static struct sysdev_attribute *bank_attrs;

static ssize_t show_bank(struct sys_device *s, struct sysdev_attribute *attr,
			 char *buf)
{
	u64 b = bank[attr - bank_attrs];
	return sprintf(buf, "%Lx\n", b);
}

static ssize_t set_bank(struct sys_device *s, struct sysdev_attribute *attr,
			const char *buf, size_t siz)
{
	char *end;
	u64 new = simple_strtoull(buf, &end, 0);
	if (end == buf)
		return -EINVAL;
	bank[attr - bank_attrs] = new;
	mce_restart();
	return end-buf;
}
962

963 964
static ssize_t show_trigger(struct sys_device *s, struct sysdev_attribute *attr,
				char *buf)
965 966 967 968 969 970
{
	strcpy(buf, trigger);
	strcat(buf, "\n");
	return strlen(trigger) + 1;
}

971 972
static ssize_t set_trigger(struct sys_device *s, struct sysdev_attribute *attr,
				const char *buf,size_t siz)
973 974 975 976 977 978 979 980 981 982 983 984
{
	char *p;
	int len;
	strncpy(trigger, buf, sizeof(trigger));
	trigger[sizeof(trigger)-1] = 0;
	len = strlen(trigger);
	p = strchr(trigger, '\n');
	if (*p) *p = 0;
	return len;
}

static SYSDEV_ATTR(trigger, 0644, show_trigger, set_trigger);
985
static SYSDEV_INT_ATTR(tolerant, 0644, tolerant);
L
Linus Torvalds 已提交
986
ACCESSOR(check_interval,check_interval,mce_restart())
987
static struct sysdev_attribute *mce_attributes[] = {
988
	&attr_tolerant.attr, &attr_check_interval, &attr_trigger,
989 990
	NULL
};
L
Linus Torvalds 已提交
991

992 993
static cpumask_t mce_device_initialized = CPU_MASK_NONE;

994 995
/* Per cpu sysdev init.  All of the cpus still share the same ctl bank */
static __cpuinit int mce_create_device(unsigned int cpu)
L
Linus Torvalds 已提交
996 997
{
	int err;
998
	int i;
999

A
Andreas Herrmann 已提交
1000
	if (!mce_available(&boot_cpu_data))
1001 1002
		return -EIO;

1003
	memset(&per_cpu(device_mce, cpu).kobj, 0, sizeof(struct kobject));
1004 1005 1006 1007
	per_cpu(device_mce,cpu).id = cpu;
	per_cpu(device_mce,cpu).cls = &mce_sysclass;

	err = sysdev_register(&per_cpu(device_mce,cpu));
1008 1009 1010 1011 1012 1013 1014 1015 1016
	if (err)
		return err;

	for (i = 0; mce_attributes[i]; i++) {
		err = sysdev_create_file(&per_cpu(device_mce,cpu),
					 mce_attributes[i]);
		if (err)
			goto error;
	}
1017 1018 1019 1020 1021 1022
	for (i = 0; i < banks; i++) {
		err = sysdev_create_file(&per_cpu(device_mce, cpu),
					&bank_attrs[i]);
		if (err)
			goto error2;
	}
1023
	cpu_set(cpu, mce_device_initialized);
1024

1025
	return 0;
1026 1027 1028 1029 1030
error2:
	while (--i >= 0) {
		sysdev_remove_file(&per_cpu(device_mce, cpu),
					&bank_attrs[i]);
	}
1031
error:
1032
	while (--i >= 0) {
1033 1034
		sysdev_remove_file(&per_cpu(device_mce,cpu),
				   mce_attributes[i]);
1035
	}
1036 1037
	sysdev_unregister(&per_cpu(device_mce,cpu));

1038 1039 1040
	return err;
}

1041
static __cpuinit void mce_remove_device(unsigned int cpu)
1042
{
1043 1044
	int i;

1045 1046 1047
	if (!cpu_isset(cpu, mce_device_initialized))
		return;

1048
	for (i = 0; mce_attributes[i]; i++)
1049
		sysdev_remove_file(&per_cpu(device_mce,cpu),
1050
			mce_attributes[i]);
1051 1052 1053
	for (i = 0; i < banks; i++)
		sysdev_remove_file(&per_cpu(device_mce, cpu),
			&bank_attrs[i]);
1054
	sysdev_unregister(&per_cpu(device_mce,cpu));
1055
	cpu_clear(cpu, mce_device_initialized);
1056 1057
}

1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078
/* Make sure there are no machine checks on offlined CPUs. */
static void __cpuexit mce_disable_cpu(void *h)
{
	int i;

	if (!mce_available(&current_cpu_data))
		return;
	for (i = 0; i < banks; i++)
		wrmsrl(MSR_IA32_MC0_CTL + i*4, 0);
}

static void __cpuexit mce_reenable_cpu(void *h)
{
	int i;

	if (!mce_available(&current_cpu_data))
		return;
	for (i = 0; i < banks; i++)
		wrmsrl(MSR_IA32_MC0_CTL + i*4, bank[i]);
}

1079
/* Get notified when a cpu comes on/off. Be hotplug friendly. */
1080 1081
static int __cpuinit mce_cpu_callback(struct notifier_block *nfb,
				      unsigned long action, void *hcpu)
1082 1083
{
	unsigned int cpu = (unsigned long)hcpu;
1084
	struct timer_list *t = &per_cpu(mce_timer, cpu);
1085 1086

	switch (action) {
1087 1088 1089
	case CPU_ONLINE:
	case CPU_ONLINE_FROZEN:
		mce_create_device(cpu);
1090 1091
		if (threshold_cpu_callback)
			threshold_cpu_callback(action, cpu);
1092 1093
		break;
	case CPU_DEAD:
1094
	case CPU_DEAD_FROZEN:
1095 1096
		if (threshold_cpu_callback)
			threshold_cpu_callback(action, cpu);
1097 1098
		mce_remove_device(cpu);
		break;
1099 1100 1101
	case CPU_DOWN_PREPARE:
	case CPU_DOWN_PREPARE_FROZEN:
		del_timer_sync(t);
1102
		smp_call_function_single(cpu, mce_disable_cpu, NULL, 1);
1103 1104 1105 1106 1107
		break;
	case CPU_DOWN_FAILED:
	case CPU_DOWN_FAILED_FROZEN:
		t->expires = round_jiffies_relative(jiffies + next_interval);
		add_timer_on(t, cpu);
1108
		smp_call_function_single(cpu, mce_reenable_cpu, NULL, 1);
1109
		break;
1110
	}
1111
	return NOTIFY_OK;
1112 1113
}

1114
static struct notifier_block mce_cpu_notifier __cpuinitdata = {
1115 1116 1117
	.notifier_call = mce_cpu_callback,
};

1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145
static __init int mce_init_banks(void)
{
	int i;

	bank_attrs = kzalloc(sizeof(struct sysdev_attribute) * banks,
				GFP_KERNEL);
	if (!bank_attrs)
		return -ENOMEM;

	for (i = 0; i < banks; i++) {
		struct sysdev_attribute *a = &bank_attrs[i];
		a->attr.name = kasprintf(GFP_KERNEL, "bank%d", i);
		if (!a->attr.name)
			goto nomem;
		a->attr.mode = 0644;
		a->show = show_bank;
		a->store = set_bank;
	}
	return 0;

nomem:
	while (--i >= 0)
		kfree(bank_attrs[i].attr.name);
	kfree(bank_attrs);
	bank_attrs = NULL;
	return -ENOMEM;
}

1146 1147 1148 1149 1150
static __init int mce_init_device(void)
{
	int err;
	int i = 0;

L
Linus Torvalds 已提交
1151 1152
	if (!mce_available(&boot_cpu_data))
		return -EIO;
1153 1154 1155 1156 1157

	err = mce_init_banks();
	if (err)
		return err;

L
Linus Torvalds 已提交
1158
	err = sysdev_class_register(&mce_sysclass);
1159 1160
	if (err)
		return err;
1161 1162

	for_each_online_cpu(i) {
1163 1164 1165
		err = mce_create_device(i);
		if (err)
			return err;
1166 1167
	}

1168
	register_hotcpu_notifier(&mce_cpu_notifier);
L
Linus Torvalds 已提交
1169 1170 1171
	misc_register(&mce_log_device);
	return err;
}
1172

L
Linus Torvalds 已提交
1173
device_initcall(mce_init_device);