mce.c 45.8 KB
Newer Older
L
Linus Torvalds 已提交
1 2
/*
 * Machine check handler.
I
Ingo Molnar 已提交
3
 *
L
Linus Torvalds 已提交
4
 * K8 parts Copyright 2002,2003 Andi Kleen, SuSE Labs.
5 6
 * Rest from unknown author(s).
 * 2004 Andi Kleen. Rewrote most of it.
7 8
 * Copyright 2008 Intel Corporation
 * Author: Andi Kleen
L
Linus Torvalds 已提交
9
 */
I
Ingo Molnar 已提交
10 11 12
#include <linux/thread_info.h>
#include <linux/capability.h>
#include <linux/miscdevice.h>
13
#include <linux/interrupt.h>
I
Ingo Molnar 已提交
14 15 16 17
#include <linux/ratelimit.h>
#include <linux/kallsyms.h>
#include <linux/rcupdate.h>
#include <linux/kobject.h>
18
#include <linux/uaccess.h>
I
Ingo Molnar 已提交
19 20 21
#include <linux/kdebug.h>
#include <linux/kernel.h>
#include <linux/percpu.h>
L
Linus Torvalds 已提交
22 23
#include <linux/string.h>
#include <linux/sysdev.h>
24
#include <linux/delay.h>
25
#include <linux/ctype.h>
I
Ingo Molnar 已提交
26
#include <linux/sched.h>
27
#include <linux/sysfs.h>
I
Ingo Molnar 已提交
28 29 30 31
#include <linux/types.h>
#include <linux/init.h>
#include <linux/kmod.h>
#include <linux/poll.h>
32
#include <linux/nmi.h>
I
Ingo Molnar 已提交
33
#include <linux/cpu.h>
34
#include <linux/smp.h>
I
Ingo Molnar 已提交
35
#include <linux/fs.h>
36
#include <linux/mm.h>
I
Ingo Molnar 已提交
37

38
#include <asm/processor.h>
39 40
#include <asm/hw_irq.h>
#include <asm/apic.h>
41
#include <asm/idle.h>
42
#include <asm/ipi.h>
I
Ingo Molnar 已提交
43 44
#include <asm/mce.h>
#include <asm/msr.h>
L
Linus Torvalds 已提交
45

46
#include "mce-internal.h"
47

H
Hidetoshi Seto 已提交
48
int mce_disabled __read_mostly;
49

I
Ingo Molnar 已提交
50
#define MISC_MCELOG_MINOR	227
51

52 53
#define SPINUNIT 100	/* 100ns */

54 55
atomic_t mce_entry;

56 57
DEFINE_PER_CPU(unsigned, mce_exception_count);

58 59 60 61 62 63 64
/*
 * Tolerant levels:
 *   0: always panic on uncorrected errors, log corrected errors
 *   1: panic or SIGBUS on uncorrected errors, log corrected errors
 *   2: SIGBUS or log uncorrected errors (if possible), log corrected errors
 *   3: never panic or SIGBUS, log all errors (for testing only)
 */
H
Hidetoshi Seto 已提交
65 66 67 68 69 70 71 72 73 74
static int			tolerant		__read_mostly = 1;
static int			banks			__read_mostly;
static int			rip_msr			__read_mostly;
static int			mce_bootlog		__read_mostly = -1;
static int			monarch_timeout		__read_mostly = -1;
static int			mce_panic_timeout	__read_mostly;
static int			mce_dont_log_ce		__read_mostly;
int				mce_cmci_disabled	__read_mostly;
int				mce_ignore_ce		__read_mostly;
int				mce_ser			__read_mostly;
75

76 77
struct mce_bank                *mce_banks		__read_mostly;

78 79 80 81
/* User mode helper program triggered by machine check event */
static unsigned long		mce_need_notify;
static char			mce_helper[128];
static char			*mce_helper_argv[2] = { mce_helper, NULL };
L
Linus Torvalds 已提交
82

83
static DECLARE_WAIT_QUEUE_HEAD(mce_wait);
84 85 86
static DEFINE_PER_CPU(struct mce, mces_seen);
static int			cpu_missing;

87

88 89 90 91 92
/* MCA banks polled by the period polling timer for corrected events */
DEFINE_PER_CPU(mce_banks_t, mce_poll_banks) = {
	[0 ... BITS_TO_LONGS(MAX_NR_BANKS)-1] = ~0UL
};

93 94
static DEFINE_PER_CPU(struct work_struct, mce_work);

95 96 97 98
/* Do initial initialization of a struct mce */
void mce_setup(struct mce *m)
{
	memset(m, 0, sizeof(struct mce));
99
	m->cpu = m->extcpu = smp_processor_id();
100
	rdtscll(m->tsc);
101 102 103 104 105 106 107 108 109
	/* We hope get_seconds stays lockless */
	m->time = get_seconds();
	m->cpuvendor = boot_cpu_data.x86_vendor;
	m->cpuid = cpuid_eax(1);
#ifdef CONFIG_SMP
	m->socketid = cpu_data(m->extcpu).phys_proc_id;
#endif
	m->apicid = cpu_data(m->extcpu).initial_apicid;
	rdmsrl(MSR_IA32_MCG_CAP, m->mcgcap);
110 111
}

112 113 114
DEFINE_PER_CPU(struct mce, injectm);
EXPORT_PER_CPU_SYMBOL_GPL(injectm);

L
Linus Torvalds 已提交
115 116 117 118 119 120
/*
 * Lockless MCE logging infrastructure.
 * This avoids deadlocks on printk locks without having to break locks. Also
 * separate MCEs from kernel messages to avoid bogus bug reports.
 */

121
static struct mce_log mcelog = {
122 123 124
	.signature	= MCE_LOG_SIGNATURE,
	.len		= MCE_LOG_LEN,
	.recordlen	= sizeof(struct mce),
125
};
L
Linus Torvalds 已提交
126 127 128 129

void mce_log(struct mce *mce)
{
	unsigned next, entry;
I
Ingo Molnar 已提交
130

L
Linus Torvalds 已提交
131
	mce->finished = 0;
M
Mike Waychison 已提交
132
	wmb();
L
Linus Torvalds 已提交
133 134
	for (;;) {
		entry = rcu_dereference(mcelog.next);
135
		for (;;) {
I
Ingo Molnar 已提交
136 137 138 139 140
			/*
			 * When the buffer fills up discard new entries.
			 * Assume that the earlier errors are the more
			 * interesting ones:
			 */
141
			if (entry >= MCE_LOG_LEN) {
142 143
				set_bit(MCE_OVERFLOW,
					(unsigned long *)&mcelog.flags);
144 145
				return;
			}
I
Ingo Molnar 已提交
146
			/* Old left over entry. Skip: */
147 148 149 150
			if (mcelog.entry[entry].finished) {
				entry++;
				continue;
			}
M
Mike Waychison 已提交
151
			break;
L
Linus Torvalds 已提交
152 153 154 155 156 157 158
		}
		smp_rmb();
		next = entry + 1;
		if (cmpxchg(&mcelog.next, entry, next) == entry)
			break;
	}
	memcpy(mcelog.entry + entry, mce, sizeof(struct mce));
M
Mike Waychison 已提交
159
	wmb();
L
Linus Torvalds 已提交
160
	mcelog.entry[entry].finished = 1;
M
Mike Waychison 已提交
161
	wmb();
L
Linus Torvalds 已提交
162

163
	mce->finished = 1;
164
	set_bit(0, &mce_need_notify);
L
Linus Torvalds 已提交
165 166
}

H
Hidetoshi Seto 已提交
167
static void print_mce(struct mce *m)
L
Linus Torvalds 已提交
168
{
169
	printk(KERN_EMERG
L
Linus Torvalds 已提交
170
	       "CPU %d: Machine Check Exception: %16Lx Bank %d: %016Lx\n",
171
	       m->extcpu, m->mcgstatus, m->bank, m->status);
172
	if (m->ip) {
173
		printk(KERN_EMERG "RIP%s %02x:<%016Lx> ",
L
Linus Torvalds 已提交
174
		       !(m->mcgstatus & MCG_STATUS_EIPV) ? " !INEXACT!" : "",
175
		       m->cs, m->ip);
L
Linus Torvalds 已提交
176
		if (m->cs == __KERNEL_CS)
177
			print_symbol("{%s}", m->ip);
L
Linus Torvalds 已提交
178 179
		printk("\n");
	}
180
	printk(KERN_EMERG "TSC %llx ", m->tsc);
L
Linus Torvalds 已提交
181
	if (m->addr)
182
		printk("ADDR %llx ", m->addr);
L
Linus Torvalds 已提交
183
	if (m->misc)
184
		printk("MISC %llx ", m->misc);
L
Linus Torvalds 已提交
185
	printk("\n");
186 187 188
	printk(KERN_EMERG "PROCESSOR %u:%x TIME %llu SOCKET %u APIC %x\n",
			m->cpuvendor, m->cpuid, m->time, m->socketid,
			m->apicid);
189 190
}

H
Hidetoshi Seto 已提交
191 192 193 194 195
static void print_mce_head(void)
{
	printk(KERN_EMERG "\n" KERN_EMERG "HARDWARE ERROR\n");
}

196 197 198 199
static void print_mce_tail(void)
{
	printk(KERN_EMERG "This is not a software problem!\n"
	       KERN_EMERG "Run through mcelog --ascii to decode and contact your hardware vendor\n");
L
Linus Torvalds 已提交
200 201
}

202 203 204 205 206 207 208 209 210 211 212 213
#define PANIC_TIMEOUT 5 /* 5 seconds */

static atomic_t mce_paniced;

/* Panic in progress. Enable interrupts and wait for final IPI */
static void wait_for_panic(void)
{
	long timeout = PANIC_TIMEOUT*USEC_PER_SEC;
	preempt_disable();
	local_irq_enable();
	while (timeout-- > 0)
		udelay(1);
214 215
	if (panic_timeout == 0)
		panic_timeout = mce_panic_timeout;
216 217 218
	panic("Panicing machine check CPU died");
}

219
static void mce_panic(char *msg, struct mce *final, char *exp)
220
{
L
Linus Torvalds 已提交
221
	int i;
222

223 224 225
	/*
	 * Make sure only one CPU runs in machine check panic
	 */
226
	if (atomic_inc_return(&mce_paniced) > 1)
227 228 229
		wait_for_panic();
	barrier();

230 231
	bust_spinlocks(1);
	console_verbose();
H
Hidetoshi Seto 已提交
232
	print_mce_head();
233
	/* First print corrected ones that are still unlogged */
L
Linus Torvalds 已提交
234
	for (i = 0; i < MCE_LOG_LEN; i++) {
235
		struct mce *m = &mcelog.entry[i];
H
Hidetoshi Seto 已提交
236 237 238 239
		if (!(m->status & MCI_STATUS_VAL))
			continue;
		if (!(m->status & MCI_STATUS_UC))
			print_mce(m);
240 241 242 243 244
	}
	/* Now print uncorrected but with the final one last */
	for (i = 0; i < MCE_LOG_LEN; i++) {
		struct mce *m = &mcelog.entry[i];
		if (!(m->status & MCI_STATUS_VAL))
L
Linus Torvalds 已提交
245
			continue;
H
Hidetoshi Seto 已提交
246 247
		if (!(m->status & MCI_STATUS_UC))
			continue;
248
		if (!final || memcmp(m, final, sizeof(struct mce)))
H
Hidetoshi Seto 已提交
249
			print_mce(m);
L
Linus Torvalds 已提交
250
	}
251
	if (final)
H
Hidetoshi Seto 已提交
252
		print_mce(final);
253 254
	if (cpu_missing)
		printk(KERN_EMERG "Some CPUs didn't answer in synchronization\n");
255
	print_mce_tail();
256 257
	if (exp)
		printk(KERN_EMERG "Machine check: %s\n", exp);
258 259
	if (panic_timeout == 0)
		panic_timeout = mce_panic_timeout;
260
	panic(msg);
261
}
L
Linus Torvalds 已提交
262

263 264 265 266 267 268 269
/* Support code for software error injection */

static int msr_to_offset(u32 msr)
{
	unsigned bank = __get_cpu_var(injectm.bank);
	if (msr == rip_msr)
		return offsetof(struct mce, ip);
270
	if (msr == MSR_IA32_MCx_STATUS(bank))
271
		return offsetof(struct mce, status);
272
	if (msr == MSR_IA32_MCx_ADDR(bank))
273
		return offsetof(struct mce, addr);
274
	if (msr == MSR_IA32_MCx_MISC(bank))
275 276 277 278 279 280
		return offsetof(struct mce, misc);
	if (msr == MSR_IA32_MCG_STATUS)
		return offsetof(struct mce, mcgstatus);
	return -1;
}

281 282 283 284
/* MSR access wrappers used for error injection */
static u64 mce_rdmsrl(u32 msr)
{
	u64 v;
285 286 287 288 289 290
	if (__get_cpu_var(injectm).finished) {
		int offset = msr_to_offset(msr);
		if (offset < 0)
			return 0;
		return *(u64 *)((char *)&__get_cpu_var(injectm) + offset);
	}
291 292 293 294 295 296
	rdmsrl(msr, v);
	return v;
}

static void mce_wrmsrl(u32 msr, u64 v)
{
297 298 299 300 301 302
	if (__get_cpu_var(injectm).finished) {
		int offset = msr_to_offset(msr);
		if (offset >= 0)
			*(u64 *)((char *)&__get_cpu_var(injectm) + offset) = v;
		return;
	}
303 304 305
	wrmsrl(msr, v);
}

306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360
/*
 * Simple lockless ring to communicate PFNs from the exception handler with the
 * process context work function. This is vastly simplified because there's
 * only a single reader and a single writer.
 */
#define MCE_RING_SIZE 16	/* we use one entry less */

struct mce_ring {
	unsigned short start;
	unsigned short end;
	unsigned long ring[MCE_RING_SIZE];
};
static DEFINE_PER_CPU(struct mce_ring, mce_ring);

/* Runs with CPU affinity in workqueue */
static int mce_ring_empty(void)
{
	struct mce_ring *r = &__get_cpu_var(mce_ring);

	return r->start == r->end;
}

static int mce_ring_get(unsigned long *pfn)
{
	struct mce_ring *r;
	int ret = 0;

	*pfn = 0;
	get_cpu();
	r = &__get_cpu_var(mce_ring);
	if (r->start == r->end)
		goto out;
	*pfn = r->ring[r->start];
	r->start = (r->start + 1) % MCE_RING_SIZE;
	ret = 1;
out:
	put_cpu();
	return ret;
}

/* Always runs in MCE context with preempt off */
static int mce_ring_add(unsigned long pfn)
{
	struct mce_ring *r = &__get_cpu_var(mce_ring);
	unsigned next;

	next = (r->end + 1) % MCE_RING_SIZE;
	if (next == r->start)
		return -1;
	r->ring[r->end] = pfn;
	wmb();
	r->end = next;
	return 0;
}

A
Andi Kleen 已提交
361
int mce_available(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
362
{
363
	if (mce_disabled)
364
		return 0;
365
	return cpu_has(c, X86_FEATURE_MCE) && cpu_has(c, X86_FEATURE_MCA);
L
Linus Torvalds 已提交
366 367
}

368 369 370 371 372 373 374 375 376
static void mce_schedule_work(void)
{
	if (!mce_ring_empty()) {
		struct work_struct *work = &__get_cpu_var(mce_work);
		if (!work_pending(work))
			schedule_work(work);
	}
}

H
Huang Ying 已提交
377 378 379 380
/*
 * Get the address of the instruction at the time of the machine check
 * error.
 */
381 382
static inline void mce_get_rip(struct mce *m, struct pt_regs *regs)
{
H
Huang Ying 已提交
383 384

	if (regs && (m->mcgstatus & (MCG_STATUS_RIPV|MCG_STATUS_EIPV))) {
385
		m->ip = regs->ip;
386 387
		m->cs = regs->cs;
	} else {
388
		m->ip = 0;
389 390
		m->cs = 0;
	}
H
Huang Ying 已提交
391
	if (rip_msr)
392
		m->ip = mce_rdmsrl(rip_msr);
393 394
}

395 396 397 398 399 400 401 402 403 404 405
#ifdef CONFIG_X86_LOCAL_APIC 
/*
 * Called after interrupts have been reenabled again
 * when a MCE happened during an interrupts off region
 * in the kernel.
 */
asmlinkage void smp_mce_self_interrupt(struct pt_regs *regs)
{
	ack_APIC_irq();
	exit_idle();
	irq_enter();
406
	mce_notify_irq();
407
	mce_schedule_work();
408 409 410 411 412 413 414
	irq_exit();
}
#endif

static void mce_report_event(struct pt_regs *regs)
{
	if (regs->flags & (X86_VM_MASK|X86_EFLAGS_IF)) {
415
		mce_notify_irq();
416 417 418 419 420 421 422
		/*
		 * Triggering the work queue here is just an insurance
		 * policy in case the syscall exit notify handler
		 * doesn't run soon enough or ends up running on the
		 * wrong CPU (can happen when audit sleeps)
		 */
		mce_schedule_work();
423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450
		return;
	}

#ifdef CONFIG_X86_LOCAL_APIC
	/*
	 * Without APIC do not notify. The event will be picked
	 * up eventually.
	 */
	if (!cpu_has_apic)
		return;

	/*
	 * When interrupts are disabled we cannot use
	 * kernel services safely. Trigger an self interrupt
	 * through the APIC to instead do the notification
	 * after interrupts are reenabled again.
	 */
	apic->send_IPI_self(MCE_SELF_VECTOR);

	/*
	 * Wait for idle afterwards again so that we don't leave the
	 * APIC in a non idle state because the normal APIC writes
	 * cannot exclude us.
	 */
	apic_wait_icr_idle();
#endif
}

451 452
DEFINE_PER_CPU(unsigned, mce_poll_count);

453
/*
454 455 456 457
 * Poll for corrected events or events that happened before reset.
 * Those are just logged through /dev/mcelog.
 *
 * This is executed in standard interrupt context.
A
Andi Kleen 已提交
458 459 460 461 462 463 464 465 466
 *
 * Note: spec recommends to panic for fatal unsignalled
 * errors here. However this would be quite problematic --
 * we would need to reimplement the Monarch handling and
 * it would mess up the exclusion between exception handler
 * and poll hander -- * so we skip this for now.
 * These cases should not happen anyways, or only when the CPU
 * is already totally * confused. In this case it's likely it will
 * not fully execute the machine check handler either.
467
 */
468
void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
469 470 471 472
{
	struct mce m;
	int i;

473 474
	__get_cpu_var(mce_poll_count)++;

475 476
	mce_setup(&m);

477
	m.mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS);
478
	for (i = 0; i < banks; i++) {
479
		if (!mce_banks[i].ctl || !test_bit(i, *b))
480 481 482 483 484 485 486 487
			continue;

		m.misc = 0;
		m.addr = 0;
		m.bank = i;
		m.tsc = 0;

		barrier();
488
		m.status = mce_rdmsrl(MSR_IA32_MCx_STATUS(i));
489 490 491 492
		if (!(m.status & MCI_STATUS_VAL))
			continue;

		/*
A
Andi Kleen 已提交
493 494
		 * Uncorrected or signalled events are handled by the exception
		 * handler when it is enabled, so don't process those here.
495 496 497
		 *
		 * TBD do the same check for MCI_STATUS_EN here?
		 */
A
Andi Kleen 已提交
498 499
		if (!(flags & MCP_UC) &&
		    (m.status & (mce_ser ? MCI_STATUS_S : MCI_STATUS_UC)))
500 501 502
			continue;

		if (m.status & MCI_STATUS_MISCV)
503
			m.misc = mce_rdmsrl(MSR_IA32_MCx_MISC(i));
504
		if (m.status & MCI_STATUS_ADDRV)
505
			m.addr = mce_rdmsrl(MSR_IA32_MCx_ADDR(i));
506 507 508 509 510 511 512

		if (!(flags & MCP_TIMESTAMP))
			m.tsc = 0;
		/*
		 * Don't get the IP here because it's unlikely to
		 * have anything to do with the actual error location.
		 */
513
		if (!(flags & MCP_DONTLOG) && !mce_dont_log_ce) {
A
Andi Kleen 已提交
514 515 516
			mce_log(&m);
			add_taint(TAINT_MACHINE_CHECK);
		}
517 518 519 520

		/*
		 * Clear state for this bank.
		 */
521
		mce_wrmsrl(MSR_IA32_MCx_STATUS(i), 0);
522 523 524 525 526 527
	}

	/*
	 * Don't clear MCG_STATUS here because it's only defined for
	 * exceptions.
	 */
528 529

	sync_core();
530
}
531
EXPORT_SYMBOL_GPL(machine_check_poll);
532

533 534 535 536 537 538 539 540 541
/*
 * Do a quick check if any of the events requires a panic.
 * This decides if we keep the events around or clear them.
 */
static int mce_no_way_out(struct mce *m, char **msg)
{
	int i;

	for (i = 0; i < banks; i++) {
542
		m->status = mce_rdmsrl(MSR_IA32_MCx_STATUS(i));
543 544 545 546 547 548
		if (mce_severity(m, tolerant, msg) >= MCE_PANIC_SEVERITY)
			return 1;
	}
	return 0;
}

549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642
/*
 * Variable to establish order between CPUs while scanning.
 * Each CPU spins initially until executing is equal its number.
 */
static atomic_t mce_executing;

/*
 * Defines order of CPUs on entry. First CPU becomes Monarch.
 */
static atomic_t mce_callin;

/*
 * Check if a timeout waiting for other CPUs happened.
 */
static int mce_timed_out(u64 *t)
{
	/*
	 * The others already did panic for some reason.
	 * Bail out like in a timeout.
	 * rmb() to tell the compiler that system_state
	 * might have been modified by someone else.
	 */
	rmb();
	if (atomic_read(&mce_paniced))
		wait_for_panic();
	if (!monarch_timeout)
		goto out;
	if ((s64)*t < SPINUNIT) {
		/* CHECKME: Make panic default for 1 too? */
		if (tolerant < 1)
			mce_panic("Timeout synchronizing machine check over CPUs",
				  NULL, NULL);
		cpu_missing = 1;
		return 1;
	}
	*t -= SPINUNIT;
out:
	touch_nmi_watchdog();
	return 0;
}

/*
 * The Monarch's reign.  The Monarch is the CPU who entered
 * the machine check handler first. It waits for the others to
 * raise the exception too and then grades them. When any
 * error is fatal panic. Only then let the others continue.
 *
 * The other CPUs entering the MCE handler will be controlled by the
 * Monarch. They are called Subjects.
 *
 * This way we prevent any potential data corruption in a unrecoverable case
 * and also makes sure always all CPU's errors are examined.
 *
 * Also this detects the case of an machine check event coming from outer
 * space (not detected by any CPUs) In this case some external agent wants
 * us to shut down, so panic too.
 *
 * The other CPUs might still decide to panic if the handler happens
 * in a unrecoverable place, but in this case the system is in a semi-stable
 * state and won't corrupt anything by itself. It's ok to let the others
 * continue for a bit first.
 *
 * All the spin loops have timeouts; when a timeout happens a CPU
 * typically elects itself to be Monarch.
 */
static void mce_reign(void)
{
	int cpu;
	struct mce *m = NULL;
	int global_worst = 0;
	char *msg = NULL;
	char *nmsg = NULL;

	/*
	 * This CPU is the Monarch and the other CPUs have run
	 * through their handlers.
	 * Grade the severity of the errors of all the CPUs.
	 */
	for_each_possible_cpu(cpu) {
		int severity = mce_severity(&per_cpu(mces_seen, cpu), tolerant,
					    &nmsg);
		if (severity > global_worst) {
			msg = nmsg;
			global_worst = severity;
			m = &per_cpu(mces_seen, cpu);
		}
	}

	/*
	 * Cannot recover? Panic here then.
	 * This dumps all the mces in the log buffer and stops the
	 * other CPUs.
	 */
	if (m && global_worst >= MCE_PANIC_SEVERITY && tolerant < 3)
643
		mce_panic("Fatal Machine check", m, msg);
644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674

	/*
	 * For UC somewhere we let the CPU who detects it handle it.
	 * Also must let continue the others, otherwise the handling
	 * CPU could deadlock on a lock.
	 */

	/*
	 * No machine check event found. Must be some external
	 * source or one CPU is hung. Panic.
	 */
	if (!m && tolerant < 3)
		mce_panic("Machine check from unknown source", NULL, NULL);

	/*
	 * Now clear all the mces_seen so that they don't reappear on
	 * the next mce.
	 */
	for_each_possible_cpu(cpu)
		memset(&per_cpu(mces_seen, cpu), 0, sizeof(struct mce));
}

static atomic_t global_nwo;

/*
 * Start of Monarch synchronization. This waits until all CPUs have
 * entered the exception handler and then determines if any of them
 * saw a fatal event that requires panic. Then it executes them
 * in the entry order.
 * TBD double check parallel CPU hotunplug
 */
H
Hidetoshi Seto 已提交
675
static int mce_start(int *no_way_out)
676
{
H
Hidetoshi Seto 已提交
677
	int order;
678 679 680
	int cpus = num_online_cpus();
	u64 timeout = (u64)monarch_timeout * NSEC_PER_USEC;

H
Hidetoshi Seto 已提交
681 682
	if (!timeout)
		return -1;
683

H
Hidetoshi Seto 已提交
684
	atomic_add(*no_way_out, &global_nwo);
685 686 687 688
	/*
	 * global_nwo should be updated before mce_callin
	 */
	smp_wmb();
689
	order = atomic_inc_return(&mce_callin);
690 691 692 693 694 695 696

	/*
	 * Wait for everyone.
	 */
	while (atomic_read(&mce_callin) != cpus) {
		if (mce_timed_out(&timeout)) {
			atomic_set(&global_nwo, 0);
H
Hidetoshi Seto 已提交
697
			return -1;
698 699 700 701
		}
		ndelay(SPINUNIT);
	}

702 703 704 705
	/*
	 * mce_callin should be read before global_nwo
	 */
	smp_rmb();
706

H
Hidetoshi Seto 已提交
707 708 709 710
	if (order == 1) {
		/*
		 * Monarch: Starts executing now, the others wait.
		 */
711
		atomic_set(&mce_executing, 1);
H
Hidetoshi Seto 已提交
712 713 714 715 716 717 718 719 720 721 722 723 724 725
	} else {
		/*
		 * Subject: Now start the scanning loop one by one in
		 * the original callin order.
		 * This way when there are any shared banks it will be
		 * only seen by one CPU before cleared, avoiding duplicates.
		 */
		while (atomic_read(&mce_executing) < order) {
			if (mce_timed_out(&timeout)) {
				atomic_set(&global_nwo, 0);
				return -1;
			}
			ndelay(SPINUNIT);
		}
726 727 728
	}

	/*
H
Hidetoshi Seto 已提交
729
	 * Cache the global no_way_out state.
730
	 */
H
Hidetoshi Seto 已提交
731 732 733
	*no_way_out = atomic_read(&global_nwo);

	return order;
734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802
}

/*
 * Synchronize between CPUs after main scanning loop.
 * This invokes the bulk of the Monarch processing.
 */
static int mce_end(int order)
{
	int ret = -1;
	u64 timeout = (u64)monarch_timeout * NSEC_PER_USEC;

	if (!timeout)
		goto reset;
	if (order < 0)
		goto reset;

	/*
	 * Allow others to run.
	 */
	atomic_inc(&mce_executing);

	if (order == 1) {
		/* CHECKME: Can this race with a parallel hotplug? */
		int cpus = num_online_cpus();

		/*
		 * Monarch: Wait for everyone to go through their scanning
		 * loops.
		 */
		while (atomic_read(&mce_executing) <= cpus) {
			if (mce_timed_out(&timeout))
				goto reset;
			ndelay(SPINUNIT);
		}

		mce_reign();
		barrier();
		ret = 0;
	} else {
		/*
		 * Subject: Wait for Monarch to finish.
		 */
		while (atomic_read(&mce_executing) != 0) {
			if (mce_timed_out(&timeout))
				goto reset;
			ndelay(SPINUNIT);
		}

		/*
		 * Don't reset anything. That's done by the Monarch.
		 */
		return 0;
	}

	/*
	 * Reset all global state.
	 */
reset:
	atomic_set(&global_nwo, 0);
	atomic_set(&mce_callin, 0);
	barrier();

	/*
	 * Let others run again.
	 */
	atomic_set(&mce_executing, 0);
	return ret;
}

803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819
/*
 * Check if the address reported by the CPU is in a format we can parse.
 * It would be possible to add code for most other cases, but all would
 * be somewhat complicated (e.g. segment offset would require an instruction
 * parser). So only support physical addresses upto page granuality for now.
 */
static int mce_usable_address(struct mce *m)
{
	if (!(m->status & MCI_STATUS_MISCV) || !(m->status & MCI_STATUS_ADDRV))
		return 0;
	if ((m->misc & 0x3f) > PAGE_SHIFT)
		return 0;
	if (((m->misc >> 6) & 7) != MCM_ADDR_PHYS)
		return 0;
	return 1;
}

820 821 822 823 824 825
static void mce_clear_state(unsigned long *toclear)
{
	int i;

	for (i = 0; i < banks; i++) {
		if (test_bit(i, toclear))
826
			mce_wrmsrl(MSR_IA32_MCx_STATUS(i), 0);
827 828 829
	}
}

830 831 832 833 834 835 836
/*
 * The actual machine check handler. This only handles real
 * exceptions when something got corrupted coming in through int 18.
 *
 * This is executed in NMI context not subject to normal locking rules. This
 * implies that most kernel services cannot be safely used. Don't even
 * think about putting a printk in there!
837 838 839 840
 *
 * On Intel systems this is entered on all CPUs in parallel through
 * MCE broadcast. However some CPUs might be broken beyond repair,
 * so be always careful when synchronizing with others.
L
Linus Torvalds 已提交
841
 */
I
Ingo Molnar 已提交
842
void do_machine_check(struct pt_regs *regs, long error_code)
L
Linus Torvalds 已提交
843
{
844
	struct mce m, *final;
L
Linus Torvalds 已提交
845
	int i;
846 847 848 849 850 851
	int worst = 0;
	int severity;
	/*
	 * Establish sequential order between the CPUs entering the machine
	 * check handler.
	 */
H
Hidetoshi Seto 已提交
852
	int order;
853 854 855 856 857 858 859 860 861 862
	/*
	 * If no_way_out gets set, there is no safe way to recover from this
	 * MCE.  If tolerant is cranked up, we'll try anyway.
	 */
	int no_way_out = 0;
	/*
	 * If kill_it gets set, there might be a way to recover from this
	 * error.
	 */
	int kill_it = 0;
863
	DECLARE_BITMAP(toclear, MAX_NR_BANKS);
864
	char *msg = "Unknown";
L
Linus Torvalds 已提交
865

866 867
	atomic_inc(&mce_entry);

868 869
	__get_cpu_var(mce_exception_count)++;

870
	if (notify_die(DIE_NMI, "machine check", regs, error_code,
871
			   18, SIGKILL) == NOTIFY_STOP)
872
		goto out;
873
	if (!banks)
874
		goto out;
L
Linus Torvalds 已提交
875

876 877
	mce_setup(&m);

878
	m.mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS);
879
	no_way_out = mce_no_way_out(&m, &msg);
880

881 882 883
	final = &__get_cpu_var(mces_seen);
	*final = m;

L
Linus Torvalds 已提交
884 885
	barrier();

A
Andi Kleen 已提交
886 887 888 889 890 891
	/*
	 * When no restart IP must always kill or panic.
	 */
	if (!(m.mcgstatus & MCG_STATUS_RIPV))
		kill_it = 1;

892 893 894 895 896
	/*
	 * Go through all the banks in exclusion of the other CPUs.
	 * This way we don't report duplicated events on shared banks
	 * because the first one to see it will clear it.
	 */
H
Hidetoshi Seto 已提交
897
	order = mce_start(&no_way_out);
L
Linus Torvalds 已提交
898
	for (i = 0; i < banks; i++) {
899
		__clear_bit(i, toclear);
900
		if (!mce_banks[i].ctl)
L
Linus Torvalds 已提交
901
			continue;
902 903

		m.misc = 0;
L
Linus Torvalds 已提交
904 905 906
		m.addr = 0;
		m.bank = i;

907
		m.status = mce_rdmsrl(MSR_IA32_MCx_STATUS(i));
L
Linus Torvalds 已提交
908 909 910
		if ((m.status & MCI_STATUS_VAL) == 0)
			continue;

911
		/*
A
Andi Kleen 已提交
912 913
		 * Non uncorrected or non signaled errors are handled by
		 * machine_check_poll. Leave them alone, unless this panics.
914
		 */
A
Andi Kleen 已提交
915 916
		if (!(m.status & (mce_ser ? MCI_STATUS_S : MCI_STATUS_UC)) &&
			!no_way_out)
917 918 919 920 921 922 923
			continue;

		/*
		 * Set taint even when machine check was not enabled.
		 */
		add_taint(TAINT_MACHINE_CHECK);

A
Andi Kleen 已提交
924
		severity = mce_severity(&m, tolerant, NULL);
925

A
Andi Kleen 已提交
926 927 928 929 930 931 932 933
		/*
		 * When machine check was for corrected handler don't touch,
		 * unless we're panicing.
		 */
		if (severity == MCE_KEEP_SEVERITY && !no_way_out)
			continue;
		__set_bit(i, toclear);
		if (severity == MCE_NO_SEVERITY) {
934 935 936 937 938
			/*
			 * Machine check event was not enabled. Clear, but
			 * ignore.
			 */
			continue;
L
Linus Torvalds 已提交
939 940
		}

A
Andi Kleen 已提交
941 942 943 944 945 946
		/*
		 * Kill on action required.
		 */
		if (severity == MCE_AR_SEVERITY)
			kill_it = 1;

L
Linus Torvalds 已提交
947
		if (m.status & MCI_STATUS_MISCV)
948
			m.misc = mce_rdmsrl(MSR_IA32_MCx_MISC(i));
L
Linus Torvalds 已提交
949
		if (m.status & MCI_STATUS_ADDRV)
950
			m.addr = mce_rdmsrl(MSR_IA32_MCx_ADDR(i));
L
Linus Torvalds 已提交
951

952 953 954 955 956 957 958 959 960 961
		/*
		 * Action optional error. Queue address for later processing.
		 * When the ring overflows we just ignore the AO error.
		 * RED-PEN add some logging mechanism when
		 * usable_address or mce_add_ring fails.
		 * RED-PEN don't ignore overflow for tolerant == 0
		 */
		if (severity == MCE_AO_SEVERITY && mce_usable_address(&m))
			mce_ring_add(m.addr >> PAGE_SHIFT);

962
		mce_get_rip(&m, regs);
963
		mce_log(&m);
L
Linus Torvalds 已提交
964

965 966 967
		if (severity > worst) {
			*final = m;
			worst = severity;
L
Linus Torvalds 已提交
968 969 970
		}
	}

971 972 973
	if (!no_way_out)
		mce_clear_state(toclear);

I
Ingo Molnar 已提交
974
	/*
975 976
	 * Do most of the synchronization with other CPUs.
	 * When there's any problem use only local no_way_out state.
I
Ingo Molnar 已提交
977
	 */
978 979
	if (mce_end(order) < 0)
		no_way_out = worst >= MCE_PANIC_SEVERITY;
980 981 982

	/*
	 * If we have decided that we just CAN'T continue, and the user
I
Ingo Molnar 已提交
983
	 * has not set tolerant to an insane level, give up and die.
984 985 986
	 *
	 * This is mainly used in the case when the system doesn't
	 * support MCE broadcasting or it has been disabled.
987 988
	 */
	if (no_way_out && tolerant < 3)
989
		mce_panic("Fatal machine check on current CPU", final, msg);
990 991 992 993 994 995 996 997

	/*
	 * If the error seems to be unrecoverable, something should be
	 * done.  Try to kill as little as possible.  If we can kill just
	 * one task, do that.  If the user has set the tolerance very
	 * high, don't try to do anything at all.
	 */

A
Andi Kleen 已提交
998 999
	if (kill_it && tolerant < 3)
		force_sig(SIGBUS, current);
L
Linus Torvalds 已提交
1000

1001 1002 1003
	/* notify userspace ASAP */
	set_thread_flag(TIF_MCE_NOTIFY);

1004 1005
	if (worst > 0)
		mce_report_event(regs);
1006
	mce_wrmsrl(MSR_IA32_MCG_STATUS, 0);
1007
out:
1008
	atomic_dec(&mce_entry);
1009
	sync_core();
L
Linus Torvalds 已提交
1010
}
1011
EXPORT_SYMBOL_GPL(do_machine_check);
L
Linus Torvalds 已提交
1012

1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042
/* dummy to break dependency. actual code is in mm/memory-failure.c */
void __attribute__((weak)) memory_failure(unsigned long pfn, int vector)
{
	printk(KERN_ERR "Action optional memory failure at %lx ignored\n", pfn);
}

/*
 * Called after mce notification in process context. This code
 * is allowed to sleep. Call the high level VM handler to process
 * any corrupted pages.
 * Assume that the work queue code only calls this one at a time
 * per CPU.
 * Note we don't disable preemption, so this code might run on the wrong
 * CPU. In this case the event is picked up by the scheduled work queue.
 * This is merely a fast path to expedite processing in some common
 * cases.
 */
void mce_notify_process(void)
{
	unsigned long pfn;
	mce_notify_irq();
	while (mce_ring_get(&pfn))
		memory_failure(pfn, MCE_VECTOR);
}

static void mce_process_work(struct work_struct *dummy)
{
	mce_notify_process();
}

1043 1044 1045
#ifdef CONFIG_X86_MCE_INTEL
/***
 * mce_log_therm_throt_event - Logs the thermal throttling event to mcelog
S
Simon Arlott 已提交
1046
 * @cpu: The CPU on which the event occurred.
1047 1048 1049 1050 1051 1052 1053 1054 1055 1056
 * @status: Event status information
 *
 * This function should be called by the thermal interrupt after the
 * event has been processed and the decision was made to log the event
 * further.
 *
 * The status parameter will be saved to the 'status' field of 'struct mce'
 * and historically has been the register value of the
 * MSR_IA32_THERMAL_STATUS (Intel) msr.
 */
1057
void mce_log_therm_throt_event(__u64 status)
1058 1059 1060
{
	struct mce m;

1061
	mce_setup(&m);
1062 1063 1064 1065 1066 1067
	m.bank = MCE_THERMAL_BANK;
	m.status = status;
	mce_log(&m);
}
#endif /* CONFIG_X86_MCE_INTEL */

L
Linus Torvalds 已提交
1068
/*
1069 1070 1071
 * Periodic polling timer for "silent" machine check errors.  If the
 * poller finds an MCE, poll 2x faster.  When the poller finds no more
 * errors, poll 2x slower (up to check_interval seconds).
L
Linus Torvalds 已提交
1072 1073
 */
static int check_interval = 5 * 60; /* 5 minutes */
I
Ingo Molnar 已提交
1074

1075
static DEFINE_PER_CPU(int, next_interval); /* in jiffies */
1076
static DEFINE_PER_CPU(struct timer_list, mce_timer);
L
Linus Torvalds 已提交
1077

1078
static void mcheck_timer(unsigned long data)
L
Linus Torvalds 已提交
1079
{
1080
	struct timer_list *t = &per_cpu(mce_timer, data);
1081
	int *n;
1082 1083 1084

	WARN_ON(smp_processor_id() != data);

I
Ingo Molnar 已提交
1085
	if (mce_available(&current_cpu_data)) {
1086 1087
		machine_check_poll(MCP_TIMESTAMP,
				&__get_cpu_var(mce_poll_banks));
I
Ingo Molnar 已提交
1088
	}
L
Linus Torvalds 已提交
1089 1090

	/*
1091 1092
	 * Alert userspace if needed.  If we logged an MCE, reduce the
	 * polling interval, otherwise increase the polling interval.
L
Linus Torvalds 已提交
1093
	 */
1094
	n = &__get_cpu_var(next_interval);
1095
	if (mce_notify_irq())
1096
		*n = max(*n/2, HZ/100);
1097
	else
1098
		*n = min(*n*2, (int)round_jiffies_relative(check_interval*HZ));
1099

1100
	t->expires = jiffies + *n;
1101
	add_timer(t);
1102 1103
}

1104 1105
static void mce_do_trigger(struct work_struct *work)
{
1106
	call_usermodehelper(mce_helper, mce_helper_argv, NULL, UMH_NO_WAIT);
1107 1108 1109 1110
}

static DECLARE_WORK(mce_trigger_work, mce_do_trigger);

1111
/*
1112 1113 1114
 * Notify the user(s) about new machine check events.
 * Can be called from interrupt context, but not from machine check/NMI
 * context.
1115
 */
1116
int mce_notify_irq(void)
1117
{
1118 1119 1120
	/* Not more than two messages every minute */
	static DEFINE_RATELIMIT_STATE(ratelimit, 60*HZ, 2);

1121
	clear_thread_flag(TIF_MCE_NOTIFY);
I
Ingo Molnar 已提交
1122

1123
	if (test_and_clear_bit(0, &mce_need_notify)) {
1124
		wake_up_interruptible(&mce_wait);
1125 1126 1127 1128 1129 1130

		/*
		 * There is no risk of missing notifications because
		 * work_pending is always cleared before the function is
		 * executed.
		 */
1131
		if (mce_helper[0] && !work_pending(&mce_trigger_work))
1132
			schedule_work(&mce_trigger_work);
1133

1134
		if (__ratelimit(&ratelimit))
1135
			printk(KERN_INFO "Machine check events logged\n");
1136 1137

		return 1;
L
Linus Torvalds 已提交
1138
	}
1139 1140
	return 0;
}
1141
EXPORT_SYMBOL_GPL(mce_notify_irq);
1142

1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157
static int mce_banks_init(void)
{
	int i;

	mce_banks = kzalloc(banks * sizeof(struct mce_bank), GFP_KERNEL);
	if (!mce_banks)
		return -ENOMEM;
	for (i = 0; i < banks; i++) {
		struct mce_bank *b = &mce_banks[i];
		b->ctl = -1ULL;
		b->init = 1;
	}
	return 0;
}

1158
/*
L
Linus Torvalds 已提交
1159 1160
 * Initialize Machine Checks for a CPU.
 */
1161
static int mce_cap_init(void)
L
Linus Torvalds 已提交
1162
{
1163
	unsigned b;
I
Ingo Molnar 已提交
1164
	u64 cap;
L
Linus Torvalds 已提交
1165 1166

	rdmsrl(MSR_IA32_MCG_CAP, cap);
1167 1168

	b = cap & MCG_BANKCNT_MASK;
1169 1170
	printk(KERN_INFO "mce: CPU supports %d MCE banks\n", b);

1171 1172 1173 1174 1175 1176 1177 1178 1179 1180
	if (b > MAX_NR_BANKS) {
		printk(KERN_WARNING
		       "MCE: Using only %u machine check banks out of %u\n",
			MAX_NR_BANKS, b);
		b = MAX_NR_BANKS;
	}

	/* Don't support asymmetric configurations today */
	WARN_ON(banks != 0 && b != banks);
	banks = b;
1181 1182 1183 1184
	if (!mce_banks) {
		int err = mce_banks_init();
		if (err)
			return err;
L
Linus Torvalds 已提交
1185
	}
1186

1187
	/* Use accurate RIP reporting if available. */
1188
	if ((cap & MCG_EXT_P) && MCG_EXT_CNT(cap) >= 9)
1189
		rip_msr = MSR_IA32_MCG_EIP;
L
Linus Torvalds 已提交
1190

A
Andi Kleen 已提交
1191 1192 1193
	if (cap & MCG_SER_P)
		mce_ser = 1;

1194 1195 1196
	return 0;
}

1197
static void mce_init(void)
1198
{
I
Ingo Molnar 已提交
1199
	mce_banks_t all_banks;
1200 1201 1202
	u64 cap;
	int i;

1203 1204 1205
	/*
	 * Log the machine checks left over from the previous reset.
	 */
1206
	bitmap_fill(all_banks, MAX_NR_BANKS);
A
Andi Kleen 已提交
1207
	machine_check_poll(MCP_UC|(!mce_bootlog ? MCP_DONTLOG : 0), &all_banks);
L
Linus Torvalds 已提交
1208 1209 1210

	set_in_cr4(X86_CR4_MCE);

1211
	rdmsrl(MSR_IA32_MCG_CAP, cap);
L
Linus Torvalds 已提交
1212 1213 1214 1215
	if (cap & MCG_CTL_P)
		wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff);

	for (i = 0; i < banks; i++) {
1216 1217
		struct mce_bank *b = &mce_banks[i];
		if (!b->init)
1218
			continue;
1219 1220
		wrmsrl(MSR_IA32_MCx_CTL(i), b->ctl);
		wrmsrl(MSR_IA32_MCx_STATUS(i), 0);
1221
	}
L
Linus Torvalds 已提交
1222 1223 1224
}

/* Add per CPU specific workarounds here */
1225
static void mce_cpu_quirks(struct cpuinfo_x86 *c)
1226
{
L
Linus Torvalds 已提交
1227
	/* This should be disabled by the BIOS, but isn't always */
1228
	if (c->x86_vendor == X86_VENDOR_AMD) {
I
Ingo Molnar 已提交
1229 1230 1231 1232 1233 1234
		if (c->x86 == 15 && banks > 4) {
			/*
			 * disable GART TBL walk error reporting, which
			 * trips off incorrectly with the IOMMU & 3ware
			 * & Cerberus:
			 */
1235
			clear_bit(10, (unsigned long *)&mce_banks[4].ctl);
I
Ingo Molnar 已提交
1236 1237 1238 1239 1240 1241
		}
		if (c->x86 <= 17 && mce_bootlog < 0) {
			/*
			 * Lots of broken BIOS around that don't clear them
			 * by default and leave crap in there. Don't log:
			 */
1242
			mce_bootlog = 0;
I
Ingo Molnar 已提交
1243
		}
1244 1245 1246 1247
		/*
		 * Various K7s with broken bank 0 around. Always disable
		 * by default.
		 */
1248
		 if (c->x86 == 6 && banks > 0)
1249
			mce_banks[0].ctl = 0;
L
Linus Torvalds 已提交
1250
	}
1251

1252 1253 1254 1255 1256 1257 1258 1259 1260 1261
	if (c->x86_vendor == X86_VENDOR_INTEL) {
		/*
		 * SDM documents that on family 6 bank 0 should not be written
		 * because it aliases to another special BIOS controlled
		 * register.
		 * But it's not aliased anymore on model 0x1a+
		 * Don't ignore bank 0 completely because there could be a
		 * valid event later, merely don't write CTL0.
		 */

1262 1263
		if (c->x86 == 6 && c->x86_model < 0x1A && banks > 0)
			mce_banks[0].init = 0;
1264 1265 1266 1267 1268 1269 1270 1271

		/*
		 * All newer Intel systems support MCE broadcasting. Enable
		 * synchronization with a one second timeout.
		 */
		if ((c->x86 > 6 || (c->x86 == 6 && c->x86_model >= 0xe)) &&
			monarch_timeout < 0)
			monarch_timeout = USEC_PER_SEC;
1272 1273 1274 1275

		/* There are also broken BIOSes on some Pentium M systems. */
		if (c->x86 == 6 && c->x86_model == 13 && mce_bootlog < 0)
			mce_bootlog = 0;
1276
	}
1277 1278
	if (monarch_timeout < 0)
		monarch_timeout = 0;
1279 1280
	if (mce_bootlog != 0)
		mce_panic_timeout = 30;
1281
}
L
Linus Torvalds 已提交
1282

1283 1284 1285 1286 1287 1288
static void __cpuinit mce_ancient_init(struct cpuinfo_x86 *c)
{
	if (c->x86 != 5)
		return;
	switch (c->x86_vendor) {
	case X86_VENDOR_INTEL:
1289
		intel_p5_mcheck_init(c);
1290 1291 1292 1293 1294 1295 1296
		break;
	case X86_VENDOR_CENTAUR:
		winchip_mcheck_init(c);
		break;
	}
}

1297
static void mce_cpu_features(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1298 1299 1300 1301 1302
{
	switch (c->x86_vendor) {
	case X86_VENDOR_INTEL:
		mce_intel_feature_init(c);
		break;
1303 1304 1305
	case X86_VENDOR_AMD:
		mce_amd_feature_init(c);
		break;
L
Linus Torvalds 已提交
1306 1307 1308 1309 1310
	default:
		break;
	}
}

1311 1312 1313
static void mce_init_timer(void)
{
	struct timer_list *t = &__get_cpu_var(mce_timer);
1314
	int *n = &__get_cpu_var(next_interval);
1315

1316 1317 1318
	if (mce_ignore_ce)
		return;

1319 1320
	*n = check_interval * HZ;
	if (!*n)
1321 1322
		return;
	setup_timer(t, mcheck_timer, smp_processor_id());
1323
	t->expires = round_jiffies(jiffies + *n);
1324 1325 1326
	add_timer(t);
}

A
Andi Kleen 已提交
1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337
/* Handle unconfigured int18 (should never happen) */
static void unexpected_machine_check(struct pt_regs *regs, long error_code)
{
	printk(KERN_ERR "CPU#%d: Unexpected int18 (Machine Check).\n",
	       smp_processor_id());
}

/* Call the installed machine check handler for this CPU setup. */
void (*machine_check_vector)(struct pt_regs *, long error_code) =
						unexpected_machine_check;

1338
/*
L
Linus Torvalds 已提交
1339
 * Called for each booted CPU to set up machine checks.
I
Ingo Molnar 已提交
1340
 * Must be called with preempt off:
L
Linus Torvalds 已提交
1341
 */
1342
void __cpuinit mcheck_init(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1343
{
1344 1345 1346 1347 1348
	if (mce_disabled)
		return;

	mce_ancient_init(c);

1349
	if (!mce_available(c))
L
Linus Torvalds 已提交
1350 1351
		return;

1352
	if (mce_cap_init() < 0) {
1353
		mce_disabled = 1;
1354 1355 1356 1357
		return;
	}
	mce_cpu_quirks(c);

1358 1359
	machine_check_vector = do_machine_check;

1360
	mce_init();
L
Linus Torvalds 已提交
1361
	mce_cpu_features(c);
1362
	mce_init_timer();
1363
	INIT_WORK(&__get_cpu_var(mce_work), mce_process_work);
L
Linus Torvalds 已提交
1364 1365 1366 1367 1368 1369
}

/*
 * Character device to read and clear the MCE log.
 */

T
Tim Hockin 已提交
1370
static DEFINE_SPINLOCK(mce_state_lock);
I
Ingo Molnar 已提交
1371 1372
static int		open_count;		/* #times opened */
static int		open_exclu;		/* already open exclusive? */
T
Tim Hockin 已提交
1373 1374 1375 1376 1377 1378 1379

static int mce_open(struct inode *inode, struct file *file)
{
	spin_lock(&mce_state_lock);

	if (open_exclu || (open_count && (file->f_flags & O_EXCL))) {
		spin_unlock(&mce_state_lock);
I
Ingo Molnar 已提交
1380

T
Tim Hockin 已提交
1381 1382 1383 1384 1385 1386 1387 1388 1389
		return -EBUSY;
	}

	if (file->f_flags & O_EXCL)
		open_exclu = 1;
	open_count++;

	spin_unlock(&mce_state_lock);

1390
	return nonseekable_open(inode, file);
T
Tim Hockin 已提交
1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404
}

static int mce_release(struct inode *inode, struct file *file)
{
	spin_lock(&mce_state_lock);

	open_count--;
	open_exclu = 0;

	spin_unlock(&mce_state_lock);

	return 0;
}

1405 1406
static void collect_tscs(void *data)
{
L
Linus Torvalds 已提交
1407
	unsigned long *cpu_tsc = (unsigned long *)data;
1408

L
Linus Torvalds 已提交
1409
	rdtscll(cpu_tsc[smp_processor_id()]);
1410
}
L
Linus Torvalds 已提交
1411

I
Ingo Molnar 已提交
1412 1413
static DEFINE_MUTEX(mce_read_mutex);

1414 1415
static ssize_t mce_read(struct file *filp, char __user *ubuf, size_t usize,
			loff_t *off)
L
Linus Torvalds 已提交
1416
{
I
Ingo Molnar 已提交
1417
	char __user *buf = ubuf;
1418
	unsigned long *cpu_tsc;
1419
	unsigned prev, next;
L
Linus Torvalds 已提交
1420 1421
	int i, err;

1422
	cpu_tsc = kmalloc(nr_cpu_ids * sizeof(long), GFP_KERNEL);
1423 1424 1425
	if (!cpu_tsc)
		return -ENOMEM;

1426
	mutex_lock(&mce_read_mutex);
L
Linus Torvalds 已提交
1427 1428 1429
	next = rcu_dereference(mcelog.next);

	/* Only supports full reads right now */
1430
	if (*off != 0 || usize < MCE_LOG_LEN*sizeof(struct mce)) {
1431
		mutex_unlock(&mce_read_mutex);
1432
		kfree(cpu_tsc);
I
Ingo Molnar 已提交
1433

L
Linus Torvalds 已提交
1434 1435 1436 1437
		return -EINVAL;
	}

	err = 0;
1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449
	prev = 0;
	do {
		for (i = prev; i < next; i++) {
			unsigned long start = jiffies;

			while (!mcelog.entry[i].finished) {
				if (time_after_eq(jiffies, start + 2)) {
					memset(mcelog.entry + i, 0,
					       sizeof(struct mce));
					goto timeout;
				}
				cpu_relax();
1450
			}
1451 1452 1453 1454 1455 1456
			smp_rmb();
			err |= copy_to_user(buf, mcelog.entry + i,
					    sizeof(struct mce));
			buf += sizeof(struct mce);
timeout:
			;
1457
		}
L
Linus Torvalds 已提交
1458

1459 1460 1461 1462 1463
		memset(mcelog.entry + prev, 0,
		       (next - prev) * sizeof(struct mce));
		prev = next;
		next = cmpxchg(&mcelog.next, prev, 0);
	} while (next != prev);
L
Linus Torvalds 已提交
1464

1465
	synchronize_sched();
L
Linus Torvalds 已提交
1466

1467 1468 1469 1470
	/*
	 * Collect entries that were still getting written before the
	 * synchronize.
	 */
1471
	on_each_cpu(collect_tscs, cpu_tsc, 1);
I
Ingo Molnar 已提交
1472

1473 1474 1475 1476 1477
	for (i = next; i < MCE_LOG_LEN; i++) {
		if (mcelog.entry[i].finished &&
		    mcelog.entry[i].tsc < cpu_tsc[mcelog.entry[i].cpu]) {
			err |= copy_to_user(buf, mcelog.entry+i,
					    sizeof(struct mce));
L
Linus Torvalds 已提交
1478 1479 1480 1481
			smp_rmb();
			buf += sizeof(struct mce);
			memset(&mcelog.entry[i], 0, sizeof(struct mce));
		}
1482
	}
1483
	mutex_unlock(&mce_read_mutex);
1484
	kfree(cpu_tsc);
I
Ingo Molnar 已提交
1485

1486
	return err ? -EFAULT : buf - ubuf;
L
Linus Torvalds 已提交
1487 1488
}

1489 1490 1491 1492 1493 1494 1495 1496
static unsigned int mce_poll(struct file *file, poll_table *wait)
{
	poll_wait(file, &mce_wait, wait);
	if (rcu_dereference(mcelog.next))
		return POLLIN | POLLRDNORM;
	return 0;
}

1497
static long mce_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
L
Linus Torvalds 已提交
1498 1499
{
	int __user *p = (int __user *)arg;
1500

L
Linus Torvalds 已提交
1501
	if (!capable(CAP_SYS_ADMIN))
1502
		return -EPERM;
I
Ingo Molnar 已提交
1503

L
Linus Torvalds 已提交
1504
	switch (cmd) {
1505
	case MCE_GET_RECORD_LEN:
L
Linus Torvalds 已提交
1506 1507
		return put_user(sizeof(struct mce), p);
	case MCE_GET_LOG_LEN:
1508
		return put_user(MCE_LOG_LEN, p);
L
Linus Torvalds 已提交
1509 1510
	case MCE_GETCLEAR_FLAGS: {
		unsigned flags;
1511 1512

		do {
L
Linus Torvalds 已提交
1513
			flags = mcelog.flags;
1514
		} while (cmpxchg(&mcelog.flags, flags, 0) != flags);
I
Ingo Molnar 已提交
1515

1516
		return put_user(flags, p);
L
Linus Torvalds 已提交
1517 1518
	}
	default:
1519 1520
		return -ENOTTY;
	}
L
Linus Torvalds 已提交
1521 1522
}

1523
/* Modified in mce-inject.c, so not static or const */
1524
struct file_operations mce_chrdev_ops = {
I
Ingo Molnar 已提交
1525 1526 1527 1528 1529
	.open			= mce_open,
	.release		= mce_release,
	.read			= mce_read,
	.poll			= mce_poll,
	.unlocked_ioctl		= mce_ioctl,
L
Linus Torvalds 已提交
1530
};
1531
EXPORT_SYMBOL_GPL(mce_chrdev_ops);
L
Linus Torvalds 已提交
1532 1533 1534 1535 1536 1537 1538

static struct miscdevice mce_log_device = {
	MISC_MCELOG_MINOR,
	"mcelog",
	&mce_chrdev_ops,
};

H
Hidetoshi Seto 已提交
1539
/*
1540 1541 1542 1543
 * mce=off Disables machine check
 * mce=no_cmci Disables CMCI
 * mce=dont_log_ce Clears corrected events silently, no log created for CEs.
 * mce=ignore_ce Disables polling and CMCI, corrected events are not cleared.
1544 1545 1546
 * mce=TOLERANCELEVEL[,monarchtimeout] (number, see above)
 *	monarchtimeout is how long to wait for other CPUs on machine
 *	check, or 0 to not wait
H
Hidetoshi Seto 已提交
1547 1548 1549
 * mce=bootlog Log MCEs from before booting. Disabled by default on AMD.
 * mce=nobootlog Don't log MCEs from before booting.
 */
L
Linus Torvalds 已提交
1550 1551
static int __init mcheck_enable(char *str)
{
1552
	if (*str == 0) {
1553
		enable_p5_mce();
1554 1555
		return 1;
	}
1556 1557
	if (*str == '=')
		str++;
L
Linus Torvalds 已提交
1558
	if (!strcmp(str, "off"))
1559
		mce_disabled = 1;
1560 1561 1562 1563 1564 1565
	else if (!strcmp(str, "no_cmci"))
		mce_cmci_disabled = 1;
	else if (!strcmp(str, "dont_log_ce"))
		mce_dont_log_ce = 1;
	else if (!strcmp(str, "ignore_ce"))
		mce_ignore_ce = 1;
H
Hidetoshi Seto 已提交
1566 1567
	else if (!strcmp(str, "bootlog") || !strcmp(str, "nobootlog"))
		mce_bootlog = (str[0] == 'b');
1568
	else if (isdigit(str[0])) {
1569
		get_option(&str, &tolerant);
1570 1571 1572 1573 1574
		if (*str == ',') {
			++str;
			get_option(&str, &monarch_timeout);
		}
	} else {
1575
		printk(KERN_INFO "mce argument %s ignored. Please use /sys\n",
H
Hidetoshi Seto 已提交
1576 1577 1578
		       str);
		return 0;
	}
1579
	return 1;
L
Linus Torvalds 已提交
1580
}
1581
__setup("mce", mcheck_enable);
L
Linus Torvalds 已提交
1582

1583
/*
L
Linus Torvalds 已提交
1584
 * Sysfs support
1585
 */
L
Linus Torvalds 已提交
1586

1587 1588 1589 1590 1591 1592 1593 1594
/*
 * Disable machine checks on suspend and shutdown. We can't really handle
 * them later.
 */
static int mce_disable(void)
{
	int i;

1595
	for (i = 0; i < banks; i++) {
1596 1597
		struct mce_bank *b = &mce_banks[i];
		if (b->init)
1598
			wrmsrl(MSR_IA32_MCx_CTL(i), 0);
1599
	}
1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612
	return 0;
}

static int mce_suspend(struct sys_device *dev, pm_message_t state)
{
	return mce_disable();
}

static int mce_shutdown(struct sys_device *dev)
{
	return mce_disable();
}

I
Ingo Molnar 已提交
1613 1614 1615 1616 1617
/*
 * On resume clear all MCE state. Don't want to see leftovers from the BIOS.
 * Only one CPU is active at this time, the others get re-added later using
 * CPU hotplug:
 */
L
Linus Torvalds 已提交
1618 1619
static int mce_resume(struct sys_device *dev)
{
1620
	mce_init();
1621
	mce_cpu_features(&current_cpu_data);
I
Ingo Molnar 已提交
1622

L
Linus Torvalds 已提交
1623 1624 1625
	return 0;
}

1626 1627 1628
static void mce_cpu_restart(void *data)
{
	del_timer_sync(&__get_cpu_var(mce_timer));
1629 1630 1631
	if (!mce_available(&current_cpu_data))
		return;
	mce_init();
1632 1633 1634
	mce_init_timer();
}

L
Linus Torvalds 已提交
1635
/* Reinit MCEs after user configuration changes */
1636 1637
static void mce_restart(void)
{
1638
	on_each_cpu(mce_cpu_restart, NULL, 1);
L
Linus Torvalds 已提交
1639 1640
}

1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660
/* Toggle features for corrected errors */
static void mce_disable_ce(void *all)
{
	if (!mce_available(&current_cpu_data))
		return;
	if (all)
		del_timer_sync(&__get_cpu_var(mce_timer));
	cmci_clear();
}

static void mce_enable_ce(void *all)
{
	if (!mce_available(&current_cpu_data))
		return;
	cmci_reenable();
	cmci_recheck();
	if (all)
		mce_init_timer();
}

L
Linus Torvalds 已提交
1661
static struct sysdev_class mce_sysclass = {
I
Ingo Molnar 已提交
1662 1663 1664 1665
	.suspend	= mce_suspend,
	.shutdown	= mce_shutdown,
	.resume		= mce_resume,
	.name		= "machinecheck",
L
Linus Torvalds 已提交
1666 1667
};

I
Ingo Molnar 已提交
1668
DEFINE_PER_CPU(struct sys_device, mce_dev);
I
Ingo Molnar 已提交
1669 1670 1671

__cpuinitdata
void (*threshold_cpu_callback)(unsigned long action, unsigned int cpu);
L
Linus Torvalds 已提交
1672

1673 1674 1675 1676
static inline struct mce_bank *attr_to_bank(struct sysdev_attribute *attr)
{
	return container_of(attr, struct mce_bank, attr);
}
1677 1678 1679 1680

static ssize_t show_bank(struct sys_device *s, struct sysdev_attribute *attr,
			 char *buf)
{
1681
	return sprintf(buf, "%llx\n", attr_to_bank(attr)->ctl);
1682 1683 1684
}

static ssize_t set_bank(struct sys_device *s, struct sysdev_attribute *attr,
H
Hidetoshi Seto 已提交
1685
			const char *buf, size_t size)
1686
{
H
Hidetoshi Seto 已提交
1687
	u64 new;
I
Ingo Molnar 已提交
1688

H
Hidetoshi Seto 已提交
1689
	if (strict_strtoull(buf, 0, &new) < 0)
1690
		return -EINVAL;
I
Ingo Molnar 已提交
1691

1692
	attr_to_bank(attr)->ctl = new;
1693
	mce_restart();
I
Ingo Molnar 已提交
1694

H
Hidetoshi Seto 已提交
1695
	return size;
1696
}
1697

I
Ingo Molnar 已提交
1698 1699
static ssize_t
show_trigger(struct sys_device *s, struct sysdev_attribute *attr, char *buf)
1700
{
1701
	strcpy(buf, mce_helper);
1702
	strcat(buf, "\n");
1703
	return strlen(mce_helper) + 1;
1704 1705
}

1706
static ssize_t set_trigger(struct sys_device *s, struct sysdev_attribute *attr,
I
Ingo Molnar 已提交
1707
				const char *buf, size_t siz)
1708 1709 1710
{
	char *p;
	int len;
I
Ingo Molnar 已提交
1711

1712 1713 1714 1715
	strncpy(mce_helper, buf, sizeof(mce_helper));
	mce_helper[sizeof(mce_helper)-1] = 0;
	len = strlen(mce_helper);
	p = strchr(mce_helper, '\n');
I
Ingo Molnar 已提交
1716 1717 1718 1719

	if (*p)
		*p = 0;

1720 1721 1722
	return len;
}

1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768
static ssize_t set_ignore_ce(struct sys_device *s,
			     struct sysdev_attribute *attr,
			     const char *buf, size_t size)
{
	u64 new;

	if (strict_strtoull(buf, 0, &new) < 0)
		return -EINVAL;

	if (mce_ignore_ce ^ !!new) {
		if (new) {
			/* disable ce features */
			on_each_cpu(mce_disable_ce, (void *)1, 1);
			mce_ignore_ce = 1;
		} else {
			/* enable ce features */
			mce_ignore_ce = 0;
			on_each_cpu(mce_enable_ce, (void *)1, 1);
		}
	}
	return size;
}

static ssize_t set_cmci_disabled(struct sys_device *s,
				 struct sysdev_attribute *attr,
				 const char *buf, size_t size)
{
	u64 new;

	if (strict_strtoull(buf, 0, &new) < 0)
		return -EINVAL;

	if (mce_cmci_disabled ^ !!new) {
		if (new) {
			/* disable cmci */
			on_each_cpu(mce_disable_ce, NULL, 1);
			mce_cmci_disabled = 1;
		} else {
			/* enable cmci */
			mce_cmci_disabled = 0;
			on_each_cpu(mce_enable_ce, NULL, 1);
		}
	}
	return size;
}

1769 1770 1771 1772 1773 1774 1775 1776 1777
static ssize_t store_int_with_restart(struct sys_device *s,
				      struct sysdev_attribute *attr,
				      const char *buf, size_t size)
{
	ssize_t ret = sysdev_store_int(s, attr, buf, size);
	mce_restart();
	return ret;
}

1778
static SYSDEV_ATTR(trigger, 0644, show_trigger, set_trigger);
1779
static SYSDEV_INT_ATTR(tolerant, 0644, tolerant);
1780
static SYSDEV_INT_ATTR(monarch_timeout, 0644, monarch_timeout);
1781
static SYSDEV_INT_ATTR(dont_log_ce, 0644, mce_dont_log_ce);
I
Ingo Molnar 已提交
1782

1783 1784 1785 1786 1787
static struct sysdev_ext_attribute attr_check_interval = {
	_SYSDEV_ATTR(check_interval, 0644, sysdev_show_int,
		     store_int_with_restart),
	&check_interval
};
I
Ingo Molnar 已提交
1788

1789 1790 1791 1792 1793 1794
static struct sysdev_ext_attribute attr_ignore_ce = {
	_SYSDEV_ATTR(ignore_ce, 0644, sysdev_show_int, set_ignore_ce),
	&mce_ignore_ce
};

static struct sysdev_ext_attribute attr_cmci_disabled = {
Y
Yinghai Lu 已提交
1795
	_SYSDEV_ATTR(cmci_disabled, 0644, sysdev_show_int, set_cmci_disabled),
1796 1797 1798
	&mce_cmci_disabled
};

I
Ingo Molnar 已提交
1799
static struct sysdev_attribute *mce_attrs[] = {
1800 1801 1802
	&attr_tolerant.attr,
	&attr_check_interval.attr,
	&attr_trigger,
1803
	&attr_monarch_timeout.attr,
1804 1805 1806
	&attr_dont_log_ce.attr,
	&attr_ignore_ce.attr,
	&attr_cmci_disabled.attr,
1807 1808
	NULL
};
L
Linus Torvalds 已提交
1809

I
Ingo Molnar 已提交
1810
static cpumask_var_t mce_dev_initialized;
1811

I
Ingo Molnar 已提交
1812
/* Per cpu sysdev init. All of the cpus still share the same ctrl bank: */
1813
static __cpuinit int mce_create_device(unsigned int cpu)
L
Linus Torvalds 已提交
1814 1815
{
	int err;
1816
	int i, j;
1817

A
Andreas Herrmann 已提交
1818
	if (!mce_available(&boot_cpu_data))
1819 1820
		return -EIO;

I
Ingo Molnar 已提交
1821 1822 1823
	memset(&per_cpu(mce_dev, cpu).kobj, 0, sizeof(struct kobject));
	per_cpu(mce_dev, cpu).id	= cpu;
	per_cpu(mce_dev, cpu).cls	= &mce_sysclass;
1824

I
Ingo Molnar 已提交
1825
	err = sysdev_register(&per_cpu(mce_dev, cpu));
1826 1827 1828
	if (err)
		return err;

I
Ingo Molnar 已提交
1829 1830
	for (i = 0; mce_attrs[i]; i++) {
		err = sysdev_create_file(&per_cpu(mce_dev, cpu), mce_attrs[i]);
1831 1832 1833
		if (err)
			goto error;
	}
1834
	for (j = 0; j < banks; j++) {
I
Ingo Molnar 已提交
1835
		err = sysdev_create_file(&per_cpu(mce_dev, cpu),
1836
					&mce_banks[j].attr);
1837 1838 1839
		if (err)
			goto error2;
	}
I
Ingo Molnar 已提交
1840
	cpumask_set_cpu(cpu, mce_dev_initialized);
1841

1842
	return 0;
1843
error2:
1844
	while (--j >= 0)
1845
		sysdev_remove_file(&per_cpu(mce_dev, cpu), &mce_banks[j].attr);
1846
error:
I
Ingo Molnar 已提交
1847
	while (--i >= 0)
1848
		sysdev_remove_file(&per_cpu(mce_dev, cpu), &mce_banks[i].attr);
I
Ingo Molnar 已提交
1849 1850

	sysdev_unregister(&per_cpu(mce_dev, cpu));
1851

1852 1853 1854
	return err;
}

1855
static __cpuinit void mce_remove_device(unsigned int cpu)
1856
{
1857 1858
	int i;

I
Ingo Molnar 已提交
1859
	if (!cpumask_test_cpu(cpu, mce_dev_initialized))
1860 1861
		return;

I
Ingo Molnar 已提交
1862 1863 1864
	for (i = 0; mce_attrs[i]; i++)
		sysdev_remove_file(&per_cpu(mce_dev, cpu), mce_attrs[i]);

1865
	for (i = 0; i < banks; i++)
1866
		sysdev_remove_file(&per_cpu(mce_dev, cpu), &mce_banks[i].attr);
I
Ingo Molnar 已提交
1867 1868 1869

	sysdev_unregister(&per_cpu(mce_dev, cpu));
	cpumask_clear_cpu(cpu, mce_dev_initialized);
1870 1871
}

1872
/* Make sure there are no machine checks on offlined CPUs. */
1873
static void mce_disable_cpu(void *h)
1874
{
A
Andi Kleen 已提交
1875
	unsigned long action = *(unsigned long *)h;
I
Ingo Molnar 已提交
1876
	int i;
1877 1878 1879

	if (!mce_available(&current_cpu_data))
		return;
A
Andi Kleen 已提交
1880 1881
	if (!(action & CPU_TASKS_FROZEN))
		cmci_clear();
1882
	for (i = 0; i < banks; i++) {
1883 1884
		struct mce_bank *b = &mce_banks[i];
		if (b->init)
1885
			wrmsrl(MSR_IA32_MCx_CTL(i), 0);
1886
	}
1887 1888
}

1889
static void mce_reenable_cpu(void *h)
1890
{
A
Andi Kleen 已提交
1891
	unsigned long action = *(unsigned long *)h;
I
Ingo Molnar 已提交
1892
	int i;
1893 1894 1895

	if (!mce_available(&current_cpu_data))
		return;
I
Ingo Molnar 已提交
1896

A
Andi Kleen 已提交
1897 1898
	if (!(action & CPU_TASKS_FROZEN))
		cmci_reenable();
1899
	for (i = 0; i < banks; i++) {
1900 1901
		struct mce_bank *b = &mce_banks[i];
		if (b->init)
1902
			wrmsrl(MSR_IA32_MCx_CTL(i), b->ctl);
1903
	}
1904 1905
}

1906
/* Get notified when a cpu comes on/off. Be hotplug friendly. */
I
Ingo Molnar 已提交
1907 1908
static int __cpuinit
mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
1909 1910
{
	unsigned int cpu = (unsigned long)hcpu;
1911
	struct timer_list *t = &per_cpu(mce_timer, cpu);
1912 1913

	switch (action) {
1914 1915 1916
	case CPU_ONLINE:
	case CPU_ONLINE_FROZEN:
		mce_create_device(cpu);
1917 1918
		if (threshold_cpu_callback)
			threshold_cpu_callback(action, cpu);
1919 1920
		break;
	case CPU_DEAD:
1921
	case CPU_DEAD_FROZEN:
1922 1923
		if (threshold_cpu_callback)
			threshold_cpu_callback(action, cpu);
1924 1925
		mce_remove_device(cpu);
		break;
1926 1927 1928
	case CPU_DOWN_PREPARE:
	case CPU_DOWN_PREPARE_FROZEN:
		del_timer_sync(t);
A
Andi Kleen 已提交
1929
		smp_call_function_single(cpu, mce_disable_cpu, &action, 1);
1930 1931 1932
		break;
	case CPU_DOWN_FAILED:
	case CPU_DOWN_FAILED_FROZEN:
1933 1934
		t->expires = round_jiffies(jiffies +
						__get_cpu_var(next_interval));
1935
		add_timer_on(t, cpu);
A
Andi Kleen 已提交
1936 1937 1938 1939 1940
		smp_call_function_single(cpu, mce_reenable_cpu, &action, 1);
		break;
	case CPU_POST_DEAD:
		/* intentionally ignoring frozen here */
		cmci_rediscover(cpu);
1941
		break;
1942
	}
1943
	return NOTIFY_OK;
1944 1945
}

1946
static struct notifier_block mce_cpu_notifier __cpuinitdata = {
1947 1948 1949
	.notifier_call = mce_cpu_callback,
};

1950
static __init void mce_init_banks(void)
1951 1952 1953 1954
{
	int i;

	for (i = 0; i < banks; i++) {
1955 1956
		struct mce_bank *b = &mce_banks[i];
		struct sysdev_attribute *a = &b->attr;
I
Ingo Molnar 已提交
1957

1958 1959
		a->attr.name	= b->attrname;
		snprintf(b->attrname, ATTR_LEN, "bank%d", i);
I
Ingo Molnar 已提交
1960 1961 1962 1963

		a->attr.mode	= 0644;
		a->show		= show_bank;
		a->store	= set_bank;
1964 1965 1966
	}
}

1967 1968 1969 1970 1971
static __init int mce_init_device(void)
{
	int err;
	int i = 0;

L
Linus Torvalds 已提交
1972 1973
	if (!mce_available(&boot_cpu_data))
		return -EIO;
1974

1975
	zalloc_cpumask_var(&mce_dev_initialized, GFP_KERNEL);
1976

1977
	mce_init_banks();
1978

L
Linus Torvalds 已提交
1979
	err = sysdev_class_register(&mce_sysclass);
1980 1981
	if (err)
		return err;
1982 1983

	for_each_online_cpu(i) {
1984 1985 1986
		err = mce_create_device(i);
		if (err)
			return err;
1987 1988
	}

1989
	register_hotcpu_notifier(&mce_cpu_notifier);
L
Linus Torvalds 已提交
1990
	misc_register(&mce_log_device);
I
Ingo Molnar 已提交
1991

L
Linus Torvalds 已提交
1992 1993
	return err;
}
1994

L
Linus Torvalds 已提交
1995
device_initcall(mce_init_device);
I
Ingo Molnar 已提交
1996

1997 1998 1999 2000 2001 2002 2003 2004 2005
/*
 * Old style boot options parsing. Only for compatibility.
 */
static int __init mcheck_disable(char *str)
{
	mce_disabled = 1;
	return 1;
}
__setup("nomce", mcheck_disable);