mce.c 48.9 KB
Newer Older
L
Linus Torvalds 已提交
1 2
/*
 * Machine check handler.
I
Ingo Molnar 已提交
3
 *
L
Linus Torvalds 已提交
4
 * K8 parts Copyright 2002,2003 Andi Kleen, SuSE Labs.
5 6
 * Rest from unknown author(s).
 * 2004 Andi Kleen. Rewrote most of it.
7 8
 * Copyright 2008 Intel Corporation
 * Author: Andi Kleen
L
Linus Torvalds 已提交
9
 */
I
Ingo Molnar 已提交
10 11 12
#include <linux/thread_info.h>
#include <linux/capability.h>
#include <linux/miscdevice.h>
13
#include <linux/interrupt.h>
I
Ingo Molnar 已提交
14 15 16 17
#include <linux/ratelimit.h>
#include <linux/kallsyms.h>
#include <linux/rcupdate.h>
#include <linux/kobject.h>
18
#include <linux/uaccess.h>
I
Ingo Molnar 已提交
19 20 21
#include <linux/kdebug.h>
#include <linux/kernel.h>
#include <linux/percpu.h>
L
Linus Torvalds 已提交
22 23
#include <linux/string.h>
#include <linux/sysdev.h>
24
#include <linux/delay.h>
25
#include <linux/ctype.h>
I
Ingo Molnar 已提交
26
#include <linux/sched.h>
27
#include <linux/sysfs.h>
I
Ingo Molnar 已提交
28
#include <linux/types.h>
29
#include <linux/slab.h>
I
Ingo Molnar 已提交
30 31 32
#include <linux/init.h>
#include <linux/kmod.h>
#include <linux/poll.h>
33
#include <linux/nmi.h>
I
Ingo Molnar 已提交
34
#include <linux/cpu.h>
35
#include <linux/smp.h>
I
Ingo Molnar 已提交
36
#include <linux/fs.h>
37
#include <linux/mm.h>
38
#include <linux/debugfs.h>
I
Ingo Molnar 已提交
39

40
#include <asm/processor.h>
41 42
#include <asm/hw_irq.h>
#include <asm/apic.h>
43
#include <asm/idle.h>
44
#include <asm/ipi.h>
I
Ingo Molnar 已提交
45 46
#include <asm/mce.h>
#include <asm/msr.h>
L
Linus Torvalds 已提交
47

48
#include "mce-internal.h"
49

50 51
static DEFINE_MUTEX(mce_read_mutex);

52 53 54 55 56
#define rcu_dereference_check_mce(p) \
	rcu_dereference_check((p), \
			      rcu_read_lock_sched_held() || \
			      lockdep_is_held(&mce_read_mutex))

57 58 59
#define CREATE_TRACE_POINTS
#include <trace/events/mce.h>

H
Hidetoshi Seto 已提交
60
int mce_disabled __read_mostly;
61

I
Ingo Molnar 已提交
62
#define MISC_MCELOG_MINOR	227
63

64 65
#define SPINUNIT 100	/* 100ns */

66 67
atomic_t mce_entry;

68 69
DEFINE_PER_CPU(unsigned, mce_exception_count);

70 71 72 73 74 75 76
/*
 * Tolerant levels:
 *   0: always panic on uncorrected errors, log corrected errors
 *   1: panic or SIGBUS on uncorrected errors, log corrected errors
 *   2: SIGBUS or log uncorrected errors (if possible), log corrected errors
 *   3: never panic or SIGBUS, log all errors (for testing only)
 */
H
Hidetoshi Seto 已提交
77 78 79 80 81 82 83 84 85 86
static int			tolerant		__read_mostly = 1;
static int			banks			__read_mostly;
static int			rip_msr			__read_mostly;
static int			mce_bootlog		__read_mostly = -1;
static int			monarch_timeout		__read_mostly = -1;
static int			mce_panic_timeout	__read_mostly;
static int			mce_dont_log_ce		__read_mostly;
int				mce_cmci_disabled	__read_mostly;
int				mce_ignore_ce		__read_mostly;
int				mce_ser			__read_mostly;
87

88 89
struct mce_bank                *mce_banks		__read_mostly;

90 91 92 93
/* User mode helper program triggered by machine check event */
static unsigned long		mce_need_notify;
static char			mce_helper[128];
static char			*mce_helper_argv[2] = { mce_helper, NULL };
L
Linus Torvalds 已提交
94

95
static DECLARE_WAIT_QUEUE_HEAD(mce_wait);
96 97 98
static DEFINE_PER_CPU(struct mce, mces_seen);
static int			cpu_missing;

99 100 101 102 103 104 105 106 107
/*
 * CPU/chipset specific EDAC code can register a notifier call here to print
 * MCE errors in a human-readable form.
 */
ATOMIC_NOTIFIER_HEAD(x86_mce_decoder_chain);
EXPORT_SYMBOL_GPL(x86_mce_decoder_chain);

static int default_decode_mce(struct notifier_block *nb, unsigned long val,
			       void *data)
108 109 110
{
	pr_emerg("No human readable MCE decoding support on this CPU type.\n");
	pr_emerg("Run the message through 'mcelog --ascii' to decode.\n");
111 112

	return NOTIFY_STOP;
113 114
}

115 116 117 118
static struct notifier_block mce_dec_nb = {
	.notifier_call = default_decode_mce,
	.priority      = -1,
};
119

120 121 122 123 124
/* MCA banks polled by the period polling timer for corrected events */
DEFINE_PER_CPU(mce_banks_t, mce_poll_banks) = {
	[0 ... BITS_TO_LONGS(MAX_NR_BANKS)-1] = ~0UL
};

125 126
static DEFINE_PER_CPU(struct work_struct, mce_work);

127 128 129 130
/* Do initial initialization of a struct mce */
void mce_setup(struct mce *m)
{
	memset(m, 0, sizeof(struct mce));
131
	m->cpu = m->extcpu = smp_processor_id();
132
	rdtscll(m->tsc);
133 134 135 136 137 138 139 140 141
	/* We hope get_seconds stays lockless */
	m->time = get_seconds();
	m->cpuvendor = boot_cpu_data.x86_vendor;
	m->cpuid = cpuid_eax(1);
#ifdef CONFIG_SMP
	m->socketid = cpu_data(m->extcpu).phys_proc_id;
#endif
	m->apicid = cpu_data(m->extcpu).initial_apicid;
	rdmsrl(MSR_IA32_MCG_CAP, m->mcgcap);
142 143
}

144 145 146
DEFINE_PER_CPU(struct mce, injectm);
EXPORT_PER_CPU_SYMBOL_GPL(injectm);

L
Linus Torvalds 已提交
147 148 149 150 151 152
/*
 * Lockless MCE logging infrastructure.
 * This avoids deadlocks on printk locks without having to break locks. Also
 * separate MCEs from kernel messages to avoid bogus bug reports.
 */

153
static struct mce_log mcelog = {
154 155 156
	.signature	= MCE_LOG_SIGNATURE,
	.len		= MCE_LOG_LEN,
	.recordlen	= sizeof(struct mce),
157
};
L
Linus Torvalds 已提交
158 159 160 161

void mce_log(struct mce *mce)
{
	unsigned next, entry;
I
Ingo Molnar 已提交
162

163 164 165
	/* Emit the trace record: */
	trace_mce_record(mce);

L
Linus Torvalds 已提交
166
	mce->finished = 0;
M
Mike Waychison 已提交
167
	wmb();
L
Linus Torvalds 已提交
168
	for (;;) {
169
		entry = rcu_dereference_check_mce(mcelog.next);
170
		for (;;) {
I
Ingo Molnar 已提交
171 172 173 174 175
			/*
			 * When the buffer fills up discard new entries.
			 * Assume that the earlier errors are the more
			 * interesting ones:
			 */
176
			if (entry >= MCE_LOG_LEN) {
177 178
				set_bit(MCE_OVERFLOW,
					(unsigned long *)&mcelog.flags);
179 180
				return;
			}
I
Ingo Molnar 已提交
181
			/* Old left over entry. Skip: */
182 183 184 185
			if (mcelog.entry[entry].finished) {
				entry++;
				continue;
			}
M
Mike Waychison 已提交
186
			break;
L
Linus Torvalds 已提交
187 188 189 190 191 192 193
		}
		smp_rmb();
		next = entry + 1;
		if (cmpxchg(&mcelog.next, entry, next) == entry)
			break;
	}
	memcpy(mcelog.entry + entry, mce, sizeof(struct mce));
M
Mike Waychison 已提交
194
	wmb();
L
Linus Torvalds 已提交
195
	mcelog.entry[entry].finished = 1;
M
Mike Waychison 已提交
196
	wmb();
L
Linus Torvalds 已提交
197

198
	mce->finished = 1;
199
	set_bit(0, &mce_need_notify);
L
Linus Torvalds 已提交
200 201
}

H
Hidetoshi Seto 已提交
202
static void print_mce(struct mce *m)
L
Linus Torvalds 已提交
203
{
204
	pr_emerg("CPU %d: Machine Check Exception: %16Lx Bank %d: %016Lx\n",
205
	       m->extcpu, m->mcgstatus, m->bank, m->status);
206

207
	if (m->ip) {
208 209 210 211
		pr_emerg("RIP%s %02x:<%016Lx> ",
			!(m->mcgstatus & MCG_STATUS_EIPV) ? " !INEXACT!" : "",
				m->cs, m->ip);

L
Linus Torvalds 已提交
212
		if (m->cs == __KERNEL_CS)
213
			print_symbol("{%s}", m->ip);
214
		pr_cont("\n");
L
Linus Torvalds 已提交
215
	}
216 217

	pr_emerg("TSC %llx ", m->tsc);
L
Linus Torvalds 已提交
218
	if (m->addr)
219
		pr_cont("ADDR %llx ", m->addr);
L
Linus Torvalds 已提交
220
	if (m->misc)
221
		pr_cont("MISC %llx ", m->misc);
222

223 224 225 226 227 228
	pr_cont("\n");
	pr_emerg("PROCESSOR %u:%x TIME %llu SOCKET %u APIC %x\n",
		m->cpuvendor, m->cpuid, m->time, m->socketid, m->apicid);

	/*
	 * Print out human-readable details about the MCE error,
229
	 * (if the CPU has an implementation for that)
230
	 */
231
	atomic_notifier_call_chain(&x86_mce_decoder_chain, 0, m);
232 233
}

H
Hidetoshi Seto 已提交
234 235
static void print_mce_head(void)
{
236
	pr_emerg("\nHARDWARE ERROR\n");
H
Hidetoshi Seto 已提交
237 238
}

239 240
static void print_mce_tail(void)
{
241
	pr_emerg("This is not a software problem!\n");
L
Linus Torvalds 已提交
242 243
}

244 245 246 247
#define PANIC_TIMEOUT 5 /* 5 seconds */

static atomic_t mce_paniced;

248 249 250
static int fake_panic;
static atomic_t mce_fake_paniced;

251 252 253 254
/* Panic in progress. Enable interrupts and wait for final IPI */
static void wait_for_panic(void)
{
	long timeout = PANIC_TIMEOUT*USEC_PER_SEC;
255

256 257 258 259
	preempt_disable();
	local_irq_enable();
	while (timeout-- > 0)
		udelay(1);
260 261
	if (panic_timeout == 0)
		panic_timeout = mce_panic_timeout;
262 263 264
	panic("Panicing machine check CPU died");
}

265
static void mce_panic(char *msg, struct mce *final, char *exp)
266
{
L
Linus Torvalds 已提交
267
	int i;
268

269 270 271 272 273 274 275
	if (!fake_panic) {
		/*
		 * Make sure only one CPU runs in machine check panic
		 */
		if (atomic_inc_return(&mce_paniced) > 1)
			wait_for_panic();
		barrier();
276

277 278 279 280 281 282 283
		bust_spinlocks(1);
		console_verbose();
	} else {
		/* Don't log too much for fake panic */
		if (atomic_inc_return(&mce_fake_paniced) > 1)
			return;
	}
H
Hidetoshi Seto 已提交
284
	print_mce_head();
285
	/* First print corrected ones that are still unlogged */
L
Linus Torvalds 已提交
286
	for (i = 0; i < MCE_LOG_LEN; i++) {
287
		struct mce *m = &mcelog.entry[i];
H
Hidetoshi Seto 已提交
288 289 290 291
		if (!(m->status & MCI_STATUS_VAL))
			continue;
		if (!(m->status & MCI_STATUS_UC))
			print_mce(m);
292 293 294 295 296
	}
	/* Now print uncorrected but with the final one last */
	for (i = 0; i < MCE_LOG_LEN; i++) {
		struct mce *m = &mcelog.entry[i];
		if (!(m->status & MCI_STATUS_VAL))
L
Linus Torvalds 已提交
297
			continue;
H
Hidetoshi Seto 已提交
298 299
		if (!(m->status & MCI_STATUS_UC))
			continue;
300
		if (!final || memcmp(m, final, sizeof(struct mce)))
H
Hidetoshi Seto 已提交
301
			print_mce(m);
L
Linus Torvalds 已提交
302
	}
303
	if (final)
H
Hidetoshi Seto 已提交
304
		print_mce(final);
305 306
	if (cpu_missing)
		printk(KERN_EMERG "Some CPUs didn't answer in synchronization\n");
307
	print_mce_tail();
308 309
	if (exp)
		printk(KERN_EMERG "Machine check: %s\n", exp);
310 311 312 313 314 315
	if (!fake_panic) {
		if (panic_timeout == 0)
			panic_timeout = mce_panic_timeout;
		panic(msg);
	} else
		printk(KERN_EMERG "Fake kernel panic: %s\n", msg);
316
}
L
Linus Torvalds 已提交
317

318 319 320 321 322
/* Support code for software error injection */

static int msr_to_offset(u32 msr)
{
	unsigned bank = __get_cpu_var(injectm.bank);
323

324 325
	if (msr == rip_msr)
		return offsetof(struct mce, ip);
326
	if (msr == MSR_IA32_MCx_STATUS(bank))
327
		return offsetof(struct mce, status);
328
	if (msr == MSR_IA32_MCx_ADDR(bank))
329
		return offsetof(struct mce, addr);
330
	if (msr == MSR_IA32_MCx_MISC(bank))
331 332 333 334 335 336
		return offsetof(struct mce, misc);
	if (msr == MSR_IA32_MCG_STATUS)
		return offsetof(struct mce, mcgstatus);
	return -1;
}

337 338 339 340
/* MSR access wrappers used for error injection */
static u64 mce_rdmsrl(u32 msr)
{
	u64 v;
341

342 343
	if (__get_cpu_var(injectm).finished) {
		int offset = msr_to_offset(msr);
344

345 346 347 348
		if (offset < 0)
			return 0;
		return *(u64 *)((char *)&__get_cpu_var(injectm) + offset);
	}
349 350 351 352 353 354 355 356 357 358 359

	if (rdmsrl_safe(msr, &v)) {
		WARN_ONCE(1, "mce: Unable to read msr %d!\n", msr);
		/*
		 * Return zero in case the access faulted. This should
		 * not happen normally but can happen if the CPU does
		 * something weird, or if the code is buggy.
		 */
		v = 0;
	}

360 361 362 363 364
	return v;
}

static void mce_wrmsrl(u32 msr, u64 v)
{
365 366
	if (__get_cpu_var(injectm).finished) {
		int offset = msr_to_offset(msr);
367

368 369 370 371
		if (offset >= 0)
			*(u64 *)((char *)&__get_cpu_var(injectm) + offset) = v;
		return;
	}
372 373 374
	wrmsrl(msr, v);
}

375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429
/*
 * Simple lockless ring to communicate PFNs from the exception handler with the
 * process context work function. This is vastly simplified because there's
 * only a single reader and a single writer.
 */
#define MCE_RING_SIZE 16	/* we use one entry less */

struct mce_ring {
	unsigned short start;
	unsigned short end;
	unsigned long ring[MCE_RING_SIZE];
};
static DEFINE_PER_CPU(struct mce_ring, mce_ring);

/* Runs with CPU affinity in workqueue */
static int mce_ring_empty(void)
{
	struct mce_ring *r = &__get_cpu_var(mce_ring);

	return r->start == r->end;
}

static int mce_ring_get(unsigned long *pfn)
{
	struct mce_ring *r;
	int ret = 0;

	*pfn = 0;
	get_cpu();
	r = &__get_cpu_var(mce_ring);
	if (r->start == r->end)
		goto out;
	*pfn = r->ring[r->start];
	r->start = (r->start + 1) % MCE_RING_SIZE;
	ret = 1;
out:
	put_cpu();
	return ret;
}

/* Always runs in MCE context with preempt off */
static int mce_ring_add(unsigned long pfn)
{
	struct mce_ring *r = &__get_cpu_var(mce_ring);
	unsigned next;

	next = (r->end + 1) % MCE_RING_SIZE;
	if (next == r->start)
		return -1;
	r->ring[r->end] = pfn;
	wmb();
	r->end = next;
	return 0;
}

A
Andi Kleen 已提交
430
int mce_available(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
431
{
432
	if (mce_disabled)
433
		return 0;
434
	return cpu_has(c, X86_FEATURE_MCE) && cpu_has(c, X86_FEATURE_MCA);
L
Linus Torvalds 已提交
435 436
}

437 438 439 440 441 442 443 444 445
static void mce_schedule_work(void)
{
	if (!mce_ring_empty()) {
		struct work_struct *work = &__get_cpu_var(mce_work);
		if (!work_pending(work))
			schedule_work(work);
	}
}

H
Huang Ying 已提交
446 447 448 449
/*
 * Get the address of the instruction at the time of the machine check
 * error.
 */
450 451
static inline void mce_get_rip(struct mce *m, struct pt_regs *regs)
{
H
Huang Ying 已提交
452 453

	if (regs && (m->mcgstatus & (MCG_STATUS_RIPV|MCG_STATUS_EIPV))) {
454
		m->ip = regs->ip;
455 456
		m->cs = regs->cs;
	} else {
457
		m->ip = 0;
458 459
		m->cs = 0;
	}
H
Huang Ying 已提交
460
	if (rip_msr)
461
		m->ip = mce_rdmsrl(rip_msr);
462 463
}

464
#ifdef CONFIG_X86_LOCAL_APIC
465 466 467 468 469 470 471 472 473 474
/*
 * Called after interrupts have been reenabled again
 * when a MCE happened during an interrupts off region
 * in the kernel.
 */
asmlinkage void smp_mce_self_interrupt(struct pt_regs *regs)
{
	ack_APIC_irq();
	exit_idle();
	irq_enter();
475
	mce_notify_irq();
476
	mce_schedule_work();
477 478 479 480 481 482 483
	irq_exit();
}
#endif

static void mce_report_event(struct pt_regs *regs)
{
	if (regs->flags & (X86_VM_MASK|X86_EFLAGS_IF)) {
484
		mce_notify_irq();
485 486 487 488 489 490 491
		/*
		 * Triggering the work queue here is just an insurance
		 * policy in case the syscall exit notify handler
		 * doesn't run soon enough or ends up running on the
		 * wrong CPU (can happen when audit sleeps)
		 */
		mce_schedule_work();
492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519
		return;
	}

#ifdef CONFIG_X86_LOCAL_APIC
	/*
	 * Without APIC do not notify. The event will be picked
	 * up eventually.
	 */
	if (!cpu_has_apic)
		return;

	/*
	 * When interrupts are disabled we cannot use
	 * kernel services safely. Trigger an self interrupt
	 * through the APIC to instead do the notification
	 * after interrupts are reenabled again.
	 */
	apic->send_IPI_self(MCE_SELF_VECTOR);

	/*
	 * Wait for idle afterwards again so that we don't leave the
	 * APIC in a non idle state because the normal APIC writes
	 * cannot exclude us.
	 */
	apic_wait_icr_idle();
#endif
}

520 521
DEFINE_PER_CPU(unsigned, mce_poll_count);

522
/*
523 524 525 526
 * Poll for corrected events or events that happened before reset.
 * Those are just logged through /dev/mcelog.
 *
 * This is executed in standard interrupt context.
A
Andi Kleen 已提交
527 528 529 530 531 532 533 534 535
 *
 * Note: spec recommends to panic for fatal unsignalled
 * errors here. However this would be quite problematic --
 * we would need to reimplement the Monarch handling and
 * it would mess up the exclusion between exception handler
 * and poll hander -- * so we skip this for now.
 * These cases should not happen anyways, or only when the CPU
 * is already totally * confused. In this case it's likely it will
 * not fully execute the machine check handler either.
536
 */
537
void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
538 539 540 541
{
	struct mce m;
	int i;

542 543
	__get_cpu_var(mce_poll_count)++;

544 545
	mce_setup(&m);

546
	m.mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS);
547
	for (i = 0; i < banks; i++) {
548
		if (!mce_banks[i].ctl || !test_bit(i, *b))
549 550 551 552 553 554 555 556
			continue;

		m.misc = 0;
		m.addr = 0;
		m.bank = i;
		m.tsc = 0;

		barrier();
557
		m.status = mce_rdmsrl(MSR_IA32_MCx_STATUS(i));
558 559 560 561
		if (!(m.status & MCI_STATUS_VAL))
			continue;

		/*
A
Andi Kleen 已提交
562 563
		 * Uncorrected or signalled events are handled by the exception
		 * handler when it is enabled, so don't process those here.
564 565 566
		 *
		 * TBD do the same check for MCI_STATUS_EN here?
		 */
A
Andi Kleen 已提交
567 568
		if (!(flags & MCP_UC) &&
		    (m.status & (mce_ser ? MCI_STATUS_S : MCI_STATUS_UC)))
569 570 571
			continue;

		if (m.status & MCI_STATUS_MISCV)
572
			m.misc = mce_rdmsrl(MSR_IA32_MCx_MISC(i));
573
		if (m.status & MCI_STATUS_ADDRV)
574
			m.addr = mce_rdmsrl(MSR_IA32_MCx_ADDR(i));
575 576 577 578 579 580 581

		if (!(flags & MCP_TIMESTAMP))
			m.tsc = 0;
		/*
		 * Don't get the IP here because it's unlikely to
		 * have anything to do with the actual error location.
		 */
582
		if (!(flags & MCP_DONTLOG) && !mce_dont_log_ce) {
A
Andi Kleen 已提交
583 584 585
			mce_log(&m);
			add_taint(TAINT_MACHINE_CHECK);
		}
586 587 588 589

		/*
		 * Clear state for this bank.
		 */
590
		mce_wrmsrl(MSR_IA32_MCx_STATUS(i), 0);
591 592 593 594 595 596
	}

	/*
	 * Don't clear MCG_STATUS here because it's only defined for
	 * exceptions.
	 */
597 598

	sync_core();
599
}
600
EXPORT_SYMBOL_GPL(machine_check_poll);
601

602 603 604 605 606 607 608 609 610
/*
 * Do a quick check if any of the events requires a panic.
 * This decides if we keep the events around or clear them.
 */
static int mce_no_way_out(struct mce *m, char **msg)
{
	int i;

	for (i = 0; i < banks; i++) {
611
		m->status = mce_rdmsrl(MSR_IA32_MCx_STATUS(i));
612 613 614 615 616 617
		if (mce_severity(m, tolerant, msg) >= MCE_PANIC_SEVERITY)
			return 1;
	}
	return 0;
}

618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670
/*
 * Variable to establish order between CPUs while scanning.
 * Each CPU spins initially until executing is equal its number.
 */
static atomic_t mce_executing;

/*
 * Defines order of CPUs on entry. First CPU becomes Monarch.
 */
static atomic_t mce_callin;

/*
 * Check if a timeout waiting for other CPUs happened.
 */
static int mce_timed_out(u64 *t)
{
	/*
	 * The others already did panic for some reason.
	 * Bail out like in a timeout.
	 * rmb() to tell the compiler that system_state
	 * might have been modified by someone else.
	 */
	rmb();
	if (atomic_read(&mce_paniced))
		wait_for_panic();
	if (!monarch_timeout)
		goto out;
	if ((s64)*t < SPINUNIT) {
		/* CHECKME: Make panic default for 1 too? */
		if (tolerant < 1)
			mce_panic("Timeout synchronizing machine check over CPUs",
				  NULL, NULL);
		cpu_missing = 1;
		return 1;
	}
	*t -= SPINUNIT;
out:
	touch_nmi_watchdog();
	return 0;
}

/*
 * The Monarch's reign.  The Monarch is the CPU who entered
 * the machine check handler first. It waits for the others to
 * raise the exception too and then grades them. When any
 * error is fatal panic. Only then let the others continue.
 *
 * The other CPUs entering the MCE handler will be controlled by the
 * Monarch. They are called Subjects.
 *
 * This way we prevent any potential data corruption in a unrecoverable case
 * and also makes sure always all CPU's errors are examined.
 *
671
 * Also this detects the case of a machine check event coming from outer
672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711
 * space (not detected by any CPUs) In this case some external agent wants
 * us to shut down, so panic too.
 *
 * The other CPUs might still decide to panic if the handler happens
 * in a unrecoverable place, but in this case the system is in a semi-stable
 * state and won't corrupt anything by itself. It's ok to let the others
 * continue for a bit first.
 *
 * All the spin loops have timeouts; when a timeout happens a CPU
 * typically elects itself to be Monarch.
 */
static void mce_reign(void)
{
	int cpu;
	struct mce *m = NULL;
	int global_worst = 0;
	char *msg = NULL;
	char *nmsg = NULL;

	/*
	 * This CPU is the Monarch and the other CPUs have run
	 * through their handlers.
	 * Grade the severity of the errors of all the CPUs.
	 */
	for_each_possible_cpu(cpu) {
		int severity = mce_severity(&per_cpu(mces_seen, cpu), tolerant,
					    &nmsg);
		if (severity > global_worst) {
			msg = nmsg;
			global_worst = severity;
			m = &per_cpu(mces_seen, cpu);
		}
	}

	/*
	 * Cannot recover? Panic here then.
	 * This dumps all the mces in the log buffer and stops the
	 * other CPUs.
	 */
	if (m && global_worst >= MCE_PANIC_SEVERITY && tolerant < 3)
712
		mce_panic("Fatal Machine check", m, msg);
713 714 715 716 717 718 719 720 721 722 723

	/*
	 * For UC somewhere we let the CPU who detects it handle it.
	 * Also must let continue the others, otherwise the handling
	 * CPU could deadlock on a lock.
	 */

	/*
	 * No machine check event found. Must be some external
	 * source or one CPU is hung. Panic.
	 */
724
	if (global_worst <= MCE_KEEP_SEVERITY && tolerant < 3)
725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743
		mce_panic("Machine check from unknown source", NULL, NULL);

	/*
	 * Now clear all the mces_seen so that they don't reappear on
	 * the next mce.
	 */
	for_each_possible_cpu(cpu)
		memset(&per_cpu(mces_seen, cpu), 0, sizeof(struct mce));
}

static atomic_t global_nwo;

/*
 * Start of Monarch synchronization. This waits until all CPUs have
 * entered the exception handler and then determines if any of them
 * saw a fatal event that requires panic. Then it executes them
 * in the entry order.
 * TBD double check parallel CPU hotunplug
 */
H
Hidetoshi Seto 已提交
744
static int mce_start(int *no_way_out)
745
{
H
Hidetoshi Seto 已提交
746
	int order;
747 748 749
	int cpus = num_online_cpus();
	u64 timeout = (u64)monarch_timeout * NSEC_PER_USEC;

H
Hidetoshi Seto 已提交
750 751
	if (!timeout)
		return -1;
752

H
Hidetoshi Seto 已提交
753
	atomic_add(*no_way_out, &global_nwo);
754 755 756 757
	/*
	 * global_nwo should be updated before mce_callin
	 */
	smp_wmb();
758
	order = atomic_inc_return(&mce_callin);
759 760 761 762 763 764 765

	/*
	 * Wait for everyone.
	 */
	while (atomic_read(&mce_callin) != cpus) {
		if (mce_timed_out(&timeout)) {
			atomic_set(&global_nwo, 0);
H
Hidetoshi Seto 已提交
766
			return -1;
767 768 769 770
		}
		ndelay(SPINUNIT);
	}

771 772 773 774
	/*
	 * mce_callin should be read before global_nwo
	 */
	smp_rmb();
775

H
Hidetoshi Seto 已提交
776 777 778 779
	if (order == 1) {
		/*
		 * Monarch: Starts executing now, the others wait.
		 */
780
		atomic_set(&mce_executing, 1);
H
Hidetoshi Seto 已提交
781 782 783 784 785 786 787 788 789 790 791 792 793 794
	} else {
		/*
		 * Subject: Now start the scanning loop one by one in
		 * the original callin order.
		 * This way when there are any shared banks it will be
		 * only seen by one CPU before cleared, avoiding duplicates.
		 */
		while (atomic_read(&mce_executing) < order) {
			if (mce_timed_out(&timeout)) {
				atomic_set(&global_nwo, 0);
				return -1;
			}
			ndelay(SPINUNIT);
		}
795 796 797
	}

	/*
H
Hidetoshi Seto 已提交
798
	 * Cache the global no_way_out state.
799
	 */
H
Hidetoshi Seto 已提交
800 801 802
	*no_way_out = atomic_read(&global_nwo);

	return order;
803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871
}

/*
 * Synchronize between CPUs after main scanning loop.
 * This invokes the bulk of the Monarch processing.
 */
static int mce_end(int order)
{
	int ret = -1;
	u64 timeout = (u64)monarch_timeout * NSEC_PER_USEC;

	if (!timeout)
		goto reset;
	if (order < 0)
		goto reset;

	/*
	 * Allow others to run.
	 */
	atomic_inc(&mce_executing);

	if (order == 1) {
		/* CHECKME: Can this race with a parallel hotplug? */
		int cpus = num_online_cpus();

		/*
		 * Monarch: Wait for everyone to go through their scanning
		 * loops.
		 */
		while (atomic_read(&mce_executing) <= cpus) {
			if (mce_timed_out(&timeout))
				goto reset;
			ndelay(SPINUNIT);
		}

		mce_reign();
		barrier();
		ret = 0;
	} else {
		/*
		 * Subject: Wait for Monarch to finish.
		 */
		while (atomic_read(&mce_executing) != 0) {
			if (mce_timed_out(&timeout))
				goto reset;
			ndelay(SPINUNIT);
		}

		/*
		 * Don't reset anything. That's done by the Monarch.
		 */
		return 0;
	}

	/*
	 * Reset all global state.
	 */
reset:
	atomic_set(&global_nwo, 0);
	atomic_set(&mce_callin, 0);
	barrier();

	/*
	 * Let others run again.
	 */
	atomic_set(&mce_executing, 0);
	return ret;
}

872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888
/*
 * Check if the address reported by the CPU is in a format we can parse.
 * It would be possible to add code for most other cases, but all would
 * be somewhat complicated (e.g. segment offset would require an instruction
 * parser). So only support physical addresses upto page granuality for now.
 */
static int mce_usable_address(struct mce *m)
{
	if (!(m->status & MCI_STATUS_MISCV) || !(m->status & MCI_STATUS_ADDRV))
		return 0;
	if ((m->misc & 0x3f) > PAGE_SHIFT)
		return 0;
	if (((m->misc >> 6) & 7) != MCM_ADDR_PHYS)
		return 0;
	return 1;
}

889 890 891 892 893 894
static void mce_clear_state(unsigned long *toclear)
{
	int i;

	for (i = 0; i < banks; i++) {
		if (test_bit(i, toclear))
895
			mce_wrmsrl(MSR_IA32_MCx_STATUS(i), 0);
896 897 898
	}
}

899 900 901 902 903 904 905
/*
 * The actual machine check handler. This only handles real
 * exceptions when something got corrupted coming in through int 18.
 *
 * This is executed in NMI context not subject to normal locking rules. This
 * implies that most kernel services cannot be safely used. Don't even
 * think about putting a printk in there!
906 907 908 909
 *
 * On Intel systems this is entered on all CPUs in parallel through
 * MCE broadcast. However some CPUs might be broken beyond repair,
 * so be always careful when synchronizing with others.
L
Linus Torvalds 已提交
910
 */
I
Ingo Molnar 已提交
911
void do_machine_check(struct pt_regs *regs, long error_code)
L
Linus Torvalds 已提交
912
{
913
	struct mce m, *final;
L
Linus Torvalds 已提交
914
	int i;
915 916 917 918 919 920
	int worst = 0;
	int severity;
	/*
	 * Establish sequential order between the CPUs entering the machine
	 * check handler.
	 */
H
Hidetoshi Seto 已提交
921
	int order;
922 923 924 925 926 927 928 929 930 931
	/*
	 * If no_way_out gets set, there is no safe way to recover from this
	 * MCE.  If tolerant is cranked up, we'll try anyway.
	 */
	int no_way_out = 0;
	/*
	 * If kill_it gets set, there might be a way to recover from this
	 * error.
	 */
	int kill_it = 0;
932
	DECLARE_BITMAP(toclear, MAX_NR_BANKS);
933
	char *msg = "Unknown";
L
Linus Torvalds 已提交
934

935 936
	atomic_inc(&mce_entry);

937 938
	__get_cpu_var(mce_exception_count)++;

939
	if (notify_die(DIE_NMI, "machine check", regs, error_code,
940
			   18, SIGKILL) == NOTIFY_STOP)
941
		goto out;
942
	if (!banks)
943
		goto out;
L
Linus Torvalds 已提交
944

945 946
	mce_setup(&m);

947
	m.mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS);
948 949 950
	final = &__get_cpu_var(mces_seen);
	*final = m;

951 952
	no_way_out = mce_no_way_out(&m, &msg);

L
Linus Torvalds 已提交
953 954
	barrier();

A
Andi Kleen 已提交
955 956 957 958 959 960
	/*
	 * When no restart IP must always kill or panic.
	 */
	if (!(m.mcgstatus & MCG_STATUS_RIPV))
		kill_it = 1;

961 962 963 964 965
	/*
	 * Go through all the banks in exclusion of the other CPUs.
	 * This way we don't report duplicated events on shared banks
	 * because the first one to see it will clear it.
	 */
H
Hidetoshi Seto 已提交
966
	order = mce_start(&no_way_out);
L
Linus Torvalds 已提交
967
	for (i = 0; i < banks; i++) {
968
		__clear_bit(i, toclear);
969
		if (!mce_banks[i].ctl)
L
Linus Torvalds 已提交
970
			continue;
971 972

		m.misc = 0;
L
Linus Torvalds 已提交
973 974 975
		m.addr = 0;
		m.bank = i;

976
		m.status = mce_rdmsrl(MSR_IA32_MCx_STATUS(i));
L
Linus Torvalds 已提交
977 978 979
		if ((m.status & MCI_STATUS_VAL) == 0)
			continue;

980
		/*
A
Andi Kleen 已提交
981 982
		 * Non uncorrected or non signaled errors are handled by
		 * machine_check_poll. Leave them alone, unless this panics.
983
		 */
A
Andi Kleen 已提交
984 985
		if (!(m.status & (mce_ser ? MCI_STATUS_S : MCI_STATUS_UC)) &&
			!no_way_out)
986 987 988 989 990 991 992
			continue;

		/*
		 * Set taint even when machine check was not enabled.
		 */
		add_taint(TAINT_MACHINE_CHECK);

A
Andi Kleen 已提交
993
		severity = mce_severity(&m, tolerant, NULL);
994

A
Andi Kleen 已提交
995 996 997 998 999 1000 1001 1002
		/*
		 * When machine check was for corrected handler don't touch,
		 * unless we're panicing.
		 */
		if (severity == MCE_KEEP_SEVERITY && !no_way_out)
			continue;
		__set_bit(i, toclear);
		if (severity == MCE_NO_SEVERITY) {
1003 1004 1005 1006 1007
			/*
			 * Machine check event was not enabled. Clear, but
			 * ignore.
			 */
			continue;
L
Linus Torvalds 已提交
1008 1009
		}

A
Andi Kleen 已提交
1010 1011 1012 1013 1014 1015
		/*
		 * Kill on action required.
		 */
		if (severity == MCE_AR_SEVERITY)
			kill_it = 1;

L
Linus Torvalds 已提交
1016
		if (m.status & MCI_STATUS_MISCV)
1017
			m.misc = mce_rdmsrl(MSR_IA32_MCx_MISC(i));
L
Linus Torvalds 已提交
1018
		if (m.status & MCI_STATUS_ADDRV)
1019
			m.addr = mce_rdmsrl(MSR_IA32_MCx_ADDR(i));
L
Linus Torvalds 已提交
1020

1021 1022 1023 1024 1025 1026 1027 1028 1029 1030
		/*
		 * Action optional error. Queue address for later processing.
		 * When the ring overflows we just ignore the AO error.
		 * RED-PEN add some logging mechanism when
		 * usable_address or mce_add_ring fails.
		 * RED-PEN don't ignore overflow for tolerant == 0
		 */
		if (severity == MCE_AO_SEVERITY && mce_usable_address(&m))
			mce_ring_add(m.addr >> PAGE_SHIFT);

1031
		mce_get_rip(&m, regs);
1032
		mce_log(&m);
L
Linus Torvalds 已提交
1033

1034 1035 1036
		if (severity > worst) {
			*final = m;
			worst = severity;
L
Linus Torvalds 已提交
1037 1038 1039
		}
	}

1040 1041 1042
	if (!no_way_out)
		mce_clear_state(toclear);

I
Ingo Molnar 已提交
1043
	/*
1044 1045
	 * Do most of the synchronization with other CPUs.
	 * When there's any problem use only local no_way_out state.
I
Ingo Molnar 已提交
1046
	 */
1047 1048
	if (mce_end(order) < 0)
		no_way_out = worst >= MCE_PANIC_SEVERITY;
1049 1050 1051

	/*
	 * If we have decided that we just CAN'T continue, and the user
I
Ingo Molnar 已提交
1052
	 * has not set tolerant to an insane level, give up and die.
1053 1054 1055
	 *
	 * This is mainly used in the case when the system doesn't
	 * support MCE broadcasting or it has been disabled.
1056 1057
	 */
	if (no_way_out && tolerant < 3)
1058
		mce_panic("Fatal machine check on current CPU", final, msg);
1059 1060 1061 1062 1063 1064 1065 1066

	/*
	 * If the error seems to be unrecoverable, something should be
	 * done.  Try to kill as little as possible.  If we can kill just
	 * one task, do that.  If the user has set the tolerance very
	 * high, don't try to do anything at all.
	 */

A
Andi Kleen 已提交
1067 1068
	if (kill_it && tolerant < 3)
		force_sig(SIGBUS, current);
L
Linus Torvalds 已提交
1069

1070 1071 1072
	/* notify userspace ASAP */
	set_thread_flag(TIF_MCE_NOTIFY);

1073 1074
	if (worst > 0)
		mce_report_event(regs);
1075
	mce_wrmsrl(MSR_IA32_MCG_STATUS, 0);
1076
out:
1077
	atomic_dec(&mce_entry);
1078
	sync_core();
L
Linus Torvalds 已提交
1079
}
1080
EXPORT_SYMBOL_GPL(do_machine_check);
L
Linus Torvalds 已提交
1081

1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111
/* dummy to break dependency. actual code is in mm/memory-failure.c */
void __attribute__((weak)) memory_failure(unsigned long pfn, int vector)
{
	printk(KERN_ERR "Action optional memory failure at %lx ignored\n", pfn);
}

/*
 * Called after mce notification in process context. This code
 * is allowed to sleep. Call the high level VM handler to process
 * any corrupted pages.
 * Assume that the work queue code only calls this one at a time
 * per CPU.
 * Note we don't disable preemption, so this code might run on the wrong
 * CPU. In this case the event is picked up by the scheduled work queue.
 * This is merely a fast path to expedite processing in some common
 * cases.
 */
void mce_notify_process(void)
{
	unsigned long pfn;
	mce_notify_irq();
	while (mce_ring_get(&pfn))
		memory_failure(pfn, MCE_VECTOR);
}

static void mce_process_work(struct work_struct *dummy)
{
	mce_notify_process();
}

1112 1113 1114
#ifdef CONFIG_X86_MCE_INTEL
/***
 * mce_log_therm_throt_event - Logs the thermal throttling event to mcelog
S
Simon Arlott 已提交
1115
 * @cpu: The CPU on which the event occurred.
1116 1117 1118 1119 1120 1121 1122 1123 1124 1125
 * @status: Event status information
 *
 * This function should be called by the thermal interrupt after the
 * event has been processed and the decision was made to log the event
 * further.
 *
 * The status parameter will be saved to the 'status' field of 'struct mce'
 * and historically has been the register value of the
 * MSR_IA32_THERMAL_STATUS (Intel) msr.
 */
1126
void mce_log_therm_throt_event(__u64 status)
1127 1128 1129
{
	struct mce m;

1130
	mce_setup(&m);
1131 1132 1133 1134 1135 1136
	m.bank = MCE_THERMAL_BANK;
	m.status = status;
	mce_log(&m);
}
#endif /* CONFIG_X86_MCE_INTEL */

L
Linus Torvalds 已提交
1137
/*
1138 1139 1140
 * Periodic polling timer for "silent" machine check errors.  If the
 * poller finds an MCE, poll 2x faster.  When the poller finds no more
 * errors, poll 2x slower (up to check_interval seconds).
L
Linus Torvalds 已提交
1141 1142
 */
static int check_interval = 5 * 60; /* 5 minutes */
I
Ingo Molnar 已提交
1143

1144
static DEFINE_PER_CPU(int, mce_next_interval); /* in jiffies */
1145
static DEFINE_PER_CPU(struct timer_list, mce_timer);
L
Linus Torvalds 已提交
1146

1147
static void mce_start_timer(unsigned long data)
L
Linus Torvalds 已提交
1148
{
1149
	struct timer_list *t = &per_cpu(mce_timer, data);
1150
	int *n;
1151 1152 1153

	WARN_ON(smp_processor_id() != data);

I
Ingo Molnar 已提交
1154
	if (mce_available(&current_cpu_data)) {
1155 1156
		machine_check_poll(MCP_TIMESTAMP,
				&__get_cpu_var(mce_poll_banks));
I
Ingo Molnar 已提交
1157
	}
L
Linus Torvalds 已提交
1158 1159

	/*
1160 1161
	 * Alert userspace if needed.  If we logged an MCE, reduce the
	 * polling interval, otherwise increase the polling interval.
L
Linus Torvalds 已提交
1162
	 */
1163
	n = &__get_cpu_var(mce_next_interval);
1164
	if (mce_notify_irq())
1165
		*n = max(*n/2, HZ/100);
1166
	else
1167
		*n = min(*n*2, (int)round_jiffies_relative(check_interval*HZ));
1168

1169
	t->expires = jiffies + *n;
1170
	add_timer_on(t, smp_processor_id());
1171 1172
}

1173 1174
static void mce_do_trigger(struct work_struct *work)
{
1175
	call_usermodehelper(mce_helper, mce_helper_argv, NULL, UMH_NO_WAIT);
1176 1177 1178 1179
}

static DECLARE_WORK(mce_trigger_work, mce_do_trigger);

1180
/*
1181 1182 1183
 * Notify the user(s) about new machine check events.
 * Can be called from interrupt context, but not from machine check/NMI
 * context.
1184
 */
1185
int mce_notify_irq(void)
1186
{
1187 1188 1189
	/* Not more than two messages every minute */
	static DEFINE_RATELIMIT_STATE(ratelimit, 60*HZ, 2);

1190
	clear_thread_flag(TIF_MCE_NOTIFY);
I
Ingo Molnar 已提交
1191

1192
	if (test_and_clear_bit(0, &mce_need_notify)) {
1193
		wake_up_interruptible(&mce_wait);
1194 1195 1196 1197 1198 1199

		/*
		 * There is no risk of missing notifications because
		 * work_pending is always cleared before the function is
		 * executed.
		 */
1200
		if (mce_helper[0] && !work_pending(&mce_trigger_work))
1201
			schedule_work(&mce_trigger_work);
1202

1203
		if (__ratelimit(&ratelimit))
1204
			printk(KERN_INFO "Machine check events logged\n");
1205 1206

		return 1;
L
Linus Torvalds 已提交
1207
	}
1208 1209
	return 0;
}
1210
EXPORT_SYMBOL_GPL(mce_notify_irq);
1211

H
Hidetoshi Seto 已提交
1212
static int __cpuinit __mcheck_cpu_mce_banks_init(void)
1213 1214 1215 1216 1217 1218 1219 1220
{
	int i;

	mce_banks = kzalloc(banks * sizeof(struct mce_bank), GFP_KERNEL);
	if (!mce_banks)
		return -ENOMEM;
	for (i = 0; i < banks; i++) {
		struct mce_bank *b = &mce_banks[i];
1221

1222 1223 1224 1225 1226 1227
		b->ctl = -1ULL;
		b->init = 1;
	}
	return 0;
}

1228
/*
L
Linus Torvalds 已提交
1229 1230
 * Initialize Machine Checks for a CPU.
 */
1231
static int __cpuinit __mcheck_cpu_cap_init(void)
L
Linus Torvalds 已提交
1232
{
1233
	unsigned b;
I
Ingo Molnar 已提交
1234
	u64 cap;
L
Linus Torvalds 已提交
1235 1236

	rdmsrl(MSR_IA32_MCG_CAP, cap);
1237 1238

	b = cap & MCG_BANKCNT_MASK;
1239 1240
	if (!banks)
		printk(KERN_INFO "mce: CPU supports %d MCE banks\n", b);
1241

1242 1243 1244 1245 1246 1247 1248 1249 1250 1251
	if (b > MAX_NR_BANKS) {
		printk(KERN_WARNING
		       "MCE: Using only %u machine check banks out of %u\n",
			MAX_NR_BANKS, b);
		b = MAX_NR_BANKS;
	}

	/* Don't support asymmetric configurations today */
	WARN_ON(banks != 0 && b != banks);
	banks = b;
1252
	if (!mce_banks) {
H
Hidetoshi Seto 已提交
1253
		int err = __mcheck_cpu_mce_banks_init();
1254

1255 1256
		if (err)
			return err;
L
Linus Torvalds 已提交
1257
	}
1258

1259
	/* Use accurate RIP reporting if available. */
1260
	if ((cap & MCG_EXT_P) && MCG_EXT_CNT(cap) >= 9)
1261
		rip_msr = MSR_IA32_MCG_EIP;
L
Linus Torvalds 已提交
1262

A
Andi Kleen 已提交
1263 1264 1265
	if (cap & MCG_SER_P)
		mce_ser = 1;

1266 1267 1268
	return 0;
}

1269
static void __mcheck_cpu_init_generic(void)
1270
{
I
Ingo Molnar 已提交
1271
	mce_banks_t all_banks;
1272 1273 1274
	u64 cap;
	int i;

1275 1276 1277
	/*
	 * Log the machine checks left over from the previous reset.
	 */
1278
	bitmap_fill(all_banks, MAX_NR_BANKS);
A
Andi Kleen 已提交
1279
	machine_check_poll(MCP_UC|(!mce_bootlog ? MCP_DONTLOG : 0), &all_banks);
L
Linus Torvalds 已提交
1280 1281 1282

	set_in_cr4(X86_CR4_MCE);

1283
	rdmsrl(MSR_IA32_MCG_CAP, cap);
L
Linus Torvalds 已提交
1284 1285 1286 1287
	if (cap & MCG_CTL_P)
		wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff);

	for (i = 0; i < banks; i++) {
1288
		struct mce_bank *b = &mce_banks[i];
1289

1290
		if (!b->init)
1291
			continue;
1292 1293
		wrmsrl(MSR_IA32_MCx_CTL(i), b->ctl);
		wrmsrl(MSR_IA32_MCx_STATUS(i), 0);
1294
	}
L
Linus Torvalds 已提交
1295 1296 1297
}

/* Add per CPU specific workarounds here */
1298
static int __cpuinit __mcheck_cpu_apply_quirks(struct cpuinfo_x86 *c)
1299
{
1300 1301 1302 1303 1304
	if (c->x86_vendor == X86_VENDOR_UNKNOWN) {
		pr_info("MCE: unknown CPU type - not enabling MCE support.\n");
		return -EOPNOTSUPP;
	}

L
Linus Torvalds 已提交
1305
	/* This should be disabled by the BIOS, but isn't always */
1306
	if (c->x86_vendor == X86_VENDOR_AMD) {
I
Ingo Molnar 已提交
1307 1308 1309 1310 1311 1312
		if (c->x86 == 15 && banks > 4) {
			/*
			 * disable GART TBL walk error reporting, which
			 * trips off incorrectly with the IOMMU & 3ware
			 * & Cerberus:
			 */
1313
			clear_bit(10, (unsigned long *)&mce_banks[4].ctl);
I
Ingo Molnar 已提交
1314 1315 1316 1317 1318 1319
		}
		if (c->x86 <= 17 && mce_bootlog < 0) {
			/*
			 * Lots of broken BIOS around that don't clear them
			 * by default and leave crap in there. Don't log:
			 */
1320
			mce_bootlog = 0;
I
Ingo Molnar 已提交
1321
		}
1322 1323 1324 1325
		/*
		 * Various K7s with broken bank 0 around. Always disable
		 * by default.
		 */
1326
		 if (c->x86 == 6 && banks > 0)
1327
			mce_banks[0].ctl = 0;
L
Linus Torvalds 已提交
1328
	}
1329

1330 1331 1332 1333 1334 1335 1336 1337 1338 1339
	if (c->x86_vendor == X86_VENDOR_INTEL) {
		/*
		 * SDM documents that on family 6 bank 0 should not be written
		 * because it aliases to another special BIOS controlled
		 * register.
		 * But it's not aliased anymore on model 0x1a+
		 * Don't ignore bank 0 completely because there could be a
		 * valid event later, merely don't write CTL0.
		 */

1340 1341
		if (c->x86 == 6 && c->x86_model < 0x1A && banks > 0)
			mce_banks[0].init = 0;
1342 1343 1344 1345 1346 1347 1348 1349

		/*
		 * All newer Intel systems support MCE broadcasting. Enable
		 * synchronization with a one second timeout.
		 */
		if ((c->x86 > 6 || (c->x86 == 6 && c->x86_model >= 0xe)) &&
			monarch_timeout < 0)
			monarch_timeout = USEC_PER_SEC;
1350

1351 1352 1353 1354 1355
		/*
		 * There are also broken BIOSes on some Pentium M and
		 * earlier systems:
		 */
		if (c->x86 == 6 && c->x86_model <= 13 && mce_bootlog < 0)
1356
			mce_bootlog = 0;
1357
	}
1358 1359
	if (monarch_timeout < 0)
		monarch_timeout = 0;
1360 1361
	if (mce_bootlog != 0)
		mce_panic_timeout = 30;
1362 1363

	return 0;
1364
}
L
Linus Torvalds 已提交
1365

1366
static void __cpuinit __mcheck_cpu_ancient_init(struct cpuinfo_x86 *c)
1367 1368 1369 1370 1371
{
	if (c->x86 != 5)
		return;
	switch (c->x86_vendor) {
	case X86_VENDOR_INTEL:
1372
		intel_p5_mcheck_init(c);
1373 1374 1375 1376 1377 1378 1379
		break;
	case X86_VENDOR_CENTAUR:
		winchip_mcheck_init(c);
		break;
	}
}

1380
static void __mcheck_cpu_init_vendor(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1381 1382 1383 1384 1385
{
	switch (c->x86_vendor) {
	case X86_VENDOR_INTEL:
		mce_intel_feature_init(c);
		break;
1386 1387 1388
	case X86_VENDOR_AMD:
		mce_amd_feature_init(c);
		break;
L
Linus Torvalds 已提交
1389 1390 1391 1392 1393
	default:
		break;
	}
}

1394
static void __mcheck_cpu_init_timer(void)
1395 1396
{
	struct timer_list *t = &__get_cpu_var(mce_timer);
1397
	int *n = &__get_cpu_var(mce_next_interval);
1398

1399 1400
	setup_timer(t, mce_start_timer, smp_processor_id());

1401 1402 1403
	if (mce_ignore_ce)
		return;

1404 1405
	*n = check_interval * HZ;
	if (!*n)
1406
		return;
1407
	t->expires = round_jiffies(jiffies + *n);
1408
	add_timer_on(t, smp_processor_id());
1409 1410
}

A
Andi Kleen 已提交
1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421
/* Handle unconfigured int18 (should never happen) */
static void unexpected_machine_check(struct pt_regs *regs, long error_code)
{
	printk(KERN_ERR "CPU#%d: Unexpected int18 (Machine Check).\n",
	       smp_processor_id());
}

/* Call the installed machine check handler for this CPU setup. */
void (*machine_check_vector)(struct pt_regs *, long error_code) =
						unexpected_machine_check;

1422
/*
L
Linus Torvalds 已提交
1423
 * Called for each booted CPU to set up machine checks.
I
Ingo Molnar 已提交
1424
 * Must be called with preempt off:
L
Linus Torvalds 已提交
1425
 */
1426
void __cpuinit mcheck_cpu_init(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1427
{
1428 1429 1430
	if (mce_disabled)
		return;

1431
	__mcheck_cpu_ancient_init(c);
1432

1433
	if (!mce_available(c))
L
Linus Torvalds 已提交
1434 1435
		return;

1436
	if (__mcheck_cpu_cap_init() < 0 || __mcheck_cpu_apply_quirks(c) < 0) {
1437
		mce_disabled = 1;
1438 1439 1440
		return;
	}

1441 1442
	machine_check_vector = do_machine_check;

1443 1444 1445
	__mcheck_cpu_init_generic();
	__mcheck_cpu_init_vendor(c);
	__mcheck_cpu_init_timer();
1446
	INIT_WORK(&__get_cpu_var(mce_work), mce_process_work);
1447

L
Linus Torvalds 已提交
1448 1449 1450 1451 1452 1453
}

/*
 * Character device to read and clear the MCE log.
 */

T
Tim Hockin 已提交
1454
static DEFINE_SPINLOCK(mce_state_lock);
I
Ingo Molnar 已提交
1455 1456
static int		open_count;		/* #times opened */
static int		open_exclu;		/* already open exclusive? */
T
Tim Hockin 已提交
1457 1458 1459 1460 1461 1462 1463

static int mce_open(struct inode *inode, struct file *file)
{
	spin_lock(&mce_state_lock);

	if (open_exclu || (open_count && (file->f_flags & O_EXCL))) {
		spin_unlock(&mce_state_lock);
I
Ingo Molnar 已提交
1464

T
Tim Hockin 已提交
1465 1466 1467 1468 1469 1470 1471 1472 1473
		return -EBUSY;
	}

	if (file->f_flags & O_EXCL)
		open_exclu = 1;
	open_count++;

	spin_unlock(&mce_state_lock);

1474
	return nonseekable_open(inode, file);
T
Tim Hockin 已提交
1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488
}

static int mce_release(struct inode *inode, struct file *file)
{
	spin_lock(&mce_state_lock);

	open_count--;
	open_exclu = 0;

	spin_unlock(&mce_state_lock);

	return 0;
}

1489 1490
static void collect_tscs(void *data)
{
L
Linus Torvalds 已提交
1491
	unsigned long *cpu_tsc = (unsigned long *)data;
1492

L
Linus Torvalds 已提交
1493
	rdtscll(cpu_tsc[smp_processor_id()]);
1494
}
L
Linus Torvalds 已提交
1495

1496 1497
static ssize_t mce_read(struct file *filp, char __user *ubuf, size_t usize,
			loff_t *off)
L
Linus Torvalds 已提交
1498
{
I
Ingo Molnar 已提交
1499
	char __user *buf = ubuf;
1500
	unsigned long *cpu_tsc;
1501
	unsigned prev, next;
L
Linus Torvalds 已提交
1502 1503
	int i, err;

1504
	cpu_tsc = kmalloc(nr_cpu_ids * sizeof(long), GFP_KERNEL);
1505 1506 1507
	if (!cpu_tsc)
		return -ENOMEM;

1508
	mutex_lock(&mce_read_mutex);
1509
	next = rcu_dereference_check_mce(mcelog.next);
L
Linus Torvalds 已提交
1510 1511

	/* Only supports full reads right now */
1512
	if (*off != 0 || usize < MCE_LOG_LEN*sizeof(struct mce)) {
1513
		mutex_unlock(&mce_read_mutex);
1514
		kfree(cpu_tsc);
I
Ingo Molnar 已提交
1515

L
Linus Torvalds 已提交
1516 1517 1518 1519
		return -EINVAL;
	}

	err = 0;
1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531
	prev = 0;
	do {
		for (i = prev; i < next; i++) {
			unsigned long start = jiffies;

			while (!mcelog.entry[i].finished) {
				if (time_after_eq(jiffies, start + 2)) {
					memset(mcelog.entry + i, 0,
					       sizeof(struct mce));
					goto timeout;
				}
				cpu_relax();
1532
			}
1533 1534 1535 1536 1537 1538
			smp_rmb();
			err |= copy_to_user(buf, mcelog.entry + i,
					    sizeof(struct mce));
			buf += sizeof(struct mce);
timeout:
			;
1539
		}
L
Linus Torvalds 已提交
1540

1541 1542 1543 1544 1545
		memset(mcelog.entry + prev, 0,
		       (next - prev) * sizeof(struct mce));
		prev = next;
		next = cmpxchg(&mcelog.next, prev, 0);
	} while (next != prev);
L
Linus Torvalds 已提交
1546

1547
	synchronize_sched();
L
Linus Torvalds 已提交
1548

1549 1550 1551 1552
	/*
	 * Collect entries that were still getting written before the
	 * synchronize.
	 */
1553
	on_each_cpu(collect_tscs, cpu_tsc, 1);
I
Ingo Molnar 已提交
1554

1555 1556 1557 1558 1559
	for (i = next; i < MCE_LOG_LEN; i++) {
		if (mcelog.entry[i].finished &&
		    mcelog.entry[i].tsc < cpu_tsc[mcelog.entry[i].cpu]) {
			err |= copy_to_user(buf, mcelog.entry+i,
					    sizeof(struct mce));
L
Linus Torvalds 已提交
1560 1561 1562 1563
			smp_rmb();
			buf += sizeof(struct mce);
			memset(&mcelog.entry[i], 0, sizeof(struct mce));
		}
1564
	}
1565
	mutex_unlock(&mce_read_mutex);
1566
	kfree(cpu_tsc);
I
Ingo Molnar 已提交
1567

1568
	return err ? -EFAULT : buf - ubuf;
L
Linus Torvalds 已提交
1569 1570
}

1571 1572 1573
static unsigned int mce_poll(struct file *file, poll_table *wait)
{
	poll_wait(file, &mce_wait, wait);
1574
	if (rcu_dereference_check_mce(mcelog.next))
1575 1576 1577 1578
		return POLLIN | POLLRDNORM;
	return 0;
}

1579
static long mce_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
L
Linus Torvalds 已提交
1580 1581
{
	int __user *p = (int __user *)arg;
1582

L
Linus Torvalds 已提交
1583
	if (!capable(CAP_SYS_ADMIN))
1584
		return -EPERM;
I
Ingo Molnar 已提交
1585

L
Linus Torvalds 已提交
1586
	switch (cmd) {
1587
	case MCE_GET_RECORD_LEN:
L
Linus Torvalds 已提交
1588 1589
		return put_user(sizeof(struct mce), p);
	case MCE_GET_LOG_LEN:
1590
		return put_user(MCE_LOG_LEN, p);
L
Linus Torvalds 已提交
1591 1592
	case MCE_GETCLEAR_FLAGS: {
		unsigned flags;
1593 1594

		do {
L
Linus Torvalds 已提交
1595
			flags = mcelog.flags;
1596
		} while (cmpxchg(&mcelog.flags, flags, 0) != flags);
I
Ingo Molnar 已提交
1597

1598
		return put_user(flags, p);
L
Linus Torvalds 已提交
1599 1600
	}
	default:
1601 1602
		return -ENOTTY;
	}
L
Linus Torvalds 已提交
1603 1604
}

1605
/* Modified in mce-inject.c, so not static or const */
1606
struct file_operations mce_chrdev_ops = {
I
Ingo Molnar 已提交
1607 1608 1609 1610 1611
	.open			= mce_open,
	.release		= mce_release,
	.read			= mce_read,
	.poll			= mce_poll,
	.unlocked_ioctl		= mce_ioctl,
L
Linus Torvalds 已提交
1612
};
1613
EXPORT_SYMBOL_GPL(mce_chrdev_ops);
L
Linus Torvalds 已提交
1614 1615 1616 1617 1618 1619 1620

static struct miscdevice mce_log_device = {
	MISC_MCELOG_MINOR,
	"mcelog",
	&mce_chrdev_ops,
};

H
Hidetoshi Seto 已提交
1621
/*
1622 1623 1624 1625
 * mce=off Disables machine check
 * mce=no_cmci Disables CMCI
 * mce=dont_log_ce Clears corrected events silently, no log created for CEs.
 * mce=ignore_ce Disables polling and CMCI, corrected events are not cleared.
1626 1627 1628
 * mce=TOLERANCELEVEL[,monarchtimeout] (number, see above)
 *	monarchtimeout is how long to wait for other CPUs on machine
 *	check, or 0 to not wait
H
Hidetoshi Seto 已提交
1629 1630 1631
 * mce=bootlog Log MCEs from before booting. Disabled by default on AMD.
 * mce=nobootlog Don't log MCEs from before booting.
 */
L
Linus Torvalds 已提交
1632 1633
static int __init mcheck_enable(char *str)
{
1634
	if (*str == 0) {
1635
		enable_p5_mce();
1636 1637
		return 1;
	}
1638 1639
	if (*str == '=')
		str++;
L
Linus Torvalds 已提交
1640
	if (!strcmp(str, "off"))
1641
		mce_disabled = 1;
1642 1643 1644 1645 1646 1647
	else if (!strcmp(str, "no_cmci"))
		mce_cmci_disabled = 1;
	else if (!strcmp(str, "dont_log_ce"))
		mce_dont_log_ce = 1;
	else if (!strcmp(str, "ignore_ce"))
		mce_ignore_ce = 1;
H
Hidetoshi Seto 已提交
1648 1649
	else if (!strcmp(str, "bootlog") || !strcmp(str, "nobootlog"))
		mce_bootlog = (str[0] == 'b');
1650
	else if (isdigit(str[0])) {
1651
		get_option(&str, &tolerant);
1652 1653 1654 1655 1656
		if (*str == ',') {
			++str;
			get_option(&str, &monarch_timeout);
		}
	} else {
1657
		printk(KERN_INFO "mce argument %s ignored. Please use /sys\n",
H
Hidetoshi Seto 已提交
1658 1659 1660
		       str);
		return 0;
	}
1661
	return 1;
L
Linus Torvalds 已提交
1662
}
1663
__setup("mce", mcheck_enable);
L
Linus Torvalds 已提交
1664

1665
int __init mcheck_init(void)
1666 1667 1668
{
	atomic_notifier_chain_register(&x86_mce_decoder_chain, &mce_dec_nb);

1669 1670
	mcheck_intel_therm_init();

1671 1672 1673
	return 0;
}

1674
/*
L
Linus Torvalds 已提交
1675
 * Sysfs support
1676
 */
L
Linus Torvalds 已提交
1677

1678 1679 1680 1681
/*
 * Disable machine checks on suspend and shutdown. We can't really handle
 * them later.
 */
1682
static int mce_disable_error_reporting(void)
1683 1684 1685
{
	int i;

1686
	for (i = 0; i < banks; i++) {
1687
		struct mce_bank *b = &mce_banks[i];
1688

1689
		if (b->init)
1690
			wrmsrl(MSR_IA32_MCx_CTL(i), 0);
1691
	}
1692 1693 1694 1695 1696
	return 0;
}

static int mce_suspend(struct sys_device *dev, pm_message_t state)
{
1697
	return mce_disable_error_reporting();
1698 1699 1700 1701
}

static int mce_shutdown(struct sys_device *dev)
{
1702
	return mce_disable_error_reporting();
1703 1704
}

I
Ingo Molnar 已提交
1705 1706 1707 1708 1709
/*
 * On resume clear all MCE state. Don't want to see leftovers from the BIOS.
 * Only one CPU is active at this time, the others get re-added later using
 * CPU hotplug:
 */
L
Linus Torvalds 已提交
1710 1711
static int mce_resume(struct sys_device *dev)
{
1712 1713
	__mcheck_cpu_init_generic();
	__mcheck_cpu_init_vendor(&current_cpu_data);
I
Ingo Molnar 已提交
1714

L
Linus Torvalds 已提交
1715 1716 1717
	return 0;
}

1718 1719 1720
static void mce_cpu_restart(void *data)
{
	del_timer_sync(&__get_cpu_var(mce_timer));
1721 1722
	if (!mce_available(&current_cpu_data))
		return;
1723 1724
	__mcheck_cpu_init_generic();
	__mcheck_cpu_init_timer();
1725 1726
}

L
Linus Torvalds 已提交
1727
/* Reinit MCEs after user configuration changes */
1728 1729
static void mce_restart(void)
{
1730
	on_each_cpu(mce_cpu_restart, NULL, 1);
L
Linus Torvalds 已提交
1731 1732
}

1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749
/* Toggle features for corrected errors */
static void mce_disable_ce(void *all)
{
	if (!mce_available(&current_cpu_data))
		return;
	if (all)
		del_timer_sync(&__get_cpu_var(mce_timer));
	cmci_clear();
}

static void mce_enable_ce(void *all)
{
	if (!mce_available(&current_cpu_data))
		return;
	cmci_reenable();
	cmci_recheck();
	if (all)
1750
		__mcheck_cpu_init_timer();
1751 1752
}

L
Linus Torvalds 已提交
1753
static struct sysdev_class mce_sysclass = {
I
Ingo Molnar 已提交
1754 1755 1756 1757
	.suspend	= mce_suspend,
	.shutdown	= mce_shutdown,
	.resume		= mce_resume,
	.name		= "machinecheck",
L
Linus Torvalds 已提交
1758 1759
};

I
Ingo Molnar 已提交
1760
DEFINE_PER_CPU(struct sys_device, mce_dev);
I
Ingo Molnar 已提交
1761 1762 1763

__cpuinitdata
void (*threshold_cpu_callback)(unsigned long action, unsigned int cpu);
L
Linus Torvalds 已提交
1764

1765 1766 1767 1768
static inline struct mce_bank *attr_to_bank(struct sysdev_attribute *attr)
{
	return container_of(attr, struct mce_bank, attr);
}
1769 1770 1771 1772

static ssize_t show_bank(struct sys_device *s, struct sysdev_attribute *attr,
			 char *buf)
{
1773
	return sprintf(buf, "%llx\n", attr_to_bank(attr)->ctl);
1774 1775 1776
}

static ssize_t set_bank(struct sys_device *s, struct sysdev_attribute *attr,
H
Hidetoshi Seto 已提交
1777
			const char *buf, size_t size)
1778
{
H
Hidetoshi Seto 已提交
1779
	u64 new;
I
Ingo Molnar 已提交
1780

H
Hidetoshi Seto 已提交
1781
	if (strict_strtoull(buf, 0, &new) < 0)
1782
		return -EINVAL;
I
Ingo Molnar 已提交
1783

1784
	attr_to_bank(attr)->ctl = new;
1785
	mce_restart();
I
Ingo Molnar 已提交
1786

H
Hidetoshi Seto 已提交
1787
	return size;
1788
}
1789

I
Ingo Molnar 已提交
1790 1791
static ssize_t
show_trigger(struct sys_device *s, struct sysdev_attribute *attr, char *buf)
1792
{
1793
	strcpy(buf, mce_helper);
1794
	strcat(buf, "\n");
1795
	return strlen(mce_helper) + 1;
1796 1797
}

1798
static ssize_t set_trigger(struct sys_device *s, struct sysdev_attribute *attr,
I
Ingo Molnar 已提交
1799
				const char *buf, size_t siz)
1800 1801
{
	char *p;
I
Ingo Molnar 已提交
1802

1803 1804 1805
	strncpy(mce_helper, buf, sizeof(mce_helper));
	mce_helper[sizeof(mce_helper)-1] = 0;
	p = strchr(mce_helper, '\n');
I
Ingo Molnar 已提交
1806

1807
	if (p)
I
Ingo Molnar 已提交
1808 1809
		*p = 0;

1810
	return strlen(mce_helper) + !!p;
1811 1812
}

1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858
static ssize_t set_ignore_ce(struct sys_device *s,
			     struct sysdev_attribute *attr,
			     const char *buf, size_t size)
{
	u64 new;

	if (strict_strtoull(buf, 0, &new) < 0)
		return -EINVAL;

	if (mce_ignore_ce ^ !!new) {
		if (new) {
			/* disable ce features */
			on_each_cpu(mce_disable_ce, (void *)1, 1);
			mce_ignore_ce = 1;
		} else {
			/* enable ce features */
			mce_ignore_ce = 0;
			on_each_cpu(mce_enable_ce, (void *)1, 1);
		}
	}
	return size;
}

static ssize_t set_cmci_disabled(struct sys_device *s,
				 struct sysdev_attribute *attr,
				 const char *buf, size_t size)
{
	u64 new;

	if (strict_strtoull(buf, 0, &new) < 0)
		return -EINVAL;

	if (mce_cmci_disabled ^ !!new) {
		if (new) {
			/* disable cmci */
			on_each_cpu(mce_disable_ce, NULL, 1);
			mce_cmci_disabled = 1;
		} else {
			/* enable cmci */
			mce_cmci_disabled = 0;
			on_each_cpu(mce_enable_ce, NULL, 1);
		}
	}
	return size;
}

1859 1860 1861 1862 1863 1864 1865 1866 1867
static ssize_t store_int_with_restart(struct sys_device *s,
				      struct sysdev_attribute *attr,
				      const char *buf, size_t size)
{
	ssize_t ret = sysdev_store_int(s, attr, buf, size);
	mce_restart();
	return ret;
}

1868
static SYSDEV_ATTR(trigger, 0644, show_trigger, set_trigger);
1869
static SYSDEV_INT_ATTR(tolerant, 0644, tolerant);
1870
static SYSDEV_INT_ATTR(monarch_timeout, 0644, monarch_timeout);
1871
static SYSDEV_INT_ATTR(dont_log_ce, 0644, mce_dont_log_ce);
I
Ingo Molnar 已提交
1872

1873 1874 1875 1876 1877
static struct sysdev_ext_attribute attr_check_interval = {
	_SYSDEV_ATTR(check_interval, 0644, sysdev_show_int,
		     store_int_with_restart),
	&check_interval
};
I
Ingo Molnar 已提交
1878

1879 1880 1881 1882 1883 1884
static struct sysdev_ext_attribute attr_ignore_ce = {
	_SYSDEV_ATTR(ignore_ce, 0644, sysdev_show_int, set_ignore_ce),
	&mce_ignore_ce
};

static struct sysdev_ext_attribute attr_cmci_disabled = {
Y
Yinghai Lu 已提交
1885
	_SYSDEV_ATTR(cmci_disabled, 0644, sysdev_show_int, set_cmci_disabled),
1886 1887 1888
	&mce_cmci_disabled
};

I
Ingo Molnar 已提交
1889
static struct sysdev_attribute *mce_attrs[] = {
1890 1891 1892
	&attr_tolerant.attr,
	&attr_check_interval.attr,
	&attr_trigger,
1893
	&attr_monarch_timeout.attr,
1894 1895 1896
	&attr_dont_log_ce.attr,
	&attr_ignore_ce.attr,
	&attr_cmci_disabled.attr,
1897 1898
	NULL
};
L
Linus Torvalds 已提交
1899

I
Ingo Molnar 已提交
1900
static cpumask_var_t mce_dev_initialized;
1901

I
Ingo Molnar 已提交
1902
/* Per cpu sysdev init. All of the cpus still share the same ctrl bank: */
1903
static __cpuinit int mce_create_device(unsigned int cpu)
L
Linus Torvalds 已提交
1904 1905
{
	int err;
1906
	int i, j;
1907

A
Andreas Herrmann 已提交
1908
	if (!mce_available(&boot_cpu_data))
1909 1910
		return -EIO;

I
Ingo Molnar 已提交
1911 1912 1913
	memset(&per_cpu(mce_dev, cpu).kobj, 0, sizeof(struct kobject));
	per_cpu(mce_dev, cpu).id	= cpu;
	per_cpu(mce_dev, cpu).cls	= &mce_sysclass;
1914

I
Ingo Molnar 已提交
1915
	err = sysdev_register(&per_cpu(mce_dev, cpu));
1916 1917 1918
	if (err)
		return err;

I
Ingo Molnar 已提交
1919 1920
	for (i = 0; mce_attrs[i]; i++) {
		err = sysdev_create_file(&per_cpu(mce_dev, cpu), mce_attrs[i]);
1921 1922 1923
		if (err)
			goto error;
	}
1924
	for (j = 0; j < banks; j++) {
I
Ingo Molnar 已提交
1925
		err = sysdev_create_file(&per_cpu(mce_dev, cpu),
1926
					&mce_banks[j].attr);
1927 1928 1929
		if (err)
			goto error2;
	}
I
Ingo Molnar 已提交
1930
	cpumask_set_cpu(cpu, mce_dev_initialized);
1931

1932
	return 0;
1933
error2:
1934
	while (--j >= 0)
1935
		sysdev_remove_file(&per_cpu(mce_dev, cpu), &mce_banks[j].attr);
1936
error:
I
Ingo Molnar 已提交
1937
	while (--i >= 0)
1938
		sysdev_remove_file(&per_cpu(mce_dev, cpu), mce_attrs[i]);
I
Ingo Molnar 已提交
1939 1940

	sysdev_unregister(&per_cpu(mce_dev, cpu));
1941

1942 1943 1944
	return err;
}

1945
static __cpuinit void mce_remove_device(unsigned int cpu)
1946
{
1947 1948
	int i;

I
Ingo Molnar 已提交
1949
	if (!cpumask_test_cpu(cpu, mce_dev_initialized))
1950 1951
		return;

I
Ingo Molnar 已提交
1952 1953 1954
	for (i = 0; mce_attrs[i]; i++)
		sysdev_remove_file(&per_cpu(mce_dev, cpu), mce_attrs[i]);

1955
	for (i = 0; i < banks; i++)
1956
		sysdev_remove_file(&per_cpu(mce_dev, cpu), &mce_banks[i].attr);
I
Ingo Molnar 已提交
1957 1958 1959

	sysdev_unregister(&per_cpu(mce_dev, cpu));
	cpumask_clear_cpu(cpu, mce_dev_initialized);
1960 1961
}

1962
/* Make sure there are no machine checks on offlined CPUs. */
1963
static void __cpuinit mce_disable_cpu(void *h)
1964
{
A
Andi Kleen 已提交
1965
	unsigned long action = *(unsigned long *)h;
I
Ingo Molnar 已提交
1966
	int i;
1967 1968 1969

	if (!mce_available(&current_cpu_data))
		return;
1970

A
Andi Kleen 已提交
1971 1972
	if (!(action & CPU_TASKS_FROZEN))
		cmci_clear();
1973
	for (i = 0; i < banks; i++) {
1974
		struct mce_bank *b = &mce_banks[i];
1975

1976
		if (b->init)
1977
			wrmsrl(MSR_IA32_MCx_CTL(i), 0);
1978
	}
1979 1980
}

1981
static void __cpuinit mce_reenable_cpu(void *h)
1982
{
A
Andi Kleen 已提交
1983
	unsigned long action = *(unsigned long *)h;
I
Ingo Molnar 已提交
1984
	int i;
1985 1986 1987

	if (!mce_available(&current_cpu_data))
		return;
I
Ingo Molnar 已提交
1988

A
Andi Kleen 已提交
1989 1990
	if (!(action & CPU_TASKS_FROZEN))
		cmci_reenable();
1991
	for (i = 0; i < banks; i++) {
1992
		struct mce_bank *b = &mce_banks[i];
1993

1994
		if (b->init)
1995
			wrmsrl(MSR_IA32_MCx_CTL(i), b->ctl);
1996
	}
1997 1998
}

1999
/* Get notified when a cpu comes on/off. Be hotplug friendly. */
I
Ingo Molnar 已提交
2000 2001
static int __cpuinit
mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
2002 2003
{
	unsigned int cpu = (unsigned long)hcpu;
2004
	struct timer_list *t = &per_cpu(mce_timer, cpu);
2005 2006

	switch (action) {
2007 2008 2009
	case CPU_ONLINE:
	case CPU_ONLINE_FROZEN:
		mce_create_device(cpu);
2010 2011
		if (threshold_cpu_callback)
			threshold_cpu_callback(action, cpu);
2012 2013
		break;
	case CPU_DEAD:
2014
	case CPU_DEAD_FROZEN:
2015 2016
		if (threshold_cpu_callback)
			threshold_cpu_callback(action, cpu);
2017 2018
		mce_remove_device(cpu);
		break;
2019 2020 2021
	case CPU_DOWN_PREPARE:
	case CPU_DOWN_PREPARE_FROZEN:
		del_timer_sync(t);
A
Andi Kleen 已提交
2022
		smp_call_function_single(cpu, mce_disable_cpu, &action, 1);
2023 2024 2025
		break;
	case CPU_DOWN_FAILED:
	case CPU_DOWN_FAILED_FROZEN:
2026 2027
		if (!mce_ignore_ce && check_interval) {
			t->expires = round_jiffies(jiffies +
2028
					   __get_cpu_var(mce_next_interval));
2029 2030
			add_timer_on(t, cpu);
		}
A
Andi Kleen 已提交
2031 2032 2033 2034 2035
		smp_call_function_single(cpu, mce_reenable_cpu, &action, 1);
		break;
	case CPU_POST_DEAD:
		/* intentionally ignoring frozen here */
		cmci_rediscover(cpu);
2036
		break;
2037
	}
2038
	return NOTIFY_OK;
2039 2040
}

2041
static struct notifier_block mce_cpu_notifier __cpuinitdata = {
2042 2043 2044
	.notifier_call = mce_cpu_callback,
};

2045
static __init void mce_init_banks(void)
2046 2047 2048 2049
{
	int i;

	for (i = 0; i < banks; i++) {
2050 2051
		struct mce_bank *b = &mce_banks[i];
		struct sysdev_attribute *a = &b->attr;
I
Ingo Molnar 已提交
2052

2053
		sysfs_attr_init(&a->attr);
2054 2055
		a->attr.name	= b->attrname;
		snprintf(b->attrname, ATTR_LEN, "bank%d", i);
I
Ingo Molnar 已提交
2056 2057 2058 2059

		a->attr.mode	= 0644;
		a->show		= show_bank;
		a->store	= set_bank;
2060 2061 2062
	}
}

2063
static __init int mcheck_init_device(void)
2064 2065 2066 2067
{
	int err;
	int i = 0;

L
Linus Torvalds 已提交
2068 2069
	if (!mce_available(&boot_cpu_data))
		return -EIO;
2070

2071
	zalloc_cpumask_var(&mce_dev_initialized, GFP_KERNEL);
2072

2073
	mce_init_banks();
2074

L
Linus Torvalds 已提交
2075
	err = sysdev_class_register(&mce_sysclass);
2076 2077
	if (err)
		return err;
2078 2079

	for_each_online_cpu(i) {
2080 2081 2082
		err = mce_create_device(i);
		if (err)
			return err;
2083 2084
	}

2085
	register_hotcpu_notifier(&mce_cpu_notifier);
L
Linus Torvalds 已提交
2086
	misc_register(&mce_log_device);
I
Ingo Molnar 已提交
2087

L
Linus Torvalds 已提交
2088 2089
	return err;
}
2090

2091
device_initcall(mcheck_init_device);
I
Ingo Molnar 已提交
2092

2093 2094 2095 2096 2097 2098 2099 2100 2101
/*
 * Old style boot options parsing. Only for compatibility.
 */
static int __init mcheck_disable(char *str)
{
	mce_disabled = 1;
	return 1;
}
__setup("nomce", mcheck_disable);
I
Ingo Molnar 已提交
2102

2103 2104
#ifdef CONFIG_DEBUG_FS
struct dentry *mce_get_debugfs_dir(void)
I
Ingo Molnar 已提交
2105
{
2106
	static struct dentry *dmce;
I
Ingo Molnar 已提交
2107

2108 2109
	if (!dmce)
		dmce = debugfs_create_dir("mce", NULL);
I
Ingo Molnar 已提交
2110

2111 2112
	return dmce;
}
I
Ingo Molnar 已提交
2113

2114 2115 2116 2117 2118 2119 2120 2121
static void mce_reset(void)
{
	cpu_missing = 0;
	atomic_set(&mce_fake_paniced, 0);
	atomic_set(&mce_executing, 0);
	atomic_set(&mce_callin, 0);
	atomic_set(&global_nwo, 0);
}
I
Ingo Molnar 已提交
2122

2123 2124 2125 2126
static int fake_panic_get(void *data, u64 *val)
{
	*val = fake_panic;
	return 0;
I
Ingo Molnar 已提交
2127 2128
}

2129
static int fake_panic_set(void *data, u64 val)
I
Ingo Molnar 已提交
2130
{
2131 2132 2133
	mce_reset();
	fake_panic = val;
	return 0;
I
Ingo Molnar 已提交
2134 2135
}

2136 2137
DEFINE_SIMPLE_ATTRIBUTE(fake_panic_fops, fake_panic_get,
			fake_panic_set, "%llu\n");
2138

2139
static int __init mcheck_debugfs_init(void)
2140
{
2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151
	struct dentry *dmce, *ffake_panic;

	dmce = mce_get_debugfs_dir();
	if (!dmce)
		return -ENOMEM;
	ffake_panic = debugfs_create_file("fake_panic", 0444, dmce, NULL,
					  &fake_panic_fops);
	if (!ffake_panic)
		return -ENOMEM;

	return 0;
2152
}
2153
late_initcall(mcheck_debugfs_init);
2154
#endif