mce.c 48.9 KB
Newer Older
L
Linus Torvalds 已提交
1 2
/*
 * Machine check handler.
I
Ingo Molnar 已提交
3
 *
L
Linus Torvalds 已提交
4
 * K8 parts Copyright 2002,2003 Andi Kleen, SuSE Labs.
5 6
 * Rest from unknown author(s).
 * 2004 Andi Kleen. Rewrote most of it.
7 8
 * Copyright 2008 Intel Corporation
 * Author: Andi Kleen
L
Linus Torvalds 已提交
9
 */
I
Ingo Molnar 已提交
10 11 12
#include <linux/thread_info.h>
#include <linux/capability.h>
#include <linux/miscdevice.h>
13
#include <linux/interrupt.h>
I
Ingo Molnar 已提交
14 15 16 17
#include <linux/ratelimit.h>
#include <linux/kallsyms.h>
#include <linux/rcupdate.h>
#include <linux/kobject.h>
18
#include <linux/uaccess.h>
I
Ingo Molnar 已提交
19 20 21
#include <linux/kdebug.h>
#include <linux/kernel.h>
#include <linux/percpu.h>
L
Linus Torvalds 已提交
22 23
#include <linux/string.h>
#include <linux/sysdev.h>
24
#include <linux/delay.h>
25
#include <linux/ctype.h>
I
Ingo Molnar 已提交
26
#include <linux/sched.h>
27
#include <linux/sysfs.h>
I
Ingo Molnar 已提交
28 29 30 31
#include <linux/types.h>
#include <linux/init.h>
#include <linux/kmod.h>
#include <linux/poll.h>
32
#include <linux/nmi.h>
I
Ingo Molnar 已提交
33
#include <linux/cpu.h>
34
#include <linux/smp.h>
I
Ingo Molnar 已提交
35
#include <linux/fs.h>
36
#include <linux/mm.h>
37
#include <linux/debugfs.h>
I
Ingo Molnar 已提交
38

39
#include <asm/processor.h>
40 41
#include <asm/hw_irq.h>
#include <asm/apic.h>
42
#include <asm/idle.h>
43
#include <asm/ipi.h>
I
Ingo Molnar 已提交
44 45
#include <asm/mce.h>
#include <asm/msr.h>
L
Linus Torvalds 已提交
46

47
#include "mce-internal.h"
48

49 50
static DEFINE_MUTEX(mce_read_mutex);

51 52 53 54 55
#define rcu_dereference_check_mce(p) \
	rcu_dereference_check((p), \
			      rcu_read_lock_sched_held() || \
			      lockdep_is_held(&mce_read_mutex))

56 57 58
#define CREATE_TRACE_POINTS
#include <trace/events/mce.h>

H
Hidetoshi Seto 已提交
59
int mce_disabled __read_mostly;
60

I
Ingo Molnar 已提交
61
#define MISC_MCELOG_MINOR	227
62

63 64
#define SPINUNIT 100	/* 100ns */

65 66
atomic_t mce_entry;

67 68
DEFINE_PER_CPU(unsigned, mce_exception_count);

69 70 71 72 73 74 75
/*
 * Tolerant levels:
 *   0: always panic on uncorrected errors, log corrected errors
 *   1: panic or SIGBUS on uncorrected errors, log corrected errors
 *   2: SIGBUS or log uncorrected errors (if possible), log corrected errors
 *   3: never panic or SIGBUS, log all errors (for testing only)
 */
H
Hidetoshi Seto 已提交
76 77 78 79 80 81 82 83 84 85
static int			tolerant		__read_mostly = 1;
static int			banks			__read_mostly;
static int			rip_msr			__read_mostly;
static int			mce_bootlog		__read_mostly = -1;
static int			monarch_timeout		__read_mostly = -1;
static int			mce_panic_timeout	__read_mostly;
static int			mce_dont_log_ce		__read_mostly;
int				mce_cmci_disabled	__read_mostly;
int				mce_ignore_ce		__read_mostly;
int				mce_ser			__read_mostly;
86

87 88
struct mce_bank                *mce_banks		__read_mostly;

89 90 91 92
/* User mode helper program triggered by machine check event */
static unsigned long		mce_need_notify;
static char			mce_helper[128];
static char			*mce_helper_argv[2] = { mce_helper, NULL };
L
Linus Torvalds 已提交
93

94
static DECLARE_WAIT_QUEUE_HEAD(mce_wait);
95 96 97
static DEFINE_PER_CPU(struct mce, mces_seen);
static int			cpu_missing;

98 99 100 101 102 103 104 105 106
/*
 * CPU/chipset specific EDAC code can register a notifier call here to print
 * MCE errors in a human-readable form.
 */
ATOMIC_NOTIFIER_HEAD(x86_mce_decoder_chain);
EXPORT_SYMBOL_GPL(x86_mce_decoder_chain);

static int default_decode_mce(struct notifier_block *nb, unsigned long val,
			       void *data)
107 108 109
{
	pr_emerg("No human readable MCE decoding support on this CPU type.\n");
	pr_emerg("Run the message through 'mcelog --ascii' to decode.\n");
110 111

	return NOTIFY_STOP;
112 113
}

114 115 116 117
static struct notifier_block mce_dec_nb = {
	.notifier_call = default_decode_mce,
	.priority      = -1,
};
118

119 120 121 122 123
/* MCA banks polled by the period polling timer for corrected events */
DEFINE_PER_CPU(mce_banks_t, mce_poll_banks) = {
	[0 ... BITS_TO_LONGS(MAX_NR_BANKS)-1] = ~0UL
};

124 125
static DEFINE_PER_CPU(struct work_struct, mce_work);

126 127 128 129
/* Do initial initialization of a struct mce */
void mce_setup(struct mce *m)
{
	memset(m, 0, sizeof(struct mce));
130
	m->cpu = m->extcpu = smp_processor_id();
131
	rdtscll(m->tsc);
132 133 134 135 136 137 138 139 140
	/* We hope get_seconds stays lockless */
	m->time = get_seconds();
	m->cpuvendor = boot_cpu_data.x86_vendor;
	m->cpuid = cpuid_eax(1);
#ifdef CONFIG_SMP
	m->socketid = cpu_data(m->extcpu).phys_proc_id;
#endif
	m->apicid = cpu_data(m->extcpu).initial_apicid;
	rdmsrl(MSR_IA32_MCG_CAP, m->mcgcap);
141 142
}

143 144 145
DEFINE_PER_CPU(struct mce, injectm);
EXPORT_PER_CPU_SYMBOL_GPL(injectm);

L
Linus Torvalds 已提交
146 147 148 149 150 151
/*
 * Lockless MCE logging infrastructure.
 * This avoids deadlocks on printk locks without having to break locks. Also
 * separate MCEs from kernel messages to avoid bogus bug reports.
 */

152
static struct mce_log mcelog = {
153 154 155
	.signature	= MCE_LOG_SIGNATURE,
	.len		= MCE_LOG_LEN,
	.recordlen	= sizeof(struct mce),
156
};
L
Linus Torvalds 已提交
157 158 159 160

void mce_log(struct mce *mce)
{
	unsigned next, entry;
I
Ingo Molnar 已提交
161

162 163 164
	/* Emit the trace record: */
	trace_mce_record(mce);

L
Linus Torvalds 已提交
165
	mce->finished = 0;
M
Mike Waychison 已提交
166
	wmb();
L
Linus Torvalds 已提交
167
	for (;;) {
168
		entry = rcu_dereference_check_mce(mcelog.next);
169
		for (;;) {
I
Ingo Molnar 已提交
170 171 172 173 174
			/*
			 * When the buffer fills up discard new entries.
			 * Assume that the earlier errors are the more
			 * interesting ones:
			 */
175
			if (entry >= MCE_LOG_LEN) {
176 177
				set_bit(MCE_OVERFLOW,
					(unsigned long *)&mcelog.flags);
178 179
				return;
			}
I
Ingo Molnar 已提交
180
			/* Old left over entry. Skip: */
181 182 183 184
			if (mcelog.entry[entry].finished) {
				entry++;
				continue;
			}
M
Mike Waychison 已提交
185
			break;
L
Linus Torvalds 已提交
186 187 188 189 190 191 192
		}
		smp_rmb();
		next = entry + 1;
		if (cmpxchg(&mcelog.next, entry, next) == entry)
			break;
	}
	memcpy(mcelog.entry + entry, mce, sizeof(struct mce));
M
Mike Waychison 已提交
193
	wmb();
L
Linus Torvalds 已提交
194
	mcelog.entry[entry].finished = 1;
M
Mike Waychison 已提交
195
	wmb();
L
Linus Torvalds 已提交
196

197
	mce->finished = 1;
198
	set_bit(0, &mce_need_notify);
L
Linus Torvalds 已提交
199 200
}

H
Hidetoshi Seto 已提交
201
static void print_mce(struct mce *m)
L
Linus Torvalds 已提交
202
{
203
	pr_emerg("CPU %d: Machine Check Exception: %16Lx Bank %d: %016Lx\n",
204
	       m->extcpu, m->mcgstatus, m->bank, m->status);
205

206
	if (m->ip) {
207 208 209 210
		pr_emerg("RIP%s %02x:<%016Lx> ",
			!(m->mcgstatus & MCG_STATUS_EIPV) ? " !INEXACT!" : "",
				m->cs, m->ip);

L
Linus Torvalds 已提交
211
		if (m->cs == __KERNEL_CS)
212
			print_symbol("{%s}", m->ip);
213
		pr_cont("\n");
L
Linus Torvalds 已提交
214
	}
215 216

	pr_emerg("TSC %llx ", m->tsc);
L
Linus Torvalds 已提交
217
	if (m->addr)
218
		pr_cont("ADDR %llx ", m->addr);
L
Linus Torvalds 已提交
219
	if (m->misc)
220
		pr_cont("MISC %llx ", m->misc);
221

222 223 224 225 226 227
	pr_cont("\n");
	pr_emerg("PROCESSOR %u:%x TIME %llu SOCKET %u APIC %x\n",
		m->cpuvendor, m->cpuid, m->time, m->socketid, m->apicid);

	/*
	 * Print out human-readable details about the MCE error,
228
	 * (if the CPU has an implementation for that)
229
	 */
230
	atomic_notifier_call_chain(&x86_mce_decoder_chain, 0, m);
231 232
}

H
Hidetoshi Seto 已提交
233 234
static void print_mce_head(void)
{
235
	pr_emerg("\nHARDWARE ERROR\n");
H
Hidetoshi Seto 已提交
236 237
}

238 239
static void print_mce_tail(void)
{
240
	pr_emerg("This is not a software problem!\n");
L
Linus Torvalds 已提交
241 242
}

243 244 245 246
#define PANIC_TIMEOUT 5 /* 5 seconds */

static atomic_t mce_paniced;

247 248 249
static int fake_panic;
static atomic_t mce_fake_paniced;

250 251 252 253
/* Panic in progress. Enable interrupts and wait for final IPI */
static void wait_for_panic(void)
{
	long timeout = PANIC_TIMEOUT*USEC_PER_SEC;
254

255 256 257 258
	preempt_disable();
	local_irq_enable();
	while (timeout-- > 0)
		udelay(1);
259 260
	if (panic_timeout == 0)
		panic_timeout = mce_panic_timeout;
261 262 263
	panic("Panicing machine check CPU died");
}

264
static void mce_panic(char *msg, struct mce *final, char *exp)
265
{
L
Linus Torvalds 已提交
266
	int i;
267

268 269 270 271 272 273 274
	if (!fake_panic) {
		/*
		 * Make sure only one CPU runs in machine check panic
		 */
		if (atomic_inc_return(&mce_paniced) > 1)
			wait_for_panic();
		barrier();
275

276 277 278 279 280 281 282
		bust_spinlocks(1);
		console_verbose();
	} else {
		/* Don't log too much for fake panic */
		if (atomic_inc_return(&mce_fake_paniced) > 1)
			return;
	}
H
Hidetoshi Seto 已提交
283
	print_mce_head();
284
	/* First print corrected ones that are still unlogged */
L
Linus Torvalds 已提交
285
	for (i = 0; i < MCE_LOG_LEN; i++) {
286
		struct mce *m = &mcelog.entry[i];
H
Hidetoshi Seto 已提交
287 288 289 290
		if (!(m->status & MCI_STATUS_VAL))
			continue;
		if (!(m->status & MCI_STATUS_UC))
			print_mce(m);
291 292 293 294 295
	}
	/* Now print uncorrected but with the final one last */
	for (i = 0; i < MCE_LOG_LEN; i++) {
		struct mce *m = &mcelog.entry[i];
		if (!(m->status & MCI_STATUS_VAL))
L
Linus Torvalds 已提交
296
			continue;
H
Hidetoshi Seto 已提交
297 298
		if (!(m->status & MCI_STATUS_UC))
			continue;
299
		if (!final || memcmp(m, final, sizeof(struct mce)))
H
Hidetoshi Seto 已提交
300
			print_mce(m);
L
Linus Torvalds 已提交
301
	}
302
	if (final)
H
Hidetoshi Seto 已提交
303
		print_mce(final);
304 305
	if (cpu_missing)
		printk(KERN_EMERG "Some CPUs didn't answer in synchronization\n");
306
	print_mce_tail();
307 308
	if (exp)
		printk(KERN_EMERG "Machine check: %s\n", exp);
309 310 311 312 313 314
	if (!fake_panic) {
		if (panic_timeout == 0)
			panic_timeout = mce_panic_timeout;
		panic(msg);
	} else
		printk(KERN_EMERG "Fake kernel panic: %s\n", msg);
315
}
L
Linus Torvalds 已提交
316

317 318 319 320 321
/* Support code for software error injection */

static int msr_to_offset(u32 msr)
{
	unsigned bank = __get_cpu_var(injectm.bank);
322

323 324
	if (msr == rip_msr)
		return offsetof(struct mce, ip);
325
	if (msr == MSR_IA32_MCx_STATUS(bank))
326
		return offsetof(struct mce, status);
327
	if (msr == MSR_IA32_MCx_ADDR(bank))
328
		return offsetof(struct mce, addr);
329
	if (msr == MSR_IA32_MCx_MISC(bank))
330 331 332 333 334 335
		return offsetof(struct mce, misc);
	if (msr == MSR_IA32_MCG_STATUS)
		return offsetof(struct mce, mcgstatus);
	return -1;
}

336 337 338 339
/* MSR access wrappers used for error injection */
static u64 mce_rdmsrl(u32 msr)
{
	u64 v;
340

341 342
	if (__get_cpu_var(injectm).finished) {
		int offset = msr_to_offset(msr);
343

344 345 346 347
		if (offset < 0)
			return 0;
		return *(u64 *)((char *)&__get_cpu_var(injectm) + offset);
	}
348 349 350 351 352 353 354 355 356 357 358

	if (rdmsrl_safe(msr, &v)) {
		WARN_ONCE(1, "mce: Unable to read msr %d!\n", msr);
		/*
		 * Return zero in case the access faulted. This should
		 * not happen normally but can happen if the CPU does
		 * something weird, or if the code is buggy.
		 */
		v = 0;
	}

359 360 361 362 363
	return v;
}

static void mce_wrmsrl(u32 msr, u64 v)
{
364 365
	if (__get_cpu_var(injectm).finished) {
		int offset = msr_to_offset(msr);
366

367 368 369 370
		if (offset >= 0)
			*(u64 *)((char *)&__get_cpu_var(injectm) + offset) = v;
		return;
	}
371 372 373
	wrmsrl(msr, v);
}

374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428
/*
 * Simple lockless ring to communicate PFNs from the exception handler with the
 * process context work function. This is vastly simplified because there's
 * only a single reader and a single writer.
 */
#define MCE_RING_SIZE 16	/* we use one entry less */

struct mce_ring {
	unsigned short start;
	unsigned short end;
	unsigned long ring[MCE_RING_SIZE];
};
static DEFINE_PER_CPU(struct mce_ring, mce_ring);

/* Runs with CPU affinity in workqueue */
static int mce_ring_empty(void)
{
	struct mce_ring *r = &__get_cpu_var(mce_ring);

	return r->start == r->end;
}

static int mce_ring_get(unsigned long *pfn)
{
	struct mce_ring *r;
	int ret = 0;

	*pfn = 0;
	get_cpu();
	r = &__get_cpu_var(mce_ring);
	if (r->start == r->end)
		goto out;
	*pfn = r->ring[r->start];
	r->start = (r->start + 1) % MCE_RING_SIZE;
	ret = 1;
out:
	put_cpu();
	return ret;
}

/* Always runs in MCE context with preempt off */
static int mce_ring_add(unsigned long pfn)
{
	struct mce_ring *r = &__get_cpu_var(mce_ring);
	unsigned next;

	next = (r->end + 1) % MCE_RING_SIZE;
	if (next == r->start)
		return -1;
	r->ring[r->end] = pfn;
	wmb();
	r->end = next;
	return 0;
}

A
Andi Kleen 已提交
429
int mce_available(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
430
{
431
	if (mce_disabled)
432
		return 0;
433
	return cpu_has(c, X86_FEATURE_MCE) && cpu_has(c, X86_FEATURE_MCA);
L
Linus Torvalds 已提交
434 435
}

436 437 438 439 440 441 442 443 444
static void mce_schedule_work(void)
{
	if (!mce_ring_empty()) {
		struct work_struct *work = &__get_cpu_var(mce_work);
		if (!work_pending(work))
			schedule_work(work);
	}
}

H
Huang Ying 已提交
445 446 447 448
/*
 * Get the address of the instruction at the time of the machine check
 * error.
 */
449 450
static inline void mce_get_rip(struct mce *m, struct pt_regs *regs)
{
H
Huang Ying 已提交
451 452

	if (regs && (m->mcgstatus & (MCG_STATUS_RIPV|MCG_STATUS_EIPV))) {
453
		m->ip = regs->ip;
454 455
		m->cs = regs->cs;
	} else {
456
		m->ip = 0;
457 458
		m->cs = 0;
	}
H
Huang Ying 已提交
459
	if (rip_msr)
460
		m->ip = mce_rdmsrl(rip_msr);
461 462
}

463
#ifdef CONFIG_X86_LOCAL_APIC
464 465 466 467 468 469 470 471 472 473
/*
 * Called after interrupts have been reenabled again
 * when a MCE happened during an interrupts off region
 * in the kernel.
 */
asmlinkage void smp_mce_self_interrupt(struct pt_regs *regs)
{
	ack_APIC_irq();
	exit_idle();
	irq_enter();
474
	mce_notify_irq();
475
	mce_schedule_work();
476 477 478 479 480 481 482
	irq_exit();
}
#endif

static void mce_report_event(struct pt_regs *regs)
{
	if (regs->flags & (X86_VM_MASK|X86_EFLAGS_IF)) {
483
		mce_notify_irq();
484 485 486 487 488 489 490
		/*
		 * Triggering the work queue here is just an insurance
		 * policy in case the syscall exit notify handler
		 * doesn't run soon enough or ends up running on the
		 * wrong CPU (can happen when audit sleeps)
		 */
		mce_schedule_work();
491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518
		return;
	}

#ifdef CONFIG_X86_LOCAL_APIC
	/*
	 * Without APIC do not notify. The event will be picked
	 * up eventually.
	 */
	if (!cpu_has_apic)
		return;

	/*
	 * When interrupts are disabled we cannot use
	 * kernel services safely. Trigger an self interrupt
	 * through the APIC to instead do the notification
	 * after interrupts are reenabled again.
	 */
	apic->send_IPI_self(MCE_SELF_VECTOR);

	/*
	 * Wait for idle afterwards again so that we don't leave the
	 * APIC in a non idle state because the normal APIC writes
	 * cannot exclude us.
	 */
	apic_wait_icr_idle();
#endif
}

519 520
DEFINE_PER_CPU(unsigned, mce_poll_count);

521
/*
522 523 524 525
 * Poll for corrected events or events that happened before reset.
 * Those are just logged through /dev/mcelog.
 *
 * This is executed in standard interrupt context.
A
Andi Kleen 已提交
526 527 528 529 530 531 532 533 534
 *
 * Note: spec recommends to panic for fatal unsignalled
 * errors here. However this would be quite problematic --
 * we would need to reimplement the Monarch handling and
 * it would mess up the exclusion between exception handler
 * and poll hander -- * so we skip this for now.
 * These cases should not happen anyways, or only when the CPU
 * is already totally * confused. In this case it's likely it will
 * not fully execute the machine check handler either.
535
 */
536
void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
537 538 539 540
{
	struct mce m;
	int i;

541 542
	__get_cpu_var(mce_poll_count)++;

543 544
	mce_setup(&m);

545
	m.mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS);
546
	for (i = 0; i < banks; i++) {
547
		if (!mce_banks[i].ctl || !test_bit(i, *b))
548 549 550 551 552 553 554 555
			continue;

		m.misc = 0;
		m.addr = 0;
		m.bank = i;
		m.tsc = 0;

		barrier();
556
		m.status = mce_rdmsrl(MSR_IA32_MCx_STATUS(i));
557 558 559 560
		if (!(m.status & MCI_STATUS_VAL))
			continue;

		/*
A
Andi Kleen 已提交
561 562
		 * Uncorrected or signalled events are handled by the exception
		 * handler when it is enabled, so don't process those here.
563 564 565
		 *
		 * TBD do the same check for MCI_STATUS_EN here?
		 */
A
Andi Kleen 已提交
566 567
		if (!(flags & MCP_UC) &&
		    (m.status & (mce_ser ? MCI_STATUS_S : MCI_STATUS_UC)))
568 569 570
			continue;

		if (m.status & MCI_STATUS_MISCV)
571
			m.misc = mce_rdmsrl(MSR_IA32_MCx_MISC(i));
572
		if (m.status & MCI_STATUS_ADDRV)
573
			m.addr = mce_rdmsrl(MSR_IA32_MCx_ADDR(i));
574 575 576 577 578 579 580

		if (!(flags & MCP_TIMESTAMP))
			m.tsc = 0;
		/*
		 * Don't get the IP here because it's unlikely to
		 * have anything to do with the actual error location.
		 */
581
		if (!(flags & MCP_DONTLOG) && !mce_dont_log_ce) {
A
Andi Kleen 已提交
582 583 584
			mce_log(&m);
			add_taint(TAINT_MACHINE_CHECK);
		}
585 586 587 588

		/*
		 * Clear state for this bank.
		 */
589
		mce_wrmsrl(MSR_IA32_MCx_STATUS(i), 0);
590 591 592 593 594 595
	}

	/*
	 * Don't clear MCG_STATUS here because it's only defined for
	 * exceptions.
	 */
596 597

	sync_core();
598
}
599
EXPORT_SYMBOL_GPL(machine_check_poll);
600

601 602 603 604 605 606 607 608 609
/*
 * Do a quick check if any of the events requires a panic.
 * This decides if we keep the events around or clear them.
 */
static int mce_no_way_out(struct mce *m, char **msg)
{
	int i;

	for (i = 0; i < banks; i++) {
610
		m->status = mce_rdmsrl(MSR_IA32_MCx_STATUS(i));
611 612 613 614 615 616
		if (mce_severity(m, tolerant, msg) >= MCE_PANIC_SEVERITY)
			return 1;
	}
	return 0;
}

617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669
/*
 * Variable to establish order between CPUs while scanning.
 * Each CPU spins initially until executing is equal its number.
 */
static atomic_t mce_executing;

/*
 * Defines order of CPUs on entry. First CPU becomes Monarch.
 */
static atomic_t mce_callin;

/*
 * Check if a timeout waiting for other CPUs happened.
 */
static int mce_timed_out(u64 *t)
{
	/*
	 * The others already did panic for some reason.
	 * Bail out like in a timeout.
	 * rmb() to tell the compiler that system_state
	 * might have been modified by someone else.
	 */
	rmb();
	if (atomic_read(&mce_paniced))
		wait_for_panic();
	if (!monarch_timeout)
		goto out;
	if ((s64)*t < SPINUNIT) {
		/* CHECKME: Make panic default for 1 too? */
		if (tolerant < 1)
			mce_panic("Timeout synchronizing machine check over CPUs",
				  NULL, NULL);
		cpu_missing = 1;
		return 1;
	}
	*t -= SPINUNIT;
out:
	touch_nmi_watchdog();
	return 0;
}

/*
 * The Monarch's reign.  The Monarch is the CPU who entered
 * the machine check handler first. It waits for the others to
 * raise the exception too and then grades them. When any
 * error is fatal panic. Only then let the others continue.
 *
 * The other CPUs entering the MCE handler will be controlled by the
 * Monarch. They are called Subjects.
 *
 * This way we prevent any potential data corruption in a unrecoverable case
 * and also makes sure always all CPU's errors are examined.
 *
670
 * Also this detects the case of a machine check event coming from outer
671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710
 * space (not detected by any CPUs) In this case some external agent wants
 * us to shut down, so panic too.
 *
 * The other CPUs might still decide to panic if the handler happens
 * in a unrecoverable place, but in this case the system is in a semi-stable
 * state and won't corrupt anything by itself. It's ok to let the others
 * continue for a bit first.
 *
 * All the spin loops have timeouts; when a timeout happens a CPU
 * typically elects itself to be Monarch.
 */
static void mce_reign(void)
{
	int cpu;
	struct mce *m = NULL;
	int global_worst = 0;
	char *msg = NULL;
	char *nmsg = NULL;

	/*
	 * This CPU is the Monarch and the other CPUs have run
	 * through their handlers.
	 * Grade the severity of the errors of all the CPUs.
	 */
	for_each_possible_cpu(cpu) {
		int severity = mce_severity(&per_cpu(mces_seen, cpu), tolerant,
					    &nmsg);
		if (severity > global_worst) {
			msg = nmsg;
			global_worst = severity;
			m = &per_cpu(mces_seen, cpu);
		}
	}

	/*
	 * Cannot recover? Panic here then.
	 * This dumps all the mces in the log buffer and stops the
	 * other CPUs.
	 */
	if (m && global_worst >= MCE_PANIC_SEVERITY && tolerant < 3)
711
		mce_panic("Fatal Machine check", m, msg);
712 713 714 715 716 717 718 719 720 721 722

	/*
	 * For UC somewhere we let the CPU who detects it handle it.
	 * Also must let continue the others, otherwise the handling
	 * CPU could deadlock on a lock.
	 */

	/*
	 * No machine check event found. Must be some external
	 * source or one CPU is hung. Panic.
	 */
723
	if (global_worst <= MCE_KEEP_SEVERITY && tolerant < 3)
724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742
		mce_panic("Machine check from unknown source", NULL, NULL);

	/*
	 * Now clear all the mces_seen so that they don't reappear on
	 * the next mce.
	 */
	for_each_possible_cpu(cpu)
		memset(&per_cpu(mces_seen, cpu), 0, sizeof(struct mce));
}

static atomic_t global_nwo;

/*
 * Start of Monarch synchronization. This waits until all CPUs have
 * entered the exception handler and then determines if any of them
 * saw a fatal event that requires panic. Then it executes them
 * in the entry order.
 * TBD double check parallel CPU hotunplug
 */
H
Hidetoshi Seto 已提交
743
static int mce_start(int *no_way_out)
744
{
H
Hidetoshi Seto 已提交
745
	int order;
746 747 748
	int cpus = num_online_cpus();
	u64 timeout = (u64)monarch_timeout * NSEC_PER_USEC;

H
Hidetoshi Seto 已提交
749 750
	if (!timeout)
		return -1;
751

H
Hidetoshi Seto 已提交
752
	atomic_add(*no_way_out, &global_nwo);
753 754 755 756
	/*
	 * global_nwo should be updated before mce_callin
	 */
	smp_wmb();
757
	order = atomic_inc_return(&mce_callin);
758 759 760 761 762 763 764

	/*
	 * Wait for everyone.
	 */
	while (atomic_read(&mce_callin) != cpus) {
		if (mce_timed_out(&timeout)) {
			atomic_set(&global_nwo, 0);
H
Hidetoshi Seto 已提交
765
			return -1;
766 767 768 769
		}
		ndelay(SPINUNIT);
	}

770 771 772 773
	/*
	 * mce_callin should be read before global_nwo
	 */
	smp_rmb();
774

H
Hidetoshi Seto 已提交
775 776 777 778
	if (order == 1) {
		/*
		 * Monarch: Starts executing now, the others wait.
		 */
779
		atomic_set(&mce_executing, 1);
H
Hidetoshi Seto 已提交
780 781 782 783 784 785 786 787 788 789 790 791 792 793
	} else {
		/*
		 * Subject: Now start the scanning loop one by one in
		 * the original callin order.
		 * This way when there are any shared banks it will be
		 * only seen by one CPU before cleared, avoiding duplicates.
		 */
		while (atomic_read(&mce_executing) < order) {
			if (mce_timed_out(&timeout)) {
				atomic_set(&global_nwo, 0);
				return -1;
			}
			ndelay(SPINUNIT);
		}
794 795 796
	}

	/*
H
Hidetoshi Seto 已提交
797
	 * Cache the global no_way_out state.
798
	 */
H
Hidetoshi Seto 已提交
799 800 801
	*no_way_out = atomic_read(&global_nwo);

	return order;
802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870
}

/*
 * Synchronize between CPUs after main scanning loop.
 * This invokes the bulk of the Monarch processing.
 */
static int mce_end(int order)
{
	int ret = -1;
	u64 timeout = (u64)monarch_timeout * NSEC_PER_USEC;

	if (!timeout)
		goto reset;
	if (order < 0)
		goto reset;

	/*
	 * Allow others to run.
	 */
	atomic_inc(&mce_executing);

	if (order == 1) {
		/* CHECKME: Can this race with a parallel hotplug? */
		int cpus = num_online_cpus();

		/*
		 * Monarch: Wait for everyone to go through their scanning
		 * loops.
		 */
		while (atomic_read(&mce_executing) <= cpus) {
			if (mce_timed_out(&timeout))
				goto reset;
			ndelay(SPINUNIT);
		}

		mce_reign();
		barrier();
		ret = 0;
	} else {
		/*
		 * Subject: Wait for Monarch to finish.
		 */
		while (atomic_read(&mce_executing) != 0) {
			if (mce_timed_out(&timeout))
				goto reset;
			ndelay(SPINUNIT);
		}

		/*
		 * Don't reset anything. That's done by the Monarch.
		 */
		return 0;
	}

	/*
	 * Reset all global state.
	 */
reset:
	atomic_set(&global_nwo, 0);
	atomic_set(&mce_callin, 0);
	barrier();

	/*
	 * Let others run again.
	 */
	atomic_set(&mce_executing, 0);
	return ret;
}

871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887
/*
 * Check if the address reported by the CPU is in a format we can parse.
 * It would be possible to add code for most other cases, but all would
 * be somewhat complicated (e.g. segment offset would require an instruction
 * parser). So only support physical addresses upto page granuality for now.
 */
static int mce_usable_address(struct mce *m)
{
	if (!(m->status & MCI_STATUS_MISCV) || !(m->status & MCI_STATUS_ADDRV))
		return 0;
	if ((m->misc & 0x3f) > PAGE_SHIFT)
		return 0;
	if (((m->misc >> 6) & 7) != MCM_ADDR_PHYS)
		return 0;
	return 1;
}

888 889 890 891 892 893
static void mce_clear_state(unsigned long *toclear)
{
	int i;

	for (i = 0; i < banks; i++) {
		if (test_bit(i, toclear))
894
			mce_wrmsrl(MSR_IA32_MCx_STATUS(i), 0);
895 896 897
	}
}

898 899 900 901 902 903 904
/*
 * The actual machine check handler. This only handles real
 * exceptions when something got corrupted coming in through int 18.
 *
 * This is executed in NMI context not subject to normal locking rules. This
 * implies that most kernel services cannot be safely used. Don't even
 * think about putting a printk in there!
905 906 907 908
 *
 * On Intel systems this is entered on all CPUs in parallel through
 * MCE broadcast. However some CPUs might be broken beyond repair,
 * so be always careful when synchronizing with others.
L
Linus Torvalds 已提交
909
 */
I
Ingo Molnar 已提交
910
void do_machine_check(struct pt_regs *regs, long error_code)
L
Linus Torvalds 已提交
911
{
912
	struct mce m, *final;
L
Linus Torvalds 已提交
913
	int i;
914 915 916 917 918 919
	int worst = 0;
	int severity;
	/*
	 * Establish sequential order between the CPUs entering the machine
	 * check handler.
	 */
H
Hidetoshi Seto 已提交
920
	int order;
921 922 923 924 925 926 927 928 929 930
	/*
	 * If no_way_out gets set, there is no safe way to recover from this
	 * MCE.  If tolerant is cranked up, we'll try anyway.
	 */
	int no_way_out = 0;
	/*
	 * If kill_it gets set, there might be a way to recover from this
	 * error.
	 */
	int kill_it = 0;
931
	DECLARE_BITMAP(toclear, MAX_NR_BANKS);
932
	char *msg = "Unknown";
L
Linus Torvalds 已提交
933

934 935
	atomic_inc(&mce_entry);

936 937
	__get_cpu_var(mce_exception_count)++;

938
	if (notify_die(DIE_NMI, "machine check", regs, error_code,
939
			   18, SIGKILL) == NOTIFY_STOP)
940
		goto out;
941
	if (!banks)
942
		goto out;
L
Linus Torvalds 已提交
943

944 945
	mce_setup(&m);

946
	m.mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS);
947 948 949
	final = &__get_cpu_var(mces_seen);
	*final = m;

950 951
	no_way_out = mce_no_way_out(&m, &msg);

L
Linus Torvalds 已提交
952 953
	barrier();

A
Andi Kleen 已提交
954 955 956 957 958 959
	/*
	 * When no restart IP must always kill or panic.
	 */
	if (!(m.mcgstatus & MCG_STATUS_RIPV))
		kill_it = 1;

960 961 962 963 964
	/*
	 * Go through all the banks in exclusion of the other CPUs.
	 * This way we don't report duplicated events on shared banks
	 * because the first one to see it will clear it.
	 */
H
Hidetoshi Seto 已提交
965
	order = mce_start(&no_way_out);
L
Linus Torvalds 已提交
966
	for (i = 0; i < banks; i++) {
967
		__clear_bit(i, toclear);
968
		if (!mce_banks[i].ctl)
L
Linus Torvalds 已提交
969
			continue;
970 971

		m.misc = 0;
L
Linus Torvalds 已提交
972 973 974
		m.addr = 0;
		m.bank = i;

975
		m.status = mce_rdmsrl(MSR_IA32_MCx_STATUS(i));
L
Linus Torvalds 已提交
976 977 978
		if ((m.status & MCI_STATUS_VAL) == 0)
			continue;

979
		/*
A
Andi Kleen 已提交
980 981
		 * Non uncorrected or non signaled errors are handled by
		 * machine_check_poll. Leave them alone, unless this panics.
982
		 */
A
Andi Kleen 已提交
983 984
		if (!(m.status & (mce_ser ? MCI_STATUS_S : MCI_STATUS_UC)) &&
			!no_way_out)
985 986 987 988 989 990 991
			continue;

		/*
		 * Set taint even when machine check was not enabled.
		 */
		add_taint(TAINT_MACHINE_CHECK);

A
Andi Kleen 已提交
992
		severity = mce_severity(&m, tolerant, NULL);
993

A
Andi Kleen 已提交
994 995 996 997 998 999 1000 1001
		/*
		 * When machine check was for corrected handler don't touch,
		 * unless we're panicing.
		 */
		if (severity == MCE_KEEP_SEVERITY && !no_way_out)
			continue;
		__set_bit(i, toclear);
		if (severity == MCE_NO_SEVERITY) {
1002 1003 1004 1005 1006
			/*
			 * Machine check event was not enabled. Clear, but
			 * ignore.
			 */
			continue;
L
Linus Torvalds 已提交
1007 1008
		}

A
Andi Kleen 已提交
1009 1010 1011 1012 1013 1014
		/*
		 * Kill on action required.
		 */
		if (severity == MCE_AR_SEVERITY)
			kill_it = 1;

L
Linus Torvalds 已提交
1015
		if (m.status & MCI_STATUS_MISCV)
1016
			m.misc = mce_rdmsrl(MSR_IA32_MCx_MISC(i));
L
Linus Torvalds 已提交
1017
		if (m.status & MCI_STATUS_ADDRV)
1018
			m.addr = mce_rdmsrl(MSR_IA32_MCx_ADDR(i));
L
Linus Torvalds 已提交
1019

1020 1021 1022 1023 1024 1025 1026 1027 1028 1029
		/*
		 * Action optional error. Queue address for later processing.
		 * When the ring overflows we just ignore the AO error.
		 * RED-PEN add some logging mechanism when
		 * usable_address or mce_add_ring fails.
		 * RED-PEN don't ignore overflow for tolerant == 0
		 */
		if (severity == MCE_AO_SEVERITY && mce_usable_address(&m))
			mce_ring_add(m.addr >> PAGE_SHIFT);

1030
		mce_get_rip(&m, regs);
1031
		mce_log(&m);
L
Linus Torvalds 已提交
1032

1033 1034 1035
		if (severity > worst) {
			*final = m;
			worst = severity;
L
Linus Torvalds 已提交
1036 1037 1038
		}
	}

1039 1040 1041
	if (!no_way_out)
		mce_clear_state(toclear);

I
Ingo Molnar 已提交
1042
	/*
1043 1044
	 * Do most of the synchronization with other CPUs.
	 * When there's any problem use only local no_way_out state.
I
Ingo Molnar 已提交
1045
	 */
1046 1047
	if (mce_end(order) < 0)
		no_way_out = worst >= MCE_PANIC_SEVERITY;
1048 1049 1050

	/*
	 * If we have decided that we just CAN'T continue, and the user
I
Ingo Molnar 已提交
1051
	 * has not set tolerant to an insane level, give up and die.
1052 1053 1054
	 *
	 * This is mainly used in the case when the system doesn't
	 * support MCE broadcasting or it has been disabled.
1055 1056
	 */
	if (no_way_out && tolerant < 3)
1057
		mce_panic("Fatal machine check on current CPU", final, msg);
1058 1059 1060 1061 1062 1063 1064 1065

	/*
	 * If the error seems to be unrecoverable, something should be
	 * done.  Try to kill as little as possible.  If we can kill just
	 * one task, do that.  If the user has set the tolerance very
	 * high, don't try to do anything at all.
	 */

A
Andi Kleen 已提交
1066 1067
	if (kill_it && tolerant < 3)
		force_sig(SIGBUS, current);
L
Linus Torvalds 已提交
1068

1069 1070 1071
	/* notify userspace ASAP */
	set_thread_flag(TIF_MCE_NOTIFY);

1072 1073
	if (worst > 0)
		mce_report_event(regs);
1074
	mce_wrmsrl(MSR_IA32_MCG_STATUS, 0);
1075
out:
1076
	atomic_dec(&mce_entry);
1077
	sync_core();
L
Linus Torvalds 已提交
1078
}
1079
EXPORT_SYMBOL_GPL(do_machine_check);
L
Linus Torvalds 已提交
1080

1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110
/* dummy to break dependency. actual code is in mm/memory-failure.c */
void __attribute__((weak)) memory_failure(unsigned long pfn, int vector)
{
	printk(KERN_ERR "Action optional memory failure at %lx ignored\n", pfn);
}

/*
 * Called after mce notification in process context. This code
 * is allowed to sleep. Call the high level VM handler to process
 * any corrupted pages.
 * Assume that the work queue code only calls this one at a time
 * per CPU.
 * Note we don't disable preemption, so this code might run on the wrong
 * CPU. In this case the event is picked up by the scheduled work queue.
 * This is merely a fast path to expedite processing in some common
 * cases.
 */
void mce_notify_process(void)
{
	unsigned long pfn;
	mce_notify_irq();
	while (mce_ring_get(&pfn))
		memory_failure(pfn, MCE_VECTOR);
}

static void mce_process_work(struct work_struct *dummy)
{
	mce_notify_process();
}

1111 1112 1113
#ifdef CONFIG_X86_MCE_INTEL
/***
 * mce_log_therm_throt_event - Logs the thermal throttling event to mcelog
S
Simon Arlott 已提交
1114
 * @cpu: The CPU on which the event occurred.
1115 1116 1117 1118 1119 1120 1121 1122 1123 1124
 * @status: Event status information
 *
 * This function should be called by the thermal interrupt after the
 * event has been processed and the decision was made to log the event
 * further.
 *
 * The status parameter will be saved to the 'status' field of 'struct mce'
 * and historically has been the register value of the
 * MSR_IA32_THERMAL_STATUS (Intel) msr.
 */
1125
void mce_log_therm_throt_event(__u64 status)
1126 1127 1128
{
	struct mce m;

1129
	mce_setup(&m);
1130 1131 1132 1133 1134 1135
	m.bank = MCE_THERMAL_BANK;
	m.status = status;
	mce_log(&m);
}
#endif /* CONFIG_X86_MCE_INTEL */

L
Linus Torvalds 已提交
1136
/*
1137 1138 1139
 * Periodic polling timer for "silent" machine check errors.  If the
 * poller finds an MCE, poll 2x faster.  When the poller finds no more
 * errors, poll 2x slower (up to check_interval seconds).
L
Linus Torvalds 已提交
1140 1141
 */
static int check_interval = 5 * 60; /* 5 minutes */
I
Ingo Molnar 已提交
1142

1143
static DEFINE_PER_CPU(int, mce_next_interval); /* in jiffies */
1144
static DEFINE_PER_CPU(struct timer_list, mce_timer);
L
Linus Torvalds 已提交
1145

1146
static void mce_start_timer(unsigned long data)
L
Linus Torvalds 已提交
1147
{
1148
	struct timer_list *t = &per_cpu(mce_timer, data);
1149
	int *n;
1150 1151 1152

	WARN_ON(smp_processor_id() != data);

I
Ingo Molnar 已提交
1153
	if (mce_available(&current_cpu_data)) {
1154 1155
		machine_check_poll(MCP_TIMESTAMP,
				&__get_cpu_var(mce_poll_banks));
I
Ingo Molnar 已提交
1156
	}
L
Linus Torvalds 已提交
1157 1158

	/*
1159 1160
	 * Alert userspace if needed.  If we logged an MCE, reduce the
	 * polling interval, otherwise increase the polling interval.
L
Linus Torvalds 已提交
1161
	 */
1162
	n = &__get_cpu_var(mce_next_interval);
1163
	if (mce_notify_irq())
1164
		*n = max(*n/2, HZ/100);
1165
	else
1166
		*n = min(*n*2, (int)round_jiffies_relative(check_interval*HZ));
1167

1168
	t->expires = jiffies + *n;
1169
	add_timer_on(t, smp_processor_id());
1170 1171
}

1172 1173
static void mce_do_trigger(struct work_struct *work)
{
1174
	call_usermodehelper(mce_helper, mce_helper_argv, NULL, UMH_NO_WAIT);
1175 1176 1177 1178
}

static DECLARE_WORK(mce_trigger_work, mce_do_trigger);

1179
/*
1180 1181 1182
 * Notify the user(s) about new machine check events.
 * Can be called from interrupt context, but not from machine check/NMI
 * context.
1183
 */
1184
int mce_notify_irq(void)
1185
{
1186 1187 1188
	/* Not more than two messages every minute */
	static DEFINE_RATELIMIT_STATE(ratelimit, 60*HZ, 2);

1189
	clear_thread_flag(TIF_MCE_NOTIFY);
I
Ingo Molnar 已提交
1190

1191
	if (test_and_clear_bit(0, &mce_need_notify)) {
1192
		wake_up_interruptible(&mce_wait);
1193 1194 1195 1196 1197 1198

		/*
		 * There is no risk of missing notifications because
		 * work_pending is always cleared before the function is
		 * executed.
		 */
1199
		if (mce_helper[0] && !work_pending(&mce_trigger_work))
1200
			schedule_work(&mce_trigger_work);
1201

1202
		if (__ratelimit(&ratelimit))
1203
			printk(KERN_INFO "Machine check events logged\n");
1204 1205

		return 1;
L
Linus Torvalds 已提交
1206
	}
1207 1208
	return 0;
}
1209
EXPORT_SYMBOL_GPL(mce_notify_irq);
1210

H
Hidetoshi Seto 已提交
1211
static int __cpuinit __mcheck_cpu_mce_banks_init(void)
1212 1213 1214 1215 1216 1217 1218 1219
{
	int i;

	mce_banks = kzalloc(banks * sizeof(struct mce_bank), GFP_KERNEL);
	if (!mce_banks)
		return -ENOMEM;
	for (i = 0; i < banks; i++) {
		struct mce_bank *b = &mce_banks[i];
1220

1221 1222 1223 1224 1225 1226
		b->ctl = -1ULL;
		b->init = 1;
	}
	return 0;
}

1227
/*
L
Linus Torvalds 已提交
1228 1229
 * Initialize Machine Checks for a CPU.
 */
1230
static int __cpuinit __mcheck_cpu_cap_init(void)
L
Linus Torvalds 已提交
1231
{
1232
	unsigned b;
I
Ingo Molnar 已提交
1233
	u64 cap;
L
Linus Torvalds 已提交
1234 1235

	rdmsrl(MSR_IA32_MCG_CAP, cap);
1236 1237

	b = cap & MCG_BANKCNT_MASK;
1238 1239
	if (!banks)
		printk(KERN_INFO "mce: CPU supports %d MCE banks\n", b);
1240

1241 1242 1243 1244 1245 1246 1247 1248 1249 1250
	if (b > MAX_NR_BANKS) {
		printk(KERN_WARNING
		       "MCE: Using only %u machine check banks out of %u\n",
			MAX_NR_BANKS, b);
		b = MAX_NR_BANKS;
	}

	/* Don't support asymmetric configurations today */
	WARN_ON(banks != 0 && b != banks);
	banks = b;
1251
	if (!mce_banks) {
H
Hidetoshi Seto 已提交
1252
		int err = __mcheck_cpu_mce_banks_init();
1253

1254 1255
		if (err)
			return err;
L
Linus Torvalds 已提交
1256
	}
1257

1258
	/* Use accurate RIP reporting if available. */
1259
	if ((cap & MCG_EXT_P) && MCG_EXT_CNT(cap) >= 9)
1260
		rip_msr = MSR_IA32_MCG_EIP;
L
Linus Torvalds 已提交
1261

A
Andi Kleen 已提交
1262 1263 1264
	if (cap & MCG_SER_P)
		mce_ser = 1;

1265 1266 1267
	return 0;
}

1268
static void __mcheck_cpu_init_generic(void)
1269
{
I
Ingo Molnar 已提交
1270
	mce_banks_t all_banks;
1271 1272 1273
	u64 cap;
	int i;

1274 1275 1276
	/*
	 * Log the machine checks left over from the previous reset.
	 */
1277
	bitmap_fill(all_banks, MAX_NR_BANKS);
A
Andi Kleen 已提交
1278
	machine_check_poll(MCP_UC|(!mce_bootlog ? MCP_DONTLOG : 0), &all_banks);
L
Linus Torvalds 已提交
1279 1280 1281

	set_in_cr4(X86_CR4_MCE);

1282
	rdmsrl(MSR_IA32_MCG_CAP, cap);
L
Linus Torvalds 已提交
1283 1284 1285 1286
	if (cap & MCG_CTL_P)
		wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff);

	for (i = 0; i < banks; i++) {
1287
		struct mce_bank *b = &mce_banks[i];
1288

1289
		if (!b->init)
1290
			continue;
1291 1292
		wrmsrl(MSR_IA32_MCx_CTL(i), b->ctl);
		wrmsrl(MSR_IA32_MCx_STATUS(i), 0);
1293
	}
L
Linus Torvalds 已提交
1294 1295 1296
}

/* Add per CPU specific workarounds here */
1297
static int __cpuinit __mcheck_cpu_apply_quirks(struct cpuinfo_x86 *c)
1298
{
1299 1300 1301 1302 1303
	if (c->x86_vendor == X86_VENDOR_UNKNOWN) {
		pr_info("MCE: unknown CPU type - not enabling MCE support.\n");
		return -EOPNOTSUPP;
	}

L
Linus Torvalds 已提交
1304
	/* This should be disabled by the BIOS, but isn't always */
1305
	if (c->x86_vendor == X86_VENDOR_AMD) {
I
Ingo Molnar 已提交
1306 1307 1308 1309 1310 1311
		if (c->x86 == 15 && banks > 4) {
			/*
			 * disable GART TBL walk error reporting, which
			 * trips off incorrectly with the IOMMU & 3ware
			 * & Cerberus:
			 */
1312
			clear_bit(10, (unsigned long *)&mce_banks[4].ctl);
I
Ingo Molnar 已提交
1313 1314 1315 1316 1317 1318
		}
		if (c->x86 <= 17 && mce_bootlog < 0) {
			/*
			 * Lots of broken BIOS around that don't clear them
			 * by default and leave crap in there. Don't log:
			 */
1319
			mce_bootlog = 0;
I
Ingo Molnar 已提交
1320
		}
1321 1322 1323 1324
		/*
		 * Various K7s with broken bank 0 around. Always disable
		 * by default.
		 */
1325
		 if (c->x86 == 6 && banks > 0)
1326
			mce_banks[0].ctl = 0;
L
Linus Torvalds 已提交
1327
	}
1328

1329 1330 1331 1332 1333 1334 1335 1336 1337 1338
	if (c->x86_vendor == X86_VENDOR_INTEL) {
		/*
		 * SDM documents that on family 6 bank 0 should not be written
		 * because it aliases to another special BIOS controlled
		 * register.
		 * But it's not aliased anymore on model 0x1a+
		 * Don't ignore bank 0 completely because there could be a
		 * valid event later, merely don't write CTL0.
		 */

1339 1340
		if (c->x86 == 6 && c->x86_model < 0x1A && banks > 0)
			mce_banks[0].init = 0;
1341 1342 1343 1344 1345 1346 1347 1348

		/*
		 * All newer Intel systems support MCE broadcasting. Enable
		 * synchronization with a one second timeout.
		 */
		if ((c->x86 > 6 || (c->x86 == 6 && c->x86_model >= 0xe)) &&
			monarch_timeout < 0)
			monarch_timeout = USEC_PER_SEC;
1349

1350 1351 1352 1353 1354
		/*
		 * There are also broken BIOSes on some Pentium M and
		 * earlier systems:
		 */
		if (c->x86 == 6 && c->x86_model <= 13 && mce_bootlog < 0)
1355
			mce_bootlog = 0;
1356
	}
1357 1358
	if (monarch_timeout < 0)
		monarch_timeout = 0;
1359 1360
	if (mce_bootlog != 0)
		mce_panic_timeout = 30;
1361 1362

	return 0;
1363
}
L
Linus Torvalds 已提交
1364

1365
static void __cpuinit __mcheck_cpu_ancient_init(struct cpuinfo_x86 *c)
1366 1367 1368 1369 1370
{
	if (c->x86 != 5)
		return;
	switch (c->x86_vendor) {
	case X86_VENDOR_INTEL:
1371
		intel_p5_mcheck_init(c);
1372 1373 1374 1375 1376 1377 1378
		break;
	case X86_VENDOR_CENTAUR:
		winchip_mcheck_init(c);
		break;
	}
}

1379
static void __mcheck_cpu_init_vendor(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1380 1381 1382 1383 1384
{
	switch (c->x86_vendor) {
	case X86_VENDOR_INTEL:
		mce_intel_feature_init(c);
		break;
1385 1386 1387
	case X86_VENDOR_AMD:
		mce_amd_feature_init(c);
		break;
L
Linus Torvalds 已提交
1388 1389 1390 1391 1392
	default:
		break;
	}
}

1393
static void __mcheck_cpu_init_timer(void)
1394 1395
{
	struct timer_list *t = &__get_cpu_var(mce_timer);
1396
	int *n = &__get_cpu_var(mce_next_interval);
1397

1398 1399
	setup_timer(t, mce_start_timer, smp_processor_id());

1400 1401 1402
	if (mce_ignore_ce)
		return;

1403 1404
	*n = check_interval * HZ;
	if (!*n)
1405
		return;
1406
	t->expires = round_jiffies(jiffies + *n);
1407
	add_timer_on(t, smp_processor_id());
1408 1409
}

A
Andi Kleen 已提交
1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420
/* Handle unconfigured int18 (should never happen) */
static void unexpected_machine_check(struct pt_regs *regs, long error_code)
{
	printk(KERN_ERR "CPU#%d: Unexpected int18 (Machine Check).\n",
	       smp_processor_id());
}

/* Call the installed machine check handler for this CPU setup. */
void (*machine_check_vector)(struct pt_regs *, long error_code) =
						unexpected_machine_check;

1421
/*
L
Linus Torvalds 已提交
1422
 * Called for each booted CPU to set up machine checks.
I
Ingo Molnar 已提交
1423
 * Must be called with preempt off:
L
Linus Torvalds 已提交
1424
 */
1425
void __cpuinit mcheck_cpu_init(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1426
{
1427 1428 1429
	if (mce_disabled)
		return;

1430
	__mcheck_cpu_ancient_init(c);
1431

1432
	if (!mce_available(c))
L
Linus Torvalds 已提交
1433 1434
		return;

1435
	if (__mcheck_cpu_cap_init() < 0 || __mcheck_cpu_apply_quirks(c) < 0) {
1436
		mce_disabled = 1;
1437 1438 1439
		return;
	}

1440 1441
	machine_check_vector = do_machine_check;

1442 1443 1444
	__mcheck_cpu_init_generic();
	__mcheck_cpu_init_vendor(c);
	__mcheck_cpu_init_timer();
1445
	INIT_WORK(&__get_cpu_var(mce_work), mce_process_work);
1446

L
Linus Torvalds 已提交
1447 1448 1449 1450 1451 1452
}

/*
 * Character device to read and clear the MCE log.
 */

T
Tim Hockin 已提交
1453
static DEFINE_SPINLOCK(mce_state_lock);
I
Ingo Molnar 已提交
1454 1455
static int		open_count;		/* #times opened */
static int		open_exclu;		/* already open exclusive? */
T
Tim Hockin 已提交
1456 1457 1458 1459 1460 1461 1462

static int mce_open(struct inode *inode, struct file *file)
{
	spin_lock(&mce_state_lock);

	if (open_exclu || (open_count && (file->f_flags & O_EXCL))) {
		spin_unlock(&mce_state_lock);
I
Ingo Molnar 已提交
1463

T
Tim Hockin 已提交
1464 1465 1466 1467 1468 1469 1470 1471 1472
		return -EBUSY;
	}

	if (file->f_flags & O_EXCL)
		open_exclu = 1;
	open_count++;

	spin_unlock(&mce_state_lock);

1473
	return nonseekable_open(inode, file);
T
Tim Hockin 已提交
1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487
}

static int mce_release(struct inode *inode, struct file *file)
{
	spin_lock(&mce_state_lock);

	open_count--;
	open_exclu = 0;

	spin_unlock(&mce_state_lock);

	return 0;
}

1488 1489
static void collect_tscs(void *data)
{
L
Linus Torvalds 已提交
1490
	unsigned long *cpu_tsc = (unsigned long *)data;
1491

L
Linus Torvalds 已提交
1492
	rdtscll(cpu_tsc[smp_processor_id()]);
1493
}
L
Linus Torvalds 已提交
1494

1495 1496
static ssize_t mce_read(struct file *filp, char __user *ubuf, size_t usize,
			loff_t *off)
L
Linus Torvalds 已提交
1497
{
I
Ingo Molnar 已提交
1498
	char __user *buf = ubuf;
1499
	unsigned long *cpu_tsc;
1500
	unsigned prev, next;
L
Linus Torvalds 已提交
1501 1502
	int i, err;

1503
	cpu_tsc = kmalloc(nr_cpu_ids * sizeof(long), GFP_KERNEL);
1504 1505 1506
	if (!cpu_tsc)
		return -ENOMEM;

1507
	mutex_lock(&mce_read_mutex);
1508
	next = rcu_dereference_check_mce(mcelog.next);
L
Linus Torvalds 已提交
1509 1510

	/* Only supports full reads right now */
1511
	if (*off != 0 || usize < MCE_LOG_LEN*sizeof(struct mce)) {
1512
		mutex_unlock(&mce_read_mutex);
1513
		kfree(cpu_tsc);
I
Ingo Molnar 已提交
1514

L
Linus Torvalds 已提交
1515 1516 1517 1518
		return -EINVAL;
	}

	err = 0;
1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530
	prev = 0;
	do {
		for (i = prev; i < next; i++) {
			unsigned long start = jiffies;

			while (!mcelog.entry[i].finished) {
				if (time_after_eq(jiffies, start + 2)) {
					memset(mcelog.entry + i, 0,
					       sizeof(struct mce));
					goto timeout;
				}
				cpu_relax();
1531
			}
1532 1533 1534 1535 1536 1537
			smp_rmb();
			err |= copy_to_user(buf, mcelog.entry + i,
					    sizeof(struct mce));
			buf += sizeof(struct mce);
timeout:
			;
1538
		}
L
Linus Torvalds 已提交
1539

1540 1541 1542 1543 1544
		memset(mcelog.entry + prev, 0,
		       (next - prev) * sizeof(struct mce));
		prev = next;
		next = cmpxchg(&mcelog.next, prev, 0);
	} while (next != prev);
L
Linus Torvalds 已提交
1545

1546
	synchronize_sched();
L
Linus Torvalds 已提交
1547

1548 1549 1550 1551
	/*
	 * Collect entries that were still getting written before the
	 * synchronize.
	 */
1552
	on_each_cpu(collect_tscs, cpu_tsc, 1);
I
Ingo Molnar 已提交
1553

1554 1555 1556 1557 1558
	for (i = next; i < MCE_LOG_LEN; i++) {
		if (mcelog.entry[i].finished &&
		    mcelog.entry[i].tsc < cpu_tsc[mcelog.entry[i].cpu]) {
			err |= copy_to_user(buf, mcelog.entry+i,
					    sizeof(struct mce));
L
Linus Torvalds 已提交
1559 1560 1561 1562
			smp_rmb();
			buf += sizeof(struct mce);
			memset(&mcelog.entry[i], 0, sizeof(struct mce));
		}
1563
	}
1564
	mutex_unlock(&mce_read_mutex);
1565
	kfree(cpu_tsc);
I
Ingo Molnar 已提交
1566

1567
	return err ? -EFAULT : buf - ubuf;
L
Linus Torvalds 已提交
1568 1569
}

1570 1571 1572
static unsigned int mce_poll(struct file *file, poll_table *wait)
{
	poll_wait(file, &mce_wait, wait);
1573
	if (rcu_dereference_check_mce(mcelog.next))
1574 1575 1576 1577
		return POLLIN | POLLRDNORM;
	return 0;
}

1578
static long mce_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
L
Linus Torvalds 已提交
1579 1580
{
	int __user *p = (int __user *)arg;
1581

L
Linus Torvalds 已提交
1582
	if (!capable(CAP_SYS_ADMIN))
1583
		return -EPERM;
I
Ingo Molnar 已提交
1584

L
Linus Torvalds 已提交
1585
	switch (cmd) {
1586
	case MCE_GET_RECORD_LEN:
L
Linus Torvalds 已提交
1587 1588
		return put_user(sizeof(struct mce), p);
	case MCE_GET_LOG_LEN:
1589
		return put_user(MCE_LOG_LEN, p);
L
Linus Torvalds 已提交
1590 1591
	case MCE_GETCLEAR_FLAGS: {
		unsigned flags;
1592 1593

		do {
L
Linus Torvalds 已提交
1594
			flags = mcelog.flags;
1595
		} while (cmpxchg(&mcelog.flags, flags, 0) != flags);
I
Ingo Molnar 已提交
1596

1597
		return put_user(flags, p);
L
Linus Torvalds 已提交
1598 1599
	}
	default:
1600 1601
		return -ENOTTY;
	}
L
Linus Torvalds 已提交
1602 1603
}

1604
/* Modified in mce-inject.c, so not static or const */
1605
struct file_operations mce_chrdev_ops = {
I
Ingo Molnar 已提交
1606 1607 1608 1609 1610
	.open			= mce_open,
	.release		= mce_release,
	.read			= mce_read,
	.poll			= mce_poll,
	.unlocked_ioctl		= mce_ioctl,
L
Linus Torvalds 已提交
1611
};
1612
EXPORT_SYMBOL_GPL(mce_chrdev_ops);
L
Linus Torvalds 已提交
1613 1614 1615 1616 1617 1618 1619

static struct miscdevice mce_log_device = {
	MISC_MCELOG_MINOR,
	"mcelog",
	&mce_chrdev_ops,
};

H
Hidetoshi Seto 已提交
1620
/*
1621 1622 1623 1624
 * mce=off Disables machine check
 * mce=no_cmci Disables CMCI
 * mce=dont_log_ce Clears corrected events silently, no log created for CEs.
 * mce=ignore_ce Disables polling and CMCI, corrected events are not cleared.
1625 1626 1627
 * mce=TOLERANCELEVEL[,monarchtimeout] (number, see above)
 *	monarchtimeout is how long to wait for other CPUs on machine
 *	check, or 0 to not wait
H
Hidetoshi Seto 已提交
1628 1629 1630
 * mce=bootlog Log MCEs from before booting. Disabled by default on AMD.
 * mce=nobootlog Don't log MCEs from before booting.
 */
L
Linus Torvalds 已提交
1631 1632
static int __init mcheck_enable(char *str)
{
1633
	if (*str == 0) {
1634
		enable_p5_mce();
1635 1636
		return 1;
	}
1637 1638
	if (*str == '=')
		str++;
L
Linus Torvalds 已提交
1639
	if (!strcmp(str, "off"))
1640
		mce_disabled = 1;
1641 1642 1643 1644 1645 1646
	else if (!strcmp(str, "no_cmci"))
		mce_cmci_disabled = 1;
	else if (!strcmp(str, "dont_log_ce"))
		mce_dont_log_ce = 1;
	else if (!strcmp(str, "ignore_ce"))
		mce_ignore_ce = 1;
H
Hidetoshi Seto 已提交
1647 1648
	else if (!strcmp(str, "bootlog") || !strcmp(str, "nobootlog"))
		mce_bootlog = (str[0] == 'b');
1649
	else if (isdigit(str[0])) {
1650
		get_option(&str, &tolerant);
1651 1652 1653 1654 1655
		if (*str == ',') {
			++str;
			get_option(&str, &monarch_timeout);
		}
	} else {
1656
		printk(KERN_INFO "mce argument %s ignored. Please use /sys\n",
H
Hidetoshi Seto 已提交
1657 1658 1659
		       str);
		return 0;
	}
1660
	return 1;
L
Linus Torvalds 已提交
1661
}
1662
__setup("mce", mcheck_enable);
L
Linus Torvalds 已提交
1663

1664
int __init mcheck_init(void)
1665 1666 1667
{
	atomic_notifier_chain_register(&x86_mce_decoder_chain, &mce_dec_nb);

1668 1669
	mcheck_intel_therm_init();

1670 1671 1672
	return 0;
}

1673
/*
L
Linus Torvalds 已提交
1674
 * Sysfs support
1675
 */
L
Linus Torvalds 已提交
1676

1677 1678 1679 1680
/*
 * Disable machine checks on suspend and shutdown. We can't really handle
 * them later.
 */
1681
static int mce_disable_error_reporting(void)
1682 1683 1684
{
	int i;

1685
	for (i = 0; i < banks; i++) {
1686
		struct mce_bank *b = &mce_banks[i];
1687

1688
		if (b->init)
1689
			wrmsrl(MSR_IA32_MCx_CTL(i), 0);
1690
	}
1691 1692 1693 1694 1695
	return 0;
}

static int mce_suspend(struct sys_device *dev, pm_message_t state)
{
1696
	return mce_disable_error_reporting();
1697 1698 1699 1700
}

static int mce_shutdown(struct sys_device *dev)
{
1701
	return mce_disable_error_reporting();
1702 1703
}

I
Ingo Molnar 已提交
1704 1705 1706 1707 1708
/*
 * On resume clear all MCE state. Don't want to see leftovers from the BIOS.
 * Only one CPU is active at this time, the others get re-added later using
 * CPU hotplug:
 */
L
Linus Torvalds 已提交
1709 1710
static int mce_resume(struct sys_device *dev)
{
1711 1712
	__mcheck_cpu_init_generic();
	__mcheck_cpu_init_vendor(&current_cpu_data);
I
Ingo Molnar 已提交
1713

L
Linus Torvalds 已提交
1714 1715 1716
	return 0;
}

1717 1718 1719
static void mce_cpu_restart(void *data)
{
	del_timer_sync(&__get_cpu_var(mce_timer));
1720 1721
	if (!mce_available(&current_cpu_data))
		return;
1722 1723
	__mcheck_cpu_init_generic();
	__mcheck_cpu_init_timer();
1724 1725
}

L
Linus Torvalds 已提交
1726
/* Reinit MCEs after user configuration changes */
1727 1728
static void mce_restart(void)
{
1729
	on_each_cpu(mce_cpu_restart, NULL, 1);
L
Linus Torvalds 已提交
1730 1731
}

1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748
/* Toggle features for corrected errors */
static void mce_disable_ce(void *all)
{
	if (!mce_available(&current_cpu_data))
		return;
	if (all)
		del_timer_sync(&__get_cpu_var(mce_timer));
	cmci_clear();
}

static void mce_enable_ce(void *all)
{
	if (!mce_available(&current_cpu_data))
		return;
	cmci_reenable();
	cmci_recheck();
	if (all)
1749
		__mcheck_cpu_init_timer();
1750 1751
}

L
Linus Torvalds 已提交
1752
static struct sysdev_class mce_sysclass = {
I
Ingo Molnar 已提交
1753 1754 1755 1756
	.suspend	= mce_suspend,
	.shutdown	= mce_shutdown,
	.resume		= mce_resume,
	.name		= "machinecheck",
L
Linus Torvalds 已提交
1757 1758
};

I
Ingo Molnar 已提交
1759
DEFINE_PER_CPU(struct sys_device, mce_dev);
I
Ingo Molnar 已提交
1760 1761 1762

__cpuinitdata
void (*threshold_cpu_callback)(unsigned long action, unsigned int cpu);
L
Linus Torvalds 已提交
1763

1764 1765 1766 1767
static inline struct mce_bank *attr_to_bank(struct sysdev_attribute *attr)
{
	return container_of(attr, struct mce_bank, attr);
}
1768 1769 1770 1771

static ssize_t show_bank(struct sys_device *s, struct sysdev_attribute *attr,
			 char *buf)
{
1772
	return sprintf(buf, "%llx\n", attr_to_bank(attr)->ctl);
1773 1774 1775
}

static ssize_t set_bank(struct sys_device *s, struct sysdev_attribute *attr,
H
Hidetoshi Seto 已提交
1776
			const char *buf, size_t size)
1777
{
H
Hidetoshi Seto 已提交
1778
	u64 new;
I
Ingo Molnar 已提交
1779

H
Hidetoshi Seto 已提交
1780
	if (strict_strtoull(buf, 0, &new) < 0)
1781
		return -EINVAL;
I
Ingo Molnar 已提交
1782

1783
	attr_to_bank(attr)->ctl = new;
1784
	mce_restart();
I
Ingo Molnar 已提交
1785

H
Hidetoshi Seto 已提交
1786
	return size;
1787
}
1788

I
Ingo Molnar 已提交
1789 1790
static ssize_t
show_trigger(struct sys_device *s, struct sysdev_attribute *attr, char *buf)
1791
{
1792
	strcpy(buf, mce_helper);
1793
	strcat(buf, "\n");
1794
	return strlen(mce_helper) + 1;
1795 1796
}

1797
static ssize_t set_trigger(struct sys_device *s, struct sysdev_attribute *attr,
I
Ingo Molnar 已提交
1798
				const char *buf, size_t siz)
1799 1800
{
	char *p;
I
Ingo Molnar 已提交
1801

1802 1803 1804
	strncpy(mce_helper, buf, sizeof(mce_helper));
	mce_helper[sizeof(mce_helper)-1] = 0;
	p = strchr(mce_helper, '\n');
I
Ingo Molnar 已提交
1805

1806
	if (p)
I
Ingo Molnar 已提交
1807 1808
		*p = 0;

1809
	return strlen(mce_helper) + !!p;
1810 1811
}

1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857
static ssize_t set_ignore_ce(struct sys_device *s,
			     struct sysdev_attribute *attr,
			     const char *buf, size_t size)
{
	u64 new;

	if (strict_strtoull(buf, 0, &new) < 0)
		return -EINVAL;

	if (mce_ignore_ce ^ !!new) {
		if (new) {
			/* disable ce features */
			on_each_cpu(mce_disable_ce, (void *)1, 1);
			mce_ignore_ce = 1;
		} else {
			/* enable ce features */
			mce_ignore_ce = 0;
			on_each_cpu(mce_enable_ce, (void *)1, 1);
		}
	}
	return size;
}

static ssize_t set_cmci_disabled(struct sys_device *s,
				 struct sysdev_attribute *attr,
				 const char *buf, size_t size)
{
	u64 new;

	if (strict_strtoull(buf, 0, &new) < 0)
		return -EINVAL;

	if (mce_cmci_disabled ^ !!new) {
		if (new) {
			/* disable cmci */
			on_each_cpu(mce_disable_ce, NULL, 1);
			mce_cmci_disabled = 1;
		} else {
			/* enable cmci */
			mce_cmci_disabled = 0;
			on_each_cpu(mce_enable_ce, NULL, 1);
		}
	}
	return size;
}

1858 1859 1860 1861 1862 1863 1864 1865 1866
static ssize_t store_int_with_restart(struct sys_device *s,
				      struct sysdev_attribute *attr,
				      const char *buf, size_t size)
{
	ssize_t ret = sysdev_store_int(s, attr, buf, size);
	mce_restart();
	return ret;
}

1867
static SYSDEV_ATTR(trigger, 0644, show_trigger, set_trigger);
1868
static SYSDEV_INT_ATTR(tolerant, 0644, tolerant);
1869
static SYSDEV_INT_ATTR(monarch_timeout, 0644, monarch_timeout);
1870
static SYSDEV_INT_ATTR(dont_log_ce, 0644, mce_dont_log_ce);
I
Ingo Molnar 已提交
1871

1872 1873 1874 1875 1876
static struct sysdev_ext_attribute attr_check_interval = {
	_SYSDEV_ATTR(check_interval, 0644, sysdev_show_int,
		     store_int_with_restart),
	&check_interval
};
I
Ingo Molnar 已提交
1877

1878 1879 1880 1881 1882 1883
static struct sysdev_ext_attribute attr_ignore_ce = {
	_SYSDEV_ATTR(ignore_ce, 0644, sysdev_show_int, set_ignore_ce),
	&mce_ignore_ce
};

static struct sysdev_ext_attribute attr_cmci_disabled = {
Y
Yinghai Lu 已提交
1884
	_SYSDEV_ATTR(cmci_disabled, 0644, sysdev_show_int, set_cmci_disabled),
1885 1886 1887
	&mce_cmci_disabled
};

I
Ingo Molnar 已提交
1888
static struct sysdev_attribute *mce_attrs[] = {
1889 1890 1891
	&attr_tolerant.attr,
	&attr_check_interval.attr,
	&attr_trigger,
1892
	&attr_monarch_timeout.attr,
1893 1894 1895
	&attr_dont_log_ce.attr,
	&attr_ignore_ce.attr,
	&attr_cmci_disabled.attr,
1896 1897
	NULL
};
L
Linus Torvalds 已提交
1898

I
Ingo Molnar 已提交
1899
static cpumask_var_t mce_dev_initialized;
1900

I
Ingo Molnar 已提交
1901
/* Per cpu sysdev init. All of the cpus still share the same ctrl bank: */
1902
static __cpuinit int mce_create_device(unsigned int cpu)
L
Linus Torvalds 已提交
1903 1904
{
	int err;
1905
	int i, j;
1906

A
Andreas Herrmann 已提交
1907
	if (!mce_available(&boot_cpu_data))
1908 1909
		return -EIO;

I
Ingo Molnar 已提交
1910 1911 1912
	memset(&per_cpu(mce_dev, cpu).kobj, 0, sizeof(struct kobject));
	per_cpu(mce_dev, cpu).id	= cpu;
	per_cpu(mce_dev, cpu).cls	= &mce_sysclass;
1913

I
Ingo Molnar 已提交
1914
	err = sysdev_register(&per_cpu(mce_dev, cpu));
1915 1916 1917
	if (err)
		return err;

I
Ingo Molnar 已提交
1918 1919
	for (i = 0; mce_attrs[i]; i++) {
		err = sysdev_create_file(&per_cpu(mce_dev, cpu), mce_attrs[i]);
1920 1921 1922
		if (err)
			goto error;
	}
1923
	for (j = 0; j < banks; j++) {
I
Ingo Molnar 已提交
1924
		err = sysdev_create_file(&per_cpu(mce_dev, cpu),
1925
					&mce_banks[j].attr);
1926 1927 1928
		if (err)
			goto error2;
	}
I
Ingo Molnar 已提交
1929
	cpumask_set_cpu(cpu, mce_dev_initialized);
1930

1931
	return 0;
1932
error2:
1933
	while (--j >= 0)
1934
		sysdev_remove_file(&per_cpu(mce_dev, cpu), &mce_banks[j].attr);
1935
error:
I
Ingo Molnar 已提交
1936
	while (--i >= 0)
1937
		sysdev_remove_file(&per_cpu(mce_dev, cpu), mce_attrs[i]);
I
Ingo Molnar 已提交
1938 1939

	sysdev_unregister(&per_cpu(mce_dev, cpu));
1940

1941 1942 1943
	return err;
}

1944
static __cpuinit void mce_remove_device(unsigned int cpu)
1945
{
1946 1947
	int i;

I
Ingo Molnar 已提交
1948
	if (!cpumask_test_cpu(cpu, mce_dev_initialized))
1949 1950
		return;

I
Ingo Molnar 已提交
1951 1952 1953
	for (i = 0; mce_attrs[i]; i++)
		sysdev_remove_file(&per_cpu(mce_dev, cpu), mce_attrs[i]);

1954
	for (i = 0; i < banks; i++)
1955
		sysdev_remove_file(&per_cpu(mce_dev, cpu), &mce_banks[i].attr);
I
Ingo Molnar 已提交
1956 1957 1958

	sysdev_unregister(&per_cpu(mce_dev, cpu));
	cpumask_clear_cpu(cpu, mce_dev_initialized);
1959 1960
}

1961
/* Make sure there are no machine checks on offlined CPUs. */
1962
static void __cpuinit mce_disable_cpu(void *h)
1963
{
A
Andi Kleen 已提交
1964
	unsigned long action = *(unsigned long *)h;
I
Ingo Molnar 已提交
1965
	int i;
1966 1967 1968

	if (!mce_available(&current_cpu_data))
		return;
1969

A
Andi Kleen 已提交
1970 1971
	if (!(action & CPU_TASKS_FROZEN))
		cmci_clear();
1972
	for (i = 0; i < banks; i++) {
1973
		struct mce_bank *b = &mce_banks[i];
1974

1975
		if (b->init)
1976
			wrmsrl(MSR_IA32_MCx_CTL(i), 0);
1977
	}
1978 1979
}

1980
static void __cpuinit mce_reenable_cpu(void *h)
1981
{
A
Andi Kleen 已提交
1982
	unsigned long action = *(unsigned long *)h;
I
Ingo Molnar 已提交
1983
	int i;
1984 1985 1986

	if (!mce_available(&current_cpu_data))
		return;
I
Ingo Molnar 已提交
1987

A
Andi Kleen 已提交
1988 1989
	if (!(action & CPU_TASKS_FROZEN))
		cmci_reenable();
1990
	for (i = 0; i < banks; i++) {
1991
		struct mce_bank *b = &mce_banks[i];
1992

1993
		if (b->init)
1994
			wrmsrl(MSR_IA32_MCx_CTL(i), b->ctl);
1995
	}
1996 1997
}

1998
/* Get notified when a cpu comes on/off. Be hotplug friendly. */
I
Ingo Molnar 已提交
1999 2000
static int __cpuinit
mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
2001 2002
{
	unsigned int cpu = (unsigned long)hcpu;
2003
	struct timer_list *t = &per_cpu(mce_timer, cpu);
2004 2005

	switch (action) {
2006 2007 2008
	case CPU_ONLINE:
	case CPU_ONLINE_FROZEN:
		mce_create_device(cpu);
2009 2010
		if (threshold_cpu_callback)
			threshold_cpu_callback(action, cpu);
2011 2012
		break;
	case CPU_DEAD:
2013
	case CPU_DEAD_FROZEN:
2014 2015
		if (threshold_cpu_callback)
			threshold_cpu_callback(action, cpu);
2016 2017
		mce_remove_device(cpu);
		break;
2018 2019 2020
	case CPU_DOWN_PREPARE:
	case CPU_DOWN_PREPARE_FROZEN:
		del_timer_sync(t);
A
Andi Kleen 已提交
2021
		smp_call_function_single(cpu, mce_disable_cpu, &action, 1);
2022 2023 2024
		break;
	case CPU_DOWN_FAILED:
	case CPU_DOWN_FAILED_FROZEN:
2025 2026
		if (!mce_ignore_ce && check_interval) {
			t->expires = round_jiffies(jiffies +
2027
					   __get_cpu_var(mce_next_interval));
2028 2029
			add_timer_on(t, cpu);
		}
A
Andi Kleen 已提交
2030 2031 2032 2033 2034
		smp_call_function_single(cpu, mce_reenable_cpu, &action, 1);
		break;
	case CPU_POST_DEAD:
		/* intentionally ignoring frozen here */
		cmci_rediscover(cpu);
2035
		break;
2036
	}
2037
	return NOTIFY_OK;
2038 2039
}

2040
static struct notifier_block mce_cpu_notifier __cpuinitdata = {
2041 2042 2043
	.notifier_call = mce_cpu_callback,
};

2044
static __init void mce_init_banks(void)
2045 2046 2047 2048
{
	int i;

	for (i = 0; i < banks; i++) {
2049 2050
		struct mce_bank *b = &mce_banks[i];
		struct sysdev_attribute *a = &b->attr;
I
Ingo Molnar 已提交
2051

2052
		sysfs_attr_init(&a->attr);
2053 2054
		a->attr.name	= b->attrname;
		snprintf(b->attrname, ATTR_LEN, "bank%d", i);
I
Ingo Molnar 已提交
2055 2056 2057 2058

		a->attr.mode	= 0644;
		a->show		= show_bank;
		a->store	= set_bank;
2059 2060 2061
	}
}

2062
static __init int mcheck_init_device(void)
2063 2064 2065 2066
{
	int err;
	int i = 0;

L
Linus Torvalds 已提交
2067 2068
	if (!mce_available(&boot_cpu_data))
		return -EIO;
2069

2070
	zalloc_cpumask_var(&mce_dev_initialized, GFP_KERNEL);
2071

2072
	mce_init_banks();
2073

L
Linus Torvalds 已提交
2074
	err = sysdev_class_register(&mce_sysclass);
2075 2076
	if (err)
		return err;
2077 2078

	for_each_online_cpu(i) {
2079 2080 2081
		err = mce_create_device(i);
		if (err)
			return err;
2082 2083
	}

2084
	register_hotcpu_notifier(&mce_cpu_notifier);
L
Linus Torvalds 已提交
2085
	misc_register(&mce_log_device);
I
Ingo Molnar 已提交
2086

L
Linus Torvalds 已提交
2087 2088
	return err;
}
2089

2090
device_initcall(mcheck_init_device);
I
Ingo Molnar 已提交
2091

2092 2093 2094 2095 2096 2097 2098 2099 2100
/*
 * Old style boot options parsing. Only for compatibility.
 */
static int __init mcheck_disable(char *str)
{
	mce_disabled = 1;
	return 1;
}
__setup("nomce", mcheck_disable);
I
Ingo Molnar 已提交
2101

2102 2103
#ifdef CONFIG_DEBUG_FS
struct dentry *mce_get_debugfs_dir(void)
I
Ingo Molnar 已提交
2104
{
2105
	static struct dentry *dmce;
I
Ingo Molnar 已提交
2106

2107 2108
	if (!dmce)
		dmce = debugfs_create_dir("mce", NULL);
I
Ingo Molnar 已提交
2109

2110 2111
	return dmce;
}
I
Ingo Molnar 已提交
2112

2113 2114 2115 2116 2117 2118 2119 2120
static void mce_reset(void)
{
	cpu_missing = 0;
	atomic_set(&mce_fake_paniced, 0);
	atomic_set(&mce_executing, 0);
	atomic_set(&mce_callin, 0);
	atomic_set(&global_nwo, 0);
}
I
Ingo Molnar 已提交
2121

2122 2123 2124 2125
static int fake_panic_get(void *data, u64 *val)
{
	*val = fake_panic;
	return 0;
I
Ingo Molnar 已提交
2126 2127
}

2128
static int fake_panic_set(void *data, u64 val)
I
Ingo Molnar 已提交
2129
{
2130 2131 2132
	mce_reset();
	fake_panic = val;
	return 0;
I
Ingo Molnar 已提交
2133 2134
}

2135 2136
DEFINE_SIMPLE_ATTRIBUTE(fake_panic_fops, fake_panic_get,
			fake_panic_set, "%llu\n");
2137

2138
static int __init mcheck_debugfs_init(void)
2139
{
2140 2141 2142 2143 2144 2145 2146 2147 2148 2149 2150
	struct dentry *dmce, *ffake_panic;

	dmce = mce_get_debugfs_dir();
	if (!dmce)
		return -ENOMEM;
	ffake_panic = debugfs_create_file("fake_panic", 0444, dmce, NULL,
					  &fake_panic_fops);
	if (!ffake_panic)
		return -ENOMEM;

	return 0;
2151
}
2152
late_initcall(mcheck_debugfs_init);
2153
#endif