mce.c 49.9 KB
Newer Older
L
Linus Torvalds 已提交
1 2
/*
 * Machine check handler.
I
Ingo Molnar 已提交
3
 *
L
Linus Torvalds 已提交
4
 * K8 parts Copyright 2002,2003 Andi Kleen, SuSE Labs.
5 6
 * Rest from unknown author(s).
 * 2004 Andi Kleen. Rewrote most of it.
7 8
 * Copyright 2008 Intel Corporation
 * Author: Andi Kleen
L
Linus Torvalds 已提交
9
 */
I
Ingo Molnar 已提交
10 11 12 13 14 15 16
#include <linux/thread_info.h>
#include <linux/capability.h>
#include <linux/miscdevice.h>
#include <linux/ratelimit.h>
#include <linux/kallsyms.h>
#include <linux/rcupdate.h>
#include <linux/kobject.h>
17
#include <linux/uaccess.h>
I
Ingo Molnar 已提交
18 19 20
#include <linux/kdebug.h>
#include <linux/kernel.h>
#include <linux/percpu.h>
L
Linus Torvalds 已提交
21 22
#include <linux/string.h>
#include <linux/sysdev.h>
23
#include <linux/syscore_ops.h>
24
#include <linux/delay.h>
25
#include <linux/ctype.h>
I
Ingo Molnar 已提交
26
#include <linux/sched.h>
27
#include <linux/sysfs.h>
I
Ingo Molnar 已提交
28
#include <linux/types.h>
29
#include <linux/slab.h>
I
Ingo Molnar 已提交
30 31 32
#include <linux/init.h>
#include <linux/kmod.h>
#include <linux/poll.h>
33
#include <linux/nmi.h>
I
Ingo Molnar 已提交
34
#include <linux/cpu.h>
35
#include <linux/smp.h>
I
Ingo Molnar 已提交
36
#include <linux/fs.h>
37
#include <linux/mm.h>
38
#include <linux/debugfs.h>
39
#include <linux/edac_mce.h>
40
#include <linux/irq_work.h>
I
Ingo Molnar 已提交
41

42
#include <asm/processor.h>
I
Ingo Molnar 已提交
43 44
#include <asm/mce.h>
#include <asm/msr.h>
L
Linus Torvalds 已提交
45

46
#include "mce-internal.h"
47

48
static DEFINE_MUTEX(mce_chrdev_read_mutex);
49

50
#define rcu_dereference_check_mce(p) \
51
	rcu_dereference_index_check((p), \
52
			      rcu_read_lock_sched_held() || \
53
			      lockdep_is_held(&mce_chrdev_read_mutex))
54

55 56 57
#define CREATE_TRACE_POINTS
#include <trace/events/mce.h>

H
Hidetoshi Seto 已提交
58
int mce_disabled __read_mostly;
59

I
Ingo Molnar 已提交
60
#define MISC_MCELOG_MINOR	227
61

62 63
#define SPINUNIT 100	/* 100ns */

64 65
atomic_t mce_entry;

66 67
DEFINE_PER_CPU(unsigned, mce_exception_count);

68 69 70 71 72 73 74
/*
 * Tolerant levels:
 *   0: always panic on uncorrected errors, log corrected errors
 *   1: panic or SIGBUS on uncorrected errors, log corrected errors
 *   2: SIGBUS or log uncorrected errors (if possible), log corrected errors
 *   3: never panic or SIGBUS, log all errors (for testing only)
 */
H
Hidetoshi Seto 已提交
75 76 77 78 79 80 81 82 83 84
static int			tolerant		__read_mostly = 1;
static int			banks			__read_mostly;
static int			rip_msr			__read_mostly;
static int			mce_bootlog		__read_mostly = -1;
static int			monarch_timeout		__read_mostly = -1;
static int			mce_panic_timeout	__read_mostly;
static int			mce_dont_log_ce		__read_mostly;
int				mce_cmci_disabled	__read_mostly;
int				mce_ignore_ce		__read_mostly;
int				mce_ser			__read_mostly;
85

86 87
struct mce_bank                *mce_banks		__read_mostly;

88 89 90 91
/* User mode helper program triggered by machine check event */
static unsigned long		mce_need_notify;
static char			mce_helper[128];
static char			*mce_helper_argv[2] = { mce_helper, NULL };
L
Linus Torvalds 已提交
92

93 94
static DECLARE_WAIT_QUEUE_HEAD(mce_chrdev_wait);

95 96 97
static DEFINE_PER_CPU(struct mce, mces_seen);
static int			cpu_missing;

98 99 100 101 102 103 104
/*
 * CPU/chipset specific EDAC code can register a notifier call here to print
 * MCE errors in a human-readable form.
 */
ATOMIC_NOTIFIER_HEAD(x86_mce_decoder_chain);
EXPORT_SYMBOL_GPL(x86_mce_decoder_chain);

105 106 107 108 109
/* MCA banks polled by the period polling timer for corrected events */
DEFINE_PER_CPU(mce_banks_t, mce_poll_banks) = {
	[0 ... BITS_TO_LONGS(MAX_NR_BANKS)-1] = ~0UL
};

110 111
static DEFINE_PER_CPU(struct work_struct, mce_work);

112 113 114 115
/* Do initial initialization of a struct mce */
void mce_setup(struct mce *m)
{
	memset(m, 0, sizeof(struct mce));
116
	m->cpu = m->extcpu = smp_processor_id();
117
	rdtscll(m->tsc);
118 119 120 121 122 123 124 125 126
	/* We hope get_seconds stays lockless */
	m->time = get_seconds();
	m->cpuvendor = boot_cpu_data.x86_vendor;
	m->cpuid = cpuid_eax(1);
#ifdef CONFIG_SMP
	m->socketid = cpu_data(m->extcpu).phys_proc_id;
#endif
	m->apicid = cpu_data(m->extcpu).initial_apicid;
	rdmsrl(MSR_IA32_MCG_CAP, m->mcgcap);
127 128
}

129 130 131
DEFINE_PER_CPU(struct mce, injectm);
EXPORT_PER_CPU_SYMBOL_GPL(injectm);

L
Linus Torvalds 已提交
132 133 134 135 136 137
/*
 * Lockless MCE logging infrastructure.
 * This avoids deadlocks on printk locks without having to break locks. Also
 * separate MCEs from kernel messages to avoid bogus bug reports.
 */

138
static struct mce_log mcelog = {
139 140 141
	.signature	= MCE_LOG_SIGNATURE,
	.len		= MCE_LOG_LEN,
	.recordlen	= sizeof(struct mce),
142
};
L
Linus Torvalds 已提交
143 144 145 146

void mce_log(struct mce *mce)
{
	unsigned next, entry;
I
Ingo Molnar 已提交
147

148 149 150
	/* Emit the trace record: */
	trace_mce_record(mce);

L
Linus Torvalds 已提交
151
	mce->finished = 0;
M
Mike Waychison 已提交
152
	wmb();
L
Linus Torvalds 已提交
153
	for (;;) {
154
		entry = rcu_dereference_check_mce(mcelog.next);
155
		for (;;) {
156 157 158 159 160 161 162 163 164
			/*
			 * If edac_mce is enabled, it will check the error type
			 * and will process it, if it is a known error.
			 * Otherwise, the error will be sent through mcelog
			 * interface
			 */
			if (edac_mce_parse(mce))
				return;

I
Ingo Molnar 已提交
165 166 167 168 169
			/*
			 * When the buffer fills up discard new entries.
			 * Assume that the earlier errors are the more
			 * interesting ones:
			 */
170
			if (entry >= MCE_LOG_LEN) {
171 172
				set_bit(MCE_OVERFLOW,
					(unsigned long *)&mcelog.flags);
173 174
				return;
			}
I
Ingo Molnar 已提交
175
			/* Old left over entry. Skip: */
176 177 178 179
			if (mcelog.entry[entry].finished) {
				entry++;
				continue;
			}
M
Mike Waychison 已提交
180
			break;
L
Linus Torvalds 已提交
181 182 183 184 185 186 187
		}
		smp_rmb();
		next = entry + 1;
		if (cmpxchg(&mcelog.next, entry, next) == entry)
			break;
	}
	memcpy(mcelog.entry + entry, mce, sizeof(struct mce));
M
Mike Waychison 已提交
188
	wmb();
L
Linus Torvalds 已提交
189
	mcelog.entry[entry].finished = 1;
M
Mike Waychison 已提交
190
	wmb();
L
Linus Torvalds 已提交
191

192
	mce->finished = 1;
193
	set_bit(0, &mce_need_notify);
L
Linus Torvalds 已提交
194 195
}

H
Hidetoshi Seto 已提交
196
static void print_mce(struct mce *m)
L
Linus Torvalds 已提交
197
{
198 199
	int ret = 0;

H
Huang Ying 已提交
200
	pr_emerg(HW_ERR "CPU %d: Machine Check Exception: %Lx Bank %d: %016Lx\n",
201
	       m->extcpu, m->mcgstatus, m->bank, m->status);
202

203
	if (m->ip) {
H
Huang Ying 已提交
204
		pr_emerg(HW_ERR "RIP%s %02x:<%016Lx> ",
205 206 207
			!(m->mcgstatus & MCG_STATUS_EIPV) ? " !INEXACT!" : "",
				m->cs, m->ip);

L
Linus Torvalds 已提交
208
		if (m->cs == __KERNEL_CS)
209
			print_symbol("{%s}", m->ip);
210
		pr_cont("\n");
L
Linus Torvalds 已提交
211
	}
212

H
Huang Ying 已提交
213
	pr_emerg(HW_ERR "TSC %llx ", m->tsc);
L
Linus Torvalds 已提交
214
	if (m->addr)
215
		pr_cont("ADDR %llx ", m->addr);
L
Linus Torvalds 已提交
216
	if (m->misc)
217
		pr_cont("MISC %llx ", m->misc);
218

219
	pr_cont("\n");
H
Huang Ying 已提交
220
	pr_emerg(HW_ERR "PROCESSOR %u:%x TIME %llu SOCKET %u APIC %x\n",
221 222 223 224
		m->cpuvendor, m->cpuid, m->time, m->socketid, m->apicid);

	/*
	 * Print out human-readable details about the MCE error,
225
	 * (if the CPU has an implementation for that)
226
	 */
227 228 229 230 231
	ret = atomic_notifier_call_chain(&x86_mce_decoder_chain, 0, m);
	if (ret == NOTIFY_STOP)
		return;

	pr_emerg_ratelimited(HW_ERR "Run the above through 'mcelog --ascii'\n");
232 233
}

234 235 236 237
#define PANIC_TIMEOUT 5 /* 5 seconds */

static atomic_t mce_paniced;

238 239 240
static int fake_panic;
static atomic_t mce_fake_paniced;

241 242 243 244
/* Panic in progress. Enable interrupts and wait for final IPI */
static void wait_for_panic(void)
{
	long timeout = PANIC_TIMEOUT*USEC_PER_SEC;
245

246 247 248 249
	preempt_disable();
	local_irq_enable();
	while (timeout-- > 0)
		udelay(1);
250 251
	if (panic_timeout == 0)
		panic_timeout = mce_panic_timeout;
252 253 254
	panic("Panicing machine check CPU died");
}

255
static void mce_panic(char *msg, struct mce *final, char *exp)
256
{
257
	int i, apei_err = 0;
258

259 260 261 262 263 264 265
	if (!fake_panic) {
		/*
		 * Make sure only one CPU runs in machine check panic
		 */
		if (atomic_inc_return(&mce_paniced) > 1)
			wait_for_panic();
		barrier();
266

267 268 269 270 271 272 273
		bust_spinlocks(1);
		console_verbose();
	} else {
		/* Don't log too much for fake panic */
		if (atomic_inc_return(&mce_fake_paniced) > 1)
			return;
	}
274
	/* First print corrected ones that are still unlogged */
L
Linus Torvalds 已提交
275
	for (i = 0; i < MCE_LOG_LEN; i++) {
276
		struct mce *m = &mcelog.entry[i];
H
Hidetoshi Seto 已提交
277 278
		if (!(m->status & MCI_STATUS_VAL))
			continue;
279
		if (!(m->status & MCI_STATUS_UC)) {
H
Hidetoshi Seto 已提交
280
			print_mce(m);
281 282 283
			if (!apei_err)
				apei_err = apei_write_mce(m);
		}
284 285 286 287 288
	}
	/* Now print uncorrected but with the final one last */
	for (i = 0; i < MCE_LOG_LEN; i++) {
		struct mce *m = &mcelog.entry[i];
		if (!(m->status & MCI_STATUS_VAL))
L
Linus Torvalds 已提交
289
			continue;
H
Hidetoshi Seto 已提交
290 291
		if (!(m->status & MCI_STATUS_UC))
			continue;
292
		if (!final || memcmp(m, final, sizeof(struct mce))) {
H
Hidetoshi Seto 已提交
293
			print_mce(m);
294 295 296
			if (!apei_err)
				apei_err = apei_write_mce(m);
		}
L
Linus Torvalds 已提交
297
	}
298
	if (final) {
H
Hidetoshi Seto 已提交
299
		print_mce(final);
300 301 302
		if (!apei_err)
			apei_err = apei_write_mce(final);
	}
303
	if (cpu_missing)
H
Huang Ying 已提交
304
		pr_emerg(HW_ERR "Some CPUs didn't answer in synchronization\n");
305
	if (exp)
H
Huang Ying 已提交
306
		pr_emerg(HW_ERR "Machine check: %s\n", exp);
307 308 309 310 311
	if (!fake_panic) {
		if (panic_timeout == 0)
			panic_timeout = mce_panic_timeout;
		panic(msg);
	} else
H
Huang Ying 已提交
312
		pr_emerg(HW_ERR "Fake kernel panic: %s\n", msg);
313
}
L
Linus Torvalds 已提交
314

315 316 317 318
/* Support code for software error injection */

static int msr_to_offset(u32 msr)
{
T
Tejun Heo 已提交
319
	unsigned bank = __this_cpu_read(injectm.bank);
320

321 322
	if (msr == rip_msr)
		return offsetof(struct mce, ip);
323
	if (msr == MSR_IA32_MCx_STATUS(bank))
324
		return offsetof(struct mce, status);
325
	if (msr == MSR_IA32_MCx_ADDR(bank))
326
		return offsetof(struct mce, addr);
327
	if (msr == MSR_IA32_MCx_MISC(bank))
328 329 330 331 332 333
		return offsetof(struct mce, misc);
	if (msr == MSR_IA32_MCG_STATUS)
		return offsetof(struct mce, mcgstatus);
	return -1;
}

334 335 336 337
/* MSR access wrappers used for error injection */
static u64 mce_rdmsrl(u32 msr)
{
	u64 v;
338

T
Tejun Heo 已提交
339
	if (__this_cpu_read(injectm.finished)) {
340
		int offset = msr_to_offset(msr);
341

342 343 344 345
		if (offset < 0)
			return 0;
		return *(u64 *)((char *)&__get_cpu_var(injectm) + offset);
	}
346 347 348 349 350 351 352 353 354 355 356

	if (rdmsrl_safe(msr, &v)) {
		WARN_ONCE(1, "mce: Unable to read msr %d!\n", msr);
		/*
		 * Return zero in case the access faulted. This should
		 * not happen normally but can happen if the CPU does
		 * something weird, or if the code is buggy.
		 */
		v = 0;
	}

357 358 359 360 361
	return v;
}

static void mce_wrmsrl(u32 msr, u64 v)
{
T
Tejun Heo 已提交
362
	if (__this_cpu_read(injectm.finished)) {
363
		int offset = msr_to_offset(msr);
364

365 366 367 368
		if (offset >= 0)
			*(u64 *)((char *)&__get_cpu_var(injectm) + offset) = v;
		return;
	}
369 370 371
	wrmsrl(msr, v);
}

372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396
/*
 * Collect all global (w.r.t. this processor) status about this machine
 * check into our "mce" struct so that we can use it later to assess
 * the severity of the problem as we read per-bank specific details.
 */
static inline void mce_gather_info(struct mce *m, struct pt_regs *regs)
{
	mce_setup(m);

	m->mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS);
	if (regs) {
		/*
		 * Get the address of the instruction at the time of
		 * the machine check error.
		 */
		if (m->mcgstatus & (MCG_STATUS_RIPV|MCG_STATUS_EIPV)) {
			m->ip = regs->ip;
			m->cs = regs->cs;
		}
		/* Use accurate RIP reporting if available. */
		if (rip_msr)
			m->ip = mce_rdmsrl(rip_msr);
	}
}

397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451
/*
 * Simple lockless ring to communicate PFNs from the exception handler with the
 * process context work function. This is vastly simplified because there's
 * only a single reader and a single writer.
 */
#define MCE_RING_SIZE 16	/* we use one entry less */

struct mce_ring {
	unsigned short start;
	unsigned short end;
	unsigned long ring[MCE_RING_SIZE];
};
static DEFINE_PER_CPU(struct mce_ring, mce_ring);

/* Runs with CPU affinity in workqueue */
static int mce_ring_empty(void)
{
	struct mce_ring *r = &__get_cpu_var(mce_ring);

	return r->start == r->end;
}

static int mce_ring_get(unsigned long *pfn)
{
	struct mce_ring *r;
	int ret = 0;

	*pfn = 0;
	get_cpu();
	r = &__get_cpu_var(mce_ring);
	if (r->start == r->end)
		goto out;
	*pfn = r->ring[r->start];
	r->start = (r->start + 1) % MCE_RING_SIZE;
	ret = 1;
out:
	put_cpu();
	return ret;
}

/* Always runs in MCE context with preempt off */
static int mce_ring_add(unsigned long pfn)
{
	struct mce_ring *r = &__get_cpu_var(mce_ring);
	unsigned next;

	next = (r->end + 1) % MCE_RING_SIZE;
	if (next == r->start)
		return -1;
	r->ring[r->end] = pfn;
	wmb();
	r->end = next;
	return 0;
}

A
Andi Kleen 已提交
452
int mce_available(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
453
{
454
	if (mce_disabled)
455
		return 0;
456
	return cpu_has(c, X86_FEATURE_MCE) && cpu_has(c, X86_FEATURE_MCA);
L
Linus Torvalds 已提交
457 458
}

459 460 461 462 463 464 465 466 467
static void mce_schedule_work(void)
{
	if (!mce_ring_empty()) {
		struct work_struct *work = &__get_cpu_var(mce_work);
		if (!work_pending(work))
			schedule_work(work);
	}
}

468 469 470
DEFINE_PER_CPU(struct irq_work, mce_irq_work);

static void mce_irq_work_cb(struct irq_work *entry)
471
{
472
	mce_notify_irq();
473
	mce_schedule_work();
474 475 476 477 478
}

static void mce_report_event(struct pt_regs *regs)
{
	if (regs->flags & (X86_VM_MASK|X86_EFLAGS_IF)) {
479
		mce_notify_irq();
480 481 482 483 484 485 486
		/*
		 * Triggering the work queue here is just an insurance
		 * policy in case the syscall exit notify handler
		 * doesn't run soon enough or ends up running on the
		 * wrong CPU (can happen when audit sleeps)
		 */
		mce_schedule_work();
487 488 489
		return;
	}

490
	irq_work_queue(&__get_cpu_var(mce_irq_work));
491 492
}

493 494
DEFINE_PER_CPU(unsigned, mce_poll_count);

495
/*
496 497 498 499
 * Poll for corrected events or events that happened before reset.
 * Those are just logged through /dev/mcelog.
 *
 * This is executed in standard interrupt context.
A
Andi Kleen 已提交
500 501 502 503 504 505 506 507 508
 *
 * Note: spec recommends to panic for fatal unsignalled
 * errors here. However this would be quite problematic --
 * we would need to reimplement the Monarch handling and
 * it would mess up the exclusion between exception handler
 * and poll hander -- * so we skip this for now.
 * These cases should not happen anyways, or only when the CPU
 * is already totally * confused. In this case it's likely it will
 * not fully execute the machine check handler either.
509
 */
510
void machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
511 512 513 514
{
	struct mce m;
	int i;

515
	percpu_inc(mce_poll_count);
516

517
	mce_gather_info(&m, NULL);
518 519

	for (i = 0; i < banks; i++) {
520
		if (!mce_banks[i].ctl || !test_bit(i, *b))
521 522 523 524 525 526 527 528
			continue;

		m.misc = 0;
		m.addr = 0;
		m.bank = i;
		m.tsc = 0;

		barrier();
529
		m.status = mce_rdmsrl(MSR_IA32_MCx_STATUS(i));
530 531 532 533
		if (!(m.status & MCI_STATUS_VAL))
			continue;

		/*
A
Andi Kleen 已提交
534 535
		 * Uncorrected or signalled events are handled by the exception
		 * handler when it is enabled, so don't process those here.
536 537 538
		 *
		 * TBD do the same check for MCI_STATUS_EN here?
		 */
A
Andi Kleen 已提交
539 540
		if (!(flags & MCP_UC) &&
		    (m.status & (mce_ser ? MCI_STATUS_S : MCI_STATUS_UC)))
541 542 543
			continue;

		if (m.status & MCI_STATUS_MISCV)
544
			m.misc = mce_rdmsrl(MSR_IA32_MCx_MISC(i));
545
		if (m.status & MCI_STATUS_ADDRV)
546
			m.addr = mce_rdmsrl(MSR_IA32_MCx_ADDR(i));
547 548 549 550 551 552 553

		if (!(flags & MCP_TIMESTAMP))
			m.tsc = 0;
		/*
		 * Don't get the IP here because it's unlikely to
		 * have anything to do with the actual error location.
		 */
554
		if (!(flags & MCP_DONTLOG) && !mce_dont_log_ce) {
A
Andi Kleen 已提交
555
			mce_log(&m);
556
			atomic_notifier_call_chain(&x86_mce_decoder_chain, 0, &m);
A
Andi Kleen 已提交
557
		}
558 559 560 561

		/*
		 * Clear state for this bank.
		 */
562
		mce_wrmsrl(MSR_IA32_MCx_STATUS(i), 0);
563 564 565 566 567 568
	}

	/*
	 * Don't clear MCG_STATUS here because it's only defined for
	 * exceptions.
	 */
569 570

	sync_core();
571
}
572
EXPORT_SYMBOL_GPL(machine_check_poll);
573

574 575 576 577 578 579 580 581 582
/*
 * Do a quick check if any of the events requires a panic.
 * This decides if we keep the events around or clear them.
 */
static int mce_no_way_out(struct mce *m, char **msg)
{
	int i;

	for (i = 0; i < banks; i++) {
583
		m->status = mce_rdmsrl(MSR_IA32_MCx_STATUS(i));
584 585 586 587 588 589
		if (mce_severity(m, tolerant, msg) >= MCE_PANIC_SEVERITY)
			return 1;
	}
	return 0;
}

590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642
/*
 * Variable to establish order between CPUs while scanning.
 * Each CPU spins initially until executing is equal its number.
 */
static atomic_t mce_executing;

/*
 * Defines order of CPUs on entry. First CPU becomes Monarch.
 */
static atomic_t mce_callin;

/*
 * Check if a timeout waiting for other CPUs happened.
 */
static int mce_timed_out(u64 *t)
{
	/*
	 * The others already did panic for some reason.
	 * Bail out like in a timeout.
	 * rmb() to tell the compiler that system_state
	 * might have been modified by someone else.
	 */
	rmb();
	if (atomic_read(&mce_paniced))
		wait_for_panic();
	if (!monarch_timeout)
		goto out;
	if ((s64)*t < SPINUNIT) {
		/* CHECKME: Make panic default for 1 too? */
		if (tolerant < 1)
			mce_panic("Timeout synchronizing machine check over CPUs",
				  NULL, NULL);
		cpu_missing = 1;
		return 1;
	}
	*t -= SPINUNIT;
out:
	touch_nmi_watchdog();
	return 0;
}

/*
 * The Monarch's reign.  The Monarch is the CPU who entered
 * the machine check handler first. It waits for the others to
 * raise the exception too and then grades them. When any
 * error is fatal panic. Only then let the others continue.
 *
 * The other CPUs entering the MCE handler will be controlled by the
 * Monarch. They are called Subjects.
 *
 * This way we prevent any potential data corruption in a unrecoverable case
 * and also makes sure always all CPU's errors are examined.
 *
643
 * Also this detects the case of a machine check event coming from outer
644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683
 * space (not detected by any CPUs) In this case some external agent wants
 * us to shut down, so panic too.
 *
 * The other CPUs might still decide to panic if the handler happens
 * in a unrecoverable place, but in this case the system is in a semi-stable
 * state and won't corrupt anything by itself. It's ok to let the others
 * continue for a bit first.
 *
 * All the spin loops have timeouts; when a timeout happens a CPU
 * typically elects itself to be Monarch.
 */
static void mce_reign(void)
{
	int cpu;
	struct mce *m = NULL;
	int global_worst = 0;
	char *msg = NULL;
	char *nmsg = NULL;

	/*
	 * This CPU is the Monarch and the other CPUs have run
	 * through their handlers.
	 * Grade the severity of the errors of all the CPUs.
	 */
	for_each_possible_cpu(cpu) {
		int severity = mce_severity(&per_cpu(mces_seen, cpu), tolerant,
					    &nmsg);
		if (severity > global_worst) {
			msg = nmsg;
			global_worst = severity;
			m = &per_cpu(mces_seen, cpu);
		}
	}

	/*
	 * Cannot recover? Panic here then.
	 * This dumps all the mces in the log buffer and stops the
	 * other CPUs.
	 */
	if (m && global_worst >= MCE_PANIC_SEVERITY && tolerant < 3)
684
		mce_panic("Fatal Machine check", m, msg);
685 686 687 688 689 690 691 692 693 694 695

	/*
	 * For UC somewhere we let the CPU who detects it handle it.
	 * Also must let continue the others, otherwise the handling
	 * CPU could deadlock on a lock.
	 */

	/*
	 * No machine check event found. Must be some external
	 * source or one CPU is hung. Panic.
	 */
696
	if (global_worst <= MCE_KEEP_SEVERITY && tolerant < 3)
697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715
		mce_panic("Machine check from unknown source", NULL, NULL);

	/*
	 * Now clear all the mces_seen so that they don't reappear on
	 * the next mce.
	 */
	for_each_possible_cpu(cpu)
		memset(&per_cpu(mces_seen, cpu), 0, sizeof(struct mce));
}

static atomic_t global_nwo;

/*
 * Start of Monarch synchronization. This waits until all CPUs have
 * entered the exception handler and then determines if any of them
 * saw a fatal event that requires panic. Then it executes them
 * in the entry order.
 * TBD double check parallel CPU hotunplug
 */
H
Hidetoshi Seto 已提交
716
static int mce_start(int *no_way_out)
717
{
H
Hidetoshi Seto 已提交
718
	int order;
719 720 721
	int cpus = num_online_cpus();
	u64 timeout = (u64)monarch_timeout * NSEC_PER_USEC;

H
Hidetoshi Seto 已提交
722 723
	if (!timeout)
		return -1;
724

H
Hidetoshi Seto 已提交
725
	atomic_add(*no_way_out, &global_nwo);
726 727 728 729
	/*
	 * global_nwo should be updated before mce_callin
	 */
	smp_wmb();
730
	order = atomic_inc_return(&mce_callin);
731 732 733 734 735 736 737

	/*
	 * Wait for everyone.
	 */
	while (atomic_read(&mce_callin) != cpus) {
		if (mce_timed_out(&timeout)) {
			atomic_set(&global_nwo, 0);
H
Hidetoshi Seto 已提交
738
			return -1;
739 740 741 742
		}
		ndelay(SPINUNIT);
	}

743 744 745 746
	/*
	 * mce_callin should be read before global_nwo
	 */
	smp_rmb();
747

H
Hidetoshi Seto 已提交
748 749 750 751
	if (order == 1) {
		/*
		 * Monarch: Starts executing now, the others wait.
		 */
752
		atomic_set(&mce_executing, 1);
H
Hidetoshi Seto 已提交
753 754 755 756 757 758 759 760 761 762 763 764 765 766
	} else {
		/*
		 * Subject: Now start the scanning loop one by one in
		 * the original callin order.
		 * This way when there are any shared banks it will be
		 * only seen by one CPU before cleared, avoiding duplicates.
		 */
		while (atomic_read(&mce_executing) < order) {
			if (mce_timed_out(&timeout)) {
				atomic_set(&global_nwo, 0);
				return -1;
			}
			ndelay(SPINUNIT);
		}
767 768 769
	}

	/*
H
Hidetoshi Seto 已提交
770
	 * Cache the global no_way_out state.
771
	 */
H
Hidetoshi Seto 已提交
772 773 774
	*no_way_out = atomic_read(&global_nwo);

	return order;
775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843
}

/*
 * Synchronize between CPUs after main scanning loop.
 * This invokes the bulk of the Monarch processing.
 */
static int mce_end(int order)
{
	int ret = -1;
	u64 timeout = (u64)monarch_timeout * NSEC_PER_USEC;

	if (!timeout)
		goto reset;
	if (order < 0)
		goto reset;

	/*
	 * Allow others to run.
	 */
	atomic_inc(&mce_executing);

	if (order == 1) {
		/* CHECKME: Can this race with a parallel hotplug? */
		int cpus = num_online_cpus();

		/*
		 * Monarch: Wait for everyone to go through their scanning
		 * loops.
		 */
		while (atomic_read(&mce_executing) <= cpus) {
			if (mce_timed_out(&timeout))
				goto reset;
			ndelay(SPINUNIT);
		}

		mce_reign();
		barrier();
		ret = 0;
	} else {
		/*
		 * Subject: Wait for Monarch to finish.
		 */
		while (atomic_read(&mce_executing) != 0) {
			if (mce_timed_out(&timeout))
				goto reset;
			ndelay(SPINUNIT);
		}

		/*
		 * Don't reset anything. That's done by the Monarch.
		 */
		return 0;
	}

	/*
	 * Reset all global state.
	 */
reset:
	atomic_set(&global_nwo, 0);
	atomic_set(&mce_callin, 0);
	barrier();

	/*
	 * Let others run again.
	 */
	atomic_set(&mce_executing, 0);
	return ret;
}

844 845 846 847
/*
 * Check if the address reported by the CPU is in a format we can parse.
 * It would be possible to add code for most other cases, but all would
 * be somewhat complicated (e.g. segment offset would require an instruction
L
Lucas De Marchi 已提交
848
 * parser). So only support physical addresses up to page granuality for now.
849 850 851 852 853
 */
static int mce_usable_address(struct mce *m)
{
	if (!(m->status & MCI_STATUS_MISCV) || !(m->status & MCI_STATUS_ADDRV))
		return 0;
854
	if (MCI_MISC_ADDR_LSB(m->misc) > PAGE_SHIFT)
855
		return 0;
856
	if (MCI_MISC_ADDR_MODE(m->misc) != MCI_MISC_ADDR_PHYS)
857 858 859 860
		return 0;
	return 1;
}

861 862 863 864 865 866
static void mce_clear_state(unsigned long *toclear)
{
	int i;

	for (i = 0; i < banks; i++) {
		if (test_bit(i, toclear))
867
			mce_wrmsrl(MSR_IA32_MCx_STATUS(i), 0);
868 869 870
	}
}

871 872 873 874 875 876 877
/*
 * The actual machine check handler. This only handles real
 * exceptions when something got corrupted coming in through int 18.
 *
 * This is executed in NMI context not subject to normal locking rules. This
 * implies that most kernel services cannot be safely used. Don't even
 * think about putting a printk in there!
878 879 880 881
 *
 * On Intel systems this is entered on all CPUs in parallel through
 * MCE broadcast. However some CPUs might be broken beyond repair,
 * so be always careful when synchronizing with others.
L
Linus Torvalds 已提交
882
 */
I
Ingo Molnar 已提交
883
void do_machine_check(struct pt_regs *regs, long error_code)
L
Linus Torvalds 已提交
884
{
885
	struct mce m, *final;
L
Linus Torvalds 已提交
886
	int i;
887 888 889 890 891 892
	int worst = 0;
	int severity;
	/*
	 * Establish sequential order between the CPUs entering the machine
	 * check handler.
	 */
H
Hidetoshi Seto 已提交
893
	int order;
894 895 896 897 898 899 900 901 902 903
	/*
	 * If no_way_out gets set, there is no safe way to recover from this
	 * MCE.  If tolerant is cranked up, we'll try anyway.
	 */
	int no_way_out = 0;
	/*
	 * If kill_it gets set, there might be a way to recover from this
	 * error.
	 */
	int kill_it = 0;
904
	DECLARE_BITMAP(toclear, MAX_NR_BANKS);
905
	char *msg = "Unknown";
L
Linus Torvalds 已提交
906

907 908
	atomic_inc(&mce_entry);

909
	percpu_inc(mce_exception_count);
910

911
	if (notify_die(DIE_NMI, "machine check", regs, error_code,
912
			   18, SIGKILL) == NOTIFY_STOP)
913
		goto out;
914
	if (!banks)
915
		goto out;
L
Linus Torvalds 已提交
916

917
	mce_gather_info(&m, regs);
918

919 920 921
	final = &__get_cpu_var(mces_seen);
	*final = m;

922 923
	no_way_out = mce_no_way_out(&m, &msg);

L
Linus Torvalds 已提交
924 925
	barrier();

A
Andi Kleen 已提交
926 927 928 929 930 931
	/*
	 * When no restart IP must always kill or panic.
	 */
	if (!(m.mcgstatus & MCG_STATUS_RIPV))
		kill_it = 1;

932 933 934 935 936
	/*
	 * Go through all the banks in exclusion of the other CPUs.
	 * This way we don't report duplicated events on shared banks
	 * because the first one to see it will clear it.
	 */
H
Hidetoshi Seto 已提交
937
	order = mce_start(&no_way_out);
L
Linus Torvalds 已提交
938
	for (i = 0; i < banks; i++) {
939
		__clear_bit(i, toclear);
940
		if (!mce_banks[i].ctl)
L
Linus Torvalds 已提交
941
			continue;
942 943

		m.misc = 0;
L
Linus Torvalds 已提交
944 945 946
		m.addr = 0;
		m.bank = i;

947
		m.status = mce_rdmsrl(MSR_IA32_MCx_STATUS(i));
L
Linus Torvalds 已提交
948 949 950
		if ((m.status & MCI_STATUS_VAL) == 0)
			continue;

951
		/*
A
Andi Kleen 已提交
952 953
		 * Non uncorrected or non signaled errors are handled by
		 * machine_check_poll. Leave them alone, unless this panics.
954
		 */
A
Andi Kleen 已提交
955 956
		if (!(m.status & (mce_ser ? MCI_STATUS_S : MCI_STATUS_UC)) &&
			!no_way_out)
957 958 959 960 961 962 963
			continue;

		/*
		 * Set taint even when machine check was not enabled.
		 */
		add_taint(TAINT_MACHINE_CHECK);

A
Andi Kleen 已提交
964
		severity = mce_severity(&m, tolerant, NULL);
965

A
Andi Kleen 已提交
966 967 968 969 970 971 972 973
		/*
		 * When machine check was for corrected handler don't touch,
		 * unless we're panicing.
		 */
		if (severity == MCE_KEEP_SEVERITY && !no_way_out)
			continue;
		__set_bit(i, toclear);
		if (severity == MCE_NO_SEVERITY) {
974 975 976 977 978
			/*
			 * Machine check event was not enabled. Clear, but
			 * ignore.
			 */
			continue;
L
Linus Torvalds 已提交
979 980
		}

A
Andi Kleen 已提交
981 982 983 984 985 986
		/*
		 * Kill on action required.
		 */
		if (severity == MCE_AR_SEVERITY)
			kill_it = 1;

L
Linus Torvalds 已提交
987
		if (m.status & MCI_STATUS_MISCV)
988
			m.misc = mce_rdmsrl(MSR_IA32_MCx_MISC(i));
L
Linus Torvalds 已提交
989
		if (m.status & MCI_STATUS_ADDRV)
990
			m.addr = mce_rdmsrl(MSR_IA32_MCx_ADDR(i));
L
Linus Torvalds 已提交
991

992 993 994 995 996 997 998 999 1000 1001
		/*
		 * Action optional error. Queue address for later processing.
		 * When the ring overflows we just ignore the AO error.
		 * RED-PEN add some logging mechanism when
		 * usable_address or mce_add_ring fails.
		 * RED-PEN don't ignore overflow for tolerant == 0
		 */
		if (severity == MCE_AO_SEVERITY && mce_usable_address(&m))
			mce_ring_add(m.addr >> PAGE_SHIFT);

1002
		mce_log(&m);
L
Linus Torvalds 已提交
1003

1004 1005 1006
		if (severity > worst) {
			*final = m;
			worst = severity;
L
Linus Torvalds 已提交
1007 1008 1009
		}
	}

1010 1011 1012
	if (!no_way_out)
		mce_clear_state(toclear);

I
Ingo Molnar 已提交
1013
	/*
1014 1015
	 * Do most of the synchronization with other CPUs.
	 * When there's any problem use only local no_way_out state.
I
Ingo Molnar 已提交
1016
	 */
1017 1018
	if (mce_end(order) < 0)
		no_way_out = worst >= MCE_PANIC_SEVERITY;
1019 1020 1021

	/*
	 * If we have decided that we just CAN'T continue, and the user
I
Ingo Molnar 已提交
1022
	 * has not set tolerant to an insane level, give up and die.
1023 1024 1025
	 *
	 * This is mainly used in the case when the system doesn't
	 * support MCE broadcasting or it has been disabled.
1026 1027
	 */
	if (no_way_out && tolerant < 3)
1028
		mce_panic("Fatal machine check on current CPU", final, msg);
1029 1030 1031 1032 1033 1034 1035 1036

	/*
	 * If the error seems to be unrecoverable, something should be
	 * done.  Try to kill as little as possible.  If we can kill just
	 * one task, do that.  If the user has set the tolerance very
	 * high, don't try to do anything at all.
	 */

A
Andi Kleen 已提交
1037 1038
	if (kill_it && tolerant < 3)
		force_sig(SIGBUS, current);
L
Linus Torvalds 已提交
1039

1040 1041 1042
	/* notify userspace ASAP */
	set_thread_flag(TIF_MCE_NOTIFY);

1043 1044
	if (worst > 0)
		mce_report_event(regs);
1045
	mce_wrmsrl(MSR_IA32_MCG_STATUS, 0);
1046
out:
1047
	atomic_dec(&mce_entry);
1048
	sync_core();
L
Linus Torvalds 已提交
1049
}
1050
EXPORT_SYMBOL_GPL(do_machine_check);
L
Linus Torvalds 已提交
1051

1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081
/* dummy to break dependency. actual code is in mm/memory-failure.c */
void __attribute__((weak)) memory_failure(unsigned long pfn, int vector)
{
	printk(KERN_ERR "Action optional memory failure at %lx ignored\n", pfn);
}

/*
 * Called after mce notification in process context. This code
 * is allowed to sleep. Call the high level VM handler to process
 * any corrupted pages.
 * Assume that the work queue code only calls this one at a time
 * per CPU.
 * Note we don't disable preemption, so this code might run on the wrong
 * CPU. In this case the event is picked up by the scheduled work queue.
 * This is merely a fast path to expedite processing in some common
 * cases.
 */
void mce_notify_process(void)
{
	unsigned long pfn;
	mce_notify_irq();
	while (mce_ring_get(&pfn))
		memory_failure(pfn, MCE_VECTOR);
}

static void mce_process_work(struct work_struct *dummy)
{
	mce_notify_process();
}

1082 1083 1084
#ifdef CONFIG_X86_MCE_INTEL
/***
 * mce_log_therm_throt_event - Logs the thermal throttling event to mcelog
S
Simon Arlott 已提交
1085
 * @cpu: The CPU on which the event occurred.
1086 1087 1088 1089 1090 1091 1092 1093 1094 1095
 * @status: Event status information
 *
 * This function should be called by the thermal interrupt after the
 * event has been processed and the decision was made to log the event
 * further.
 *
 * The status parameter will be saved to the 'status' field of 'struct mce'
 * and historically has been the register value of the
 * MSR_IA32_THERMAL_STATUS (Intel) msr.
 */
1096
void mce_log_therm_throt_event(__u64 status)
1097 1098 1099
{
	struct mce m;

1100
	mce_setup(&m);
1101 1102 1103 1104 1105 1106
	m.bank = MCE_THERMAL_BANK;
	m.status = status;
	mce_log(&m);
}
#endif /* CONFIG_X86_MCE_INTEL */

L
Linus Torvalds 已提交
1107
/*
1108 1109 1110
 * Periodic polling timer for "silent" machine check errors.  If the
 * poller finds an MCE, poll 2x faster.  When the poller finds no more
 * errors, poll 2x slower (up to check_interval seconds).
L
Linus Torvalds 已提交
1111 1112
 */
static int check_interval = 5 * 60; /* 5 minutes */
I
Ingo Molnar 已提交
1113

1114
static DEFINE_PER_CPU(int, mce_next_interval); /* in jiffies */
1115
static DEFINE_PER_CPU(struct timer_list, mce_timer);
L
Linus Torvalds 已提交
1116

1117
static void mce_start_timer(unsigned long data)
L
Linus Torvalds 已提交
1118
{
1119
	struct timer_list *t = &per_cpu(mce_timer, data);
1120
	int *n;
1121 1122 1123

	WARN_ON(smp_processor_id() != data);

1124
	if (mce_available(__this_cpu_ptr(&cpu_info))) {
1125 1126
		machine_check_poll(MCP_TIMESTAMP,
				&__get_cpu_var(mce_poll_banks));
I
Ingo Molnar 已提交
1127
	}
L
Linus Torvalds 已提交
1128 1129

	/*
1130 1131
	 * Alert userspace if needed.  If we logged an MCE, reduce the
	 * polling interval, otherwise increase the polling interval.
L
Linus Torvalds 已提交
1132
	 */
1133
	n = &__get_cpu_var(mce_next_interval);
1134
	if (mce_notify_irq())
1135
		*n = max(*n/2, HZ/100);
1136
	else
1137
		*n = min(*n*2, (int)round_jiffies_relative(check_interval*HZ));
1138

1139
	t->expires = jiffies + *n;
1140
	add_timer_on(t, smp_processor_id());
1141 1142
}

1143 1144
static void mce_do_trigger(struct work_struct *work)
{
1145
	call_usermodehelper(mce_helper, mce_helper_argv, NULL, UMH_NO_WAIT);
1146 1147 1148 1149
}

static DECLARE_WORK(mce_trigger_work, mce_do_trigger);

1150
/*
1151 1152 1153
 * Notify the user(s) about new machine check events.
 * Can be called from interrupt context, but not from machine check/NMI
 * context.
1154
 */
1155
int mce_notify_irq(void)
1156
{
1157 1158 1159
	/* Not more than two messages every minute */
	static DEFINE_RATELIMIT_STATE(ratelimit, 60*HZ, 2);

1160
	clear_thread_flag(TIF_MCE_NOTIFY);
I
Ingo Molnar 已提交
1161

1162
	if (test_and_clear_bit(0, &mce_need_notify)) {
1163 1164
		/* wake processes polling /dev/mcelog */
		wake_up_interruptible(&mce_chrdev_wait);
1165 1166 1167 1168 1169 1170

		/*
		 * There is no risk of missing notifications because
		 * work_pending is always cleared before the function is
		 * executed.
		 */
1171
		if (mce_helper[0] && !work_pending(&mce_trigger_work))
1172
			schedule_work(&mce_trigger_work);
1173

1174
		if (__ratelimit(&ratelimit))
H
Huang Ying 已提交
1175
			pr_info(HW_ERR "Machine check events logged\n");
1176 1177

		return 1;
L
Linus Torvalds 已提交
1178
	}
1179 1180
	return 0;
}
1181
EXPORT_SYMBOL_GPL(mce_notify_irq);
1182

H
Hidetoshi Seto 已提交
1183
static int __cpuinit __mcheck_cpu_mce_banks_init(void)
1184 1185 1186 1187 1188 1189 1190 1191
{
	int i;

	mce_banks = kzalloc(banks * sizeof(struct mce_bank), GFP_KERNEL);
	if (!mce_banks)
		return -ENOMEM;
	for (i = 0; i < banks; i++) {
		struct mce_bank *b = &mce_banks[i];
1192

1193 1194 1195 1196 1197 1198
		b->ctl = -1ULL;
		b->init = 1;
	}
	return 0;
}

1199
/*
L
Linus Torvalds 已提交
1200 1201
 * Initialize Machine Checks for a CPU.
 */
1202
static int __cpuinit __mcheck_cpu_cap_init(void)
L
Linus Torvalds 已提交
1203
{
1204
	unsigned b;
I
Ingo Molnar 已提交
1205
	u64 cap;
L
Linus Torvalds 已提交
1206 1207

	rdmsrl(MSR_IA32_MCG_CAP, cap);
1208 1209

	b = cap & MCG_BANKCNT_MASK;
1210 1211
	if (!banks)
		printk(KERN_INFO "mce: CPU supports %d MCE banks\n", b);
1212

1213 1214 1215 1216 1217 1218 1219 1220 1221 1222
	if (b > MAX_NR_BANKS) {
		printk(KERN_WARNING
		       "MCE: Using only %u machine check banks out of %u\n",
			MAX_NR_BANKS, b);
		b = MAX_NR_BANKS;
	}

	/* Don't support asymmetric configurations today */
	WARN_ON(banks != 0 && b != banks);
	banks = b;
1223
	if (!mce_banks) {
H
Hidetoshi Seto 已提交
1224
		int err = __mcheck_cpu_mce_banks_init();
1225

1226 1227
		if (err)
			return err;
L
Linus Torvalds 已提交
1228
	}
1229

1230
	/* Use accurate RIP reporting if available. */
1231
	if ((cap & MCG_EXT_P) && MCG_EXT_CNT(cap) >= 9)
1232
		rip_msr = MSR_IA32_MCG_EIP;
L
Linus Torvalds 已提交
1233

A
Andi Kleen 已提交
1234 1235 1236
	if (cap & MCG_SER_P)
		mce_ser = 1;

1237 1238 1239
	return 0;
}

1240
static void __mcheck_cpu_init_generic(void)
1241
{
I
Ingo Molnar 已提交
1242
	mce_banks_t all_banks;
1243 1244 1245
	u64 cap;
	int i;

1246 1247 1248
	/*
	 * Log the machine checks left over from the previous reset.
	 */
1249
	bitmap_fill(all_banks, MAX_NR_BANKS);
A
Andi Kleen 已提交
1250
	machine_check_poll(MCP_UC|(!mce_bootlog ? MCP_DONTLOG : 0), &all_banks);
L
Linus Torvalds 已提交
1251 1252 1253

	set_in_cr4(X86_CR4_MCE);

1254
	rdmsrl(MSR_IA32_MCG_CAP, cap);
L
Linus Torvalds 已提交
1255 1256 1257 1258
	if (cap & MCG_CTL_P)
		wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff);

	for (i = 0; i < banks; i++) {
1259
		struct mce_bank *b = &mce_banks[i];
1260

1261
		if (!b->init)
1262
			continue;
1263 1264
		wrmsrl(MSR_IA32_MCx_CTL(i), b->ctl);
		wrmsrl(MSR_IA32_MCx_STATUS(i), 0);
1265
	}
L
Linus Torvalds 已提交
1266 1267 1268
}

/* Add per CPU specific workarounds here */
1269
static int __cpuinit __mcheck_cpu_apply_quirks(struct cpuinfo_x86 *c)
1270
{
1271 1272 1273 1274 1275
	if (c->x86_vendor == X86_VENDOR_UNKNOWN) {
		pr_info("MCE: unknown CPU type - not enabling MCE support.\n");
		return -EOPNOTSUPP;
	}

L
Linus Torvalds 已提交
1276
	/* This should be disabled by the BIOS, but isn't always */
1277
	if (c->x86_vendor == X86_VENDOR_AMD) {
I
Ingo Molnar 已提交
1278 1279 1280 1281 1282 1283
		if (c->x86 == 15 && banks > 4) {
			/*
			 * disable GART TBL walk error reporting, which
			 * trips off incorrectly with the IOMMU & 3ware
			 * & Cerberus:
			 */
1284
			clear_bit(10, (unsigned long *)&mce_banks[4].ctl);
I
Ingo Molnar 已提交
1285 1286 1287 1288 1289 1290
		}
		if (c->x86 <= 17 && mce_bootlog < 0) {
			/*
			 * Lots of broken BIOS around that don't clear them
			 * by default and leave crap in there. Don't log:
			 */
1291
			mce_bootlog = 0;
I
Ingo Molnar 已提交
1292
		}
1293 1294 1295 1296
		/*
		 * Various K7s with broken bank 0 around. Always disable
		 * by default.
		 */
1297
		 if (c->x86 == 6 && banks > 0)
1298
			mce_banks[0].ctl = 0;
L
Linus Torvalds 已提交
1299
	}
1300

1301 1302 1303 1304 1305 1306 1307 1308 1309 1310
	if (c->x86_vendor == X86_VENDOR_INTEL) {
		/*
		 * SDM documents that on family 6 bank 0 should not be written
		 * because it aliases to another special BIOS controlled
		 * register.
		 * But it's not aliased anymore on model 0x1a+
		 * Don't ignore bank 0 completely because there could be a
		 * valid event later, merely don't write CTL0.
		 */

1311 1312
		if (c->x86 == 6 && c->x86_model < 0x1A && banks > 0)
			mce_banks[0].init = 0;
1313 1314 1315 1316 1317 1318 1319 1320

		/*
		 * All newer Intel systems support MCE broadcasting. Enable
		 * synchronization with a one second timeout.
		 */
		if ((c->x86 > 6 || (c->x86 == 6 && c->x86_model >= 0xe)) &&
			monarch_timeout < 0)
			monarch_timeout = USEC_PER_SEC;
1321

1322 1323 1324 1325 1326
		/*
		 * There are also broken BIOSes on some Pentium M and
		 * earlier systems:
		 */
		if (c->x86 == 6 && c->x86_model <= 13 && mce_bootlog < 0)
1327
			mce_bootlog = 0;
1328
	}
1329 1330
	if (monarch_timeout < 0)
		monarch_timeout = 0;
1331 1332
	if (mce_bootlog != 0)
		mce_panic_timeout = 30;
1333 1334

	return 0;
1335
}
L
Linus Torvalds 已提交
1336

1337
static int __cpuinit __mcheck_cpu_ancient_init(struct cpuinfo_x86 *c)
1338 1339
{
	if (c->x86 != 5)
1340 1341
		return 0;

1342 1343
	switch (c->x86_vendor) {
	case X86_VENDOR_INTEL:
1344
		intel_p5_mcheck_init(c);
1345
		return 1;
1346 1347 1348
		break;
	case X86_VENDOR_CENTAUR:
		winchip_mcheck_init(c);
1349
		return 1;
1350 1351
		break;
	}
1352 1353

	return 0;
1354 1355
}

1356
static void __mcheck_cpu_init_vendor(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1357 1358 1359 1360 1361
{
	switch (c->x86_vendor) {
	case X86_VENDOR_INTEL:
		mce_intel_feature_init(c);
		break;
1362 1363 1364
	case X86_VENDOR_AMD:
		mce_amd_feature_init(c);
		break;
L
Linus Torvalds 已提交
1365 1366 1367 1368 1369
	default:
		break;
	}
}

1370
static void __mcheck_cpu_init_timer(void)
1371 1372
{
	struct timer_list *t = &__get_cpu_var(mce_timer);
1373
	int *n = &__get_cpu_var(mce_next_interval);
1374

1375 1376
	setup_timer(t, mce_start_timer, smp_processor_id());

1377 1378 1379
	if (mce_ignore_ce)
		return;

1380 1381
	*n = check_interval * HZ;
	if (!*n)
1382
		return;
1383
	t->expires = round_jiffies(jiffies + *n);
1384
	add_timer_on(t, smp_processor_id());
1385 1386
}

A
Andi Kleen 已提交
1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397
/* Handle unconfigured int18 (should never happen) */
static void unexpected_machine_check(struct pt_regs *regs, long error_code)
{
	printk(KERN_ERR "CPU#%d: Unexpected int18 (Machine Check).\n",
	       smp_processor_id());
}

/* Call the installed machine check handler for this CPU setup. */
void (*machine_check_vector)(struct pt_regs *, long error_code) =
						unexpected_machine_check;

1398
/*
L
Linus Torvalds 已提交
1399
 * Called for each booted CPU to set up machine checks.
I
Ingo Molnar 已提交
1400
 * Must be called with preempt off:
L
Linus Torvalds 已提交
1401
 */
1402
void __cpuinit mcheck_cpu_init(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1403
{
1404 1405 1406
	if (mce_disabled)
		return;

1407 1408
	if (__mcheck_cpu_ancient_init(c))
		return;
1409

1410
	if (!mce_available(c))
L
Linus Torvalds 已提交
1411 1412
		return;

1413
	if (__mcheck_cpu_cap_init() < 0 || __mcheck_cpu_apply_quirks(c) < 0) {
1414
		mce_disabled = 1;
1415 1416 1417
		return;
	}

1418 1419
	machine_check_vector = do_machine_check;

1420 1421 1422
	__mcheck_cpu_init_generic();
	__mcheck_cpu_init_vendor(c);
	__mcheck_cpu_init_timer();
1423
	INIT_WORK(&__get_cpu_var(mce_work), mce_process_work);
1424
	init_irq_work(&__get_cpu_var(mce_irq_work), &mce_irq_work_cb);
L
Linus Torvalds 已提交
1425 1426 1427
}

/*
1428
 * mce_chrdev: Character device /dev/mcelog to read and clear the MCE log.
L
Linus Torvalds 已提交
1429 1430
 */

1431 1432 1433
static DEFINE_SPINLOCK(mce_chrdev_state_lock);
static int mce_chrdev_open_count;	/* #times opened */
static int mce_chrdev_open_exclu;	/* already open exclusive? */
T
Tim Hockin 已提交
1434

1435
static int mce_chrdev_open(struct inode *inode, struct file *file)
T
Tim Hockin 已提交
1436
{
1437
	spin_lock(&mce_chrdev_state_lock);
T
Tim Hockin 已提交
1438

1439 1440 1441
	if (mce_chrdev_open_exclu ||
	    (mce_chrdev_open_count && (file->f_flags & O_EXCL))) {
		spin_unlock(&mce_chrdev_state_lock);
I
Ingo Molnar 已提交
1442

T
Tim Hockin 已提交
1443 1444 1445 1446
		return -EBUSY;
	}

	if (file->f_flags & O_EXCL)
1447 1448
		mce_chrdev_open_exclu = 1;
	mce_chrdev_open_count++;
T
Tim Hockin 已提交
1449

1450
	spin_unlock(&mce_chrdev_state_lock);
T
Tim Hockin 已提交
1451

1452
	return nonseekable_open(inode, file);
T
Tim Hockin 已提交
1453 1454
}

1455
static int mce_chrdev_release(struct inode *inode, struct file *file)
T
Tim Hockin 已提交
1456
{
1457
	spin_lock(&mce_chrdev_state_lock);
T
Tim Hockin 已提交
1458

1459 1460
	mce_chrdev_open_count--;
	mce_chrdev_open_exclu = 0;
T
Tim Hockin 已提交
1461

1462
	spin_unlock(&mce_chrdev_state_lock);
T
Tim Hockin 已提交
1463 1464 1465 1466

	return 0;
}

1467 1468
static void collect_tscs(void *data)
{
L
Linus Torvalds 已提交
1469
	unsigned long *cpu_tsc = (unsigned long *)data;
1470

L
Linus Torvalds 已提交
1471
	rdtscll(cpu_tsc[smp_processor_id()]);
1472
}
L
Linus Torvalds 已提交
1473

1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510
static int mce_apei_read_done;

/* Collect MCE record of previous boot in persistent storage via APEI ERST. */
static int __mce_read_apei(char __user **ubuf, size_t usize)
{
	int rc;
	u64 record_id;
	struct mce m;

	if (usize < sizeof(struct mce))
		return -EINVAL;

	rc = apei_read_mce(&m, &record_id);
	/* Error or no more MCE record */
	if (rc <= 0) {
		mce_apei_read_done = 1;
		return rc;
	}
	rc = -EFAULT;
	if (copy_to_user(*ubuf, &m, sizeof(struct mce)))
		return rc;
	/*
	 * In fact, we should have cleared the record after that has
	 * been flushed to the disk or sent to network in
	 * /sbin/mcelog, but we have no interface to support that now,
	 * so just clear it to avoid duplication.
	 */
	rc = apei_clear_mce(record_id);
	if (rc) {
		mce_apei_read_done = 1;
		return rc;
	}
	*ubuf += sizeof(struct mce);

	return 0;
}

1511 1512
static ssize_t mce_chrdev_read(struct file *filp, char __user *ubuf,
				size_t usize, loff_t *off)
L
Linus Torvalds 已提交
1513
{
I
Ingo Molnar 已提交
1514
	char __user *buf = ubuf;
1515
	unsigned long *cpu_tsc;
1516
	unsigned prev, next;
L
Linus Torvalds 已提交
1517 1518
	int i, err;

1519
	cpu_tsc = kmalloc(nr_cpu_ids * sizeof(long), GFP_KERNEL);
1520 1521 1522
	if (!cpu_tsc)
		return -ENOMEM;

1523
	mutex_lock(&mce_chrdev_read_mutex);
1524 1525 1526 1527 1528 1529 1530

	if (!mce_apei_read_done) {
		err = __mce_read_apei(&buf, usize);
		if (err || buf != ubuf)
			goto out;
	}

1531
	next = rcu_dereference_check_mce(mcelog.next);
L
Linus Torvalds 已提交
1532 1533

	/* Only supports full reads right now */
1534 1535 1536
	err = -EINVAL;
	if (*off != 0 || usize < MCE_LOG_LEN*sizeof(struct mce))
		goto out;
L
Linus Torvalds 已提交
1537 1538

	err = 0;
1539 1540 1541 1542
	prev = 0;
	do {
		for (i = prev; i < next; i++) {
			unsigned long start = jiffies;
H
Hidetoshi Seto 已提交
1543
			struct mce *m = &mcelog.entry[i];
1544

H
Hidetoshi Seto 已提交
1545
			while (!m->finished) {
1546
				if (time_after_eq(jiffies, start + 2)) {
H
Hidetoshi Seto 已提交
1547
					memset(m, 0, sizeof(*m));
1548 1549 1550
					goto timeout;
				}
				cpu_relax();
1551
			}
1552
			smp_rmb();
H
Hidetoshi Seto 已提交
1553 1554
			err |= copy_to_user(buf, m, sizeof(*m));
			buf += sizeof(*m);
1555 1556
timeout:
			;
1557
		}
L
Linus Torvalds 已提交
1558

1559 1560 1561 1562 1563
		memset(mcelog.entry + prev, 0,
		       (next - prev) * sizeof(struct mce));
		prev = next;
		next = cmpxchg(&mcelog.next, prev, 0);
	} while (next != prev);
L
Linus Torvalds 已提交
1564

1565
	synchronize_sched();
L
Linus Torvalds 已提交
1566

1567 1568 1569 1570
	/*
	 * Collect entries that were still getting written before the
	 * synchronize.
	 */
1571
	on_each_cpu(collect_tscs, cpu_tsc, 1);
I
Ingo Molnar 已提交
1572

1573
	for (i = next; i < MCE_LOG_LEN; i++) {
H
Hidetoshi Seto 已提交
1574 1575 1576 1577
		struct mce *m = &mcelog.entry[i];

		if (m->finished && m->tsc < cpu_tsc[m->cpu]) {
			err |= copy_to_user(buf, m, sizeof(*m));
L
Linus Torvalds 已提交
1578
			smp_rmb();
H
Hidetoshi Seto 已提交
1579 1580
			buf += sizeof(*m);
			memset(m, 0, sizeof(*m));
L
Linus Torvalds 已提交
1581
		}
1582
	}
1583 1584 1585 1586 1587

	if (err)
		err = -EFAULT;

out:
1588
	mutex_unlock(&mce_chrdev_read_mutex);
1589
	kfree(cpu_tsc);
I
Ingo Molnar 已提交
1590

1591
	return err ? err : buf - ubuf;
L
Linus Torvalds 已提交
1592 1593
}

1594
static unsigned int mce_chrdev_poll(struct file *file, poll_table *wait)
1595
{
1596
	poll_wait(file, &mce_chrdev_wait, wait);
1597
	if (rcu_access_index(mcelog.next))
1598
		return POLLIN | POLLRDNORM;
1599 1600
	if (!mce_apei_read_done && apei_check_mce())
		return POLLIN | POLLRDNORM;
1601 1602 1603
	return 0;
}

1604 1605
static long mce_chrdev_ioctl(struct file *f, unsigned int cmd,
				unsigned long arg)
L
Linus Torvalds 已提交
1606 1607
{
	int __user *p = (int __user *)arg;
1608

L
Linus Torvalds 已提交
1609
	if (!capable(CAP_SYS_ADMIN))
1610
		return -EPERM;
I
Ingo Molnar 已提交
1611

L
Linus Torvalds 已提交
1612
	switch (cmd) {
1613
	case MCE_GET_RECORD_LEN:
L
Linus Torvalds 已提交
1614 1615
		return put_user(sizeof(struct mce), p);
	case MCE_GET_LOG_LEN:
1616
		return put_user(MCE_LOG_LEN, p);
L
Linus Torvalds 已提交
1617 1618
	case MCE_GETCLEAR_FLAGS: {
		unsigned flags;
1619 1620

		do {
L
Linus Torvalds 已提交
1621
			flags = mcelog.flags;
1622
		} while (cmpxchg(&mcelog.flags, flags, 0) != flags);
I
Ingo Molnar 已提交
1623

1624
		return put_user(flags, p);
L
Linus Torvalds 已提交
1625 1626
	}
	default:
1627 1628
		return -ENOTTY;
	}
L
Linus Torvalds 已提交
1629 1630
}

1631
/* Modified in mce-inject.c, so not static or const */
1632
struct file_operations mce_chrdev_ops = {
1633 1634 1635 1636 1637 1638
	.open			= mce_chrdev_open,
	.release		= mce_chrdev_release,
	.read			= mce_chrdev_read,
	.poll			= mce_chrdev_poll,
	.unlocked_ioctl		= mce_chrdev_ioctl,
	.llseek			= no_llseek,
L
Linus Torvalds 已提交
1639
};
1640
EXPORT_SYMBOL_GPL(mce_chrdev_ops);
L
Linus Torvalds 已提交
1641

1642
static struct miscdevice mce_chrdev_device = {
L
Linus Torvalds 已提交
1643 1644 1645 1646 1647
	MISC_MCELOG_MINOR,
	"mcelog",
	&mce_chrdev_ops,
};

H
Hidetoshi Seto 已提交
1648
/*
1649 1650 1651 1652
 * mce=off Disables machine check
 * mce=no_cmci Disables CMCI
 * mce=dont_log_ce Clears corrected events silently, no log created for CEs.
 * mce=ignore_ce Disables polling and CMCI, corrected events are not cleared.
1653 1654 1655
 * mce=TOLERANCELEVEL[,monarchtimeout] (number, see above)
 *	monarchtimeout is how long to wait for other CPUs on machine
 *	check, or 0 to not wait
H
Hidetoshi Seto 已提交
1656 1657 1658
 * mce=bootlog Log MCEs from before booting. Disabled by default on AMD.
 * mce=nobootlog Don't log MCEs from before booting.
 */
L
Linus Torvalds 已提交
1659 1660
static int __init mcheck_enable(char *str)
{
1661
	if (*str == 0) {
1662
		enable_p5_mce();
1663 1664
		return 1;
	}
1665 1666
	if (*str == '=')
		str++;
L
Linus Torvalds 已提交
1667
	if (!strcmp(str, "off"))
1668
		mce_disabled = 1;
1669 1670 1671 1672 1673 1674
	else if (!strcmp(str, "no_cmci"))
		mce_cmci_disabled = 1;
	else if (!strcmp(str, "dont_log_ce"))
		mce_dont_log_ce = 1;
	else if (!strcmp(str, "ignore_ce"))
		mce_ignore_ce = 1;
H
Hidetoshi Seto 已提交
1675 1676
	else if (!strcmp(str, "bootlog") || !strcmp(str, "nobootlog"))
		mce_bootlog = (str[0] == 'b');
1677
	else if (isdigit(str[0])) {
1678
		get_option(&str, &tolerant);
1679 1680 1681 1682 1683
		if (*str == ',') {
			++str;
			get_option(&str, &monarch_timeout);
		}
	} else {
1684
		printk(KERN_INFO "mce argument %s ignored. Please use /sys\n",
H
Hidetoshi Seto 已提交
1685 1686 1687
		       str);
		return 0;
	}
1688
	return 1;
L
Linus Torvalds 已提交
1689
}
1690
__setup("mce", mcheck_enable);
L
Linus Torvalds 已提交
1691

1692
int __init mcheck_init(void)
1693
{
1694 1695
	mcheck_intel_therm_init();

1696 1697 1698
	return 0;
}

1699
/*
1700
 * mce_syscore: PM support
1701
 */
L
Linus Torvalds 已提交
1702

1703 1704 1705 1706
/*
 * Disable machine checks on suspend and shutdown. We can't really handle
 * them later.
 */
1707
static int mce_disable_error_reporting(void)
1708 1709 1710
{
	int i;

1711
	for (i = 0; i < banks; i++) {
1712
		struct mce_bank *b = &mce_banks[i];
1713

1714
		if (b->init)
1715
			wrmsrl(MSR_IA32_MCx_CTL(i), 0);
1716
	}
1717 1718 1719
	return 0;
}

1720
static int mce_syscore_suspend(void)
1721
{
1722
	return mce_disable_error_reporting();
1723 1724
}

1725
static void mce_syscore_shutdown(void)
1726
{
1727
	mce_disable_error_reporting();
1728 1729
}

I
Ingo Molnar 已提交
1730 1731 1732 1733 1734
/*
 * On resume clear all MCE state. Don't want to see leftovers from the BIOS.
 * Only one CPU is active at this time, the others get re-added later using
 * CPU hotplug:
 */
1735
static void mce_syscore_resume(void)
L
Linus Torvalds 已提交
1736
{
1737
	__mcheck_cpu_init_generic();
1738
	__mcheck_cpu_init_vendor(__this_cpu_ptr(&cpu_info));
L
Linus Torvalds 已提交
1739 1740
}

1741
static struct syscore_ops mce_syscore_ops = {
1742 1743 1744
	.suspend	= mce_syscore_suspend,
	.shutdown	= mce_syscore_shutdown,
	.resume		= mce_syscore_resume,
1745 1746
};

1747 1748 1749 1750
/*
 * mce_sysdev: Sysfs support
 */

1751 1752 1753
static void mce_cpu_restart(void *data)
{
	del_timer_sync(&__get_cpu_var(mce_timer));
1754
	if (!mce_available(__this_cpu_ptr(&cpu_info)))
1755
		return;
1756 1757
	__mcheck_cpu_init_generic();
	__mcheck_cpu_init_timer();
1758 1759
}

L
Linus Torvalds 已提交
1760
/* Reinit MCEs after user configuration changes */
1761 1762
static void mce_restart(void)
{
1763
	on_each_cpu(mce_cpu_restart, NULL, 1);
L
Linus Torvalds 已提交
1764 1765
}

1766 1767 1768
/* Toggle features for corrected errors */
static void mce_disable_ce(void *all)
{
1769
	if (!mce_available(__this_cpu_ptr(&cpu_info)))
1770 1771 1772 1773 1774 1775 1776 1777
		return;
	if (all)
		del_timer_sync(&__get_cpu_var(mce_timer));
	cmci_clear();
}

static void mce_enable_ce(void *all)
{
1778
	if (!mce_available(__this_cpu_ptr(&cpu_info)))
1779 1780 1781 1782
		return;
	cmci_reenable();
	cmci_recheck();
	if (all)
1783
		__mcheck_cpu_init_timer();
1784 1785
}

1786
static struct sysdev_class mce_sysdev_class = {
I
Ingo Molnar 已提交
1787
	.name		= "machinecheck",
L
Linus Torvalds 已提交
1788 1789
};

1790
DEFINE_PER_CPU(struct sys_device, mce_sysdev);
I
Ingo Molnar 已提交
1791 1792 1793

__cpuinitdata
void (*threshold_cpu_callback)(unsigned long action, unsigned int cpu);
L
Linus Torvalds 已提交
1794

1795 1796 1797 1798
static inline struct mce_bank *attr_to_bank(struct sysdev_attribute *attr)
{
	return container_of(attr, struct mce_bank, attr);
}
1799 1800 1801 1802

static ssize_t show_bank(struct sys_device *s, struct sysdev_attribute *attr,
			 char *buf)
{
1803
	return sprintf(buf, "%llx\n", attr_to_bank(attr)->ctl);
1804 1805 1806
}

static ssize_t set_bank(struct sys_device *s, struct sysdev_attribute *attr,
H
Hidetoshi Seto 已提交
1807
			const char *buf, size_t size)
1808
{
H
Hidetoshi Seto 已提交
1809
	u64 new;
I
Ingo Molnar 已提交
1810

H
Hidetoshi Seto 已提交
1811
	if (strict_strtoull(buf, 0, &new) < 0)
1812
		return -EINVAL;
I
Ingo Molnar 已提交
1813

1814
	attr_to_bank(attr)->ctl = new;
1815
	mce_restart();
I
Ingo Molnar 已提交
1816

H
Hidetoshi Seto 已提交
1817
	return size;
1818
}
1819

I
Ingo Molnar 已提交
1820 1821
static ssize_t
show_trigger(struct sys_device *s, struct sysdev_attribute *attr, char *buf)
1822
{
1823
	strcpy(buf, mce_helper);
1824
	strcat(buf, "\n");
1825
	return strlen(mce_helper) + 1;
1826 1827
}

1828
static ssize_t set_trigger(struct sys_device *s, struct sysdev_attribute *attr,
I
Ingo Molnar 已提交
1829
				const char *buf, size_t siz)
1830 1831
{
	char *p;
I
Ingo Molnar 已提交
1832

1833 1834 1835
	strncpy(mce_helper, buf, sizeof(mce_helper));
	mce_helper[sizeof(mce_helper)-1] = 0;
	p = strchr(mce_helper, '\n');
I
Ingo Molnar 已提交
1836

1837
	if (p)
I
Ingo Molnar 已提交
1838 1839
		*p = 0;

1840
	return strlen(mce_helper) + !!p;
1841 1842
}

1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888
static ssize_t set_ignore_ce(struct sys_device *s,
			     struct sysdev_attribute *attr,
			     const char *buf, size_t size)
{
	u64 new;

	if (strict_strtoull(buf, 0, &new) < 0)
		return -EINVAL;

	if (mce_ignore_ce ^ !!new) {
		if (new) {
			/* disable ce features */
			on_each_cpu(mce_disable_ce, (void *)1, 1);
			mce_ignore_ce = 1;
		} else {
			/* enable ce features */
			mce_ignore_ce = 0;
			on_each_cpu(mce_enable_ce, (void *)1, 1);
		}
	}
	return size;
}

static ssize_t set_cmci_disabled(struct sys_device *s,
				 struct sysdev_attribute *attr,
				 const char *buf, size_t size)
{
	u64 new;

	if (strict_strtoull(buf, 0, &new) < 0)
		return -EINVAL;

	if (mce_cmci_disabled ^ !!new) {
		if (new) {
			/* disable cmci */
			on_each_cpu(mce_disable_ce, NULL, 1);
			mce_cmci_disabled = 1;
		} else {
			/* enable cmci */
			mce_cmci_disabled = 0;
			on_each_cpu(mce_enable_ce, NULL, 1);
		}
	}
	return size;
}

1889 1890 1891 1892 1893 1894 1895 1896 1897
static ssize_t store_int_with_restart(struct sys_device *s,
				      struct sysdev_attribute *attr,
				      const char *buf, size_t size)
{
	ssize_t ret = sysdev_store_int(s, attr, buf, size);
	mce_restart();
	return ret;
}

1898
static SYSDEV_ATTR(trigger, 0644, show_trigger, set_trigger);
1899
static SYSDEV_INT_ATTR(tolerant, 0644, tolerant);
1900
static SYSDEV_INT_ATTR(monarch_timeout, 0644, monarch_timeout);
1901
static SYSDEV_INT_ATTR(dont_log_ce, 0644, mce_dont_log_ce);
I
Ingo Molnar 已提交
1902

1903 1904 1905 1906 1907
static struct sysdev_ext_attribute attr_check_interval = {
	_SYSDEV_ATTR(check_interval, 0644, sysdev_show_int,
		     store_int_with_restart),
	&check_interval
};
I
Ingo Molnar 已提交
1908

1909 1910 1911 1912 1913 1914
static struct sysdev_ext_attribute attr_ignore_ce = {
	_SYSDEV_ATTR(ignore_ce, 0644, sysdev_show_int, set_ignore_ce),
	&mce_ignore_ce
};

static struct sysdev_ext_attribute attr_cmci_disabled = {
Y
Yinghai Lu 已提交
1915
	_SYSDEV_ATTR(cmci_disabled, 0644, sysdev_show_int, set_cmci_disabled),
1916 1917 1918
	&mce_cmci_disabled
};

1919
static struct sysdev_attribute *mce_sysdev_attrs[] = {
1920 1921 1922
	&attr_tolerant.attr,
	&attr_check_interval.attr,
	&attr_trigger,
1923
	&attr_monarch_timeout.attr,
1924 1925 1926
	&attr_dont_log_ce.attr,
	&attr_ignore_ce.attr,
	&attr_cmci_disabled.attr,
1927 1928
	NULL
};
L
Linus Torvalds 已提交
1929

1930
static cpumask_var_t mce_sysdev_initialized;
1931

I
Ingo Molnar 已提交
1932
/* Per cpu sysdev init. All of the cpus still share the same ctrl bank: */
1933
static __cpuinit int mce_sysdev_create(unsigned int cpu)
L
Linus Torvalds 已提交
1934
{
1935
	struct sys_device *sysdev = &per_cpu(mce_sysdev, cpu);
L
Linus Torvalds 已提交
1936
	int err;
1937
	int i, j;
1938

A
Andreas Herrmann 已提交
1939
	if (!mce_available(&boot_cpu_data))
1940 1941
		return -EIO;

1942 1943
	memset(&sysdev->kobj, 0, sizeof(struct kobject));
	sysdev->id  = cpu;
1944
	sysdev->cls = &mce_sysdev_class;
1945

1946
	err = sysdev_register(sysdev);
1947 1948 1949
	if (err)
		return err;

1950 1951
	for (i = 0; mce_sysdev_attrs[i]; i++) {
		err = sysdev_create_file(sysdev, mce_sysdev_attrs[i]);
1952 1953 1954
		if (err)
			goto error;
	}
1955
	for (j = 0; j < banks; j++) {
1956
		err = sysdev_create_file(sysdev, &mce_banks[j].attr);
1957 1958 1959
		if (err)
			goto error2;
	}
1960
	cpumask_set_cpu(cpu, mce_sysdev_initialized);
1961

1962
	return 0;
1963
error2:
1964
	while (--j >= 0)
1965
		sysdev_remove_file(sysdev, &mce_banks[j].attr);
1966
error:
I
Ingo Molnar 已提交
1967
	while (--i >= 0)
1968
		sysdev_remove_file(sysdev, mce_sysdev_attrs[i]);
I
Ingo Molnar 已提交
1969

1970
	sysdev_unregister(sysdev);
1971

1972 1973 1974
	return err;
}

1975
static __cpuinit void mce_sysdev_remove(unsigned int cpu)
1976
{
1977
	struct sys_device *sysdev = &per_cpu(mce_sysdev, cpu);
1978 1979
	int i;

1980
	if (!cpumask_test_cpu(cpu, mce_sysdev_initialized))
1981 1982
		return;

1983 1984
	for (i = 0; mce_sysdev_attrs[i]; i++)
		sysdev_remove_file(sysdev, mce_sysdev_attrs[i]);
I
Ingo Molnar 已提交
1985

1986
	for (i = 0; i < banks; i++)
1987
		sysdev_remove_file(sysdev, &mce_banks[i].attr);
I
Ingo Molnar 已提交
1988

1989
	sysdev_unregister(sysdev);
1990
	cpumask_clear_cpu(cpu, mce_sysdev_initialized);
1991 1992
}

1993
/* Make sure there are no machine checks on offlined CPUs. */
1994
static void __cpuinit mce_disable_cpu(void *h)
1995
{
A
Andi Kleen 已提交
1996
	unsigned long action = *(unsigned long *)h;
I
Ingo Molnar 已提交
1997
	int i;
1998

1999
	if (!mce_available(__this_cpu_ptr(&cpu_info)))
2000
		return;
2001

A
Andi Kleen 已提交
2002 2003
	if (!(action & CPU_TASKS_FROZEN))
		cmci_clear();
2004
	for (i = 0; i < banks; i++) {
2005
		struct mce_bank *b = &mce_banks[i];
2006

2007
		if (b->init)
2008
			wrmsrl(MSR_IA32_MCx_CTL(i), 0);
2009
	}
2010 2011
}

2012
static void __cpuinit mce_reenable_cpu(void *h)
2013
{
A
Andi Kleen 已提交
2014
	unsigned long action = *(unsigned long *)h;
I
Ingo Molnar 已提交
2015
	int i;
2016

2017
	if (!mce_available(__this_cpu_ptr(&cpu_info)))
2018
		return;
I
Ingo Molnar 已提交
2019

A
Andi Kleen 已提交
2020 2021
	if (!(action & CPU_TASKS_FROZEN))
		cmci_reenable();
2022
	for (i = 0; i < banks; i++) {
2023
		struct mce_bank *b = &mce_banks[i];
2024

2025
		if (b->init)
2026
			wrmsrl(MSR_IA32_MCx_CTL(i), b->ctl);
2027
	}
2028 2029
}

2030
/* Get notified when a cpu comes on/off. Be hotplug friendly. */
I
Ingo Molnar 已提交
2031 2032
static int __cpuinit
mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
2033 2034
{
	unsigned int cpu = (unsigned long)hcpu;
2035
	struct timer_list *t = &per_cpu(mce_timer, cpu);
2036 2037

	switch (action) {
2038 2039
	case CPU_ONLINE:
	case CPU_ONLINE_FROZEN:
2040
		mce_sysdev_create(cpu);
2041 2042
		if (threshold_cpu_callback)
			threshold_cpu_callback(action, cpu);
2043 2044
		break;
	case CPU_DEAD:
2045
	case CPU_DEAD_FROZEN:
2046 2047
		if (threshold_cpu_callback)
			threshold_cpu_callback(action, cpu);
2048
		mce_sysdev_remove(cpu);
2049
		break;
2050 2051 2052
	case CPU_DOWN_PREPARE:
	case CPU_DOWN_PREPARE_FROZEN:
		del_timer_sync(t);
A
Andi Kleen 已提交
2053
		smp_call_function_single(cpu, mce_disable_cpu, &action, 1);
2054 2055 2056
		break;
	case CPU_DOWN_FAILED:
	case CPU_DOWN_FAILED_FROZEN:
2057 2058
		if (!mce_ignore_ce && check_interval) {
			t->expires = round_jiffies(jiffies +
2059
					   __get_cpu_var(mce_next_interval));
2060 2061
			add_timer_on(t, cpu);
		}
A
Andi Kleen 已提交
2062 2063 2064 2065 2066
		smp_call_function_single(cpu, mce_reenable_cpu, &action, 1);
		break;
	case CPU_POST_DEAD:
		/* intentionally ignoring frozen here */
		cmci_rediscover(cpu);
2067
		break;
2068
	}
2069
	return NOTIFY_OK;
2070 2071
}

2072
static struct notifier_block mce_cpu_notifier __cpuinitdata = {
2073 2074 2075
	.notifier_call = mce_cpu_callback,
};

2076
static __init void mce_init_banks(void)
2077 2078 2079 2080
{
	int i;

	for (i = 0; i < banks; i++) {
2081 2082
		struct mce_bank *b = &mce_banks[i];
		struct sysdev_attribute *a = &b->attr;
I
Ingo Molnar 已提交
2083

2084
		sysfs_attr_init(&a->attr);
2085 2086
		a->attr.name	= b->attrname;
		snprintf(b->attrname, ATTR_LEN, "bank%d", i);
I
Ingo Molnar 已提交
2087 2088 2089 2090

		a->attr.mode	= 0644;
		a->show		= show_bank;
		a->store	= set_bank;
2091 2092 2093
	}
}

2094
static __init int mcheck_init_device(void)
2095 2096 2097 2098
{
	int err;
	int i = 0;

L
Linus Torvalds 已提交
2099 2100
	if (!mce_available(&boot_cpu_data))
		return -EIO;
2101

2102
	zalloc_cpumask_var(&mce_sysdev_initialized, GFP_KERNEL);
2103

2104
	mce_init_banks();
2105

2106
	err = sysdev_class_register(&mce_sysdev_class);
2107 2108
	if (err)
		return err;
2109 2110

	for_each_online_cpu(i) {
2111
		err = mce_sysdev_create(i);
2112 2113
		if (err)
			return err;
2114 2115
	}

2116
	register_syscore_ops(&mce_syscore_ops);
2117
	register_hotcpu_notifier(&mce_cpu_notifier);
2118 2119 2120

	/* register character device /dev/mcelog */
	misc_register(&mce_chrdev_device);
I
Ingo Molnar 已提交
2121

L
Linus Torvalds 已提交
2122 2123
	return err;
}
2124
device_initcall(mcheck_init_device);
I
Ingo Molnar 已提交
2125

2126 2127 2128 2129 2130 2131 2132 2133 2134
/*
 * Old style boot options parsing. Only for compatibility.
 */
static int __init mcheck_disable(char *str)
{
	mce_disabled = 1;
	return 1;
}
__setup("nomce", mcheck_disable);
I
Ingo Molnar 已提交
2135

2136 2137
#ifdef CONFIG_DEBUG_FS
struct dentry *mce_get_debugfs_dir(void)
I
Ingo Molnar 已提交
2138
{
2139
	static struct dentry *dmce;
I
Ingo Molnar 已提交
2140

2141 2142
	if (!dmce)
		dmce = debugfs_create_dir("mce", NULL);
I
Ingo Molnar 已提交
2143

2144 2145
	return dmce;
}
I
Ingo Molnar 已提交
2146

2147 2148 2149 2150 2151 2152 2153 2154
static void mce_reset(void)
{
	cpu_missing = 0;
	atomic_set(&mce_fake_paniced, 0);
	atomic_set(&mce_executing, 0);
	atomic_set(&mce_callin, 0);
	atomic_set(&global_nwo, 0);
}
I
Ingo Molnar 已提交
2155

2156 2157 2158 2159
static int fake_panic_get(void *data, u64 *val)
{
	*val = fake_panic;
	return 0;
I
Ingo Molnar 已提交
2160 2161
}

2162
static int fake_panic_set(void *data, u64 val)
I
Ingo Molnar 已提交
2163
{
2164 2165 2166
	mce_reset();
	fake_panic = val;
	return 0;
I
Ingo Molnar 已提交
2167 2168
}

2169 2170
DEFINE_SIMPLE_ATTRIBUTE(fake_panic_fops, fake_panic_get,
			fake_panic_set, "%llu\n");
2171

2172
static int __init mcheck_debugfs_init(void)
2173
{
2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184
	struct dentry *dmce, *ffake_panic;

	dmce = mce_get_debugfs_dir();
	if (!dmce)
		return -ENOMEM;
	ffake_panic = debugfs_create_file("fake_panic", 0444, dmce, NULL,
					  &fake_panic_fops);
	if (!ffake_panic)
		return -ENOMEM;

	return 0;
2185
}
2186
late_initcall(mcheck_debugfs_init);
2187
#endif