mce.c 59.7 KB
Newer Older
L
Linus Torvalds 已提交
1 2
/*
 * Machine check handler.
I
Ingo Molnar 已提交
3
 *
L
Linus Torvalds 已提交
4
 * K8 parts Copyright 2002,2003 Andi Kleen, SuSE Labs.
5 6
 * Rest from unknown author(s).
 * 2004 Andi Kleen. Rewrote most of it.
7 8
 * Copyright 2008 Intel Corporation
 * Author: Andi Kleen
L
Linus Torvalds 已提交
9
 */
10 11 12

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

I
Ingo Molnar 已提交
13 14 15 16 17 18 19
#include <linux/thread_info.h>
#include <linux/capability.h>
#include <linux/miscdevice.h>
#include <linux/ratelimit.h>
#include <linux/kallsyms.h>
#include <linux/rcupdate.h>
#include <linux/kobject.h>
20
#include <linux/uaccess.h>
I
Ingo Molnar 已提交
21 22 23
#include <linux/kdebug.h>
#include <linux/kernel.h>
#include <linux/percpu.h>
L
Linus Torvalds 已提交
24
#include <linux/string.h>
25
#include <linux/device.h>
26
#include <linux/syscore_ops.h>
27
#include <linux/delay.h>
28
#include <linux/ctype.h>
I
Ingo Molnar 已提交
29
#include <linux/sched.h>
30
#include <linux/sysfs.h>
I
Ingo Molnar 已提交
31
#include <linux/types.h>
32
#include <linux/slab.h>
I
Ingo Molnar 已提交
33 34 35
#include <linux/init.h>
#include <linux/kmod.h>
#include <linux/poll.h>
36
#include <linux/nmi.h>
I
Ingo Molnar 已提交
37
#include <linux/cpu.h>
38
#include <linux/smp.h>
I
Ingo Molnar 已提交
39
#include <linux/fs.h>
40
#include <linux/mm.h>
41
#include <linux/debugfs.h>
42
#include <linux/irq_work.h>
43
#include <linux/export.h>
I
Ingo Molnar 已提交
44

45
#include <asm/processor.h>
46
#include <asm/traps.h>
A
Andy Lutomirski 已提交
47
#include <asm/tlbflush.h>
I
Ingo Molnar 已提交
48 49
#include <asm/mce.h>
#include <asm/msr.h>
L
Linus Torvalds 已提交
50

51
#include "mce-internal.h"
52

53
static DEFINE_MUTEX(mce_chrdev_read_mutex);
54

55
#define mce_log_get_idx_check(p) \
56
({ \
57 58
	RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() && \
			 !lockdep_is_held(&mce_chrdev_read_mutex), \
59
			 "suspicious mce_log_get_idx_check() usage"); \
60 61
	smp_load_acquire(&(p)); \
})
62

63 64 65
#define CREATE_TRACE_POINTS
#include <trace/events/mce.h>

66
#define SPINUNIT		100	/* 100ns */
67

68 69
DEFINE_PER_CPU(unsigned, mce_exception_count);

70
struct mce_bank *mce_banks __read_mostly;
71
struct mce_vendor_flags mce_flags __read_mostly;
72

73
struct mca_config mca_cfg __read_mostly = {
74
	.bootlog  = -1,
75 76 77 78 79 80 81
	/*
	 * Tolerant levels:
	 * 0: always panic on uncorrected errors, log corrected errors
	 * 1: panic or SIGBUS on uncorrected errors, log corrected errors
	 * 2: SIGBUS or log uncorrected errors (if possible), log corr. errors
	 * 3: never panic or SIGBUS, log all errors (for testing only)
	 */
82 83
	.tolerant = 1,
	.monarch_timeout = -1
84 85
};

86 87 88 89
/* User mode helper program triggered by machine check event */
static unsigned long		mce_need_notify;
static char			mce_helper[128];
static char			*mce_helper_argv[2] = { mce_helper, NULL };
L
Linus Torvalds 已提交
90

91 92
static DECLARE_WAIT_QUEUE_HEAD(mce_chrdev_wait);

93 94 95
static DEFINE_PER_CPU(struct mce, mces_seen);
static int			cpu_missing;

96 97 98 99
/*
 * MCA banks polled by the period polling timer for corrected events.
 * With Intel CMCI, this only has MCA banks which do not support CMCI (if any).
 */
100 101 102 103
DEFINE_PER_CPU(mce_banks_t, mce_poll_banks) = {
	[0 ... BITS_TO_LONGS(MAX_NR_BANKS)-1] = ~0UL
};

104 105 106 107 108 109 110 111 112
/*
 * MCA banks controlled through firmware first for corrected errors.
 * This is a global list of banks for which we won't enable CMCI and we
 * won't poll. Firmware controls these banks and is responsible for
 * reporting corrected errors through GHES. Uncorrected/recoverable
 * errors are still notified through a machine check.
 */
mce_banks_t mce_banks_ce_disabled;

113 114
static struct work_struct mce_work;
static struct irq_work mce_irq_work;
115

116
static void (*quirk_no_way_out)(int bank, struct mce *m, struct pt_regs *regs);
117
static int mce_usable_address(struct mce *m);
118

119 120 121 122
/*
 * CPU/chipset specific EDAC code can register a notifier call here to print
 * MCE errors in a human-readable form.
 */
123
ATOMIC_NOTIFIER_HEAD(x86_mce_decoder_chain);
124

125 126 127 128
/* Do initial initialization of a struct mce */
void mce_setup(struct mce *m)
{
	memset(m, 0, sizeof(struct mce));
129
	m->cpu = m->extcpu = smp_processor_id();
130
	m->tsc = rdtsc();
131 132 133 134 135 136 137
	/* We hope get_seconds stays lockless */
	m->time = get_seconds();
	m->cpuvendor = boot_cpu_data.x86_vendor;
	m->cpuid = cpuid_eax(1);
	m->socketid = cpu_data(m->extcpu).phys_proc_id;
	m->apicid = cpu_data(m->extcpu).initial_apicid;
	rdmsrl(MSR_IA32_MCG_CAP, m->mcgcap);
138 139
}

140 141 142
DEFINE_PER_CPU(struct mce, injectm);
EXPORT_PER_CPU_SYMBOL_GPL(injectm);

L
Linus Torvalds 已提交
143 144 145 146 147 148
/*
 * Lockless MCE logging infrastructure.
 * This avoids deadlocks on printk locks without having to break locks. Also
 * separate MCEs from kernel messages to avoid bogus bug reports.
 */

149
static struct mce_log mcelog = {
150 151 152
	.signature	= MCE_LOG_SIGNATURE,
	.len		= MCE_LOG_LEN,
	.recordlen	= sizeof(struct mce),
153
};
L
Linus Torvalds 已提交
154 155 156 157

void mce_log(struct mce *mce)
{
	unsigned next, entry;
I
Ingo Molnar 已提交
158

159 160 161
	/* Emit the trace record: */
	trace_mce_record(mce);

162 163
	if (!mce_gen_pool_add(mce))
		irq_work_queue(&mce_irq_work);
164

L
Linus Torvalds 已提交
165
	mce->finished = 0;
M
Mike Waychison 已提交
166
	wmb();
L
Linus Torvalds 已提交
167
	for (;;) {
168
		entry = mce_log_get_idx_check(mcelog.next);
169
		for (;;) {
170

I
Ingo Molnar 已提交
171 172 173 174 175
			/*
			 * When the buffer fills up discard new entries.
			 * Assume that the earlier errors are the more
			 * interesting ones:
			 */
176
			if (entry >= MCE_LOG_LEN) {
177 178
				set_bit(MCE_OVERFLOW,
					(unsigned long *)&mcelog.flags);
179 180
				return;
			}
I
Ingo Molnar 已提交
181
			/* Old left over entry. Skip: */
182 183 184 185
			if (mcelog.entry[entry].finished) {
				entry++;
				continue;
			}
M
Mike Waychison 已提交
186
			break;
L
Linus Torvalds 已提交
187 188 189 190 191 192 193
		}
		smp_rmb();
		next = entry + 1;
		if (cmpxchg(&mcelog.next, entry, next) == entry)
			break;
	}
	memcpy(mcelog.entry + entry, mce, sizeof(struct mce));
M
Mike Waychison 已提交
194
	wmb();
L
Linus Torvalds 已提交
195
	mcelog.entry[entry].finished = 1;
M
Mike Waychison 已提交
196
	wmb();
L
Linus Torvalds 已提交
197

198
	mce->finished = 1;
199
	set_bit(0, &mce_need_notify);
L
Linus Torvalds 已提交
200 201
}

202
void mce_inject_log(struct mce *m)
B
Borislav Petkov 已提交
203
{
204 205 206
	mutex_lock(&mce_chrdev_read_mutex);
	mce_log(m);
	mutex_unlock(&mce_chrdev_read_mutex);
B
Borislav Petkov 已提交
207
}
208
EXPORT_SYMBOL_GPL(mce_inject_log);
B
Borislav Petkov 已提交
209

210
static struct notifier_block mce_srao_nb;
B
Borislav Petkov 已提交
211

212 213
void mce_register_decode_chain(struct notifier_block *nb)
{
214 215 216 217
	/* Ensure SRAO notifier has the highest priority in the decode chain. */
	if (nb != &mce_srao_nb && nb->priority == INT_MAX)
		nb->priority -= 1;

218 219 220 221 222 223 224 225 226 227
	atomic_notifier_chain_register(&x86_mce_decoder_chain, nb);
}
EXPORT_SYMBOL_GPL(mce_register_decode_chain);

void mce_unregister_decode_chain(struct notifier_block *nb)
{
	atomic_notifier_chain_unregister(&x86_mce_decoder_chain, nb);
}
EXPORT_SYMBOL_GPL(mce_unregister_decode_chain);

H
Hidetoshi Seto 已提交
228
static void print_mce(struct mce *m)
L
Linus Torvalds 已提交
229
{
230 231
	int ret = 0;

H
Huang Ying 已提交
232
	pr_emerg(HW_ERR "CPU %d: Machine Check Exception: %Lx Bank %d: %016Lx\n",
233
	       m->extcpu, m->mcgstatus, m->bank, m->status);
234

235
	if (m->ip) {
H
Huang Ying 已提交
236
		pr_emerg(HW_ERR "RIP%s %02x:<%016Lx> ",
237 238 239
			!(m->mcgstatus & MCG_STATUS_EIPV) ? " !INEXACT!" : "",
				m->cs, m->ip);

L
Linus Torvalds 已提交
240
		if (m->cs == __KERNEL_CS)
241
			print_symbol("{%s}", m->ip);
242
		pr_cont("\n");
L
Linus Torvalds 已提交
243
	}
244

H
Huang Ying 已提交
245
	pr_emerg(HW_ERR "TSC %llx ", m->tsc);
L
Linus Torvalds 已提交
246
	if (m->addr)
247
		pr_cont("ADDR %llx ", m->addr);
L
Linus Torvalds 已提交
248
	if (m->misc)
249
		pr_cont("MISC %llx ", m->misc);
250

251
	pr_cont("\n");
252 253 254 255
	/*
	 * Note this output is parsed by external tools and old fields
	 * should not be changed.
	 */
256
	pr_emerg(HW_ERR "PROCESSOR %u:%x TIME %llu SOCKET %u APIC %x microcode %x\n",
257 258
		m->cpuvendor, m->cpuid, m->time, m->socketid, m->apicid,
		cpu_data(m->extcpu).microcode);
259 260 261

	/*
	 * Print out human-readable details about the MCE error,
262
	 * (if the CPU has an implementation for that)
263
	 */
264 265 266 267 268
	ret = atomic_notifier_call_chain(&x86_mce_decoder_chain, 0, m);
	if (ret == NOTIFY_STOP)
		return;

	pr_emerg_ratelimited(HW_ERR "Run the above through 'mcelog --ascii'\n");
269 270
}

271 272
#define PANIC_TIMEOUT 5 /* 5 seconds */

273
static atomic_t mce_panicked;
274

275
static int fake_panic;
276
static atomic_t mce_fake_panicked;
277

278 279 280 281
/* Panic in progress. Enable interrupts and wait for final IPI */
static void wait_for_panic(void)
{
	long timeout = PANIC_TIMEOUT*USEC_PER_SEC;
282

283 284 285 286
	preempt_disable();
	local_irq_enable();
	while (timeout-- > 0)
		udelay(1);
287
	if (panic_timeout == 0)
288
		panic_timeout = mca_cfg.panic_timeout;
289 290 291
	panic("Panicing machine check CPU died");
}

292
static void mce_panic(const char *msg, struct mce *final, char *exp)
293
{
294
	int i, apei_err = 0;
295

296 297 298 299
	if (!fake_panic) {
		/*
		 * Make sure only one CPU runs in machine check panic
		 */
300
		if (atomic_inc_return(&mce_panicked) > 1)
301 302
			wait_for_panic();
		barrier();
303

304 305 306 307
		bust_spinlocks(1);
		console_verbose();
	} else {
		/* Don't log too much for fake panic */
308
		if (atomic_inc_return(&mce_fake_panicked) > 1)
309 310
			return;
	}
311
	/* First print corrected ones that are still unlogged */
L
Linus Torvalds 已提交
312
	for (i = 0; i < MCE_LOG_LEN; i++) {
313
		struct mce *m = &mcelog.entry[i];
H
Hidetoshi Seto 已提交
314 315
		if (!(m->status & MCI_STATUS_VAL))
			continue;
316
		if (!(m->status & MCI_STATUS_UC)) {
H
Hidetoshi Seto 已提交
317
			print_mce(m);
318 319 320
			if (!apei_err)
				apei_err = apei_write_mce(m);
		}
321 322 323 324 325
	}
	/* Now print uncorrected but with the final one last */
	for (i = 0; i < MCE_LOG_LEN; i++) {
		struct mce *m = &mcelog.entry[i];
		if (!(m->status & MCI_STATUS_VAL))
L
Linus Torvalds 已提交
326
			continue;
H
Hidetoshi Seto 已提交
327 328
		if (!(m->status & MCI_STATUS_UC))
			continue;
329
		if (!final || memcmp(m, final, sizeof(struct mce))) {
H
Hidetoshi Seto 已提交
330
			print_mce(m);
331 332 333
			if (!apei_err)
				apei_err = apei_write_mce(m);
		}
L
Linus Torvalds 已提交
334
	}
335
	if (final) {
H
Hidetoshi Seto 已提交
336
		print_mce(final);
337 338 339
		if (!apei_err)
			apei_err = apei_write_mce(final);
	}
340
	if (cpu_missing)
H
Huang Ying 已提交
341
		pr_emerg(HW_ERR "Some CPUs didn't answer in synchronization\n");
342
	if (exp)
H
Huang Ying 已提交
343
		pr_emerg(HW_ERR "Machine check: %s\n", exp);
344 345
	if (!fake_panic) {
		if (panic_timeout == 0)
346
			panic_timeout = mca_cfg.panic_timeout;
347 348
		panic(msg);
	} else
H
Huang Ying 已提交
349
		pr_emerg(HW_ERR "Fake kernel panic: %s\n", msg);
350
}
L
Linus Torvalds 已提交
351

352 353 354 355
/* Support code for software error injection */

static int msr_to_offset(u32 msr)
{
T
Tejun Heo 已提交
356
	unsigned bank = __this_cpu_read(injectm.bank);
357

358
	if (msr == mca_cfg.rip_msr)
359
		return offsetof(struct mce, ip);
360
	if (msr == MSR_IA32_MCx_STATUS(bank))
361
		return offsetof(struct mce, status);
362
	if (msr == MSR_IA32_MCx_ADDR(bank))
363
		return offsetof(struct mce, addr);
364
	if (msr == MSR_IA32_MCx_MISC(bank))
365 366 367 368 369 370
		return offsetof(struct mce, misc);
	if (msr == MSR_IA32_MCG_STATUS)
		return offsetof(struct mce, mcgstatus);
	return -1;
}

371 372 373 374
/* MSR access wrappers used for error injection */
static u64 mce_rdmsrl(u32 msr)
{
	u64 v;
375

T
Tejun Heo 已提交
376
	if (__this_cpu_read(injectm.finished)) {
377
		int offset = msr_to_offset(msr);
378

379 380
		if (offset < 0)
			return 0;
381
		return *(u64 *)((char *)this_cpu_ptr(&injectm) + offset);
382
	}
383 384 385 386 387 388 389 390 391 392 393

	if (rdmsrl_safe(msr, &v)) {
		WARN_ONCE(1, "mce: Unable to read msr %d!\n", msr);
		/*
		 * Return zero in case the access faulted. This should
		 * not happen normally but can happen if the CPU does
		 * something weird, or if the code is buggy.
		 */
		v = 0;
	}

394 395 396 397 398
	return v;
}

static void mce_wrmsrl(u32 msr, u64 v)
{
T
Tejun Heo 已提交
399
	if (__this_cpu_read(injectm.finished)) {
400
		int offset = msr_to_offset(msr);
401

402
		if (offset >= 0)
403
			*(u64 *)((char *)this_cpu_ptr(&injectm) + offset) = v;
404 405
		return;
	}
406 407 408
	wrmsrl(msr, v);
}

409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426
/*
 * Collect all global (w.r.t. this processor) status about this machine
 * check into our "mce" struct so that we can use it later to assess
 * the severity of the problem as we read per-bank specific details.
 */
static inline void mce_gather_info(struct mce *m, struct pt_regs *regs)
{
	mce_setup(m);

	m->mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS);
	if (regs) {
		/*
		 * Get the address of the instruction at the time of
		 * the machine check error.
		 */
		if (m->mcgstatus & (MCG_STATUS_RIPV|MCG_STATUS_EIPV)) {
			m->ip = regs->ip;
			m->cs = regs->cs;
427 428 429 430 431 432 433 434

			/*
			 * When in VM86 mode make the cs look like ring 3
			 * always. This is a lie, but it's better than passing
			 * the additional vm86 bit around everywhere.
			 */
			if (v8086_mode(regs))
				m->cs |= 3;
435 436
		}
		/* Use accurate RIP reporting if available. */
437 438
		if (mca_cfg.rip_msr)
			m->ip = mce_rdmsrl(mca_cfg.rip_msr);
439 440 441
	}
}

A
Andi Kleen 已提交
442
int mce_available(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
443
{
444
	if (mca_cfg.disabled)
445
		return 0;
446
	return cpu_has(c, X86_FEATURE_MCE) && cpu_has(c, X86_FEATURE_MCA);
L
Linus Torvalds 已提交
447 448
}

449 450
static void mce_schedule_work(void)
{
451
	if (!mce_gen_pool_empty() && keventd_up())
452
		schedule_work(&mce_work);
453 454
}

455
static void mce_irq_work_cb(struct irq_work *entry)
456
{
457
	mce_notify_irq();
458
	mce_schedule_work();
459 460 461 462 463
}

static void mce_report_event(struct pt_regs *regs)
{
	if (regs->flags & (X86_VM_MASK|X86_EFLAGS_IF)) {
464
		mce_notify_irq();
465 466 467 468 469 470 471
		/*
		 * Triggering the work queue here is just an insurance
		 * policy in case the syscall exit notify handler
		 * doesn't run soon enough or ends up running on the
		 * wrong CPU (can happen when audit sleeps)
		 */
		mce_schedule_work();
472 473 474
		return;
	}

475
	irq_work_queue(&mce_irq_work);
476 477
}

478 479 480 481 482 483 484 485 486
static int srao_decode_notifier(struct notifier_block *nb, unsigned long val,
				void *data)
{
	struct mce *mce = (struct mce *)data;
	unsigned long pfn;

	if (!mce)
		return NOTIFY_DONE;

B
Borislav Petkov 已提交
487
	if (mce_usable_address(mce) && (mce->severity == MCE_AO_SEVERITY)) {
488 489 490 491 492
		pfn = mce->addr >> PAGE_SHIFT;
		memory_failure(pfn, MCE_VECTOR, 0);
	}

	return NOTIFY_OK;
493
}
494 495 496 497
static struct notifier_block mce_srao_nb = {
	.notifier_call	= srao_decode_notifier,
	.priority = INT_MAX,
};
498

499 500 501 502 503 504 505 506 507 508 509 510 511
/*
 * Read ADDR and MISC registers.
 */
static void mce_read_aux(struct mce *m, int i)
{
	if (m->status & MCI_STATUS_MISCV)
		m->misc = mce_rdmsrl(MSR_IA32_MCx_MISC(i));
	if (m->status & MCI_STATUS_ADDRV) {
		m->addr = mce_rdmsrl(MSR_IA32_MCx_ADDR(i));

		/*
		 * Mask the reported address by the reported granularity.
		 */
512
		if (mca_cfg.ser && (m->status & MCI_STATUS_MISCV)) {
513 514 515 516 517 518 519
			u8 shift = MCI_MISC_ADDR_LSB(m->misc);
			m->addr >>= shift;
			m->addr <<= shift;
		}
	}
}

520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550
static bool memory_error(struct mce *m)
{
	struct cpuinfo_x86 *c = &boot_cpu_data;

	if (c->x86_vendor == X86_VENDOR_AMD) {
		/*
		 * coming soon
		 */
		return false;
	} else if (c->x86_vendor == X86_VENDOR_INTEL) {
		/*
		 * Intel SDM Volume 3B - 15.9.2 Compound Error Codes
		 *
		 * Bit 7 of the MCACOD field of IA32_MCi_STATUS is used for
		 * indicating a memory error. Bit 8 is used for indicating a
		 * cache hierarchy error. The combination of bit 2 and bit 3
		 * is used for indicating a `generic' cache hierarchy error
		 * But we can't just blindly check the above bits, because if
		 * bit 11 is set, then it is a bus/interconnect error - and
		 * either way the above bits just gives more detail on what
		 * bus/interconnect error happened. Note that bit 12 can be
		 * ignored, as it's the "filter" bit.
		 */
		return (m->status & 0xef80) == BIT(7) ||
		       (m->status & 0xef00) == BIT(8) ||
		       (m->status & 0xeffc) == 0xc;
	}

	return false;
}

551 552
DEFINE_PER_CPU(unsigned, mce_poll_count);

553
/*
554 555 556 557
 * Poll for corrected events or events that happened before reset.
 * Those are just logged through /dev/mcelog.
 *
 * This is executed in standard interrupt context.
A
Andi Kleen 已提交
558 559 560 561 562 563 564 565 566
 *
 * Note: spec recommends to panic for fatal unsignalled
 * errors here. However this would be quite problematic --
 * we would need to reimplement the Monarch handling and
 * it would mess up the exclusion between exception handler
 * and poll hander -- * so we skip this for now.
 * These cases should not happen anyways, or only when the CPU
 * is already totally * confused. In this case it's likely it will
 * not fully execute the machine check handler either.
567
 */
568
bool machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
569
{
570
	bool error_seen = false;
571
	struct mce m;
572
	int severity;
573 574
	int i;

575
	this_cpu_inc(mce_poll_count);
576

577
	mce_gather_info(&m, NULL);
578

579
	for (i = 0; i < mca_cfg.banks; i++) {
580
		if (!mce_banks[i].ctl || !test_bit(i, *b))
581 582 583 584 585 586 587 588
			continue;

		m.misc = 0;
		m.addr = 0;
		m.bank = i;
		m.tsc = 0;

		barrier();
589
		m.status = mce_rdmsrl(MSR_IA32_MCx_STATUS(i));
590 591 592
		if (!(m.status & MCI_STATUS_VAL))
			continue;

593

594
		/*
A
Andi Kleen 已提交
595 596
		 * Uncorrected or signalled events are handled by the exception
		 * handler when it is enabled, so don't process those here.
597 598 599
		 *
		 * TBD do the same check for MCI_STATUS_EN here?
		 */
A
Andi Kleen 已提交
600
		if (!(flags & MCP_UC) &&
601
		    (m.status & (mca_cfg.ser ? MCI_STATUS_S : MCI_STATUS_UC)))
602 603
			continue;

604 605
		error_seen = true;

606
		mce_read_aux(&m, i);
607 608 609

		if (!(flags & MCP_TIMESTAMP))
			m.tsc = 0;
610 611 612

		severity = mce_severity(&m, mca_cfg.tolerant, NULL, false);

B
Borislav Petkov 已提交
613 614
		if (severity == MCE_DEFERRED_SEVERITY && memory_error(&m))
			if (m.status & MCI_STATUS_ADDRV)
615
				m.severity = severity;
616

617 618 619 620
		/*
		 * Don't get the IP here because it's unlikely to
		 * have anything to do with the actual error location.
		 */
621
		if (!(flags & MCP_DONTLOG) && !mca_cfg.dont_log_ce)
A
Andi Kleen 已提交
622
			mce_log(&m);
B
Borislav Petkov 已提交
623
		else if (mce_usable_address(&m)) {
624 625 626 627 628 629 630
			/*
			 * Although we skipped logging this, we still want
			 * to take action. Add to the pool so the registered
			 * notifiers will see it.
			 */
			if (!mce_gen_pool_add(&m))
				mce_schedule_work();
631
		}
632 633 634 635

		/*
		 * Clear state for this bank.
		 */
636
		mce_wrmsrl(MSR_IA32_MCx_STATUS(i), 0);
637 638 639 640 641 642
	}

	/*
	 * Don't clear MCG_STATUS here because it's only defined for
	 * exceptions.
	 */
643 644

	sync_core();
645

646
	return error_seen;
647
}
648
EXPORT_SYMBOL_GPL(machine_check_poll);
649

650 651 652 653
/*
 * Do a quick check if any of the events requires a panic.
 * This decides if we keep the events around or clear them.
 */
654 655
static int mce_no_way_out(struct mce *m, char **msg, unsigned long *validp,
			  struct pt_regs *regs)
656
{
657
	int i, ret = 0;
658
	char *tmp;
659

660
	for (i = 0; i < mca_cfg.banks; i++) {
661
		m->status = mce_rdmsrl(MSR_IA32_MCx_STATUS(i));
662
		if (m->status & MCI_STATUS_VAL) {
663
			__set_bit(i, validp);
664 665 666
			if (quirk_no_way_out)
				quirk_no_way_out(i, m, regs);
		}
667 668 669

		if (mce_severity(m, mca_cfg.tolerant, &tmp, true) >= MCE_PANIC_SEVERITY) {
			*msg = tmp;
670
			ret = 1;
671
		}
672
	}
673
	return ret;
674 675
}

676 677 678 679 680 681 682 683 684 685 686 687 688 689
/*
 * Variable to establish order between CPUs while scanning.
 * Each CPU spins initially until executing is equal its number.
 */
static atomic_t mce_executing;

/*
 * Defines order of CPUs on entry. First CPU becomes Monarch.
 */
static atomic_t mce_callin;

/*
 * Check if a timeout waiting for other CPUs happened.
 */
690
static int mce_timed_out(u64 *t, const char *msg)
691 692 693 694 695 696 697 698
{
	/*
	 * The others already did panic for some reason.
	 * Bail out like in a timeout.
	 * rmb() to tell the compiler that system_state
	 * might have been modified by someone else.
	 */
	rmb();
699
	if (atomic_read(&mce_panicked))
700
		wait_for_panic();
701
	if (!mca_cfg.monarch_timeout)
702 703
		goto out;
	if ((s64)*t < SPINUNIT) {
704
		if (mca_cfg.tolerant <= 1)
705
			mce_panic(msg, NULL, NULL);
706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726
		cpu_missing = 1;
		return 1;
	}
	*t -= SPINUNIT;
out:
	touch_nmi_watchdog();
	return 0;
}

/*
 * The Monarch's reign.  The Monarch is the CPU who entered
 * the machine check handler first. It waits for the others to
 * raise the exception too and then grades them. When any
 * error is fatal panic. Only then let the others continue.
 *
 * The other CPUs entering the MCE handler will be controlled by the
 * Monarch. They are called Subjects.
 *
 * This way we prevent any potential data corruption in a unrecoverable case
 * and also makes sure always all CPU's errors are examined.
 *
727
 * Also this detects the case of a machine check event coming from outer
728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752
 * space (not detected by any CPUs) In this case some external agent wants
 * us to shut down, so panic too.
 *
 * The other CPUs might still decide to panic if the handler happens
 * in a unrecoverable place, but in this case the system is in a semi-stable
 * state and won't corrupt anything by itself. It's ok to let the others
 * continue for a bit first.
 *
 * All the spin loops have timeouts; when a timeout happens a CPU
 * typically elects itself to be Monarch.
 */
static void mce_reign(void)
{
	int cpu;
	struct mce *m = NULL;
	int global_worst = 0;
	char *msg = NULL;
	char *nmsg = NULL;

	/*
	 * This CPU is the Monarch and the other CPUs have run
	 * through their handlers.
	 * Grade the severity of the errors of all the CPUs.
	 */
	for_each_possible_cpu(cpu) {
753 754
		int severity = mce_severity(&per_cpu(mces_seen, cpu),
					    mca_cfg.tolerant,
755
					    &nmsg, true);
756 757 758 759 760 761 762 763 764 765 766 767
		if (severity > global_worst) {
			msg = nmsg;
			global_worst = severity;
			m = &per_cpu(mces_seen, cpu);
		}
	}

	/*
	 * Cannot recover? Panic here then.
	 * This dumps all the mces in the log buffer and stops the
	 * other CPUs.
	 */
768
	if (m && global_worst >= MCE_PANIC_SEVERITY && mca_cfg.tolerant < 3)
769
		mce_panic("Fatal machine check", m, msg);
770 771 772 773 774 775 776 777 778 779 780

	/*
	 * For UC somewhere we let the CPU who detects it handle it.
	 * Also must let continue the others, otherwise the handling
	 * CPU could deadlock on a lock.
	 */

	/*
	 * No machine check event found. Must be some external
	 * source or one CPU is hung. Panic.
	 */
781
	if (global_worst <= MCE_KEEP_SEVERITY && mca_cfg.tolerant < 3)
782
		mce_panic("Fatal machine check from unknown source", NULL, NULL);
783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800

	/*
	 * Now clear all the mces_seen so that they don't reappear on
	 * the next mce.
	 */
	for_each_possible_cpu(cpu)
		memset(&per_cpu(mces_seen, cpu), 0, sizeof(struct mce));
}

static atomic_t global_nwo;

/*
 * Start of Monarch synchronization. This waits until all CPUs have
 * entered the exception handler and then determines if any of them
 * saw a fatal event that requires panic. Then it executes them
 * in the entry order.
 * TBD double check parallel CPU hotunplug
 */
H
Hidetoshi Seto 已提交
801
static int mce_start(int *no_way_out)
802
{
H
Hidetoshi Seto 已提交
803
	int order;
804
	int cpus = num_online_cpus();
805
	u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC;
806

H
Hidetoshi Seto 已提交
807 808
	if (!timeout)
		return -1;
809

H
Hidetoshi Seto 已提交
810
	atomic_add(*no_way_out, &global_nwo);
811 812 813 814
	/*
	 * global_nwo should be updated before mce_callin
	 */
	smp_wmb();
815
	order = atomic_inc_return(&mce_callin);
816 817 818 819 820

	/*
	 * Wait for everyone.
	 */
	while (atomic_read(&mce_callin) != cpus) {
821 822
		if (mce_timed_out(&timeout,
				  "Timeout: Not all CPUs entered broadcast exception handler")) {
823
			atomic_set(&global_nwo, 0);
H
Hidetoshi Seto 已提交
824
			return -1;
825 826 827 828
		}
		ndelay(SPINUNIT);
	}

829 830 831 832
	/*
	 * mce_callin should be read before global_nwo
	 */
	smp_rmb();
833

H
Hidetoshi Seto 已提交
834 835 836 837
	if (order == 1) {
		/*
		 * Monarch: Starts executing now, the others wait.
		 */
838
		atomic_set(&mce_executing, 1);
H
Hidetoshi Seto 已提交
839 840 841 842 843 844 845 846
	} else {
		/*
		 * Subject: Now start the scanning loop one by one in
		 * the original callin order.
		 * This way when there are any shared banks it will be
		 * only seen by one CPU before cleared, avoiding duplicates.
		 */
		while (atomic_read(&mce_executing) < order) {
847 848
			if (mce_timed_out(&timeout,
					  "Timeout: Subject CPUs unable to finish machine check processing")) {
H
Hidetoshi Seto 已提交
849 850 851 852 853
				atomic_set(&global_nwo, 0);
				return -1;
			}
			ndelay(SPINUNIT);
		}
854 855 856
	}

	/*
H
Hidetoshi Seto 已提交
857
	 * Cache the global no_way_out state.
858
	 */
H
Hidetoshi Seto 已提交
859 860 861
	*no_way_out = atomic_read(&global_nwo);

	return order;
862 863 864 865 866 867 868 869 870
}

/*
 * Synchronize between CPUs after main scanning loop.
 * This invokes the bulk of the Monarch processing.
 */
static int mce_end(int order)
{
	int ret = -1;
871
	u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC;
872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891

	if (!timeout)
		goto reset;
	if (order < 0)
		goto reset;

	/*
	 * Allow others to run.
	 */
	atomic_inc(&mce_executing);

	if (order == 1) {
		/* CHECKME: Can this race with a parallel hotplug? */
		int cpus = num_online_cpus();

		/*
		 * Monarch: Wait for everyone to go through their scanning
		 * loops.
		 */
		while (atomic_read(&mce_executing) <= cpus) {
892 893
			if (mce_timed_out(&timeout,
					  "Timeout: Monarch CPU unable to finish machine check processing"))
894 895 896 897 898 899 900 901 902 903 904 905
				goto reset;
			ndelay(SPINUNIT);
		}

		mce_reign();
		barrier();
		ret = 0;
	} else {
		/*
		 * Subject: Wait for Monarch to finish.
		 */
		while (atomic_read(&mce_executing) != 0) {
906 907
			if (mce_timed_out(&timeout,
					  "Timeout: Monarch CPU did not finish machine check processing"))
908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932
				goto reset;
			ndelay(SPINUNIT);
		}

		/*
		 * Don't reset anything. That's done by the Monarch.
		 */
		return 0;
	}

	/*
	 * Reset all global state.
	 */
reset:
	atomic_set(&global_nwo, 0);
	atomic_set(&mce_callin, 0);
	barrier();

	/*
	 * Let others run again.
	 */
	atomic_set(&mce_executing, 0);
	return ret;
}

933 934 935 936
/*
 * Check if the address reported by the CPU is in a format we can parse.
 * It would be possible to add code for most other cases, but all would
 * be somewhat complicated (e.g. segment offset would require an instruction
L
Lucas De Marchi 已提交
937
 * parser). So only support physical addresses up to page granuality for now.
938 939 940 941 942
 */
static int mce_usable_address(struct mce *m)
{
	if (!(m->status & MCI_STATUS_MISCV) || !(m->status & MCI_STATUS_ADDRV))
		return 0;
943
	if (MCI_MISC_ADDR_LSB(m->misc) > PAGE_SHIFT)
944
		return 0;
945
	if (MCI_MISC_ADDR_MODE(m->misc) != MCI_MISC_ADDR_PHYS)
946 947 948 949
		return 0;
	return 1;
}

950 951 952 953
static void mce_clear_state(unsigned long *toclear)
{
	int i;

954
	for (i = 0; i < mca_cfg.banks; i++) {
955
		if (test_bit(i, toclear))
956
			mce_wrmsrl(MSR_IA32_MCx_STATUS(i), 0);
957 958 959
	}
}

960 961 962 963 964 965 966
/*
 * The actual machine check handler. This only handles real
 * exceptions when something got corrupted coming in through int 18.
 *
 * This is executed in NMI context not subject to normal locking rules. This
 * implies that most kernel services cannot be safely used. Don't even
 * think about putting a printk in there!
967 968 969 970
 *
 * On Intel systems this is entered on all CPUs in parallel through
 * MCE broadcast. However some CPUs might be broken beyond repair,
 * so be always careful when synchronizing with others.
L
Linus Torvalds 已提交
971
 */
I
Ingo Molnar 已提交
972
void do_machine_check(struct pt_regs *regs, long error_code)
L
Linus Torvalds 已提交
973
{
974
	struct mca_config *cfg = &mca_cfg;
975
	struct mce m, *final;
L
Linus Torvalds 已提交
976
	int i;
977 978 979 980 981 982
	int worst = 0;
	int severity;
	/*
	 * Establish sequential order between the CPUs entering the machine
	 * check handler.
	 */
H
Hidetoshi Seto 已提交
983
	int order;
984 985
	/*
	 * If no_way_out gets set, there is no safe way to recover from this
986
	 * MCE.  If mca_cfg.tolerant is cranked up, we'll try anyway.
987 988 989 990 991 992 993
	 */
	int no_way_out = 0;
	/*
	 * If kill_it gets set, there might be a way to recover from this
	 * error.
	 */
	int kill_it = 0;
994
	DECLARE_BITMAP(toclear, MAX_NR_BANKS);
995
	DECLARE_BITMAP(valid_banks, MAX_NR_BANKS);
996
	char *msg = "Unknown";
997 998
	u64 recover_paddr = ~0ull;
	int flags = MF_ACTION_REQUIRED;
A
Ashok Raj 已提交
999
	int lmce = 0;
L
Linus Torvalds 已提交
1000

1001
	ist_enter(regs);
1002

1003
	this_cpu_inc(mce_exception_count);
1004

1005
	if (!cfg->banks)
1006
		goto out;
L
Linus Torvalds 已提交
1007

1008
	mce_gather_info(&m, regs);
1009

1010
	final = this_cpu_ptr(&mces_seen);
1011 1012
	*final = m;

1013
	memset(valid_banks, 0, sizeof(valid_banks));
1014
	no_way_out = mce_no_way_out(&m, &msg, valid_banks, regs);
1015

L
Linus Torvalds 已提交
1016 1017
	barrier();

A
Andi Kleen 已提交
1018
	/*
1019 1020 1021
	 * When no restart IP might need to kill or panic.
	 * Assume the worst for now, but if we find the
	 * severity is MCE_AR_SEVERITY we have other options.
A
Andi Kleen 已提交
1022 1023 1024 1025
	 */
	if (!(m.mcgstatus & MCG_STATUS_RIPV))
		kill_it = 1;

1026
	/*
A
Ashok Raj 已提交
1027
	 * Check if this MCE is signaled to only this logical processor
1028
	 */
A
Ashok Raj 已提交
1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040
	if (m.mcgstatus & MCG_STATUS_LMCES)
		lmce = 1;
	else {
		/*
		 * Go through all the banks in exclusion of the other CPUs.
		 * This way we don't report duplicated events on shared banks
		 * because the first one to see it will clear it.
		 * If this is a Local MCE, then no need to perform rendezvous.
		 */
		order = mce_start(&no_way_out);
	}

1041
	for (i = 0; i < cfg->banks; i++) {
1042
		__clear_bit(i, toclear);
1043 1044
		if (!test_bit(i, valid_banks))
			continue;
1045
		if (!mce_banks[i].ctl)
L
Linus Torvalds 已提交
1046
			continue;
1047 1048

		m.misc = 0;
L
Linus Torvalds 已提交
1049 1050 1051
		m.addr = 0;
		m.bank = i;

1052
		m.status = mce_rdmsrl(MSR_IA32_MCx_STATUS(i));
L
Linus Torvalds 已提交
1053 1054 1055
		if ((m.status & MCI_STATUS_VAL) == 0)
			continue;

1056
		/*
A
Andi Kleen 已提交
1057 1058
		 * Non uncorrected or non signaled errors are handled by
		 * machine_check_poll. Leave them alone, unless this panics.
1059
		 */
1060
		if (!(m.status & (cfg->ser ? MCI_STATUS_S : MCI_STATUS_UC)) &&
A
Andi Kleen 已提交
1061
			!no_way_out)
1062 1063 1064 1065 1066
			continue;

		/*
		 * Set taint even when machine check was not enabled.
		 */
1067
		add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE);
1068

1069
		severity = mce_severity(&m, cfg->tolerant, NULL, true);
1070

A
Andi Kleen 已提交
1071
		/*
1072 1073
		 * When machine check was for corrected/deferred handler don't
		 * touch, unless we're panicing.
A
Andi Kleen 已提交
1074
		 */
1075 1076
		if ((severity == MCE_KEEP_SEVERITY ||
		     severity == MCE_UCNA_SEVERITY) && !no_way_out)
A
Andi Kleen 已提交
1077 1078 1079
			continue;
		__set_bit(i, toclear);
		if (severity == MCE_NO_SEVERITY) {
1080 1081 1082 1083 1084
			/*
			 * Machine check event was not enabled. Clear, but
			 * ignore.
			 */
			continue;
L
Linus Torvalds 已提交
1085 1086
		}

1087
		mce_read_aux(&m, i);
L
Linus Torvalds 已提交
1088

1089 1090
		/* assuming valid severity level != 0 */
		m.severity = severity;
1091

1092
		mce_log(&m);
L
Linus Torvalds 已提交
1093

1094 1095 1096
		if (severity > worst) {
			*final = m;
			worst = severity;
L
Linus Torvalds 已提交
1097 1098 1099
		}
	}

1100 1101 1102
	/* mce_clear_state will clear *final, save locally for use later */
	m = *final;

1103 1104 1105
	if (!no_way_out)
		mce_clear_state(toclear);

I
Ingo Molnar 已提交
1106
	/*
1107 1108
	 * Do most of the synchronization with other CPUs.
	 * When there's any problem use only local no_way_out state.
I
Ingo Molnar 已提交
1109
	 */
A
Ashok Raj 已提交
1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121
	if (!lmce) {
		if (mce_end(order) < 0)
			no_way_out = worst >= MCE_PANIC_SEVERITY;
	} else {
		/*
		 * Local MCE skipped calling mce_reign()
		 * If we found a fatal error, we need to panic here.
		 */
		 if (worst >= MCE_PANIC_SEVERITY && mca_cfg.tolerant < 3)
			mce_panic("Machine check from unknown source",
				NULL, NULL);
	}
1122 1123

	/*
1124 1125 1126 1127
	 * At insane "tolerant" levels we take no action. Otherwise
	 * we only die if we have no other choice. For less serious
	 * issues we try to recover, or limit damage to the current
	 * process.
1128
	 */
1129
	if (cfg->tolerant < 3) {
1130 1131 1132
		if (no_way_out)
			mce_panic("Fatal machine check on current CPU", &m, msg);
		if (worst == MCE_AR_SEVERITY) {
1133 1134 1135
			recover_paddr = m.addr;
			if (!(m.mcgstatus & MCG_STATUS_RIPV))
				flags |= MF_MUST_KILL;
1136 1137 1138 1139
		} else if (kill_it) {
			force_sig(SIGBUS, current);
		}
	}
1140

1141 1142
	if (worst > 0)
		mce_report_event(regs);
1143
	mce_wrmsrl(MSR_IA32_MCG_STATUS, 0);
1144
out:
1145
	sync_core();
1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165

	if (recover_paddr == ~0ull)
		goto done;

	pr_err("Uncorrected hardware memory error in user-access at %llx",
		 recover_paddr);
	/*
	 * We must call memory_failure() here even if the current process is
	 * doomed. We still need to mark the page as poisoned and alert any
	 * other users of the page.
	 */
	ist_begin_non_atomic(regs);
	local_irq_enable();
	if (memory_failure(recover_paddr >> PAGE_SHIFT, MCE_VECTOR, flags) < 0) {
		pr_err("Memory error not recovered");
		force_sig(SIGBUS, current);
	}
	local_irq_disable();
	ist_end_non_atomic();
done:
1166
	ist_exit(regs);
L
Linus Torvalds 已提交
1167
}
1168
EXPORT_SYMBOL_GPL(do_machine_check);
L
Linus Torvalds 已提交
1169

1170 1171
#ifndef CONFIG_MEMORY_FAILURE
int memory_failure(unsigned long pfn, int vector, int flags)
1172
{
1173 1174
	/* mce_severity() should not hand us an ACTION_REQUIRED error */
	BUG_ON(flags & MF_ACTION_REQUIRED);
1175 1176 1177
	pr_err("Uncorrected memory error in page 0x%lx ignored\n"
	       "Rebuild kernel with CONFIG_MEMORY_FAILURE=y for smarter handling\n",
	       pfn);
1178 1179

	return 0;
1180
}
1181
#endif
1182

1183 1184 1185
/*
 * Action optional processing happens here (picking up
 * from the list of faulting pages that do_machine_check()
1186
 * placed into the genpool).
1187
 */
1188 1189
static void mce_process_work(struct work_struct *dummy)
{
1190
	mce_gen_pool_process();
1191 1192
}

1193 1194 1195
#ifdef CONFIG_X86_MCE_INTEL
/***
 * mce_log_therm_throt_event - Logs the thermal throttling event to mcelog
S
Simon Arlott 已提交
1196
 * @cpu: The CPU on which the event occurred.
1197 1198 1199 1200 1201 1202 1203 1204 1205 1206
 * @status: Event status information
 *
 * This function should be called by the thermal interrupt after the
 * event has been processed and the decision was made to log the event
 * further.
 *
 * The status parameter will be saved to the 'status' field of 'struct mce'
 * and historically has been the register value of the
 * MSR_IA32_THERMAL_STATUS (Intel) msr.
 */
1207
void mce_log_therm_throt_event(__u64 status)
1208 1209 1210
{
	struct mce m;

1211
	mce_setup(&m);
1212 1213 1214 1215 1216 1217
	m.bank = MCE_THERMAL_BANK;
	m.status = status;
	mce_log(&m);
}
#endif /* CONFIG_X86_MCE_INTEL */

L
Linus Torvalds 已提交
1218
/*
1219 1220 1221
 * Periodic polling timer for "silent" machine check errors.  If the
 * poller finds an MCE, poll 2x faster.  When the poller finds no more
 * errors, poll 2x slower (up to check_interval seconds).
L
Linus Torvalds 已提交
1222
 */
1223
static unsigned long check_interval = INITIAL_CHECK_INTERVAL;
I
Ingo Molnar 已提交
1224

T
Thomas Gleixner 已提交
1225
static DEFINE_PER_CPU(unsigned long, mce_next_interval); /* in jiffies */
1226
static DEFINE_PER_CPU(struct timer_list, mce_timer);
L
Linus Torvalds 已提交
1227

C
Chen Gong 已提交
1228 1229 1230 1231 1232
static unsigned long mce_adjust_timer_default(unsigned long interval)
{
	return interval;
}

1233
static unsigned long (*mce_adjust_timer)(unsigned long interval) = mce_adjust_timer_default;
C
Chen Gong 已提交
1234

1235
static void __restart_timer(struct timer_list *t, unsigned long interval)
1236
{
1237 1238
	unsigned long when = jiffies + interval;
	unsigned long flags;
1239

1240
	local_irq_save(flags);
1241

1242 1243 1244 1245 1246 1247 1248 1249 1250
	if (timer_pending(t)) {
		if (time_before(when, t->expires))
			mod_timer_pinned(t, when);
	} else {
		t->expires = round_jiffies(when);
		add_timer_on(t, smp_processor_id());
	}

	local_irq_restore(flags);
1251 1252
}

T
Thomas Gleixner 已提交
1253
static void mce_timer_fn(unsigned long data)
L
Linus Torvalds 已提交
1254
{
1255
	struct timer_list *t = this_cpu_ptr(&mce_timer);
1256
	int cpu = smp_processor_id();
T
Thomas Gleixner 已提交
1257
	unsigned long iv;
1258

1259 1260 1261
	WARN_ON(cpu != data);

	iv = __this_cpu_read(mce_next_interval);
1262

1263
	if (mce_available(this_cpu_ptr(&cpu_info))) {
1264 1265 1266 1267 1268 1269
		machine_check_poll(MCP_TIMESTAMP, this_cpu_ptr(&mce_poll_banks));

		if (mce_intel_cmci_poll()) {
			iv = mce_adjust_timer(iv);
			goto done;
		}
I
Ingo Molnar 已提交
1270
	}
L
Linus Torvalds 已提交
1271 1272

	/*
1273 1274
	 * Alert userspace if needed. If we logged an MCE, reduce the polling
	 * interval, otherwise increase the polling interval.
L
Linus Torvalds 已提交
1275
	 */
1276
	if (mce_notify_irq())
1277
		iv = max(iv / 2, (unsigned long) HZ/100);
1278
	else
T
Thomas Gleixner 已提交
1279
		iv = min(iv * 2, round_jiffies_relative(check_interval * HZ));
1280 1281

done:
T
Thomas Gleixner 已提交
1282
	__this_cpu_write(mce_next_interval, iv);
1283
	__restart_timer(t, iv);
C
Chen Gong 已提交
1284
}
1285

C
Chen Gong 已提交
1286 1287 1288 1289 1290
/*
 * Ensure that the timer is firing in @interval from now.
 */
void mce_timer_kick(unsigned long interval)
{
1291
	struct timer_list *t = this_cpu_ptr(&mce_timer);
C
Chen Gong 已提交
1292 1293
	unsigned long iv = __this_cpu_read(mce_next_interval);

1294 1295
	__restart_timer(t, interval);

C
Chen Gong 已提交
1296 1297
	if (interval < iv)
		__this_cpu_write(mce_next_interval, interval);
1298 1299
}

1300 1301 1302 1303 1304 1305 1306 1307 1308
/* Must not be called in IRQ context where del_timer_sync() can deadlock */
static void mce_timer_delete_all(void)
{
	int cpu;

	for_each_online_cpu(cpu)
		del_timer_sync(&per_cpu(mce_timer, cpu));
}

1309 1310
static void mce_do_trigger(struct work_struct *work)
{
1311
	call_usermodehelper(mce_helper, mce_helper_argv, NULL, UMH_NO_WAIT);
1312 1313 1314 1315
}

static DECLARE_WORK(mce_trigger_work, mce_do_trigger);

1316
/*
1317 1318 1319
 * Notify the user(s) about new machine check events.
 * Can be called from interrupt context, but not from machine check/NMI
 * context.
1320
 */
1321
int mce_notify_irq(void)
1322
{
1323 1324 1325
	/* Not more than two messages every minute */
	static DEFINE_RATELIMIT_STATE(ratelimit, 60*HZ, 2);

1326
	if (test_and_clear_bit(0, &mce_need_notify)) {
1327 1328
		/* wake processes polling /dev/mcelog */
		wake_up_interruptible(&mce_chrdev_wait);
1329

1330
		if (mce_helper[0])
1331
			schedule_work(&mce_trigger_work);
1332

1333
		if (__ratelimit(&ratelimit))
H
Huang Ying 已提交
1334
			pr_info(HW_ERR "Machine check events logged\n");
1335 1336

		return 1;
L
Linus Torvalds 已提交
1337
	}
1338 1339
	return 0;
}
1340
EXPORT_SYMBOL_GPL(mce_notify_irq);
1341

1342
static int __mcheck_cpu_mce_banks_init(void)
1343 1344
{
	int i;
1345
	u8 num_banks = mca_cfg.banks;
1346

1347
	mce_banks = kzalloc(num_banks * sizeof(struct mce_bank), GFP_KERNEL);
1348 1349
	if (!mce_banks)
		return -ENOMEM;
1350 1351

	for (i = 0; i < num_banks; i++) {
1352
		struct mce_bank *b = &mce_banks[i];
1353

1354 1355 1356 1357 1358 1359
		b->ctl = -1ULL;
		b->init = 1;
	}
	return 0;
}

1360
/*
L
Linus Torvalds 已提交
1361 1362
 * Initialize Machine Checks for a CPU.
 */
1363
static int __mcheck_cpu_cap_init(void)
L
Linus Torvalds 已提交
1364
{
1365
	unsigned b;
I
Ingo Molnar 已提交
1366
	u64 cap;
L
Linus Torvalds 已提交
1367 1368

	rdmsrl(MSR_IA32_MCG_CAP, cap);
1369 1370

	b = cap & MCG_BANKCNT_MASK;
1371
	if (!mca_cfg.banks)
1372
		pr_info("CPU supports %d MCE banks\n", b);
1373

1374
	if (b > MAX_NR_BANKS) {
1375
		pr_warn("Using only %u machine check banks out of %u\n",
1376 1377 1378 1379 1380
			MAX_NR_BANKS, b);
		b = MAX_NR_BANKS;
	}

	/* Don't support asymmetric configurations today */
1381 1382 1383
	WARN_ON(mca_cfg.banks != 0 && b != mca_cfg.banks);
	mca_cfg.banks = b;

1384
	if (!mce_banks) {
H
Hidetoshi Seto 已提交
1385
		int err = __mcheck_cpu_mce_banks_init();
1386

1387 1388
		if (err)
			return err;
L
Linus Torvalds 已提交
1389
	}
1390

1391
	/* Use accurate RIP reporting if available. */
1392
	if ((cap & MCG_EXT_P) && MCG_EXT_CNT(cap) >= 9)
1393
		mca_cfg.rip_msr = MSR_IA32_MCG_EIP;
L
Linus Torvalds 已提交
1394

A
Andi Kleen 已提交
1395
	if (cap & MCG_SER_P)
1396
		mca_cfg.ser = true;
A
Andi Kleen 已提交
1397

1398 1399 1400
	return 0;
}

1401
static void __mcheck_cpu_init_generic(void)
1402
{
1403
	enum mcp_flags m_fl = 0;
I
Ingo Molnar 已提交
1404
	mce_banks_t all_banks;
1405 1406 1407
	u64 cap;
	int i;

1408 1409 1410
	if (!mca_cfg.bootlog)
		m_fl = MCP_DONTLOG;

1411 1412 1413
	/*
	 * Log the machine checks left over from the previous reset.
	 */
1414
	bitmap_fill(all_banks, MAX_NR_BANKS);
1415
	machine_check_poll(MCP_UC | m_fl, &all_banks);
L
Linus Torvalds 已提交
1416

A
Andy Lutomirski 已提交
1417
	cr4_set_bits(X86_CR4_MCE);
L
Linus Torvalds 已提交
1418

1419
	rdmsrl(MSR_IA32_MCG_CAP, cap);
L
Linus Torvalds 已提交
1420 1421 1422
	if (cap & MCG_CTL_P)
		wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff);

1423
	for (i = 0; i < mca_cfg.banks; i++) {
1424
		struct mce_bank *b = &mce_banks[i];
1425

1426
		if (!b->init)
1427
			continue;
1428 1429
		wrmsrl(MSR_IA32_MCx_CTL(i), b->ctl);
		wrmsrl(MSR_IA32_MCx_STATUS(i), 0);
1430
	}
L
Linus Torvalds 已提交
1431 1432
}

1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460
/*
 * During IFU recovery Sandy Bridge -EP4S processors set the RIPV and
 * EIPV bits in MCG_STATUS to zero on the affected logical processor (SDM
 * Vol 3B Table 15-20). But this confuses both the code that determines
 * whether the machine check occurred in kernel or user mode, and also
 * the severity assessment code. Pretend that EIPV was set, and take the
 * ip/cs values from the pt_regs that mce_gather_info() ignored earlier.
 */
static void quirk_sandybridge_ifu(int bank, struct mce *m, struct pt_regs *regs)
{
	if (bank != 0)
		return;
	if ((m->mcgstatus & (MCG_STATUS_EIPV|MCG_STATUS_RIPV)) != 0)
		return;
	if ((m->status & (MCI_STATUS_OVER|MCI_STATUS_UC|
		          MCI_STATUS_EN|MCI_STATUS_MISCV|MCI_STATUS_ADDRV|
			  MCI_STATUS_PCC|MCI_STATUS_S|MCI_STATUS_AR|
			  MCACOD)) !=
			 (MCI_STATUS_UC|MCI_STATUS_EN|
			  MCI_STATUS_MISCV|MCI_STATUS_ADDRV|MCI_STATUS_S|
			  MCI_STATUS_AR|MCACOD_INSTR))
		return;

	m->mcgstatus |= MCG_STATUS_EIPV;
	m->ip = regs->ip;
	m->cs = regs->cs;
}

L
Linus Torvalds 已提交
1461
/* Add per CPU specific workarounds here */
1462
static int __mcheck_cpu_apply_quirks(struct cpuinfo_x86 *c)
1463
{
1464 1465
	struct mca_config *cfg = &mca_cfg;

1466
	if (c->x86_vendor == X86_VENDOR_UNKNOWN) {
1467
		pr_info("unknown CPU type - not enabling MCE support\n");
1468 1469 1470
		return -EOPNOTSUPP;
	}

L
Linus Torvalds 已提交
1471
	/* This should be disabled by the BIOS, but isn't always */
1472
	if (c->x86_vendor == X86_VENDOR_AMD) {
1473
		if (c->x86 == 15 && cfg->banks > 4) {
I
Ingo Molnar 已提交
1474 1475 1476 1477 1478
			/*
			 * disable GART TBL walk error reporting, which
			 * trips off incorrectly with the IOMMU & 3ware
			 * & Cerberus:
			 */
1479
			clear_bit(10, (unsigned long *)&mce_banks[4].ctl);
I
Ingo Molnar 已提交
1480
		}
1481
		if (c->x86 <= 17 && cfg->bootlog < 0) {
I
Ingo Molnar 已提交
1482 1483 1484 1485
			/*
			 * Lots of broken BIOS around that don't clear them
			 * by default and leave crap in there. Don't log:
			 */
1486
			cfg->bootlog = 0;
I
Ingo Molnar 已提交
1487
		}
1488 1489 1490 1491
		/*
		 * Various K7s with broken bank 0 around. Always disable
		 * by default.
		 */
1492
		if (c->x86 == 6 && cfg->banks > 0)
1493
			mce_banks[0].ctl = 0;
1494

1495 1496 1497 1498 1499 1500 1501
		/*
		 * overflow_recov is supported for F15h Models 00h-0fh
		 * even though we don't have a CPUID bit for it.
		 */
		if (c->x86 == 0x15 && c->x86_model <= 0xf)
			mce_flags.overflow_recov = 1;

1502 1503 1504 1505 1506 1507 1508 1509 1510 1511
		/*
		 * Turn off MC4_MISC thresholding banks on those models since
		 * they're not supported there.
		 */
		if (c->x86 == 0x15 &&
		    (c->x86_model >= 0x10 && c->x86_model <= 0x1f)) {
			int i;
			u64 hwcr;
			bool need_toggle;
			u32 msrs[] = {
1512 1513
				0x00000413, /* MC4_MISC0 */
				0xc0000408, /* MC4_MISC1 */
1514
			};
1515

1516
			rdmsrl(MSR_K7_HWCR, hwcr);
1517

1518 1519
			/* McStatusWrEn has to be set */
			need_toggle = !(hwcr & BIT(18));
1520

1521 1522
			if (need_toggle)
				wrmsrl(MSR_K7_HWCR, hwcr | BIT(18));
1523

1524 1525 1526
			/* Clear CntP bit safely */
			for (i = 0; i < ARRAY_SIZE(msrs); i++)
				msr_clear_bit(msrs[i], 62);
1527

1528 1529 1530 1531
			/* restore old settings */
			if (need_toggle)
				wrmsrl(MSR_K7_HWCR, hwcr);
		}
L
Linus Torvalds 已提交
1532
	}
1533

1534 1535 1536 1537 1538 1539 1540 1541 1542 1543
	if (c->x86_vendor == X86_VENDOR_INTEL) {
		/*
		 * SDM documents that on family 6 bank 0 should not be written
		 * because it aliases to another special BIOS controlled
		 * register.
		 * But it's not aliased anymore on model 0x1a+
		 * Don't ignore bank 0 completely because there could be a
		 * valid event later, merely don't write CTL0.
		 */

1544
		if (c->x86 == 6 && c->x86_model < 0x1A && cfg->banks > 0)
1545
			mce_banks[0].init = 0;
1546 1547 1548 1549 1550 1551

		/*
		 * All newer Intel systems support MCE broadcasting. Enable
		 * synchronization with a one second timeout.
		 */
		if ((c->x86 > 6 || (c->x86 == 6 && c->x86_model >= 0xe)) &&
1552 1553
			cfg->monarch_timeout < 0)
			cfg->monarch_timeout = USEC_PER_SEC;
1554

1555 1556 1557 1558
		/*
		 * There are also broken BIOSes on some Pentium M and
		 * earlier systems:
		 */
1559 1560
		if (c->x86 == 6 && c->x86_model <= 13 && cfg->bootlog < 0)
			cfg->bootlog = 0;
1561 1562 1563

		if (c->x86 == 6 && c->x86_model == 45)
			quirk_no_way_out = quirk_sandybridge_ifu;
1564
	}
1565 1566 1567
	if (cfg->monarch_timeout < 0)
		cfg->monarch_timeout = 0;
	if (cfg->bootlog != 0)
1568
		cfg->panic_timeout = 30;
1569 1570

	return 0;
1571
}
L
Linus Torvalds 已提交
1572

1573
static int __mcheck_cpu_ancient_init(struct cpuinfo_x86 *c)
1574 1575
{
	if (c->x86 != 5)
1576 1577
		return 0;

1578 1579
	switch (c->x86_vendor) {
	case X86_VENDOR_INTEL:
1580
		intel_p5_mcheck_init(c);
1581
		return 1;
1582 1583 1584
		break;
	case X86_VENDOR_CENTAUR:
		winchip_mcheck_init(c);
1585
		return 1;
1586
		break;
1587 1588
	default:
		return 0;
1589
	}
1590 1591

	return 0;
1592 1593
}

1594
static void __mcheck_cpu_init_vendor(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1595 1596 1597 1598
{
	switch (c->x86_vendor) {
	case X86_VENDOR_INTEL:
		mce_intel_feature_init(c);
1599
		mce_adjust_timer = cmci_intel_adjust_timer;
L
Linus Torvalds 已提交
1600
		break;
1601 1602 1603 1604

	case X86_VENDOR_AMD: {
		u32 ebx = cpuid_ebx(0x80000007);

1605
		mce_amd_feature_init(c);
1606 1607
		mce_flags.overflow_recov = !!(ebx & BIT(0));
		mce_flags.succor	 = !!(ebx & BIT(1));
1608 1609
		mce_flags.smca		 = !!(ebx & BIT(3));

1610
		break;
1611 1612
		}

L
Linus Torvalds 已提交
1613 1614 1615 1616 1617
	default:
		break;
	}
}

1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628
static void __mcheck_cpu_clear_vendor(struct cpuinfo_x86 *c)
{
	switch (c->x86_vendor) {
	case X86_VENDOR_INTEL:
		mce_intel_feature_clear(c);
		break;
	default:
		break;
	}
}

T
Thomas Gleixner 已提交
1629
static void mce_start_timer(unsigned int cpu, struct timer_list *t)
1630
{
1631
	unsigned long iv = check_interval * HZ;
1632

1633
	if (mca_cfg.ignore_ce || !iv)
1634 1635
		return;

1636 1637
	per_cpu(mce_next_interval, cpu) = iv;

T
Thomas Gleixner 已提交
1638
	t->expires = round_jiffies(jiffies + iv);
1639
	add_timer_on(t, cpu);
1640 1641
}

T
Thomas Gleixner 已提交
1642 1643
static void __mcheck_cpu_init_timer(void)
{
1644
	struct timer_list *t = this_cpu_ptr(&mce_timer);
T
Thomas Gleixner 已提交
1645 1646 1647 1648 1649 1650
	unsigned int cpu = smp_processor_id();

	setup_timer(t, mce_timer_fn, cpu);
	mce_start_timer(cpu, t);
}

A
Andi Kleen 已提交
1651 1652 1653
/* Handle unconfigured int18 (should never happen) */
static void unexpected_machine_check(struct pt_regs *regs, long error_code)
{
1654
	pr_err("CPU#%d: Unexpected int18 (Machine Check)\n",
A
Andi Kleen 已提交
1655 1656 1657 1658 1659 1660 1661
	       smp_processor_id());
}

/* Call the installed machine check handler for this CPU setup. */
void (*machine_check_vector)(struct pt_regs *, long error_code) =
						unexpected_machine_check;

1662
/*
L
Linus Torvalds 已提交
1663
 * Called for each booted CPU to set up machine checks.
I
Ingo Molnar 已提交
1664
 * Must be called with preempt off:
L
Linus Torvalds 已提交
1665
 */
1666
void mcheck_cpu_init(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1667
{
1668
	if (mca_cfg.disabled)
1669 1670
		return;

1671 1672
	if (__mcheck_cpu_ancient_init(c))
		return;
1673

1674
	if (!mce_available(c))
L
Linus Torvalds 已提交
1675 1676
		return;

1677
	if (__mcheck_cpu_cap_init() < 0 || __mcheck_cpu_apply_quirks(c) < 0) {
1678
		mca_cfg.disabled = true;
1679 1680 1681
		return;
	}

1682 1683 1684 1685 1686 1687
	if (mce_gen_pool_init()) {
		mca_cfg.disabled = true;
		pr_emerg("Couldn't allocate MCE records pool!\n");
		return;
	}

1688 1689
	machine_check_vector = do_machine_check;

1690 1691 1692
	__mcheck_cpu_init_generic();
	__mcheck_cpu_init_vendor(c);
	__mcheck_cpu_init_timer();
L
Linus Torvalds 已提交
1693 1694
}

1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711
/*
 * Called for each booted CPU to clear some machine checks opt-ins
 */
void mcheck_cpu_clear(struct cpuinfo_x86 *c)
{
	if (mca_cfg.disabled)
		return;

	if (!mce_available(c))
		return;

	/*
	 * Possibly to clear general settings generic to x86
	 * __mcheck_cpu_clear_generic(c);
	 */
	__mcheck_cpu_clear_vendor(c);

L
Linus Torvalds 已提交
1712 1713 1714
}

/*
1715
 * mce_chrdev: Character device /dev/mcelog to read and clear the MCE log.
L
Linus Torvalds 已提交
1716 1717
 */

1718 1719 1720
static DEFINE_SPINLOCK(mce_chrdev_state_lock);
static int mce_chrdev_open_count;	/* #times opened */
static int mce_chrdev_open_exclu;	/* already open exclusive? */
T
Tim Hockin 已提交
1721

1722
static int mce_chrdev_open(struct inode *inode, struct file *file)
T
Tim Hockin 已提交
1723
{
1724
	spin_lock(&mce_chrdev_state_lock);
T
Tim Hockin 已提交
1725

1726 1727 1728
	if (mce_chrdev_open_exclu ||
	    (mce_chrdev_open_count && (file->f_flags & O_EXCL))) {
		spin_unlock(&mce_chrdev_state_lock);
I
Ingo Molnar 已提交
1729

T
Tim Hockin 已提交
1730 1731 1732 1733
		return -EBUSY;
	}

	if (file->f_flags & O_EXCL)
1734 1735
		mce_chrdev_open_exclu = 1;
	mce_chrdev_open_count++;
T
Tim Hockin 已提交
1736

1737
	spin_unlock(&mce_chrdev_state_lock);
T
Tim Hockin 已提交
1738

1739
	return nonseekable_open(inode, file);
T
Tim Hockin 已提交
1740 1741
}

1742
static int mce_chrdev_release(struct inode *inode, struct file *file)
T
Tim Hockin 已提交
1743
{
1744
	spin_lock(&mce_chrdev_state_lock);
T
Tim Hockin 已提交
1745

1746 1747
	mce_chrdev_open_count--;
	mce_chrdev_open_exclu = 0;
T
Tim Hockin 已提交
1748

1749
	spin_unlock(&mce_chrdev_state_lock);
T
Tim Hockin 已提交
1750 1751 1752 1753

	return 0;
}

1754 1755
static void collect_tscs(void *data)
{
L
Linus Torvalds 已提交
1756
	unsigned long *cpu_tsc = (unsigned long *)data;
1757

1758
	cpu_tsc[smp_processor_id()] = rdtsc();
1759
}
L
Linus Torvalds 已提交
1760

1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776
static int mce_apei_read_done;

/* Collect MCE record of previous boot in persistent storage via APEI ERST. */
static int __mce_read_apei(char __user **ubuf, size_t usize)
{
	int rc;
	u64 record_id;
	struct mce m;

	if (usize < sizeof(struct mce))
		return -EINVAL;

	rc = apei_read_mce(&m, &record_id);
	/* Error or no more MCE record */
	if (rc <= 0) {
		mce_apei_read_done = 1;
1777 1778 1779 1780 1781 1782
		/*
		 * When ERST is disabled, mce_chrdev_read() should return
		 * "no record" instead of "no device."
		 */
		if (rc == -ENODEV)
			return 0;
1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803
		return rc;
	}
	rc = -EFAULT;
	if (copy_to_user(*ubuf, &m, sizeof(struct mce)))
		return rc;
	/*
	 * In fact, we should have cleared the record after that has
	 * been flushed to the disk or sent to network in
	 * /sbin/mcelog, but we have no interface to support that now,
	 * so just clear it to avoid duplication.
	 */
	rc = apei_clear_mce(record_id);
	if (rc) {
		mce_apei_read_done = 1;
		return rc;
	}
	*ubuf += sizeof(struct mce);

	return 0;
}

1804 1805
static ssize_t mce_chrdev_read(struct file *filp, char __user *ubuf,
				size_t usize, loff_t *off)
L
Linus Torvalds 已提交
1806
{
I
Ingo Molnar 已提交
1807
	char __user *buf = ubuf;
1808
	unsigned long *cpu_tsc;
1809
	unsigned prev, next;
L
Linus Torvalds 已提交
1810 1811
	int i, err;

1812
	cpu_tsc = kmalloc(nr_cpu_ids * sizeof(long), GFP_KERNEL);
1813 1814 1815
	if (!cpu_tsc)
		return -ENOMEM;

1816
	mutex_lock(&mce_chrdev_read_mutex);
1817 1818 1819 1820 1821 1822 1823

	if (!mce_apei_read_done) {
		err = __mce_read_apei(&buf, usize);
		if (err || buf != ubuf)
			goto out;
	}

1824
	next = mce_log_get_idx_check(mcelog.next);
L
Linus Torvalds 已提交
1825 1826

	/* Only supports full reads right now */
1827 1828 1829
	err = -EINVAL;
	if (*off != 0 || usize < MCE_LOG_LEN*sizeof(struct mce))
		goto out;
L
Linus Torvalds 已提交
1830 1831

	err = 0;
1832 1833 1834 1835
	prev = 0;
	do {
		for (i = prev; i < next; i++) {
			unsigned long start = jiffies;
H
Hidetoshi Seto 已提交
1836
			struct mce *m = &mcelog.entry[i];
1837

H
Hidetoshi Seto 已提交
1838
			while (!m->finished) {
1839
				if (time_after_eq(jiffies, start + 2)) {
H
Hidetoshi Seto 已提交
1840
					memset(m, 0, sizeof(*m));
1841 1842 1843
					goto timeout;
				}
				cpu_relax();
1844
			}
1845
			smp_rmb();
H
Hidetoshi Seto 已提交
1846 1847
			err |= copy_to_user(buf, m, sizeof(*m));
			buf += sizeof(*m);
1848 1849
timeout:
			;
1850
		}
L
Linus Torvalds 已提交
1851

1852 1853 1854 1855 1856
		memset(mcelog.entry + prev, 0,
		       (next - prev) * sizeof(struct mce));
		prev = next;
		next = cmpxchg(&mcelog.next, prev, 0);
	} while (next != prev);
L
Linus Torvalds 已提交
1857

1858
	synchronize_sched();
L
Linus Torvalds 已提交
1859

1860 1861 1862 1863
	/*
	 * Collect entries that were still getting written before the
	 * synchronize.
	 */
1864
	on_each_cpu(collect_tscs, cpu_tsc, 1);
I
Ingo Molnar 已提交
1865

1866
	for (i = next; i < MCE_LOG_LEN; i++) {
H
Hidetoshi Seto 已提交
1867 1868 1869 1870
		struct mce *m = &mcelog.entry[i];

		if (m->finished && m->tsc < cpu_tsc[m->cpu]) {
			err |= copy_to_user(buf, m, sizeof(*m));
L
Linus Torvalds 已提交
1871
			smp_rmb();
H
Hidetoshi Seto 已提交
1872 1873
			buf += sizeof(*m);
			memset(m, 0, sizeof(*m));
L
Linus Torvalds 已提交
1874
		}
1875
	}
1876 1877 1878 1879 1880

	if (err)
		err = -EFAULT;

out:
1881
	mutex_unlock(&mce_chrdev_read_mutex);
1882
	kfree(cpu_tsc);
I
Ingo Molnar 已提交
1883

1884
	return err ? err : buf - ubuf;
L
Linus Torvalds 已提交
1885 1886
}

1887
static unsigned int mce_chrdev_poll(struct file *file, poll_table *wait)
1888
{
1889
	poll_wait(file, &mce_chrdev_wait, wait);
1890
	if (READ_ONCE(mcelog.next))
1891
		return POLLIN | POLLRDNORM;
1892 1893
	if (!mce_apei_read_done && apei_check_mce())
		return POLLIN | POLLRDNORM;
1894 1895 1896
	return 0;
}

1897 1898
static long mce_chrdev_ioctl(struct file *f, unsigned int cmd,
				unsigned long arg)
L
Linus Torvalds 已提交
1899 1900
{
	int __user *p = (int __user *)arg;
1901

L
Linus Torvalds 已提交
1902
	if (!capable(CAP_SYS_ADMIN))
1903
		return -EPERM;
I
Ingo Molnar 已提交
1904

L
Linus Torvalds 已提交
1905
	switch (cmd) {
1906
	case MCE_GET_RECORD_LEN:
L
Linus Torvalds 已提交
1907 1908
		return put_user(sizeof(struct mce), p);
	case MCE_GET_LOG_LEN:
1909
		return put_user(MCE_LOG_LEN, p);
L
Linus Torvalds 已提交
1910 1911
	case MCE_GETCLEAR_FLAGS: {
		unsigned flags;
1912 1913

		do {
L
Linus Torvalds 已提交
1914
			flags = mcelog.flags;
1915
		} while (cmpxchg(&mcelog.flags, flags, 0) != flags);
I
Ingo Molnar 已提交
1916

1917
		return put_user(flags, p);
L
Linus Torvalds 已提交
1918 1919
	}
	default:
1920 1921
		return -ENOTTY;
	}
L
Linus Torvalds 已提交
1922 1923
}

1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934
static ssize_t (*mce_write)(struct file *filp, const char __user *ubuf,
			    size_t usize, loff_t *off);

void register_mce_write_callback(ssize_t (*fn)(struct file *filp,
			     const char __user *ubuf,
			     size_t usize, loff_t *off))
{
	mce_write = fn;
}
EXPORT_SYMBOL_GPL(register_mce_write_callback);

1935 1936
static ssize_t mce_chrdev_write(struct file *filp, const char __user *ubuf,
				size_t usize, loff_t *off)
1937 1938 1939 1940 1941 1942 1943 1944
{
	if (mce_write)
		return mce_write(filp, ubuf, usize, off);
	else
		return -EINVAL;
}

static const struct file_operations mce_chrdev_ops = {
1945 1946 1947
	.open			= mce_chrdev_open,
	.release		= mce_chrdev_release,
	.read			= mce_chrdev_read,
1948
	.write			= mce_chrdev_write,
1949 1950 1951
	.poll			= mce_chrdev_poll,
	.unlocked_ioctl		= mce_chrdev_ioctl,
	.llseek			= no_llseek,
L
Linus Torvalds 已提交
1952 1953
};

1954
static struct miscdevice mce_chrdev_device = {
L
Linus Torvalds 已提交
1955 1956 1957 1958 1959
	MISC_MCELOG_MINOR,
	"mcelog",
	&mce_chrdev_ops,
};

1960 1961 1962
static void __mce_disable_bank(void *arg)
{
	int bank = *((int *)arg);
1963
	__clear_bit(bank, this_cpu_ptr(mce_poll_banks));
1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978
	cmci_disable_bank(bank);
}

void mce_disable_bank(int bank)
{
	if (bank >= mca_cfg.banks) {
		pr_warn(FW_BUG
			"Ignoring request to disable invalid MCA bank %d.\n",
			bank);
		return;
	}
	set_bit(bank, mce_banks_ce_disabled);
	on_each_cpu(__mce_disable_bank, &bank, 1);
}

H
Hidetoshi Seto 已提交
1979
/*
1980 1981
 * mce=off Disables machine check
 * mce=no_cmci Disables CMCI
1982
 * mce=no_lmce Disables LMCE
1983 1984
 * mce=dont_log_ce Clears corrected events silently, no log created for CEs.
 * mce=ignore_ce Disables polling and CMCI, corrected events are not cleared.
1985 1986 1987
 * mce=TOLERANCELEVEL[,monarchtimeout] (number, see above)
 *	monarchtimeout is how long to wait for other CPUs on machine
 *	check, or 0 to not wait
H
Hidetoshi Seto 已提交
1988 1989
 * mce=bootlog Log MCEs from before booting. Disabled by default on AMD.
 * mce=nobootlog Don't log MCEs from before booting.
1990
 * mce=bios_cmci_threshold Don't program the CMCI threshold
H
Hidetoshi Seto 已提交
1991
 */
L
Linus Torvalds 已提交
1992 1993
static int __init mcheck_enable(char *str)
{
1994 1995
	struct mca_config *cfg = &mca_cfg;

1996
	if (*str == 0) {
1997
		enable_p5_mce();
1998 1999
		return 1;
	}
2000 2001
	if (*str == '=')
		str++;
L
Linus Torvalds 已提交
2002
	if (!strcmp(str, "off"))
2003
		cfg->disabled = true;
2004
	else if (!strcmp(str, "no_cmci"))
2005
		cfg->cmci_disabled = true;
2006 2007
	else if (!strcmp(str, "no_lmce"))
		cfg->lmce_disabled = true;
2008
	else if (!strcmp(str, "dont_log_ce"))
2009
		cfg->dont_log_ce = true;
2010
	else if (!strcmp(str, "ignore_ce"))
2011
		cfg->ignore_ce = true;
H
Hidetoshi Seto 已提交
2012
	else if (!strcmp(str, "bootlog") || !strcmp(str, "nobootlog"))
2013
		cfg->bootlog = (str[0] == 'b');
2014
	else if (!strcmp(str, "bios_cmci_threshold"))
2015
		cfg->bios_cmci_threshold = true;
2016
	else if (isdigit(str[0])) {
2017
		if (get_option(&str, &cfg->tolerant) == 2)
2018
			get_option(&str, &(cfg->monarch_timeout));
2019
	} else {
2020
		pr_info("mce argument %s ignored. Please use /sys\n", str);
H
Hidetoshi Seto 已提交
2021 2022
		return 0;
	}
2023
	return 1;
L
Linus Torvalds 已提交
2024
}
2025
__setup("mce", mcheck_enable);
L
Linus Torvalds 已提交
2026

2027
int __init mcheck_init(void)
2028
{
2029
	mcheck_intel_therm_init();
2030
	mce_register_decode_chain(&mce_srao_nb);
2031
	mcheck_vendor_init_severity();
2032

2033 2034 2035
	INIT_WORK(&mce_work, mce_process_work);
	init_irq_work(&mce_irq_work, mce_irq_work_cb);

2036 2037 2038
	return 0;
}

2039
/*
2040
 * mce_syscore: PM support
2041
 */
L
Linus Torvalds 已提交
2042

2043 2044 2045 2046
/*
 * Disable machine checks on suspend and shutdown. We can't really handle
 * them later.
 */
2047
static void mce_disable_error_reporting(void)
2048 2049 2050
{
	int i;

2051
	for (i = 0; i < mca_cfg.banks; i++) {
2052
		struct mce_bank *b = &mce_banks[i];
2053

2054
		if (b->init)
2055
			wrmsrl(MSR_IA32_MCx_CTL(i), 0);
2056
	}
2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071
	return;
}

static void vendor_disable_error_reporting(void)
{
	/*
	 * Don't clear on Intel CPUs. Some of these MSRs are socket-wide.
	 * Disabling them for just a single offlined CPU is bad, since it will
	 * inhibit reporting for all shared resources on the socket like the
	 * last level cache (LLC), the integrated memory controller (iMC), etc.
	 */
	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
		return;

	mce_disable_error_reporting();
2072 2073
}

2074
static int mce_syscore_suspend(void)
2075
{
2076 2077
	vendor_disable_error_reporting();
	return 0;
2078 2079
}

2080
static void mce_syscore_shutdown(void)
2081
{
2082
	vendor_disable_error_reporting();
2083 2084
}

I
Ingo Molnar 已提交
2085 2086 2087 2088 2089
/*
 * On resume clear all MCE state. Don't want to see leftovers from the BIOS.
 * Only one CPU is active at this time, the others get re-added later using
 * CPU hotplug:
 */
2090
static void mce_syscore_resume(void)
L
Linus Torvalds 已提交
2091
{
2092
	__mcheck_cpu_init_generic();
2093
	__mcheck_cpu_init_vendor(raw_cpu_ptr(&cpu_info));
L
Linus Torvalds 已提交
2094 2095
}

2096
static struct syscore_ops mce_syscore_ops = {
2097 2098 2099
	.suspend	= mce_syscore_suspend,
	.shutdown	= mce_syscore_shutdown,
	.resume		= mce_syscore_resume,
2100 2101
};

2102
/*
2103
 * mce_device: Sysfs support
2104 2105
 */

2106 2107
static void mce_cpu_restart(void *data)
{
2108
	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2109
		return;
2110 2111
	__mcheck_cpu_init_generic();
	__mcheck_cpu_init_timer();
2112 2113
}

L
Linus Torvalds 已提交
2114
/* Reinit MCEs after user configuration changes */
2115 2116
static void mce_restart(void)
{
2117
	mce_timer_delete_all();
2118
	on_each_cpu(mce_cpu_restart, NULL, 1);
L
Linus Torvalds 已提交
2119 2120
}

2121
/* Toggle features for corrected errors */
2122
static void mce_disable_cmci(void *data)
2123
{
2124
	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2125 2126 2127 2128 2129 2130
		return;
	cmci_clear();
}

static void mce_enable_ce(void *all)
{
2131
	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2132 2133 2134 2135
		return;
	cmci_reenable();
	cmci_recheck();
	if (all)
2136
		__mcheck_cpu_init_timer();
2137 2138
}

2139
static struct bus_type mce_subsys = {
I
Ingo Molnar 已提交
2140
	.name		= "machinecheck",
2141
	.dev_name	= "machinecheck",
L
Linus Torvalds 已提交
2142 2143
};

2144
DEFINE_PER_CPU(struct device *, mce_device);
I
Ingo Molnar 已提交
2145 2146

void (*threshold_cpu_callback)(unsigned long action, unsigned int cpu);
L
Linus Torvalds 已提交
2147

2148
static inline struct mce_bank *attr_to_bank(struct device_attribute *attr)
2149 2150 2151
{
	return container_of(attr, struct mce_bank, attr);
}
2152

2153
static ssize_t show_bank(struct device *s, struct device_attribute *attr,
2154 2155
			 char *buf)
{
2156
	return sprintf(buf, "%llx\n", attr_to_bank(attr)->ctl);
2157 2158
}

2159
static ssize_t set_bank(struct device *s, struct device_attribute *attr,
H
Hidetoshi Seto 已提交
2160
			const char *buf, size_t size)
2161
{
H
Hidetoshi Seto 已提交
2162
	u64 new;
I
Ingo Molnar 已提交
2163

2164
	if (kstrtou64(buf, 0, &new) < 0)
2165
		return -EINVAL;
I
Ingo Molnar 已提交
2166

2167
	attr_to_bank(attr)->ctl = new;
2168
	mce_restart();
I
Ingo Molnar 已提交
2169

H
Hidetoshi Seto 已提交
2170
	return size;
2171
}
2172

I
Ingo Molnar 已提交
2173
static ssize_t
2174
show_trigger(struct device *s, struct device_attribute *attr, char *buf)
2175
{
2176
	strcpy(buf, mce_helper);
2177
	strcat(buf, "\n");
2178
	return strlen(mce_helper) + 1;
2179 2180
}

2181
static ssize_t set_trigger(struct device *s, struct device_attribute *attr,
I
Ingo Molnar 已提交
2182
				const char *buf, size_t siz)
2183 2184
{
	char *p;
I
Ingo Molnar 已提交
2185

2186 2187 2188
	strncpy(mce_helper, buf, sizeof(mce_helper));
	mce_helper[sizeof(mce_helper)-1] = 0;
	p = strchr(mce_helper, '\n');
I
Ingo Molnar 已提交
2189

2190
	if (p)
I
Ingo Molnar 已提交
2191 2192
		*p = 0;

2193
	return strlen(mce_helper) + !!p;
2194 2195
}

2196 2197
static ssize_t set_ignore_ce(struct device *s,
			     struct device_attribute *attr,
2198 2199 2200 2201
			     const char *buf, size_t size)
{
	u64 new;

2202
	if (kstrtou64(buf, 0, &new) < 0)
2203 2204
		return -EINVAL;

2205
	if (mca_cfg.ignore_ce ^ !!new) {
2206 2207
		if (new) {
			/* disable ce features */
2208 2209
			mce_timer_delete_all();
			on_each_cpu(mce_disable_cmci, NULL, 1);
2210
			mca_cfg.ignore_ce = true;
2211 2212
		} else {
			/* enable ce features */
2213
			mca_cfg.ignore_ce = false;
2214 2215 2216 2217 2218 2219
			on_each_cpu(mce_enable_ce, (void *)1, 1);
		}
	}
	return size;
}

2220 2221
static ssize_t set_cmci_disabled(struct device *s,
				 struct device_attribute *attr,
2222 2223 2224 2225
				 const char *buf, size_t size)
{
	u64 new;

2226
	if (kstrtou64(buf, 0, &new) < 0)
2227 2228
		return -EINVAL;

2229
	if (mca_cfg.cmci_disabled ^ !!new) {
2230 2231
		if (new) {
			/* disable cmci */
2232
			on_each_cpu(mce_disable_cmci, NULL, 1);
2233
			mca_cfg.cmci_disabled = true;
2234 2235
		} else {
			/* enable cmci */
2236
			mca_cfg.cmci_disabled = false;
2237 2238 2239 2240 2241 2242
			on_each_cpu(mce_enable_ce, NULL, 1);
		}
	}
	return size;
}

2243 2244
static ssize_t store_int_with_restart(struct device *s,
				      struct device_attribute *attr,
2245 2246
				      const char *buf, size_t size)
{
2247
	ssize_t ret = device_store_int(s, attr, buf, size);
2248 2249 2250 2251
	mce_restart();
	return ret;
}

2252
static DEVICE_ATTR(trigger, 0644, show_trigger, set_trigger);
2253
static DEVICE_INT_ATTR(tolerant, 0644, mca_cfg.tolerant);
2254
static DEVICE_INT_ATTR(monarch_timeout, 0644, mca_cfg.monarch_timeout);
2255
static DEVICE_BOOL_ATTR(dont_log_ce, 0644, mca_cfg.dont_log_ce);
I
Ingo Molnar 已提交
2256

2257 2258
static struct dev_ext_attribute dev_attr_check_interval = {
	__ATTR(check_interval, 0644, device_show_int, store_int_with_restart),
2259 2260
	&check_interval
};
I
Ingo Molnar 已提交
2261

2262
static struct dev_ext_attribute dev_attr_ignore_ce = {
2263 2264
	__ATTR(ignore_ce, 0644, device_show_bool, set_ignore_ce),
	&mca_cfg.ignore_ce
2265 2266
};

2267
static struct dev_ext_attribute dev_attr_cmci_disabled = {
2268 2269
	__ATTR(cmci_disabled, 0644, device_show_bool, set_cmci_disabled),
	&mca_cfg.cmci_disabled
2270 2271
};

2272 2273 2274 2275 2276 2277 2278 2279
static struct device_attribute *mce_device_attrs[] = {
	&dev_attr_tolerant.attr,
	&dev_attr_check_interval.attr,
	&dev_attr_trigger,
	&dev_attr_monarch_timeout.attr,
	&dev_attr_dont_log_ce.attr,
	&dev_attr_ignore_ce.attr,
	&dev_attr_cmci_disabled.attr,
2280 2281
	NULL
};
L
Linus Torvalds 已提交
2282

2283
static cpumask_var_t mce_device_initialized;
2284

2285 2286 2287 2288 2289
static void mce_device_release(struct device *dev)
{
	kfree(dev);
}

2290
/* Per cpu device init. All of the cpus still share the same ctrl bank: */
2291
static int mce_device_create(unsigned int cpu)
L
Linus Torvalds 已提交
2292
{
2293
	struct device *dev;
L
Linus Torvalds 已提交
2294
	int err;
2295
	int i, j;
2296

A
Andreas Herrmann 已提交
2297
	if (!mce_available(&boot_cpu_data))
2298 2299
		return -EIO;

2300 2301 2302
	dev = kzalloc(sizeof *dev, GFP_KERNEL);
	if (!dev)
		return -ENOMEM;
2303 2304
	dev->id  = cpu;
	dev->bus = &mce_subsys;
2305
	dev->release = &mce_device_release;
2306

2307
	err = device_register(dev);
2308 2309
	if (err) {
		put_device(dev);
2310
		return err;
2311
	}
2312

2313 2314
	for (i = 0; mce_device_attrs[i]; i++) {
		err = device_create_file(dev, mce_device_attrs[i]);
2315 2316 2317
		if (err)
			goto error;
	}
2318
	for (j = 0; j < mca_cfg.banks; j++) {
2319
		err = device_create_file(dev, &mce_banks[j].attr);
2320 2321 2322
		if (err)
			goto error2;
	}
2323
	cpumask_set_cpu(cpu, mce_device_initialized);
2324
	per_cpu(mce_device, cpu) = dev;
2325

2326
	return 0;
2327
error2:
2328
	while (--j >= 0)
2329
		device_remove_file(dev, &mce_banks[j].attr);
2330
error:
I
Ingo Molnar 已提交
2331
	while (--i >= 0)
2332
		device_remove_file(dev, mce_device_attrs[i]);
I
Ingo Molnar 已提交
2333

2334
	device_unregister(dev);
2335

2336 2337 2338
	return err;
}

2339
static void mce_device_remove(unsigned int cpu)
2340
{
2341
	struct device *dev = per_cpu(mce_device, cpu);
2342 2343
	int i;

2344
	if (!cpumask_test_cpu(cpu, mce_device_initialized))
2345 2346
		return;

2347 2348
	for (i = 0; mce_device_attrs[i]; i++)
		device_remove_file(dev, mce_device_attrs[i]);
I
Ingo Molnar 已提交
2349

2350
	for (i = 0; i < mca_cfg.banks; i++)
2351
		device_remove_file(dev, &mce_banks[i].attr);
I
Ingo Molnar 已提交
2352

2353 2354
	device_unregister(dev);
	cpumask_clear_cpu(cpu, mce_device_initialized);
2355
	per_cpu(mce_device, cpu) = NULL;
2356 2357
}

2358
/* Make sure there are no machine checks on offlined CPUs. */
2359
static void mce_disable_cpu(void *h)
2360
{
A
Andi Kleen 已提交
2361
	unsigned long action = *(unsigned long *)h;
2362

2363
	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2364
		return;
2365

A
Andi Kleen 已提交
2366 2367
	if (!(action & CPU_TASKS_FROZEN))
		cmci_clear();
2368

2369
	vendor_disable_error_reporting();
2370 2371
}

2372
static void mce_reenable_cpu(void *h)
2373
{
A
Andi Kleen 已提交
2374
	unsigned long action = *(unsigned long *)h;
I
Ingo Molnar 已提交
2375
	int i;
2376

2377
	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2378
		return;
I
Ingo Molnar 已提交
2379

A
Andi Kleen 已提交
2380 2381
	if (!(action & CPU_TASKS_FROZEN))
		cmci_reenable();
2382
	for (i = 0; i < mca_cfg.banks; i++) {
2383
		struct mce_bank *b = &mce_banks[i];
2384

2385
		if (b->init)
2386
			wrmsrl(MSR_IA32_MCx_CTL(i), b->ctl);
2387
	}
2388 2389
}

2390
/* Get notified when a cpu comes on/off. Be hotplug friendly. */
2391
static int
I
Ingo Molnar 已提交
2392
mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
2393 2394
{
	unsigned int cpu = (unsigned long)hcpu;
2395
	struct timer_list *t = &per_cpu(mce_timer, cpu);
2396

2397
	switch (action & ~CPU_TASKS_FROZEN) {
2398
	case CPU_ONLINE:
2399
		mce_device_create(cpu);
2400 2401
		if (threshold_cpu_callback)
			threshold_cpu_callback(action, cpu);
2402 2403
		break;
	case CPU_DEAD:
2404 2405
		if (threshold_cpu_callback)
			threshold_cpu_callback(action, cpu);
2406
		mce_device_remove(cpu);
C
Chen Gong 已提交
2407
		mce_intel_hcpu_update(cpu);
B
Borislav Petkov 已提交
2408 2409 2410 2411

		/* intentionally ignoring frozen here */
		if (!(action & CPU_TASKS_FROZEN))
			cmci_rediscover();
2412
		break;
2413
	case CPU_DOWN_PREPARE:
A
Andi Kleen 已提交
2414
		smp_call_function_single(cpu, mce_disable_cpu, &action, 1);
C
Chen Gong 已提交
2415
		del_timer_sync(t);
2416 2417
		break;
	case CPU_DOWN_FAILED:
A
Andi Kleen 已提交
2418
		smp_call_function_single(cpu, mce_reenable_cpu, &action, 1);
T
Thomas Gleixner 已提交
2419
		mce_start_timer(cpu, t);
A
Andi Kleen 已提交
2420
		break;
2421 2422
	}

2423
	return NOTIFY_OK;
2424 2425
}

2426
static struct notifier_block mce_cpu_notifier = {
2427 2428 2429
	.notifier_call = mce_cpu_callback,
};

2430
static __init void mce_init_banks(void)
2431 2432 2433
{
	int i;

2434
	for (i = 0; i < mca_cfg.banks; i++) {
2435
		struct mce_bank *b = &mce_banks[i];
2436
		struct device_attribute *a = &b->attr;
I
Ingo Molnar 已提交
2437

2438
		sysfs_attr_init(&a->attr);
2439 2440
		a->attr.name	= b->attrname;
		snprintf(b->attrname, ATTR_LEN, "bank%d", i);
I
Ingo Molnar 已提交
2441 2442 2443 2444

		a->attr.mode	= 0644;
		a->show		= show_bank;
		a->store	= set_bank;
2445 2446 2447
	}
}

2448
static __init int mcheck_init_device(void)
2449 2450 2451 2452
{
	int err;
	int i = 0;

2453 2454 2455 2456
	if (!mce_available(&boot_cpu_data)) {
		err = -EIO;
		goto err_out;
	}
2457

2458 2459 2460 2461
	if (!zalloc_cpumask_var(&mce_device_initialized, GFP_KERNEL)) {
		err = -ENOMEM;
		goto err_out;
	}
2462

2463
	mce_init_banks();
2464

2465
	err = subsys_system_register(&mce_subsys, NULL);
2466
	if (err)
2467
		goto err_out_mem;
2468

2469
	cpu_notifier_register_begin();
2470
	for_each_online_cpu(i) {
2471
		err = mce_device_create(i);
2472
		if (err) {
2473 2474 2475 2476 2477 2478
			/*
			 * Register notifier anyway (and do not unreg it) so
			 * that we don't leave undeleted timers, see notifier
			 * callback above.
			 */
			__register_hotcpu_notifier(&mce_cpu_notifier);
2479
			cpu_notifier_register_done();
2480
			goto err_device_create;
2481
		}
2482 2483
	}

2484 2485
	__register_hotcpu_notifier(&mce_cpu_notifier);
	cpu_notifier_register_done();
2486

2487 2488
	register_syscore_ops(&mce_syscore_ops);

2489
	/* register character device /dev/mcelog */
2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513
	err = misc_register(&mce_chrdev_device);
	if (err)
		goto err_register;

	return 0;

err_register:
	unregister_syscore_ops(&mce_syscore_ops);

err_device_create:
	/*
	 * We didn't keep track of which devices were created above, but
	 * even if we had, the set of online cpus might have changed.
	 * Play safe and remove for every possible cpu, since
	 * mce_device_remove() will do the right thing.
	 */
	for_each_possible_cpu(i)
		mce_device_remove(i);

err_out_mem:
	free_cpumask_var(mce_device_initialized);

err_out:
	pr_err("Unable to init device /dev/mcelog (rc: %d)\n", err);
I
Ingo Molnar 已提交
2514

L
Linus Torvalds 已提交
2515 2516
	return err;
}
2517
device_initcall_sync(mcheck_init_device);
I
Ingo Molnar 已提交
2518

2519 2520 2521 2522 2523
/*
 * Old style boot options parsing. Only for compatibility.
 */
static int __init mcheck_disable(char *str)
{
2524
	mca_cfg.disabled = true;
2525 2526 2527
	return 1;
}
__setup("nomce", mcheck_disable);
I
Ingo Molnar 已提交
2528

2529 2530
#ifdef CONFIG_DEBUG_FS
struct dentry *mce_get_debugfs_dir(void)
I
Ingo Molnar 已提交
2531
{
2532
	static struct dentry *dmce;
I
Ingo Molnar 已提交
2533

2534 2535
	if (!dmce)
		dmce = debugfs_create_dir("mce", NULL);
I
Ingo Molnar 已提交
2536

2537 2538
	return dmce;
}
I
Ingo Molnar 已提交
2539

2540 2541 2542
static void mce_reset(void)
{
	cpu_missing = 0;
2543
	atomic_set(&mce_fake_panicked, 0);
2544 2545 2546 2547
	atomic_set(&mce_executing, 0);
	atomic_set(&mce_callin, 0);
	atomic_set(&global_nwo, 0);
}
I
Ingo Molnar 已提交
2548

2549 2550 2551 2552
static int fake_panic_get(void *data, u64 *val)
{
	*val = fake_panic;
	return 0;
I
Ingo Molnar 已提交
2553 2554
}

2555
static int fake_panic_set(void *data, u64 val)
I
Ingo Molnar 已提交
2556
{
2557 2558 2559
	mce_reset();
	fake_panic = val;
	return 0;
I
Ingo Molnar 已提交
2560 2561
}

2562 2563
DEFINE_SIMPLE_ATTRIBUTE(fake_panic_fops, fake_panic_get,
			fake_panic_set, "%llu\n");
2564

2565
static int __init mcheck_debugfs_init(void)
2566
{
2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577
	struct dentry *dmce, *ffake_panic;

	dmce = mce_get_debugfs_dir();
	if (!dmce)
		return -ENOMEM;
	ffake_panic = debugfs_create_file("fake_panic", 0444, dmce, NULL,
					  &fake_panic_fops);
	if (!ffake_panic)
		return -ENOMEM;

	return 0;
2578
}
2579 2580
#else
static int __init mcheck_debugfs_init(void) { return -EINVAL; }
2581
#endif
2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595

static int __init mcheck_late_init(void)
{
	mcheck_debugfs_init();

	/*
	 * Flush out everything that has been logged during early boot, now that
	 * everything has been initialized (workqueues, decoders, ...).
	 */
	mce_schedule_work();

	return 0;
}
late_initcall(mcheck_late_init);