mce.c 61.7 KB
Newer Older
L
Linus Torvalds 已提交
1 2
/*
 * Machine check handler.
I
Ingo Molnar 已提交
3
 *
L
Linus Torvalds 已提交
4
 * K8 parts Copyright 2002,2003 Andi Kleen, SuSE Labs.
5 6
 * Rest from unknown author(s).
 * 2004 Andi Kleen. Rewrote most of it.
7 8
 * Copyright 2008 Intel Corporation
 * Author: Andi Kleen
L
Linus Torvalds 已提交
9
 */
10 11 12

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

I
Ingo Molnar 已提交
13 14 15 16 17 18 19
#include <linux/thread_info.h>
#include <linux/capability.h>
#include <linux/miscdevice.h>
#include <linux/ratelimit.h>
#include <linux/kallsyms.h>
#include <linux/rcupdate.h>
#include <linux/kobject.h>
20
#include <linux/uaccess.h>
I
Ingo Molnar 已提交
21 22 23
#include <linux/kdebug.h>
#include <linux/kernel.h>
#include <linux/percpu.h>
L
Linus Torvalds 已提交
24
#include <linux/string.h>
25
#include <linux/device.h>
26
#include <linux/syscore_ops.h>
27
#include <linux/delay.h>
28
#include <linux/ctype.h>
I
Ingo Molnar 已提交
29
#include <linux/sched.h>
30
#include <linux/sysfs.h>
I
Ingo Molnar 已提交
31
#include <linux/types.h>
32
#include <linux/slab.h>
I
Ingo Molnar 已提交
33 34 35
#include <linux/init.h>
#include <linux/kmod.h>
#include <linux/poll.h>
36
#include <linux/nmi.h>
I
Ingo Molnar 已提交
37
#include <linux/cpu.h>
38
#include <linux/smp.h>
I
Ingo Molnar 已提交
39
#include <linux/fs.h>
40
#include <linux/mm.h>
41
#include <linux/debugfs.h>
42
#include <linux/irq_work.h>
43
#include <linux/export.h>
I
Ingo Molnar 已提交
44

45
#include <asm/processor.h>
46
#include <asm/traps.h>
A
Andy Lutomirski 已提交
47
#include <asm/tlbflush.h>
I
Ingo Molnar 已提交
48 49
#include <asm/mce.h>
#include <asm/msr.h>
L
Linus Torvalds 已提交
50

51
#include "mce-internal.h"
52

53
static DEFINE_MUTEX(mce_chrdev_read_mutex);
54

55
#define mce_log_get_idx_check(p) \
56
({ \
57 58
	RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() && \
			 !lockdep_is_held(&mce_chrdev_read_mutex), \
59
			 "suspicious mce_log_get_idx_check() usage"); \
60 61
	smp_load_acquire(&(p)); \
})
62

63 64 65
#define CREATE_TRACE_POINTS
#include <trace/events/mce.h>

66
#define SPINUNIT		100	/* 100ns */
67

68 69
DEFINE_PER_CPU(unsigned, mce_exception_count);

70
struct mce_bank *mce_banks __read_mostly;
71
struct mce_vendor_flags mce_flags __read_mostly;
72

73
struct mca_config mca_cfg __read_mostly = {
74
	.bootlog  = -1,
75 76 77 78 79 80 81
	/*
	 * Tolerant levels:
	 * 0: always panic on uncorrected errors, log corrected errors
	 * 1: panic or SIGBUS on uncorrected errors, log corrected errors
	 * 2: SIGBUS or log uncorrected errors (if possible), log corr. errors
	 * 3: never panic or SIGBUS, log all errors (for testing only)
	 */
82 83
	.tolerant = 1,
	.monarch_timeout = -1
84 85
};

86 87 88 89
/* User mode helper program triggered by machine check event */
static unsigned long		mce_need_notify;
static char			mce_helper[128];
static char			*mce_helper_argv[2] = { mce_helper, NULL };
L
Linus Torvalds 已提交
90

91 92
static DECLARE_WAIT_QUEUE_HEAD(mce_chrdev_wait);

93 94 95
static DEFINE_PER_CPU(struct mce, mces_seen);
static int			cpu_missing;

96 97 98 99
/*
 * MCA banks polled by the period polling timer for corrected events.
 * With Intel CMCI, this only has MCA banks which do not support CMCI (if any).
 */
100 101 102 103
DEFINE_PER_CPU(mce_banks_t, mce_poll_banks) = {
	[0 ... BITS_TO_LONGS(MAX_NR_BANKS)-1] = ~0UL
};

104 105 106 107 108 109 110 111 112
/*
 * MCA banks controlled through firmware first for corrected errors.
 * This is a global list of banks for which we won't enable CMCI and we
 * won't poll. Firmware controls these banks and is responsible for
 * reporting corrected errors through GHES. Uncorrected/recoverable
 * errors are still notified through a machine check.
 */
mce_banks_t mce_banks_ce_disabled;

113 114
static struct work_struct mce_work;
static struct irq_work mce_irq_work;
115

116 117
static void (*quirk_no_way_out)(int bank, struct mce *m, struct pt_regs *regs);

118 119 120 121
/*
 * CPU/chipset specific EDAC code can register a notifier call here to print
 * MCE errors in a human-readable form.
 */
122
ATOMIC_NOTIFIER_HEAD(x86_mce_decoder_chain);
123

124 125 126 127
/* Do initial initialization of a struct mce */
void mce_setup(struct mce *m)
{
	memset(m, 0, sizeof(struct mce));
128
	m->cpu = m->extcpu = smp_processor_id();
129
	m->tsc = rdtsc();
130 131 132 133 134 135 136
	/* We hope get_seconds stays lockless */
	m->time = get_seconds();
	m->cpuvendor = boot_cpu_data.x86_vendor;
	m->cpuid = cpuid_eax(1);
	m->socketid = cpu_data(m->extcpu).phys_proc_id;
	m->apicid = cpu_data(m->extcpu).initial_apicid;
	rdmsrl(MSR_IA32_MCG_CAP, m->mcgcap);
137 138
}

139 140 141
DEFINE_PER_CPU(struct mce, injectm);
EXPORT_PER_CPU_SYMBOL_GPL(injectm);

L
Linus Torvalds 已提交
142 143 144 145 146 147
/*
 * Lockless MCE logging infrastructure.
 * This avoids deadlocks on printk locks without having to break locks. Also
 * separate MCEs from kernel messages to avoid bogus bug reports.
 */

148
static struct mce_log mcelog = {
149 150 151
	.signature	= MCE_LOG_SIGNATURE,
	.len		= MCE_LOG_LEN,
	.recordlen	= sizeof(struct mce),
152
};
L
Linus Torvalds 已提交
153 154 155 156

void mce_log(struct mce *mce)
{
	unsigned next, entry;
I
Ingo Molnar 已提交
157

158 159 160
	/* Emit the trace record: */
	trace_mce_record(mce);

161 162
	if (!mce_gen_pool_add(mce))
		irq_work_queue(&mce_irq_work);
163

M
Mike Waychison 已提交
164
	wmb();
L
Linus Torvalds 已提交
165
	for (;;) {
166
		entry = mce_log_get_idx_check(mcelog.next);
167
		for (;;) {
168

I
Ingo Molnar 已提交
169 170 171 172 173
			/*
			 * When the buffer fills up discard new entries.
			 * Assume that the earlier errors are the more
			 * interesting ones:
			 */
174
			if (entry >= MCE_LOG_LEN) {
175 176
				set_bit(MCE_OVERFLOW,
					(unsigned long *)&mcelog.flags);
177 178
				return;
			}
I
Ingo Molnar 已提交
179
			/* Old left over entry. Skip: */
180 181 182 183
			if (mcelog.entry[entry].finished) {
				entry++;
				continue;
			}
M
Mike Waychison 已提交
184
			break;
L
Linus Torvalds 已提交
185 186 187 188 189 190 191
		}
		smp_rmb();
		next = entry + 1;
		if (cmpxchg(&mcelog.next, entry, next) == entry)
			break;
	}
	memcpy(mcelog.entry + entry, mce, sizeof(struct mce));
M
Mike Waychison 已提交
192
	wmb();
L
Linus Torvalds 已提交
193
	mcelog.entry[entry].finished = 1;
M
Mike Waychison 已提交
194
	wmb();
L
Linus Torvalds 已提交
195

196
	set_bit(0, &mce_need_notify);
L
Linus Torvalds 已提交
197 198
}

199
void mce_inject_log(struct mce *m)
B
Borislav Petkov 已提交
200
{
201 202 203
	mutex_lock(&mce_chrdev_read_mutex);
	mce_log(m);
	mutex_unlock(&mce_chrdev_read_mutex);
B
Borislav Petkov 已提交
204
}
205
EXPORT_SYMBOL_GPL(mce_inject_log);
B
Borislav Petkov 已提交
206

207
static struct notifier_block mce_srao_nb;
B
Borislav Petkov 已提交
208

209 210
void mce_register_decode_chain(struct notifier_block *nb)
{
211 212 213 214
	/* Ensure SRAO notifier has the highest priority in the decode chain. */
	if (nb != &mce_srao_nb && nb->priority == INT_MAX)
		nb->priority -= 1;

215 216 217 218 219 220 221 222 223 224
	atomic_notifier_chain_register(&x86_mce_decoder_chain, nb);
}
EXPORT_SYMBOL_GPL(mce_register_decode_chain);

void mce_unregister_decode_chain(struct notifier_block *nb)
{
	atomic_notifier_chain_unregister(&x86_mce_decoder_chain, nb);
}
EXPORT_SYMBOL_GPL(mce_unregister_decode_chain);

225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271
static inline u32 ctl_reg(int bank)
{
	return MSR_IA32_MCx_CTL(bank);
}

static inline u32 status_reg(int bank)
{
	return MSR_IA32_MCx_STATUS(bank);
}

static inline u32 addr_reg(int bank)
{
	return MSR_IA32_MCx_ADDR(bank);
}

static inline u32 misc_reg(int bank)
{
	return MSR_IA32_MCx_MISC(bank);
}

static inline u32 smca_ctl_reg(int bank)
{
	return MSR_AMD64_SMCA_MCx_CTL(bank);
}

static inline u32 smca_status_reg(int bank)
{
	return MSR_AMD64_SMCA_MCx_STATUS(bank);
}

static inline u32 smca_addr_reg(int bank)
{
	return MSR_AMD64_SMCA_MCx_ADDR(bank);
}

static inline u32 smca_misc_reg(int bank)
{
	return MSR_AMD64_SMCA_MCx_MISC(bank);
}

struct mca_msr_regs msr_ops = {
	.ctl	= ctl_reg,
	.status	= status_reg,
	.addr	= addr_reg,
	.misc	= misc_reg
};

H
Hidetoshi Seto 已提交
272
static void print_mce(struct mce *m)
L
Linus Torvalds 已提交
273
{
274 275
	int ret = 0;

H
Huang Ying 已提交
276
	pr_emerg(HW_ERR "CPU %d: Machine Check Exception: %Lx Bank %d: %016Lx\n",
277
	       m->extcpu, m->mcgstatus, m->bank, m->status);
278

279
	if (m->ip) {
H
Huang Ying 已提交
280
		pr_emerg(HW_ERR "RIP%s %02x:<%016Lx> ",
281 282 283
			!(m->mcgstatus & MCG_STATUS_EIPV) ? " !INEXACT!" : "",
				m->cs, m->ip);

L
Linus Torvalds 已提交
284
		if (m->cs == __KERNEL_CS)
285
			print_symbol("{%s}", m->ip);
286
		pr_cont("\n");
L
Linus Torvalds 已提交
287
	}
288

H
Huang Ying 已提交
289
	pr_emerg(HW_ERR "TSC %llx ", m->tsc);
L
Linus Torvalds 已提交
290
	if (m->addr)
291
		pr_cont("ADDR %llx ", m->addr);
L
Linus Torvalds 已提交
292
	if (m->misc)
293
		pr_cont("MISC %llx ", m->misc);
294

295
	pr_cont("\n");
296 297 298 299
	/*
	 * Note this output is parsed by external tools and old fields
	 * should not be changed.
	 */
300
	pr_emerg(HW_ERR "PROCESSOR %u:%x TIME %llu SOCKET %u APIC %x microcode %x\n",
301 302
		m->cpuvendor, m->cpuid, m->time, m->socketid, m->apicid,
		cpu_data(m->extcpu).microcode);
303 304 305

	/*
	 * Print out human-readable details about the MCE error,
306
	 * (if the CPU has an implementation for that)
307
	 */
308 309 310 311 312
	ret = atomic_notifier_call_chain(&x86_mce_decoder_chain, 0, m);
	if (ret == NOTIFY_STOP)
		return;

	pr_emerg_ratelimited(HW_ERR "Run the above through 'mcelog --ascii'\n");
313 314
}

315 316
#define PANIC_TIMEOUT 5 /* 5 seconds */

317
static atomic_t mce_panicked;
318

319
static int fake_panic;
320
static atomic_t mce_fake_panicked;
321

322 323 324 325
/* Panic in progress. Enable interrupts and wait for final IPI */
static void wait_for_panic(void)
{
	long timeout = PANIC_TIMEOUT*USEC_PER_SEC;
326

327 328 329 330
	preempt_disable();
	local_irq_enable();
	while (timeout-- > 0)
		udelay(1);
331
	if (panic_timeout == 0)
332
		panic_timeout = mca_cfg.panic_timeout;
333 334 335
	panic("Panicing machine check CPU died");
}

336
static void mce_panic(const char *msg, struct mce *final, char *exp)
337
{
338 339 340
	int apei_err = 0;
	struct llist_node *pending;
	struct mce_evt_llist *l;
341

342 343 344 345
	if (!fake_panic) {
		/*
		 * Make sure only one CPU runs in machine check panic
		 */
346
		if (atomic_inc_return(&mce_panicked) > 1)
347 348
			wait_for_panic();
		barrier();
349

350 351 352 353
		bust_spinlocks(1);
		console_verbose();
	} else {
		/* Don't log too much for fake panic */
354
		if (atomic_inc_return(&mce_fake_panicked) > 1)
355 356
			return;
	}
357
	pending = mce_gen_pool_prepare_records();
358
	/* First print corrected ones that are still unlogged */
359 360
	llist_for_each_entry(l, pending, llnode) {
		struct mce *m = &l->mce;
361
		if (!(m->status & MCI_STATUS_UC)) {
H
Hidetoshi Seto 已提交
362
			print_mce(m);
363 364 365
			if (!apei_err)
				apei_err = apei_write_mce(m);
		}
366 367
	}
	/* Now print uncorrected but with the final one last */
368 369
	llist_for_each_entry(l, pending, llnode) {
		struct mce *m = &l->mce;
H
Hidetoshi Seto 已提交
370 371
		if (!(m->status & MCI_STATUS_UC))
			continue;
372
		if (!final || mce_cmp(m, final)) {
H
Hidetoshi Seto 已提交
373
			print_mce(m);
374 375 376
			if (!apei_err)
				apei_err = apei_write_mce(m);
		}
L
Linus Torvalds 已提交
377
	}
378
	if (final) {
H
Hidetoshi Seto 已提交
379
		print_mce(final);
380 381 382
		if (!apei_err)
			apei_err = apei_write_mce(final);
	}
383
	if (cpu_missing)
H
Huang Ying 已提交
384
		pr_emerg(HW_ERR "Some CPUs didn't answer in synchronization\n");
385
	if (exp)
H
Huang Ying 已提交
386
		pr_emerg(HW_ERR "Machine check: %s\n", exp);
387 388
	if (!fake_panic) {
		if (panic_timeout == 0)
389
			panic_timeout = mca_cfg.panic_timeout;
390 391
		panic(msg);
	} else
H
Huang Ying 已提交
392
		pr_emerg(HW_ERR "Fake kernel panic: %s\n", msg);
393
}
L
Linus Torvalds 已提交
394

395 396 397 398
/* Support code for software error injection */

static int msr_to_offset(u32 msr)
{
T
Tejun Heo 已提交
399
	unsigned bank = __this_cpu_read(injectm.bank);
400

401
	if (msr == mca_cfg.rip_msr)
402
		return offsetof(struct mce, ip);
403
	if (msr == msr_ops.status(bank))
404
		return offsetof(struct mce, status);
405
	if (msr == msr_ops.addr(bank))
406
		return offsetof(struct mce, addr);
407
	if (msr == msr_ops.misc(bank))
408 409 410 411 412 413
		return offsetof(struct mce, misc);
	if (msr == MSR_IA32_MCG_STATUS)
		return offsetof(struct mce, mcgstatus);
	return -1;
}

414 415 416 417
/* MSR access wrappers used for error injection */
static u64 mce_rdmsrl(u32 msr)
{
	u64 v;
418

T
Tejun Heo 已提交
419
	if (__this_cpu_read(injectm.finished)) {
420
		int offset = msr_to_offset(msr);
421

422 423
		if (offset < 0)
			return 0;
424
		return *(u64 *)((char *)this_cpu_ptr(&injectm) + offset);
425
	}
426 427 428 429 430 431 432 433 434 435 436

	if (rdmsrl_safe(msr, &v)) {
		WARN_ONCE(1, "mce: Unable to read msr %d!\n", msr);
		/*
		 * Return zero in case the access faulted. This should
		 * not happen normally but can happen if the CPU does
		 * something weird, or if the code is buggy.
		 */
		v = 0;
	}

437 438 439 440 441
	return v;
}

static void mce_wrmsrl(u32 msr, u64 v)
{
T
Tejun Heo 已提交
442
	if (__this_cpu_read(injectm.finished)) {
443
		int offset = msr_to_offset(msr);
444

445
		if (offset >= 0)
446
			*(u64 *)((char *)this_cpu_ptr(&injectm) + offset) = v;
447 448
		return;
	}
449 450 451
	wrmsrl(msr, v);
}

452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469
/*
 * Collect all global (w.r.t. this processor) status about this machine
 * check into our "mce" struct so that we can use it later to assess
 * the severity of the problem as we read per-bank specific details.
 */
static inline void mce_gather_info(struct mce *m, struct pt_regs *regs)
{
	mce_setup(m);

	m->mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS);
	if (regs) {
		/*
		 * Get the address of the instruction at the time of
		 * the machine check error.
		 */
		if (m->mcgstatus & (MCG_STATUS_RIPV|MCG_STATUS_EIPV)) {
			m->ip = regs->ip;
			m->cs = regs->cs;
470 471 472 473 474 475 476 477

			/*
			 * When in VM86 mode make the cs look like ring 3
			 * always. This is a lie, but it's better than passing
			 * the additional vm86 bit around everywhere.
			 */
			if (v8086_mode(regs))
				m->cs |= 3;
478 479
		}
		/* Use accurate RIP reporting if available. */
480 481
		if (mca_cfg.rip_msr)
			m->ip = mce_rdmsrl(mca_cfg.rip_msr);
482 483 484
	}
}

A
Andi Kleen 已提交
485
int mce_available(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
486
{
487
	if (mca_cfg.disabled)
488
		return 0;
489
	return cpu_has(c, X86_FEATURE_MCE) && cpu_has(c, X86_FEATURE_MCA);
L
Linus Torvalds 已提交
490 491
}

492 493
static void mce_schedule_work(void)
{
494
	if (!mce_gen_pool_empty() && keventd_up())
495
		schedule_work(&mce_work);
496 497
}

498
static void mce_irq_work_cb(struct irq_work *entry)
499
{
500
	mce_notify_irq();
501
	mce_schedule_work();
502 503 504 505 506
}

static void mce_report_event(struct pt_regs *regs)
{
	if (regs->flags & (X86_VM_MASK|X86_EFLAGS_IF)) {
507
		mce_notify_irq();
508 509 510 511 512 513 514
		/*
		 * Triggering the work queue here is just an insurance
		 * policy in case the syscall exit notify handler
		 * doesn't run soon enough or ends up running on the
		 * wrong CPU (can happen when audit sleeps)
		 */
		mce_schedule_work();
515 516 517
		return;
	}

518
	irq_work_queue(&mce_irq_work);
519 520
}

521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542
/*
 * Check if the address reported by the CPU is in a format we can parse.
 * It would be possible to add code for most other cases, but all would
 * be somewhat complicated (e.g. segment offset would require an instruction
 * parser). So only support physical addresses up to page granuality for now.
 */
static int mce_usable_address(struct mce *m)
{
	if (!(m->status & MCI_STATUS_MISCV) || !(m->status & MCI_STATUS_ADDRV))
		return 0;

	/* Checks after this one are Intel-specific: */
	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
		return 1;

	if (MCI_MISC_ADDR_LSB(m->misc) > PAGE_SHIFT)
		return 0;
	if (MCI_MISC_ADDR_MODE(m->misc) != MCI_MISC_ADDR_PHYS)
		return 0;
	return 1;
}

543 544 545 546 547 548 549 550 551
static int srao_decode_notifier(struct notifier_block *nb, unsigned long val,
				void *data)
{
	struct mce *mce = (struct mce *)data;
	unsigned long pfn;

	if (!mce)
		return NOTIFY_DONE;

B
Borislav Petkov 已提交
552
	if (mce_usable_address(mce) && (mce->severity == MCE_AO_SEVERITY)) {
553 554 555 556 557
		pfn = mce->addr >> PAGE_SHIFT;
		memory_failure(pfn, MCE_VECTOR, 0);
	}

	return NOTIFY_OK;
558
}
559 560 561 562
static struct notifier_block mce_srao_nb = {
	.notifier_call	= srao_decode_notifier,
	.priority = INT_MAX,
};
563

564 565 566 567 568 569
/*
 * Read ADDR and MISC registers.
 */
static void mce_read_aux(struct mce *m, int i)
{
	if (m->status & MCI_STATUS_MISCV)
570
		m->misc = mce_rdmsrl(msr_ops.misc(i));
571
	if (m->status & MCI_STATUS_ADDRV) {
572
		m->addr = mce_rdmsrl(msr_ops.addr(i));
573 574 575 576

		/*
		 * Mask the reported address by the reported granularity.
		 */
577
		if (mca_cfg.ser && (m->status & MCI_STATUS_MISCV)) {
578 579 580 581 582 583 584
			u8 shift = MCI_MISC_ADDR_LSB(m->misc);
			m->addr >>= shift;
			m->addr <<= shift;
		}
	}
}

585 586 587 588 589
static bool memory_error(struct mce *m)
{
	struct cpuinfo_x86 *c = &boot_cpu_data;

	if (c->x86_vendor == X86_VENDOR_AMD) {
590 591 592 593
		/* ErrCodeExt[20:16] */
		u8 xec = (m->status >> 16) & 0x1f;

		return (xec == 0x0 || xec == 0x8);
594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615
	} else if (c->x86_vendor == X86_VENDOR_INTEL) {
		/*
		 * Intel SDM Volume 3B - 15.9.2 Compound Error Codes
		 *
		 * Bit 7 of the MCACOD field of IA32_MCi_STATUS is used for
		 * indicating a memory error. Bit 8 is used for indicating a
		 * cache hierarchy error. The combination of bit 2 and bit 3
		 * is used for indicating a `generic' cache hierarchy error
		 * But we can't just blindly check the above bits, because if
		 * bit 11 is set, then it is a bus/interconnect error - and
		 * either way the above bits just gives more detail on what
		 * bus/interconnect error happened. Note that bit 12 can be
		 * ignored, as it's the "filter" bit.
		 */
		return (m->status & 0xef80) == BIT(7) ||
		       (m->status & 0xef00) == BIT(8) ||
		       (m->status & 0xeffc) == 0xc;
	}

	return false;
}

616 617
DEFINE_PER_CPU(unsigned, mce_poll_count);

618
/*
619 620 621 622
 * Poll for corrected events or events that happened before reset.
 * Those are just logged through /dev/mcelog.
 *
 * This is executed in standard interrupt context.
A
Andi Kleen 已提交
623 624 625 626 627 628 629 630 631
 *
 * Note: spec recommends to panic for fatal unsignalled
 * errors here. However this would be quite problematic --
 * we would need to reimplement the Monarch handling and
 * it would mess up the exclusion between exception handler
 * and poll hander -- * so we skip this for now.
 * These cases should not happen anyways, or only when the CPU
 * is already totally * confused. In this case it's likely it will
 * not fully execute the machine check handler either.
632
 */
633
bool machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
634
{
635
	bool error_seen = false;
636
	struct mce m;
637
	int severity;
638 639
	int i;

640
	this_cpu_inc(mce_poll_count);
641

642
	mce_gather_info(&m, NULL);
643

644
	for (i = 0; i < mca_cfg.banks; i++) {
645
		if (!mce_banks[i].ctl || !test_bit(i, *b))
646 647 648 649 650 651 652 653
			continue;

		m.misc = 0;
		m.addr = 0;
		m.bank = i;
		m.tsc = 0;

		barrier();
654
		m.status = mce_rdmsrl(msr_ops.status(i));
655 656 657
		if (!(m.status & MCI_STATUS_VAL))
			continue;

658

659
		/*
A
Andi Kleen 已提交
660 661
		 * Uncorrected or signalled events are handled by the exception
		 * handler when it is enabled, so don't process those here.
662 663 664
		 *
		 * TBD do the same check for MCI_STATUS_EN here?
		 */
A
Andi Kleen 已提交
665
		if (!(flags & MCP_UC) &&
666
		    (m.status & (mca_cfg.ser ? MCI_STATUS_S : MCI_STATUS_UC)))
667 668
			continue;

669 670
		error_seen = true;

671
		mce_read_aux(&m, i);
672 673 674

		if (!(flags & MCP_TIMESTAMP))
			m.tsc = 0;
675 676 677

		severity = mce_severity(&m, mca_cfg.tolerant, NULL, false);

B
Borislav Petkov 已提交
678 679
		if (severity == MCE_DEFERRED_SEVERITY && memory_error(&m))
			if (m.status & MCI_STATUS_ADDRV)
680
				m.severity = severity;
681

682 683 684 685
		/*
		 * Don't get the IP here because it's unlikely to
		 * have anything to do with the actual error location.
		 */
686
		if (!(flags & MCP_DONTLOG) && !mca_cfg.dont_log_ce)
A
Andi Kleen 已提交
687
			mce_log(&m);
B
Borislav Petkov 已提交
688
		else if (mce_usable_address(&m)) {
689 690 691 692 693 694 695
			/*
			 * Although we skipped logging this, we still want
			 * to take action. Add to the pool so the registered
			 * notifiers will see it.
			 */
			if (!mce_gen_pool_add(&m))
				mce_schedule_work();
696
		}
697 698 699 700

		/*
		 * Clear state for this bank.
		 */
701
		mce_wrmsrl(msr_ops.status(i), 0);
702 703 704 705 706 707
	}

	/*
	 * Don't clear MCG_STATUS here because it's only defined for
	 * exceptions.
	 */
708 709

	sync_core();
710

711
	return error_seen;
712
}
713
EXPORT_SYMBOL_GPL(machine_check_poll);
714

715 716 717 718
/*
 * Do a quick check if any of the events requires a panic.
 * This decides if we keep the events around or clear them.
 */
719 720
static int mce_no_way_out(struct mce *m, char **msg, unsigned long *validp,
			  struct pt_regs *regs)
721
{
722
	int i, ret = 0;
723
	char *tmp;
724

725
	for (i = 0; i < mca_cfg.banks; i++) {
726
		m->status = mce_rdmsrl(msr_ops.status(i));
727
		if (m->status & MCI_STATUS_VAL) {
728
			__set_bit(i, validp);
729 730 731
			if (quirk_no_way_out)
				quirk_no_way_out(i, m, regs);
		}
732 733 734

		if (mce_severity(m, mca_cfg.tolerant, &tmp, true) >= MCE_PANIC_SEVERITY) {
			*msg = tmp;
735
			ret = 1;
736
		}
737
	}
738
	return ret;
739 740
}

741 742 743 744 745 746 747 748 749 750 751 752 753 754
/*
 * Variable to establish order between CPUs while scanning.
 * Each CPU spins initially until executing is equal its number.
 */
static atomic_t mce_executing;

/*
 * Defines order of CPUs on entry. First CPU becomes Monarch.
 */
static atomic_t mce_callin;

/*
 * Check if a timeout waiting for other CPUs happened.
 */
755
static int mce_timed_out(u64 *t, const char *msg)
756 757 758 759 760 761 762 763
{
	/*
	 * The others already did panic for some reason.
	 * Bail out like in a timeout.
	 * rmb() to tell the compiler that system_state
	 * might have been modified by someone else.
	 */
	rmb();
764
	if (atomic_read(&mce_panicked))
765
		wait_for_panic();
766
	if (!mca_cfg.monarch_timeout)
767 768
		goto out;
	if ((s64)*t < SPINUNIT) {
769
		if (mca_cfg.tolerant <= 1)
770
			mce_panic(msg, NULL, NULL);
771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791
		cpu_missing = 1;
		return 1;
	}
	*t -= SPINUNIT;
out:
	touch_nmi_watchdog();
	return 0;
}

/*
 * The Monarch's reign.  The Monarch is the CPU who entered
 * the machine check handler first. It waits for the others to
 * raise the exception too and then grades them. When any
 * error is fatal panic. Only then let the others continue.
 *
 * The other CPUs entering the MCE handler will be controlled by the
 * Monarch. They are called Subjects.
 *
 * This way we prevent any potential data corruption in a unrecoverable case
 * and also makes sure always all CPU's errors are examined.
 *
792
 * Also this detects the case of a machine check event coming from outer
793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817
 * space (not detected by any CPUs) In this case some external agent wants
 * us to shut down, so panic too.
 *
 * The other CPUs might still decide to panic if the handler happens
 * in a unrecoverable place, but in this case the system is in a semi-stable
 * state and won't corrupt anything by itself. It's ok to let the others
 * continue for a bit first.
 *
 * All the spin loops have timeouts; when a timeout happens a CPU
 * typically elects itself to be Monarch.
 */
static void mce_reign(void)
{
	int cpu;
	struct mce *m = NULL;
	int global_worst = 0;
	char *msg = NULL;
	char *nmsg = NULL;

	/*
	 * This CPU is the Monarch and the other CPUs have run
	 * through their handlers.
	 * Grade the severity of the errors of all the CPUs.
	 */
	for_each_possible_cpu(cpu) {
818 819
		int severity = mce_severity(&per_cpu(mces_seen, cpu),
					    mca_cfg.tolerant,
820
					    &nmsg, true);
821 822 823 824 825 826 827 828 829 830 831 832
		if (severity > global_worst) {
			msg = nmsg;
			global_worst = severity;
			m = &per_cpu(mces_seen, cpu);
		}
	}

	/*
	 * Cannot recover? Panic here then.
	 * This dumps all the mces in the log buffer and stops the
	 * other CPUs.
	 */
833
	if (m && global_worst >= MCE_PANIC_SEVERITY && mca_cfg.tolerant < 3)
834
		mce_panic("Fatal machine check", m, msg);
835 836 837 838 839 840 841 842 843 844 845

	/*
	 * For UC somewhere we let the CPU who detects it handle it.
	 * Also must let continue the others, otherwise the handling
	 * CPU could deadlock on a lock.
	 */

	/*
	 * No machine check event found. Must be some external
	 * source or one CPU is hung. Panic.
	 */
846
	if (global_worst <= MCE_KEEP_SEVERITY && mca_cfg.tolerant < 3)
847
		mce_panic("Fatal machine check from unknown source", NULL, NULL);
848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865

	/*
	 * Now clear all the mces_seen so that they don't reappear on
	 * the next mce.
	 */
	for_each_possible_cpu(cpu)
		memset(&per_cpu(mces_seen, cpu), 0, sizeof(struct mce));
}

static atomic_t global_nwo;

/*
 * Start of Monarch synchronization. This waits until all CPUs have
 * entered the exception handler and then determines if any of them
 * saw a fatal event that requires panic. Then it executes them
 * in the entry order.
 * TBD double check parallel CPU hotunplug
 */
H
Hidetoshi Seto 已提交
866
static int mce_start(int *no_way_out)
867
{
H
Hidetoshi Seto 已提交
868
	int order;
869
	int cpus = num_online_cpus();
870
	u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC;
871

H
Hidetoshi Seto 已提交
872 873
	if (!timeout)
		return -1;
874

H
Hidetoshi Seto 已提交
875
	atomic_add(*no_way_out, &global_nwo);
876
	/*
877 878
	 * Rely on the implied barrier below, such that global_nwo
	 * is updated before mce_callin.
879
	 */
880
	order = atomic_inc_return(&mce_callin);
881 882 883 884 885

	/*
	 * Wait for everyone.
	 */
	while (atomic_read(&mce_callin) != cpus) {
886 887
		if (mce_timed_out(&timeout,
				  "Timeout: Not all CPUs entered broadcast exception handler")) {
888
			atomic_set(&global_nwo, 0);
H
Hidetoshi Seto 已提交
889
			return -1;
890 891 892 893
		}
		ndelay(SPINUNIT);
	}

894 895 896 897
	/*
	 * mce_callin should be read before global_nwo
	 */
	smp_rmb();
898

H
Hidetoshi Seto 已提交
899 900 901 902
	if (order == 1) {
		/*
		 * Monarch: Starts executing now, the others wait.
		 */
903
		atomic_set(&mce_executing, 1);
H
Hidetoshi Seto 已提交
904 905 906 907 908 909 910 911
	} else {
		/*
		 * Subject: Now start the scanning loop one by one in
		 * the original callin order.
		 * This way when there are any shared banks it will be
		 * only seen by one CPU before cleared, avoiding duplicates.
		 */
		while (atomic_read(&mce_executing) < order) {
912 913
			if (mce_timed_out(&timeout,
					  "Timeout: Subject CPUs unable to finish machine check processing")) {
H
Hidetoshi Seto 已提交
914 915 916 917 918
				atomic_set(&global_nwo, 0);
				return -1;
			}
			ndelay(SPINUNIT);
		}
919 920 921
	}

	/*
H
Hidetoshi Seto 已提交
922
	 * Cache the global no_way_out state.
923
	 */
H
Hidetoshi Seto 已提交
924 925 926
	*no_way_out = atomic_read(&global_nwo);

	return order;
927 928 929 930 931 932 933 934 935
}

/*
 * Synchronize between CPUs after main scanning loop.
 * This invokes the bulk of the Monarch processing.
 */
static int mce_end(int order)
{
	int ret = -1;
936
	u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC;
937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956

	if (!timeout)
		goto reset;
	if (order < 0)
		goto reset;

	/*
	 * Allow others to run.
	 */
	atomic_inc(&mce_executing);

	if (order == 1) {
		/* CHECKME: Can this race with a parallel hotplug? */
		int cpus = num_online_cpus();

		/*
		 * Monarch: Wait for everyone to go through their scanning
		 * loops.
		 */
		while (atomic_read(&mce_executing) <= cpus) {
957 958
			if (mce_timed_out(&timeout,
					  "Timeout: Monarch CPU unable to finish machine check processing"))
959 960 961 962 963 964 965 966 967 968 969 970
				goto reset;
			ndelay(SPINUNIT);
		}

		mce_reign();
		barrier();
		ret = 0;
	} else {
		/*
		 * Subject: Wait for Monarch to finish.
		 */
		while (atomic_read(&mce_executing) != 0) {
971 972
			if (mce_timed_out(&timeout,
					  "Timeout: Monarch CPU did not finish machine check processing"))
973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001
				goto reset;
			ndelay(SPINUNIT);
		}

		/*
		 * Don't reset anything. That's done by the Monarch.
		 */
		return 0;
	}

	/*
	 * Reset all global state.
	 */
reset:
	atomic_set(&global_nwo, 0);
	atomic_set(&mce_callin, 0);
	barrier();

	/*
	 * Let others run again.
	 */
	atomic_set(&mce_executing, 0);
	return ret;
}

static void mce_clear_state(unsigned long *toclear)
{
	int i;

1002
	for (i = 0; i < mca_cfg.banks; i++) {
1003
		if (test_bit(i, toclear))
1004
			mce_wrmsrl(msr_ops.status(i), 0);
1005 1006 1007
	}
}

1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021
static int do_memory_failure(struct mce *m)
{
	int flags = MF_ACTION_REQUIRED;
	int ret;

	pr_err("Uncorrected hardware memory error in user-access at %llx", m->addr);
	if (!(m->mcgstatus & MCG_STATUS_RIPV))
		flags |= MF_MUST_KILL;
	ret = memory_failure(m->addr >> PAGE_SHIFT, MCE_VECTOR, flags);
	if (ret)
		pr_err("Memory error not recovered");
	return ret;
}

1022 1023 1024 1025 1026 1027 1028
/*
 * The actual machine check handler. This only handles real
 * exceptions when something got corrupted coming in through int 18.
 *
 * This is executed in NMI context not subject to normal locking rules. This
 * implies that most kernel services cannot be safely used. Don't even
 * think about putting a printk in there!
1029 1030 1031 1032
 *
 * On Intel systems this is entered on all CPUs in parallel through
 * MCE broadcast. However some CPUs might be broken beyond repair,
 * so be always careful when synchronizing with others.
L
Linus Torvalds 已提交
1033
 */
I
Ingo Molnar 已提交
1034
void do_machine_check(struct pt_regs *regs, long error_code)
L
Linus Torvalds 已提交
1035
{
1036
	struct mca_config *cfg = &mca_cfg;
1037
	struct mce m, *final;
L
Linus Torvalds 已提交
1038
	int i;
1039 1040
	int worst = 0;
	int severity;
1041

1042 1043 1044 1045
	/*
	 * Establish sequential order between the CPUs entering the machine
	 * check handler.
	 */
1046
	int order = -1;
1047 1048
	/*
	 * If no_way_out gets set, there is no safe way to recover from this
1049
	 * MCE.  If mca_cfg.tolerant is cranked up, we'll try anyway.
1050 1051 1052 1053 1054 1055 1056
	 */
	int no_way_out = 0;
	/*
	 * If kill_it gets set, there might be a way to recover from this
	 * error.
	 */
	int kill_it = 0;
1057
	DECLARE_BITMAP(toclear, MAX_NR_BANKS);
1058
	DECLARE_BITMAP(valid_banks, MAX_NR_BANKS);
1059
	char *msg = "Unknown";
1060 1061 1062 1063 1064 1065

	/*
	 * MCEs are always local on AMD. Same is determined by MCG_STATUS_LMCES
	 * on Intel.
	 */
	int lmce = 1;
L
Linus Torvalds 已提交
1066

1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077
	/* If this CPU is offline, just bail out. */
	if (cpu_is_offline(smp_processor_id())) {
		u64 mcgstatus;

		mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS);
		if (mcgstatus & MCG_STATUS_RIPV) {
			mce_wrmsrl(MSR_IA32_MCG_STATUS, 0);
			return;
		}
	}

1078
	ist_enter(regs);
1079

1080
	this_cpu_inc(mce_exception_count);
1081

1082
	if (!cfg->banks)
1083
		goto out;
L
Linus Torvalds 已提交
1084

1085
	mce_gather_info(&m, regs);
1086

1087
	final = this_cpu_ptr(&mces_seen);
1088 1089
	*final = m;

1090
	memset(valid_banks, 0, sizeof(valid_banks));
1091
	no_way_out = mce_no_way_out(&m, &msg, valid_banks, regs);
1092

L
Linus Torvalds 已提交
1093 1094
	barrier();

A
Andi Kleen 已提交
1095
	/*
1096 1097 1098
	 * When no restart IP might need to kill or panic.
	 * Assume the worst for now, but if we find the
	 * severity is MCE_AR_SEVERITY we have other options.
A
Andi Kleen 已提交
1099 1100 1101 1102
	 */
	if (!(m.mcgstatus & MCG_STATUS_RIPV))
		kill_it = 1;

1103
	/*
1104 1105
	 * Check if this MCE is signaled to only this logical processor,
	 * on Intel only.
1106
	 */
1107 1108 1109 1110 1111 1112 1113 1114 1115 1116
	if (m.cpuvendor == X86_VENDOR_INTEL)
		lmce = m.mcgstatus & MCG_STATUS_LMCES;

	/*
	 * Go through all banks in exclusion of the other CPUs. This way we
	 * don't report duplicated events on shared banks because the first one
	 * to see it will clear it. If this is a Local MCE, then no need to
	 * perform rendezvous.
	 */
	if (!lmce)
A
Ashok Raj 已提交
1117 1118
		order = mce_start(&no_way_out);

1119
	for (i = 0; i < cfg->banks; i++) {
1120
		__clear_bit(i, toclear);
1121 1122
		if (!test_bit(i, valid_banks))
			continue;
1123
		if (!mce_banks[i].ctl)
L
Linus Torvalds 已提交
1124
			continue;
1125 1126

		m.misc = 0;
L
Linus Torvalds 已提交
1127 1128 1129
		m.addr = 0;
		m.bank = i;

1130
		m.status = mce_rdmsrl(msr_ops.status(i));
L
Linus Torvalds 已提交
1131 1132 1133
		if ((m.status & MCI_STATUS_VAL) == 0)
			continue;

1134
		/*
A
Andi Kleen 已提交
1135 1136
		 * Non uncorrected or non signaled errors are handled by
		 * machine_check_poll. Leave them alone, unless this panics.
1137
		 */
1138
		if (!(m.status & (cfg->ser ? MCI_STATUS_S : MCI_STATUS_UC)) &&
A
Andi Kleen 已提交
1139
			!no_way_out)
1140 1141 1142 1143 1144
			continue;

		/*
		 * Set taint even when machine check was not enabled.
		 */
1145
		add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE);
1146

1147
		severity = mce_severity(&m, cfg->tolerant, NULL, true);
1148

A
Andi Kleen 已提交
1149
		/*
1150 1151
		 * When machine check was for corrected/deferred handler don't
		 * touch, unless we're panicing.
A
Andi Kleen 已提交
1152
		 */
1153 1154
		if ((severity == MCE_KEEP_SEVERITY ||
		     severity == MCE_UCNA_SEVERITY) && !no_way_out)
A
Andi Kleen 已提交
1155 1156 1157
			continue;
		__set_bit(i, toclear);
		if (severity == MCE_NO_SEVERITY) {
1158 1159 1160 1161 1162
			/*
			 * Machine check event was not enabled. Clear, but
			 * ignore.
			 */
			continue;
L
Linus Torvalds 已提交
1163 1164
		}

1165
		mce_read_aux(&m, i);
L
Linus Torvalds 已提交
1166

1167 1168
		/* assuming valid severity level != 0 */
		m.severity = severity;
1169

1170
		mce_log(&m);
L
Linus Torvalds 已提交
1171

1172 1173 1174
		if (severity > worst) {
			*final = m;
			worst = severity;
L
Linus Torvalds 已提交
1175 1176 1177
		}
	}

1178 1179 1180
	/* mce_clear_state will clear *final, save locally for use later */
	m = *final;

1181 1182 1183
	if (!no_way_out)
		mce_clear_state(toclear);

I
Ingo Molnar 已提交
1184
	/*
1185 1186
	 * Do most of the synchronization with other CPUs.
	 * When there's any problem use only local no_way_out state.
I
Ingo Molnar 已提交
1187
	 */
A
Ashok Raj 已提交
1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199
	if (!lmce) {
		if (mce_end(order) < 0)
			no_way_out = worst >= MCE_PANIC_SEVERITY;
	} else {
		/*
		 * Local MCE skipped calling mce_reign()
		 * If we found a fatal error, we need to panic here.
		 */
		 if (worst >= MCE_PANIC_SEVERITY && mca_cfg.tolerant < 3)
			mce_panic("Machine check from unknown source",
				NULL, NULL);
	}
1200 1201

	/*
1202 1203
	 * If tolerant is at an insane level we drop requests to kill
	 * processes and continue even when there is no way out.
1204
	 */
1205 1206 1207 1208
	if (cfg->tolerant == 3)
		kill_it = 0;
	else if (no_way_out)
		mce_panic("Fatal machine check on current CPU", &m, msg);
1209

1210 1211
	if (worst > 0)
		mce_report_event(regs);
1212
	mce_wrmsrl(MSR_IA32_MCG_STATUS, 0);
1213
out:
1214
	sync_core();
1215

1216 1217
	if (worst != MCE_AR_SEVERITY && !kill_it)
		goto out_ist;
1218

1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230
	/* Fault was in user mode and we need to take some action */
	if ((m.cs & 3) == 3) {
		ist_begin_non_atomic(regs);
		local_irq_enable();

		if (kill_it || do_memory_failure(&m))
			force_sig(SIGBUS, current);
		local_irq_disable();
		ist_end_non_atomic();
	} else {
		if (!fixup_exception(regs, X86_TRAP_MC))
			mce_panic("Failed kernel mode recovery", &m, NULL);
1231
	}
1232 1233

out_ist:
1234
	ist_exit(regs);
L
Linus Torvalds 已提交
1235
}
1236
EXPORT_SYMBOL_GPL(do_machine_check);
L
Linus Torvalds 已提交
1237

1238 1239
#ifndef CONFIG_MEMORY_FAILURE
int memory_failure(unsigned long pfn, int vector, int flags)
1240
{
1241 1242
	/* mce_severity() should not hand us an ACTION_REQUIRED error */
	BUG_ON(flags & MF_ACTION_REQUIRED);
1243 1244 1245
	pr_err("Uncorrected memory error in page 0x%lx ignored\n"
	       "Rebuild kernel with CONFIG_MEMORY_FAILURE=y for smarter handling\n",
	       pfn);
1246 1247

	return 0;
1248
}
1249
#endif
1250

1251 1252 1253
/*
 * Action optional processing happens here (picking up
 * from the list of faulting pages that do_machine_check()
1254
 * placed into the genpool).
1255
 */
1256 1257
static void mce_process_work(struct work_struct *dummy)
{
1258
	mce_gen_pool_process();
1259 1260
}

1261 1262 1263
#ifdef CONFIG_X86_MCE_INTEL
/***
 * mce_log_therm_throt_event - Logs the thermal throttling event to mcelog
S
Simon Arlott 已提交
1264
 * @cpu: The CPU on which the event occurred.
1265 1266 1267 1268 1269 1270 1271 1272 1273 1274
 * @status: Event status information
 *
 * This function should be called by the thermal interrupt after the
 * event has been processed and the decision was made to log the event
 * further.
 *
 * The status parameter will be saved to the 'status' field of 'struct mce'
 * and historically has been the register value of the
 * MSR_IA32_THERMAL_STATUS (Intel) msr.
 */
1275
void mce_log_therm_throt_event(__u64 status)
1276 1277 1278
{
	struct mce m;

1279
	mce_setup(&m);
1280 1281 1282 1283 1284 1285
	m.bank = MCE_THERMAL_BANK;
	m.status = status;
	mce_log(&m);
}
#endif /* CONFIG_X86_MCE_INTEL */

L
Linus Torvalds 已提交
1286
/*
1287 1288 1289
 * Periodic polling timer for "silent" machine check errors.  If the
 * poller finds an MCE, poll 2x faster.  When the poller finds no more
 * errors, poll 2x slower (up to check_interval seconds).
L
Linus Torvalds 已提交
1290
 */
1291
static unsigned long check_interval = INITIAL_CHECK_INTERVAL;
I
Ingo Molnar 已提交
1292

T
Thomas Gleixner 已提交
1293
static DEFINE_PER_CPU(unsigned long, mce_next_interval); /* in jiffies */
1294
static DEFINE_PER_CPU(struct timer_list, mce_timer);
L
Linus Torvalds 已提交
1295

C
Chen Gong 已提交
1296 1297 1298 1299 1300
static unsigned long mce_adjust_timer_default(unsigned long interval)
{
	return interval;
}

1301
static unsigned long (*mce_adjust_timer)(unsigned long interval) = mce_adjust_timer_default;
C
Chen Gong 已提交
1302

1303
static void __restart_timer(struct timer_list *t, unsigned long interval)
1304
{
1305 1306
	unsigned long when = jiffies + interval;
	unsigned long flags;
1307

1308
	local_irq_save(flags);
1309

1310 1311
	if (timer_pending(t)) {
		if (time_before(when, t->expires))
1312
			mod_timer(t, when);
1313 1314 1315 1316 1317 1318
	} else {
		t->expires = round_jiffies(when);
		add_timer_on(t, smp_processor_id());
	}

	local_irq_restore(flags);
1319 1320
}

T
Thomas Gleixner 已提交
1321
static void mce_timer_fn(unsigned long data)
L
Linus Torvalds 已提交
1322
{
1323
	struct timer_list *t = this_cpu_ptr(&mce_timer);
1324
	int cpu = smp_processor_id();
T
Thomas Gleixner 已提交
1325
	unsigned long iv;
1326

1327 1328 1329
	WARN_ON(cpu != data);

	iv = __this_cpu_read(mce_next_interval);
1330

1331
	if (mce_available(this_cpu_ptr(&cpu_info))) {
1332 1333 1334 1335 1336 1337
		machine_check_poll(MCP_TIMESTAMP, this_cpu_ptr(&mce_poll_banks));

		if (mce_intel_cmci_poll()) {
			iv = mce_adjust_timer(iv);
			goto done;
		}
I
Ingo Molnar 已提交
1338
	}
L
Linus Torvalds 已提交
1339 1340

	/*
1341 1342
	 * Alert userspace if needed. If we logged an MCE, reduce the polling
	 * interval, otherwise increase the polling interval.
L
Linus Torvalds 已提交
1343
	 */
1344
	if (mce_notify_irq())
1345
		iv = max(iv / 2, (unsigned long) HZ/100);
1346
	else
T
Thomas Gleixner 已提交
1347
		iv = min(iv * 2, round_jiffies_relative(check_interval * HZ));
1348 1349

done:
T
Thomas Gleixner 已提交
1350
	__this_cpu_write(mce_next_interval, iv);
1351
	__restart_timer(t, iv);
C
Chen Gong 已提交
1352
}
1353

C
Chen Gong 已提交
1354 1355 1356 1357 1358
/*
 * Ensure that the timer is firing in @interval from now.
 */
void mce_timer_kick(unsigned long interval)
{
1359
	struct timer_list *t = this_cpu_ptr(&mce_timer);
C
Chen Gong 已提交
1360 1361
	unsigned long iv = __this_cpu_read(mce_next_interval);

1362 1363
	__restart_timer(t, interval);

C
Chen Gong 已提交
1364 1365
	if (interval < iv)
		__this_cpu_write(mce_next_interval, interval);
1366 1367
}

1368 1369 1370 1371 1372 1373 1374 1375 1376
/* Must not be called in IRQ context where del_timer_sync() can deadlock */
static void mce_timer_delete_all(void)
{
	int cpu;

	for_each_online_cpu(cpu)
		del_timer_sync(&per_cpu(mce_timer, cpu));
}

1377 1378
static void mce_do_trigger(struct work_struct *work)
{
1379
	call_usermodehelper(mce_helper, mce_helper_argv, NULL, UMH_NO_WAIT);
1380 1381 1382 1383
}

static DECLARE_WORK(mce_trigger_work, mce_do_trigger);

1384
/*
1385 1386 1387
 * Notify the user(s) about new machine check events.
 * Can be called from interrupt context, but not from machine check/NMI
 * context.
1388
 */
1389
int mce_notify_irq(void)
1390
{
1391 1392 1393
	/* Not more than two messages every minute */
	static DEFINE_RATELIMIT_STATE(ratelimit, 60*HZ, 2);

1394
	if (test_and_clear_bit(0, &mce_need_notify)) {
1395 1396
		/* wake processes polling /dev/mcelog */
		wake_up_interruptible(&mce_chrdev_wait);
1397

1398
		if (mce_helper[0])
1399
			schedule_work(&mce_trigger_work);
1400

1401
		if (__ratelimit(&ratelimit))
H
Huang Ying 已提交
1402
			pr_info(HW_ERR "Machine check events logged\n");
1403 1404

		return 1;
L
Linus Torvalds 已提交
1405
	}
1406 1407
	return 0;
}
1408
EXPORT_SYMBOL_GPL(mce_notify_irq);
1409

1410
static int __mcheck_cpu_mce_banks_init(void)
1411 1412
{
	int i;
1413
	u8 num_banks = mca_cfg.banks;
1414

1415
	mce_banks = kzalloc(num_banks * sizeof(struct mce_bank), GFP_KERNEL);
1416 1417
	if (!mce_banks)
		return -ENOMEM;
1418 1419

	for (i = 0; i < num_banks; i++) {
1420
		struct mce_bank *b = &mce_banks[i];
1421

1422 1423 1424 1425 1426 1427
		b->ctl = -1ULL;
		b->init = 1;
	}
	return 0;
}

1428
/*
L
Linus Torvalds 已提交
1429 1430
 * Initialize Machine Checks for a CPU.
 */
1431
static int __mcheck_cpu_cap_init(void)
L
Linus Torvalds 已提交
1432
{
1433
	unsigned b;
I
Ingo Molnar 已提交
1434
	u64 cap;
L
Linus Torvalds 已提交
1435 1436

	rdmsrl(MSR_IA32_MCG_CAP, cap);
1437 1438

	b = cap & MCG_BANKCNT_MASK;
1439
	if (!mca_cfg.banks)
1440
		pr_info("CPU supports %d MCE banks\n", b);
1441

1442
	if (b > MAX_NR_BANKS) {
1443
		pr_warn("Using only %u machine check banks out of %u\n",
1444 1445 1446 1447 1448
			MAX_NR_BANKS, b);
		b = MAX_NR_BANKS;
	}

	/* Don't support asymmetric configurations today */
1449 1450 1451
	WARN_ON(mca_cfg.banks != 0 && b != mca_cfg.banks);
	mca_cfg.banks = b;

1452
	if (!mce_banks) {
H
Hidetoshi Seto 已提交
1453
		int err = __mcheck_cpu_mce_banks_init();
1454

1455 1456
		if (err)
			return err;
L
Linus Torvalds 已提交
1457
	}
1458

1459
	/* Use accurate RIP reporting if available. */
1460
	if ((cap & MCG_EXT_P) && MCG_EXT_CNT(cap) >= 9)
1461
		mca_cfg.rip_msr = MSR_IA32_MCG_EIP;
L
Linus Torvalds 已提交
1462

A
Andi Kleen 已提交
1463
	if (cap & MCG_SER_P)
1464
		mca_cfg.ser = true;
A
Andi Kleen 已提交
1465

1466 1467 1468
	return 0;
}

1469
static void __mcheck_cpu_init_generic(void)
1470
{
1471
	enum mcp_flags m_fl = 0;
I
Ingo Molnar 已提交
1472
	mce_banks_t all_banks;
1473 1474
	u64 cap;

1475 1476 1477
	if (!mca_cfg.bootlog)
		m_fl = MCP_DONTLOG;

1478 1479 1480
	/*
	 * Log the machine checks left over from the previous reset.
	 */
1481
	bitmap_fill(all_banks, MAX_NR_BANKS);
1482
	machine_check_poll(MCP_UC | m_fl, &all_banks);
L
Linus Torvalds 已提交
1483

A
Andy Lutomirski 已提交
1484
	cr4_set_bits(X86_CR4_MCE);
L
Linus Torvalds 已提交
1485

1486
	rdmsrl(MSR_IA32_MCG_CAP, cap);
L
Linus Torvalds 已提交
1487 1488
	if (cap & MCG_CTL_P)
		wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff);
1489 1490 1491 1492 1493
}

static void __mcheck_cpu_init_clear_banks(void)
{
	int i;
L
Linus Torvalds 已提交
1494

1495
	for (i = 0; i < mca_cfg.banks; i++) {
1496
		struct mce_bank *b = &mce_banks[i];
1497

1498
		if (!b->init)
1499
			continue;
1500 1501
		wrmsrl(msr_ops.ctl(i), b->ctl);
		wrmsrl(msr_ops.status(i), 0);
1502
	}
L
Linus Torvalds 已提交
1503 1504
}

1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532
/*
 * During IFU recovery Sandy Bridge -EP4S processors set the RIPV and
 * EIPV bits in MCG_STATUS to zero on the affected logical processor (SDM
 * Vol 3B Table 15-20). But this confuses both the code that determines
 * whether the machine check occurred in kernel or user mode, and also
 * the severity assessment code. Pretend that EIPV was set, and take the
 * ip/cs values from the pt_regs that mce_gather_info() ignored earlier.
 */
static void quirk_sandybridge_ifu(int bank, struct mce *m, struct pt_regs *regs)
{
	if (bank != 0)
		return;
	if ((m->mcgstatus & (MCG_STATUS_EIPV|MCG_STATUS_RIPV)) != 0)
		return;
	if ((m->status & (MCI_STATUS_OVER|MCI_STATUS_UC|
		          MCI_STATUS_EN|MCI_STATUS_MISCV|MCI_STATUS_ADDRV|
			  MCI_STATUS_PCC|MCI_STATUS_S|MCI_STATUS_AR|
			  MCACOD)) !=
			 (MCI_STATUS_UC|MCI_STATUS_EN|
			  MCI_STATUS_MISCV|MCI_STATUS_ADDRV|MCI_STATUS_S|
			  MCI_STATUS_AR|MCACOD_INSTR))
		return;

	m->mcgstatus |= MCG_STATUS_EIPV;
	m->ip = regs->ip;
	m->cs = regs->cs;
}

L
Linus Torvalds 已提交
1533
/* Add per CPU specific workarounds here */
1534
static int __mcheck_cpu_apply_quirks(struct cpuinfo_x86 *c)
1535
{
1536 1537
	struct mca_config *cfg = &mca_cfg;

1538
	if (c->x86_vendor == X86_VENDOR_UNKNOWN) {
1539
		pr_info("unknown CPU type - not enabling MCE support\n");
1540 1541 1542
		return -EOPNOTSUPP;
	}

L
Linus Torvalds 已提交
1543
	/* This should be disabled by the BIOS, but isn't always */
1544
	if (c->x86_vendor == X86_VENDOR_AMD) {
1545
		if (c->x86 == 15 && cfg->banks > 4) {
I
Ingo Molnar 已提交
1546 1547 1548 1549 1550
			/*
			 * disable GART TBL walk error reporting, which
			 * trips off incorrectly with the IOMMU & 3ware
			 * & Cerberus:
			 */
1551
			clear_bit(10, (unsigned long *)&mce_banks[4].ctl);
I
Ingo Molnar 已提交
1552
		}
1553
		if (c->x86 < 17 && cfg->bootlog < 0) {
I
Ingo Molnar 已提交
1554 1555 1556 1557
			/*
			 * Lots of broken BIOS around that don't clear them
			 * by default and leave crap in there. Don't log:
			 */
1558
			cfg->bootlog = 0;
I
Ingo Molnar 已提交
1559
		}
1560 1561 1562 1563
		/*
		 * Various K7s with broken bank 0 around. Always disable
		 * by default.
		 */
1564
		if (c->x86 == 6 && cfg->banks > 0)
1565
			mce_banks[0].ctl = 0;
1566

1567 1568 1569 1570 1571 1572 1573
		/*
		 * overflow_recov is supported for F15h Models 00h-0fh
		 * even though we don't have a CPUID bit for it.
		 */
		if (c->x86 == 0x15 && c->x86_model <= 0xf)
			mce_flags.overflow_recov = 1;

1574 1575 1576 1577 1578 1579 1580 1581 1582 1583
		/*
		 * Turn off MC4_MISC thresholding banks on those models since
		 * they're not supported there.
		 */
		if (c->x86 == 0x15 &&
		    (c->x86_model >= 0x10 && c->x86_model <= 0x1f)) {
			int i;
			u64 hwcr;
			bool need_toggle;
			u32 msrs[] = {
1584 1585
				0x00000413, /* MC4_MISC0 */
				0xc0000408, /* MC4_MISC1 */
1586
			};
1587

1588
			rdmsrl(MSR_K7_HWCR, hwcr);
1589

1590 1591
			/* McStatusWrEn has to be set */
			need_toggle = !(hwcr & BIT(18));
1592

1593 1594
			if (need_toggle)
				wrmsrl(MSR_K7_HWCR, hwcr | BIT(18));
1595

1596 1597 1598
			/* Clear CntP bit safely */
			for (i = 0; i < ARRAY_SIZE(msrs); i++)
				msr_clear_bit(msrs[i], 62);
1599

1600 1601 1602 1603
			/* restore old settings */
			if (need_toggle)
				wrmsrl(MSR_K7_HWCR, hwcr);
		}
L
Linus Torvalds 已提交
1604
	}
1605

1606 1607 1608 1609 1610 1611 1612 1613 1614 1615
	if (c->x86_vendor == X86_VENDOR_INTEL) {
		/*
		 * SDM documents that on family 6 bank 0 should not be written
		 * because it aliases to another special BIOS controlled
		 * register.
		 * But it's not aliased anymore on model 0x1a+
		 * Don't ignore bank 0 completely because there could be a
		 * valid event later, merely don't write CTL0.
		 */

1616
		if (c->x86 == 6 && c->x86_model < 0x1A && cfg->banks > 0)
1617
			mce_banks[0].init = 0;
1618 1619 1620 1621 1622 1623

		/*
		 * All newer Intel systems support MCE broadcasting. Enable
		 * synchronization with a one second timeout.
		 */
		if ((c->x86 > 6 || (c->x86 == 6 && c->x86_model >= 0xe)) &&
1624 1625
			cfg->monarch_timeout < 0)
			cfg->monarch_timeout = USEC_PER_SEC;
1626

1627 1628 1629 1630
		/*
		 * There are also broken BIOSes on some Pentium M and
		 * earlier systems:
		 */
1631 1632
		if (c->x86 == 6 && c->x86_model <= 13 && cfg->bootlog < 0)
			cfg->bootlog = 0;
1633 1634 1635

		if (c->x86 == 6 && c->x86_model == 45)
			quirk_no_way_out = quirk_sandybridge_ifu;
1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646
		/*
		 * MCG_CAP.MCG_SER_P is necessary but not sufficient to know
		 * whether this processor will actually generate recoverable
		 * machine checks. Check to see if this is an E7 model Xeon.
		 * We can't do a model number check because E5 and E7 use the
		 * same model number. E5 doesn't support recovery, E7 does.
		 */
		if (mca_cfg.recovery || (mca_cfg.ser &&
			!strncmp(c->x86_model_id,
				 "Intel(R) Xeon(R) CPU E7-", 24)))
			set_cpu_cap(c, X86_FEATURE_MCE_RECOVERY);
1647
	}
1648 1649 1650
	if (cfg->monarch_timeout < 0)
		cfg->monarch_timeout = 0;
	if (cfg->bootlog != 0)
1651
		cfg->panic_timeout = 30;
1652 1653

	return 0;
1654
}
L
Linus Torvalds 已提交
1655

1656
static int __mcheck_cpu_ancient_init(struct cpuinfo_x86 *c)
1657 1658
{
	if (c->x86 != 5)
1659 1660
		return 0;

1661 1662
	switch (c->x86_vendor) {
	case X86_VENDOR_INTEL:
1663
		intel_p5_mcheck_init(c);
1664
		return 1;
1665 1666 1667
		break;
	case X86_VENDOR_CENTAUR:
		winchip_mcheck_init(c);
1668
		return 1;
1669
		break;
1670 1671
	default:
		return 0;
1672
	}
1673 1674

	return 0;
1675 1676
}

1677
static void __mcheck_cpu_init_vendor(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1678 1679 1680 1681
{
	switch (c->x86_vendor) {
	case X86_VENDOR_INTEL:
		mce_intel_feature_init(c);
1682
		mce_adjust_timer = cmci_intel_adjust_timer;
L
Linus Torvalds 已提交
1683
		break;
1684 1685

	case X86_VENDOR_AMD: {
1686 1687 1688
		mce_flags.overflow_recov = !!cpu_has(c, X86_FEATURE_OVERFLOW_RECOV);
		mce_flags.succor	 = !!cpu_has(c, X86_FEATURE_SUCCOR);
		mce_flags.smca		 = !!cpu_has(c, X86_FEATURE_SMCA);
1689 1690 1691 1692 1693 1694 1695 1696 1697 1698

		/*
		 * Install proper ops for Scalable MCA enabled processors
		 */
		if (mce_flags.smca) {
			msr_ops.ctl	= smca_ctl_reg;
			msr_ops.status	= smca_status_reg;
			msr_ops.addr	= smca_addr_reg;
			msr_ops.misc	= smca_misc_reg;
		}
1699
		mce_amd_feature_init(c);
1700

1701
		break;
1702 1703
		}

L
Linus Torvalds 已提交
1704 1705 1706 1707 1708
	default:
		break;
	}
}

1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719
static void __mcheck_cpu_clear_vendor(struct cpuinfo_x86 *c)
{
	switch (c->x86_vendor) {
	case X86_VENDOR_INTEL:
		mce_intel_feature_clear(c);
		break;
	default:
		break;
	}
}

T
Thomas Gleixner 已提交
1720
static void mce_start_timer(unsigned int cpu, struct timer_list *t)
1721
{
1722
	unsigned long iv = check_interval * HZ;
1723

1724
	if (mca_cfg.ignore_ce || !iv)
1725 1726
		return;

1727 1728
	per_cpu(mce_next_interval, cpu) = iv;

T
Thomas Gleixner 已提交
1729
	t->expires = round_jiffies(jiffies + iv);
1730
	add_timer_on(t, cpu);
1731 1732
}

T
Thomas Gleixner 已提交
1733 1734
static void __mcheck_cpu_init_timer(void)
{
1735
	struct timer_list *t = this_cpu_ptr(&mce_timer);
T
Thomas Gleixner 已提交
1736 1737
	unsigned int cpu = smp_processor_id();

1738
	setup_pinned_timer(t, mce_timer_fn, cpu);
T
Thomas Gleixner 已提交
1739 1740 1741
	mce_start_timer(cpu, t);
}

A
Andi Kleen 已提交
1742 1743 1744
/* Handle unconfigured int18 (should never happen) */
static void unexpected_machine_check(struct pt_regs *regs, long error_code)
{
1745
	pr_err("CPU#%d: Unexpected int18 (Machine Check)\n",
A
Andi Kleen 已提交
1746 1747 1748 1749 1750 1751 1752
	       smp_processor_id());
}

/* Call the installed machine check handler for this CPU setup. */
void (*machine_check_vector)(struct pt_regs *, long error_code) =
						unexpected_machine_check;

1753
/*
L
Linus Torvalds 已提交
1754
 * Called for each booted CPU to set up machine checks.
I
Ingo Molnar 已提交
1755
 * Must be called with preempt off:
L
Linus Torvalds 已提交
1756
 */
1757
void mcheck_cpu_init(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1758
{
1759
	if (mca_cfg.disabled)
1760 1761
		return;

1762 1763
	if (__mcheck_cpu_ancient_init(c))
		return;
1764

1765
	if (!mce_available(c))
L
Linus Torvalds 已提交
1766 1767
		return;

1768
	if (__mcheck_cpu_cap_init() < 0 || __mcheck_cpu_apply_quirks(c) < 0) {
1769
		mca_cfg.disabled = true;
1770 1771 1772
		return;
	}

1773 1774 1775 1776 1777 1778
	if (mce_gen_pool_init()) {
		mca_cfg.disabled = true;
		pr_emerg("Couldn't allocate MCE records pool!\n");
		return;
	}

1779 1780
	machine_check_vector = do_machine_check;

1781 1782
	__mcheck_cpu_init_generic();
	__mcheck_cpu_init_vendor(c);
1783
	__mcheck_cpu_init_clear_banks();
1784
	__mcheck_cpu_init_timer();
L
Linus Torvalds 已提交
1785 1786
}

1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800 1801 1802 1803
/*
 * Called for each booted CPU to clear some machine checks opt-ins
 */
void mcheck_cpu_clear(struct cpuinfo_x86 *c)
{
	if (mca_cfg.disabled)
		return;

	if (!mce_available(c))
		return;

	/*
	 * Possibly to clear general settings generic to x86
	 * __mcheck_cpu_clear_generic(c);
	 */
	__mcheck_cpu_clear_vendor(c);

L
Linus Torvalds 已提交
1804 1805 1806
}

/*
1807
 * mce_chrdev: Character device /dev/mcelog to read and clear the MCE log.
L
Linus Torvalds 已提交
1808 1809
 */

1810 1811 1812
static DEFINE_SPINLOCK(mce_chrdev_state_lock);
static int mce_chrdev_open_count;	/* #times opened */
static int mce_chrdev_open_exclu;	/* already open exclusive? */
T
Tim Hockin 已提交
1813

1814
static int mce_chrdev_open(struct inode *inode, struct file *file)
T
Tim Hockin 已提交
1815
{
1816
	spin_lock(&mce_chrdev_state_lock);
T
Tim Hockin 已提交
1817

1818 1819 1820
	if (mce_chrdev_open_exclu ||
	    (mce_chrdev_open_count && (file->f_flags & O_EXCL))) {
		spin_unlock(&mce_chrdev_state_lock);
I
Ingo Molnar 已提交
1821

T
Tim Hockin 已提交
1822 1823 1824 1825
		return -EBUSY;
	}

	if (file->f_flags & O_EXCL)
1826 1827
		mce_chrdev_open_exclu = 1;
	mce_chrdev_open_count++;
T
Tim Hockin 已提交
1828

1829
	spin_unlock(&mce_chrdev_state_lock);
T
Tim Hockin 已提交
1830

1831
	return nonseekable_open(inode, file);
T
Tim Hockin 已提交
1832 1833
}

1834
static int mce_chrdev_release(struct inode *inode, struct file *file)
T
Tim Hockin 已提交
1835
{
1836
	spin_lock(&mce_chrdev_state_lock);
T
Tim Hockin 已提交
1837

1838 1839
	mce_chrdev_open_count--;
	mce_chrdev_open_exclu = 0;
T
Tim Hockin 已提交
1840

1841
	spin_unlock(&mce_chrdev_state_lock);
T
Tim Hockin 已提交
1842 1843 1844 1845

	return 0;
}

1846 1847
static void collect_tscs(void *data)
{
L
Linus Torvalds 已提交
1848
	unsigned long *cpu_tsc = (unsigned long *)data;
1849

1850
	cpu_tsc[smp_processor_id()] = rdtsc();
1851
}
L
Linus Torvalds 已提交
1852

1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868
static int mce_apei_read_done;

/* Collect MCE record of previous boot in persistent storage via APEI ERST. */
static int __mce_read_apei(char __user **ubuf, size_t usize)
{
	int rc;
	u64 record_id;
	struct mce m;

	if (usize < sizeof(struct mce))
		return -EINVAL;

	rc = apei_read_mce(&m, &record_id);
	/* Error or no more MCE record */
	if (rc <= 0) {
		mce_apei_read_done = 1;
1869 1870 1871 1872 1873 1874
		/*
		 * When ERST is disabled, mce_chrdev_read() should return
		 * "no record" instead of "no device."
		 */
		if (rc == -ENODEV)
			return 0;
1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895
		return rc;
	}
	rc = -EFAULT;
	if (copy_to_user(*ubuf, &m, sizeof(struct mce)))
		return rc;
	/*
	 * In fact, we should have cleared the record after that has
	 * been flushed to the disk or sent to network in
	 * /sbin/mcelog, but we have no interface to support that now,
	 * so just clear it to avoid duplication.
	 */
	rc = apei_clear_mce(record_id);
	if (rc) {
		mce_apei_read_done = 1;
		return rc;
	}
	*ubuf += sizeof(struct mce);

	return 0;
}

1896 1897
static ssize_t mce_chrdev_read(struct file *filp, char __user *ubuf,
				size_t usize, loff_t *off)
L
Linus Torvalds 已提交
1898
{
I
Ingo Molnar 已提交
1899
	char __user *buf = ubuf;
1900
	unsigned long *cpu_tsc;
1901
	unsigned prev, next;
L
Linus Torvalds 已提交
1902 1903
	int i, err;

1904
	cpu_tsc = kmalloc(nr_cpu_ids * sizeof(long), GFP_KERNEL);
1905 1906 1907
	if (!cpu_tsc)
		return -ENOMEM;

1908
	mutex_lock(&mce_chrdev_read_mutex);
1909 1910 1911 1912 1913 1914 1915

	if (!mce_apei_read_done) {
		err = __mce_read_apei(&buf, usize);
		if (err || buf != ubuf)
			goto out;
	}

1916
	next = mce_log_get_idx_check(mcelog.next);
L
Linus Torvalds 已提交
1917 1918

	/* Only supports full reads right now */
1919 1920 1921
	err = -EINVAL;
	if (*off != 0 || usize < MCE_LOG_LEN*sizeof(struct mce))
		goto out;
L
Linus Torvalds 已提交
1922 1923

	err = 0;
1924 1925 1926 1927
	prev = 0;
	do {
		for (i = prev; i < next; i++) {
			unsigned long start = jiffies;
H
Hidetoshi Seto 已提交
1928
			struct mce *m = &mcelog.entry[i];
1929

H
Hidetoshi Seto 已提交
1930
			while (!m->finished) {
1931
				if (time_after_eq(jiffies, start + 2)) {
H
Hidetoshi Seto 已提交
1932
					memset(m, 0, sizeof(*m));
1933 1934 1935
					goto timeout;
				}
				cpu_relax();
1936
			}
1937
			smp_rmb();
H
Hidetoshi Seto 已提交
1938 1939
			err |= copy_to_user(buf, m, sizeof(*m));
			buf += sizeof(*m);
1940 1941
timeout:
			;
1942
		}
L
Linus Torvalds 已提交
1943

1944 1945 1946 1947 1948
		memset(mcelog.entry + prev, 0,
		       (next - prev) * sizeof(struct mce));
		prev = next;
		next = cmpxchg(&mcelog.next, prev, 0);
	} while (next != prev);
L
Linus Torvalds 已提交
1949

1950
	synchronize_sched();
L
Linus Torvalds 已提交
1951

1952 1953 1954 1955
	/*
	 * Collect entries that were still getting written before the
	 * synchronize.
	 */
1956
	on_each_cpu(collect_tscs, cpu_tsc, 1);
I
Ingo Molnar 已提交
1957

1958
	for (i = next; i < MCE_LOG_LEN; i++) {
H
Hidetoshi Seto 已提交
1959 1960 1961 1962
		struct mce *m = &mcelog.entry[i];

		if (m->finished && m->tsc < cpu_tsc[m->cpu]) {
			err |= copy_to_user(buf, m, sizeof(*m));
L
Linus Torvalds 已提交
1963
			smp_rmb();
H
Hidetoshi Seto 已提交
1964 1965
			buf += sizeof(*m);
			memset(m, 0, sizeof(*m));
L
Linus Torvalds 已提交
1966
		}
1967
	}
1968 1969 1970 1971 1972

	if (err)
		err = -EFAULT;

out:
1973
	mutex_unlock(&mce_chrdev_read_mutex);
1974
	kfree(cpu_tsc);
I
Ingo Molnar 已提交
1975

1976
	return err ? err : buf - ubuf;
L
Linus Torvalds 已提交
1977 1978
}

1979
static unsigned int mce_chrdev_poll(struct file *file, poll_table *wait)
1980
{
1981
	poll_wait(file, &mce_chrdev_wait, wait);
1982
	if (READ_ONCE(mcelog.next))
1983
		return POLLIN | POLLRDNORM;
1984 1985
	if (!mce_apei_read_done && apei_check_mce())
		return POLLIN | POLLRDNORM;
1986 1987 1988
	return 0;
}

1989 1990
static long mce_chrdev_ioctl(struct file *f, unsigned int cmd,
				unsigned long arg)
L
Linus Torvalds 已提交
1991 1992
{
	int __user *p = (int __user *)arg;
1993

L
Linus Torvalds 已提交
1994
	if (!capable(CAP_SYS_ADMIN))
1995
		return -EPERM;
I
Ingo Molnar 已提交
1996

L
Linus Torvalds 已提交
1997
	switch (cmd) {
1998
	case MCE_GET_RECORD_LEN:
L
Linus Torvalds 已提交
1999 2000
		return put_user(sizeof(struct mce), p);
	case MCE_GET_LOG_LEN:
2001
		return put_user(MCE_LOG_LEN, p);
L
Linus Torvalds 已提交
2002 2003
	case MCE_GETCLEAR_FLAGS: {
		unsigned flags;
2004 2005

		do {
L
Linus Torvalds 已提交
2006
			flags = mcelog.flags;
2007
		} while (cmpxchg(&mcelog.flags, flags, 0) != flags);
I
Ingo Molnar 已提交
2008

2009
		return put_user(flags, p);
L
Linus Torvalds 已提交
2010 2011
	}
	default:
2012 2013
		return -ENOTTY;
	}
L
Linus Torvalds 已提交
2014 2015
}

2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026
static ssize_t (*mce_write)(struct file *filp, const char __user *ubuf,
			    size_t usize, loff_t *off);

void register_mce_write_callback(ssize_t (*fn)(struct file *filp,
			     const char __user *ubuf,
			     size_t usize, loff_t *off))
{
	mce_write = fn;
}
EXPORT_SYMBOL_GPL(register_mce_write_callback);

2027 2028
static ssize_t mce_chrdev_write(struct file *filp, const char __user *ubuf,
				size_t usize, loff_t *off)
2029 2030 2031 2032 2033 2034 2035 2036
{
	if (mce_write)
		return mce_write(filp, ubuf, usize, off);
	else
		return -EINVAL;
}

static const struct file_operations mce_chrdev_ops = {
2037 2038 2039
	.open			= mce_chrdev_open,
	.release		= mce_chrdev_release,
	.read			= mce_chrdev_read,
2040
	.write			= mce_chrdev_write,
2041 2042 2043
	.poll			= mce_chrdev_poll,
	.unlocked_ioctl		= mce_chrdev_ioctl,
	.llseek			= no_llseek,
L
Linus Torvalds 已提交
2044 2045
};

2046
static struct miscdevice mce_chrdev_device = {
L
Linus Torvalds 已提交
2047 2048 2049 2050 2051
	MISC_MCELOG_MINOR,
	"mcelog",
	&mce_chrdev_ops,
};

2052 2053 2054
static void __mce_disable_bank(void *arg)
{
	int bank = *((int *)arg);
2055
	__clear_bit(bank, this_cpu_ptr(mce_poll_banks));
2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070
	cmci_disable_bank(bank);
}

void mce_disable_bank(int bank)
{
	if (bank >= mca_cfg.banks) {
		pr_warn(FW_BUG
			"Ignoring request to disable invalid MCA bank %d.\n",
			bank);
		return;
	}
	set_bit(bank, mce_banks_ce_disabled);
	on_each_cpu(__mce_disable_bank, &bank, 1);
}

H
Hidetoshi Seto 已提交
2071
/*
2072 2073
 * mce=off Disables machine check
 * mce=no_cmci Disables CMCI
2074
 * mce=no_lmce Disables LMCE
2075 2076
 * mce=dont_log_ce Clears corrected events silently, no log created for CEs.
 * mce=ignore_ce Disables polling and CMCI, corrected events are not cleared.
2077 2078 2079
 * mce=TOLERANCELEVEL[,monarchtimeout] (number, see above)
 *	monarchtimeout is how long to wait for other CPUs on machine
 *	check, or 0 to not wait
H
Hidetoshi Seto 已提交
2080 2081
 * mce=bootlog Log MCEs from before booting. Disabled by default on AMD.
 * mce=nobootlog Don't log MCEs from before booting.
2082
 * mce=bios_cmci_threshold Don't program the CMCI threshold
H
Hidetoshi Seto 已提交
2083
 */
L
Linus Torvalds 已提交
2084 2085
static int __init mcheck_enable(char *str)
{
2086 2087
	struct mca_config *cfg = &mca_cfg;

2088
	if (*str == 0) {
2089
		enable_p5_mce();
2090 2091
		return 1;
	}
2092 2093
	if (*str == '=')
		str++;
L
Linus Torvalds 已提交
2094
	if (!strcmp(str, "off"))
2095
		cfg->disabled = true;
2096
	else if (!strcmp(str, "no_cmci"))
2097
		cfg->cmci_disabled = true;
2098 2099
	else if (!strcmp(str, "no_lmce"))
		cfg->lmce_disabled = true;
2100
	else if (!strcmp(str, "dont_log_ce"))
2101
		cfg->dont_log_ce = true;
2102
	else if (!strcmp(str, "ignore_ce"))
2103
		cfg->ignore_ce = true;
H
Hidetoshi Seto 已提交
2104
	else if (!strcmp(str, "bootlog") || !strcmp(str, "nobootlog"))
2105
		cfg->bootlog = (str[0] == 'b');
2106
	else if (!strcmp(str, "bios_cmci_threshold"))
2107
		cfg->bios_cmci_threshold = true;
2108 2109
	else if (!strcmp(str, "recovery"))
		cfg->recovery = true;
2110
	else if (isdigit(str[0])) {
2111
		if (get_option(&str, &cfg->tolerant) == 2)
2112
			get_option(&str, &(cfg->monarch_timeout));
2113
	} else {
2114
		pr_info("mce argument %s ignored. Please use /sys\n", str);
H
Hidetoshi Seto 已提交
2115 2116
		return 0;
	}
2117
	return 1;
L
Linus Torvalds 已提交
2118
}
2119
__setup("mce", mcheck_enable);
L
Linus Torvalds 已提交
2120

2121
int __init mcheck_init(void)
2122
{
2123
	mcheck_intel_therm_init();
2124
	mce_register_decode_chain(&mce_srao_nb);
2125
	mcheck_vendor_init_severity();
2126

2127 2128 2129
	INIT_WORK(&mce_work, mce_process_work);
	init_irq_work(&mce_irq_work, mce_irq_work_cb);

2130 2131 2132
	return 0;
}

2133
/*
2134
 * mce_syscore: PM support
2135
 */
L
Linus Torvalds 已提交
2136

2137 2138 2139 2140
/*
 * Disable machine checks on suspend and shutdown. We can't really handle
 * them later.
 */
2141
static void mce_disable_error_reporting(void)
2142 2143 2144
{
	int i;

2145
	for (i = 0; i < mca_cfg.banks; i++) {
2146
		struct mce_bank *b = &mce_banks[i];
2147

2148
		if (b->init)
2149
			wrmsrl(msr_ops.ctl(i), 0);
2150
	}
2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165
	return;
}

static void vendor_disable_error_reporting(void)
{
	/*
	 * Don't clear on Intel CPUs. Some of these MSRs are socket-wide.
	 * Disabling them for just a single offlined CPU is bad, since it will
	 * inhibit reporting for all shared resources on the socket like the
	 * last level cache (LLC), the integrated memory controller (iMC), etc.
	 */
	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
		return;

	mce_disable_error_reporting();
2166 2167
}

2168
static int mce_syscore_suspend(void)
2169
{
2170 2171
	vendor_disable_error_reporting();
	return 0;
2172 2173
}

2174
static void mce_syscore_shutdown(void)
2175
{
2176
	vendor_disable_error_reporting();
2177 2178
}

I
Ingo Molnar 已提交
2179 2180 2181 2182 2183
/*
 * On resume clear all MCE state. Don't want to see leftovers from the BIOS.
 * Only one CPU is active at this time, the others get re-added later using
 * CPU hotplug:
 */
2184
static void mce_syscore_resume(void)
L
Linus Torvalds 已提交
2185
{
2186
	__mcheck_cpu_init_generic();
2187
	__mcheck_cpu_init_vendor(raw_cpu_ptr(&cpu_info));
2188
	__mcheck_cpu_init_clear_banks();
L
Linus Torvalds 已提交
2189 2190
}

2191
static struct syscore_ops mce_syscore_ops = {
2192 2193 2194
	.suspend	= mce_syscore_suspend,
	.shutdown	= mce_syscore_shutdown,
	.resume		= mce_syscore_resume,
2195 2196
};

2197
/*
2198
 * mce_device: Sysfs support
2199 2200
 */

2201 2202
static void mce_cpu_restart(void *data)
{
2203
	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2204
		return;
2205
	__mcheck_cpu_init_generic();
2206
	__mcheck_cpu_init_clear_banks();
2207
	__mcheck_cpu_init_timer();
2208 2209
}

L
Linus Torvalds 已提交
2210
/* Reinit MCEs after user configuration changes */
2211 2212
static void mce_restart(void)
{
2213
	mce_timer_delete_all();
2214
	on_each_cpu(mce_cpu_restart, NULL, 1);
L
Linus Torvalds 已提交
2215 2216
}

2217
/* Toggle features for corrected errors */
2218
static void mce_disable_cmci(void *data)
2219
{
2220
	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2221 2222 2223 2224 2225 2226
		return;
	cmci_clear();
}

static void mce_enable_ce(void *all)
{
2227
	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2228 2229 2230 2231
		return;
	cmci_reenable();
	cmci_recheck();
	if (all)
2232
		__mcheck_cpu_init_timer();
2233 2234
}

2235
static struct bus_type mce_subsys = {
I
Ingo Molnar 已提交
2236
	.name		= "machinecheck",
2237
	.dev_name	= "machinecheck",
L
Linus Torvalds 已提交
2238 2239
};

2240
DEFINE_PER_CPU(struct device *, mce_device);
I
Ingo Molnar 已提交
2241 2242

void (*threshold_cpu_callback)(unsigned long action, unsigned int cpu);
L
Linus Torvalds 已提交
2243

2244
static inline struct mce_bank *attr_to_bank(struct device_attribute *attr)
2245 2246 2247
{
	return container_of(attr, struct mce_bank, attr);
}
2248

2249
static ssize_t show_bank(struct device *s, struct device_attribute *attr,
2250 2251
			 char *buf)
{
2252
	return sprintf(buf, "%llx\n", attr_to_bank(attr)->ctl);
2253 2254
}

2255
static ssize_t set_bank(struct device *s, struct device_attribute *attr,
H
Hidetoshi Seto 已提交
2256
			const char *buf, size_t size)
2257
{
H
Hidetoshi Seto 已提交
2258
	u64 new;
I
Ingo Molnar 已提交
2259

2260
	if (kstrtou64(buf, 0, &new) < 0)
2261
		return -EINVAL;
I
Ingo Molnar 已提交
2262

2263
	attr_to_bank(attr)->ctl = new;
2264
	mce_restart();
I
Ingo Molnar 已提交
2265

H
Hidetoshi Seto 已提交
2266
	return size;
2267
}
2268

I
Ingo Molnar 已提交
2269
static ssize_t
2270
show_trigger(struct device *s, struct device_attribute *attr, char *buf)
2271
{
2272
	strcpy(buf, mce_helper);
2273
	strcat(buf, "\n");
2274
	return strlen(mce_helper) + 1;
2275 2276
}

2277
static ssize_t set_trigger(struct device *s, struct device_attribute *attr,
I
Ingo Molnar 已提交
2278
				const char *buf, size_t siz)
2279 2280
{
	char *p;
I
Ingo Molnar 已提交
2281

2282 2283 2284
	strncpy(mce_helper, buf, sizeof(mce_helper));
	mce_helper[sizeof(mce_helper)-1] = 0;
	p = strchr(mce_helper, '\n');
I
Ingo Molnar 已提交
2285

2286
	if (p)
I
Ingo Molnar 已提交
2287 2288
		*p = 0;

2289
	return strlen(mce_helper) + !!p;
2290 2291
}

2292 2293
static ssize_t set_ignore_ce(struct device *s,
			     struct device_attribute *attr,
2294 2295 2296 2297
			     const char *buf, size_t size)
{
	u64 new;

2298
	if (kstrtou64(buf, 0, &new) < 0)
2299 2300
		return -EINVAL;

2301
	if (mca_cfg.ignore_ce ^ !!new) {
2302 2303
		if (new) {
			/* disable ce features */
2304 2305
			mce_timer_delete_all();
			on_each_cpu(mce_disable_cmci, NULL, 1);
2306
			mca_cfg.ignore_ce = true;
2307 2308
		} else {
			/* enable ce features */
2309
			mca_cfg.ignore_ce = false;
2310 2311 2312 2313 2314 2315
			on_each_cpu(mce_enable_ce, (void *)1, 1);
		}
	}
	return size;
}

2316 2317
static ssize_t set_cmci_disabled(struct device *s,
				 struct device_attribute *attr,
2318 2319 2320 2321
				 const char *buf, size_t size)
{
	u64 new;

2322
	if (kstrtou64(buf, 0, &new) < 0)
2323 2324
		return -EINVAL;

2325
	if (mca_cfg.cmci_disabled ^ !!new) {
2326 2327
		if (new) {
			/* disable cmci */
2328
			on_each_cpu(mce_disable_cmci, NULL, 1);
2329
			mca_cfg.cmci_disabled = true;
2330 2331
		} else {
			/* enable cmci */
2332
			mca_cfg.cmci_disabled = false;
2333 2334 2335 2336 2337 2338
			on_each_cpu(mce_enable_ce, NULL, 1);
		}
	}
	return size;
}

2339 2340
static ssize_t store_int_with_restart(struct device *s,
				      struct device_attribute *attr,
2341 2342
				      const char *buf, size_t size)
{
2343
	ssize_t ret = device_store_int(s, attr, buf, size);
2344 2345 2346 2347
	mce_restart();
	return ret;
}

2348
static DEVICE_ATTR(trigger, 0644, show_trigger, set_trigger);
2349
static DEVICE_INT_ATTR(tolerant, 0644, mca_cfg.tolerant);
2350
static DEVICE_INT_ATTR(monarch_timeout, 0644, mca_cfg.monarch_timeout);
2351
static DEVICE_BOOL_ATTR(dont_log_ce, 0644, mca_cfg.dont_log_ce);
I
Ingo Molnar 已提交
2352

2353 2354
static struct dev_ext_attribute dev_attr_check_interval = {
	__ATTR(check_interval, 0644, device_show_int, store_int_with_restart),
2355 2356
	&check_interval
};
I
Ingo Molnar 已提交
2357

2358
static struct dev_ext_attribute dev_attr_ignore_ce = {
2359 2360
	__ATTR(ignore_ce, 0644, device_show_bool, set_ignore_ce),
	&mca_cfg.ignore_ce
2361 2362
};

2363
static struct dev_ext_attribute dev_attr_cmci_disabled = {
2364 2365
	__ATTR(cmci_disabled, 0644, device_show_bool, set_cmci_disabled),
	&mca_cfg.cmci_disabled
2366 2367
};

2368 2369 2370 2371 2372 2373 2374 2375
static struct device_attribute *mce_device_attrs[] = {
	&dev_attr_tolerant.attr,
	&dev_attr_check_interval.attr,
	&dev_attr_trigger,
	&dev_attr_monarch_timeout.attr,
	&dev_attr_dont_log_ce.attr,
	&dev_attr_ignore_ce.attr,
	&dev_attr_cmci_disabled.attr,
2376 2377
	NULL
};
L
Linus Torvalds 已提交
2378

2379
static cpumask_var_t mce_device_initialized;
2380

2381 2382 2383 2384 2385
static void mce_device_release(struct device *dev)
{
	kfree(dev);
}

2386
/* Per cpu device init. All of the cpus still share the same ctrl bank: */
2387
static int mce_device_create(unsigned int cpu)
L
Linus Torvalds 已提交
2388
{
2389
	struct device *dev;
L
Linus Torvalds 已提交
2390
	int err;
2391
	int i, j;
2392

A
Andreas Herrmann 已提交
2393
	if (!mce_available(&boot_cpu_data))
2394 2395
		return -EIO;

2396 2397 2398
	dev = kzalloc(sizeof *dev, GFP_KERNEL);
	if (!dev)
		return -ENOMEM;
2399 2400
	dev->id  = cpu;
	dev->bus = &mce_subsys;
2401
	dev->release = &mce_device_release;
2402

2403
	err = device_register(dev);
2404 2405
	if (err) {
		put_device(dev);
2406
		return err;
2407
	}
2408

2409 2410
	for (i = 0; mce_device_attrs[i]; i++) {
		err = device_create_file(dev, mce_device_attrs[i]);
2411 2412 2413
		if (err)
			goto error;
	}
2414
	for (j = 0; j < mca_cfg.banks; j++) {
2415
		err = device_create_file(dev, &mce_banks[j].attr);
2416 2417 2418
		if (err)
			goto error2;
	}
2419
	cpumask_set_cpu(cpu, mce_device_initialized);
2420
	per_cpu(mce_device, cpu) = dev;
2421

2422
	return 0;
2423
error2:
2424
	while (--j >= 0)
2425
		device_remove_file(dev, &mce_banks[j].attr);
2426
error:
I
Ingo Molnar 已提交
2427
	while (--i >= 0)
2428
		device_remove_file(dev, mce_device_attrs[i]);
I
Ingo Molnar 已提交
2429

2430
	device_unregister(dev);
2431

2432 2433 2434
	return err;
}

2435
static void mce_device_remove(unsigned int cpu)
2436
{
2437
	struct device *dev = per_cpu(mce_device, cpu);
2438 2439
	int i;

2440
	if (!cpumask_test_cpu(cpu, mce_device_initialized))
2441 2442
		return;

2443 2444
	for (i = 0; mce_device_attrs[i]; i++)
		device_remove_file(dev, mce_device_attrs[i]);
I
Ingo Molnar 已提交
2445

2446
	for (i = 0; i < mca_cfg.banks; i++)
2447
		device_remove_file(dev, &mce_banks[i].attr);
I
Ingo Molnar 已提交
2448

2449 2450
	device_unregister(dev);
	cpumask_clear_cpu(cpu, mce_device_initialized);
2451
	per_cpu(mce_device, cpu) = NULL;
2452 2453
}

2454
/* Make sure there are no machine checks on offlined CPUs. */
2455
static void mce_disable_cpu(void *h)
2456
{
A
Andi Kleen 已提交
2457
	unsigned long action = *(unsigned long *)h;
2458

2459
	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2460
		return;
2461

A
Andi Kleen 已提交
2462 2463
	if (!(action & CPU_TASKS_FROZEN))
		cmci_clear();
2464

2465
	vendor_disable_error_reporting();
2466 2467
}

2468
static void mce_reenable_cpu(void *h)
2469
{
A
Andi Kleen 已提交
2470
	unsigned long action = *(unsigned long *)h;
I
Ingo Molnar 已提交
2471
	int i;
2472

2473
	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2474
		return;
I
Ingo Molnar 已提交
2475

A
Andi Kleen 已提交
2476 2477
	if (!(action & CPU_TASKS_FROZEN))
		cmci_reenable();
2478
	for (i = 0; i < mca_cfg.banks; i++) {
2479
		struct mce_bank *b = &mce_banks[i];
2480

2481
		if (b->init)
2482
			wrmsrl(msr_ops.ctl(i), b->ctl);
2483
	}
2484 2485
}

2486
/* Get notified when a cpu comes on/off. Be hotplug friendly. */
2487
static int
I
Ingo Molnar 已提交
2488
mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
2489 2490
{
	unsigned int cpu = (unsigned long)hcpu;
2491
	struct timer_list *t = &per_cpu(mce_timer, cpu);
2492

2493
	switch (action & ~CPU_TASKS_FROZEN) {
2494
	case CPU_ONLINE:
2495
		mce_device_create(cpu);
2496 2497
		if (threshold_cpu_callback)
			threshold_cpu_callback(action, cpu);
2498 2499
		break;
	case CPU_DEAD:
2500 2501
		if (threshold_cpu_callback)
			threshold_cpu_callback(action, cpu);
2502
		mce_device_remove(cpu);
C
Chen Gong 已提交
2503
		mce_intel_hcpu_update(cpu);
B
Borislav Petkov 已提交
2504 2505 2506 2507

		/* intentionally ignoring frozen here */
		if (!(action & CPU_TASKS_FROZEN))
			cmci_rediscover();
2508
		break;
2509
	case CPU_DOWN_PREPARE:
A
Andi Kleen 已提交
2510
		smp_call_function_single(cpu, mce_disable_cpu, &action, 1);
C
Chen Gong 已提交
2511
		del_timer_sync(t);
2512 2513
		break;
	case CPU_DOWN_FAILED:
A
Andi Kleen 已提交
2514
		smp_call_function_single(cpu, mce_reenable_cpu, &action, 1);
T
Thomas Gleixner 已提交
2515
		mce_start_timer(cpu, t);
A
Andi Kleen 已提交
2516
		break;
2517 2518
	}

2519
	return NOTIFY_OK;
2520 2521
}

2522
static struct notifier_block mce_cpu_notifier = {
2523 2524 2525
	.notifier_call = mce_cpu_callback,
};

2526
static __init void mce_init_banks(void)
2527 2528 2529
{
	int i;

2530
	for (i = 0; i < mca_cfg.banks; i++) {
2531
		struct mce_bank *b = &mce_banks[i];
2532
		struct device_attribute *a = &b->attr;
I
Ingo Molnar 已提交
2533

2534
		sysfs_attr_init(&a->attr);
2535 2536
		a->attr.name	= b->attrname;
		snprintf(b->attrname, ATTR_LEN, "bank%d", i);
I
Ingo Molnar 已提交
2537 2538 2539 2540

		a->attr.mode	= 0644;
		a->show		= show_bank;
		a->store	= set_bank;
2541 2542 2543
	}
}

2544
static __init int mcheck_init_device(void)
2545 2546 2547 2548
{
	int err;
	int i = 0;

2549 2550 2551 2552
	if (!mce_available(&boot_cpu_data)) {
		err = -EIO;
		goto err_out;
	}
2553

2554 2555 2556 2557
	if (!zalloc_cpumask_var(&mce_device_initialized, GFP_KERNEL)) {
		err = -ENOMEM;
		goto err_out;
	}
2558

2559
	mce_init_banks();
2560

2561
	err = subsys_system_register(&mce_subsys, NULL);
2562
	if (err)
2563
		goto err_out_mem;
2564

2565
	cpu_notifier_register_begin();
2566
	for_each_online_cpu(i) {
2567
		err = mce_device_create(i);
2568
		if (err) {
2569 2570 2571 2572 2573 2574
			/*
			 * Register notifier anyway (and do not unreg it) so
			 * that we don't leave undeleted timers, see notifier
			 * callback above.
			 */
			__register_hotcpu_notifier(&mce_cpu_notifier);
2575
			cpu_notifier_register_done();
2576
			goto err_device_create;
2577
		}
2578 2579
	}

2580 2581
	__register_hotcpu_notifier(&mce_cpu_notifier);
	cpu_notifier_register_done();
2582

2583 2584
	register_syscore_ops(&mce_syscore_ops);

2585
	/* register character device /dev/mcelog */
2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609
	err = misc_register(&mce_chrdev_device);
	if (err)
		goto err_register;

	return 0;

err_register:
	unregister_syscore_ops(&mce_syscore_ops);

err_device_create:
	/*
	 * We didn't keep track of which devices were created above, but
	 * even if we had, the set of online cpus might have changed.
	 * Play safe and remove for every possible cpu, since
	 * mce_device_remove() will do the right thing.
	 */
	for_each_possible_cpu(i)
		mce_device_remove(i);

err_out_mem:
	free_cpumask_var(mce_device_initialized);

err_out:
	pr_err("Unable to init device /dev/mcelog (rc: %d)\n", err);
I
Ingo Molnar 已提交
2610

L
Linus Torvalds 已提交
2611 2612
	return err;
}
2613
device_initcall_sync(mcheck_init_device);
I
Ingo Molnar 已提交
2614

2615 2616 2617 2618 2619
/*
 * Old style boot options parsing. Only for compatibility.
 */
static int __init mcheck_disable(char *str)
{
2620
	mca_cfg.disabled = true;
2621 2622 2623
	return 1;
}
__setup("nomce", mcheck_disable);
I
Ingo Molnar 已提交
2624

2625 2626
#ifdef CONFIG_DEBUG_FS
struct dentry *mce_get_debugfs_dir(void)
I
Ingo Molnar 已提交
2627
{
2628
	static struct dentry *dmce;
I
Ingo Molnar 已提交
2629

2630 2631
	if (!dmce)
		dmce = debugfs_create_dir("mce", NULL);
I
Ingo Molnar 已提交
2632

2633 2634
	return dmce;
}
I
Ingo Molnar 已提交
2635

2636 2637 2638
static void mce_reset(void)
{
	cpu_missing = 0;
2639
	atomic_set(&mce_fake_panicked, 0);
2640 2641 2642 2643
	atomic_set(&mce_executing, 0);
	atomic_set(&mce_callin, 0);
	atomic_set(&global_nwo, 0);
}
I
Ingo Molnar 已提交
2644

2645 2646 2647 2648
static int fake_panic_get(void *data, u64 *val)
{
	*val = fake_panic;
	return 0;
I
Ingo Molnar 已提交
2649 2650
}

2651
static int fake_panic_set(void *data, u64 val)
I
Ingo Molnar 已提交
2652
{
2653 2654 2655
	mce_reset();
	fake_panic = val;
	return 0;
I
Ingo Molnar 已提交
2656 2657
}

2658 2659
DEFINE_SIMPLE_ATTRIBUTE(fake_panic_fops, fake_panic_get,
			fake_panic_set, "%llu\n");
2660

2661
static int __init mcheck_debugfs_init(void)
2662
{
2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673
	struct dentry *dmce, *ffake_panic;

	dmce = mce_get_debugfs_dir();
	if (!dmce)
		return -ENOMEM;
	ffake_panic = debugfs_create_file("fake_panic", 0444, dmce, NULL,
					  &fake_panic_fops);
	if (!ffake_panic)
		return -ENOMEM;

	return 0;
2674
}
2675 2676
#else
static int __init mcheck_debugfs_init(void) { return -EINVAL; }
2677
#endif
2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691

static int __init mcheck_late_init(void)
{
	mcheck_debugfs_init();

	/*
	 * Flush out everything that has been logged during early boot, now that
	 * everything has been initialized (workqueues, decoders, ...).
	 */
	mce_schedule_work();

	return 0;
}
late_initcall(mcheck_late_init);