mce.c 60.9 KB
Newer Older
L
Linus Torvalds 已提交
1 2
/*
 * Machine check handler.
I
Ingo Molnar 已提交
3
 *
L
Linus Torvalds 已提交
4
 * K8 parts Copyright 2002,2003 Andi Kleen, SuSE Labs.
5 6
 * Rest from unknown author(s).
 * 2004 Andi Kleen. Rewrote most of it.
7 8
 * Copyright 2008 Intel Corporation
 * Author: Andi Kleen
L
Linus Torvalds 已提交
9
 */
10 11 12

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

I
Ingo Molnar 已提交
13 14 15 16 17 18 19
#include <linux/thread_info.h>
#include <linux/capability.h>
#include <linux/miscdevice.h>
#include <linux/ratelimit.h>
#include <linux/kallsyms.h>
#include <linux/rcupdate.h>
#include <linux/kobject.h>
20
#include <linux/uaccess.h>
I
Ingo Molnar 已提交
21 22 23
#include <linux/kdebug.h>
#include <linux/kernel.h>
#include <linux/percpu.h>
L
Linus Torvalds 已提交
24
#include <linux/string.h>
25
#include <linux/device.h>
26
#include <linux/syscore_ops.h>
27
#include <linux/delay.h>
28
#include <linux/ctype.h>
I
Ingo Molnar 已提交
29
#include <linux/sched.h>
30
#include <linux/sysfs.h>
I
Ingo Molnar 已提交
31
#include <linux/types.h>
32
#include <linux/slab.h>
I
Ingo Molnar 已提交
33 34 35
#include <linux/init.h>
#include <linux/kmod.h>
#include <linux/poll.h>
36
#include <linux/nmi.h>
I
Ingo Molnar 已提交
37
#include <linux/cpu.h>
38
#include <linux/smp.h>
I
Ingo Molnar 已提交
39
#include <linux/fs.h>
40
#include <linux/mm.h>
41
#include <linux/debugfs.h>
42
#include <linux/irq_work.h>
43
#include <linux/export.h>
44
#include <linux/jump_label.h>
I
Ingo Molnar 已提交
45

46
#include <asm/intel-family.h>
47
#include <asm/processor.h>
48
#include <asm/traps.h>
A
Andy Lutomirski 已提交
49
#include <asm/tlbflush.h>
I
Ingo Molnar 已提交
50 51
#include <asm/mce.h>
#include <asm/msr.h>
L
Linus Torvalds 已提交
52

53
#include "mce-internal.h"
54

55
static DEFINE_MUTEX(mce_chrdev_read_mutex);
56

57 58
static int mce_chrdev_open_count;	/* #times opened */

59
#define mce_log_get_idx_check(p) \
60
({ \
61 62
	RCU_LOCKDEP_WARN(!rcu_read_lock_sched_held() && \
			 !lockdep_is_held(&mce_chrdev_read_mutex), \
63
			 "suspicious mce_log_get_idx_check() usage"); \
64 65
	smp_load_acquire(&(p)); \
})
66

67 68 69
#define CREATE_TRACE_POINTS
#include <trace/events/mce.h>

70
#define SPINUNIT		100	/* 100ns */
71

72 73
DEFINE_PER_CPU(unsigned, mce_exception_count);

74
struct mce_bank *mce_banks __read_mostly;
75
struct mce_vendor_flags mce_flags __read_mostly;
76

77
struct mca_config mca_cfg __read_mostly = {
78
	.bootlog  = -1,
79 80 81 82 83 84 85
	/*
	 * Tolerant levels:
	 * 0: always panic on uncorrected errors, log corrected errors
	 * 1: panic or SIGBUS on uncorrected errors, log corrected errors
	 * 2: SIGBUS or log uncorrected errors (if possible), log corr. errors
	 * 3: never panic or SIGBUS, log all errors (for testing only)
	 */
86 87
	.tolerant = 1,
	.monarch_timeout = -1
88 89
};

90 91 92 93
/* User mode helper program triggered by machine check event */
static unsigned long		mce_need_notify;
static char			mce_helper[128];
static char			*mce_helper_argv[2] = { mce_helper, NULL };
L
Linus Torvalds 已提交
94

95 96
static DECLARE_WAIT_QUEUE_HEAD(mce_chrdev_wait);

97 98 99
static DEFINE_PER_CPU(struct mce, mces_seen);
static int			cpu_missing;

100 101 102 103
/*
 * MCA banks polled by the period polling timer for corrected events.
 * With Intel CMCI, this only has MCA banks which do not support CMCI (if any).
 */
104 105 106 107
DEFINE_PER_CPU(mce_banks_t, mce_poll_banks) = {
	[0 ... BITS_TO_LONGS(MAX_NR_BANKS)-1] = ~0UL
};

108 109 110 111 112 113 114 115 116
/*
 * MCA banks controlled through firmware first for corrected errors.
 * This is a global list of banks for which we won't enable CMCI and we
 * won't poll. Firmware controls these banks and is responsible for
 * reporting corrected errors through GHES. Uncorrected/recoverable
 * errors are still notified through a machine check.
 */
mce_banks_t mce_banks_ce_disabled;

117 118
static struct work_struct mce_work;
static struct irq_work mce_irq_work;
119

120 121
static void (*quirk_no_way_out)(int bank, struct mce *m, struct pt_regs *regs);

122 123 124 125
/*
 * CPU/chipset specific EDAC code can register a notifier call here to print
 * MCE errors in a human-readable form.
 */
126
BLOCKING_NOTIFIER_HEAD(x86_mce_decoder_chain);
127

128 129 130 131
/* Do initial initialization of a struct mce */
void mce_setup(struct mce *m)
{
	memset(m, 0, sizeof(struct mce));
132
	m->cpu = m->extcpu = smp_processor_id();
133 134 135 136 137 138 139
	/* We hope get_seconds stays lockless */
	m->time = get_seconds();
	m->cpuvendor = boot_cpu_data.x86_vendor;
	m->cpuid = cpuid_eax(1);
	m->socketid = cpu_data(m->extcpu).phys_proc_id;
	m->apicid = cpu_data(m->extcpu).initial_apicid;
	rdmsrl(MSR_IA32_MCG_CAP, m->mcgcap);
140 141 142

	if (this_cpu_has(X86_FEATURE_INTEL_PPIN))
		rdmsrl(MSR_PPIN, m->ppin);
143 144
}

145 146 147
DEFINE_PER_CPU(struct mce, injectm);
EXPORT_PER_CPU_SYMBOL_GPL(injectm);

L
Linus Torvalds 已提交
148 149 150 151 152 153
/*
 * Lockless MCE logging infrastructure.
 * This avoids deadlocks on printk locks without having to break locks. Also
 * separate MCEs from kernel messages to avoid bogus bug reports.
 */

154
static struct mce_log mcelog = {
155 156 157
	.signature	= MCE_LOG_SIGNATURE,
	.len		= MCE_LOG_LEN,
	.recordlen	= sizeof(struct mce),
158
};
L
Linus Torvalds 已提交
159 160 161 162

void mce_log(struct mce *mce)
{
	unsigned next, entry;
I
Ingo Molnar 已提交
163

164 165 166
	/* Emit the trace record: */
	trace_mce_record(mce);

167 168
	if (!mce_gen_pool_add(mce))
		irq_work_queue(&mce_irq_work);
169

M
Mike Waychison 已提交
170
	wmb();
L
Linus Torvalds 已提交
171
	for (;;) {
172
		entry = mce_log_get_idx_check(mcelog.next);
173
		for (;;) {
174

I
Ingo Molnar 已提交
175 176 177 178 179
			/*
			 * When the buffer fills up discard new entries.
			 * Assume that the earlier errors are the more
			 * interesting ones:
			 */
180
			if (entry >= MCE_LOG_LEN) {
181 182
				set_bit(MCE_OVERFLOW,
					(unsigned long *)&mcelog.flags);
183 184
				return;
			}
I
Ingo Molnar 已提交
185
			/* Old left over entry. Skip: */
186 187 188 189
			if (mcelog.entry[entry].finished) {
				entry++;
				continue;
			}
M
Mike Waychison 已提交
190
			break;
L
Linus Torvalds 已提交
191 192 193 194 195 196 197
		}
		smp_rmb();
		next = entry + 1;
		if (cmpxchg(&mcelog.next, entry, next) == entry)
			break;
	}
	memcpy(mcelog.entry + entry, mce, sizeof(struct mce));
M
Mike Waychison 已提交
198
	wmb();
L
Linus Torvalds 已提交
199
	mcelog.entry[entry].finished = 1;
M
Mike Waychison 已提交
200
	wmb();
L
Linus Torvalds 已提交
201

202
	set_bit(0, &mce_need_notify);
L
Linus Torvalds 已提交
203 204
}

205
void mce_inject_log(struct mce *m)
B
Borislav Petkov 已提交
206
{
207 208 209
	mutex_lock(&mce_chrdev_read_mutex);
	mce_log(m);
	mutex_unlock(&mce_chrdev_read_mutex);
B
Borislav Petkov 已提交
210
}
211
EXPORT_SYMBOL_GPL(mce_inject_log);
B
Borislav Petkov 已提交
212

213
static struct notifier_block mce_srao_nb;
B
Borislav Petkov 已提交
214

215 216
static atomic_t num_notifiers;

217 218
void mce_register_decode_chain(struct notifier_block *nb)
{
219 220
	atomic_inc(&num_notifiers);

221
	WARN_ON(nb->priority > MCE_PRIO_LOWEST && nb->priority < MCE_PRIO_EDAC);
222

223
	blocking_notifier_chain_register(&x86_mce_decoder_chain, nb);
224 225 226 227 228
}
EXPORT_SYMBOL_GPL(mce_register_decode_chain);

void mce_unregister_decode_chain(struct notifier_block *nb)
{
229 230
	atomic_dec(&num_notifiers);

231
	blocking_notifier_chain_unregister(&x86_mce_decoder_chain, nb);
232 233 234
}
EXPORT_SYMBOL_GPL(mce_unregister_decode_chain);

235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281
static inline u32 ctl_reg(int bank)
{
	return MSR_IA32_MCx_CTL(bank);
}

static inline u32 status_reg(int bank)
{
	return MSR_IA32_MCx_STATUS(bank);
}

static inline u32 addr_reg(int bank)
{
	return MSR_IA32_MCx_ADDR(bank);
}

static inline u32 misc_reg(int bank)
{
	return MSR_IA32_MCx_MISC(bank);
}

static inline u32 smca_ctl_reg(int bank)
{
	return MSR_AMD64_SMCA_MCx_CTL(bank);
}

static inline u32 smca_status_reg(int bank)
{
	return MSR_AMD64_SMCA_MCx_STATUS(bank);
}

static inline u32 smca_addr_reg(int bank)
{
	return MSR_AMD64_SMCA_MCx_ADDR(bank);
}

static inline u32 smca_misc_reg(int bank)
{
	return MSR_AMD64_SMCA_MCx_MISC(bank);
}

struct mca_msr_regs msr_ops = {
	.ctl	= ctl_reg,
	.status	= status_reg,
	.addr	= addr_reg,
	.misc	= misc_reg
};

282
static void __print_mce(struct mce *m)
L
Linus Torvalds 已提交
283
{
284 285 286 287
	pr_emerg(HW_ERR "CPU %d: Machine Check%s: %Lx Bank %d: %016Lx\n",
		 m->extcpu,
		 (m->mcgstatus & MCG_STATUS_MCIP ? " Exception" : ""),
		 m->mcgstatus, m->bank, m->status);
288

289
	if (m->ip) {
H
Huang Ying 已提交
290
		pr_emerg(HW_ERR "RIP%s %02x:<%016Lx> ",
291
			!(m->mcgstatus & MCG_STATUS_EIPV) ? " !INEXACT!" : "",
292
			m->cs, m->ip);
293

L
Linus Torvalds 已提交
294
		if (m->cs == __KERNEL_CS)
295
			print_symbol("{%s}", m->ip);
296
		pr_cont("\n");
L
Linus Torvalds 已提交
297
	}
298

H
Huang Ying 已提交
299
	pr_emerg(HW_ERR "TSC %llx ", m->tsc);
L
Linus Torvalds 已提交
300
	if (m->addr)
301
		pr_cont("ADDR %llx ", m->addr);
L
Linus Torvalds 已提交
302
	if (m->misc)
303
		pr_cont("MISC %llx ", m->misc);
304

305 306 307 308 309 310 311
	if (mce_flags.smca) {
		if (m->synd)
			pr_cont("SYND %llx ", m->synd);
		if (m->ipid)
			pr_cont("IPID %llx ", m->ipid);
	}

312
	pr_cont("\n");
313 314 315 316
	/*
	 * Note this output is parsed by external tools and old fields
	 * should not be changed.
	 */
317
	pr_emerg(HW_ERR "PROCESSOR %u:%x TIME %llu SOCKET %u APIC %x microcode %x\n",
318 319
		m->cpuvendor, m->cpuid, m->time, m->socketid, m->apicid,
		cpu_data(m->extcpu).microcode);
320 321 322 323 324
}

static void print_mce(struct mce *m)
{
	__print_mce(m);
325
	pr_emerg_ratelimited(HW_ERR "Run the above through 'mcelog --ascii'\n");
326 327
}

328 329
#define PANIC_TIMEOUT 5 /* 5 seconds */

330
static atomic_t mce_panicked;
331

332
static int fake_panic;
333
static atomic_t mce_fake_panicked;
334

335 336 337 338
/* Panic in progress. Enable interrupts and wait for final IPI */
static void wait_for_panic(void)
{
	long timeout = PANIC_TIMEOUT*USEC_PER_SEC;
339

340 341 342 343
	preempt_disable();
	local_irq_enable();
	while (timeout-- > 0)
		udelay(1);
344
	if (panic_timeout == 0)
345
		panic_timeout = mca_cfg.panic_timeout;
346 347 348
	panic("Panicing machine check CPU died");
}

349
static void mce_panic(const char *msg, struct mce *final, char *exp)
350
{
351 352 353
	int apei_err = 0;
	struct llist_node *pending;
	struct mce_evt_llist *l;
354

355 356 357 358
	if (!fake_panic) {
		/*
		 * Make sure only one CPU runs in machine check panic
		 */
359
		if (atomic_inc_return(&mce_panicked) > 1)
360 361
			wait_for_panic();
		barrier();
362

363 364 365 366
		bust_spinlocks(1);
		console_verbose();
	} else {
		/* Don't log too much for fake panic */
367
		if (atomic_inc_return(&mce_fake_panicked) > 1)
368 369
			return;
	}
370
	pending = mce_gen_pool_prepare_records();
371
	/* First print corrected ones that are still unlogged */
372 373
	llist_for_each_entry(l, pending, llnode) {
		struct mce *m = &l->mce;
374
		if (!(m->status & MCI_STATUS_UC)) {
H
Hidetoshi Seto 已提交
375
			print_mce(m);
376 377 378
			if (!apei_err)
				apei_err = apei_write_mce(m);
		}
379 380
	}
	/* Now print uncorrected but with the final one last */
381 382
	llist_for_each_entry(l, pending, llnode) {
		struct mce *m = &l->mce;
H
Hidetoshi Seto 已提交
383 384
		if (!(m->status & MCI_STATUS_UC))
			continue;
385
		if (!final || mce_cmp(m, final)) {
H
Hidetoshi Seto 已提交
386
			print_mce(m);
387 388 389
			if (!apei_err)
				apei_err = apei_write_mce(m);
		}
L
Linus Torvalds 已提交
390
	}
391
	if (final) {
H
Hidetoshi Seto 已提交
392
		print_mce(final);
393 394 395
		if (!apei_err)
			apei_err = apei_write_mce(final);
	}
396
	if (cpu_missing)
H
Huang Ying 已提交
397
		pr_emerg(HW_ERR "Some CPUs didn't answer in synchronization\n");
398
	if (exp)
H
Huang Ying 已提交
399
		pr_emerg(HW_ERR "Machine check: %s\n", exp);
400 401
	if (!fake_panic) {
		if (panic_timeout == 0)
402
			panic_timeout = mca_cfg.panic_timeout;
403 404
		panic(msg);
	} else
H
Huang Ying 已提交
405
		pr_emerg(HW_ERR "Fake kernel panic: %s\n", msg);
406
}
L
Linus Torvalds 已提交
407

408 409 410 411
/* Support code for software error injection */

static int msr_to_offset(u32 msr)
{
T
Tejun Heo 已提交
412
	unsigned bank = __this_cpu_read(injectm.bank);
413

414
	if (msr == mca_cfg.rip_msr)
415
		return offsetof(struct mce, ip);
416
	if (msr == msr_ops.status(bank))
417
		return offsetof(struct mce, status);
418
	if (msr == msr_ops.addr(bank))
419
		return offsetof(struct mce, addr);
420
	if (msr == msr_ops.misc(bank))
421 422 423 424 425 426
		return offsetof(struct mce, misc);
	if (msr == MSR_IA32_MCG_STATUS)
		return offsetof(struct mce, mcgstatus);
	return -1;
}

427 428 429 430
/* MSR access wrappers used for error injection */
static u64 mce_rdmsrl(u32 msr)
{
	u64 v;
431

T
Tejun Heo 已提交
432
	if (__this_cpu_read(injectm.finished)) {
433
		int offset = msr_to_offset(msr);
434

435 436
		if (offset < 0)
			return 0;
437
		return *(u64 *)((char *)this_cpu_ptr(&injectm) + offset);
438
	}
439 440

	if (rdmsrl_safe(msr, &v)) {
441
		WARN_ONCE(1, "mce: Unable to read MSR 0x%x!\n", msr);
442 443 444 445 446 447 448 449
		/*
		 * Return zero in case the access faulted. This should
		 * not happen normally but can happen if the CPU does
		 * something weird, or if the code is buggy.
		 */
		v = 0;
	}

450 451 452 453 454
	return v;
}

static void mce_wrmsrl(u32 msr, u64 v)
{
T
Tejun Heo 已提交
455
	if (__this_cpu_read(injectm.finished)) {
456
		int offset = msr_to_offset(msr);
457

458
		if (offset >= 0)
459
			*(u64 *)((char *)this_cpu_ptr(&injectm) + offset) = v;
460 461
		return;
	}
462 463 464
	wrmsrl(msr, v);
}

465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482
/*
 * Collect all global (w.r.t. this processor) status about this machine
 * check into our "mce" struct so that we can use it later to assess
 * the severity of the problem as we read per-bank specific details.
 */
static inline void mce_gather_info(struct mce *m, struct pt_regs *regs)
{
	mce_setup(m);

	m->mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS);
	if (regs) {
		/*
		 * Get the address of the instruction at the time of
		 * the machine check error.
		 */
		if (m->mcgstatus & (MCG_STATUS_RIPV|MCG_STATUS_EIPV)) {
			m->ip = regs->ip;
			m->cs = regs->cs;
483 484 485 486 487 488 489 490

			/*
			 * When in VM86 mode make the cs look like ring 3
			 * always. This is a lie, but it's better than passing
			 * the additional vm86 bit around everywhere.
			 */
			if (v8086_mode(regs))
				m->cs |= 3;
491 492
		}
		/* Use accurate RIP reporting if available. */
493 494
		if (mca_cfg.rip_msr)
			m->ip = mce_rdmsrl(mca_cfg.rip_msr);
495 496 497
	}
}

A
Andi Kleen 已提交
498
int mce_available(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
499
{
500
	if (mca_cfg.disabled)
501
		return 0;
502
	return cpu_has(c, X86_FEATURE_MCE) && cpu_has(c, X86_FEATURE_MCA);
L
Linus Torvalds 已提交
503 504
}

505 506
static void mce_schedule_work(void)
{
507
	if (!mce_gen_pool_empty())
508
		schedule_work(&mce_work);
509 510
}

511
static void mce_irq_work_cb(struct irq_work *entry)
512
{
513
	mce_notify_irq();
514
	mce_schedule_work();
515 516 517 518 519
}

static void mce_report_event(struct pt_regs *regs)
{
	if (regs->flags & (X86_VM_MASK|X86_EFLAGS_IF)) {
520
		mce_notify_irq();
521 522 523 524 525 526 527
		/*
		 * Triggering the work queue here is just an insurance
		 * policy in case the syscall exit notify handler
		 * doesn't run soon enough or ends up running on the
		 * wrong CPU (can happen when audit sleeps)
		 */
		mce_schedule_work();
528 529 530
		return;
	}

531
	irq_work_queue(&mce_irq_work);
532 533
}

534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555
/*
 * Check if the address reported by the CPU is in a format we can parse.
 * It would be possible to add code for most other cases, but all would
 * be somewhat complicated (e.g. segment offset would require an instruction
 * parser). So only support physical addresses up to page granuality for now.
 */
static int mce_usable_address(struct mce *m)
{
	if (!(m->status & MCI_STATUS_MISCV) || !(m->status & MCI_STATUS_ADDRV))
		return 0;

	/* Checks after this one are Intel-specific: */
	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
		return 1;

	if (MCI_MISC_ADDR_LSB(m->misc) > PAGE_SHIFT)
		return 0;
	if (MCI_MISC_ADDR_MODE(m->misc) != MCI_MISC_ADDR_PHYS)
		return 0;
	return 1;
}

556 557 558 559 560 561 562 563 564
static int srao_decode_notifier(struct notifier_block *nb, unsigned long val,
				void *data)
{
	struct mce *mce = (struct mce *)data;
	unsigned long pfn;

	if (!mce)
		return NOTIFY_DONE;

B
Borislav Petkov 已提交
565
	if (mce_usable_address(mce) && (mce->severity == MCE_AO_SEVERITY)) {
566 567 568 569 570
		pfn = mce->addr >> PAGE_SHIFT;
		memory_failure(pfn, MCE_VECTOR, 0);
	}

	return NOTIFY_OK;
571
}
572 573
static struct notifier_block mce_srao_nb = {
	.notifier_call	= srao_decode_notifier,
574
	.priority	= MCE_PRIO_SRAO,
575
};
576

577 578 579 580 581 582 583 584 585 586 587 588 589 590 591
static int mce_default_notifier(struct notifier_block *nb, unsigned long val,
				void *data)
{
	struct mce *m = (struct mce *)data;

	if (!m)
		return NOTIFY_DONE;

	/*
	 * Run the default notifier if we have only the SRAO
	 * notifier and us registered.
	 */
	if (atomic_read(&num_notifiers) > 2)
		return NOTIFY_DONE;

592 593 594 595
	/* Don't print when mcelog is running */
	if (mce_chrdev_open_count > 0)
		return NOTIFY_DONE;

596 597 598 599 600 601 602 603
	__print_mce(m);

	return NOTIFY_DONE;
}

static struct notifier_block mce_default_nb = {
	.notifier_call	= mce_default_notifier,
	/* lowest prio, we want it to run last. */
604
	.priority	= MCE_PRIO_LOWEST,
605 606
};

607 608 609 610 611 612
/*
 * Read ADDR and MISC registers.
 */
static void mce_read_aux(struct mce *m, int i)
{
	if (m->status & MCI_STATUS_MISCV)
613
		m->misc = mce_rdmsrl(msr_ops.misc(i));
614

615
	if (m->status & MCI_STATUS_ADDRV) {
616
		m->addr = mce_rdmsrl(msr_ops.addr(i));
617 618 619 620

		/*
		 * Mask the reported address by the reported granularity.
		 */
621
		if (mca_cfg.ser && (m->status & MCI_STATUS_MISCV)) {
622 623 624 625
			u8 shift = MCI_MISC_ADDR_LSB(m->misc);
			m->addr >>= shift;
			m->addr <<= shift;
		}
626 627 628 629 630 631 632 633 634 635

		/*
		 * Extract [55:<lsb>] where lsb is the least significant
		 * *valid* bit of the address bits.
		 */
		if (mce_flags.smca) {
			u8 lsb = (m->addr >> 56) & 0x3f;

			m->addr &= GENMASK_ULL(55, lsb);
		}
636
	}
637

638 639 640 641 642 643
	if (mce_flags.smca) {
		m->ipid = mce_rdmsrl(MSR_AMD64_SMCA_MCx_IPID(i));

		if (m->status & MCI_STATUS_SYNDV)
			m->synd = mce_rdmsrl(MSR_AMD64_SMCA_MCx_SYND(i));
	}
644 645
}

646 647 648 649 650
static bool memory_error(struct mce *m)
{
	struct cpuinfo_x86 *c = &boot_cpu_data;

	if (c->x86_vendor == X86_VENDOR_AMD) {
651 652 653 654
		/* ErrCodeExt[20:16] */
		u8 xec = (m->status >> 16) & 0x1f;

		return (xec == 0x0 || xec == 0x8);
655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676
	} else if (c->x86_vendor == X86_VENDOR_INTEL) {
		/*
		 * Intel SDM Volume 3B - 15.9.2 Compound Error Codes
		 *
		 * Bit 7 of the MCACOD field of IA32_MCi_STATUS is used for
		 * indicating a memory error. Bit 8 is used for indicating a
		 * cache hierarchy error. The combination of bit 2 and bit 3
		 * is used for indicating a `generic' cache hierarchy error
		 * But we can't just blindly check the above bits, because if
		 * bit 11 is set, then it is a bus/interconnect error - and
		 * either way the above bits just gives more detail on what
		 * bus/interconnect error happened. Note that bit 12 can be
		 * ignored, as it's the "filter" bit.
		 */
		return (m->status & 0xef80) == BIT(7) ||
		       (m->status & 0xef00) == BIT(8) ||
		       (m->status & 0xeffc) == 0xc;
	}

	return false;
}

677 678
DEFINE_PER_CPU(unsigned, mce_poll_count);

679
/*
680 681 682 683
 * Poll for corrected events or events that happened before reset.
 * Those are just logged through /dev/mcelog.
 *
 * This is executed in standard interrupt context.
A
Andi Kleen 已提交
684 685 686 687 688 689 690 691 692
 *
 * Note: spec recommends to panic for fatal unsignalled
 * errors here. However this would be quite problematic --
 * we would need to reimplement the Monarch handling and
 * it would mess up the exclusion between exception handler
 * and poll hander -- * so we skip this for now.
 * These cases should not happen anyways, or only when the CPU
 * is already totally * confused. In this case it's likely it will
 * not fully execute the machine check handler either.
693
 */
694
bool machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
695
{
696
	bool error_seen = false;
697
	struct mce m;
698
	int severity;
699 700
	int i;

701
	this_cpu_inc(mce_poll_count);
702

703
	mce_gather_info(&m, NULL);
704

705 706
	if (flags & MCP_TIMESTAMP)
		m.tsc = rdtsc();
707

708
	for (i = 0; i < mca_cfg.banks; i++) {
709
		if (!mce_banks[i].ctl || !test_bit(i, *b))
710 711 712 713 714 715 716
			continue;

		m.misc = 0;
		m.addr = 0;
		m.bank = i;

		barrier();
717
		m.status = mce_rdmsrl(msr_ops.status(i));
718 719 720 721
		if (!(m.status & MCI_STATUS_VAL))
			continue;

		/*
A
Andi Kleen 已提交
722 723
		 * Uncorrected or signalled events are handled by the exception
		 * handler when it is enabled, so don't process those here.
724 725 726
		 *
		 * TBD do the same check for MCI_STATUS_EN here?
		 */
A
Andi Kleen 已提交
727
		if (!(flags & MCP_UC) &&
728
		    (m.status & (mca_cfg.ser ? MCI_STATUS_S : MCI_STATUS_UC)))
729 730
			continue;

731 732
		error_seen = true;

733
		mce_read_aux(&m, i);
734

735 736
		severity = mce_severity(&m, mca_cfg.tolerant, NULL, false);

B
Borislav Petkov 已提交
737 738
		if (severity == MCE_DEFERRED_SEVERITY && memory_error(&m))
			if (m.status & MCI_STATUS_ADDRV)
739
				m.severity = severity;
740

741 742 743 744
		/*
		 * Don't get the IP here because it's unlikely to
		 * have anything to do with the actual error location.
		 */
745
		if (!(flags & MCP_DONTLOG) && !mca_cfg.dont_log_ce)
A
Andi Kleen 已提交
746
			mce_log(&m);
B
Borislav Petkov 已提交
747
		else if (mce_usable_address(&m)) {
748 749 750 751 752 753 754
			/*
			 * Although we skipped logging this, we still want
			 * to take action. Add to the pool so the registered
			 * notifiers will see it.
			 */
			if (!mce_gen_pool_add(&m))
				mce_schedule_work();
755
		}
756 757 758 759

		/*
		 * Clear state for this bank.
		 */
760
		mce_wrmsrl(msr_ops.status(i), 0);
761 762 763 764 765 766
	}

	/*
	 * Don't clear MCG_STATUS here because it's only defined for
	 * exceptions.
	 */
767 768

	sync_core();
769

770
	return error_seen;
771
}
772
EXPORT_SYMBOL_GPL(machine_check_poll);
773

774 775 776 777
/*
 * Do a quick check if any of the events requires a panic.
 * This decides if we keep the events around or clear them.
 */
778 779
static int mce_no_way_out(struct mce *m, char **msg, unsigned long *validp,
			  struct pt_regs *regs)
780
{
781
	int i, ret = 0;
782
	char *tmp;
783

784
	for (i = 0; i < mca_cfg.banks; i++) {
785
		m->status = mce_rdmsrl(msr_ops.status(i));
786
		if (m->status & MCI_STATUS_VAL) {
787
			__set_bit(i, validp);
788 789 790
			if (quirk_no_way_out)
				quirk_no_way_out(i, m, regs);
		}
791 792 793

		if (mce_severity(m, mca_cfg.tolerant, &tmp, true) >= MCE_PANIC_SEVERITY) {
			*msg = tmp;
794
			ret = 1;
795
		}
796
	}
797
	return ret;
798 799
}

800 801 802 803 804 805 806 807 808 809 810 811 812 813
/*
 * Variable to establish order between CPUs while scanning.
 * Each CPU spins initially until executing is equal its number.
 */
static atomic_t mce_executing;

/*
 * Defines order of CPUs on entry. First CPU becomes Monarch.
 */
static atomic_t mce_callin;

/*
 * Check if a timeout waiting for other CPUs happened.
 */
814
static int mce_timed_out(u64 *t, const char *msg)
815 816 817 818 819 820 821 822
{
	/*
	 * The others already did panic for some reason.
	 * Bail out like in a timeout.
	 * rmb() to tell the compiler that system_state
	 * might have been modified by someone else.
	 */
	rmb();
823
	if (atomic_read(&mce_panicked))
824
		wait_for_panic();
825
	if (!mca_cfg.monarch_timeout)
826 827
		goto out;
	if ((s64)*t < SPINUNIT) {
828
		if (mca_cfg.tolerant <= 1)
829
			mce_panic(msg, NULL, NULL);
830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850
		cpu_missing = 1;
		return 1;
	}
	*t -= SPINUNIT;
out:
	touch_nmi_watchdog();
	return 0;
}

/*
 * The Monarch's reign.  The Monarch is the CPU who entered
 * the machine check handler first. It waits for the others to
 * raise the exception too and then grades them. When any
 * error is fatal panic. Only then let the others continue.
 *
 * The other CPUs entering the MCE handler will be controlled by the
 * Monarch. They are called Subjects.
 *
 * This way we prevent any potential data corruption in a unrecoverable case
 * and also makes sure always all CPU's errors are examined.
 *
851
 * Also this detects the case of a machine check event coming from outer
852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876
 * space (not detected by any CPUs) In this case some external agent wants
 * us to shut down, so panic too.
 *
 * The other CPUs might still decide to panic if the handler happens
 * in a unrecoverable place, but in this case the system is in a semi-stable
 * state and won't corrupt anything by itself. It's ok to let the others
 * continue for a bit first.
 *
 * All the spin loops have timeouts; when a timeout happens a CPU
 * typically elects itself to be Monarch.
 */
static void mce_reign(void)
{
	int cpu;
	struct mce *m = NULL;
	int global_worst = 0;
	char *msg = NULL;
	char *nmsg = NULL;

	/*
	 * This CPU is the Monarch and the other CPUs have run
	 * through their handlers.
	 * Grade the severity of the errors of all the CPUs.
	 */
	for_each_possible_cpu(cpu) {
877 878
		int severity = mce_severity(&per_cpu(mces_seen, cpu),
					    mca_cfg.tolerant,
879
					    &nmsg, true);
880 881 882 883 884 885 886 887 888 889 890 891
		if (severity > global_worst) {
			msg = nmsg;
			global_worst = severity;
			m = &per_cpu(mces_seen, cpu);
		}
	}

	/*
	 * Cannot recover? Panic here then.
	 * This dumps all the mces in the log buffer and stops the
	 * other CPUs.
	 */
892
	if (m && global_worst >= MCE_PANIC_SEVERITY && mca_cfg.tolerant < 3)
893
		mce_panic("Fatal machine check", m, msg);
894 895 896 897 898 899 900 901 902 903 904

	/*
	 * For UC somewhere we let the CPU who detects it handle it.
	 * Also must let continue the others, otherwise the handling
	 * CPU could deadlock on a lock.
	 */

	/*
	 * No machine check event found. Must be some external
	 * source or one CPU is hung. Panic.
	 */
905
	if (global_worst <= MCE_KEEP_SEVERITY && mca_cfg.tolerant < 3)
906
		mce_panic("Fatal machine check from unknown source", NULL, NULL);
907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924

	/*
	 * Now clear all the mces_seen so that they don't reappear on
	 * the next mce.
	 */
	for_each_possible_cpu(cpu)
		memset(&per_cpu(mces_seen, cpu), 0, sizeof(struct mce));
}

static atomic_t global_nwo;

/*
 * Start of Monarch synchronization. This waits until all CPUs have
 * entered the exception handler and then determines if any of them
 * saw a fatal event that requires panic. Then it executes them
 * in the entry order.
 * TBD double check parallel CPU hotunplug
 */
H
Hidetoshi Seto 已提交
925
static int mce_start(int *no_way_out)
926
{
H
Hidetoshi Seto 已提交
927
	int order;
928
	int cpus = num_online_cpus();
929
	u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC;
930

H
Hidetoshi Seto 已提交
931 932
	if (!timeout)
		return -1;
933

H
Hidetoshi Seto 已提交
934
	atomic_add(*no_way_out, &global_nwo);
935
	/*
936 937
	 * Rely on the implied barrier below, such that global_nwo
	 * is updated before mce_callin.
938
	 */
939
	order = atomic_inc_return(&mce_callin);
940 941 942 943 944

	/*
	 * Wait for everyone.
	 */
	while (atomic_read(&mce_callin) != cpus) {
945 946
		if (mce_timed_out(&timeout,
				  "Timeout: Not all CPUs entered broadcast exception handler")) {
947
			atomic_set(&global_nwo, 0);
H
Hidetoshi Seto 已提交
948
			return -1;
949 950 951 952
		}
		ndelay(SPINUNIT);
	}

953 954 955 956
	/*
	 * mce_callin should be read before global_nwo
	 */
	smp_rmb();
957

H
Hidetoshi Seto 已提交
958 959 960 961
	if (order == 1) {
		/*
		 * Monarch: Starts executing now, the others wait.
		 */
962
		atomic_set(&mce_executing, 1);
H
Hidetoshi Seto 已提交
963 964 965 966 967 968 969 970
	} else {
		/*
		 * Subject: Now start the scanning loop one by one in
		 * the original callin order.
		 * This way when there are any shared banks it will be
		 * only seen by one CPU before cleared, avoiding duplicates.
		 */
		while (atomic_read(&mce_executing) < order) {
971 972
			if (mce_timed_out(&timeout,
					  "Timeout: Subject CPUs unable to finish machine check processing")) {
H
Hidetoshi Seto 已提交
973 974 975 976 977
				atomic_set(&global_nwo, 0);
				return -1;
			}
			ndelay(SPINUNIT);
		}
978 979 980
	}

	/*
H
Hidetoshi Seto 已提交
981
	 * Cache the global no_way_out state.
982
	 */
H
Hidetoshi Seto 已提交
983 984 985
	*no_way_out = atomic_read(&global_nwo);

	return order;
986 987 988 989 990 991 992 993 994
}

/*
 * Synchronize between CPUs after main scanning loop.
 * This invokes the bulk of the Monarch processing.
 */
static int mce_end(int order)
{
	int ret = -1;
995
	u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC;
996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015

	if (!timeout)
		goto reset;
	if (order < 0)
		goto reset;

	/*
	 * Allow others to run.
	 */
	atomic_inc(&mce_executing);

	if (order == 1) {
		/* CHECKME: Can this race with a parallel hotplug? */
		int cpus = num_online_cpus();

		/*
		 * Monarch: Wait for everyone to go through their scanning
		 * loops.
		 */
		while (atomic_read(&mce_executing) <= cpus) {
1016 1017
			if (mce_timed_out(&timeout,
					  "Timeout: Monarch CPU unable to finish machine check processing"))
1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029
				goto reset;
			ndelay(SPINUNIT);
		}

		mce_reign();
		barrier();
		ret = 0;
	} else {
		/*
		 * Subject: Wait for Monarch to finish.
		 */
		while (atomic_read(&mce_executing) != 0) {
1030 1031
			if (mce_timed_out(&timeout,
					  "Timeout: Monarch CPU did not finish machine check processing"))
1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060
				goto reset;
			ndelay(SPINUNIT);
		}

		/*
		 * Don't reset anything. That's done by the Monarch.
		 */
		return 0;
	}

	/*
	 * Reset all global state.
	 */
reset:
	atomic_set(&global_nwo, 0);
	atomic_set(&mce_callin, 0);
	barrier();

	/*
	 * Let others run again.
	 */
	atomic_set(&mce_executing, 0);
	return ret;
}

static void mce_clear_state(unsigned long *toclear)
{
	int i;

1061
	for (i = 0; i < mca_cfg.banks; i++) {
1062
		if (test_bit(i, toclear))
1063
			mce_wrmsrl(msr_ops.status(i), 0);
1064 1065 1066
	}
}

1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080
static int do_memory_failure(struct mce *m)
{
	int flags = MF_ACTION_REQUIRED;
	int ret;

	pr_err("Uncorrected hardware memory error in user-access at %llx", m->addr);
	if (!(m->mcgstatus & MCG_STATUS_RIPV))
		flags |= MF_MUST_KILL;
	ret = memory_failure(m->addr >> PAGE_SHIFT, MCE_VECTOR, flags);
	if (ret)
		pr_err("Memory error not recovered");
	return ret;
}

1081 1082 1083 1084 1085 1086 1087
/*
 * The actual machine check handler. This only handles real
 * exceptions when something got corrupted coming in through int 18.
 *
 * This is executed in NMI context not subject to normal locking rules. This
 * implies that most kernel services cannot be safely used. Don't even
 * think about putting a printk in there!
1088 1089 1090 1091
 *
 * On Intel systems this is entered on all CPUs in parallel through
 * MCE broadcast. However some CPUs might be broken beyond repair,
 * so be always careful when synchronizing with others.
L
Linus Torvalds 已提交
1092
 */
I
Ingo Molnar 已提交
1093
void do_machine_check(struct pt_regs *regs, long error_code)
L
Linus Torvalds 已提交
1094
{
1095
	struct mca_config *cfg = &mca_cfg;
1096
	struct mce m, *final;
L
Linus Torvalds 已提交
1097
	int i;
1098 1099
	int worst = 0;
	int severity;
1100

1101 1102 1103 1104
	/*
	 * Establish sequential order between the CPUs entering the machine
	 * check handler.
	 */
1105
	int order = -1;
1106 1107
	/*
	 * If no_way_out gets set, there is no safe way to recover from this
1108
	 * MCE.  If mca_cfg.tolerant is cranked up, we'll try anyway.
1109 1110 1111 1112 1113 1114 1115
	 */
	int no_way_out = 0;
	/*
	 * If kill_it gets set, there might be a way to recover from this
	 * error.
	 */
	int kill_it = 0;
1116
	DECLARE_BITMAP(toclear, MAX_NR_BANKS);
1117
	DECLARE_BITMAP(valid_banks, MAX_NR_BANKS);
1118
	char *msg = "Unknown";
1119 1120 1121 1122 1123 1124

	/*
	 * MCEs are always local on AMD. Same is determined by MCG_STATUS_LMCES
	 * on Intel.
	 */
	int lmce = 1;
L
Linus Torvalds 已提交
1125

1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136
	/* If this CPU is offline, just bail out. */
	if (cpu_is_offline(smp_processor_id())) {
		u64 mcgstatus;

		mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS);
		if (mcgstatus & MCG_STATUS_RIPV) {
			mce_wrmsrl(MSR_IA32_MCG_STATUS, 0);
			return;
		}
	}

1137
	ist_enter(regs);
1138

1139
	this_cpu_inc(mce_exception_count);
1140

1141
	if (!cfg->banks)
1142
		goto out;
L
Linus Torvalds 已提交
1143

1144
	mce_gather_info(&m, regs);
1145
	m.tsc = rdtsc();
1146

1147
	final = this_cpu_ptr(&mces_seen);
1148 1149
	*final = m;

1150
	memset(valid_banks, 0, sizeof(valid_banks));
1151
	no_way_out = mce_no_way_out(&m, &msg, valid_banks, regs);
1152

L
Linus Torvalds 已提交
1153 1154
	barrier();

A
Andi Kleen 已提交
1155
	/*
1156 1157 1158
	 * When no restart IP might need to kill or panic.
	 * Assume the worst for now, but if we find the
	 * severity is MCE_AR_SEVERITY we have other options.
A
Andi Kleen 已提交
1159 1160 1161 1162
	 */
	if (!(m.mcgstatus & MCG_STATUS_RIPV))
		kill_it = 1;

1163
	/*
1164 1165
	 * Check if this MCE is signaled to only this logical processor,
	 * on Intel only.
1166
	 */
1167 1168 1169 1170 1171 1172 1173 1174 1175 1176
	if (m.cpuvendor == X86_VENDOR_INTEL)
		lmce = m.mcgstatus & MCG_STATUS_LMCES;

	/*
	 * Go through all banks in exclusion of the other CPUs. This way we
	 * don't report duplicated events on shared banks because the first one
	 * to see it will clear it. If this is a Local MCE, then no need to
	 * perform rendezvous.
	 */
	if (!lmce)
A
Ashok Raj 已提交
1177 1178
		order = mce_start(&no_way_out);

1179
	for (i = 0; i < cfg->banks; i++) {
1180
		__clear_bit(i, toclear);
1181 1182
		if (!test_bit(i, valid_banks))
			continue;
1183
		if (!mce_banks[i].ctl)
L
Linus Torvalds 已提交
1184
			continue;
1185 1186

		m.misc = 0;
L
Linus Torvalds 已提交
1187 1188 1189
		m.addr = 0;
		m.bank = i;

1190
		m.status = mce_rdmsrl(msr_ops.status(i));
L
Linus Torvalds 已提交
1191 1192 1193
		if ((m.status & MCI_STATUS_VAL) == 0)
			continue;

1194
		/*
A
Andi Kleen 已提交
1195 1196
		 * Non uncorrected or non signaled errors are handled by
		 * machine_check_poll. Leave them alone, unless this panics.
1197
		 */
1198
		if (!(m.status & (cfg->ser ? MCI_STATUS_S : MCI_STATUS_UC)) &&
A
Andi Kleen 已提交
1199
			!no_way_out)
1200 1201 1202 1203 1204
			continue;

		/*
		 * Set taint even when machine check was not enabled.
		 */
1205
		add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE);
1206

1207
		severity = mce_severity(&m, cfg->tolerant, NULL, true);
1208

A
Andi Kleen 已提交
1209
		/*
1210 1211
		 * When machine check was for corrected/deferred handler don't
		 * touch, unless we're panicing.
A
Andi Kleen 已提交
1212
		 */
1213 1214
		if ((severity == MCE_KEEP_SEVERITY ||
		     severity == MCE_UCNA_SEVERITY) && !no_way_out)
A
Andi Kleen 已提交
1215 1216 1217
			continue;
		__set_bit(i, toclear);
		if (severity == MCE_NO_SEVERITY) {
1218 1219 1220 1221 1222
			/*
			 * Machine check event was not enabled. Clear, but
			 * ignore.
			 */
			continue;
L
Linus Torvalds 已提交
1223 1224
		}

1225
		mce_read_aux(&m, i);
L
Linus Torvalds 已提交
1226

1227 1228
		/* assuming valid severity level != 0 */
		m.severity = severity;
1229

1230
		mce_log(&m);
L
Linus Torvalds 已提交
1231

1232 1233 1234
		if (severity > worst) {
			*final = m;
			worst = severity;
L
Linus Torvalds 已提交
1235 1236 1237
		}
	}

1238 1239 1240
	/* mce_clear_state will clear *final, save locally for use later */
	m = *final;

1241 1242 1243
	if (!no_way_out)
		mce_clear_state(toclear);

I
Ingo Molnar 已提交
1244
	/*
1245 1246
	 * Do most of the synchronization with other CPUs.
	 * When there's any problem use only local no_way_out state.
I
Ingo Molnar 已提交
1247
	 */
A
Ashok Raj 已提交
1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259
	if (!lmce) {
		if (mce_end(order) < 0)
			no_way_out = worst >= MCE_PANIC_SEVERITY;
	} else {
		/*
		 * Local MCE skipped calling mce_reign()
		 * If we found a fatal error, we need to panic here.
		 */
		 if (worst >= MCE_PANIC_SEVERITY && mca_cfg.tolerant < 3)
			mce_panic("Machine check from unknown source",
				NULL, NULL);
	}
1260 1261

	/*
1262 1263
	 * If tolerant is at an insane level we drop requests to kill
	 * processes and continue even when there is no way out.
1264
	 */
1265 1266 1267 1268
	if (cfg->tolerant == 3)
		kill_it = 0;
	else if (no_way_out)
		mce_panic("Fatal machine check on current CPU", &m, msg);
1269

1270 1271
	if (worst > 0)
		mce_report_event(regs);
1272
	mce_wrmsrl(MSR_IA32_MCG_STATUS, 0);
1273
out:
1274
	sync_core();
1275

1276 1277
	if (worst != MCE_AR_SEVERITY && !kill_it)
		goto out_ist;
1278

1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290
	/* Fault was in user mode and we need to take some action */
	if ((m.cs & 3) == 3) {
		ist_begin_non_atomic(regs);
		local_irq_enable();

		if (kill_it || do_memory_failure(&m))
			force_sig(SIGBUS, current);
		local_irq_disable();
		ist_end_non_atomic();
	} else {
		if (!fixup_exception(regs, X86_TRAP_MC))
			mce_panic("Failed kernel mode recovery", &m, NULL);
1291
	}
1292 1293

out_ist:
1294
	ist_exit(regs);
L
Linus Torvalds 已提交
1295
}
1296
EXPORT_SYMBOL_GPL(do_machine_check);
L
Linus Torvalds 已提交
1297

1298 1299
#ifndef CONFIG_MEMORY_FAILURE
int memory_failure(unsigned long pfn, int vector, int flags)
1300
{
1301 1302
	/* mce_severity() should not hand us an ACTION_REQUIRED error */
	BUG_ON(flags & MF_ACTION_REQUIRED);
1303 1304 1305
	pr_err("Uncorrected memory error in page 0x%lx ignored\n"
	       "Rebuild kernel with CONFIG_MEMORY_FAILURE=y for smarter handling\n",
	       pfn);
1306 1307

	return 0;
1308
}
1309
#endif
1310

L
Linus Torvalds 已提交
1311
/*
1312 1313 1314
 * Periodic polling timer for "silent" machine check errors.  If the
 * poller finds an MCE, poll 2x faster.  When the poller finds no more
 * errors, poll 2x slower (up to check_interval seconds).
L
Linus Torvalds 已提交
1315
 */
1316
static unsigned long check_interval = INITIAL_CHECK_INTERVAL;
I
Ingo Molnar 已提交
1317

T
Thomas Gleixner 已提交
1318
static DEFINE_PER_CPU(unsigned long, mce_next_interval); /* in jiffies */
1319
static DEFINE_PER_CPU(struct timer_list, mce_timer);
L
Linus Torvalds 已提交
1320

C
Chen Gong 已提交
1321 1322 1323 1324 1325
static unsigned long mce_adjust_timer_default(unsigned long interval)
{
	return interval;
}

1326
static unsigned long (*mce_adjust_timer)(unsigned long interval) = mce_adjust_timer_default;
C
Chen Gong 已提交
1327

1328
static void __start_timer(struct timer_list *t, unsigned long interval)
1329
{
1330 1331
	unsigned long when = jiffies + interval;
	unsigned long flags;
1332

1333
	local_irq_save(flags);
1334

1335 1336
	if (!timer_pending(t) || time_before(when, t->expires))
		mod_timer(t, round_jiffies(when));
1337 1338

	local_irq_restore(flags);
1339 1340
}

T
Thomas Gleixner 已提交
1341
static void mce_timer_fn(unsigned long data)
L
Linus Torvalds 已提交
1342
{
1343
	struct timer_list *t = this_cpu_ptr(&mce_timer);
1344
	int cpu = smp_processor_id();
T
Thomas Gleixner 已提交
1345
	unsigned long iv;
1346

1347 1348 1349
	WARN_ON(cpu != data);

	iv = __this_cpu_read(mce_next_interval);
1350

1351
	if (mce_available(this_cpu_ptr(&cpu_info))) {
1352
		machine_check_poll(0, this_cpu_ptr(&mce_poll_banks));
1353 1354 1355 1356 1357

		if (mce_intel_cmci_poll()) {
			iv = mce_adjust_timer(iv);
			goto done;
		}
I
Ingo Molnar 已提交
1358
	}
L
Linus Torvalds 已提交
1359 1360

	/*
1361 1362
	 * Alert userspace if needed. If we logged an MCE, reduce the polling
	 * interval, otherwise increase the polling interval.
L
Linus Torvalds 已提交
1363
	 */
1364
	if (mce_notify_irq())
1365
		iv = max(iv / 2, (unsigned long) HZ/100);
1366
	else
T
Thomas Gleixner 已提交
1367
		iv = min(iv * 2, round_jiffies_relative(check_interval * HZ));
1368 1369

done:
T
Thomas Gleixner 已提交
1370
	__this_cpu_write(mce_next_interval, iv);
1371
	__start_timer(t, iv);
C
Chen Gong 已提交
1372
}
1373

C
Chen Gong 已提交
1374 1375 1376 1377 1378
/*
 * Ensure that the timer is firing in @interval from now.
 */
void mce_timer_kick(unsigned long interval)
{
1379
	struct timer_list *t = this_cpu_ptr(&mce_timer);
C
Chen Gong 已提交
1380 1381
	unsigned long iv = __this_cpu_read(mce_next_interval);

1382
	__start_timer(t, interval);
1383

C
Chen Gong 已提交
1384 1385
	if (interval < iv)
		__this_cpu_write(mce_next_interval, interval);
1386 1387
}

1388 1389 1390 1391 1392 1393 1394 1395 1396
/* Must not be called in IRQ context where del_timer_sync() can deadlock */
static void mce_timer_delete_all(void)
{
	int cpu;

	for_each_online_cpu(cpu)
		del_timer_sync(&per_cpu(mce_timer, cpu));
}

1397 1398
static void mce_do_trigger(struct work_struct *work)
{
1399
	call_usermodehelper(mce_helper, mce_helper_argv, NULL, UMH_NO_WAIT);
1400 1401 1402 1403
}

static DECLARE_WORK(mce_trigger_work, mce_do_trigger);

1404
/*
1405 1406 1407
 * Notify the user(s) about new machine check events.
 * Can be called from interrupt context, but not from machine check/NMI
 * context.
1408
 */
1409
int mce_notify_irq(void)
1410
{
1411 1412 1413
	/* Not more than two messages every minute */
	static DEFINE_RATELIMIT_STATE(ratelimit, 60*HZ, 2);

1414
	if (test_and_clear_bit(0, &mce_need_notify)) {
1415 1416
		/* wake processes polling /dev/mcelog */
		wake_up_interruptible(&mce_chrdev_wait);
1417

1418
		if (mce_helper[0])
1419
			schedule_work(&mce_trigger_work);
1420

1421
		if (__ratelimit(&ratelimit))
H
Huang Ying 已提交
1422
			pr_info(HW_ERR "Machine check events logged\n");
1423 1424

		return 1;
L
Linus Torvalds 已提交
1425
	}
1426 1427
	return 0;
}
1428
EXPORT_SYMBOL_GPL(mce_notify_irq);
1429

1430
static int __mcheck_cpu_mce_banks_init(void)
1431 1432
{
	int i;
1433
	u8 num_banks = mca_cfg.banks;
1434

1435
	mce_banks = kzalloc(num_banks * sizeof(struct mce_bank), GFP_KERNEL);
1436 1437
	if (!mce_banks)
		return -ENOMEM;
1438 1439

	for (i = 0; i < num_banks; i++) {
1440
		struct mce_bank *b = &mce_banks[i];
1441

1442 1443 1444 1445 1446 1447
		b->ctl = -1ULL;
		b->init = 1;
	}
	return 0;
}

1448
/*
L
Linus Torvalds 已提交
1449 1450
 * Initialize Machine Checks for a CPU.
 */
1451
static int __mcheck_cpu_cap_init(void)
L
Linus Torvalds 已提交
1452
{
1453
	unsigned b;
I
Ingo Molnar 已提交
1454
	u64 cap;
L
Linus Torvalds 已提交
1455 1456

	rdmsrl(MSR_IA32_MCG_CAP, cap);
1457 1458

	b = cap & MCG_BANKCNT_MASK;
1459
	if (!mca_cfg.banks)
1460
		pr_info("CPU supports %d MCE banks\n", b);
1461

1462
	if (b > MAX_NR_BANKS) {
1463
		pr_warn("Using only %u machine check banks out of %u\n",
1464 1465 1466 1467 1468
			MAX_NR_BANKS, b);
		b = MAX_NR_BANKS;
	}

	/* Don't support asymmetric configurations today */
1469 1470 1471
	WARN_ON(mca_cfg.banks != 0 && b != mca_cfg.banks);
	mca_cfg.banks = b;

1472
	if (!mce_banks) {
H
Hidetoshi Seto 已提交
1473
		int err = __mcheck_cpu_mce_banks_init();
1474

1475 1476
		if (err)
			return err;
L
Linus Torvalds 已提交
1477
	}
1478

1479
	/* Use accurate RIP reporting if available. */
1480
	if ((cap & MCG_EXT_P) && MCG_EXT_CNT(cap) >= 9)
1481
		mca_cfg.rip_msr = MSR_IA32_MCG_EIP;
L
Linus Torvalds 已提交
1482

A
Andi Kleen 已提交
1483
	if (cap & MCG_SER_P)
1484
		mca_cfg.ser = true;
A
Andi Kleen 已提交
1485

1486 1487 1488
	return 0;
}

1489
static void __mcheck_cpu_init_generic(void)
1490
{
1491
	enum mcp_flags m_fl = 0;
I
Ingo Molnar 已提交
1492
	mce_banks_t all_banks;
1493 1494
	u64 cap;

1495 1496 1497
	if (!mca_cfg.bootlog)
		m_fl = MCP_DONTLOG;

1498 1499 1500
	/*
	 * Log the machine checks left over from the previous reset.
	 */
1501
	bitmap_fill(all_banks, MAX_NR_BANKS);
1502
	machine_check_poll(MCP_UC | m_fl, &all_banks);
L
Linus Torvalds 已提交
1503

A
Andy Lutomirski 已提交
1504
	cr4_set_bits(X86_CR4_MCE);
L
Linus Torvalds 已提交
1505

1506
	rdmsrl(MSR_IA32_MCG_CAP, cap);
L
Linus Torvalds 已提交
1507 1508
	if (cap & MCG_CTL_P)
		wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff);
1509 1510 1511 1512 1513
}

static void __mcheck_cpu_init_clear_banks(void)
{
	int i;
L
Linus Torvalds 已提交
1514

1515
	for (i = 0; i < mca_cfg.banks; i++) {
1516
		struct mce_bank *b = &mce_banks[i];
1517

1518
		if (!b->init)
1519
			continue;
1520 1521
		wrmsrl(msr_ops.ctl(i), b->ctl);
		wrmsrl(msr_ops.status(i), 0);
1522
	}
L
Linus Torvalds 已提交
1523 1524
}

1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552
/*
 * During IFU recovery Sandy Bridge -EP4S processors set the RIPV and
 * EIPV bits in MCG_STATUS to zero on the affected logical processor (SDM
 * Vol 3B Table 15-20). But this confuses both the code that determines
 * whether the machine check occurred in kernel or user mode, and also
 * the severity assessment code. Pretend that EIPV was set, and take the
 * ip/cs values from the pt_regs that mce_gather_info() ignored earlier.
 */
static void quirk_sandybridge_ifu(int bank, struct mce *m, struct pt_regs *regs)
{
	if (bank != 0)
		return;
	if ((m->mcgstatus & (MCG_STATUS_EIPV|MCG_STATUS_RIPV)) != 0)
		return;
	if ((m->status & (MCI_STATUS_OVER|MCI_STATUS_UC|
		          MCI_STATUS_EN|MCI_STATUS_MISCV|MCI_STATUS_ADDRV|
			  MCI_STATUS_PCC|MCI_STATUS_S|MCI_STATUS_AR|
			  MCACOD)) !=
			 (MCI_STATUS_UC|MCI_STATUS_EN|
			  MCI_STATUS_MISCV|MCI_STATUS_ADDRV|MCI_STATUS_S|
			  MCI_STATUS_AR|MCACOD_INSTR))
		return;

	m->mcgstatus |= MCG_STATUS_EIPV;
	m->ip = regs->ip;
	m->cs = regs->cs;
}

L
Linus Torvalds 已提交
1553
/* Add per CPU specific workarounds here */
1554
static int __mcheck_cpu_apply_quirks(struct cpuinfo_x86 *c)
1555
{
1556 1557
	struct mca_config *cfg = &mca_cfg;

1558
	if (c->x86_vendor == X86_VENDOR_UNKNOWN) {
1559
		pr_info("unknown CPU type - not enabling MCE support\n");
1560 1561 1562
		return -EOPNOTSUPP;
	}

L
Linus Torvalds 已提交
1563
	/* This should be disabled by the BIOS, but isn't always */
1564
	if (c->x86_vendor == X86_VENDOR_AMD) {
1565
		if (c->x86 == 15 && cfg->banks > 4) {
I
Ingo Molnar 已提交
1566 1567 1568 1569 1570
			/*
			 * disable GART TBL walk error reporting, which
			 * trips off incorrectly with the IOMMU & 3ware
			 * & Cerberus:
			 */
1571
			clear_bit(10, (unsigned long *)&mce_banks[4].ctl);
I
Ingo Molnar 已提交
1572
		}
1573
		if (c->x86 < 17 && cfg->bootlog < 0) {
I
Ingo Molnar 已提交
1574 1575 1576 1577
			/*
			 * Lots of broken BIOS around that don't clear them
			 * by default and leave crap in there. Don't log:
			 */
1578
			cfg->bootlog = 0;
I
Ingo Molnar 已提交
1579
		}
1580 1581 1582 1583
		/*
		 * Various K7s with broken bank 0 around. Always disable
		 * by default.
		 */
1584
		if (c->x86 == 6 && cfg->banks > 0)
1585
			mce_banks[0].ctl = 0;
1586

1587 1588 1589 1590 1591 1592 1593
		/*
		 * overflow_recov is supported for F15h Models 00h-0fh
		 * even though we don't have a CPUID bit for it.
		 */
		if (c->x86 == 0x15 && c->x86_model <= 0xf)
			mce_flags.overflow_recov = 1;

1594 1595 1596 1597 1598 1599 1600 1601 1602 1603
		/*
		 * Turn off MC4_MISC thresholding banks on those models since
		 * they're not supported there.
		 */
		if (c->x86 == 0x15 &&
		    (c->x86_model >= 0x10 && c->x86_model <= 0x1f)) {
			int i;
			u64 hwcr;
			bool need_toggle;
			u32 msrs[] = {
1604 1605
				0x00000413, /* MC4_MISC0 */
				0xc0000408, /* MC4_MISC1 */
1606
			};
1607

1608
			rdmsrl(MSR_K7_HWCR, hwcr);
1609

1610 1611
			/* McStatusWrEn has to be set */
			need_toggle = !(hwcr & BIT(18));
1612

1613 1614
			if (need_toggle)
				wrmsrl(MSR_K7_HWCR, hwcr | BIT(18));
1615

1616 1617 1618
			/* Clear CntP bit safely */
			for (i = 0; i < ARRAY_SIZE(msrs); i++)
				msr_clear_bit(msrs[i], 62);
1619

1620 1621 1622 1623
			/* restore old settings */
			if (need_toggle)
				wrmsrl(MSR_K7_HWCR, hwcr);
		}
L
Linus Torvalds 已提交
1624
	}
1625

1626 1627 1628 1629 1630 1631 1632 1633 1634 1635
	if (c->x86_vendor == X86_VENDOR_INTEL) {
		/*
		 * SDM documents that on family 6 bank 0 should not be written
		 * because it aliases to another special BIOS controlled
		 * register.
		 * But it's not aliased anymore on model 0x1a+
		 * Don't ignore bank 0 completely because there could be a
		 * valid event later, merely don't write CTL0.
		 */

1636
		if (c->x86 == 6 && c->x86_model < 0x1A && cfg->banks > 0)
1637
			mce_banks[0].init = 0;
1638 1639 1640 1641 1642 1643

		/*
		 * All newer Intel systems support MCE broadcasting. Enable
		 * synchronization with a one second timeout.
		 */
		if ((c->x86 > 6 || (c->x86 == 6 && c->x86_model >= 0xe)) &&
1644 1645
			cfg->monarch_timeout < 0)
			cfg->monarch_timeout = USEC_PER_SEC;
1646

1647 1648 1649 1650
		/*
		 * There are also broken BIOSes on some Pentium M and
		 * earlier systems:
		 */
1651 1652
		if (c->x86 == 6 && c->x86_model <= 13 && cfg->bootlog < 0)
			cfg->bootlog = 0;
1653 1654 1655

		if (c->x86 == 6 && c->x86_model == 45)
			quirk_no_way_out = quirk_sandybridge_ifu;
1656
	}
1657 1658 1659
	if (cfg->monarch_timeout < 0)
		cfg->monarch_timeout = 0;
	if (cfg->bootlog != 0)
1660
		cfg->panic_timeout = 30;
1661 1662

	return 0;
1663
}
L
Linus Torvalds 已提交
1664

1665
static int __mcheck_cpu_ancient_init(struct cpuinfo_x86 *c)
1666 1667
{
	if (c->x86 != 5)
1668 1669
		return 0;

1670 1671
	switch (c->x86_vendor) {
	case X86_VENDOR_INTEL:
1672
		intel_p5_mcheck_init(c);
1673
		return 1;
1674 1675 1676
		break;
	case X86_VENDOR_CENTAUR:
		winchip_mcheck_init(c);
1677
		return 1;
1678
		break;
1679 1680
	default:
		return 0;
1681
	}
1682 1683

	return 0;
1684 1685
}

1686
static void __mcheck_cpu_init_vendor(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1687 1688 1689 1690
{
	switch (c->x86_vendor) {
	case X86_VENDOR_INTEL:
		mce_intel_feature_init(c);
1691
		mce_adjust_timer = cmci_intel_adjust_timer;
L
Linus Torvalds 已提交
1692
		break;
1693 1694

	case X86_VENDOR_AMD: {
1695 1696 1697
		mce_flags.overflow_recov = !!cpu_has(c, X86_FEATURE_OVERFLOW_RECOV);
		mce_flags.succor	 = !!cpu_has(c, X86_FEATURE_SUCCOR);
		mce_flags.smca		 = !!cpu_has(c, X86_FEATURE_SMCA);
1698 1699 1700 1701 1702 1703 1704 1705 1706 1707

		/*
		 * Install proper ops for Scalable MCA enabled processors
		 */
		if (mce_flags.smca) {
			msr_ops.ctl	= smca_ctl_reg;
			msr_ops.status	= smca_status_reg;
			msr_ops.addr	= smca_addr_reg;
			msr_ops.misc	= smca_misc_reg;
		}
1708
		mce_amd_feature_init(c);
1709

1710
		break;
1711 1712
		}

L
Linus Torvalds 已提交
1713 1714 1715 1716 1717
	default:
		break;
	}
}

1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728
static void __mcheck_cpu_clear_vendor(struct cpuinfo_x86 *c)
{
	switch (c->x86_vendor) {
	case X86_VENDOR_INTEL:
		mce_intel_feature_clear(c);
		break;
	default:
		break;
	}
}

1729
static void mce_start_timer(struct timer_list *t)
1730
{
1731
	unsigned long iv = check_interval * HZ;
1732

1733
	if (mca_cfg.ignore_ce || !iv)
1734 1735
		return;

1736 1737
	this_cpu_write(mce_next_interval, iv);
	__start_timer(t, iv);
1738 1739
}

1740 1741 1742 1743 1744 1745 1746 1747
static void __mcheck_cpu_setup_timer(void)
{
	struct timer_list *t = this_cpu_ptr(&mce_timer);
	unsigned int cpu = smp_processor_id();

	setup_pinned_timer(t, mce_timer_fn, cpu);
}

T
Thomas Gleixner 已提交
1748 1749
static void __mcheck_cpu_init_timer(void)
{
1750
	struct timer_list *t = this_cpu_ptr(&mce_timer);
T
Thomas Gleixner 已提交
1751 1752
	unsigned int cpu = smp_processor_id();

1753
	setup_pinned_timer(t, mce_timer_fn, cpu);
1754
	mce_start_timer(t);
T
Thomas Gleixner 已提交
1755 1756
}

A
Andi Kleen 已提交
1757 1758 1759
/* Handle unconfigured int18 (should never happen) */
static void unexpected_machine_check(struct pt_regs *regs, long error_code)
{
1760
	pr_err("CPU#%d: Unexpected int18 (Machine Check)\n",
A
Andi Kleen 已提交
1761 1762 1763 1764 1765 1766 1767
	       smp_processor_id());
}

/* Call the installed machine check handler for this CPU setup. */
void (*machine_check_vector)(struct pt_regs *, long error_code) =
						unexpected_machine_check;

1768
/*
L
Linus Torvalds 已提交
1769
 * Called for each booted CPU to set up machine checks.
I
Ingo Molnar 已提交
1770
 * Must be called with preempt off:
L
Linus Torvalds 已提交
1771
 */
1772
void mcheck_cpu_init(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1773
{
1774
	if (mca_cfg.disabled)
1775 1776
		return;

1777 1778
	if (__mcheck_cpu_ancient_init(c))
		return;
1779

1780
	if (!mce_available(c))
L
Linus Torvalds 已提交
1781 1782
		return;

1783
	if (__mcheck_cpu_cap_init() < 0 || __mcheck_cpu_apply_quirks(c) < 0) {
1784
		mca_cfg.disabled = true;
1785 1786 1787
		return;
	}

1788 1789 1790 1791 1792 1793
	if (mce_gen_pool_init()) {
		mca_cfg.disabled = true;
		pr_emerg("Couldn't allocate MCE records pool!\n");
		return;
	}

1794 1795
	machine_check_vector = do_machine_check;

1796 1797
	__mcheck_cpu_init_generic();
	__mcheck_cpu_init_vendor(c);
1798
	__mcheck_cpu_init_clear_banks();
1799
	__mcheck_cpu_setup_timer();
L
Linus Torvalds 已提交
1800 1801
}

1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818
/*
 * Called for each booted CPU to clear some machine checks opt-ins
 */
void mcheck_cpu_clear(struct cpuinfo_x86 *c)
{
	if (mca_cfg.disabled)
		return;

	if (!mce_available(c))
		return;

	/*
	 * Possibly to clear general settings generic to x86
	 * __mcheck_cpu_clear_generic(c);
	 */
	__mcheck_cpu_clear_vendor(c);

L
Linus Torvalds 已提交
1819 1820 1821
}

/*
1822
 * mce_chrdev: Character device /dev/mcelog to read and clear the MCE log.
L
Linus Torvalds 已提交
1823 1824
 */

1825 1826
static DEFINE_SPINLOCK(mce_chrdev_state_lock);
static int mce_chrdev_open_exclu;	/* already open exclusive? */
T
Tim Hockin 已提交
1827

1828
static int mce_chrdev_open(struct inode *inode, struct file *file)
T
Tim Hockin 已提交
1829
{
1830
	spin_lock(&mce_chrdev_state_lock);
T
Tim Hockin 已提交
1831

1832 1833 1834
	if (mce_chrdev_open_exclu ||
	    (mce_chrdev_open_count && (file->f_flags & O_EXCL))) {
		spin_unlock(&mce_chrdev_state_lock);
I
Ingo Molnar 已提交
1835

T
Tim Hockin 已提交
1836 1837 1838 1839
		return -EBUSY;
	}

	if (file->f_flags & O_EXCL)
1840 1841
		mce_chrdev_open_exclu = 1;
	mce_chrdev_open_count++;
T
Tim Hockin 已提交
1842

1843
	spin_unlock(&mce_chrdev_state_lock);
T
Tim Hockin 已提交
1844

1845
	return nonseekable_open(inode, file);
T
Tim Hockin 已提交
1846 1847
}

1848
static int mce_chrdev_release(struct inode *inode, struct file *file)
T
Tim Hockin 已提交
1849
{
1850
	spin_lock(&mce_chrdev_state_lock);
T
Tim Hockin 已提交
1851

1852 1853
	mce_chrdev_open_count--;
	mce_chrdev_open_exclu = 0;
T
Tim Hockin 已提交
1854

1855
	spin_unlock(&mce_chrdev_state_lock);
T
Tim Hockin 已提交
1856 1857 1858 1859

	return 0;
}

1860 1861
static void collect_tscs(void *data)
{
L
Linus Torvalds 已提交
1862
	unsigned long *cpu_tsc = (unsigned long *)data;
1863

1864
	cpu_tsc[smp_processor_id()] = rdtsc();
1865
}
L
Linus Torvalds 已提交
1866

1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882
static int mce_apei_read_done;

/* Collect MCE record of previous boot in persistent storage via APEI ERST. */
static int __mce_read_apei(char __user **ubuf, size_t usize)
{
	int rc;
	u64 record_id;
	struct mce m;

	if (usize < sizeof(struct mce))
		return -EINVAL;

	rc = apei_read_mce(&m, &record_id);
	/* Error or no more MCE record */
	if (rc <= 0) {
		mce_apei_read_done = 1;
1883 1884 1885 1886 1887 1888
		/*
		 * When ERST is disabled, mce_chrdev_read() should return
		 * "no record" instead of "no device."
		 */
		if (rc == -ENODEV)
			return 0;
1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909
		return rc;
	}
	rc = -EFAULT;
	if (copy_to_user(*ubuf, &m, sizeof(struct mce)))
		return rc;
	/*
	 * In fact, we should have cleared the record after that has
	 * been flushed to the disk or sent to network in
	 * /sbin/mcelog, but we have no interface to support that now,
	 * so just clear it to avoid duplication.
	 */
	rc = apei_clear_mce(record_id);
	if (rc) {
		mce_apei_read_done = 1;
		return rc;
	}
	*ubuf += sizeof(struct mce);

	return 0;
}

1910 1911
static ssize_t mce_chrdev_read(struct file *filp, char __user *ubuf,
				size_t usize, loff_t *off)
L
Linus Torvalds 已提交
1912
{
I
Ingo Molnar 已提交
1913
	char __user *buf = ubuf;
1914
	unsigned long *cpu_tsc;
1915
	unsigned prev, next;
L
Linus Torvalds 已提交
1916 1917
	int i, err;

1918
	cpu_tsc = kmalloc(nr_cpu_ids * sizeof(long), GFP_KERNEL);
1919 1920 1921
	if (!cpu_tsc)
		return -ENOMEM;

1922
	mutex_lock(&mce_chrdev_read_mutex);
1923 1924 1925 1926 1927 1928 1929

	if (!mce_apei_read_done) {
		err = __mce_read_apei(&buf, usize);
		if (err || buf != ubuf)
			goto out;
	}

1930
	next = mce_log_get_idx_check(mcelog.next);
L
Linus Torvalds 已提交
1931 1932

	/* Only supports full reads right now */
1933 1934 1935
	err = -EINVAL;
	if (*off != 0 || usize < MCE_LOG_LEN*sizeof(struct mce))
		goto out;
L
Linus Torvalds 已提交
1936 1937

	err = 0;
1938 1939 1940 1941
	prev = 0;
	do {
		for (i = prev; i < next; i++) {
			unsigned long start = jiffies;
H
Hidetoshi Seto 已提交
1942
			struct mce *m = &mcelog.entry[i];
1943

H
Hidetoshi Seto 已提交
1944
			while (!m->finished) {
1945
				if (time_after_eq(jiffies, start + 2)) {
H
Hidetoshi Seto 已提交
1946
					memset(m, 0, sizeof(*m));
1947 1948 1949
					goto timeout;
				}
				cpu_relax();
1950
			}
1951
			smp_rmb();
H
Hidetoshi Seto 已提交
1952 1953
			err |= copy_to_user(buf, m, sizeof(*m));
			buf += sizeof(*m);
1954 1955
timeout:
			;
1956
		}
L
Linus Torvalds 已提交
1957

1958 1959 1960 1961 1962
		memset(mcelog.entry + prev, 0,
		       (next - prev) * sizeof(struct mce));
		prev = next;
		next = cmpxchg(&mcelog.next, prev, 0);
	} while (next != prev);
L
Linus Torvalds 已提交
1963

1964
	synchronize_sched();
L
Linus Torvalds 已提交
1965

1966 1967 1968 1969
	/*
	 * Collect entries that were still getting written before the
	 * synchronize.
	 */
1970
	on_each_cpu(collect_tscs, cpu_tsc, 1);
I
Ingo Molnar 已提交
1971

1972
	for (i = next; i < MCE_LOG_LEN; i++) {
H
Hidetoshi Seto 已提交
1973 1974 1975 1976
		struct mce *m = &mcelog.entry[i];

		if (m->finished && m->tsc < cpu_tsc[m->cpu]) {
			err |= copy_to_user(buf, m, sizeof(*m));
L
Linus Torvalds 已提交
1977
			smp_rmb();
H
Hidetoshi Seto 已提交
1978 1979
			buf += sizeof(*m);
			memset(m, 0, sizeof(*m));
L
Linus Torvalds 已提交
1980
		}
1981
	}
1982 1983 1984 1985 1986

	if (err)
		err = -EFAULT;

out:
1987
	mutex_unlock(&mce_chrdev_read_mutex);
1988
	kfree(cpu_tsc);
I
Ingo Molnar 已提交
1989

1990
	return err ? err : buf - ubuf;
L
Linus Torvalds 已提交
1991 1992
}

1993
static unsigned int mce_chrdev_poll(struct file *file, poll_table *wait)
1994
{
1995
	poll_wait(file, &mce_chrdev_wait, wait);
1996
	if (READ_ONCE(mcelog.next))
1997
		return POLLIN | POLLRDNORM;
1998 1999
	if (!mce_apei_read_done && apei_check_mce())
		return POLLIN | POLLRDNORM;
2000 2001 2002
	return 0;
}

2003 2004
static long mce_chrdev_ioctl(struct file *f, unsigned int cmd,
				unsigned long arg)
L
Linus Torvalds 已提交
2005 2006
{
	int __user *p = (int __user *)arg;
2007

L
Linus Torvalds 已提交
2008
	if (!capable(CAP_SYS_ADMIN))
2009
		return -EPERM;
I
Ingo Molnar 已提交
2010

L
Linus Torvalds 已提交
2011
	switch (cmd) {
2012
	case MCE_GET_RECORD_LEN:
L
Linus Torvalds 已提交
2013 2014
		return put_user(sizeof(struct mce), p);
	case MCE_GET_LOG_LEN:
2015
		return put_user(MCE_LOG_LEN, p);
L
Linus Torvalds 已提交
2016 2017
	case MCE_GETCLEAR_FLAGS: {
		unsigned flags;
2018 2019

		do {
L
Linus Torvalds 已提交
2020
			flags = mcelog.flags;
2021
		} while (cmpxchg(&mcelog.flags, flags, 0) != flags);
I
Ingo Molnar 已提交
2022

2023
		return put_user(flags, p);
L
Linus Torvalds 已提交
2024 2025
	}
	default:
2026 2027
		return -ENOTTY;
	}
L
Linus Torvalds 已提交
2028 2029
}

2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040
static ssize_t (*mce_write)(struct file *filp, const char __user *ubuf,
			    size_t usize, loff_t *off);

void register_mce_write_callback(ssize_t (*fn)(struct file *filp,
			     const char __user *ubuf,
			     size_t usize, loff_t *off))
{
	mce_write = fn;
}
EXPORT_SYMBOL_GPL(register_mce_write_callback);

2041 2042
static ssize_t mce_chrdev_write(struct file *filp, const char __user *ubuf,
				size_t usize, loff_t *off)
2043 2044 2045 2046 2047 2048 2049 2050
{
	if (mce_write)
		return mce_write(filp, ubuf, usize, off);
	else
		return -EINVAL;
}

static const struct file_operations mce_chrdev_ops = {
2051 2052 2053
	.open			= mce_chrdev_open,
	.release		= mce_chrdev_release,
	.read			= mce_chrdev_read,
2054
	.write			= mce_chrdev_write,
2055 2056 2057
	.poll			= mce_chrdev_poll,
	.unlocked_ioctl		= mce_chrdev_ioctl,
	.llseek			= no_llseek,
L
Linus Torvalds 已提交
2058 2059
};

2060
static struct miscdevice mce_chrdev_device = {
L
Linus Torvalds 已提交
2061 2062 2063 2064 2065
	MISC_MCELOG_MINOR,
	"mcelog",
	&mce_chrdev_ops,
};

2066 2067 2068
static void __mce_disable_bank(void *arg)
{
	int bank = *((int *)arg);
2069
	__clear_bit(bank, this_cpu_ptr(mce_poll_banks));
2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084
	cmci_disable_bank(bank);
}

void mce_disable_bank(int bank)
{
	if (bank >= mca_cfg.banks) {
		pr_warn(FW_BUG
			"Ignoring request to disable invalid MCA bank %d.\n",
			bank);
		return;
	}
	set_bit(bank, mce_banks_ce_disabled);
	on_each_cpu(__mce_disable_bank, &bank, 1);
}

H
Hidetoshi Seto 已提交
2085
/*
2086 2087
 * mce=off Disables machine check
 * mce=no_cmci Disables CMCI
2088
 * mce=no_lmce Disables LMCE
2089 2090
 * mce=dont_log_ce Clears corrected events silently, no log created for CEs.
 * mce=ignore_ce Disables polling and CMCI, corrected events are not cleared.
2091 2092 2093
 * mce=TOLERANCELEVEL[,monarchtimeout] (number, see above)
 *	monarchtimeout is how long to wait for other CPUs on machine
 *	check, or 0 to not wait
H
Hidetoshi Seto 已提交
2094 2095
 * mce=bootlog Log MCEs from before booting. Disabled by default on AMD.
 * mce=nobootlog Don't log MCEs from before booting.
2096
 * mce=bios_cmci_threshold Don't program the CMCI threshold
2097
 * mce=recovery force enable memcpy_mcsafe()
H
Hidetoshi Seto 已提交
2098
 */
L
Linus Torvalds 已提交
2099 2100
static int __init mcheck_enable(char *str)
{
2101 2102
	struct mca_config *cfg = &mca_cfg;

2103
	if (*str == 0) {
2104
		enable_p5_mce();
2105 2106
		return 1;
	}
2107 2108
	if (*str == '=')
		str++;
L
Linus Torvalds 已提交
2109
	if (!strcmp(str, "off"))
2110
		cfg->disabled = true;
2111
	else if (!strcmp(str, "no_cmci"))
2112
		cfg->cmci_disabled = true;
2113 2114
	else if (!strcmp(str, "no_lmce"))
		cfg->lmce_disabled = true;
2115
	else if (!strcmp(str, "dont_log_ce"))
2116
		cfg->dont_log_ce = true;
2117
	else if (!strcmp(str, "ignore_ce"))
2118
		cfg->ignore_ce = true;
H
Hidetoshi Seto 已提交
2119
	else if (!strcmp(str, "bootlog") || !strcmp(str, "nobootlog"))
2120
		cfg->bootlog = (str[0] == 'b');
2121
	else if (!strcmp(str, "bios_cmci_threshold"))
2122
		cfg->bios_cmci_threshold = true;
2123 2124
	else if (!strcmp(str, "recovery"))
		cfg->recovery = true;
2125
	else if (isdigit(str[0])) {
2126
		if (get_option(&str, &cfg->tolerant) == 2)
2127
			get_option(&str, &(cfg->monarch_timeout));
2128
	} else {
2129
		pr_info("mce argument %s ignored. Please use /sys\n", str);
H
Hidetoshi Seto 已提交
2130 2131
		return 0;
	}
2132
	return 1;
L
Linus Torvalds 已提交
2133
}
2134
__setup("mce", mcheck_enable);
L
Linus Torvalds 已提交
2135

2136
int __init mcheck_init(void)
2137
{
2138
	mcheck_intel_therm_init();
2139
	mce_register_decode_chain(&mce_srao_nb);
2140
	mce_register_decode_chain(&mce_default_nb);
2141
	mcheck_vendor_init_severity();
2142

2143
	INIT_WORK(&mce_work, mce_gen_pool_process);
2144 2145
	init_irq_work(&mce_irq_work, mce_irq_work_cb);

2146 2147 2148
	return 0;
}

2149
/*
2150
 * mce_syscore: PM support
2151
 */
L
Linus Torvalds 已提交
2152

2153 2154 2155 2156
/*
 * Disable machine checks on suspend and shutdown. We can't really handle
 * them later.
 */
2157
static void mce_disable_error_reporting(void)
2158 2159 2160
{
	int i;

2161
	for (i = 0; i < mca_cfg.banks; i++) {
2162
		struct mce_bank *b = &mce_banks[i];
2163

2164
		if (b->init)
2165
			wrmsrl(msr_ops.ctl(i), 0);
2166
	}
2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181
	return;
}

static void vendor_disable_error_reporting(void)
{
	/*
	 * Don't clear on Intel CPUs. Some of these MSRs are socket-wide.
	 * Disabling them for just a single offlined CPU is bad, since it will
	 * inhibit reporting for all shared resources on the socket like the
	 * last level cache (LLC), the integrated memory controller (iMC), etc.
	 */
	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
		return;

	mce_disable_error_reporting();
2182 2183
}

2184
static int mce_syscore_suspend(void)
2185
{
2186 2187
	vendor_disable_error_reporting();
	return 0;
2188 2189
}

2190
static void mce_syscore_shutdown(void)
2191
{
2192
	vendor_disable_error_reporting();
2193 2194
}

I
Ingo Molnar 已提交
2195 2196 2197 2198 2199
/*
 * On resume clear all MCE state. Don't want to see leftovers from the BIOS.
 * Only one CPU is active at this time, the others get re-added later using
 * CPU hotplug:
 */
2200
static void mce_syscore_resume(void)
L
Linus Torvalds 已提交
2201
{
2202
	__mcheck_cpu_init_generic();
2203
	__mcheck_cpu_init_vendor(raw_cpu_ptr(&cpu_info));
2204
	__mcheck_cpu_init_clear_banks();
L
Linus Torvalds 已提交
2205 2206
}

2207
static struct syscore_ops mce_syscore_ops = {
2208 2209 2210
	.suspend	= mce_syscore_suspend,
	.shutdown	= mce_syscore_shutdown,
	.resume		= mce_syscore_resume,
2211 2212
};

2213
/*
2214
 * mce_device: Sysfs support
2215 2216
 */

2217 2218
static void mce_cpu_restart(void *data)
{
2219
	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2220
		return;
2221
	__mcheck_cpu_init_generic();
2222
	__mcheck_cpu_init_clear_banks();
2223
	__mcheck_cpu_init_timer();
2224 2225
}

L
Linus Torvalds 已提交
2226
/* Reinit MCEs after user configuration changes */
2227 2228
static void mce_restart(void)
{
2229
	mce_timer_delete_all();
2230
	on_each_cpu(mce_cpu_restart, NULL, 1);
L
Linus Torvalds 已提交
2231 2232
}

2233
/* Toggle features for corrected errors */
2234
static void mce_disable_cmci(void *data)
2235
{
2236
	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2237 2238 2239 2240 2241 2242
		return;
	cmci_clear();
}

static void mce_enable_ce(void *all)
{
2243
	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2244 2245 2246 2247
		return;
	cmci_reenable();
	cmci_recheck();
	if (all)
2248
		__mcheck_cpu_init_timer();
2249 2250
}

2251
static struct bus_type mce_subsys = {
I
Ingo Molnar 已提交
2252
	.name		= "machinecheck",
2253
	.dev_name	= "machinecheck",
L
Linus Torvalds 已提交
2254 2255
};

2256
DEFINE_PER_CPU(struct device *, mce_device);
I
Ingo Molnar 已提交
2257

2258
static inline struct mce_bank *attr_to_bank(struct device_attribute *attr)
2259 2260 2261
{
	return container_of(attr, struct mce_bank, attr);
}
2262

2263
static ssize_t show_bank(struct device *s, struct device_attribute *attr,
2264 2265
			 char *buf)
{
2266
	return sprintf(buf, "%llx\n", attr_to_bank(attr)->ctl);
2267 2268
}

2269
static ssize_t set_bank(struct device *s, struct device_attribute *attr,
H
Hidetoshi Seto 已提交
2270
			const char *buf, size_t size)
2271
{
H
Hidetoshi Seto 已提交
2272
	u64 new;
I
Ingo Molnar 已提交
2273

2274
	if (kstrtou64(buf, 0, &new) < 0)
2275
		return -EINVAL;
I
Ingo Molnar 已提交
2276

2277
	attr_to_bank(attr)->ctl = new;
2278
	mce_restart();
I
Ingo Molnar 已提交
2279

H
Hidetoshi Seto 已提交
2280
	return size;
2281
}
2282

I
Ingo Molnar 已提交
2283
static ssize_t
2284
show_trigger(struct device *s, struct device_attribute *attr, char *buf)
2285
{
2286
	strcpy(buf, mce_helper);
2287
	strcat(buf, "\n");
2288
	return strlen(mce_helper) + 1;
2289 2290
}

2291
static ssize_t set_trigger(struct device *s, struct device_attribute *attr,
I
Ingo Molnar 已提交
2292
				const char *buf, size_t siz)
2293 2294
{
	char *p;
I
Ingo Molnar 已提交
2295

2296 2297 2298
	strncpy(mce_helper, buf, sizeof(mce_helper));
	mce_helper[sizeof(mce_helper)-1] = 0;
	p = strchr(mce_helper, '\n');
I
Ingo Molnar 已提交
2299

2300
	if (p)
I
Ingo Molnar 已提交
2301 2302
		*p = 0;

2303
	return strlen(mce_helper) + !!p;
2304 2305
}

2306 2307
static ssize_t set_ignore_ce(struct device *s,
			     struct device_attribute *attr,
2308 2309 2310 2311
			     const char *buf, size_t size)
{
	u64 new;

2312
	if (kstrtou64(buf, 0, &new) < 0)
2313 2314
		return -EINVAL;

2315
	if (mca_cfg.ignore_ce ^ !!new) {
2316 2317
		if (new) {
			/* disable ce features */
2318 2319
			mce_timer_delete_all();
			on_each_cpu(mce_disable_cmci, NULL, 1);
2320
			mca_cfg.ignore_ce = true;
2321 2322
		} else {
			/* enable ce features */
2323
			mca_cfg.ignore_ce = false;
2324 2325 2326 2327 2328 2329
			on_each_cpu(mce_enable_ce, (void *)1, 1);
		}
	}
	return size;
}

2330 2331
static ssize_t set_cmci_disabled(struct device *s,
				 struct device_attribute *attr,
2332 2333 2334 2335
				 const char *buf, size_t size)
{
	u64 new;

2336
	if (kstrtou64(buf, 0, &new) < 0)
2337 2338
		return -EINVAL;

2339
	if (mca_cfg.cmci_disabled ^ !!new) {
2340 2341
		if (new) {
			/* disable cmci */
2342
			on_each_cpu(mce_disable_cmci, NULL, 1);
2343
			mca_cfg.cmci_disabled = true;
2344 2345
		} else {
			/* enable cmci */
2346
			mca_cfg.cmci_disabled = false;
2347 2348 2349 2350 2351 2352
			on_each_cpu(mce_enable_ce, NULL, 1);
		}
	}
	return size;
}

2353 2354
static ssize_t store_int_with_restart(struct device *s,
				      struct device_attribute *attr,
2355 2356
				      const char *buf, size_t size)
{
2357
	ssize_t ret = device_store_int(s, attr, buf, size);
2358 2359 2360 2361
	mce_restart();
	return ret;
}

2362
static DEVICE_ATTR(trigger, 0644, show_trigger, set_trigger);
2363
static DEVICE_INT_ATTR(tolerant, 0644, mca_cfg.tolerant);
2364
static DEVICE_INT_ATTR(monarch_timeout, 0644, mca_cfg.monarch_timeout);
2365
static DEVICE_BOOL_ATTR(dont_log_ce, 0644, mca_cfg.dont_log_ce);
I
Ingo Molnar 已提交
2366

2367 2368
static struct dev_ext_attribute dev_attr_check_interval = {
	__ATTR(check_interval, 0644, device_show_int, store_int_with_restart),
2369 2370
	&check_interval
};
I
Ingo Molnar 已提交
2371

2372
static struct dev_ext_attribute dev_attr_ignore_ce = {
2373 2374
	__ATTR(ignore_ce, 0644, device_show_bool, set_ignore_ce),
	&mca_cfg.ignore_ce
2375 2376
};

2377
static struct dev_ext_attribute dev_attr_cmci_disabled = {
2378 2379
	__ATTR(cmci_disabled, 0644, device_show_bool, set_cmci_disabled),
	&mca_cfg.cmci_disabled
2380 2381
};

2382 2383 2384 2385 2386 2387 2388 2389
static struct device_attribute *mce_device_attrs[] = {
	&dev_attr_tolerant.attr,
	&dev_attr_check_interval.attr,
	&dev_attr_trigger,
	&dev_attr_monarch_timeout.attr,
	&dev_attr_dont_log_ce.attr,
	&dev_attr_ignore_ce.attr,
	&dev_attr_cmci_disabled.attr,
2390 2391
	NULL
};
L
Linus Torvalds 已提交
2392

2393
static cpumask_var_t mce_device_initialized;
2394

2395 2396 2397 2398 2399
static void mce_device_release(struct device *dev)
{
	kfree(dev);
}

2400
/* Per cpu device init. All of the cpus still share the same ctrl bank: */
2401
static int mce_device_create(unsigned int cpu)
L
Linus Torvalds 已提交
2402
{
2403
	struct device *dev;
L
Linus Torvalds 已提交
2404
	int err;
2405
	int i, j;
2406

A
Andreas Herrmann 已提交
2407
	if (!mce_available(&boot_cpu_data))
2408 2409
		return -EIO;

2410 2411 2412 2413
	dev = per_cpu(mce_device, cpu);
	if (dev)
		return 0;

2414 2415 2416
	dev = kzalloc(sizeof *dev, GFP_KERNEL);
	if (!dev)
		return -ENOMEM;
2417 2418
	dev->id  = cpu;
	dev->bus = &mce_subsys;
2419
	dev->release = &mce_device_release;
2420

2421
	err = device_register(dev);
2422 2423
	if (err) {
		put_device(dev);
2424
		return err;
2425
	}
2426

2427 2428
	for (i = 0; mce_device_attrs[i]; i++) {
		err = device_create_file(dev, mce_device_attrs[i]);
2429 2430 2431
		if (err)
			goto error;
	}
2432
	for (j = 0; j < mca_cfg.banks; j++) {
2433
		err = device_create_file(dev, &mce_banks[j].attr);
2434 2435 2436
		if (err)
			goto error2;
	}
2437
	cpumask_set_cpu(cpu, mce_device_initialized);
2438
	per_cpu(mce_device, cpu) = dev;
2439

2440
	return 0;
2441
error2:
2442
	while (--j >= 0)
2443
		device_remove_file(dev, &mce_banks[j].attr);
2444
error:
I
Ingo Molnar 已提交
2445
	while (--i >= 0)
2446
		device_remove_file(dev, mce_device_attrs[i]);
I
Ingo Molnar 已提交
2447

2448
	device_unregister(dev);
2449

2450 2451 2452
	return err;
}

2453
static void mce_device_remove(unsigned int cpu)
2454
{
2455
	struct device *dev = per_cpu(mce_device, cpu);
2456 2457
	int i;

2458
	if (!cpumask_test_cpu(cpu, mce_device_initialized))
2459 2460
		return;

2461 2462
	for (i = 0; mce_device_attrs[i]; i++)
		device_remove_file(dev, mce_device_attrs[i]);
I
Ingo Molnar 已提交
2463

2464
	for (i = 0; i < mca_cfg.banks; i++)
2465
		device_remove_file(dev, &mce_banks[i].attr);
I
Ingo Molnar 已提交
2466

2467 2468
	device_unregister(dev);
	cpumask_clear_cpu(cpu, mce_device_initialized);
2469
	per_cpu(mce_device, cpu) = NULL;
2470 2471
}

2472
/* Make sure there are no machine checks on offlined CPUs. */
2473
static void mce_disable_cpu(void)
2474
{
2475
	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2476
		return;
2477

2478
	if (!cpuhp_tasks_frozen)
A
Andi Kleen 已提交
2479
		cmci_clear();
2480

2481
	vendor_disable_error_reporting();
2482 2483
}

2484
static void mce_reenable_cpu(void)
2485
{
I
Ingo Molnar 已提交
2486
	int i;
2487

2488
	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2489
		return;
I
Ingo Molnar 已提交
2490

2491
	if (!cpuhp_tasks_frozen)
A
Andi Kleen 已提交
2492
		cmci_reenable();
2493
	for (i = 0; i < mca_cfg.banks; i++) {
2494
		struct mce_bank *b = &mce_banks[i];
2495

2496
		if (b->init)
2497
			wrmsrl(msr_ops.ctl(i), b->ctl);
2498
	}
2499 2500
}

2501
static int mce_cpu_dead(unsigned int cpu)
2502
{
2503
	mce_intel_hcpu_update(cpu);
2504

2505 2506 2507 2508
	/* intentionally ignoring frozen here */
	if (!cpuhp_tasks_frozen)
		cmci_rediscover();
	return 0;
2509 2510
}

2511
static int mce_cpu_online(unsigned int cpu)
2512
{
2513
	struct timer_list *t = this_cpu_ptr(&mce_timer);
2514
	int ret;
2515

2516
	mce_device_create(cpu);
B
Borislav Petkov 已提交
2517

2518 2519 2520 2521
	ret = mce_threshold_create_device(cpu);
	if (ret) {
		mce_device_remove(cpu);
		return ret;
2522
	}
2523
	mce_reenable_cpu();
2524
	mce_start_timer(t);
2525
	return 0;
2526 2527
}

2528 2529
static int mce_cpu_pre_down(unsigned int cpu)
{
2530
	struct timer_list *t = this_cpu_ptr(&mce_timer);
2531 2532 2533 2534 2535 2536 2537

	mce_disable_cpu();
	del_timer_sync(t);
	mce_threshold_remove_device(cpu);
	mce_device_remove(cpu);
	return 0;
}
2538

2539
static __init void mce_init_banks(void)
2540 2541 2542
{
	int i;

2543
	for (i = 0; i < mca_cfg.banks; i++) {
2544
		struct mce_bank *b = &mce_banks[i];
2545
		struct device_attribute *a = &b->attr;
I
Ingo Molnar 已提交
2546

2547
		sysfs_attr_init(&a->attr);
2548 2549
		a->attr.name	= b->attrname;
		snprintf(b->attrname, ATTR_LEN, "bank%d", i);
I
Ingo Molnar 已提交
2550 2551 2552 2553

		a->attr.mode	= 0644;
		a->show		= show_bank;
		a->store	= set_bank;
2554 2555 2556
	}
}

2557
static __init int mcheck_init_device(void)
2558
{
2559
	enum cpuhp_state hp_online;
2560 2561
	int err;

2562 2563 2564 2565
	if (!mce_available(&boot_cpu_data)) {
		err = -EIO;
		goto err_out;
	}
2566

2567 2568 2569 2570
	if (!zalloc_cpumask_var(&mce_device_initialized, GFP_KERNEL)) {
		err = -ENOMEM;
		goto err_out;
	}
2571

2572
	mce_init_banks();
2573

2574
	err = subsys_system_register(&mce_subsys, NULL);
2575
	if (err)
2576
		goto err_out_mem;
2577

2578 2579 2580 2581
	err = cpuhp_setup_state(CPUHP_X86_MCE_DEAD, "x86/mce:dead", NULL,
				mce_cpu_dead);
	if (err)
		goto err_out_mem;
2582

2583 2584 2585
	err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/mce:online",
				mce_cpu_online, mce_cpu_pre_down);
	if (err < 0)
2586
		goto err_out_online;
2587
	hp_online = err;
2588

2589 2590
	register_syscore_ops(&mce_syscore_ops);

2591
	/* register character device /dev/mcelog */
2592 2593 2594 2595 2596 2597 2598 2599
	err = misc_register(&mce_chrdev_device);
	if (err)
		goto err_register;

	return 0;

err_register:
	unregister_syscore_ops(&mce_syscore_ops);
2600
	cpuhp_remove_state(hp_online);
2601

2602 2603
err_out_online:
	cpuhp_remove_state(CPUHP_X86_MCE_DEAD);
2604 2605 2606 2607 2608 2609

err_out_mem:
	free_cpumask_var(mce_device_initialized);

err_out:
	pr_err("Unable to init device /dev/mcelog (rc: %d)\n", err);
I
Ingo Molnar 已提交
2610

L
Linus Torvalds 已提交
2611 2612
	return err;
}
2613
device_initcall_sync(mcheck_init_device);
I
Ingo Molnar 已提交
2614

2615 2616 2617 2618 2619
/*
 * Old style boot options parsing. Only for compatibility.
 */
static int __init mcheck_disable(char *str)
{
2620
	mca_cfg.disabled = true;
2621 2622 2623
	return 1;
}
__setup("nomce", mcheck_disable);
I
Ingo Molnar 已提交
2624

2625 2626
#ifdef CONFIG_DEBUG_FS
struct dentry *mce_get_debugfs_dir(void)
I
Ingo Molnar 已提交
2627
{
2628
	static struct dentry *dmce;
I
Ingo Molnar 已提交
2629

2630 2631
	if (!dmce)
		dmce = debugfs_create_dir("mce", NULL);
I
Ingo Molnar 已提交
2632

2633 2634
	return dmce;
}
I
Ingo Molnar 已提交
2635

2636 2637 2638
static void mce_reset(void)
{
	cpu_missing = 0;
2639
	atomic_set(&mce_fake_panicked, 0);
2640 2641 2642 2643
	atomic_set(&mce_executing, 0);
	atomic_set(&mce_callin, 0);
	atomic_set(&global_nwo, 0);
}
I
Ingo Molnar 已提交
2644

2645 2646 2647 2648
static int fake_panic_get(void *data, u64 *val)
{
	*val = fake_panic;
	return 0;
I
Ingo Molnar 已提交
2649 2650
}

2651
static int fake_panic_set(void *data, u64 val)
I
Ingo Molnar 已提交
2652
{
2653 2654 2655
	mce_reset();
	fake_panic = val;
	return 0;
I
Ingo Molnar 已提交
2656 2657
}

2658 2659
DEFINE_SIMPLE_ATTRIBUTE(fake_panic_fops, fake_panic_get,
			fake_panic_set, "%llu\n");
2660

2661
static int __init mcheck_debugfs_init(void)
2662
{
2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673
	struct dentry *dmce, *ffake_panic;

	dmce = mce_get_debugfs_dir();
	if (!dmce)
		return -ENOMEM;
	ffake_panic = debugfs_create_file("fake_panic", 0444, dmce, NULL,
					  &fake_panic_fops);
	if (!ffake_panic)
		return -ENOMEM;

	return 0;
2674
}
2675 2676
#else
static int __init mcheck_debugfs_init(void) { return -EINVAL; }
2677
#endif
2678

2679 2680 2681
DEFINE_STATIC_KEY_FALSE(mcsafe_key);
EXPORT_SYMBOL_GPL(mcsafe_key);

2682 2683
static int __init mcheck_late_init(void)
{
2684 2685 2686
	if (mca_cfg.recovery)
		static_branch_inc(&mcsafe_key);

2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697
	mcheck_debugfs_init();

	/*
	 * Flush out everything that has been logged during early boot, now that
	 * everything has been initialized (workqueues, decoders, ...).
	 */
	mce_schedule_work();

	return 0;
}
late_initcall(mcheck_late_init);