mce.c 56.8 KB
Newer Older
L
Linus Torvalds 已提交
1 2
/*
 * Machine check handler.
I
Ingo Molnar 已提交
3
 *
L
Linus Torvalds 已提交
4
 * K8 parts Copyright 2002,2003 Andi Kleen, SuSE Labs.
5 6
 * Rest from unknown author(s).
 * 2004 Andi Kleen. Rewrote most of it.
7 8
 * Copyright 2008 Intel Corporation
 * Author: Andi Kleen
L
Linus Torvalds 已提交
9
 */
10 11 12

#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

I
Ingo Molnar 已提交
13 14 15 16 17 18
#include <linux/thread_info.h>
#include <linux/capability.h>
#include <linux/miscdevice.h>
#include <linux/ratelimit.h>
#include <linux/rcupdate.h>
#include <linux/kobject.h>
19
#include <linux/uaccess.h>
I
Ingo Molnar 已提交
20 21 22
#include <linux/kdebug.h>
#include <linux/kernel.h>
#include <linux/percpu.h>
L
Linus Torvalds 已提交
23
#include <linux/string.h>
24
#include <linux/device.h>
25
#include <linux/syscore_ops.h>
26
#include <linux/delay.h>
27
#include <linux/ctype.h>
I
Ingo Molnar 已提交
28
#include <linux/sched.h>
29
#include <linux/sysfs.h>
I
Ingo Molnar 已提交
30
#include <linux/types.h>
31
#include <linux/slab.h>
I
Ingo Molnar 已提交
32 33 34
#include <linux/init.h>
#include <linux/kmod.h>
#include <linux/poll.h>
35
#include <linux/nmi.h>
I
Ingo Molnar 已提交
36
#include <linux/cpu.h>
37
#include <linux/ras.h>
38
#include <linux/smp.h>
I
Ingo Molnar 已提交
39
#include <linux/fs.h>
40
#include <linux/mm.h>
41
#include <linux/debugfs.h>
42
#include <linux/irq_work.h>
43
#include <linux/export.h>
44
#include <linux/jump_label.h>
I
Ingo Molnar 已提交
45

46
#include <asm/intel-family.h>
47
#include <asm/processor.h>
48
#include <asm/traps.h>
49
#include <asm/tlbflush.h>
I
Ingo Molnar 已提交
50 51
#include <asm/mce.h>
#include <asm/msr.h>
52
#include <asm/reboot.h>
53
#include <asm/set_memory.h>
L
Linus Torvalds 已提交
54

55
#include "mce-internal.h"
56

57
static DEFINE_MUTEX(mce_log_mutex);
58

59 60 61
/* sysfs synchronization */
static DEFINE_MUTEX(mce_sysfs_mutex);

62 63 64
#define CREATE_TRACE_POINTS
#include <trace/events/mce.h>

65
#define SPINUNIT		100	/* 100ns */
66

67 68
DEFINE_PER_CPU(unsigned, mce_exception_count);

69
struct mce_bank *mce_banks __read_mostly;
70
struct mce_vendor_flags mce_flags __read_mostly;
71

72
struct mca_config mca_cfg __read_mostly = {
73
	.bootlog  = -1,
74 75 76 77 78 79 80
	/*
	 * Tolerant levels:
	 * 0: always panic on uncorrected errors, log corrected errors
	 * 1: panic or SIGBUS on uncorrected errors, log corrected errors
	 * 2: SIGBUS or log uncorrected errors (if possible), log corr. errors
	 * 3: never panic or SIGBUS, log all errors (for testing only)
	 */
81 82
	.tolerant = 1,
	.monarch_timeout = -1
83 84
};

85
static DEFINE_PER_CPU(struct mce, mces_seen);
86 87
static unsigned long mce_need_notify;
static int cpu_missing;
88

89 90 91 92
/*
 * MCA banks polled by the period polling timer for corrected events.
 * With Intel CMCI, this only has MCA banks which do not support CMCI (if any).
 */
93 94 95 96
DEFINE_PER_CPU(mce_banks_t, mce_poll_banks) = {
	[0 ... BITS_TO_LONGS(MAX_NR_BANKS)-1] = ~0UL
};

97 98 99 100 101 102 103 104 105
/*
 * MCA banks controlled through firmware first for corrected errors.
 * This is a global list of banks for which we won't enable CMCI and we
 * won't poll. Firmware controls these banks and is responsible for
 * reporting corrected errors through GHES. Uncorrected/recoverable
 * errors are still notified through a machine check.
 */
mce_banks_t mce_banks_ce_disabled;

106 107
static struct work_struct mce_work;
static struct irq_work mce_irq_work;
108

109 110
static void (*quirk_no_way_out)(int bank, struct mce *m, struct pt_regs *regs);

111 112 113 114
#ifndef mce_unmap_kpfn
static void mce_unmap_kpfn(unsigned long pfn);
#endif

115 116 117 118
/*
 * CPU/chipset specific EDAC code can register a notifier call here to print
 * MCE errors in a human-readable form.
 */
119
BLOCKING_NOTIFIER_HEAD(x86_mce_decoder_chain);
120

121 122 123 124
/* Do initial initialization of a struct mce */
void mce_setup(struct mce *m)
{
	memset(m, 0, sizeof(struct mce));
125
	m->cpu = m->extcpu = smp_processor_id();
126 127 128 129 130 131 132
	/* We hope get_seconds stays lockless */
	m->time = get_seconds();
	m->cpuvendor = boot_cpu_data.x86_vendor;
	m->cpuid = cpuid_eax(1);
	m->socketid = cpu_data(m->extcpu).phys_proc_id;
	m->apicid = cpu_data(m->extcpu).initial_apicid;
	rdmsrl(MSR_IA32_MCG_CAP, m->mcgcap);
133 134 135

	if (this_cpu_has(X86_FEATURE_INTEL_PPIN))
		rdmsrl(MSR_PPIN, m->ppin);
136 137

	m->microcode = boot_cpu_data.microcode;
138 139
}

140 141 142
DEFINE_PER_CPU(struct mce, injectm);
EXPORT_PER_CPU_SYMBOL_GPL(injectm);

143
void mce_log(struct mce *m)
L
Linus Torvalds 已提交
144
{
145
	if (!mce_gen_pool_add(m))
146
		irq_work_queue(&mce_irq_work);
L
Linus Torvalds 已提交
147 148
}

149
void mce_inject_log(struct mce *m)
150
{
151
	mutex_lock(&mce_log_mutex);
152
	mce_log(m);
153
	mutex_unlock(&mce_log_mutex);
154
}
155
EXPORT_SYMBOL_GPL(mce_inject_log);
156

157
static struct notifier_block mce_srao_nb;
158

159 160 161 162 163 164
/*
 * We run the default notifier if we have only the SRAO, the first and the
 * default notifier registered. I.e., the mandatory NUM_DEFAULT_NOTIFIERS
 * notifiers registered on the chain.
 */
#define NUM_DEFAULT_NOTIFIERS	3
165 166
static atomic_t num_notifiers;

167 168
void mce_register_decode_chain(struct notifier_block *nb)
{
169
	if (WARN_ON(nb->priority > MCE_PRIO_MCELOG && nb->priority < MCE_PRIO_EDAC))
170
		return;
171

172
	atomic_inc(&num_notifiers);
173

174
	blocking_notifier_chain_register(&x86_mce_decoder_chain, nb);
175 176 177 178 179
}
EXPORT_SYMBOL_GPL(mce_register_decode_chain);

void mce_unregister_decode_chain(struct notifier_block *nb)
{
180 181
	atomic_dec(&num_notifiers);

182
	blocking_notifier_chain_unregister(&x86_mce_decoder_chain, nb);
183 184 185
}
EXPORT_SYMBOL_GPL(mce_unregister_decode_chain);

186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232
static inline u32 ctl_reg(int bank)
{
	return MSR_IA32_MCx_CTL(bank);
}

static inline u32 status_reg(int bank)
{
	return MSR_IA32_MCx_STATUS(bank);
}

static inline u32 addr_reg(int bank)
{
	return MSR_IA32_MCx_ADDR(bank);
}

static inline u32 misc_reg(int bank)
{
	return MSR_IA32_MCx_MISC(bank);
}

static inline u32 smca_ctl_reg(int bank)
{
	return MSR_AMD64_SMCA_MCx_CTL(bank);
}

static inline u32 smca_status_reg(int bank)
{
	return MSR_AMD64_SMCA_MCx_STATUS(bank);
}

static inline u32 smca_addr_reg(int bank)
{
	return MSR_AMD64_SMCA_MCx_ADDR(bank);
}

static inline u32 smca_misc_reg(int bank)
{
	return MSR_AMD64_SMCA_MCx_MISC(bank);
}

struct mca_msr_regs msr_ops = {
	.ctl	= ctl_reg,
	.status	= status_reg,
	.addr	= addr_reg,
	.misc	= misc_reg
};

233
static void __print_mce(struct mce *m)
L
Linus Torvalds 已提交
234
{
235 236 237 238
	pr_emerg(HW_ERR "CPU %d: Machine Check%s: %Lx Bank %d: %016Lx\n",
		 m->extcpu,
		 (m->mcgstatus & MCG_STATUS_MCIP ? " Exception" : ""),
		 m->mcgstatus, m->bank, m->status);
239

240
	if (m->ip) {
241
		pr_emerg(HW_ERR "RIP%s %02x:<%016Lx> ",
242
			!(m->mcgstatus & MCG_STATUS_EIPV) ? " !INEXACT!" : "",
243
			m->cs, m->ip);
244

L
Linus Torvalds 已提交
245
		if (m->cs == __KERNEL_CS)
246
			pr_cont("{%pS}", (void *)(unsigned long)m->ip);
247
		pr_cont("\n");
L
Linus Torvalds 已提交
248
	}
249

250
	pr_emerg(HW_ERR "TSC %llx ", m->tsc);
L
Linus Torvalds 已提交
251
	if (m->addr)
252
		pr_cont("ADDR %llx ", m->addr);
L
Linus Torvalds 已提交
253
	if (m->misc)
254
		pr_cont("MISC %llx ", m->misc);
255

256 257 258 259 260 261 262
	if (mce_flags.smca) {
		if (m->synd)
			pr_cont("SYND %llx ", m->synd);
		if (m->ipid)
			pr_cont("IPID %llx ", m->ipid);
	}

263
	pr_cont("\n");
264 265 266 267
	/*
	 * Note this output is parsed by external tools and old fields
	 * should not be changed.
	 */
268
	pr_emerg(HW_ERR "PROCESSOR %u:%x TIME %llu SOCKET %u APIC %x microcode %x\n",
269
		m->cpuvendor, m->cpuid, m->time, m->socketid, m->apicid,
270
		m->microcode);
271 272 273 274 275
}

static void print_mce(struct mce *m)
{
	__print_mce(m);
276 277 278

	if (m->cpuvendor != X86_VENDOR_AMD)
		pr_emerg_ratelimited(HW_ERR "Run the above through 'mcelog --ascii'\n");
279 280
}

281 282
#define PANIC_TIMEOUT 5 /* 5 seconds */

283
static atomic_t mce_panicked;
284

285
static int fake_panic;
286
static atomic_t mce_fake_panicked;
287

288 289 290 291
/* Panic in progress. Enable interrupts and wait for final IPI */
static void wait_for_panic(void)
{
	long timeout = PANIC_TIMEOUT*USEC_PER_SEC;
292

293 294 295 296
	preempt_disable();
	local_irq_enable();
	while (timeout-- > 0)
		udelay(1);
297
	if (panic_timeout == 0)
298
		panic_timeout = mca_cfg.panic_timeout;
299 300 301
	panic("Panicing machine check CPU died");
}

302
static void mce_panic(const char *msg, struct mce *final, char *exp)
303
{
304 305 306
	int apei_err = 0;
	struct llist_node *pending;
	struct mce_evt_llist *l;
307

308 309 310 311
	if (!fake_panic) {
		/*
		 * Make sure only one CPU runs in machine check panic
		 */
312
		if (atomic_inc_return(&mce_panicked) > 1)
313 314
			wait_for_panic();
		barrier();
315

316 317 318 319
		bust_spinlocks(1);
		console_verbose();
	} else {
		/* Don't log too much for fake panic */
320
		if (atomic_inc_return(&mce_fake_panicked) > 1)
321 322
			return;
	}
323
	pending = mce_gen_pool_prepare_records();
324
	/* First print corrected ones that are still unlogged */
325 326
	llist_for_each_entry(l, pending, llnode) {
		struct mce *m = &l->mce;
327
		if (!(m->status & MCI_STATUS_UC)) {
328
			print_mce(m);
329 330 331
			if (!apei_err)
				apei_err = apei_write_mce(m);
		}
332 333
	}
	/* Now print uncorrected but with the final one last */
334 335
	llist_for_each_entry(l, pending, llnode) {
		struct mce *m = &l->mce;
336 337
		if (!(m->status & MCI_STATUS_UC))
			continue;
338
		if (!final || mce_cmp(m, final)) {
339
			print_mce(m);
340 341 342
			if (!apei_err)
				apei_err = apei_write_mce(m);
		}
L
Linus Torvalds 已提交
343
	}
344
	if (final) {
345
		print_mce(final);
346 347 348
		if (!apei_err)
			apei_err = apei_write_mce(final);
	}
349
	if (cpu_missing)
350
		pr_emerg(HW_ERR "Some CPUs didn't answer in synchronization\n");
351
	if (exp)
352
		pr_emerg(HW_ERR "Machine check: %s\n", exp);
353 354
	if (!fake_panic) {
		if (panic_timeout == 0)
355
			panic_timeout = mca_cfg.panic_timeout;
356 357
		panic(msg);
	} else
358
		pr_emerg(HW_ERR "Fake kernel panic: %s\n", msg);
359
}
L
Linus Torvalds 已提交
360

361 362 363 364
/* Support code for software error injection */

static int msr_to_offset(u32 msr)
{
365
	unsigned bank = __this_cpu_read(injectm.bank);
366

367
	if (msr == mca_cfg.rip_msr)
368
		return offsetof(struct mce, ip);
369
	if (msr == msr_ops.status(bank))
370
		return offsetof(struct mce, status);
371
	if (msr == msr_ops.addr(bank))
372
		return offsetof(struct mce, addr);
373
	if (msr == msr_ops.misc(bank))
374 375 376 377 378 379
		return offsetof(struct mce, misc);
	if (msr == MSR_IA32_MCG_STATUS)
		return offsetof(struct mce, mcgstatus);
	return -1;
}

380 381 382 383
/* MSR access wrappers used for error injection */
static u64 mce_rdmsrl(u32 msr)
{
	u64 v;
384

385
	if (__this_cpu_read(injectm.finished)) {
386
		int offset = msr_to_offset(msr);
387

388 389
		if (offset < 0)
			return 0;
390
		return *(u64 *)((char *)this_cpu_ptr(&injectm) + offset);
391
	}
392 393

	if (rdmsrl_safe(msr, &v)) {
394
		WARN_ONCE(1, "mce: Unable to read MSR 0x%x!\n", msr);
395 396 397 398 399 400 401 402
		/*
		 * Return zero in case the access faulted. This should
		 * not happen normally but can happen if the CPU does
		 * something weird, or if the code is buggy.
		 */
		v = 0;
	}

403 404 405 406 407
	return v;
}

static void mce_wrmsrl(u32 msr, u64 v)
{
408
	if (__this_cpu_read(injectm.finished)) {
409
		int offset = msr_to_offset(msr);
410

411
		if (offset >= 0)
412
			*(u64 *)((char *)this_cpu_ptr(&injectm) + offset) = v;
413 414
		return;
	}
415 416 417
	wrmsrl(msr, v);
}

418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435
/*
 * Collect all global (w.r.t. this processor) status about this machine
 * check into our "mce" struct so that we can use it later to assess
 * the severity of the problem as we read per-bank specific details.
 */
static inline void mce_gather_info(struct mce *m, struct pt_regs *regs)
{
	mce_setup(m);

	m->mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS);
	if (regs) {
		/*
		 * Get the address of the instruction at the time of
		 * the machine check error.
		 */
		if (m->mcgstatus & (MCG_STATUS_RIPV|MCG_STATUS_EIPV)) {
			m->ip = regs->ip;
			m->cs = regs->cs;
436 437 438 439 440 441 442 443

			/*
			 * When in VM86 mode make the cs look like ring 3
			 * always. This is a lie, but it's better than passing
			 * the additional vm86 bit around everywhere.
			 */
			if (v8086_mode(regs))
				m->cs |= 3;
444 445
		}
		/* Use accurate RIP reporting if available. */
446 447
		if (mca_cfg.rip_msr)
			m->ip = mce_rdmsrl(mca_cfg.rip_msr);
448 449 450
	}
}

451
int mce_available(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
452
{
453
	if (mca_cfg.disabled)
454
		return 0;
455
	return cpu_has(c, X86_FEATURE_MCE) && cpu_has(c, X86_FEATURE_MCA);
L
Linus Torvalds 已提交
456 457
}

458 459
static void mce_schedule_work(void)
{
460
	if (!mce_gen_pool_empty())
461
		schedule_work(&mce_work);
462 463
}

464
static void mce_irq_work_cb(struct irq_work *entry)
465
{
466
	mce_schedule_work();
467 468 469 470 471
}

static void mce_report_event(struct pt_regs *regs)
{
	if (regs->flags & (X86_VM_MASK|X86_EFLAGS_IF)) {
472
		mce_notify_irq();
473 474 475 476 477 478 479
		/*
		 * Triggering the work queue here is just an insurance
		 * policy in case the syscall exit notify handler
		 * doesn't run soon enough or ends up running on the
		 * wrong CPU (can happen when audit sleeps)
		 */
		mce_schedule_work();
480 481 482
		return;
	}

483
	irq_work_queue(&mce_irq_work);
484 485
}

486 487 488 489 490 491 492 493
/*
 * Check if the address reported by the CPU is in a format we can parse.
 * It would be possible to add code for most other cases, but all would
 * be somewhat complicated (e.g. segment offset would require an instruction
 * parser). So only support physical addresses up to page granuality for now.
 */
static int mce_usable_address(struct mce *m)
{
494
	if (!(m->status & MCI_STATUS_ADDRV))
495 496 497 498 499 500
		return 0;

	/* Checks after this one are Intel-specific: */
	if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
		return 1;

501 502 503
	if (!(m->status & MCI_STATUS_MISCV))
		return 0;

504 505
	if (MCI_MISC_ADDR_LSB(m->misc) > PAGE_SHIFT)
		return 0;
506

507 508
	if (MCI_MISC_ADDR_MODE(m->misc) != MCI_MISC_ADDR_PHYS)
		return 0;
509

510 511 512
	return 1;
}

513
bool mce_is_memory_error(struct mce *m)
514
{
515
	if (m->cpuvendor == X86_VENDOR_AMD) {
516
		return amd_mce_is_memory_error(m);
517

518
	} else if (m->cpuvendor == X86_VENDOR_INTEL) {
519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538
		/*
		 * Intel SDM Volume 3B - 15.9.2 Compound Error Codes
		 *
		 * Bit 7 of the MCACOD field of IA32_MCi_STATUS is used for
		 * indicating a memory error. Bit 8 is used for indicating a
		 * cache hierarchy error. The combination of bit 2 and bit 3
		 * is used for indicating a `generic' cache hierarchy error
		 * But we can't just blindly check the above bits, because if
		 * bit 11 is set, then it is a bus/interconnect error - and
		 * either way the above bits just gives more detail on what
		 * bus/interconnect error happened. Note that bit 12 can be
		 * ignored, as it's the "filter" bit.
		 */
		return (m->status & 0xef80) == BIT(7) ||
		       (m->status & 0xef00) == BIT(8) ||
		       (m->status & 0xeffc) == 0xc;
	}

	return false;
}
539
EXPORT_SYMBOL_GPL(mce_is_memory_error);
540

541 542 543 544 545 546 547 548 549 550 551
static bool mce_is_correctable(struct mce *m)
{
	if (m->cpuvendor == X86_VENDOR_AMD && m->status & MCI_STATUS_DEFERRED)
		return false;

	if (m->status & MCI_STATUS_UC)
		return false;

	return true;
}

552 553 554 555 556 557
static bool cec_add_mce(struct mce *m)
{
	if (!m)
		return false;

	/* We eat only correctable DRAM errors with usable addresses. */
558
	if (mce_is_memory_error(m) &&
559
	    mce_is_correctable(m)  &&
560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592
	    mce_usable_address(m))
		if (!cec_add_elem(m->addr >> PAGE_SHIFT))
			return true;

	return false;
}

static int mce_first_notifier(struct notifier_block *nb, unsigned long val,
			      void *data)
{
	struct mce *m = (struct mce *)data;

	if (!m)
		return NOTIFY_DONE;

	if (cec_add_mce(m))
		return NOTIFY_STOP;

	/* Emit the trace record: */
	trace_mce_record(m);

	set_bit(0, &mce_need_notify);

	mce_notify_irq();

	return NOTIFY_DONE;
}

static struct notifier_block first_nb = {
	.notifier_call	= mce_first_notifier,
	.priority	= MCE_PRIO_FIRST,
};

593 594 595 596 597 598 599 600 601
static int srao_decode_notifier(struct notifier_block *nb, unsigned long val,
				void *data)
{
	struct mce *mce = (struct mce *)data;
	unsigned long pfn;

	if (!mce)
		return NOTIFY_DONE;

602
	if (mce_usable_address(mce) && (mce->severity == MCE_AO_SEVERITY)) {
603
		pfn = mce->addr >> PAGE_SHIFT;
604 605
		if (!memory_failure(pfn, 0))
			mce_unmap_kpfn(pfn);
606 607 608
	}

	return NOTIFY_OK;
609
}
610 611
static struct notifier_block mce_srao_nb = {
	.notifier_call	= srao_decode_notifier,
612
	.priority	= MCE_PRIO_SRAO,
613
};
614

615 616 617 618 619 620 621 622
static int mce_default_notifier(struct notifier_block *nb, unsigned long val,
				void *data)
{
	struct mce *m = (struct mce *)data;

	if (!m)
		return NOTIFY_DONE;

623
	if (atomic_read(&num_notifiers) > NUM_DEFAULT_NOTIFIERS)
624 625
		return NOTIFY_DONE;

626 627 628 629 630 631 632 633
	__print_mce(m);

	return NOTIFY_DONE;
}

static struct notifier_block mce_default_nb = {
	.notifier_call	= mce_default_notifier,
	/* lowest prio, we want it to run last. */
634
	.priority	= MCE_PRIO_LOWEST,
635 636
};

637 638 639 640 641 642
/*
 * Read ADDR and MISC registers.
 */
static void mce_read_aux(struct mce *m, int i)
{
	if (m->status & MCI_STATUS_MISCV)
643
		m->misc = mce_rdmsrl(msr_ops.misc(i));
644

645
	if (m->status & MCI_STATUS_ADDRV) {
646
		m->addr = mce_rdmsrl(msr_ops.addr(i));
647 648 649 650

		/*
		 * Mask the reported address by the reported granularity.
		 */
651
		if (mca_cfg.ser && (m->status & MCI_STATUS_MISCV)) {
652 653 654 655
			u8 shift = MCI_MISC_ADDR_LSB(m->misc);
			m->addr >>= shift;
			m->addr <<= shift;
		}
656 657 658 659 660 661 662 663 664 665

		/*
		 * Extract [55:<lsb>] where lsb is the least significant
		 * *valid* bit of the address bits.
		 */
		if (mce_flags.smca) {
			u8 lsb = (m->addr >> 56) & 0x3f;

			m->addr &= GENMASK_ULL(55, lsb);
		}
666
	}
667

668 669 670 671 672 673
	if (mce_flags.smca) {
		m->ipid = mce_rdmsrl(MSR_AMD64_SMCA_MCx_IPID(i));

		if (m->status & MCI_STATUS_SYNDV)
			m->synd = mce_rdmsrl(MSR_AMD64_SMCA_MCx_SYND(i));
	}
674 675
}

676 677
DEFINE_PER_CPU(unsigned, mce_poll_count);

678
/*
679 680 681 682
 * Poll for corrected events or events that happened before reset.
 * Those are just logged through /dev/mcelog.
 *
 * This is executed in standard interrupt context.
683 684 685 686 687 688 689 690 691
 *
 * Note: spec recommends to panic for fatal unsignalled
 * errors here. However this would be quite problematic --
 * we would need to reimplement the Monarch handling and
 * it would mess up the exclusion between exception handler
 * and poll hander -- * so we skip this for now.
 * These cases should not happen anyways, or only when the CPU
 * is already totally * confused. In this case it's likely it will
 * not fully execute the machine check handler either.
692
 */
693
bool machine_check_poll(enum mcp_flags flags, mce_banks_t *b)
694
{
695
	bool error_seen = false;
696 697 698
	struct mce m;
	int i;

699
	this_cpu_inc(mce_poll_count);
700

701
	mce_gather_info(&m, NULL);
702

703 704
	if (flags & MCP_TIMESTAMP)
		m.tsc = rdtsc();
705

706
	for (i = 0; i < mca_cfg.banks; i++) {
707
		if (!mce_banks[i].ctl || !test_bit(i, *b))
708 709 710 711 712 713 714
			continue;

		m.misc = 0;
		m.addr = 0;
		m.bank = i;

		barrier();
715
		m.status = mce_rdmsrl(msr_ops.status(i));
716 717 718 719
		if (!(m.status & MCI_STATUS_VAL))
			continue;

		/*
720 721
		 * Uncorrected or signalled events are handled by the exception
		 * handler when it is enabled, so don't process those here.
722 723 724
		 *
		 * TBD do the same check for MCI_STATUS_EN here?
		 */
725
		if (!(flags & MCP_UC) &&
726
		    (m.status & (mca_cfg.ser ? MCI_STATUS_S : MCI_STATUS_UC)))
727 728
			continue;

729 730
		error_seen = true;

731
		mce_read_aux(&m, i);
732

733
		m.severity = mce_severity(&m, mca_cfg.tolerant, NULL, false);
734

735 736 737 738
		/*
		 * Don't get the IP here because it's unlikely to
		 * have anything to do with the actual error location.
		 */
739
		if (!(flags & MCP_DONTLOG) && !mca_cfg.dont_log_ce)
740
			mce_log(&m);
741
		else if (mce_usable_address(&m)) {
742 743 744 745 746 747 748
			/*
			 * Although we skipped logging this, we still want
			 * to take action. Add to the pool so the registered
			 * notifiers will see it.
			 */
			if (!mce_gen_pool_add(&m))
				mce_schedule_work();
749
		}
750 751 752 753

		/*
		 * Clear state for this bank.
		 */
754
		mce_wrmsrl(msr_ops.status(i), 0);
755 756 757 758 759 760
	}

	/*
	 * Don't clear MCG_STATUS here because it's only defined for
	 * exceptions.
	 */
761 762

	sync_core();
763

764
	return error_seen;
765
}
766
EXPORT_SYMBOL_GPL(machine_check_poll);
767

768 769 770 771
/*
 * Do a quick check if any of the events requires a panic.
 * This decides if we keep the events around or clear them.
 */
772 773
static int mce_no_way_out(struct mce *m, char **msg, unsigned long *validp,
			  struct pt_regs *regs)
774
{
775
	int i, ret = 0;
776
	char *tmp;
777

778
	for (i = 0; i < mca_cfg.banks; i++) {
779
		m->status = mce_rdmsrl(msr_ops.status(i));
780
		if (m->status & MCI_STATUS_VAL) {
781
			__set_bit(i, validp);
782 783 784
			if (quirk_no_way_out)
				quirk_no_way_out(i, m, regs);
		}
785 786 787

		if (mce_severity(m, mca_cfg.tolerant, &tmp, true) >= MCE_PANIC_SEVERITY) {
			*msg = tmp;
788
			ret = 1;
789
		}
790
	}
791
	return ret;
792 793
}

794 795 796 797 798 799 800 801 802 803 804 805 806 807
/*
 * Variable to establish order between CPUs while scanning.
 * Each CPU spins initially until executing is equal its number.
 */
static atomic_t mce_executing;

/*
 * Defines order of CPUs on entry. First CPU becomes Monarch.
 */
static atomic_t mce_callin;

/*
 * Check if a timeout waiting for other CPUs happened.
 */
808
static int mce_timed_out(u64 *t, const char *msg)
809 810 811 812 813 814 815 816
{
	/*
	 * The others already did panic for some reason.
	 * Bail out like in a timeout.
	 * rmb() to tell the compiler that system_state
	 * might have been modified by someone else.
	 */
	rmb();
817
	if (atomic_read(&mce_panicked))
818
		wait_for_panic();
819
	if (!mca_cfg.monarch_timeout)
820 821
		goto out;
	if ((s64)*t < SPINUNIT) {
822
		if (mca_cfg.tolerant <= 1)
823
			mce_panic(msg, NULL, NULL);
824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844
		cpu_missing = 1;
		return 1;
	}
	*t -= SPINUNIT;
out:
	touch_nmi_watchdog();
	return 0;
}

/*
 * The Monarch's reign.  The Monarch is the CPU who entered
 * the machine check handler first. It waits for the others to
 * raise the exception too and then grades them. When any
 * error is fatal panic. Only then let the others continue.
 *
 * The other CPUs entering the MCE handler will be controlled by the
 * Monarch. They are called Subjects.
 *
 * This way we prevent any potential data corruption in a unrecoverable case
 * and also makes sure always all CPU's errors are examined.
 *
845
 * Also this detects the case of a machine check event coming from outer
846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870
 * space (not detected by any CPUs) In this case some external agent wants
 * us to shut down, so panic too.
 *
 * The other CPUs might still decide to panic if the handler happens
 * in a unrecoverable place, but in this case the system is in a semi-stable
 * state and won't corrupt anything by itself. It's ok to let the others
 * continue for a bit first.
 *
 * All the spin loops have timeouts; when a timeout happens a CPU
 * typically elects itself to be Monarch.
 */
static void mce_reign(void)
{
	int cpu;
	struct mce *m = NULL;
	int global_worst = 0;
	char *msg = NULL;
	char *nmsg = NULL;

	/*
	 * This CPU is the Monarch and the other CPUs have run
	 * through their handlers.
	 * Grade the severity of the errors of all the CPUs.
	 */
	for_each_possible_cpu(cpu) {
871 872
		int severity = mce_severity(&per_cpu(mces_seen, cpu),
					    mca_cfg.tolerant,
873
					    &nmsg, true);
874 875 876 877 878 879 880 881 882 883 884 885
		if (severity > global_worst) {
			msg = nmsg;
			global_worst = severity;
			m = &per_cpu(mces_seen, cpu);
		}
	}

	/*
	 * Cannot recover? Panic here then.
	 * This dumps all the mces in the log buffer and stops the
	 * other CPUs.
	 */
886
	if (m && global_worst >= MCE_PANIC_SEVERITY && mca_cfg.tolerant < 3)
887
		mce_panic("Fatal machine check", m, msg);
888 889 890 891 892 893 894 895 896 897 898

	/*
	 * For UC somewhere we let the CPU who detects it handle it.
	 * Also must let continue the others, otherwise the handling
	 * CPU could deadlock on a lock.
	 */

	/*
	 * No machine check event found. Must be some external
	 * source or one CPU is hung. Panic.
	 */
899
	if (global_worst <= MCE_KEEP_SEVERITY && mca_cfg.tolerant < 3)
900
		mce_panic("Fatal machine check from unknown source", NULL, NULL);
901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918

	/*
	 * Now clear all the mces_seen so that they don't reappear on
	 * the next mce.
	 */
	for_each_possible_cpu(cpu)
		memset(&per_cpu(mces_seen, cpu), 0, sizeof(struct mce));
}

static atomic_t global_nwo;

/*
 * Start of Monarch synchronization. This waits until all CPUs have
 * entered the exception handler and then determines if any of them
 * saw a fatal event that requires panic. Then it executes them
 * in the entry order.
 * TBD double check parallel CPU hotunplug
 */
919
static int mce_start(int *no_way_out)
920
{
921
	int order;
922
	int cpus = num_online_cpus();
923
	u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC;
924

925 926
	if (!timeout)
		return -1;
927

928
	atomic_add(*no_way_out, &global_nwo);
929
	/*
930 931
	 * Rely on the implied barrier below, such that global_nwo
	 * is updated before mce_callin.
932
	 */
933
	order = atomic_inc_return(&mce_callin);
934 935 936 937 938

	/*
	 * Wait for everyone.
	 */
	while (atomic_read(&mce_callin) != cpus) {
939 940
		if (mce_timed_out(&timeout,
				  "Timeout: Not all CPUs entered broadcast exception handler")) {
941
			atomic_set(&global_nwo, 0);
942
			return -1;
943 944 945 946
		}
		ndelay(SPINUNIT);
	}

947 948 949 950
	/*
	 * mce_callin should be read before global_nwo
	 */
	smp_rmb();
951

952 953 954 955
	if (order == 1) {
		/*
		 * Monarch: Starts executing now, the others wait.
		 */
956
		atomic_set(&mce_executing, 1);
957 958 959 960 961 962 963 964
	} else {
		/*
		 * Subject: Now start the scanning loop one by one in
		 * the original callin order.
		 * This way when there are any shared banks it will be
		 * only seen by one CPU before cleared, avoiding duplicates.
		 */
		while (atomic_read(&mce_executing) < order) {
965 966
			if (mce_timed_out(&timeout,
					  "Timeout: Subject CPUs unable to finish machine check processing")) {
967 968 969 970 971
				atomic_set(&global_nwo, 0);
				return -1;
			}
			ndelay(SPINUNIT);
		}
972 973 974
	}

	/*
975
	 * Cache the global no_way_out state.
976
	 */
977 978 979
	*no_way_out = atomic_read(&global_nwo);

	return order;
980 981 982 983 984 985 986 987 988
}

/*
 * Synchronize between CPUs after main scanning loop.
 * This invokes the bulk of the Monarch processing.
 */
static int mce_end(int order)
{
	int ret = -1;
989
	u64 timeout = (u64)mca_cfg.monarch_timeout * NSEC_PER_USEC;
990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009

	if (!timeout)
		goto reset;
	if (order < 0)
		goto reset;

	/*
	 * Allow others to run.
	 */
	atomic_inc(&mce_executing);

	if (order == 1) {
		/* CHECKME: Can this race with a parallel hotplug? */
		int cpus = num_online_cpus();

		/*
		 * Monarch: Wait for everyone to go through their scanning
		 * loops.
		 */
		while (atomic_read(&mce_executing) <= cpus) {
1010 1011
			if (mce_timed_out(&timeout,
					  "Timeout: Monarch CPU unable to finish machine check processing"))
1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023
				goto reset;
			ndelay(SPINUNIT);
		}

		mce_reign();
		barrier();
		ret = 0;
	} else {
		/*
		 * Subject: Wait for Monarch to finish.
		 */
		while (atomic_read(&mce_executing) != 0) {
1024 1025
			if (mce_timed_out(&timeout,
					  "Timeout: Monarch CPU did not finish machine check processing"))
1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054
				goto reset;
			ndelay(SPINUNIT);
		}

		/*
		 * Don't reset anything. That's done by the Monarch.
		 */
		return 0;
	}

	/*
	 * Reset all global state.
	 */
reset:
	atomic_set(&global_nwo, 0);
	atomic_set(&mce_callin, 0);
	barrier();

	/*
	 * Let others run again.
	 */
	atomic_set(&mce_executing, 0);
	return ret;
}

static void mce_clear_state(unsigned long *toclear)
{
	int i;

1055
	for (i = 0; i < mca_cfg.banks; i++) {
1056
		if (test_bit(i, toclear))
1057
			mce_wrmsrl(msr_ops.status(i), 0);
1058 1059 1060
	}
}

1061 1062 1063 1064 1065 1066 1067 1068
static int do_memory_failure(struct mce *m)
{
	int flags = MF_ACTION_REQUIRED;
	int ret;

	pr_err("Uncorrected hardware memory error in user-access at %llx", m->addr);
	if (!(m->mcgstatus & MCG_STATUS_RIPV))
		flags |= MF_MUST_KILL;
1069
	ret = memory_failure(m->addr >> PAGE_SHIFT, flags);
1070 1071
	if (ret)
		pr_err("Memory error not recovered");
1072 1073
	else
		mce_unmap_kpfn(m->addr >> PAGE_SHIFT);
1074 1075 1076
	return ret;
}

1077 1078
#ifndef mce_unmap_kpfn
static void mce_unmap_kpfn(unsigned long pfn)
1079 1080 1081 1082 1083 1084 1085 1086 1087 1088
{
	unsigned long decoy_addr;

	/*
	 * Unmap this page from the kernel 1:1 mappings to make sure
	 * we don't log more errors because of speculative access to
	 * the page.
	 * We would like to just call:
	 *	set_memory_np((unsigned long)pfn_to_kaddr(pfn), 1);
	 * but doing that would radically increase the odds of a
1089
	 * speculative access to the poison page because we'd have
1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104
	 * the virtual address of the kernel 1:1 mapping sitting
	 * around in registers.
	 * Instead we get tricky.  We create a non-canonical address
	 * that looks just like the one we want, but has bit 63 flipped.
	 * This relies on set_memory_np() not checking whether we passed
	 * a legal address.
	 */

	decoy_addr = (pfn << PAGE_SHIFT) + (PAGE_OFFSET ^ BIT(63));

	if (set_memory_np(decoy_addr, 1))
		pr_warn("Could not invalidate pfn=0x%lx from 1:1 map\n", pfn);
}
#endif

1105 1106 1107 1108 1109 1110 1111
/*
 * The actual machine check handler. This only handles real
 * exceptions when something got corrupted coming in through int 18.
 *
 * This is executed in NMI context not subject to normal locking rules. This
 * implies that most kernel services cannot be safely used. Don't even
 * think about putting a printk in there!
1112 1113 1114 1115
 *
 * On Intel systems this is entered on all CPUs in parallel through
 * MCE broadcast. However some CPUs might be broken beyond repair,
 * so be always careful when synchronizing with others.
L
Linus Torvalds 已提交
1116
 */
I
Ingo Molnar 已提交
1117
void do_machine_check(struct pt_regs *regs, long error_code)
L
Linus Torvalds 已提交
1118
{
1119
	struct mca_config *cfg = &mca_cfg;
1120
	struct mce m, *final;
L
Linus Torvalds 已提交
1121
	int i;
1122 1123
	int worst = 0;
	int severity;
1124

1125 1126 1127 1128
	/*
	 * Establish sequential order between the CPUs entering the machine
	 * check handler.
	 */
1129
	int order = -1;
1130 1131
	/*
	 * If no_way_out gets set, there is no safe way to recover from this
1132
	 * MCE.  If mca_cfg.tolerant is cranked up, we'll try anyway.
1133 1134 1135 1136 1137 1138 1139
	 */
	int no_way_out = 0;
	/*
	 * If kill_it gets set, there might be a way to recover from this
	 * error.
	 */
	int kill_it = 0;
1140
	DECLARE_BITMAP(toclear, MAX_NR_BANKS);
1141
	DECLARE_BITMAP(valid_banks, MAX_NR_BANKS);
1142
	char *msg = "Unknown";
1143 1144 1145 1146 1147 1148

	/*
	 * MCEs are always local on AMD. Same is determined by MCG_STATUS_LMCES
	 * on Intel.
	 */
	int lmce = 1;
1149
	int cpu = smp_processor_id();
L
Linus Torvalds 已提交
1150

1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164
	/*
	 * Cases where we avoid rendezvous handler timeout:
	 * 1) If this CPU is offline.
	 *
	 * 2) If crashing_cpu was set, e.g. we're entering kdump and we need to
	 *  skip those CPUs which remain looping in the 1st kernel - see
	 *  crash_nmi_callback().
	 *
	 * Note: there still is a small window between kexec-ing and the new,
	 * kdump kernel establishing a new #MC handler where a broadcasted MCE
	 * might not get handled properly.
	 */
	if (cpu_is_offline(cpu) ||
	    (crashing_cpu != -1 && crashing_cpu != cpu)) {
1165 1166 1167 1168 1169 1170 1171 1172 1173
		u64 mcgstatus;

		mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS);
		if (mcgstatus & MCG_STATUS_RIPV) {
			mce_wrmsrl(MSR_IA32_MCG_STATUS, 0);
			return;
		}
	}

1174
	ist_enter(regs);
1175

1176
	this_cpu_inc(mce_exception_count);
1177

1178
	if (!cfg->banks)
1179
		goto out;
L
Linus Torvalds 已提交
1180

1181
	mce_gather_info(&m, regs);
1182
	m.tsc = rdtsc();
1183

1184
	final = this_cpu_ptr(&mces_seen);
1185 1186
	*final = m;

1187
	memset(valid_banks, 0, sizeof(valid_banks));
1188
	no_way_out = mce_no_way_out(&m, &msg, valid_banks, regs);
1189

L
Linus Torvalds 已提交
1190 1191
	barrier();

1192
	/*
1193 1194 1195
	 * When no restart IP might need to kill or panic.
	 * Assume the worst for now, but if we find the
	 * severity is MCE_AR_SEVERITY we have other options.
1196 1197 1198 1199
	 */
	if (!(m.mcgstatus & MCG_STATUS_RIPV))
		kill_it = 1;

1200
	/*
1201 1202
	 * Check if this MCE is signaled to only this logical processor,
	 * on Intel only.
1203
	 */
1204 1205 1206 1207 1208 1209 1210 1211 1212 1213
	if (m.cpuvendor == X86_VENDOR_INTEL)
		lmce = m.mcgstatus & MCG_STATUS_LMCES;

	/*
	 * Go through all banks in exclusion of the other CPUs. This way we
	 * don't report duplicated events on shared banks because the first one
	 * to see it will clear it. If this is a Local MCE, then no need to
	 * perform rendezvous.
	 */
	if (!lmce)
1214 1215
		order = mce_start(&no_way_out);

1216
	for (i = 0; i < cfg->banks; i++) {
1217
		__clear_bit(i, toclear);
1218 1219
		if (!test_bit(i, valid_banks))
			continue;
1220
		if (!mce_banks[i].ctl)
L
Linus Torvalds 已提交
1221
			continue;
1222 1223

		m.misc = 0;
L
Linus Torvalds 已提交
1224 1225 1226
		m.addr = 0;
		m.bank = i;

1227
		m.status = mce_rdmsrl(msr_ops.status(i));
L
Linus Torvalds 已提交
1228 1229 1230
		if ((m.status & MCI_STATUS_VAL) == 0)
			continue;

1231
		/*
1232 1233
		 * Non uncorrected or non signaled errors are handled by
		 * machine_check_poll. Leave them alone, unless this panics.
1234
		 */
1235
		if (!(m.status & (cfg->ser ? MCI_STATUS_S : MCI_STATUS_UC)) &&
1236
			!no_way_out)
1237 1238 1239 1240 1241
			continue;

		/*
		 * Set taint even when machine check was not enabled.
		 */
1242
		add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE);
1243

1244
		severity = mce_severity(&m, cfg->tolerant, NULL, true);
1245

1246
		/*
1247 1248
		 * When machine check was for corrected/deferred handler don't
		 * touch, unless we're panicing.
1249
		 */
1250 1251
		if ((severity == MCE_KEEP_SEVERITY ||
		     severity == MCE_UCNA_SEVERITY) && !no_way_out)
1252 1253 1254
			continue;
		__set_bit(i, toclear);
		if (severity == MCE_NO_SEVERITY) {
1255 1256 1257 1258 1259
			/*
			 * Machine check event was not enabled. Clear, but
			 * ignore.
			 */
			continue;
L
Linus Torvalds 已提交
1260 1261
		}

1262
		mce_read_aux(&m, i);
L
Linus Torvalds 已提交
1263

1264 1265
		/* assuming valid severity level != 0 */
		m.severity = severity;
1266

1267
		mce_log(&m);
L
Linus Torvalds 已提交
1268

1269 1270 1271
		if (severity > worst) {
			*final = m;
			worst = severity;
L
Linus Torvalds 已提交
1272 1273 1274
		}
	}

1275 1276 1277
	/* mce_clear_state will clear *final, save locally for use later */
	m = *final;

1278 1279 1280
	if (!no_way_out)
		mce_clear_state(toclear);

I
Ingo Molnar 已提交
1281
	/*
1282 1283
	 * Do most of the synchronization with other CPUs.
	 * When there's any problem use only local no_way_out state.
I
Ingo Molnar 已提交
1284
	 */
1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296
	if (!lmce) {
		if (mce_end(order) < 0)
			no_way_out = worst >= MCE_PANIC_SEVERITY;
	} else {
		/*
		 * Local MCE skipped calling mce_reign()
		 * If we found a fatal error, we need to panic here.
		 */
		 if (worst >= MCE_PANIC_SEVERITY && mca_cfg.tolerant < 3)
			mce_panic("Machine check from unknown source",
				NULL, NULL);
	}
1297 1298

	/*
1299 1300
	 * If tolerant is at an insane level we drop requests to kill
	 * processes and continue even when there is no way out.
1301
	 */
1302 1303 1304 1305
	if (cfg->tolerant == 3)
		kill_it = 0;
	else if (no_way_out)
		mce_panic("Fatal machine check on current CPU", &m, msg);
1306

1307 1308
	if (worst > 0)
		mce_report_event(regs);
1309
	mce_wrmsrl(MSR_IA32_MCG_STATUS, 0);
1310
out:
1311
	sync_core();
1312

1313 1314
	if (worst != MCE_AR_SEVERITY && !kill_it)
		goto out_ist;
1315

1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327
	/* Fault was in user mode and we need to take some action */
	if ((m.cs & 3) == 3) {
		ist_begin_non_atomic(regs);
		local_irq_enable();

		if (kill_it || do_memory_failure(&m))
			force_sig(SIGBUS, current);
		local_irq_disable();
		ist_end_non_atomic();
	} else {
		if (!fixup_exception(regs, X86_TRAP_MC))
			mce_panic("Failed kernel mode recovery", &m, NULL);
1328
	}
1329 1330

out_ist:
1331
	ist_exit(regs);
L
Linus Torvalds 已提交
1332
}
1333
EXPORT_SYMBOL_GPL(do_machine_check);
L
Linus Torvalds 已提交
1334

1335
#ifndef CONFIG_MEMORY_FAILURE
1336
int memory_failure(unsigned long pfn, int flags)
1337
{
1338 1339
	/* mce_severity() should not hand us an ACTION_REQUIRED error */
	BUG_ON(flags & MF_ACTION_REQUIRED);
1340 1341 1342
	pr_err("Uncorrected memory error in page 0x%lx ignored\n"
	       "Rebuild kernel with CONFIG_MEMORY_FAILURE=y for smarter handling\n",
	       pfn);
1343 1344

	return 0;
1345
}
1346
#endif
1347

L
Linus Torvalds 已提交
1348
/*
1349 1350 1351
 * Periodic polling timer for "silent" machine check errors.  If the
 * poller finds an MCE, poll 2x faster.  When the poller finds no more
 * errors, poll 2x slower (up to check_interval seconds).
L
Linus Torvalds 已提交
1352
 */
1353
static unsigned long check_interval = INITIAL_CHECK_INTERVAL;
I
Ingo Molnar 已提交
1354

1355
static DEFINE_PER_CPU(unsigned long, mce_next_interval); /* in jiffies */
1356
static DEFINE_PER_CPU(struct timer_list, mce_timer);
L
Linus Torvalds 已提交
1357

C
Chen Gong 已提交
1358 1359 1360 1361 1362
static unsigned long mce_adjust_timer_default(unsigned long interval)
{
	return interval;
}

1363
static unsigned long (*mce_adjust_timer)(unsigned long interval) = mce_adjust_timer_default;
C
Chen Gong 已提交
1364

1365
static void __start_timer(struct timer_list *t, unsigned long interval)
1366
{
1367 1368
	unsigned long when = jiffies + interval;
	unsigned long flags;
1369

1370
	local_irq_save(flags);
1371

1372 1373
	if (!timer_pending(t) || time_before(when, t->expires))
		mod_timer(t, round_jiffies(when));
1374 1375

	local_irq_restore(flags);
1376 1377
}

1378
static void mce_timer_fn(struct timer_list *t)
L
Linus Torvalds 已提交
1379
{
1380
	struct timer_list *cpu_t = this_cpu_ptr(&mce_timer);
1381
	unsigned long iv;
1382

1383
	WARN_ON(cpu_t != t);
1384 1385

	iv = __this_cpu_read(mce_next_interval);
1386

1387
	if (mce_available(this_cpu_ptr(&cpu_info))) {
1388
		machine_check_poll(0, this_cpu_ptr(&mce_poll_banks));
1389 1390 1391 1392 1393

		if (mce_intel_cmci_poll()) {
			iv = mce_adjust_timer(iv);
			goto done;
		}
I
Ingo Molnar 已提交
1394
	}
L
Linus Torvalds 已提交
1395 1396

	/*
1397 1398
	 * Alert userspace if needed. If we logged an MCE, reduce the polling
	 * interval, otherwise increase the polling interval.
L
Linus Torvalds 已提交
1399
	 */
1400
	if (mce_notify_irq())
1401
		iv = max(iv / 2, (unsigned long) HZ/100);
1402
	else
1403
		iv = min(iv * 2, round_jiffies_relative(check_interval * HZ));
1404 1405

done:
1406
	__this_cpu_write(mce_next_interval, iv);
1407
	__start_timer(t, iv);
C
Chen Gong 已提交
1408
}
1409

C
Chen Gong 已提交
1410 1411 1412 1413 1414
/*
 * Ensure that the timer is firing in @interval from now.
 */
void mce_timer_kick(unsigned long interval)
{
1415
	struct timer_list *t = this_cpu_ptr(&mce_timer);
C
Chen Gong 已提交
1416 1417
	unsigned long iv = __this_cpu_read(mce_next_interval);

1418
	__start_timer(t, interval);
1419

C
Chen Gong 已提交
1420 1421
	if (interval < iv)
		__this_cpu_write(mce_next_interval, interval);
1422 1423
}

1424 1425 1426 1427 1428 1429 1430 1431 1432
/* Must not be called in IRQ context where del_timer_sync() can deadlock */
static void mce_timer_delete_all(void)
{
	int cpu;

	for_each_online_cpu(cpu)
		del_timer_sync(&per_cpu(mce_timer, cpu));
}

1433
/*
1434 1435 1436
 * Notify the user(s) about new machine check events.
 * Can be called from interrupt context, but not from machine check/NMI
 * context.
1437
 */
1438
int mce_notify_irq(void)
1439
{
1440 1441 1442
	/* Not more than two messages every minute */
	static DEFINE_RATELIMIT_STATE(ratelimit, 60*HZ, 2);

1443
	if (test_and_clear_bit(0, &mce_need_notify)) {
1444
		mce_work_trigger();
1445

1446
		if (__ratelimit(&ratelimit))
1447
			pr_info(HW_ERR "Machine check events logged\n");
1448 1449

		return 1;
L
Linus Torvalds 已提交
1450
	}
1451 1452
	return 0;
}
1453
EXPORT_SYMBOL_GPL(mce_notify_irq);
1454

1455
static int __mcheck_cpu_mce_banks_init(void)
1456 1457
{
	int i;
1458
	u8 num_banks = mca_cfg.banks;
1459

1460
	mce_banks = kzalloc(num_banks * sizeof(struct mce_bank), GFP_KERNEL);
1461 1462
	if (!mce_banks)
		return -ENOMEM;
1463 1464

	for (i = 0; i < num_banks; i++) {
1465
		struct mce_bank *b = &mce_banks[i];
1466

1467 1468 1469 1470 1471 1472
		b->ctl = -1ULL;
		b->init = 1;
	}
	return 0;
}

1473
/*
L
Linus Torvalds 已提交
1474 1475
 * Initialize Machine Checks for a CPU.
 */
1476
static int __mcheck_cpu_cap_init(void)
L
Linus Torvalds 已提交
1477
{
1478
	unsigned b;
I
Ingo Molnar 已提交
1479
	u64 cap;
L
Linus Torvalds 已提交
1480 1481

	rdmsrl(MSR_IA32_MCG_CAP, cap);
1482 1483

	b = cap & MCG_BANKCNT_MASK;
1484
	if (!mca_cfg.banks)
1485
		pr_info("CPU supports %d MCE banks\n", b);
1486

1487
	if (b > MAX_NR_BANKS) {
1488
		pr_warn("Using only %u machine check banks out of %u\n",
1489 1490 1491 1492 1493
			MAX_NR_BANKS, b);
		b = MAX_NR_BANKS;
	}

	/* Don't support asymmetric configurations today */
1494 1495 1496
	WARN_ON(mca_cfg.banks != 0 && b != mca_cfg.banks);
	mca_cfg.banks = b;

1497
	if (!mce_banks) {
1498
		int err = __mcheck_cpu_mce_banks_init();
1499

1500 1501
		if (err)
			return err;
L
Linus Torvalds 已提交
1502
	}
1503

1504
	/* Use accurate RIP reporting if available. */
1505
	if ((cap & MCG_EXT_P) && MCG_EXT_CNT(cap) >= 9)
1506
		mca_cfg.rip_msr = MSR_IA32_MCG_EIP;
L
Linus Torvalds 已提交
1507

1508
	if (cap & MCG_SER_P)
1509
		mca_cfg.ser = 1;
1510

1511 1512 1513
	return 0;
}

1514
static void __mcheck_cpu_init_generic(void)
1515
{
1516
	enum mcp_flags m_fl = 0;
I
Ingo Molnar 已提交
1517
	mce_banks_t all_banks;
1518 1519
	u64 cap;

1520 1521 1522
	if (!mca_cfg.bootlog)
		m_fl = MCP_DONTLOG;

1523 1524 1525
	/*
	 * Log the machine checks left over from the previous reset.
	 */
1526
	bitmap_fill(all_banks, MAX_NR_BANKS);
1527
	machine_check_poll(MCP_UC | m_fl, &all_banks);
L
Linus Torvalds 已提交
1528

1529
	cr4_set_bits(X86_CR4_MCE);
L
Linus Torvalds 已提交
1530

1531
	rdmsrl(MSR_IA32_MCG_CAP, cap);
L
Linus Torvalds 已提交
1532 1533
	if (cap & MCG_CTL_P)
		wrmsr(MSR_IA32_MCG_CTL, 0xffffffff, 0xffffffff);
1534 1535 1536 1537 1538
}

static void __mcheck_cpu_init_clear_banks(void)
{
	int i;
L
Linus Torvalds 已提交
1539

1540
	for (i = 0; i < mca_cfg.banks; i++) {
1541
		struct mce_bank *b = &mce_banks[i];
1542

1543
		if (!b->init)
1544
			continue;
1545 1546
		wrmsrl(msr_ops.ctl(i), b->ctl);
		wrmsrl(msr_ops.status(i), 0);
1547
	}
L
Linus Torvalds 已提交
1548 1549
}

1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577
/*
 * During IFU recovery Sandy Bridge -EP4S processors set the RIPV and
 * EIPV bits in MCG_STATUS to zero on the affected logical processor (SDM
 * Vol 3B Table 15-20). But this confuses both the code that determines
 * whether the machine check occurred in kernel or user mode, and also
 * the severity assessment code. Pretend that EIPV was set, and take the
 * ip/cs values from the pt_regs that mce_gather_info() ignored earlier.
 */
static void quirk_sandybridge_ifu(int bank, struct mce *m, struct pt_regs *regs)
{
	if (bank != 0)
		return;
	if ((m->mcgstatus & (MCG_STATUS_EIPV|MCG_STATUS_RIPV)) != 0)
		return;
	if ((m->status & (MCI_STATUS_OVER|MCI_STATUS_UC|
		          MCI_STATUS_EN|MCI_STATUS_MISCV|MCI_STATUS_ADDRV|
			  MCI_STATUS_PCC|MCI_STATUS_S|MCI_STATUS_AR|
			  MCACOD)) !=
			 (MCI_STATUS_UC|MCI_STATUS_EN|
			  MCI_STATUS_MISCV|MCI_STATUS_ADDRV|MCI_STATUS_S|
			  MCI_STATUS_AR|MCACOD_INSTR))
		return;

	m->mcgstatus |= MCG_STATUS_EIPV;
	m->ip = regs->ip;
	m->cs = regs->cs;
}

L
Linus Torvalds 已提交
1578
/* Add per CPU specific workarounds here */
1579
static int __mcheck_cpu_apply_quirks(struct cpuinfo_x86 *c)
1580
{
1581 1582
	struct mca_config *cfg = &mca_cfg;

1583
	if (c->x86_vendor == X86_VENDOR_UNKNOWN) {
1584
		pr_info("unknown CPU type - not enabling MCE support\n");
1585 1586 1587
		return -EOPNOTSUPP;
	}

L
Linus Torvalds 已提交
1588
	/* This should be disabled by the BIOS, but isn't always */
1589
	if (c->x86_vendor == X86_VENDOR_AMD) {
1590
		if (c->x86 == 15 && cfg->banks > 4) {
I
Ingo Molnar 已提交
1591 1592 1593 1594 1595
			/*
			 * disable GART TBL walk error reporting, which
			 * trips off incorrectly with the IOMMU & 3ware
			 * & Cerberus:
			 */
1596
			clear_bit(10, (unsigned long *)&mce_banks[4].ctl);
I
Ingo Molnar 已提交
1597
		}
1598
		if (c->x86 < 0x11 && cfg->bootlog < 0) {
I
Ingo Molnar 已提交
1599 1600 1601 1602
			/*
			 * Lots of broken BIOS around that don't clear them
			 * by default and leave crap in there. Don't log:
			 */
1603
			cfg->bootlog = 0;
I
Ingo Molnar 已提交
1604
		}
1605 1606 1607 1608
		/*
		 * Various K7s with broken bank 0 around. Always disable
		 * by default.
		 */
1609
		if (c->x86 == 6 && cfg->banks > 0)
1610
			mce_banks[0].ctl = 0;
1611

1612 1613 1614 1615 1616 1617 1618
		/*
		 * overflow_recov is supported for F15h Models 00h-0fh
		 * even though we don't have a CPUID bit for it.
		 */
		if (c->x86 == 0x15 && c->x86_model <= 0xf)
			mce_flags.overflow_recov = 1;

1619 1620 1621 1622 1623 1624 1625 1626 1627 1628
		/*
		 * Turn off MC4_MISC thresholding banks on those models since
		 * they're not supported there.
		 */
		if (c->x86 == 0x15 &&
		    (c->x86_model >= 0x10 && c->x86_model <= 0x1f)) {
			int i;
			u64 hwcr;
			bool need_toggle;
			u32 msrs[] = {
1629 1630
				0x00000413, /* MC4_MISC0 */
				0xc0000408, /* MC4_MISC1 */
1631
			};
1632

1633
			rdmsrl(MSR_K7_HWCR, hwcr);
1634

1635 1636
			/* McStatusWrEn has to be set */
			need_toggle = !(hwcr & BIT(18));
1637

1638 1639
			if (need_toggle)
				wrmsrl(MSR_K7_HWCR, hwcr | BIT(18));
1640

1641 1642 1643
			/* Clear CntP bit safely */
			for (i = 0; i < ARRAY_SIZE(msrs); i++)
				msr_clear_bit(msrs[i], 62);
1644

1645 1646 1647 1648
			/* restore old settings */
			if (need_toggle)
				wrmsrl(MSR_K7_HWCR, hwcr);
		}
L
Linus Torvalds 已提交
1649
	}
1650

1651 1652 1653 1654 1655 1656 1657 1658 1659 1660
	if (c->x86_vendor == X86_VENDOR_INTEL) {
		/*
		 * SDM documents that on family 6 bank 0 should not be written
		 * because it aliases to another special BIOS controlled
		 * register.
		 * But it's not aliased anymore on model 0x1a+
		 * Don't ignore bank 0 completely because there could be a
		 * valid event later, merely don't write CTL0.
		 */

1661
		if (c->x86 == 6 && c->x86_model < 0x1A && cfg->banks > 0)
1662
			mce_banks[0].init = 0;
1663 1664 1665 1666 1667 1668

		/*
		 * All newer Intel systems support MCE broadcasting. Enable
		 * synchronization with a one second timeout.
		 */
		if ((c->x86 > 6 || (c->x86 == 6 && c->x86_model >= 0xe)) &&
1669 1670
			cfg->monarch_timeout < 0)
			cfg->monarch_timeout = USEC_PER_SEC;
1671

1672 1673 1674 1675
		/*
		 * There are also broken BIOSes on some Pentium M and
		 * earlier systems:
		 */
1676 1677
		if (c->x86 == 6 && c->x86_model <= 13 && cfg->bootlog < 0)
			cfg->bootlog = 0;
1678 1679 1680

		if (c->x86 == 6 && c->x86_model == 45)
			quirk_no_way_out = quirk_sandybridge_ifu;
1681
	}
1682 1683 1684
	if (cfg->monarch_timeout < 0)
		cfg->monarch_timeout = 0;
	if (cfg->bootlog != 0)
1685
		cfg->panic_timeout = 30;
1686 1687

	return 0;
1688
}
L
Linus Torvalds 已提交
1689

1690
static int __mcheck_cpu_ancient_init(struct cpuinfo_x86 *c)
1691 1692
{
	if (c->x86 != 5)
1693 1694
		return 0;

1695 1696
	switch (c->x86_vendor) {
	case X86_VENDOR_INTEL:
1697
		intel_p5_mcheck_init(c);
1698
		return 1;
1699 1700 1701
		break;
	case X86_VENDOR_CENTAUR:
		winchip_mcheck_init(c);
1702
		return 1;
1703
		break;
1704 1705
	default:
		return 0;
1706
	}
1707 1708

	return 0;
1709 1710
}

1711 1712 1713 1714
/*
 * Init basic CPU features needed for early decoding of MCEs.
 */
static void __mcheck_cpu_init_early(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1715
{
1716
	if (c->x86_vendor == X86_VENDOR_AMD) {
1717 1718 1719
		mce_flags.overflow_recov = !!cpu_has(c, X86_FEATURE_OVERFLOW_RECOV);
		mce_flags.succor	 = !!cpu_has(c, X86_FEATURE_SUCCOR);
		mce_flags.smca		 = !!cpu_has(c, X86_FEATURE_SMCA);
1720 1721 1722 1723 1724 1725 1726

		if (mce_flags.smca) {
			msr_ops.ctl	= smca_ctl_reg;
			msr_ops.status	= smca_status_reg;
			msr_ops.addr	= smca_addr_reg;
			msr_ops.misc	= smca_misc_reg;
		}
1727 1728
	}
}
1729

1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744
static void mce_centaur_feature_init(struct cpuinfo_x86 *c)
{
	struct mca_config *cfg = &mca_cfg;

	 /*
	  * All newer Centaur CPUs support MCE broadcasting. Enable
	  * synchronization with a one second timeout.
	  */
	if ((c->x86 == 6 && c->x86_model == 0xf && c->x86_stepping >= 0xe) ||
	     c->x86 > 6) {
		if (cfg->monarch_timeout < 0)
			cfg->monarch_timeout = USEC_PER_SEC;
	}
}

1745 1746 1747 1748 1749 1750 1751
static void __mcheck_cpu_init_vendor(struct cpuinfo_x86 *c)
{
	switch (c->x86_vendor) {
	case X86_VENDOR_INTEL:
		mce_intel_feature_init(c);
		mce_adjust_timer = cmci_intel_adjust_timer;
		break;
1752

1753 1754
	case X86_VENDOR_AMD: {
		mce_amd_feature_init(c);
1755
		break;
1756
		}
1757 1758 1759
	case X86_VENDOR_CENTAUR:
		mce_centaur_feature_init(c);
		break;
1760

L
Linus Torvalds 已提交
1761 1762 1763 1764 1765
	default:
		break;
	}
}

1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776
static void __mcheck_cpu_clear_vendor(struct cpuinfo_x86 *c)
{
	switch (c->x86_vendor) {
	case X86_VENDOR_INTEL:
		mce_intel_feature_clear(c);
		break;
	default:
		break;
	}
}

1777
static void mce_start_timer(struct timer_list *t)
1778
{
1779
	unsigned long iv = check_interval * HZ;
1780

1781
	if (mca_cfg.ignore_ce || !iv)
1782 1783
		return;

1784 1785
	this_cpu_write(mce_next_interval, iv);
	__start_timer(t, iv);
1786 1787
}

1788 1789 1790 1791
static void __mcheck_cpu_setup_timer(void)
{
	struct timer_list *t = this_cpu_ptr(&mce_timer);

1792
	timer_setup(t, mce_timer_fn, TIMER_PINNED);
1793 1794
}

1795 1796
static void __mcheck_cpu_init_timer(void)
{
1797
	struct timer_list *t = this_cpu_ptr(&mce_timer);
1798

1799
	timer_setup(t, mce_timer_fn, TIMER_PINNED);
1800
	mce_start_timer(t);
1801 1802
}

A
Andi Kleen 已提交
1803 1804 1805
/* Handle unconfigured int18 (should never happen) */
static void unexpected_machine_check(struct pt_regs *regs, long error_code)
{
1806
	pr_err("CPU#%d: Unexpected int18 (Machine Check)\n",
A
Andi Kleen 已提交
1807 1808 1809 1810 1811 1812 1813
	       smp_processor_id());
}

/* Call the installed machine check handler for this CPU setup. */
void (*machine_check_vector)(struct pt_regs *, long error_code) =
						unexpected_machine_check;

1814 1815 1816 1817 1818
dotraplinkage void do_mce(struct pt_regs *regs, long error_code)
{
	machine_check_vector(regs, error_code);
}

1819
/*
L
Linus Torvalds 已提交
1820
 * Called for each booted CPU to set up machine checks.
I
Ingo Molnar 已提交
1821
 * Must be called with preempt off:
L
Linus Torvalds 已提交
1822
 */
1823
void mcheck_cpu_init(struct cpuinfo_x86 *c)
L
Linus Torvalds 已提交
1824
{
1825
	if (mca_cfg.disabled)
1826 1827
		return;

1828 1829
	if (__mcheck_cpu_ancient_init(c))
		return;
1830

1831
	if (!mce_available(c))
L
Linus Torvalds 已提交
1832 1833
		return;

1834
	if (__mcheck_cpu_cap_init() < 0 || __mcheck_cpu_apply_quirks(c) < 0) {
1835
		mca_cfg.disabled = 1;
1836 1837 1838
		return;
	}

1839
	if (mce_gen_pool_init()) {
1840
		mca_cfg.disabled = 1;
1841 1842 1843 1844
		pr_emerg("Couldn't allocate MCE records pool!\n");
		return;
	}

1845 1846
	machine_check_vector = do_machine_check;

1847
	__mcheck_cpu_init_early(c);
1848 1849
	__mcheck_cpu_init_generic();
	__mcheck_cpu_init_vendor(c);
1850
	__mcheck_cpu_init_clear_banks();
1851
	__mcheck_cpu_setup_timer();
L
Linus Torvalds 已提交
1852 1853
}

1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870
/*
 * Called for each booted CPU to clear some machine checks opt-ins
 */
void mcheck_cpu_clear(struct cpuinfo_x86 *c)
{
	if (mca_cfg.disabled)
		return;

	if (!mce_available(c))
		return;

	/*
	 * Possibly to clear general settings generic to x86
	 * __mcheck_cpu_clear_generic(c);
	 */
	__mcheck_cpu_clear_vendor(c);

L
Linus Torvalds 已提交
1871 1872
}

1873 1874 1875
static void __mce_disable_bank(void *arg)
{
	int bank = *((int *)arg);
1876
	__clear_bit(bank, this_cpu_ptr(mce_poll_banks));
1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891
	cmci_disable_bank(bank);
}

void mce_disable_bank(int bank)
{
	if (bank >= mca_cfg.banks) {
		pr_warn(FW_BUG
			"Ignoring request to disable invalid MCA bank %d.\n",
			bank);
		return;
	}
	set_bit(bank, mce_banks_ce_disabled);
	on_each_cpu(__mce_disable_bank, &bank, 1);
}

1892
/*
1893 1894
 * mce=off Disables machine check
 * mce=no_cmci Disables CMCI
1895
 * mce=no_lmce Disables LMCE
1896 1897
 * mce=dont_log_ce Clears corrected events silently, no log created for CEs.
 * mce=ignore_ce Disables polling and CMCI, corrected events are not cleared.
1898 1899 1900
 * mce=TOLERANCELEVEL[,monarchtimeout] (number, see above)
 *	monarchtimeout is how long to wait for other CPUs on machine
 *	check, or 0 to not wait
1901 1902
 * mce=bootlog Log MCEs from before booting. Disabled by default on AMD Fam10h
	and older.
1903
 * mce=nobootlog Don't log MCEs from before booting.
1904
 * mce=bios_cmci_threshold Don't program the CMCI threshold
1905
 * mce=recovery force enable memcpy_mcsafe()
1906
 */
L
Linus Torvalds 已提交
1907 1908
static int __init mcheck_enable(char *str)
{
1909 1910
	struct mca_config *cfg = &mca_cfg;

1911
	if (*str == 0) {
1912
		enable_p5_mce();
1913 1914
		return 1;
	}
1915 1916
	if (*str == '=')
		str++;
L
Linus Torvalds 已提交
1917
	if (!strcmp(str, "off"))
1918
		cfg->disabled = 1;
1919
	else if (!strcmp(str, "no_cmci"))
1920
		cfg->cmci_disabled = true;
1921
	else if (!strcmp(str, "no_lmce"))
1922
		cfg->lmce_disabled = 1;
1923
	else if (!strcmp(str, "dont_log_ce"))
1924
		cfg->dont_log_ce = true;
1925
	else if (!strcmp(str, "ignore_ce"))
1926
		cfg->ignore_ce = true;
1927
	else if (!strcmp(str, "bootlog") || !strcmp(str, "nobootlog"))
1928
		cfg->bootlog = (str[0] == 'b');
1929
	else if (!strcmp(str, "bios_cmci_threshold"))
1930
		cfg->bios_cmci_threshold = 1;
1931
	else if (!strcmp(str, "recovery"))
1932
		cfg->recovery = 1;
1933
	else if (isdigit(str[0])) {
1934
		if (get_option(&str, &cfg->tolerant) == 2)
1935
			get_option(&str, &(cfg->monarch_timeout));
1936
	} else {
1937
		pr_info("mce argument %s ignored. Please use /sys\n", str);
1938 1939
		return 0;
	}
1940
	return 1;
L
Linus Torvalds 已提交
1941
}
1942
__setup("mce", mcheck_enable);
L
Linus Torvalds 已提交
1943

1944
int __init mcheck_init(void)
1945
{
1946
	mcheck_intel_therm_init();
1947
	mce_register_decode_chain(&first_nb);
1948
	mce_register_decode_chain(&mce_srao_nb);
1949
	mce_register_decode_chain(&mce_default_nb);
1950
	mcheck_vendor_init_severity();
1951

1952
	INIT_WORK(&mce_work, mce_gen_pool_process);
1953 1954
	init_irq_work(&mce_irq_work, mce_irq_work_cb);

1955 1956 1957
	return 0;
}

1958
/*
1959
 * mce_syscore: PM support
1960
 */
L
Linus Torvalds 已提交
1961

1962 1963 1964 1965
/*
 * Disable machine checks on suspend and shutdown. We can't really handle
 * them later.
 */
1966
static void mce_disable_error_reporting(void)
1967 1968 1969
{
	int i;

1970
	for (i = 0; i < mca_cfg.banks; i++) {
1971
		struct mce_bank *b = &mce_banks[i];
1972

1973
		if (b->init)
1974
			wrmsrl(msr_ops.ctl(i), 0);
1975
	}
1976 1977 1978 1979 1980 1981
	return;
}

static void vendor_disable_error_reporting(void)
{
	/*
1982
	 * Don't clear on Intel or AMD CPUs. Some of these MSRs are socket-wide.
1983 1984 1985 1986
	 * Disabling them for just a single offlined CPU is bad, since it will
	 * inhibit reporting for all shared resources on the socket like the
	 * last level cache (LLC), the integrated memory controller (iMC), etc.
	 */
1987 1988
	if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL ||
	    boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
1989 1990 1991
		return;

	mce_disable_error_reporting();
1992 1993
}

1994
static int mce_syscore_suspend(void)
1995
{
1996 1997
	vendor_disable_error_reporting();
	return 0;
1998 1999
}

2000
static void mce_syscore_shutdown(void)
2001
{
2002
	vendor_disable_error_reporting();
2003 2004
}

I
Ingo Molnar 已提交
2005 2006 2007 2008 2009
/*
 * On resume clear all MCE state. Don't want to see leftovers from the BIOS.
 * Only one CPU is active at this time, the others get re-added later using
 * CPU hotplug:
 */
2010
static void mce_syscore_resume(void)
L
Linus Torvalds 已提交
2011
{
2012
	__mcheck_cpu_init_generic();
2013
	__mcheck_cpu_init_vendor(raw_cpu_ptr(&cpu_info));
2014
	__mcheck_cpu_init_clear_banks();
L
Linus Torvalds 已提交
2015 2016
}

2017
static struct syscore_ops mce_syscore_ops = {
2018 2019 2020
	.suspend	= mce_syscore_suspend,
	.shutdown	= mce_syscore_shutdown,
	.resume		= mce_syscore_resume,
2021 2022
};

2023
/*
2024
 * mce_device: Sysfs support
2025 2026
 */

2027 2028
static void mce_cpu_restart(void *data)
{
2029
	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2030
		return;
2031
	__mcheck_cpu_init_generic();
2032
	__mcheck_cpu_init_clear_banks();
2033
	__mcheck_cpu_init_timer();
2034 2035
}

L
Linus Torvalds 已提交
2036
/* Reinit MCEs after user configuration changes */
2037 2038
static void mce_restart(void)
{
2039
	mce_timer_delete_all();
2040
	on_each_cpu(mce_cpu_restart, NULL, 1);
L
Linus Torvalds 已提交
2041 2042
}

2043
/* Toggle features for corrected errors */
2044
static void mce_disable_cmci(void *data)
2045
{
2046
	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2047 2048 2049 2050 2051 2052
		return;
	cmci_clear();
}

static void mce_enable_ce(void *all)
{
2053
	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2054 2055 2056 2057
		return;
	cmci_reenable();
	cmci_recheck();
	if (all)
2058
		__mcheck_cpu_init_timer();
2059 2060
}

2061
static struct bus_type mce_subsys = {
I
Ingo Molnar 已提交
2062
	.name		= "machinecheck",
2063
	.dev_name	= "machinecheck",
L
Linus Torvalds 已提交
2064 2065
};

2066
DEFINE_PER_CPU(struct device *, mce_device);
I
Ingo Molnar 已提交
2067

2068
static inline struct mce_bank *attr_to_bank(struct device_attribute *attr)
2069 2070 2071
{
	return container_of(attr, struct mce_bank, attr);
}
2072

2073
static ssize_t show_bank(struct device *s, struct device_attribute *attr,
2074 2075
			 char *buf)
{
2076
	return sprintf(buf, "%llx\n", attr_to_bank(attr)->ctl);
2077 2078
}

2079
static ssize_t set_bank(struct device *s, struct device_attribute *attr,
2080
			const char *buf, size_t size)
2081
{
2082
	u64 new;
I
Ingo Molnar 已提交
2083

2084
	if (kstrtou64(buf, 0, &new) < 0)
2085
		return -EINVAL;
I
Ingo Molnar 已提交
2086

2087
	attr_to_bank(attr)->ctl = new;
2088
	mce_restart();
I
Ingo Molnar 已提交
2089

2090
	return size;
2091
}
2092

2093 2094
static ssize_t set_ignore_ce(struct device *s,
			     struct device_attribute *attr,
2095 2096 2097 2098
			     const char *buf, size_t size)
{
	u64 new;

2099
	if (kstrtou64(buf, 0, &new) < 0)
2100 2101
		return -EINVAL;

2102
	mutex_lock(&mce_sysfs_mutex);
2103
	if (mca_cfg.ignore_ce ^ !!new) {
2104 2105
		if (new) {
			/* disable ce features */
2106 2107
			mce_timer_delete_all();
			on_each_cpu(mce_disable_cmci, NULL, 1);
2108
			mca_cfg.ignore_ce = true;
2109 2110
		} else {
			/* enable ce features */
2111
			mca_cfg.ignore_ce = false;
2112 2113 2114
			on_each_cpu(mce_enable_ce, (void *)1, 1);
		}
	}
2115 2116
	mutex_unlock(&mce_sysfs_mutex);

2117 2118 2119
	return size;
}

2120 2121
static ssize_t set_cmci_disabled(struct device *s,
				 struct device_attribute *attr,
2122 2123 2124 2125
				 const char *buf, size_t size)
{
	u64 new;

2126
	if (kstrtou64(buf, 0, &new) < 0)
2127 2128
		return -EINVAL;

2129
	mutex_lock(&mce_sysfs_mutex);
2130
	if (mca_cfg.cmci_disabled ^ !!new) {
2131 2132
		if (new) {
			/* disable cmci */
2133
			on_each_cpu(mce_disable_cmci, NULL, 1);
2134
			mca_cfg.cmci_disabled = true;
2135 2136
		} else {
			/* enable cmci */
2137
			mca_cfg.cmci_disabled = false;
2138 2139 2140
			on_each_cpu(mce_enable_ce, NULL, 1);
		}
	}
2141 2142
	mutex_unlock(&mce_sysfs_mutex);

2143 2144 2145
	return size;
}

2146 2147
static ssize_t store_int_with_restart(struct device *s,
				      struct device_attribute *attr,
2148 2149
				      const char *buf, size_t size)
{
2150 2151 2152 2153 2154 2155 2156 2157 2158 2159
	unsigned long old_check_interval = check_interval;
	ssize_t ret = device_store_ulong(s, attr, buf, size);

	if (check_interval == old_check_interval)
		return ret;

	if (check_interval < 1)
		check_interval = 1;

	mutex_lock(&mce_sysfs_mutex);
2160
	mce_restart();
2161 2162
	mutex_unlock(&mce_sysfs_mutex);

2163 2164 2165
	return ret;
}

2166
static DEVICE_INT_ATTR(tolerant, 0644, mca_cfg.tolerant);
2167
static DEVICE_INT_ATTR(monarch_timeout, 0644, mca_cfg.monarch_timeout);
2168
static DEVICE_BOOL_ATTR(dont_log_ce, 0644, mca_cfg.dont_log_ce);
I
Ingo Molnar 已提交
2169

2170 2171
static struct dev_ext_attribute dev_attr_check_interval = {
	__ATTR(check_interval, 0644, device_show_int, store_int_with_restart),
2172 2173
	&check_interval
};
I
Ingo Molnar 已提交
2174

2175
static struct dev_ext_attribute dev_attr_ignore_ce = {
2176 2177
	__ATTR(ignore_ce, 0644, device_show_bool, set_ignore_ce),
	&mca_cfg.ignore_ce
2178 2179
};

2180
static struct dev_ext_attribute dev_attr_cmci_disabled = {
2181 2182
	__ATTR(cmci_disabled, 0644, device_show_bool, set_cmci_disabled),
	&mca_cfg.cmci_disabled
2183 2184
};

2185 2186 2187
static struct device_attribute *mce_device_attrs[] = {
	&dev_attr_tolerant.attr,
	&dev_attr_check_interval.attr,
2188
#ifdef CONFIG_X86_MCELOG_LEGACY
2189
	&dev_attr_trigger,
2190
#endif
2191 2192 2193 2194
	&dev_attr_monarch_timeout.attr,
	&dev_attr_dont_log_ce.attr,
	&dev_attr_ignore_ce.attr,
	&dev_attr_cmci_disabled.attr,
2195 2196
	NULL
};
L
Linus Torvalds 已提交
2197

2198
static cpumask_var_t mce_device_initialized;
2199

2200 2201 2202 2203 2204
static void mce_device_release(struct device *dev)
{
	kfree(dev);
}

2205
/* Per cpu device init. All of the cpus still share the same ctrl bank: */
2206
static int mce_device_create(unsigned int cpu)
L
Linus Torvalds 已提交
2207
{
2208
	struct device *dev;
L
Linus Torvalds 已提交
2209
	int err;
2210
	int i, j;
2211

2212
	if (!mce_available(&boot_cpu_data))
2213 2214
		return -EIO;

2215 2216 2217 2218
	dev = per_cpu(mce_device, cpu);
	if (dev)
		return 0;

2219 2220 2221
	dev = kzalloc(sizeof *dev, GFP_KERNEL);
	if (!dev)
		return -ENOMEM;
2222 2223
	dev->id  = cpu;
	dev->bus = &mce_subsys;
2224
	dev->release = &mce_device_release;
2225

2226
	err = device_register(dev);
2227 2228
	if (err) {
		put_device(dev);
2229
		return err;
2230
	}
2231

2232 2233
	for (i = 0; mce_device_attrs[i]; i++) {
		err = device_create_file(dev, mce_device_attrs[i]);
2234 2235 2236
		if (err)
			goto error;
	}
2237
	for (j = 0; j < mca_cfg.banks; j++) {
2238
		err = device_create_file(dev, &mce_banks[j].attr);
2239 2240 2241
		if (err)
			goto error2;
	}
2242
	cpumask_set_cpu(cpu, mce_device_initialized);
2243
	per_cpu(mce_device, cpu) = dev;
2244

2245
	return 0;
2246
error2:
2247
	while (--j >= 0)
2248
		device_remove_file(dev, &mce_banks[j].attr);
2249
error:
2250
	while (--i >= 0)
2251
		device_remove_file(dev, mce_device_attrs[i]);
2252

2253
	device_unregister(dev);
2254

2255 2256 2257
	return err;
}

2258
static void mce_device_remove(unsigned int cpu)
2259
{
2260
	struct device *dev = per_cpu(mce_device, cpu);
2261 2262
	int i;

2263
	if (!cpumask_test_cpu(cpu, mce_device_initialized))
2264 2265
		return;

2266 2267
	for (i = 0; mce_device_attrs[i]; i++)
		device_remove_file(dev, mce_device_attrs[i]);
2268

2269
	for (i = 0; i < mca_cfg.banks; i++)
2270
		device_remove_file(dev, &mce_banks[i].attr);
2271

2272 2273
	device_unregister(dev);
	cpumask_clear_cpu(cpu, mce_device_initialized);
2274
	per_cpu(mce_device, cpu) = NULL;
2275 2276
}

2277
/* Make sure there are no machine checks on offlined CPUs. */
2278
static void mce_disable_cpu(void)
2279
{
2280
	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2281
		return;
2282

2283
	if (!cpuhp_tasks_frozen)
2284
		cmci_clear();
2285

2286
	vendor_disable_error_reporting();
2287 2288
}

2289
static void mce_reenable_cpu(void)
2290
{
I
Ingo Molnar 已提交
2291
	int i;
2292

2293
	if (!mce_available(raw_cpu_ptr(&cpu_info)))
2294
		return;
I
Ingo Molnar 已提交
2295

2296
	if (!cpuhp_tasks_frozen)
2297
		cmci_reenable();
2298
	for (i = 0; i < mca_cfg.banks; i++) {
2299
		struct mce_bank *b = &mce_banks[i];
2300

2301
		if (b->init)
2302
			wrmsrl(msr_ops.ctl(i), b->ctl);
2303
	}
2304 2305
}

2306
static int mce_cpu_dead(unsigned int cpu)
2307
{
2308
	mce_intel_hcpu_update(cpu);
2309

2310 2311 2312 2313
	/* intentionally ignoring frozen here */
	if (!cpuhp_tasks_frozen)
		cmci_rediscover();
	return 0;
2314 2315
}

2316
static int mce_cpu_online(unsigned int cpu)
2317
{
2318
	struct timer_list *t = this_cpu_ptr(&mce_timer);
2319
	int ret;
2320

2321
	mce_device_create(cpu);
2322

2323 2324 2325 2326
	ret = mce_threshold_create_device(cpu);
	if (ret) {
		mce_device_remove(cpu);
		return ret;
2327
	}
2328
	mce_reenable_cpu();
2329
	mce_start_timer(t);
2330
	return 0;
2331 2332
}

2333 2334
static int mce_cpu_pre_down(unsigned int cpu)
{
2335
	struct timer_list *t = this_cpu_ptr(&mce_timer);
2336 2337 2338 2339 2340 2341 2342

	mce_disable_cpu();
	del_timer_sync(t);
	mce_threshold_remove_device(cpu);
	mce_device_remove(cpu);
	return 0;
}
2343

2344
static __init void mce_init_banks(void)
2345 2346 2347
{
	int i;

2348
	for (i = 0; i < mca_cfg.banks; i++) {
2349
		struct mce_bank *b = &mce_banks[i];
2350
		struct device_attribute *a = &b->attr;
I
Ingo Molnar 已提交
2351

2352
		sysfs_attr_init(&a->attr);
2353 2354
		a->attr.name	= b->attrname;
		snprintf(b->attrname, ATTR_LEN, "bank%d", i);
I
Ingo Molnar 已提交
2355 2356 2357 2358

		a->attr.mode	= 0644;
		a->show		= show_bank;
		a->store	= set_bank;
2359 2360 2361
	}
}

2362
static __init int mcheck_init_device(void)
2363 2364 2365
{
	int err;

2366 2367 2368 2369 2370 2371
	/*
	 * Check if we have a spare virtual bit. This will only become
	 * a problem if/when we move beyond 5-level page tables.
	 */
	MAYBE_BUILD_BUG_ON(__VIRTUAL_MASK_SHIFT >= 63);

2372 2373 2374 2375
	if (!mce_available(&boot_cpu_data)) {
		err = -EIO;
		goto err_out;
	}
2376

2377 2378 2379 2380
	if (!zalloc_cpumask_var(&mce_device_initialized, GFP_KERNEL)) {
		err = -ENOMEM;
		goto err_out;
	}
2381

2382
	mce_init_banks();
2383

2384
	err = subsys_system_register(&mce_subsys, NULL);
2385
	if (err)
2386
		goto err_out_mem;
2387

2388 2389 2390 2391
	err = cpuhp_setup_state(CPUHP_X86_MCE_DEAD, "x86/mce:dead", NULL,
				mce_cpu_dead);
	if (err)
		goto err_out_mem;
2392

2393 2394 2395
	err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/mce:online",
				mce_cpu_online, mce_cpu_pre_down);
	if (err < 0)
2396
		goto err_out_online;
2397

2398 2399 2400 2401
	register_syscore_ops(&mce_syscore_ops);

	return 0;

2402 2403
err_out_online:
	cpuhp_remove_state(CPUHP_X86_MCE_DEAD);
2404 2405 2406 2407 2408

err_out_mem:
	free_cpumask_var(mce_device_initialized);

err_out:
2409
	pr_err("Unable to init MCE device (rc: %d)\n", err);
I
Ingo Molnar 已提交
2410

L
Linus Torvalds 已提交
2411 2412
	return err;
}
2413
device_initcall_sync(mcheck_init_device);
2414

2415 2416 2417 2418 2419
/*
 * Old style boot options parsing. Only for compatibility.
 */
static int __init mcheck_disable(char *str)
{
2420
	mca_cfg.disabled = 1;
2421 2422 2423
	return 1;
}
__setup("nomce", mcheck_disable);
2424

2425 2426
#ifdef CONFIG_DEBUG_FS
struct dentry *mce_get_debugfs_dir(void)
2427
{
2428
	static struct dentry *dmce;
2429

2430 2431
	if (!dmce)
		dmce = debugfs_create_dir("mce", NULL);
2432

2433 2434
	return dmce;
}
2435

2436 2437 2438
static void mce_reset(void)
{
	cpu_missing = 0;
2439
	atomic_set(&mce_fake_panicked, 0);
2440 2441 2442 2443
	atomic_set(&mce_executing, 0);
	atomic_set(&mce_callin, 0);
	atomic_set(&global_nwo, 0);
}
2444

2445 2446 2447 2448
static int fake_panic_get(void *data, u64 *val)
{
	*val = fake_panic;
	return 0;
2449 2450
}

2451
static int fake_panic_set(void *data, u64 val)
2452
{
2453 2454 2455
	mce_reset();
	fake_panic = val;
	return 0;
2456 2457
}

2458 2459
DEFINE_SIMPLE_ATTRIBUTE(fake_panic_fops, fake_panic_get,
			fake_panic_set, "%llu\n");
2460

2461
static int __init mcheck_debugfs_init(void)
2462
{
2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473
	struct dentry *dmce, *ffake_panic;

	dmce = mce_get_debugfs_dir();
	if (!dmce)
		return -ENOMEM;
	ffake_panic = debugfs_create_file("fake_panic", 0444, dmce, NULL,
					  &fake_panic_fops);
	if (!ffake_panic)
		return -ENOMEM;

	return 0;
2474
}
2475 2476
#else
static int __init mcheck_debugfs_init(void) { return -EINVAL; }
2477
#endif
2478

2479 2480 2481
DEFINE_STATIC_KEY_FALSE(mcsafe_key);
EXPORT_SYMBOL_GPL(mcsafe_key);

2482 2483
static int __init mcheck_late_init(void)
{
2484 2485 2486
	if (mca_cfg.recovery)
		static_branch_inc(&mcsafe_key);

2487
	mcheck_debugfs_init();
2488
	cec_init();
2489 2490 2491 2492 2493 2494 2495 2496 2497 2498

	/*
	 * Flush out everything that has been logged during early boot, now that
	 * everything has been initialized (workqueues, decoders, ...).
	 */
	mce_schedule_work();

	return 0;
}
late_initcall(mcheck_late_init);
新手
引导
客服 返回
顶部