fault.c 25.1 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0-only
2 3 4 5 6 7 8 9
/*
 * Based on arch/arm/mm/fault.c
 *
 * Copyright (C) 1995  Linus Torvalds
 * Copyright (C) 1995-2004 Russell King
 * Copyright (C) 2012 ARM Ltd.
 */

10
#include <linux/acpi.h>
11
#include <linux/bitfield.h>
12
#include <linux/extable.h>
13
#include <linux/kfence.h>
14 15 16 17 18 19 20
#include <linux/signal.h>
#include <linux/mm.h>
#include <linux/hardirq.h>
#include <linux/init.h>
#include <linux/kprobes.h>
#include <linux/uaccess.h>
#include <linux/page-flags.h>
21
#include <linux/sched/signal.h>
22
#include <linux/sched/debug.h>
23 24
#include <linux/highmem.h>
#include <linux/perf_event.h>
25
#include <linux/preempt.h>
26
#include <linux/hugetlb.h>
27

28
#include <asm/acpi.h>
29
#include <asm/bug.h>
30
#include <asm/cmpxchg.h>
31
#include <asm/cpufeature.h>
32
#include <asm/exception.h>
33
#include <asm/daifflags.h>
34
#include <asm/debug-monitors.h>
35
#include <asm/esr.h>
36
#include <asm/kprobes.h>
37
#include <asm/processor.h>
38
#include <asm/sysreg.h>
39 40
#include <asm/system_misc.h>
#include <asm/tlbflush.h>
41
#include <asm/traps.h>
42

43 44 45 46 47 48 49 50 51
struct fault_info {
	int	(*fn)(unsigned long addr, unsigned int esr,
		      struct pt_regs *regs);
	int	sig;
	int	code;
	const char *name;
};

static const struct fault_info fault_info[];
52
static struct fault_info debug_fault_info[];
53 54 55

static inline const struct fault_info *esr_to_fault_info(unsigned int esr)
{
56
	return fault_info + (esr & ESR_ELx_FSC);
57
}
58

59 60 61 62 63
static inline const struct fault_info *esr_to_debug_fault_info(unsigned int esr)
{
	return debug_fault_info + DBG_ESR_EVT(esr);
}

64 65 66 67 68 69 70 71 72 73 74 75 76 77
static void data_abort_decode(unsigned int esr)
{
	pr_alert("Data abort info:\n");

	if (esr & ESR_ELx_ISV) {
		pr_alert("  Access size = %u byte(s)\n",
			 1U << ((esr & ESR_ELx_SAS) >> ESR_ELx_SAS_SHIFT));
		pr_alert("  SSE = %lu, SRT = %lu\n",
			 (esr & ESR_ELx_SSE) >> ESR_ELx_SSE_SHIFT,
			 (esr & ESR_ELx_SRT_MASK) >> ESR_ELx_SRT_SHIFT);
		pr_alert("  SF = %lu, AR = %lu\n",
			 (esr & ESR_ELx_SF) >> ESR_ELx_SF_SHIFT,
			 (esr & ESR_ELx_AR) >> ESR_ELx_AR_SHIFT);
	} else {
78
		pr_alert("  ISV = 0, ISS = 0x%08lx\n", esr & ESR_ELx_ISS_MASK);
79 80 81 82 83 84 85 86 87 88 89
	}

	pr_alert("  CM = %lu, WnR = %lu\n",
		 (esr & ESR_ELx_CM) >> ESR_ELx_CM_SHIFT,
		 (esr & ESR_ELx_WNR) >> ESR_ELx_WNR_SHIFT);
}

static void mem_abort_decode(unsigned int esr)
{
	pr_alert("Mem abort info:\n");

90
	pr_alert("  ESR = 0x%08x\n", esr);
91 92
	pr_alert("  EC = 0x%02lx: %s, IL = %u bits\n",
		 ESR_ELx_EC(esr), esr_get_class_string(esr),
93 94 95 96 97 98 99 100 101 102 103 104
		 (esr & ESR_ELx_IL) ? 32 : 16);
	pr_alert("  SET = %lu, FnV = %lu\n",
		 (esr & ESR_ELx_SET_MASK) >> ESR_ELx_SET_SHIFT,
		 (esr & ESR_ELx_FnV) >> ESR_ELx_FnV_SHIFT);
	pr_alert("  EA = %lu, S1PTW = %lu\n",
		 (esr & ESR_ELx_EA) >> ESR_ELx_EA_SHIFT,
		 (esr & ESR_ELx_S1PTW) >> ESR_ELx_S1PTW_SHIFT);

	if (esr_is_data_abort(esr))
		data_abort_decode(esr);
}

105 106 107 108 109 110 111 112 113
static inline unsigned long mm_to_pgd_phys(struct mm_struct *mm)
{
	/* Either init_pg_dir or swapper_pg_dir */
	if (mm == &init_mm)
		return __pa_symbol(mm->pgd);

	return (unsigned long)virt_to_phys(mm->pgd);
}

114
/*
115
 * Dump out the page tables associated with 'addr' in the currently active mm.
116
 */
117
static void show_pte(unsigned long addr)
118
{
119
	struct mm_struct *mm;
120 121
	pgd_t *pgdp;
	pgd_t pgd;
122

123
	if (is_ttbr0_addr(addr)) {
124 125 126 127 128 129 130
		/* TTBR0 */
		mm = current->active_mm;
		if (mm == &init_mm) {
			pr_alert("[%016lx] user address but active_mm is swapper\n",
				 addr);
			return;
		}
131
	} else if (is_ttbr1_addr(addr)) {
132
		/* TTBR1 */
133
		mm = &init_mm;
134 135 136 137 138
	} else {
		pr_alert("[%016lx] address between user and kernel address ranges\n",
			 addr);
		return;
	}
139

140
	pr_alert("%s pgtable: %luk pages, %llu-bit VAs, pgdp=%016lx\n",
141
		 mm == &init_mm ? "swapper" : "user", PAGE_SIZE / SZ_1K,
142
		 vabits_actual, mm_to_pgd_phys(mm));
143 144 145
	pgdp = pgd_offset(mm, addr);
	pgd = READ_ONCE(*pgdp);
	pr_alert("[%016lx] pgd=%016llx", addr, pgd_val(pgd));
146 147

	do {
148
		p4d_t *p4dp, p4d;
149 150 151
		pud_t *pudp, pud;
		pmd_t *pmdp, pmd;
		pte_t *ptep, pte;
152

153
		if (pgd_none(pgd) || pgd_bad(pgd))
154 155
			break;

156 157 158 159 160 161 162
		p4dp = p4d_offset(pgdp, addr);
		p4d = READ_ONCE(*p4dp);
		pr_cont(", p4d=%016llx", p4d_val(p4d));
		if (p4d_none(p4d) || p4d_bad(p4d))
			break;

		pudp = pud_offset(p4dp, addr);
163 164 165
		pud = READ_ONCE(*pudp);
		pr_cont(", pud=%016llx", pud_val(pud));
		if (pud_none(pud) || pud_bad(pud))
166 167
			break;

168 169 170 171
		pmdp = pmd_offset(pudp, addr);
		pmd = READ_ONCE(*pmdp);
		pr_cont(", pmd=%016llx", pmd_val(pmd));
		if (pmd_none(pmd) || pmd_bad(pmd))
172 173
			break;

174 175 176 177
		ptep = pte_offset_map(pmdp, addr);
		pte = READ_ONCE(*ptep);
		pr_cont(", pte=%016llx", pte_val(pte));
		pte_unmap(ptep);
178 179
	} while(0);

180
	pr_cont("\n");
181 182
}

183 184 185 186 187 188 189 190 191 192 193 194 195 196
/*
 * This function sets the access flags (dirty, accessed), as well as write
 * permission, and only to a more permissive setting.
 *
 * It needs to cope with hardware update of the accessed/dirty state by other
 * agents in the system and can safely skip the __sync_icache_dcache() call as,
 * like set_pte_at(), the PTE is never changed from no-exec to exec here.
 *
 * Returns whether or not the PTE actually changed.
 */
int ptep_set_access_flags(struct vm_area_struct *vma,
			  unsigned long address, pte_t *ptep,
			  pte_t entry, int dirty)
{
197
	pteval_t old_pteval, pteval;
198
	pte_t pte = READ_ONCE(*ptep);
199

200
	if (pte_same(pte, entry))
201 202 203
		return 0;

	/* only preserve the access flags and write permission */
204
	pte_val(entry) &= PTE_RDONLY | PTE_AF | PTE_WRITE | PTE_DIRTY;
205 206 207

	/*
	 * Setting the flags must be done atomically to avoid racing with the
208 209 210
	 * hardware update of the access/dirty state. The PTE_RDONLY bit must
	 * be set to the most permissive (lowest value) of *ptep and entry
	 * (calculated as: a & b == ~(~a | ~b)).
211
	 */
212
	pte_val(entry) ^= PTE_RDONLY;
213
	pteval = pte_val(pte);
214 215 216 217 218 219 220
	do {
		old_pteval = pteval;
		pteval ^= PTE_RDONLY;
		pteval |= pte_val(entry);
		pteval ^= PTE_RDONLY;
		pteval = cmpxchg_relaxed(&pte_val(*ptep), old_pteval, pteval);
	} while (pteval != old_pteval);
221

222 223 224
	/* Invalidate a stale read-only entry */
	if (dirty)
		flush_tlb_page(vma, address);
225 226 227
	return 1;
}

228 229 230 231 232
static bool is_el1_instruction_abort(unsigned int esr)
{
	return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_CUR;
}

233 234
static inline bool is_el1_permission_fault(unsigned long addr, unsigned int esr,
					   struct pt_regs *regs)
235 236 237 238 239 240 241 242 243 244
{
	unsigned int ec       = ESR_ELx_EC(esr);
	unsigned int fsc_type = esr & ESR_ELx_FSC_TYPE;

	if (ec != ESR_ELx_EC_DABT_CUR && ec != ESR_ELx_EC_IABT_CUR)
		return false;

	if (fsc_type == ESR_ELx_FSC_PERM)
		return true;

245
	if (is_ttbr0_addr(addr) && system_uses_ttbr0_pan())
246 247 248 249 250 251
		return fsc_type == ESR_ELx_FSC_FAULT &&
			(regs->pstate & PSR_PAN_BIT);

	return false;
}

252 253 254 255 256 257 258 259 260 261 262 263 264 265
static bool __kprobes is_spurious_el1_translation_fault(unsigned long addr,
							unsigned int esr,
							struct pt_regs *regs)
{
	unsigned long flags;
	u64 par, dfsc;

	if (ESR_ELx_EC(esr) != ESR_ELx_EC_DABT_CUR ||
	    (esr & ESR_ELx_FSC_TYPE) != ESR_ELx_FSC_FAULT)
		return false;

	local_irq_save(flags);
	asm volatile("at s1e1r, %0" :: "r" (addr));
	isb();
266
	par = read_sysreg_par();
267 268
	local_irq_restore(flags);

269 270 271 272
	/*
	 * If we now have a valid translation, treat the translation fault as
	 * spurious.
	 */
273
	if (!(par & SYS_PAR_EL1_F))
274
		return true;
275 276 277 278 279

	/*
	 * If we got a different type of fault from the AT instruction,
	 * treat the translation fault as spurious.
	 */
280
	dfsc = FIELD_GET(SYS_PAR_EL1_FST, par);
281 282 283
	return (dfsc & ESR_ELx_FSC_TYPE) != ESR_ELx_FSC_FAULT;
}

284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299
static void die_kernel_fault(const char *msg, unsigned long addr,
			     unsigned int esr, struct pt_regs *regs)
{
	bust_spinlocks(1);

	pr_alert("Unable to handle kernel %s at virtual address %016lx\n", msg,
		 addr);

	mem_abort_decode(esr);

	show_pte(addr);
	die("Oops", regs, esr);
	bust_spinlocks(0);
	do_exit(SIGKILL);
}

300 301
static void __do_kernel_fault(unsigned long addr, unsigned int esr,
			      struct pt_regs *regs)
302
{
303 304
	const char *msg;

305 306
	/*
	 * Are we prepared to handle this kernel fault?
307
	 * We are almost certainly not prepared to handle instruction faults.
308
	 */
309
	if (!is_el1_instruction_abort(esr) && fixup_exception(regs))
310 311
		return;

312 313 314 315
	if (WARN_RATELIMIT(is_spurious_el1_translation_fault(addr, esr, regs),
	    "Ignoring spurious kernel translation fault at virtual address %016lx\n", addr))
		return;

316
	if (is_el1_permission_fault(addr, esr, regs)) {
317 318
		if (esr & ESR_ELx_WNR)
			msg = "write to read-only memory";
319 320
		else if (is_el1_instruction_abort(esr))
			msg = "execute from non-executable memory";
321 322 323 324 325
		else
			msg = "read from unreadable memory";
	} else if (addr < PAGE_SIZE) {
		msg = "NULL pointer dereference";
	} else {
M
Marco Elver 已提交
326
		if (kfence_handle_page_fault(addr, esr & ESR_ELx_WNR, regs))
327 328
			return;

329 330 331
		msg = "paging request";
	}

332
	die_kernel_fault(msg, addr, esr, regs);
333 334
}

335
static void set_thread_esr(unsigned long address, unsigned int esr)
336
{
337
	current->thread.fault_address = address;
338 339 340 341 342 343 344 345 346 347 348 349 350

	/*
	 * If the faulting address is in the kernel, we must sanitize the ESR.
	 * From userspace's point of view, kernel-only mappings don't exist
	 * at all, so we report them as level 0 translation faults.
	 * (This is not quite the way that "no mapping there at all" behaves:
	 * an alignment fault not caused by the memory type would take
	 * precedence over translation fault for a real access to empty
	 * space. Unfortunately we can't easily distinguish "alignment fault
	 * not caused by memory type" from "alignment fault caused by memory
	 * type", so we ignore this wrinkle and just return the translation
	 * fault.)
	 */
351
	if (!is_ttbr0_addr(current->thread.fault_address)) {
352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388
		switch (ESR_ELx_EC(esr)) {
		case ESR_ELx_EC_DABT_LOW:
			/*
			 * These bits provide only information about the
			 * faulting instruction, which userspace knows already.
			 * We explicitly clear bits which are architecturally
			 * RES0 in case they are given meanings in future.
			 * We always report the ESR as if the fault was taken
			 * to EL1 and so ISV and the bits in ISS[23:14] are
			 * clear. (In fact it always will be a fault to EL1.)
			 */
			esr &= ESR_ELx_EC_MASK | ESR_ELx_IL |
				ESR_ELx_CM | ESR_ELx_WNR;
			esr |= ESR_ELx_FSC_FAULT;
			break;
		case ESR_ELx_EC_IABT_LOW:
			/*
			 * Claim a level 0 translation fault.
			 * All other bits are architecturally RES0 for faults
			 * reported with that DFSC value, so we clear them.
			 */
			esr &= ESR_ELx_EC_MASK | ESR_ELx_IL;
			esr |= ESR_ELx_FSC_FAULT;
			break;
		default:
			/*
			 * This should never happen (entry.S only brings us
			 * into this code for insn and data aborts from a lower
			 * exception level). Fail safe by not providing an ESR
			 * context record at all.
			 */
			WARN(1, "ESR 0x%x is not DABT or IABT from EL0\n", esr);
			esr = 0;
			break;
		}
	}

389
	current->thread.fault_code = esr;
390 391
}

392
static void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *regs)
393 394 395 396 397
{
	/*
	 * If we are in kernel mode at this point, we have no context to
	 * handle this fault with.
	 */
398
	if (user_mode(regs)) {
399
		const struct fault_info *inf = esr_to_fault_info(esr);
400

401
		set_thread_esr(addr, esr);
402 403
		arm64_force_sig_fault(inf->sig, inf->code, (void __user *)addr,
				      inf->name);
404
	} else {
405
		__do_kernel_fault(addr, esr, regs);
406
	}
407 408 409 410 411
}

#define VM_FAULT_BADMAP		0x010000
#define VM_FAULT_BADACCESS	0x020000

412
static vm_fault_t __do_page_fault(struct mm_struct *mm, unsigned long addr,
413 414
				  unsigned int mm_flags, unsigned long vm_flags,
				  struct pt_regs *regs)
415
{
416
	struct vm_area_struct *vma = find_vma(mm, addr);
417 418

	if (unlikely(!vma))
419
		return VM_FAULT_BADMAP;
420 421 422 423 424

	/*
	 * Ok, we have a good vm_area for this memory access, so we can handle
	 * it.
	 */
425 426 427 428 429 430 431
	if (unlikely(vma->vm_start > addr)) {
		if (!(vma->vm_flags & VM_GROWSDOWN))
			return VM_FAULT_BADMAP;
		if (expand_stack(vma, addr))
			return VM_FAULT_BADMAP;
	}

432 433
	/*
	 * Check that the permissions on the VMA allow for the fault which
434
	 * occurred.
435
	 */
436 437
	if (!(vma->vm_flags & vm_flags))
		return VM_FAULT_BADACCESS;
438
	return handle_mm_fault(vma, addr & PAGE_MASK, mm_flags, regs);
439 440
}

M
Mark Rutland 已提交
441 442 443 444 445
static bool is_el0_instruction_abort(unsigned int esr)
{
	return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW;
}

446 447 448 449 450 451 452 453 454
/*
 * Note: not valid for EL1 DC IVAC, but we never use that such that it
 * should fault. EL0 cannot issue DC IVAC (undef).
 */
static bool is_write_abort(unsigned int esr)
{
	return (esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM);
}

455 456 457
static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
				   struct pt_regs *regs)
{
458
	const struct fault_info *inf;
459
	struct mm_struct *mm = current->mm;
460
	vm_fault_t fault;
461
	unsigned long vm_flags;
P
Peter Xu 已提交
462
	unsigned int mm_flags = FAULT_FLAG_DEFAULT;
463

464
	if (kprobe_page_fault(regs, esr))
465 466
		return 0;

467 468 469 470
	/*
	 * If we're in an interrupt or have no user context, we must not take
	 * the fault.
	 */
471
	if (faulthandler_disabled() || !mm)
472 473
		goto no_context;

474 475 476
	if (user_mode(regs))
		mm_flags |= FAULT_FLAG_USER;

477 478 479 480 481 482
	/*
	 * vm_flags tells us what bits we must have in vma->vm_flags
	 * for the fault to be benign, __do_page_fault() would check
	 * vma->vm_flags & vm_flags and returns an error if the
	 * intersection is empty
	 */
M
Mark Rutland 已提交
483
	if (is_el0_instruction_abort(esr)) {
484
		/* It was exec fault */
485
		vm_flags = VM_EXEC;
486
		mm_flags |= FAULT_FLAG_INSTRUCTION;
487
	} else if (is_write_abort(esr)) {
488
		/* It was write fault */
489 490
		vm_flags = VM_WRITE;
		mm_flags |= FAULT_FLAG_WRITE;
491 492 493 494 495 496 497 498
	} else {
		/* It was read fault */
		vm_flags = VM_READ;
		/* Write implies read */
		vm_flags |= VM_WRITE;
		/* If EPAN is absent then exec implies read */
		if (!cpus_have_const_cap(ARM64_HAS_EPAN))
			vm_flags |= VM_EXEC;
499 500
	}

501
	if (is_ttbr0_addr(addr) && is_el1_permission_fault(addr, esr, regs)) {
502 503
		/* regs->orig_addr_limit may be 0 if we entered from EL0 */
		if (regs->orig_addr_limit == KERNEL_DS)
504 505
			die_kernel_fault("access to user memory with fs=KERNEL_DS",
					 addr, esr, regs);
506

507
		if (is_el1_instruction_abort(esr))
508 509
			die_kernel_fault("execution of user memory",
					 addr, esr, regs);
510

511
		if (!search_exception_tables(regs->pc))
512 513
			die_kernel_fault("access to user memory outside uaccess routines",
					 addr, esr, regs);
514
	}
515

516 517
	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);

518 519 520 521 522
	/*
	 * As per x86, we may deadlock here. However, since the kernel only
	 * validly references user space from well defined areas of the code,
	 * we can bug out early if this is from code which shouldn't.
	 */
523
	if (!mmap_read_trylock(mm)) {
524 525 526
		if (!user_mode(regs) && !search_exception_tables(regs->pc))
			goto no_context;
retry:
527
		mmap_read_lock(mm);
528 529 530 531 532 533 534
	} else {
		/*
		 * The above down_read_trylock() might have succeeded in which
		 * case, we'll have missed the might_sleep() from down_read().
		 */
		might_sleep();
#ifdef CONFIG_DEBUG_VM
535
		if (!user_mode(regs) && !search_exception_tables(regs->pc)) {
536
			mmap_read_unlock(mm);
537
			goto no_context;
538
		}
539 540 541
#endif
	}

542
	fault = __do_page_fault(mm, addr, mm_flags, vm_flags, regs);
543

544 545 546 547 548 549
	/* Quick path to respond to signals */
	if (fault_signal_pending(fault, regs)) {
		if (!user_mode(regs))
			goto no_context;
		return 0;
	}
550

551
	if (fault & VM_FAULT_RETRY) {
552 553 554 555 556
		if (mm_flags & FAULT_FLAG_ALLOW_RETRY) {
			mm_flags |= FAULT_FLAG_TRIED;
			goto retry;
		}
	}
557
	mmap_read_unlock(mm);
558 559

	/*
560
	 * Handle the "normal" (no error) case first.
561
	 */
562
	if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP |
563
			      VM_FAULT_BADACCESS))))
564 565
		return 0;

566 567 568 569 570 571 572
	/*
	 * If we are in kernel mode at this point, we have no context to
	 * handle this fault with.
	 */
	if (!user_mode(regs))
		goto no_context;

573 574 575 576 577 578 579 580 581 582
	if (fault & VM_FAULT_OOM) {
		/*
		 * We ran out of memory, call the OOM killer, and return to
		 * userspace (which will retry the fault, or kill us if we got
		 * oom-killed).
		 */
		pagefault_out_of_memory();
		return 0;
	}

583
	inf = esr_to_fault_info(esr);
584
	set_thread_esr(addr, esr);
585 586 587 588 589
	if (fault & VM_FAULT_SIGBUS) {
		/*
		 * We had some memory, but were unable to successfully fix up
		 * this page fault.
		 */
590 591
		arm64_force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)addr,
				      inf->name);
592 593 594 595 596 597
	} else if (fault & (VM_FAULT_HWPOISON_LARGE | VM_FAULT_HWPOISON)) {
		unsigned int lsb;

		lsb = PAGE_SHIFT;
		if (fault & VM_FAULT_HWPOISON_LARGE)
			lsb = hstate_index_to_shift(VM_FAULT_GET_HINDEX(fault));
598

599 600
		arm64_force_sig_mceerr(BUS_MCEERR_AR, (void __user *)addr, lsb,
				       inf->name);
601 602 603 604 605
	} else {
		/*
		 * Something tried to access memory that isn't in our memory
		 * map.
		 */
606 607 608 609
		arm64_force_sig_fault(SIGSEGV,
				      fault == VM_FAULT_BADACCESS ? SEGV_ACCERR : SEGV_MAPERR,
				      (void __user *)addr,
				      inf->name);
610 611 612 613 614
	}

	return 0;

no_context:
615
	__do_kernel_fault(addr, esr, regs);
616 617 618 619 620 621 622
	return 0;
}

static int __kprobes do_translation_fault(unsigned long addr,
					  unsigned int esr,
					  struct pt_regs *regs)
{
623
	if (is_ttbr0_addr(addr))
624 625 626 627 628 629
		return do_page_fault(addr, esr, regs);

	do_bad_area(addr, esr, regs);
	return 0;
}

630 631 632 633 634 635 636
static int do_alignment_fault(unsigned long addr, unsigned int esr,
			      struct pt_regs *regs)
{
	do_bad_area(addr, esr, regs);
	return 0;
}

637 638
static int do_bad(unsigned long addr, unsigned int esr, struct pt_regs *regs)
{
639
	return 1; /* "fault" */
640 641
}

642 643 644
static int do_sea(unsigned long addr, unsigned int esr, struct pt_regs *regs)
{
	const struct fault_info *inf;
645
	void __user *siaddr;
646 647 648

	inf = esr_to_fault_info(esr);

649 650 651 652 653 654 655
	if (user_mode(regs) && apei_claim_sea(regs) == 0) {
		/*
		 * APEI claimed this as a firmware-first notification.
		 * Some processing deferred to task_work before ret_to_user().
		 */
		return 0;
	}
656

657
	if (esr & ESR_ELx_FnV)
658
		siaddr = NULL;
659
	else
660 661
		siaddr  = (void __user *)addr;
	arm64_notify_die(inf->name, regs, inf->sig, inf->code, siaddr, esr);
662

663
	return 0;
664 665
}

666 667 668 669 670 671 672
static int do_tag_check_fault(unsigned long addr, unsigned int esr,
			      struct pt_regs *regs)
{
	do_bad_area(addr, esr, regs);
	return 0;
}

673
static const struct fault_info fault_info[] = {
674 675 676 677
	{ do_bad,		SIGKILL, SI_KERNEL,	"ttbr address size fault"	},
	{ do_bad,		SIGKILL, SI_KERNEL,	"level 1 address size fault"	},
	{ do_bad,		SIGKILL, SI_KERNEL,	"level 2 address size fault"	},
	{ do_bad,		SIGKILL, SI_KERNEL,	"level 3 address size fault"	},
678
	{ do_translation_fault,	SIGSEGV, SEGV_MAPERR,	"level 0 translation fault"	},
679 680
	{ do_translation_fault,	SIGSEGV, SEGV_MAPERR,	"level 1 translation fault"	},
	{ do_translation_fault,	SIGSEGV, SEGV_MAPERR,	"level 2 translation fault"	},
681
	{ do_translation_fault,	SIGSEGV, SEGV_MAPERR,	"level 3 translation fault"	},
682
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 8"			},
S
Steve Capper 已提交
683 684
	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 1 access flag fault"	},
	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 2 access flag fault"	},
685
	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 3 access flag fault"	},
686
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 12"			},
S
Steve Capper 已提交
687 688
	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 1 permission fault"	},
	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 2 permission fault"	},
689
	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 3 permission fault"	},
690
	{ do_sea,		SIGBUS,  BUS_OBJERR,	"synchronous external abort"	},
691
	{ do_tag_check_fault,	SIGSEGV, SEGV_MTESERR,	"synchronous tag check fault"	},
692 693 694 695 696 697 698 699 700 701 702 703 704 705 706
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 18"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 19"			},
	{ do_sea,		SIGKILL, SI_KERNEL,	"level 0 (translation table walk)"	},
	{ do_sea,		SIGKILL, SI_KERNEL,	"level 1 (translation table walk)"	},
	{ do_sea,		SIGKILL, SI_KERNEL,	"level 2 (translation table walk)"	},
	{ do_sea,		SIGKILL, SI_KERNEL,	"level 3 (translation table walk)"	},
	{ do_sea,		SIGBUS,  BUS_OBJERR,	"synchronous parity or ECC error" },	// Reserved when RAS is implemented
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 25"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 26"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 27"			},
	{ do_sea,		SIGKILL, SI_KERNEL,	"level 0 synchronous parity error (translation table walk)"	},	// Reserved when RAS is implemented
	{ do_sea,		SIGKILL, SI_KERNEL,	"level 1 synchronous parity error (translation table walk)"	},	// Reserved when RAS is implemented
	{ do_sea,		SIGKILL, SI_KERNEL,	"level 2 synchronous parity error (translation table walk)"	},	// Reserved when RAS is implemented
	{ do_sea,		SIGKILL, SI_KERNEL,	"level 3 synchronous parity error (translation table walk)"	},	// Reserved when RAS is implemented
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 32"			},
707
	{ do_alignment_fault,	SIGBUS,  BUS_ADRALN,	"alignment fault"		},
708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 34"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 35"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 36"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 37"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 38"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 39"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 40"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 41"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 42"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 43"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 44"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 45"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 46"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 47"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"TLB conflict abort"		},
	{ do_bad,		SIGKILL, SI_KERNEL,	"Unsupported atomic hardware update fault"	},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 50"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 51"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"implementation fault (lockdown abort)" },
	{ do_bad,		SIGBUS,  BUS_OBJERR,	"implementation fault (unsupported exclusive)" },
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 54"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 55"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 56"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 57"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 58" 			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 59"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 60"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"section domain fault"		},
	{ do_bad,		SIGKILL, SI_KERNEL,	"page domain fault"		},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 63"			},
738 739
};

740
void do_mem_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs)
741
{
742
	const struct fault_info *inf = esr_to_fault_info(esr);
743 744 745 746

	if (!inf->fn(addr, esr, regs))
		return;

747 748 749
	if (!user_mode(regs)) {
		pr_alert("Unhandled fault at 0x%016lx\n", addr);
		mem_abort_decode(esr);
750
		show_pte(addr);
751
	}
752

753 754
	arm64_notify_die(inf->name, regs,
			 inf->sig, inf->code, (void __user *)addr, esr);
755
}
756
NOKPROBE_SYMBOL(do_mem_abort);
757

758
void do_el0_irq_bp_hardening(void)
759 760 761 762
{
	/* PC has already been checked in entry.S */
	arm64_apply_bp_hardening();
}
763
NOKPROBE_SYMBOL(do_el0_irq_bp_hardening);
764

765
void do_sp_pc_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs)
766
{
767 768
	arm64_notify_die("SP/PC alignment exception", regs,
			 SIGBUS, BUS_ADRALN, (void __user *)addr, esr);
769
}
770
NOKPROBE_SYMBOL(do_sp_pc_abort);
771

772 773 774 775 776 777 778 779 780
int __init early_brk64(unsigned long addr, unsigned int esr,
		       struct pt_regs *regs);

/*
 * __refdata because early_brk64 is __init, but the reference to it is
 * clobbered at arch_initcall time.
 * See traps.c and debug-monitors.c:debug_traps_init().
 */
static struct fault_info __refdata debug_fault_info[] = {
781 782 783
	{ do_bad,	SIGTRAP,	TRAP_HWBKPT,	"hardware breakpoint"	},
	{ do_bad,	SIGTRAP,	TRAP_HWBKPT,	"hardware single-step"	},
	{ do_bad,	SIGTRAP,	TRAP_HWBKPT,	"hardware watchpoint"	},
784
	{ do_bad,	SIGKILL,	SI_KERNEL,	"unknown 3"		},
785
	{ do_bad,	SIGTRAP,	TRAP_BRKPT,	"aarch32 BKPT"		},
786
	{ do_bad,	SIGKILL,	SI_KERNEL,	"aarch32 vector catch"	},
787
	{ early_brk64,	SIGTRAP,	TRAP_BRKPT,	"aarch64 BRK"		},
788
	{ do_bad,	SIGKILL,	SI_KERNEL,	"unknown 7"		},
789 790 791 792 793 794 795 796 797 798 799 800 801 802
};

void __init hook_debug_fault_code(int nr,
				  int (*fn)(unsigned long, unsigned int, struct pt_regs *),
				  int sig, int code, const char *name)
{
	BUG_ON(nr < 0 || nr >= ARRAY_SIZE(debug_fault_info));

	debug_fault_info[nr].fn		= fn;
	debug_fault_info[nr].sig	= sig;
	debug_fault_info[nr].code	= code;
	debug_fault_info[nr].name	= name;
}

803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824
/*
 * In debug exception context, we explicitly disable preemption despite
 * having interrupts disabled.
 * This serves two purposes: it makes it much less likely that we would
 * accidentally schedule in exception context and it will force a warning
 * if we somehow manage to schedule by accident.
 */
static void debug_exception_enter(struct pt_regs *regs)
{
	preempt_disable();

	/* This code is a bit fragile.  Test it. */
	RCU_LOCKDEP_WARN(!rcu_is_watching(), "exception_enter didn't work");
}
NOKPROBE_SYMBOL(debug_exception_enter);

static void debug_exception_exit(struct pt_regs *regs)
{
	preempt_enable_no_resched();
}
NOKPROBE_SYMBOL(debug_exception_exit);

825 826 827
#ifdef CONFIG_ARM64_ERRATUM_1463225
DECLARE_PER_CPU(int, __in_cortex_a76_erratum_1463225_wa);

828
static int cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)
829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846
{
	if (user_mode(regs))
		return 0;

	if (!__this_cpu_read(__in_cortex_a76_erratum_1463225_wa))
		return 0;

	/*
	 * We've taken a dummy step exception from the kernel to ensure
	 * that interrupts are re-enabled on the syscall path. Return back
	 * to cortex_a76_erratum_1463225_svc_handler() with debug exceptions
	 * masked so that we can safely restore the mdscr and get on with
	 * handling the syscall.
	 */
	regs->pstate |= PSR_D_BIT;
	return 1;
}
#else
847
static int cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs)
848 849 850 851
{
	return 0;
}
#endif /* CONFIG_ARM64_ERRATUM_1463225 */
852
NOKPROBE_SYMBOL(cortex_a76_erratum_1463225_debug_handler);
853

854 855
void do_debug_exception(unsigned long addr_if_watchpoint, unsigned int esr,
			struct pt_regs *regs)
856
{
857
	const struct fault_info *inf = esr_to_debug_fault_info(esr);
858
	unsigned long pc = instruction_pointer(regs);
859

860 861 862
	if (cortex_a76_erratum_1463225_debug_handler(regs))
		return;

863
	debug_exception_enter(regs);
864

865
	if (user_mode(regs) && !is_ttbr0_addr(pc))
866 867
		arm64_apply_bp_hardening();

868
	if (inf->fn(addr_if_watchpoint, esr, regs)) {
869
		arm64_notify_die(inf->name, regs,
870
				 inf->sig, inf->code, (void __user *)pc, esr);
871
	}
872

873
	debug_exception_exit(regs);
874
}
875
NOKPROBE_SYMBOL(do_debug_exception);