fault.c 23.9 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
/*
 * Based on arch/arm/mm/fault.c
 *
 * Copyright (C) 1995  Linus Torvalds
 * Copyright (C) 1995-2004 Russell King
 * Copyright (C) 2012 ARM Ltd.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

21
#include <linux/extable.h>
22 23 24 25 26 27 28
#include <linux/signal.h>
#include <linux/mm.h>
#include <linux/hardirq.h>
#include <linux/init.h>
#include <linux/kprobes.h>
#include <linux/uaccess.h>
#include <linux/page-flags.h>
29
#include <linux/sched/signal.h>
30
#include <linux/sched/debug.h>
31 32
#include <linux/highmem.h>
#include <linux/perf_event.h>
33
#include <linux/preempt.h>
34
#include <linux/hugetlb.h>
35

36
#include <asm/bug.h>
37
#include <asm/cmpxchg.h>
38
#include <asm/cpufeature.h>
39
#include <asm/exception.h>
40
#include <asm/daifflags.h>
41
#include <asm/debug-monitors.h>
42
#include <asm/esr.h>
43
#include <asm/sysreg.h>
44 45 46
#include <asm/system_misc.h>
#include <asm/pgtable.h>
#include <asm/tlbflush.h>
47
#include <asm/traps.h>
48

49 50
#include <acpi/ghes.h>

51 52 53 54 55 56 57 58 59
struct fault_info {
	int	(*fn)(unsigned long addr, unsigned int esr,
		      struct pt_regs *regs);
	int	sig;
	int	code;
	const char *name;
};

static const struct fault_info fault_info[];
60
static struct fault_info debug_fault_info[];
61 62 63

static inline const struct fault_info *esr_to_fault_info(unsigned int esr)
{
64
	return fault_info + (esr & ESR_ELx_FSC);
65
}
66

67 68 69 70 71
static inline const struct fault_info *esr_to_debug_fault_info(unsigned int esr)
{
	return debug_fault_info + DBG_ESR_EVT(esr);
}

72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93
#ifdef CONFIG_KPROBES
static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr)
{
	int ret = 0;

	/* kprobe_running() needs smp_processor_id() */
	if (!user_mode(regs)) {
		preempt_disable();
		if (kprobe_running() && kprobe_fault_handler(regs, esr))
			ret = 1;
		preempt_enable();
	}

	return ret;
}
#else
static inline int notify_page_fault(struct pt_regs *regs, unsigned int esr)
{
	return 0;
}
#endif

94 95 96 97 98 99 100 101 102 103 104 105 106 107
static void data_abort_decode(unsigned int esr)
{
	pr_alert("Data abort info:\n");

	if (esr & ESR_ELx_ISV) {
		pr_alert("  Access size = %u byte(s)\n",
			 1U << ((esr & ESR_ELx_SAS) >> ESR_ELx_SAS_SHIFT));
		pr_alert("  SSE = %lu, SRT = %lu\n",
			 (esr & ESR_ELx_SSE) >> ESR_ELx_SSE_SHIFT,
			 (esr & ESR_ELx_SRT_MASK) >> ESR_ELx_SRT_SHIFT);
		pr_alert("  SF = %lu, AR = %lu\n",
			 (esr & ESR_ELx_SF) >> ESR_ELx_SF_SHIFT,
			 (esr & ESR_ELx_AR) >> ESR_ELx_AR_SHIFT);
	} else {
108
		pr_alert("  ISV = 0, ISS = 0x%08lx\n", esr & ESR_ELx_ISS_MASK);
109 110 111 112 113 114 115 116 117 118 119
	}

	pr_alert("  CM = %lu, WnR = %lu\n",
		 (esr & ESR_ELx_CM) >> ESR_ELx_CM_SHIFT,
		 (esr & ESR_ELx_WNR) >> ESR_ELx_WNR_SHIFT);
}

static void mem_abort_decode(unsigned int esr)
{
	pr_alert("Mem abort info:\n");

120
	pr_alert("  ESR = 0x%08x\n", esr);
121 122 123 124 125 126 127 128 129 130 131 132 133 134
	pr_alert("  Exception class = %s, IL = %u bits\n",
		 esr_get_class_string(esr),
		 (esr & ESR_ELx_IL) ? 32 : 16);
	pr_alert("  SET = %lu, FnV = %lu\n",
		 (esr & ESR_ELx_SET_MASK) >> ESR_ELx_SET_SHIFT,
		 (esr & ESR_ELx_FnV) >> ESR_ELx_FnV_SHIFT);
	pr_alert("  EA = %lu, S1PTW = %lu\n",
		 (esr & ESR_ELx_EA) >> ESR_ELx_EA_SHIFT,
		 (esr & ESR_ELx_S1PTW) >> ESR_ELx_S1PTW_SHIFT);

	if (esr_is_data_abort(esr))
		data_abort_decode(esr);
}

135
/*
136
 * Dump out the page tables associated with 'addr' in the currently active mm.
137
 */
138
void show_pte(unsigned long addr)
139
{
140
	struct mm_struct *mm;
141 142
	pgd_t *pgdp;
	pgd_t pgd;
143

144 145 146 147 148 149 150 151 152 153
	if (addr < TASK_SIZE) {
		/* TTBR0 */
		mm = current->active_mm;
		if (mm == &init_mm) {
			pr_alert("[%016lx] user address but active_mm is swapper\n",
				 addr);
			return;
		}
	} else if (addr >= VA_START) {
		/* TTBR1 */
154
		mm = &init_mm;
155 156 157 158 159
	} else {
		pr_alert("[%016lx] address between user and kernel address ranges\n",
			 addr);
		return;
	}
160

161
	pr_alert("%s pgtable: %luk pages, %u-bit VAs, pgdp = %p\n",
162 163
		 mm == &init_mm ? "swapper" : "user", PAGE_SIZE / SZ_1K,
		 VA_BITS, mm->pgd);
164 165 166
	pgdp = pgd_offset(mm, addr);
	pgd = READ_ONCE(*pgdp);
	pr_alert("[%016lx] pgd=%016llx", addr, pgd_val(pgd));
167 168

	do {
169 170 171
		pud_t *pudp, pud;
		pmd_t *pmdp, pmd;
		pte_t *ptep, pte;
172

173
		if (pgd_none(pgd) || pgd_bad(pgd))
174 175
			break;

176 177 178 179
		pudp = pud_offset(pgdp, addr);
		pud = READ_ONCE(*pudp);
		pr_cont(", pud=%016llx", pud_val(pud));
		if (pud_none(pud) || pud_bad(pud))
180 181
			break;

182 183 184 185
		pmdp = pmd_offset(pudp, addr);
		pmd = READ_ONCE(*pmdp);
		pr_cont(", pmd=%016llx", pmd_val(pmd));
		if (pmd_none(pmd) || pmd_bad(pmd))
186 187
			break;

188 189 190 191
		ptep = pte_offset_map(pmdp, addr);
		pte = READ_ONCE(*ptep);
		pr_cont(", pte=%016llx", pte_val(pte));
		pte_unmap(ptep);
192 193
	} while(0);

194
	pr_cont("\n");
195 196
}

197 198 199 200 201 202 203 204 205 206 207 208 209 210
/*
 * This function sets the access flags (dirty, accessed), as well as write
 * permission, and only to a more permissive setting.
 *
 * It needs to cope with hardware update of the accessed/dirty state by other
 * agents in the system and can safely skip the __sync_icache_dcache() call as,
 * like set_pte_at(), the PTE is never changed from no-exec to exec here.
 *
 * Returns whether or not the PTE actually changed.
 */
int ptep_set_access_flags(struct vm_area_struct *vma,
			  unsigned long address, pte_t *ptep,
			  pte_t entry, int dirty)
{
211
	pteval_t old_pteval, pteval;
212
	pte_t pte = READ_ONCE(*ptep);
213

214
	if (pte_same(pte, entry))
215 216 217
		return 0;

	/* only preserve the access flags and write permission */
218
	pte_val(entry) &= PTE_RDONLY | PTE_AF | PTE_WRITE | PTE_DIRTY;
219 220 221

	/*
	 * Setting the flags must be done atomically to avoid racing with the
222 223 224
	 * hardware update of the access/dirty state. The PTE_RDONLY bit must
	 * be set to the most permissive (lowest value) of *ptep and entry
	 * (calculated as: a & b == ~(~a | ~b)).
225
	 */
226
	pte_val(entry) ^= PTE_RDONLY;
227
	pteval = pte_val(pte);
228 229 230 231 232 233 234
	do {
		old_pteval = pteval;
		pteval ^= PTE_RDONLY;
		pteval |= pte_val(entry);
		pteval ^= PTE_RDONLY;
		pteval = cmpxchg_relaxed(&pte_val(*ptep), old_pteval, pteval);
	} while (pteval != old_pteval);
235 236 237 238 239

	flush_tlb_fix_spurious_fault(vma, address);
	return 1;
}

240 241 242 243 244
static bool is_el1_instruction_abort(unsigned int esr)
{
	return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_CUR;
}

245 246
static inline bool is_el1_permission_fault(unsigned long addr, unsigned int esr,
					   struct pt_regs *regs)
247 248 249 250 251 252 253 254 255 256
{
	unsigned int ec       = ESR_ELx_EC(esr);
	unsigned int fsc_type = esr & ESR_ELx_FSC_TYPE;

	if (ec != ESR_ELx_EC_DABT_CUR && ec != ESR_ELx_EC_IABT_CUR)
		return false;

	if (fsc_type == ESR_ELx_FSC_PERM)
		return true;

257
	if (addr < TASK_SIZE && system_uses_ttbr0_pan())
258 259 260 261 262 263
		return fsc_type == ESR_ELx_FSC_FAULT &&
			(regs->pstate & PSR_PAN_BIT);

	return false;
}

264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279
static void die_kernel_fault(const char *msg, unsigned long addr,
			     unsigned int esr, struct pt_regs *regs)
{
	bust_spinlocks(1);

	pr_alert("Unable to handle kernel %s at virtual address %016lx\n", msg,
		 addr);

	mem_abort_decode(esr);

	show_pte(addr);
	die("Oops", regs, esr);
	bust_spinlocks(0);
	do_exit(SIGKILL);
}

280 281
static void __do_kernel_fault(unsigned long addr, unsigned int esr,
			      struct pt_regs *regs)
282
{
283 284
	const char *msg;

285 286
	/*
	 * Are we prepared to handle this kernel fault?
287
	 * We are almost certainly not prepared to handle instruction faults.
288
	 */
289
	if (!is_el1_instruction_abort(esr) && fixup_exception(regs))
290 291
		return;

292
	if (is_el1_permission_fault(addr, esr, regs)) {
293 294 295 296 297 298 299 300 301 302
		if (esr & ESR_ELx_WNR)
			msg = "write to read-only memory";
		else
			msg = "read from unreadable memory";
	} else if (addr < PAGE_SIZE) {
		msg = "NULL pointer dereference";
	} else {
		msg = "paging request";
	}

303
	die_kernel_fault(msg, addr, esr, regs);
304 305
}

306
static void set_thread_esr(unsigned long address, unsigned int esr)
307
{
308
	current->thread.fault_address = address;
309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359

	/*
	 * If the faulting address is in the kernel, we must sanitize the ESR.
	 * From userspace's point of view, kernel-only mappings don't exist
	 * at all, so we report them as level 0 translation faults.
	 * (This is not quite the way that "no mapping there at all" behaves:
	 * an alignment fault not caused by the memory type would take
	 * precedence over translation fault for a real access to empty
	 * space. Unfortunately we can't easily distinguish "alignment fault
	 * not caused by memory type" from "alignment fault caused by memory
	 * type", so we ignore this wrinkle and just return the translation
	 * fault.)
	 */
	if (current->thread.fault_address >= TASK_SIZE) {
		switch (ESR_ELx_EC(esr)) {
		case ESR_ELx_EC_DABT_LOW:
			/*
			 * These bits provide only information about the
			 * faulting instruction, which userspace knows already.
			 * We explicitly clear bits which are architecturally
			 * RES0 in case they are given meanings in future.
			 * We always report the ESR as if the fault was taken
			 * to EL1 and so ISV and the bits in ISS[23:14] are
			 * clear. (In fact it always will be a fault to EL1.)
			 */
			esr &= ESR_ELx_EC_MASK | ESR_ELx_IL |
				ESR_ELx_CM | ESR_ELx_WNR;
			esr |= ESR_ELx_FSC_FAULT;
			break;
		case ESR_ELx_EC_IABT_LOW:
			/*
			 * Claim a level 0 translation fault.
			 * All other bits are architecturally RES0 for faults
			 * reported with that DFSC value, so we clear them.
			 */
			esr &= ESR_ELx_EC_MASK | ESR_ELx_IL;
			esr |= ESR_ELx_FSC_FAULT;
			break;
		default:
			/*
			 * This should never happen (entry.S only brings us
			 * into this code for insn and data aborts from a lower
			 * exception level). Fail safe by not providing an ESR
			 * context record at all.
			 */
			WARN(1, "ESR 0x%x is not DABT or IABT from EL0\n", esr);
			esr = 0;
			break;
		}
	}

360
	current->thread.fault_code = esr;
361 362
}

363
static void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *regs)
364 365 366 367 368
{
	/*
	 * If we are in kernel mode at this point, we have no context to
	 * handle this fault with.
	 */
369
	if (user_mode(regs)) {
370
		const struct fault_info *inf = esr_to_fault_info(esr);
371

372
		set_thread_esr(addr, esr);
373 374
		arm64_force_sig_fault(inf->sig, inf->code, (void __user *)addr,
				      inf->name);
375
	} else {
376
		__do_kernel_fault(addr, esr, regs);
377
	}
378 379 380 381 382
}

#define VM_FAULT_BADMAP		0x010000
#define VM_FAULT_BADACCESS	0x020000

383
static vm_fault_t __do_page_fault(struct mm_struct *mm, unsigned long addr,
384
			   unsigned int mm_flags, unsigned long vm_flags,
385 386 387
			   struct task_struct *tsk)
{
	struct vm_area_struct *vma;
388
	vm_fault_t fault;
389 390 391 392 393 394 395 396 397 398 399 400 401

	vma = find_vma(mm, addr);
	fault = VM_FAULT_BADMAP;
	if (unlikely(!vma))
		goto out;
	if (unlikely(vma->vm_start > addr))
		goto check_stack;

	/*
	 * Ok, we have a good vm_area for this memory access, so we can handle
	 * it.
	 */
good_area:
402 403
	/*
	 * Check that the permissions on the VMA allow for the fault which
404
	 * occurred.
405 406
	 */
	if (!(vma->vm_flags & vm_flags)) {
407 408 409 410
		fault = VM_FAULT_BADACCESS;
		goto out;
	}

411
	return handle_mm_fault(vma, addr & PAGE_MASK, mm_flags);
412 413 414 415 416 417 418 419

check_stack:
	if (vma->vm_flags & VM_GROWSDOWN && !expand_stack(vma, addr))
		goto good_area;
out:
	return fault;
}

M
Mark Rutland 已提交
420 421 422 423 424
static bool is_el0_instruction_abort(unsigned int esr)
{
	return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_LOW;
}

425 426 427
static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
				   struct pt_regs *regs)
{
428
	const struct fault_info *inf;
429 430
	struct task_struct *tsk;
	struct mm_struct *mm;
431
	vm_fault_t fault, major = 0;
432
	unsigned long vm_flags = VM_READ | VM_WRITE;
433 434
	unsigned int mm_flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;

435 436 437
	if (notify_page_fault(regs, esr))
		return 0;

438 439 440 441 442 443 444
	tsk = current;
	mm  = tsk->mm;

	/*
	 * If we're in an interrupt or have no user context, we must not take
	 * the fault.
	 */
445
	if (faulthandler_disabled() || !mm)
446 447
		goto no_context;

448 449 450
	if (user_mode(regs))
		mm_flags |= FAULT_FLAG_USER;

M
Mark Rutland 已提交
451
	if (is_el0_instruction_abort(esr)) {
452
		vm_flags = VM_EXEC;
M
Mark Rutland 已提交
453
	} else if ((esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM)) {
454 455 456 457
		vm_flags = VM_WRITE;
		mm_flags |= FAULT_FLAG_WRITE;
	}

458
	if (addr < TASK_SIZE && is_el1_permission_fault(addr, esr, regs)) {
459 460
		/* regs->orig_addr_limit may be 0 if we entered from EL0 */
		if (regs->orig_addr_limit == KERNEL_DS)
461 462
			die_kernel_fault("access to user memory with fs=KERNEL_DS",
					 addr, esr, regs);
463

464
		if (is_el1_instruction_abort(esr))
465 466
			die_kernel_fault("execution of user memory",
					 addr, esr, regs);
467

468
		if (!search_exception_tables(regs->pc))
469 470
			die_kernel_fault("access to user memory outside uaccess routines",
					 addr, esr, regs);
471
	}
472

473 474
	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);

475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496
	/*
	 * As per x86, we may deadlock here. However, since the kernel only
	 * validly references user space from well defined areas of the code,
	 * we can bug out early if this is from code which shouldn't.
	 */
	if (!down_read_trylock(&mm->mmap_sem)) {
		if (!user_mode(regs) && !search_exception_tables(regs->pc))
			goto no_context;
retry:
		down_read(&mm->mmap_sem);
	} else {
		/*
		 * The above down_read_trylock() might have succeeded in which
		 * case, we'll have missed the might_sleep() from down_read().
		 */
		might_sleep();
#ifdef CONFIG_DEBUG_VM
		if (!user_mode(regs) && !search_exception_tables(regs->pc))
			goto no_context;
#endif
	}

497
	fault = __do_page_fault(mm, addr, mm_flags, vm_flags, tsk);
498
	major |= fault & VM_FAULT_MAJOR;
499

500 501 502 503 504 505 506
	if (fault & VM_FAULT_RETRY) {
		/*
		 * If we need to retry but a fatal signal is pending,
		 * handle the signal first. We do not need to release
		 * the mmap_sem because it would already be released
		 * in __lock_page_or_retry in mm/filemap.c.
		 */
507 508 509
		if (fatal_signal_pending(current)) {
			if (!user_mode(regs))
				goto no_context;
510
			return 0;
511
		}
512 513 514 515 516 517 518 519 520 521 522 523

		/*
		 * Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk of
		 * starvation.
		 */
		if (mm_flags & FAULT_FLAG_ALLOW_RETRY) {
			mm_flags &= ~FAULT_FLAG_ALLOW_RETRY;
			mm_flags |= FAULT_FLAG_TRIED;
			goto retry;
		}
	}
	up_read(&mm->mmap_sem);
524 525

	/*
526
	 * Handle the "normal" (no error) case first.
527
	 */
528 529 530 531 532 533 534 535 536
	if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP |
			      VM_FAULT_BADACCESS)))) {
		/*
		 * Major/minor page fault accounting is only done
		 * once. If we go through a retry, it is extremely
		 * likely that the page will be found in page cache at
		 * that point.
		 */
		if (major) {
537 538 539 540 541 542 543 544 545 546
			tsk->maj_flt++;
			perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, regs,
				      addr);
		} else {
			tsk->min_flt++;
			perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, regs,
				      addr);
		}

		return 0;
547
	}
548

549 550 551 552 553 554 555
	/*
	 * If we are in kernel mode at this point, we have no context to
	 * handle this fault with.
	 */
	if (!user_mode(regs))
		goto no_context;

556 557 558 559 560 561 562 563 564 565
	if (fault & VM_FAULT_OOM) {
		/*
		 * We ran out of memory, call the OOM killer, and return to
		 * userspace (which will retry the fault, or kill us if we got
		 * oom-killed).
		 */
		pagefault_out_of_memory();
		return 0;
	}

566
	inf = esr_to_fault_info(esr);
567
	set_thread_esr(addr, esr);
568 569 570 571 572
	if (fault & VM_FAULT_SIGBUS) {
		/*
		 * We had some memory, but were unable to successfully fix up
		 * this page fault.
		 */
573 574
		arm64_force_sig_fault(SIGBUS, BUS_ADRERR, (void __user *)addr,
				      inf->name);
575 576 577 578 579 580
	} else if (fault & (VM_FAULT_HWPOISON_LARGE | VM_FAULT_HWPOISON)) {
		unsigned int lsb;

		lsb = PAGE_SHIFT;
		if (fault & VM_FAULT_HWPOISON_LARGE)
			lsb = hstate_index_to_shift(VM_FAULT_GET_HINDEX(fault));
581

582 583
		arm64_force_sig_mceerr(BUS_MCEERR_AR, (void __user *)addr, lsb,
				       inf->name);
584 585 586 587 588
	} else {
		/*
		 * Something tried to access memory that isn't in our memory
		 * map.
		 */
589 590 591 592
		arm64_force_sig_fault(SIGSEGV,
				      fault == VM_FAULT_BADACCESS ? SEGV_ACCERR : SEGV_MAPERR,
				      (void __user *)addr,
				      inf->name);
593 594 595 596 597
	}

	return 0;

no_context:
598
	__do_kernel_fault(addr, esr, regs);
599 600 601 602 603 604 605 606 607 608 609 610 611 612
	return 0;
}

static int __kprobes do_translation_fault(unsigned long addr,
					  unsigned int esr,
					  struct pt_regs *regs)
{
	if (addr < TASK_SIZE)
		return do_page_fault(addr, esr, regs);

	do_bad_area(addr, esr, regs);
	return 0;
}

613 614 615 616 617 618 619
static int do_alignment_fault(unsigned long addr, unsigned int esr,
			      struct pt_regs *regs)
{
	do_bad_area(addr, esr, regs);
	return 0;
}

620 621
static int do_bad(unsigned long addr, unsigned int esr, struct pt_regs *regs)
{
622
	return 1; /* "fault" */
623 624
}

625 626 627
static int do_sea(unsigned long addr, unsigned int esr, struct pt_regs *regs)
{
	const struct fault_info *inf;
628
	void __user *siaddr;
629 630 631

	inf = esr_to_fault_info(esr);

632 633 634 635 636 637 638 639 640
	/*
	 * Synchronous aborts may interrupt code which had interrupts masked.
	 * Before calling out into the wider kernel tell the interested
	 * subsystems.
	 */
	if (IS_ENABLED(CONFIG_ACPI_APEI_SEA)) {
		if (interrupts_enabled(regs))
			nmi_enter();

641
		ghes_notify_sea();
642 643 644 645 646

		if (interrupts_enabled(regs))
			nmi_exit();
	}

647
	if (esr & ESR_ELx_FnV)
648
		siaddr = NULL;
649
	else
650 651
		siaddr  = (void __user *)addr;
	arm64_notify_die(inf->name, regs, inf->sig, inf->code, siaddr, esr);
652

653
	return 0;
654 655
}

656
static const struct fault_info fault_info[] = {
657 658 659 660
	{ do_bad,		SIGKILL, SI_KERNEL,	"ttbr address size fault"	},
	{ do_bad,		SIGKILL, SI_KERNEL,	"level 1 address size fault"	},
	{ do_bad,		SIGKILL, SI_KERNEL,	"level 2 address size fault"	},
	{ do_bad,		SIGKILL, SI_KERNEL,	"level 3 address size fault"	},
661
	{ do_translation_fault,	SIGSEGV, SEGV_MAPERR,	"level 0 translation fault"	},
662 663
	{ do_translation_fault,	SIGSEGV, SEGV_MAPERR,	"level 1 translation fault"	},
	{ do_translation_fault,	SIGSEGV, SEGV_MAPERR,	"level 2 translation fault"	},
664
	{ do_translation_fault,	SIGSEGV, SEGV_MAPERR,	"level 3 translation fault"	},
665
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 8"			},
S
Steve Capper 已提交
666 667
	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 1 access flag fault"	},
	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 2 access flag fault"	},
668
	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 3 access flag fault"	},
669
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 12"			},
S
Steve Capper 已提交
670 671
	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 1 permission fault"	},
	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 2 permission fault"	},
672
	{ do_page_fault,	SIGSEGV, SEGV_ACCERR,	"level 3 permission fault"	},
673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689
	{ do_sea,		SIGBUS,  BUS_OBJERR,	"synchronous external abort"	},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 17"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 18"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 19"			},
	{ do_sea,		SIGKILL, SI_KERNEL,	"level 0 (translation table walk)"	},
	{ do_sea,		SIGKILL, SI_KERNEL,	"level 1 (translation table walk)"	},
	{ do_sea,		SIGKILL, SI_KERNEL,	"level 2 (translation table walk)"	},
	{ do_sea,		SIGKILL, SI_KERNEL,	"level 3 (translation table walk)"	},
	{ do_sea,		SIGBUS,  BUS_OBJERR,	"synchronous parity or ECC error" },	// Reserved when RAS is implemented
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 25"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 26"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 27"			},
	{ do_sea,		SIGKILL, SI_KERNEL,	"level 0 synchronous parity error (translation table walk)"	},	// Reserved when RAS is implemented
	{ do_sea,		SIGKILL, SI_KERNEL,	"level 1 synchronous parity error (translation table walk)"	},	// Reserved when RAS is implemented
	{ do_sea,		SIGKILL, SI_KERNEL,	"level 2 synchronous parity error (translation table walk)"	},	// Reserved when RAS is implemented
	{ do_sea,		SIGKILL, SI_KERNEL,	"level 3 synchronous parity error (translation table walk)"	},	// Reserved when RAS is implemented
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 32"			},
690
	{ do_alignment_fault,	SIGBUS,  BUS_ADRALN,	"alignment fault"		},
691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 34"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 35"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 36"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 37"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 38"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 39"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 40"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 41"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 42"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 43"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 44"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 45"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 46"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 47"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"TLB conflict abort"		},
	{ do_bad,		SIGKILL, SI_KERNEL,	"Unsupported atomic hardware update fault"	},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 50"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 51"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"implementation fault (lockdown abort)" },
	{ do_bad,		SIGBUS,  BUS_OBJERR,	"implementation fault (unsupported exclusive)" },
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 54"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 55"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 56"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 57"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 58" 			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 59"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 60"			},
	{ do_bad,		SIGKILL, SI_KERNEL,	"section domain fault"		},
	{ do_bad,		SIGKILL, SI_KERNEL,	"page domain fault"		},
	{ do_bad,		SIGKILL, SI_KERNEL,	"unknown 63"			},
721 722
};

723 724
int handle_guest_sea(phys_addr_t addr, unsigned int esr)
{
725
	return ghes_notify_sea();
726 727
}

728 729 730
asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr,
					 struct pt_regs *regs)
{
731
	const struct fault_info *inf = esr_to_fault_info(esr);
732 733 734 735

	if (!inf->fn(addr, esr, regs))
		return;

736 737 738
	if (!user_mode(regs)) {
		pr_alert("Unhandled fault at 0x%016lx\n", addr);
		mem_abort_decode(esr);
739
		show_pte(addr);
740
	}
741

742 743
	arm64_notify_die(inf->name, regs,
			 inf->sig, inf->code, (void __user *)addr, esr);
744 745
}

746 747 748 749 750 751
asmlinkage void __exception do_el0_irq_bp_hardening(void)
{
	/* PC has already been checked in entry.S */
	arm64_apply_bp_hardening();
}

752 753 754 755 756 757 758 759 760 761 762 763
asmlinkage void __exception do_el0_ia_bp_hardening(unsigned long addr,
						   unsigned int esr,
						   struct pt_regs *regs)
{
	/*
	 * We've taken an instruction abort from userspace and not yet
	 * re-enabled IRQs. If the address is a kernel address, apply
	 * BP hardening prior to enabling IRQs and pre-emption.
	 */
	if (addr > TASK_SIZE)
		arm64_apply_bp_hardening();

764
	local_daif_restore(DAIF_PROCCTX);
765 766 767 768
	do_mem_abort(addr, esr, regs);
}


769 770 771 772
asmlinkage void __exception do_sp_pc_abort(unsigned long addr,
					   unsigned int esr,
					   struct pt_regs *regs)
{
773 774 775
	if (user_mode(regs)) {
		if (instruction_pointer(regs) > TASK_SIZE)
			arm64_apply_bp_hardening();
776
		local_daif_restore(DAIF_PROCCTX);
777 778
	}

779 780
	arm64_notify_die("SP/PC alignment exception", regs,
			 SIGBUS, BUS_ADRALN, (void __user *)addr, esr);
781 782
}

783 784 785 786 787 788 789 790 791
int __init early_brk64(unsigned long addr, unsigned int esr,
		       struct pt_regs *regs);

/*
 * __refdata because early_brk64 is __init, but the reference to it is
 * clobbered at arch_initcall time.
 * See traps.c and debug-monitors.c:debug_traps_init().
 */
static struct fault_info __refdata debug_fault_info[] = {
792 793 794
	{ do_bad,	SIGTRAP,	TRAP_HWBKPT,	"hardware breakpoint"	},
	{ do_bad,	SIGTRAP,	TRAP_HWBKPT,	"hardware single-step"	},
	{ do_bad,	SIGTRAP,	TRAP_HWBKPT,	"hardware watchpoint"	},
795
	{ do_bad,	SIGKILL,	SI_KERNEL,	"unknown 3"		},
796
	{ do_bad,	SIGTRAP,	TRAP_BRKPT,	"aarch32 BKPT"		},
797
	{ do_bad,	SIGKILL,	SI_KERNEL,	"aarch32 vector catch"	},
798
	{ early_brk64,	SIGTRAP,	TRAP_BRKPT,	"aarch64 BRK"		},
799
	{ do_bad,	SIGKILL,	SI_KERNEL,	"unknown 7"		},
800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817
};

void __init hook_debug_fault_code(int nr,
				  int (*fn)(unsigned long, unsigned int, struct pt_regs *),
				  int sig, int code, const char *name)
{
	BUG_ON(nr < 0 || nr >= ARRAY_SIZE(debug_fault_info));

	debug_fault_info[nr].fn		= fn;
	debug_fault_info[nr].sig	= sig;
	debug_fault_info[nr].code	= code;
	debug_fault_info[nr].name	= name;
}

asmlinkage int __exception do_debug_exception(unsigned long addr,
					      unsigned int esr,
					      struct pt_regs *regs)
{
818
	const struct fault_info *inf = esr_to_debug_fault_info(esr);
819
	int rv;
820

821 822 823 824 825 826
	/*
	 * Tell lockdep we disabled irqs in entry.S. Do nothing if they were
	 * already disabled to preserve the last enabled/disabled addresses.
	 */
	if (interrupts_enabled(regs))
		trace_hardirqs_off();
827

828 829 830
	if (user_mode(regs) && instruction_pointer(regs) > TASK_SIZE)
		arm64_apply_bp_hardening();

831 832 833
	if (!inf->fn(addr, esr, regs)) {
		rv = 1;
	} else {
834 835
		arm64_notify_die(inf->name, regs,
				 inf->sig, inf->code, (void __user *)addr, esr);
836 837
		rv = 0;
	}
838

839 840
	if (interrupts_enabled(regs))
		trace_hardirqs_on();
841

842
	return rv;
843
}
844
NOKPROBE_SYMBOL(do_debug_exception);