fault.c 6.7 KB
Newer Older
1 2 3 4
// SPDX-License-Identifier: GPL-2.0
// Copyright (C) 2018 Hangzhou C-SKY Microsystems co.,ltd.

#include <linux/extable.h>
G
Guo Ren 已提交
5
#include <linux/kprobes.h>
6 7
#include <linux/mmu_context.h>
#include <linux/perf_event.h>
8 9 10 11 12 13 14 15 16 17 18 19 20 21 22

int fixup_exception(struct pt_regs *regs)
{
	const struct exception_table_entry *fixup;

	fixup = search_exception_tables(instruction_pointer(regs));
	if (fixup) {
		regs->pc = fixup->nextinsn;

		return 1;
	}

	return 0;
}

23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53
static inline bool is_write(struct pt_regs *regs)
{
	switch (trap_no(regs)) {
	case VEC_TLBINVALIDS:
		return true;
	case VEC_TLBMODIFIED:
		return true;
	}

	return false;
}

#ifdef CONFIG_CPU_HAS_LDSTEX
static inline void csky_cmpxchg_fixup(struct pt_regs *regs)
{
	return;
}
#else
extern unsigned long csky_cmpxchg_ldw;
extern unsigned long csky_cmpxchg_stw;
static inline void csky_cmpxchg_fixup(struct pt_regs *regs)
{
	if (trap_no(regs) != VEC_TLBMODIFIED)
		return;

	if (instruction_pointer(regs) == csky_cmpxchg_stw)
		instruction_pointer_set(regs, csky_cmpxchg_ldw);
	return;
}
#endif

54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183
static inline void no_context(struct pt_regs *regs, unsigned long addr)
{
	current->thread.trap_no = trap_no(regs);

	/* Are we prepared to handle this kernel fault? */
	if (fixup_exception(regs))
		return;

	/*
	 * Oops. The kernel tried to access some bad page. We'll have to
	 * terminate things with extreme prejudice.
	 */
	bust_spinlocks(1);
	pr_alert("Unable to handle kernel paging request at virtual "
		 "addr 0x%08lx, pc: 0x%08lx\n", addr, regs->pc);
	die(regs, "Oops");
	do_exit(SIGKILL);
}

static inline void mm_fault_error(struct pt_regs *regs, unsigned long addr, vm_fault_t fault)
{
	current->thread.trap_no = trap_no(regs);

	if (fault & VM_FAULT_OOM) {
		/*
		 * We ran out of memory, call the OOM killer, and return the userspace
		 * (which will retry the fault, or kill us if we got oom-killed).
		 */
		if (!user_mode(regs)) {
			no_context(regs, addr);
			return;
		}
		pagefault_out_of_memory();
		return;
	} else if (fault & VM_FAULT_SIGBUS) {
		/* Kernel mode? Handle exceptions or die */
		if (!user_mode(regs)) {
			no_context(regs, addr);
			return;
		}
		do_trap(regs, SIGBUS, BUS_ADRERR, addr);
		return;
	}
	BUG();
}

static inline void bad_area(struct pt_regs *regs, struct mm_struct *mm, int code, unsigned long addr)
{
	/*
	 * Something tried to access memory that isn't in our memory map.
	 * Fix it, but check if it's kernel or user first.
	 */
	mmap_read_unlock(mm);
	/* User mode accesses just cause a SIGSEGV */
	if (user_mode(regs)) {
		do_trap(regs, SIGSEGV, code, addr);
		return;
	}

	no_context(regs, addr);
}

static inline void vmalloc_fault(struct pt_regs *regs, int code, unsigned long addr)
{
	pgd_t *pgd, *pgd_k;
	pud_t *pud, *pud_k;
	pmd_t *pmd, *pmd_k;
	pte_t *pte_k;
	int offset;

	/* User mode accesses just cause a SIGSEGV */
	if (user_mode(regs)) {
		do_trap(regs, SIGSEGV, code, addr);
		return;
	}

	/*
	 * Synchronize this task's top level page-table
	 * with the 'reference' page table.
	 *
	 * Do _not_ use "tsk" here. We might be inside
	 * an interrupt in the middle of a task switch..
	 */
	offset = pgd_index(addr);

	pgd = get_pgd() + offset;
	pgd_k = init_mm.pgd + offset;

	if (!pgd_present(*pgd_k)) {
		no_context(regs, addr);
		return;
	}
	set_pgd(pgd, *pgd_k);

	pud = (pud_t *)pgd;
	pud_k = (pud_t *)pgd_k;
	if (!pud_present(*pud_k)) {
		no_context(regs, addr);
		return;
	}

	pmd = pmd_offset(pud, addr);
	pmd_k = pmd_offset(pud_k, addr);
	if (!pmd_present(*pmd_k)) {
		no_context(regs, addr);
		return;
	}
	set_pmd(pmd, *pmd_k);

	pte_k = pte_offset_kernel(pmd_k, addr);
	if (!pte_present(*pte_k)) {
		no_context(regs, addr);
		return;
	}

	flush_tlb_one(addr);
}

static inline bool access_error(struct pt_regs *regs, struct vm_area_struct *vma)
{
	if (is_write(regs)) {
		if (!(vma->vm_flags & VM_WRITE))
			return true;
	} else {
		if (unlikely(!vma_is_accessible(vma)))
			return true;
	}
	return false;
}

184
/*
185 186
 * This routine handles page faults.  It determines the address and the
 * problem, and then passes it off to one of the appropriate routines.
187
 */
188
asmlinkage void do_page_fault(struct pt_regs *regs)
189
{
190 191 192 193
	struct task_struct *tsk;
	struct vm_area_struct *vma;
	struct mm_struct *mm;
	unsigned long addr = read_mmu_entryhi() & PAGE_MASK;
194
	unsigned int flags = FAULT_FLAG_DEFAULT;
195 196 197 198 199
	int code = SEGV_MAPERR;
	vm_fault_t fault;

	tsk = current;
	mm = tsk->mm;
200 201

	csky_cmpxchg_fixup(regs);
202

G
Guo Ren 已提交
203 204 205
	if (kprobe_page_fault(regs, tsk->thread.trap_no))
		return;

206
	/*
207 208
	 * Fault-in kernel-space virtual memory on-demand.
	 * The 'reference' page table is init_mm.pgd.
209 210 211 212 213 214
	 *
	 * NOTE! We MUST NOT take any locks for this case. We may
	 * be in an interrupt or a critical region, and should
	 * only copy the information from the master page table,
	 * nothing more.
	 */
215 216
	if (unlikely((addr >= VMALLOC_START) && (addr <= VMALLOC_END))) {
		vmalloc_fault(regs, code, addr);
217 218
		return;
	}
219

220 221 222 223
	/* Enable interrupts if they were enabled in the parent context. */
	if (likely(regs->sr & BIT(6)))
		local_irq_enable();

224
	/*
225 226
	 * If we're in an interrupt, have no user context, or are running
	 * in an atomic region, then we must not take the fault.
227
	 */
228 229 230 231
	if (unlikely(faulthandler_disabled() || !mm)) {
		no_context(regs, addr);
		return;
	}
232

233 234 235
	if (user_mode(regs))
		flags |= FAULT_FLAG_USER;

236 237
	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);

238 239 240
	if (is_write(regs))
		flags |= FAULT_FLAG_WRITE;
retry:
241
	mmap_read_lock(mm);
242 243 244 245 246 247
	vma = find_vma(mm, addr);
	if (unlikely(!vma)) {
		bad_area(regs, mm, code, addr);
		return;
	}
	if (likely(vma->vm_start <= addr))
248
		goto good_area;
249 250 251 252 253 254 255 256 257
	if (unlikely(!(vma->vm_flags & VM_GROWSDOWN))) {
		bad_area(regs, mm, code, addr);
		return;
	}
	if (unlikely(expand_stack(vma, addr))) {
		bad_area(regs, mm, code, addr);
		return;
	}

258 259
	/*
	 * Ok, we have a good vm_area for this memory access, so
260
	 * we can handle it.
261 262
	 */
good_area:
263
	code = SEGV_ACCERR;
264

265 266 267
	if (unlikely(access_error(regs, vma))) {
		bad_area(regs, mm, code, addr);
		return;
268 269 270
	}

	/*
271
	 * If for any reason at all we could not handle the fault,
272 273 274
	 * make sure we exit gracefully rather than endlessly redo
	 * the fault.
	 */
275 276 277 278 279 280 281
	fault = handle_mm_fault(vma, addr, flags, regs);

	/*
	 * If we need to retry but a fatal signal is pending, handle the
	 * signal first. We do not need to release the mmap_lock because it
	 * would already be released in __lock_page_or_retry in mm/filemap.c.
	 */
282 283 284
	if (fault_signal_pending(fault, regs)) {
		if (!user_mode(regs))
			no_context(regs, addr);
285
		return;
286
	}
287 288 289 290 291 292 293 294 295 296 297 298

	if (unlikely((fault & VM_FAULT_RETRY) && (flags & FAULT_FLAG_ALLOW_RETRY))) {
		flags |= FAULT_FLAG_TRIED;

		/*
		 * No need to mmap_read_unlock(mm) as we would
		 * have already released it in __lock_page_or_retry
		 * in mm/filemap.c.
		 */
		goto retry;
	}

299
	mmap_read_unlock(mm);
300

301 302
	if (unlikely(fault & VM_FAULT_ERROR)) {
		mm_fault_error(regs, addr, fault);
303 304 305 306
		return;
	}
	return;
}
新手
引导
客服 返回
顶部