fault.c 19.9 KB
Newer Older
L
Linus Torvalds 已提交
1 2
/*
 *  S390 version
3
 *    Copyright IBM Corp. 1999
L
Linus Torvalds 已提交
4 5 6 7 8 9 10
 *    Author(s): Hartmut Penner (hp@de.ibm.com)
 *               Ulrich Weigand (uweigand@de.ibm.com)
 *
 *  Derived from "arch/i386/mm/fault.c"
 *    Copyright (C) 1995  Linus Torvalds
 */

11
#include <linux/kernel_stat.h>
12
#include <linux/perf_event.h>
L
Linus Torvalds 已提交
13 14 15 16 17 18 19 20 21
#include <linux/signal.h>
#include <linux/sched.h>
#include <linux/kernel.h>
#include <linux/errno.h>
#include <linux/string.h>
#include <linux/types.h>
#include <linux/ptrace.h>
#include <linux/mman.h>
#include <linux/mm.h>
H
Heiko Carstens 已提交
22
#include <linux/compat.h>
L
Linus Torvalds 已提交
23
#include <linux/smp.h>
24
#include <linux/kdebug.h>
L
Linus Torvalds 已提交
25 26 27 28
#include <linux/init.h>
#include <linux/console.h>
#include <linux/module.h>
#include <linux/hardirq.h>
M
Michael Grundy 已提交
29
#include <linux/kprobes.h>
30
#include <linux/uaccess.h>
31
#include <linux/hugetlb.h>
32
#include <asm/asm-offsets.h>
33
#include <asm/diag.h>
L
Linus Torvalds 已提交
34
#include <asm/pgtable.h>
35
#include <asm/gmap.h>
36
#include <asm/irq.h>
M
Martin Schwidefsky 已提交
37
#include <asm/mmu_context.h>
38
#include <asm/facility.h>
39
#include "../kernel/entry.h"
L
Linus Torvalds 已提交
40 41 42 43 44

#define __FAIL_ADDR_MASK -4096L
#define __SUBCODE_MASK 0x0600
#define __PF_RES_FIELD 0x8000000000000000ULL

45 46 47
#define VM_FAULT_BADCONTEXT	0x010000
#define VM_FAULT_BADMAP		0x020000
#define VM_FAULT_BADACCESS	0x040000
48
#define VM_FAULT_SIGNAL		0x080000
49
#define VM_FAULT_PFAULT		0x100000
50

51
static unsigned long store_indication __read_mostly;
52

53
static int __init fault_init(void)
54
{
55
	if (test_facility(75))
56
		store_indication = 0xc00;
57
	return 0;
58
}
59
early_initcall(fault_init);
60

61
static inline int notify_page_fault(struct pt_regs *regs)
62
{
63 64 65
	int ret = 0;

	/* kprobe_running() needs smp_processor_id() */
66
	if (kprobes_built_in() && !user_mode(regs)) {
67 68 69 70 71 72
		preempt_disable();
		if (kprobe_running() && kprobe_fault_handler(regs, 14))
			ret = 1;
		preempt_enable();
	}
	return ret;
M
Michael Grundy 已提交
73 74
}

L
Linus Torvalds 已提交
75 76 77

/*
 * Unlock any spinlocks which will prevent us from getting the
78
 * message out.
L
Linus Torvalds 已提交
79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99
 */
void bust_spinlocks(int yes)
{
	if (yes) {
		oops_in_progress = 1;
	} else {
		int loglevel_save = console_loglevel;
		console_unblank();
		oops_in_progress = 0;
		/*
		 * OK, the message is on the console.  Now we call printk()
		 * without oops_in_progress set so that printk will give klogd
		 * a poke.  Hold onto your hats...
		 */
		console_loglevel = 15;
		printk(" ");
		console_loglevel = loglevel_save;
	}
}

/*
100
 * Returns the address space associated with the fault.
101
 * Returns 0 for kernel space and 1 for user space.
L
Linus Torvalds 已提交
102
 */
103
static inline int user_space_fault(struct pt_regs *regs)
L
Linus Torvalds 已提交
104
{
105 106
	unsigned long trans_exc_code;

L
Linus Torvalds 已提交
107
	/*
108 109
	 * The lowest two bits of the translation exception
	 * identification indicate which paging table was used.
L
Linus Torvalds 已提交
110
	 */
111 112 113 114 115 116
	trans_exc_code = regs->int_parm_long & 3;
	if (trans_exc_code == 3) /* home space -> kernel */
		return 0;
	if (user_mode(regs))
		return 1;
	if (trans_exc_code == 2) /* secondary space -> set_fs */
117
		return current->thread.mm_segment.ar4;
118 119 120
	if (current->flags & PF_VCPU)
		return 1;
	return 0;
L
Linus Torvalds 已提交
121 122
}

123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166
static int bad_address(void *p)
{
	unsigned long dummy;

	return probe_kernel_address((unsigned long *)p, dummy);
}

static void dump_pagetable(unsigned long asce, unsigned long address)
{
	unsigned long *table = __va(asce & PAGE_MASK);

	pr_alert("AS:%016lx ", asce);
	switch (asce & _ASCE_TYPE_MASK) {
	case _ASCE_TYPE_REGION1:
		table = table + ((address >> 53) & 0x7ff);
		if (bad_address(table))
			goto bad;
		pr_cont("R1:%016lx ", *table);
		if (*table & _REGION_ENTRY_INVALID)
			goto out;
		table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
		/* fallthrough */
	case _ASCE_TYPE_REGION2:
		table = table + ((address >> 42) & 0x7ff);
		if (bad_address(table))
			goto bad;
		pr_cont("R2:%016lx ", *table);
		if (*table & _REGION_ENTRY_INVALID)
			goto out;
		table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
		/* fallthrough */
	case _ASCE_TYPE_REGION3:
		table = table + ((address >> 31) & 0x7ff);
		if (bad_address(table))
			goto bad;
		pr_cont("R3:%016lx ", *table);
		if (*table & (_REGION_ENTRY_INVALID | _REGION3_ENTRY_LARGE))
			goto out;
		table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
		/* fallthrough */
	case _ASCE_TYPE_SEGMENT:
		table = table + ((address >> 20) & 0x7ff);
		if (bad_address(table))
			goto bad;
J
Joe Perches 已提交
167
		pr_cont("S:%016lx ", *table);
168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186
		if (*table & (_SEGMENT_ENTRY_INVALID | _SEGMENT_ENTRY_LARGE))
			goto out;
		table = (unsigned long *)(*table & _SEGMENT_ENTRY_ORIGIN);
	}
	table = table + ((address >> 12) & 0xff);
	if (bad_address(table))
		goto bad;
	pr_cont("P:%016lx ", *table);
out:
	pr_cont("\n");
	return;
bad:
	pr_cont("BAD\n");
}

static void dump_fault_info(struct pt_regs *regs)
{
	unsigned long asce;

187 188
	pr_alert("Failing address: %016lx TEID: %016lx\n",
		 regs->int_parm_long & __FAIL_ADDR_MASK, regs->int_parm_long);
189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223
	pr_alert("Fault in ");
	switch (regs->int_parm_long & 3) {
	case 3:
		pr_cont("home space ");
		break;
	case 2:
		pr_cont("secondary space ");
		break;
	case 1:
		pr_cont("access register ");
		break;
	case 0:
		pr_cont("primary space ");
		break;
	}
	pr_cont("mode while using ");
	if (!user_space_fault(regs)) {
		asce = S390_lowcore.kernel_asce;
		pr_cont("kernel ");
	}
#ifdef CONFIG_PGSTE
	else if ((current->flags & PF_VCPU) && S390_lowcore.gmap) {
		struct gmap *gmap = (struct gmap *)S390_lowcore.gmap;
		asce = gmap->asce;
		pr_cont("gmap ");
	}
#endif
	else {
		asce = S390_lowcore.user_asce;
		pr_cont("user ");
	}
	pr_cont("ASCE.\n");
	dump_pagetable(asce, regs->int_parm_long & __FAIL_ADDR_MASK);
}

224 225 226
int show_unhandled_signals = 1;

void report_user_fault(struct pt_regs *regs, long signr, int is_mm_fault)
227 228 229 230 231 232 233
{
	if ((task_pid_nr(current) > 1) && !show_unhandled_signals)
		return;
	if (!unhandled_signal(current, signr))
		return;
	if (!printk_ratelimit())
		return;
234
	printk(KERN_ALERT "User process fault: interruption code %04x ilc:%d ",
235
	       regs->int_code & 0xffff, regs->int_code >> 17);
236
	print_vma_addr(KERN_CONT "in ", regs->psw.addr);
M
Martin Schwidefsky 已提交
237
	printk(KERN_CONT "\n");
238 239
	if (is_mm_fault)
		dump_fault_info(regs);
240 241 242
	show_regs(regs);
}

L
Linus Torvalds 已提交
243 244 245 246
/*
 * Send SIGSEGV to task.  This is an external routine
 * to keep the stack usage of do_page_fault small.
 */
M
Martin Schwidefsky 已提交
247
static noinline void do_sigsegv(struct pt_regs *regs, int si_code)
L
Linus Torvalds 已提交
248 249 250
{
	struct siginfo si;

251
	report_user_fault(regs, SIGSEGV, 1);
L
Linus Torvalds 已提交
252
	si.si_signo = SIGSEGV;
M
Michal Hocko 已提交
253
	si.si_errno = 0;
L
Linus Torvalds 已提交
254
	si.si_code = si_code;
M
Martin Schwidefsky 已提交
255
	si.si_addr = (void __user *)(regs->int_parm_long & __FAIL_ADDR_MASK);
L
Linus Torvalds 已提交
256 257 258
	force_sig_info(SIGSEGV, &si, current);
}

M
Martin Schwidefsky 已提交
259
static noinline void do_no_context(struct pt_regs *regs)
260 261 262 263
{
	const struct exception_table_entry *fixup;

	/* Are we prepared to handle this kernel fault?  */
264
	fixup = search_exception_tables(regs->psw.addr);
265
	if (fixup) {
266
		regs->psw.addr = extable_fixup(fixup);
267 268 269 270 271 272 273
		return;
	}

	/*
	 * Oops. The kernel tried to access some bad page. We'll have to
	 * terminate things with extreme prejudice.
	 */
274
	if (!user_space_fault(regs))
275
		printk(KERN_ALERT "Unable to handle kernel pointer dereference"
276
		       " in virtual kernel address space\n");
277 278
	else
		printk(KERN_ALERT "Unable to handle kernel paging request"
279 280
		       " in virtual user address space\n");
	dump_fault_info(regs);
M
Martin Schwidefsky 已提交
281
	die(regs, "Oops");
282 283 284
	do_exit(SIGKILL);
}

M
Martin Schwidefsky 已提交
285
static noinline void do_low_address(struct pt_regs *regs)
286 287 288 289 290
{
	/* Low-address protection hit in kernel mode means
	   NULL pointer write access in kernel mode.  */
	if (regs->psw.mask & PSW_MASK_PSTATE) {
		/* Low-address protection hit in user mode 'cannot happen'. */
M
Martin Schwidefsky 已提交
291
		die (regs, "Low-address protection");
292 293 294
		do_exit(SIGKILL);
	}

M
Martin Schwidefsky 已提交
295
	do_no_context(regs);
296 297
}

M
Martin Schwidefsky 已提交
298
static noinline void do_sigbus(struct pt_regs *regs)
299 300
{
	struct task_struct *tsk = current;
M
Martin Schwidefsky 已提交
301
	struct siginfo si;
302 303 304 305 306

	/*
	 * Send a sigbus, regardless of whether we were in kernel
	 * or user mode.
	 */
M
Martin Schwidefsky 已提交
307 308 309
	si.si_signo = SIGBUS;
	si.si_errno = 0;
	si.si_code = BUS_ADRERR;
M
Martin Schwidefsky 已提交
310
	si.si_addr = (void __user *)(regs->int_parm_long & __FAIL_ADDR_MASK);
M
Martin Schwidefsky 已提交
311
	force_sig_info(SIGBUS, &si, tsk);
312 313
}

M
Martin Schwidefsky 已提交
314
static noinline void do_fault_error(struct pt_regs *regs, int fault)
315 316 317 318 319 320 321
{
	int si_code;

	switch (fault) {
	case VM_FAULT_BADACCESS:
	case VM_FAULT_BADMAP:
		/* Bad memory access. Check if it is kernel or user space. */
322
		if (user_mode(regs)) {
323 324 325
			/* User mode accesses just cause a SIGSEGV */
			si_code = (fault == VM_FAULT_BADMAP) ?
				SEGV_MAPERR : SEGV_ACCERR;
M
Martin Schwidefsky 已提交
326
			do_sigsegv(regs, si_code);
327 328 329
			return;
		}
	case VM_FAULT_BADCONTEXT:
330
	case VM_FAULT_PFAULT:
M
Martin Schwidefsky 已提交
331
		do_no_context(regs);
332
		break;
333 334 335 336
	case VM_FAULT_SIGNAL:
		if (!user_mode(regs))
			do_no_context(regs);
		break;
337
	default: /* fault & VM_FAULT_ERROR */
338
		if (fault & VM_FAULT_OOM) {
339
			if (!user_mode(regs))
M
Martin Schwidefsky 已提交
340
				do_no_context(regs);
341 342
			else
				pagefault_out_of_memory();
343 344 345 346 347 348
		} else if (fault & VM_FAULT_SIGSEGV) {
			/* Kernel mode? Handle exceptions or die */
			if (!user_mode(regs))
				do_no_context(regs);
			else
				do_sigsegv(regs, SEGV_MAPERR);
349
		} else if (fault & VM_FAULT_SIGBUS) {
350
			/* Kernel mode? Handle exceptions or die */
351
			if (!user_mode(regs))
M
Martin Schwidefsky 已提交
352
				do_no_context(regs);
M
Martin Schwidefsky 已提交
353
			else
M
Martin Schwidefsky 已提交
354
				do_sigbus(regs);
355 356 357 358 359 360
		} else
			BUG();
		break;
	}
}

L
Linus Torvalds 已提交
361 362 363 364 365
/*
 * This routine handles page faults.  It determines the address,
 * and the problem, and then passes it off to one of the appropriate
 * routines.
 *
366
 * interruption code (int_code):
L
Linus Torvalds 已提交
367 368 369 370 371
 *   04       Protection           ->  Write-Protection  (suprression)
 *   10       Segment translation  ->  Not present       (nullification)
 *   11       Page translation     ->  Not present       (nullification)
 *   3b       Region third trans.  ->  Not present       (nullification)
 */
M
Martin Schwidefsky 已提交
372
static inline int do_exception(struct pt_regs *regs, int access)
L
Linus Torvalds 已提交
373
{
374 375 376
#ifdef CONFIG_PGSTE
	struct gmap *gmap;
#endif
377 378 379
	struct task_struct *tsk;
	struct mm_struct *mm;
	struct vm_area_struct *vma;
M
Martin Schwidefsky 已提交
380
	unsigned long trans_exc_code;
381
	unsigned long address;
382 383
	unsigned int flags;
	int fault;
L
Linus Torvalds 已提交
384

385 386 387 388 389
	tsk = current;
	/*
	 * The instruction that caused the program check has
	 * been nullified. Don't signal single step via SIGTRAP.
	 */
390
	clear_pt_regs_flag(regs, PIF_PER_TRAP);
391

392
	if (notify_page_fault(regs))
393
		return 0;
M
Michael Grundy 已提交
394

395
	mm = tsk->mm;
M
Martin Schwidefsky 已提交
396
	trans_exc_code = regs->int_parm_long;
L
Linus Torvalds 已提交
397 398 399 400 401 402

	/*
	 * Verify that the fault happened in user space, that
	 * we are not in an interrupt and that there is a 
	 * user context.
	 */
403
	fault = VM_FAULT_BADCONTEXT;
404
	if (unlikely(!user_space_fault(regs) || faulthandler_disabled() || !mm))
405
		goto out;
L
Linus Torvalds 已提交
406

407
	address = trans_exc_code & __FAIL_ADDR_MASK;
408
	perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
409
	flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
410 411
	if (user_mode(regs))
		flags |= FAULT_FLAG_USER;
412 413
	if (access == VM_WRITE || (trans_exc_code & store_indication) == 0x400)
		flags |= FAULT_FLAG_WRITE;
414
	down_read(&mm->mmap_sem);
L
Linus Torvalds 已提交
415

416
#ifdef CONFIG_PGSTE
417 418
	gmap = (current->flags & PF_VCPU) ?
		(struct gmap *) S390_lowcore.gmap : NULL;
419
	if (gmap) {
420
		current->thread.gmap_addr = address;
421
		current->thread.gmap_write_flag = !!(flags & FAULT_FLAG_WRITE);
422
		address = __gmap_translate(gmap, address);
423 424 425 426
		if (address == -EFAULT) {
			fault = VM_FAULT_BADMAP;
			goto out_up;
		}
427 428
		if (gmap->pfault_enabled)
			flags |= FAULT_FLAG_RETRY_NOWAIT;
429 430 431 432
	}
#endif

retry:
433
	fault = VM_FAULT_BADMAP;
434 435
	vma = find_vma(mm, address);
	if (!vma)
436
		goto out_up;
G
Gerald Schaefer 已提交
437

438 439 440 441 442 443 444 445 446 447 448 449
	if (unlikely(vma->vm_start > address)) {
		if (!(vma->vm_flags & VM_GROWSDOWN))
			goto out_up;
		if (expand_stack(vma, address))
			goto out_up;
	}

	/*
	 * Ok, we have a good vm_area for this memory access, so
	 * we can handle it..
	 */
	fault = VM_FAULT_BADACCESS;
450
	if (unlikely(!(vma->vm_flags & access)))
451
		goto out_up;
L
Linus Torvalds 已提交
452

453 454
	if (is_vm_hugetlb_page(vma))
		address &= HPAGE_MASK;
L
Linus Torvalds 已提交
455 456 457 458 459
	/*
	 * If for any reason at all we couldn't handle the fault,
	 * make sure we exit gracefully rather than endlessly redo
	 * the fault.
	 */
460
	fault = handle_mm_fault(mm, vma, address, flags);
461 462 463 464 465
	/* No reason to continue if interrupted by SIGKILL. */
	if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) {
		fault = VM_FAULT_SIGNAL;
		goto out;
	}
466 467 468
	if (unlikely(fault & VM_FAULT_ERROR))
		goto out_up;

469 470 471 472 473 474 475 476
	/*
	 * Major/minor page fault accounting is only done on the
	 * initial attempt. If we go through a retry, it is extremely
	 * likely that the page will be found in page cache at that point.
	 */
	if (flags & FAULT_FLAG_ALLOW_RETRY) {
		if (fault & VM_FAULT_MAJOR) {
			tsk->maj_flt++;
477
			perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1,
478 479 480
				      regs, address);
		} else {
			tsk->min_flt++;
481
			perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1,
482 483 484
				      regs, address);
		}
		if (fault & VM_FAULT_RETRY) {
485 486 487 488 489 490 491 492 493
#ifdef CONFIG_PGSTE
			if (gmap && (flags & FAULT_FLAG_RETRY_NOWAIT)) {
				/* FAULT_FLAG_RETRY_NOWAIT has been set,
				 * mmap_sem has not been released */
				current->thread.gmap_pfault = 1;
				fault = VM_FAULT_PFAULT;
				goto out_up;
			}
#endif
494 495
			/* Clear FAULT_FLAG_ALLOW_RETRY to avoid any risk
			 * of starvation. */
496 497
			flags &= ~(FAULT_FLAG_ALLOW_RETRY |
				   FAULT_FLAG_RETRY_NOWAIT);
498
			flags |= FAULT_FLAG_TRIED;
499
			down_read(&mm->mmap_sem);
500 501
			goto retry;
		}
502
	}
503 504 505 506 507 508 509 510 511 512 513 514 515 516
#ifdef CONFIG_PGSTE
	if (gmap) {
		address =  __gmap_link(gmap, current->thread.gmap_addr,
				       address);
		if (address == -EFAULT) {
			fault = VM_FAULT_BADMAP;
			goto out_up;
		}
		if (address == -ENOMEM) {
			fault = VM_FAULT_OOM;
			goto out_up;
		}
	}
#endif
517 518
	fault = 0;
out_up:
519
	up_read(&mm->mmap_sem);
520 521
out:
	return fault;
L
Linus Torvalds 已提交
522 523
}

524
void do_protection_exception(struct pt_regs *regs)
L
Linus Torvalds 已提交
525
{
M
Martin Schwidefsky 已提交
526
	unsigned long trans_exc_code;
527
	int fault;
528

M
Martin Schwidefsky 已提交
529
	trans_exc_code = regs->int_parm_long;
530 531 532 533 534 535 536
	/*
	 * Protection exceptions are suppressing, decrement psw address.
	 * The exception to this rule are aborted transactions, for these
	 * the PSW already points to the correct location.
	 */
	if (!(regs->int_code & 0x200))
		regs->psw.addr = __rewind_psw(regs->psw, regs->int_code >> 16);
537 538 539 540 541
	/*
	 * Check for low-address protection.  This needs to be treated
	 * as a special case because the translation exception code
	 * field is not guaranteed to contain valid data in this case.
	 */
542
	if (unlikely(!(trans_exc_code & 4))) {
M
Martin Schwidefsky 已提交
543
		do_low_address(regs);
544 545
		return;
	}
M
Martin Schwidefsky 已提交
546
	fault = do_exception(regs, VM_WRITE);
547
	if (unlikely(fault))
M
Martin Schwidefsky 已提交
548
		do_fault_error(regs, fault);
L
Linus Torvalds 已提交
549
}
550
NOKPROBE_SYMBOL(do_protection_exception);
L
Linus Torvalds 已提交
551

552
void do_dat_exception(struct pt_regs *regs)
L
Linus Torvalds 已提交
553
{
554
	int access, fault;
555

556
	access = VM_READ | VM_EXEC | VM_WRITE;
M
Martin Schwidefsky 已提交
557
	fault = do_exception(regs, access);
558
	if (unlikely(fault))
M
Martin Schwidefsky 已提交
559
		do_fault_error(regs, fault);
L
Linus Torvalds 已提交
560
}
561
NOKPROBE_SYMBOL(do_dat_exception);
L
Linus Torvalds 已提交
562 563 564 565 566

#ifdef CONFIG_PFAULT 
/*
 * 'pfault' pseudo page faults routines.
 */
567
static int pfault_disable;
L
Linus Torvalds 已提交
568 569 570 571 572 573 574 575 576

static int __init nopfault(char *str)
{
	pfault_disable = 1;
	return 1;
}

__setup("nopfault", nopfault);

H
Heiko Carstens 已提交
577 578 579 580 581 582 583 584 585 586
struct pfault_refbk {
	u16 refdiagc;
	u16 reffcode;
	u16 refdwlen;
	u16 refversn;
	u64 refgaddr;
	u64 refselmk;
	u64 refcmpmk;
	u64 reserved;
} __attribute__ ((packed, aligned(8)));
L
Linus Torvalds 已提交
587 588 589

int pfault_init(void)
{
H
Heiko Carstens 已提交
590 591 592 593 594
	struct pfault_refbk refbk = {
		.refdiagc = 0x258,
		.reffcode = 0,
		.refdwlen = 5,
		.refversn = 2,
595
		.refgaddr = __LC_LPP,
H
Heiko Carstens 已提交
596 597 598
		.refselmk = 1ULL << 48,
		.refcmpmk = 1ULL << 48,
		.reserved = __PF_RES_FIELD };
L
Linus Torvalds 已提交
599 600
        int rc;

601
	if (pfault_disable)
L
Linus Torvalds 已提交
602
		return -1;
603
	diag_stat_inc(DIAG_STAT_X258);
604 605 606 607
	asm volatile(
		"	diag	%1,%0,0x258\n"
		"0:	j	2f\n"
		"1:	la	%0,8\n"
L
Linus Torvalds 已提交
608
		"2:\n"
609 610
		EX_TABLE(0b,1b)
		: "=d" (rc) : "a" (&refbk), "m" (refbk) : "cc");
L
Linus Torvalds 已提交
611 612 613 614 615
        return rc;
}

void pfault_fini(void)
{
H
Heiko Carstens 已提交
616 617 618 619 620 621
	struct pfault_refbk refbk = {
		.refdiagc = 0x258,
		.reffcode = 1,
		.refdwlen = 5,
		.refversn = 2,
	};
L
Linus Torvalds 已提交
622

623
	if (pfault_disable)
L
Linus Torvalds 已提交
624
		return;
625
	diag_stat_inc(DIAG_STAT_X258);
626 627
	asm volatile(
		"	diag	%0,0,0x258\n"
L
Linus Torvalds 已提交
628
		"0:\n"
629 630
		EX_TABLE(0b,0b)
		: : "a" (&refbk), "m" (refbk) : "cc");
L
Linus Torvalds 已提交
631 632
}

633 634 635
static DEFINE_SPINLOCK(pfault_lock);
static LIST_HEAD(pfault_list);

636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658
#define PF_COMPLETE	0x0080

/*
 * The mechanism of our pfault code: if Linux is running as guest, runs a user
 * space process and the user space process accesses a page that the host has
 * paged out we get a pfault interrupt.
 *
 * This allows us, within the guest, to schedule a different process. Without
 * this mechanism the host would have to suspend the whole virtual cpu until
 * the page has been paged in.
 *
 * So when we get such an interrupt then we set the state of the current task
 * to uninterruptible and also set the need_resched flag. Both happens within
 * interrupt context(!). If we later on want to return to user space we
 * recognize the need_resched flag and then call schedule().  It's not very
 * obvious how this works...
 *
 * Of course we have a lot of additional fun with the completion interrupt (->
 * host signals that a page of a process has been paged in and the process can
 * continue to run). This interrupt can arrive on any cpu and, since we have
 * virtual cpus, actually appear before the interrupt that signals that a page
 * is missing.
 */
659
static void pfault_interrupt(struct ext_code ext_code,
660
			     unsigned int param32, unsigned long param64)
L
Linus Torvalds 已提交
661 662 663
{
	struct task_struct *tsk;
	__u16 subcode;
664
	pid_t pid;
L
Linus Torvalds 已提交
665 666

	/*
667 668 669
	 * Get the external interruption subcode & pfault initial/completion
	 * signal bit. VM stores this in the 'cpu address' field associated
	 * with the external interrupt.
L
Linus Torvalds 已提交
670
	 */
671
	subcode = ext_code.subcode;
L
Linus Torvalds 已提交
672 673
	if ((subcode & 0xff00) != __SUBCODE_MASK)
		return;
674
	inc_irq_stat(IRQEXT_PFL);
675
	/* Get the token (= pid of the affected task). */
676
	pid = param64 & LPP_PFAULT_PID_MASK;
677 678 679 680 681 682 683
	rcu_read_lock();
	tsk = find_task_by_pid_ns(pid, &init_pid_ns);
	if (tsk)
		get_task_struct(tsk);
	rcu_read_unlock();
	if (!tsk)
		return;
684
	spin_lock(&pfault_lock);
685
	if (subcode & PF_COMPLETE) {
L
Linus Torvalds 已提交
686
		/* signal bit is set -> a page has been swapped in by VM */
687
		if (tsk->thread.pfault_wait == 1) {
L
Linus Torvalds 已提交
688 689 690 691
			/* Initial interrupt was faster than the completion
			 * interrupt. pfault_wait is valid. Set pfault_wait
			 * back to zero and wake up the process. This can
			 * safely be done because the task is still sleeping
692
			 * and can't produce new pfaults. */
L
Linus Torvalds 已提交
693
			tsk->thread.pfault_wait = 0;
694
			list_del(&tsk->thread.list);
L
Linus Torvalds 已提交
695
			wake_up_process(tsk);
696
			put_task_struct(tsk);
697 698 699
		} else {
			/* Completion interrupt was faster than initial
			 * interrupt. Set pfault_wait to -1 so the initial
700 701 702 703 704 705 706
			 * interrupt doesn't put the task to sleep.
			 * If the task is not running, ignore the completion
			 * interrupt since it must be a leftover of a PFAULT
			 * CANCEL operation which didn't remove all pending
			 * completion interrupts. */
			if (tsk->state == TASK_RUNNING)
				tsk->thread.pfault_wait = -1;
L
Linus Torvalds 已提交
707 708 709
		}
	} else {
		/* signal bit not set -> a real page is missing. */
H
Heiko Carstens 已提交
710 711
		if (WARN_ON_ONCE(tsk != current))
			goto out;
712 713
		if (tsk->thread.pfault_wait == 1) {
			/* Already on the list with a reference: put to sleep */
714
			goto block;
715
		} else if (tsk->thread.pfault_wait == -1) {
L
Linus Torvalds 已提交
716
			/* Completion interrupt was faster than the initial
717 718
			 * interrupt (pfault_wait == -1). Set pfault_wait
			 * back to zero and exit. */
L
Linus Torvalds 已提交
719
			tsk->thread.pfault_wait = 0;
720 721
		} else {
			/* Initial interrupt arrived before completion
722 723 724 725 726
			 * interrupt. Let the task sleep.
			 * An extra task reference is needed since a different
			 * cpu may set the task state to TASK_RUNNING again
			 * before the scheduler is reached. */
			get_task_struct(tsk);
727 728
			tsk->thread.pfault_wait = 1;
			list_add(&tsk->thread.list, &pfault_list);
729 730 731 732 733
block:
			/* Since this must be a userspace fault, there
			 * is no kernel task state to trample. Rely on the
			 * return to userspace schedule() to block. */
			__set_current_state(TASK_UNINTERRUPTIBLE);
L
Linus Torvalds 已提交
734
			set_tsk_need_resched(tsk);
735 736
		}
	}
H
Heiko Carstens 已提交
737
out:
738
	spin_unlock(&pfault_lock);
739
	put_task_struct(tsk);
740 741
}

742 743
static int pfault_cpu_notify(struct notifier_block *self, unsigned long action,
			     void *hcpu)
744 745 746 747
{
	struct thread_struct *thread, *next;
	struct task_struct *tsk;

748
	switch (action & ~CPU_TASKS_FROZEN) {
749 750 751 752 753 754 755
	case CPU_DEAD:
		spin_lock_irq(&pfault_lock);
		list_for_each_entry_safe(thread, next, &pfault_list, list) {
			thread->pfault_wait = 0;
			list_del(&thread->list);
			tsk = container_of(thread, struct task_struct, thread);
			wake_up_process(tsk);
756
			put_task_struct(tsk);
757 758 759 760 761
		}
		spin_unlock_irq(&pfault_lock);
		break;
	default:
		break;
L
Linus Torvalds 已提交
762
	}
763
	return NOTIFY_OK;
L
Linus Torvalds 已提交
764 765
}

766
static int __init pfault_irq_init(void)
H
Heiko Carstens 已提交
767
{
768
	int rc;
H
Heiko Carstens 已提交
769

770
	rc = register_external_irq(EXT_IRQ_CP_SERVICE, pfault_interrupt);
H
Heiko Carstens 已提交
771 772 773 774 775
	if (rc)
		goto out_extint;
	rc = pfault_init() == 0 ? 0 : -EOPNOTSUPP;
	if (rc)
		goto out_pfault;
776
	irq_subclass_register(IRQ_SUBCLASS_SERVICE_SIGNAL);
H
Heiko Carstens 已提交
777 778
	hotcpu_notifier(pfault_cpu_notify, 0);
	return 0;
H
Heiko Carstens 已提交
779

H
Heiko Carstens 已提交
780
out_pfault:
781
	unregister_external_irq(EXT_IRQ_CP_SERVICE, pfault_interrupt);
H
Heiko Carstens 已提交
782 783 784
out_extint:
	pfault_disable = 1;
	return rc;
H
Heiko Carstens 已提交
785
}
786 787
early_initcall(pfault_irq_init);

H
Heiko Carstens 已提交
788
#endif /* CONFIG_PFAULT */