entry.S 35.8 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3
/*
 *    S390 low-level entry points.
 *
4
 *    Copyright IBM Corp. 1999, 2012
L
Linus Torvalds 已提交
5
 *    Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
H
Heiko Carstens 已提交
6 7
 *		 Hartmut Penner (hp@de.ibm.com),
 *		 Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
8
 *		 Heiko Carstens <heiko.carstens@de.ibm.com>
L
Linus Torvalds 已提交
9 10
 */

11
#include <linux/init.h>
12
#include <linux/linkage.h>
13
#include <asm/processor.h>
L
Linus Torvalds 已提交
14 15 16 17
#include <asm/cache.h>
#include <asm/errno.h>
#include <asm/ptrace.h>
#include <asm/thread_info.h>
18
#include <asm/asm-offsets.h>
L
Linus Torvalds 已提交
19 20
#include <asm/unistd.h>
#include <asm/page.h>
21
#include <asm/sigp.h>
22
#include <asm/irq.h>
23
#include <asm/vx-insn.h>
24 25
#include <asm/setup.h>
#include <asm/nmi.h>
A
Al Viro 已提交
26
#include <asm/export.h>
L
Linus Torvalds 已提交
27

28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43
__PT_R0      =	__PT_GPRS
__PT_R1      =	__PT_GPRS + 8
__PT_R2      =	__PT_GPRS + 16
__PT_R3      =	__PT_GPRS + 24
__PT_R4      =	__PT_GPRS + 32
__PT_R5      =	__PT_GPRS + 40
__PT_R6      =	__PT_GPRS + 48
__PT_R7      =	__PT_GPRS + 56
__PT_R8      =	__PT_GPRS + 64
__PT_R9      =	__PT_GPRS + 72
__PT_R10     =	__PT_GPRS + 80
__PT_R11     =	__PT_GPRS + 88
__PT_R12     =	__PT_GPRS + 96
__PT_R13     =	__PT_GPRS + 104
__PT_R14     =	__PT_GPRS + 112
__PT_R15     =	__PT_GPRS + 120
L
Linus Torvalds 已提交
44

45
STACK_SHIFT = PAGE_SHIFT + THREAD_SIZE_ORDER
L
Linus Torvalds 已提交
46
STACK_SIZE  = 1 << STACK_SHIFT
47
STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE
L
Linus Torvalds 已提交
48

49
_TIF_WORK	= (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
50
		   _TIF_UPROBE | _TIF_GUARDED_STORAGE | _TIF_PATCH_PENDING)
51 52
_TIF_TRACE	= (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \
		   _TIF_SYSCALL_TRACEPOINT)
53 54
_CIF_WORK	= (_CIF_MCCK_PENDING | _CIF_ASCE_PRIMARY | \
		   _CIF_ASCE_SECONDARY | _CIF_FPU)
55
_PIF_WORK	= (_PIF_PER_TRAP | _PIF_SYSCALL_RESTART)
L
Linus Torvalds 已提交
56

57
#define BASED(name) name-cleanup_critical(%r13)
L
Linus Torvalds 已提交
58

59
	.macro	TRACE_IRQS_ON
60
#ifdef CONFIG_TRACE_IRQFLAGS
61 62
	basr	%r2,%r0
	brasl	%r14,trace_hardirqs_on_caller
63
#endif
64 65 66
	.endm

	.macro	TRACE_IRQS_OFF
67
#ifdef CONFIG_TRACE_IRQFLAGS
68 69
	basr	%r2,%r0
	brasl	%r14,trace_hardirqs_off_caller
70
#endif
71
	.endm
72 73

	.macro	LOCKDEP_SYS_EXIT
74 75 76
#ifdef CONFIG_LOCKDEP
	tm	__PT_PSW+1(%r11),0x01	# returning to user ?
	jz	.+10
77
	brasl	%r14,lockdep_sys_exit
78
#endif
L
Linus Torvalds 已提交
79 80
	.endm

81
	.macro	CHECK_STACK stacksize,savearea
82
#ifdef CONFIG_CHECK_STACK
83 84 85
	tml	%r15,\stacksize - CONFIG_STACK_GUARD
	lghi	%r14,\savearea
	jz	stack_overflow
86 87 88
#endif
	.endm

89
	.macro	SWITCH_ASYNC savearea,timer
90 91 92 93 94
	tmhh	%r8,0x0001		# interrupting from user ?
	jnz	1f
	lgr	%r14,%r9
	slg	%r14,BASED(.Lcritical_start)
	clg	%r14,BASED(.Lcritical_length)
L
Linus Torvalds 已提交
95
	jhe	0f
96
	lghi	%r11,\savearea		# inside critical section, do cleanup
L
Linus Torvalds 已提交
97
	brasl	%r14,cleanup_critical
98
	tmhh	%r8,0x0001		# retest problem state after cleanup
L
Linus Torvalds 已提交
99
	jnz	1f
100
0:	lg	%r14,__LC_ASYNC_STACK	# are we already on the async stack?
L
Linus Torvalds 已提交
101
	slgr	%r14,%r15
102
	srag	%r14,%r14,STACK_SHIFT
103
	jnz	2f
104
	CHECK_STACK 1<<STACK_SHIFT,\savearea
105
	aghi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
106
	j	3f
107
1:	UPDATE_VTIME %r14,%r15,\timer
108
2:	lg	%r15,__LC_ASYNC_STACK	# load async stack
109
3:	la	%r11,STACK_FRAME_OVERHEAD(%r15)
H
Heiko Carstens 已提交
110
	.endm
L
Linus Torvalds 已提交
111

112 113 114 115 116 117 118 119 120
	.macro UPDATE_VTIME w1,w2,enter_timer
	lg	\w1,__LC_EXIT_TIMER
	lg	\w2,__LC_LAST_UPDATE_TIMER
	slg	\w1,\enter_timer
	slg	\w2,__LC_EXIT_TIMER
	alg	\w1,__LC_USER_TIMER
	alg	\w2,__LC_SYSTEM_TIMER
	stg	\w1,__LC_USER_TIMER
	stg	\w2,__LC_SYSTEM_TIMER
121
	mvc	__LC_LAST_UPDATE_TIMER(8),\enter_timer
L
Linus Torvalds 已提交
122 123
	.endm

124
	.macro REENABLE_IRQS
125 126 127
	stg	%r8,__LC_RETURN_PSW
	ni	__LC_RETURN_PSW,0xbf
	ssm	__LC_RETURN_PSW
128 129
	.endm

130
	.macro STCK savearea
131
#ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES
132 133 134 135 136 137
	.insn	s,0xb27c0000,\savearea		# store clock fast
#else
	.insn	s,0xb2050000,\savearea		# store clock
#endif
	.endm

138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159
	/*
	 * The TSTMSK macro generates a test-under-mask instruction by
	 * calculating the memory offset for the specified mask value.
	 * Mask value can be any constant.  The macro shifts the mask
	 * value to calculate the memory offset for the test-under-mask
	 * instruction.
	 */
	.macro TSTMSK addr, mask, size=8, bytepos=0
		.if (\bytepos < \size) && (\mask >> 8)
			.if (\mask & 0xff)
				.error "Mask exceeds byte boundary"
			.endif
			TSTMSK \addr, "(\mask >> 8)", \size, "(\bytepos + 1)"
			.exitm
		.endif
		.ifeq \mask
			.error "Mask must not be zero"
		.endif
		off = \size - \bytepos - 1
		tm	off+\addr, \mask
	.endm

160
	.section .kprobes.text, "ax"
161 162 163 164 165 166 167 168 169 170
.Ldummy:
	/*
	 * This nop exists only in order to avoid that __switch_to starts at
	 * the beginning of the kprobes text section. In that case we would
	 * have several symbols at the same address. E.g. objdump would take
	 * an arbitrary symbol name when disassembling this code.
	 * With the added nop in between the __switch_to symbol is unique
	 * again.
	 */
	nop	0
171

L
Linus Torvalds 已提交
172 173 174 175 176 177 178
/*
 * Scheduler resume function, called by switch_to
 *  gpr2 = (task_struct *) prev
 *  gpr3 = (task_struct *) next
 * Returns:
 *  gpr2 = prev
 */
179
ENTRY(__switch_to)
180
	stmg	%r6,%r15,__SF_GPRS(%r15)	# store gprs of prev task
181 182
	lgr	%r1,%r2
	aghi	%r1,__TASK_thread		# thread_struct of prev task
183
	lg	%r5,__TASK_stack(%r3)		# start of kernel stack of next
184 185 186
	stg	%r15,__THREAD_ksp(%r1)		# store kernel stack of prev
	lgr	%r1,%r3
	aghi	%r1,__TASK_thread		# thread_struct of next task
187
	lgr	%r15,%r5
188
	aghi	%r15,STACK_INIT			# end of kernel stack of next
189 190
	stg	%r3,__LC_CURRENT		# store task struct of next
	stg	%r15,__LC_KERNEL_STACK		# store end of kernel stack
191
	lg	%r15,__THREAD_ksp(%r1)		# load kernel stack of next
192
	mvc	__LC_CURRENT_PID(4,%r0),__TASK_pid(%r3) # store pid of next
193
	lmg	%r6,%r15,__SF_GPRS(%r15)	# load gprs of next task
194 195 196
	TSTMSK	__LC_MACHINE_FLAGS,MACHINE_FLAG_LPP
	bzr	%r14
	.insn	s,0xb2800000,__LC_LPP		# set program parameter
L
Linus Torvalds 已提交
197 198
	br	%r14

199
.L__critical_start:
200 201 202 203 204 205 206 207 208 209 210

#if IS_ENABLED(CONFIG_KVM)
/*
 * sie64a calling convention:
 * %r2 pointer to sie control block
 * %r3 guest register save area
 */
ENTRY(sie64a)
	stmg	%r6,%r14,__SF_GPRS(%r15)	# save kernel registers
	stg	%r2,__SF_EMPTY(%r15)		# save control block pointer
	stg	%r3,__SF_EMPTY+8(%r15)		# save guest register save area
211
	xc	__SF_EMPTY+16(8,%r15),__SF_EMPTY+16(%r15) # reason code = 0
212
	TSTMSK	__LC_CPU_FLAGS,_CIF_FPU		# load guest fp/vx registers ?
213 214 215 216 217 218 219 220 221 222 223 224 225
	jno	.Lsie_load_guest_gprs
	brasl	%r14,load_fpu_regs		# load guest fp/vx regs
.Lsie_load_guest_gprs:
	lmg	%r0,%r13,0(%r3)			# load guest gprs 0-13
	lg	%r14,__LC_GMAP			# get gmap pointer
	ltgr	%r14,%r14
	jz	.Lsie_gmap
	lctlg	%c1,%c1,__GMAP_ASCE(%r14)	# load primary asce
.Lsie_gmap:
	lg	%r14,__SF_EMPTY(%r15)		# get control block pointer
	oi	__SIE_PROG0C+3(%r14),1		# we are going into SIE now
	tm	__SIE_PROG20+3(%r14),3		# last exit...
	jnz	.Lsie_skip
226
	TSTMSK	__LC_CPU_FLAGS,_CIF_FPU
227
	jo	.Lsie_skip			# exit if fp/vx regs changed
228
.Lsie_entry:
229 230 231 232 233 234
	sie	0(%r14)
.Lsie_skip:
	ni	__SIE_PROG0C+3(%r14),0xfe	# no longer in SIE
	lctlg	%c1,%c1,__LC_USER_ASCE		# load primary asce
.Lsie_done:
# some program checks are suppressing. C code (e.g. do_protection_exception)
235 236 237 238
# will rewind the PSW by the ILC, which is often 4 bytes in case of SIE. There
# are some corner cases (e.g. runtime instrumentation) where ILC is unpredictable.
# Other instructions between sie64a and .Lsie_done should not cause program
# interrupts. So lets use 3 nops as a landing pad for all possible rewinds.
239
# See also .Lcleanup_sie
240 241 242 243 244 245
.Lrewind_pad6:
	nopr	7
.Lrewind_pad4:
	nopr	7
.Lrewind_pad2:
	nopr	7
246 247 248 249 250
	.globl sie_exit
sie_exit:
	lg	%r14,__SF_EMPTY+8(%r15)		# load guest register save area
	stmg	%r0,%r13,0(%r14)		# save guest gprs 0-13
	lmg	%r6,%r14,__SF_GPRS(%r15)	# restore kernel registers
251
	lg	%r2,__SF_EMPTY+16(%r15)		# return exit reason code
252 253 254
	br	%r14
.Lsie_fault:
	lghi	%r14,-EFAULT
255
	stg	%r14,__SF_EMPTY+16(%r15)	# set exit reason code
256 257
	j	sie_exit

258 259 260
	EX_TABLE(.Lrewind_pad6,.Lsie_fault)
	EX_TABLE(.Lrewind_pad4,.Lsie_fault)
	EX_TABLE(.Lrewind_pad2,.Lsie_fault)
261
	EX_TABLE(sie_exit,.Lsie_fault)
A
Al Viro 已提交
262 263
EXPORT_SYMBOL(sie64a)
EXPORT_SYMBOL(sie_exit)
264 265
#endif

L
Linus Torvalds 已提交
266 267 268 269 270
/*
 * SVC interrupt handler routine. System calls are synchronous events and
 * are executed with interrupts enabled.
 */

271
ENTRY(system_call)
M
Martin Schwidefsky 已提交
272
	stpt	__LC_SYNC_ENTER_TIMER
273
.Lsysc_stmg:
274
	stmg	%r8,%r15,__LC_SAVE_AREA_SYNC
275
	lg	%r12,__LC_CURRENT
276
	lghi	%r13,__TASK_thread
277
	lghi	%r14,_PIF_SYSCALL
278
.Lsysc_per:
279 280
	lg	%r15,__LC_KERNEL_STACK
	la	%r11,STACK_FRAME_OVERHEAD(%r15)	# pointer to pt_regs
281
.Lsysc_vtime:
282
	UPDATE_VTIME %r8,%r9,__LC_SYNC_ENTER_TIMER
283 284 285
	stmg	%r0,%r7,__PT_R0(%r11)
	mvc	__PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
	mvc	__PT_PSW(16,%r11),__LC_SVC_OLD_PSW
M
Martin Schwidefsky 已提交
286
	mvc	__PT_INT_CODE(4,%r11),__LC_SVC_ILC
287
	stg	%r14,__PT_FLAGS(%r11)
288
.Lsysc_do_svc:
289 290
	# load address of system call table
	lg	%r10,__THREAD_sysc_table(%r13,%r12)
M
Martin Schwidefsky 已提交
291
	llgh	%r8,__PT_INT_CODE+2(%r11)
292
	slag	%r8,%r8,2			# shift and test for svc 0
293
	jnz	.Lsysc_nr_ok
L
Linus Torvalds 已提交
294
	# svc 0: system call number in %r1
295
	llgfr	%r1,%r1				# clear high word in r1
296
	cghi	%r1,NR_syscalls
297
	jnl	.Lsysc_nr_ok
M
Martin Schwidefsky 已提交
298
	sth	%r1,__PT_INT_CODE+2(%r11)
299
	slag	%r8,%r1,2
300
.Lsysc_nr_ok:
301 302 303 304
	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
	stg	%r2,__PT_ORIG_GPR2(%r11)
	stg	%r7,STACK_FRAME_OVERHEAD(%r15)
	lgf	%r9,0(%r8,%r10)			# get system call add.
305
	TSTMSK	__TI_flags(%r12),_TIF_TRACE
306
	jnz	.Lsysc_tracesys
307 308
	basr	%r14,%r9			# call sys_xxxx
	stg	%r2,__PT_R2(%r11)		# store return value
L
Linus Torvalds 已提交
309

310
.Lsysc_return:
311
	LOCKDEP_SYS_EXIT
312
.Lsysc_tif:
313
	TSTMSK	__PT_FLAGS(%r11),_PIF_WORK
314
	jnz	.Lsysc_work
315
	TSTMSK	__TI_flags(%r12),_TIF_WORK
316
	jnz	.Lsysc_work			# check for work
317
	TSTMSK	__LC_CPU_FLAGS,_CIF_WORK
318 319
	jnz	.Lsysc_work
.Lsysc_restore:
320 321 322
	lg	%r14,__LC_VDSO_PER_CPU
	lmg	%r0,%r10,__PT_R0(%r11)
	mvc	__LC_RETURN_PSW(16),__PT_PSW(%r11)
323
.Lsysc_exit_timer:
324 325 326 327
	stpt	__LC_EXIT_TIMER
	mvc	__VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
	lmg	%r11,%r15,__PT_R11(%r11)
	lpswe	__LC_RETURN_PSW
328
.Lsysc_done:
329

330 331 332
#
# One of the work bits is on. Find out which one.
#
333
.Lsysc_work:
334
	TSTMSK	__LC_CPU_FLAGS,_CIF_MCCK_PENDING
335
	jo	.Lsysc_mcck_pending
336
	TSTMSK	__TI_flags(%r12),_TIF_NEED_RESCHED
337
	jo	.Lsysc_reschedule
338 339
	TSTMSK	__PT_FLAGS(%r11),_PIF_SYSCALL_RESTART
	jo	.Lsysc_syscall_restart
340
#ifdef CONFIG_UPROBES
341
	TSTMSK	__TI_flags(%r12),_TIF_UPROBE
342
	jo	.Lsysc_uprobe_notify
343
#endif
344 345
	TSTMSK	__TI_flags(%r12),_TIF_GUARDED_STORAGE
	jo	.Lsysc_guarded_storage
346
	TSTMSK	__PT_FLAGS(%r11),_PIF_PER_TRAP
347
	jo	.Lsysc_singlestep
348 349 350 351 352
#ifdef CONFIG_LIVEPATCH
	TSTMSK	__TI_flags(%r12),_TIF_PATCH_PENDING
	jo	.Lsysc_patch_pending	# handle live patching just before
					# signals and possible syscall restart
#endif
353 354
	TSTMSK	__PT_FLAGS(%r11),_PIF_SYSCALL_RESTART
	jo	.Lsysc_syscall_restart
355
	TSTMSK	__TI_flags(%r12),_TIF_SIGPENDING
356
	jo	.Lsysc_sigpending
357
	TSTMSK	__TI_flags(%r12),_TIF_NOTIFY_RESUME
358
	jo	.Lsysc_notify_resume
359
	TSTMSK	__LC_CPU_FLAGS,_CIF_FPU
360
	jo	.Lsysc_vxrs
361 362
	TSTMSK	__LC_CPU_FLAGS,(_CIF_ASCE_PRIMARY|_CIF_ASCE_SECONDARY)
	jnz	.Lsysc_asce
363
	j	.Lsysc_return		# beware of critical section cleanup
L
Linus Torvalds 已提交
364 365 366

#
# _TIF_NEED_RESCHED is set, call schedule
H
Heiko Carstens 已提交
367
#
368 369
.Lsysc_reschedule:
	larl	%r14,.Lsysc_return
370
	jg	schedule
L
Linus Torvalds 已提交
371

372
#
373
# _CIF_MCCK_PENDING is set, call handler
374
#
375 376
.Lsysc_mcck_pending:
	larl	%r14,.Lsysc_return
H
Heiko Carstens 已提交
377
	jg	s390_handle_mcck	# TIF bit will be cleared by handler
378

379
#
380
# _CIF_ASCE_PRIMARY and/or CIF_ASCE_SECONDARY set, load user space asce
381
#
382
.Lsysc_asce:
383
	ni	__LC_CPU_FLAGS+7,255-_CIF_ASCE_PRIMARY
384
	lctlg	%c1,%c1,__LC_USER_ASCE		# load primary asce
385 386 387 388
	TSTMSK	__LC_CPU_FLAGS,_CIF_ASCE_SECONDARY
	jz	.Lsysc_return
	larl	%r14,.Lsysc_return
	jg	set_fs_fixup
389

390 391 392 393 394 395 396
#
# CIF_FPU is set, restore floating-point controls and floating-point registers.
#
.Lsysc_vxrs:
	larl	%r14,.Lsysc_return
	jg	load_fpu_regs

L
Linus Torvalds 已提交
397
#
398
# _TIF_SIGPENDING is set, call do_signal
L
Linus Torvalds 已提交
399
#
400
.Lsysc_sigpending:
401 402
	lgr	%r2,%r11		# pass pointer to pt_regs
	brasl	%r14,do_signal
403
	TSTMSK	__PT_FLAGS(%r11),_PIF_SYSCALL
404
	jno	.Lsysc_return
405 406
.Lsysc_do_syscall:
	lghi	%r13,__TASK_thread
407
	lmg	%r2,%r7,__PT_R2(%r11)	# load svc arguments
408 409
	lghi	%r1,0			# svc 0 returns -ENOSYS
	j	.Lsysc_do_svc
L
Linus Torvalds 已提交
410

M
Martin Schwidefsky 已提交
411 412 413
#
# _TIF_NOTIFY_RESUME is set, call do_notify_resume
#
414
.Lsysc_notify_resume:
415
	lgr	%r2,%r11		# pass pointer to pt_regs
416
	larl	%r14,.Lsysc_return
417
	jg	do_notify_resume
M
Martin Schwidefsky 已提交
418

419 420 421 422
#
# _TIF_UPROBE is set, call uprobe_notify_resume
#
#ifdef CONFIG_UPROBES
423
.Lsysc_uprobe_notify:
424
	lgr	%r2,%r11		# pass pointer to pt_regs
425
	larl	%r14,.Lsysc_return
426 427 428
	jg	uprobe_notify_resume
#endif

429 430 431 432 433 434 435
#
# _TIF_GUARDED_STORAGE is set, call guarded_storage_load
#
.Lsysc_guarded_storage:
	lgr	%r2,%r11		# pass pointer to pt_regs
	larl	%r14,.Lsysc_return
	jg	gs_load_bc_cb
436 437 438 439 440 441 442 443 444
#
# _TIF_PATCH_PENDING is set, call klp_update_patch_state
#
#ifdef CONFIG_LIVEPATCH
.Lsysc_patch_pending:
	lg	%r2,__LC_CURRENT	# pass pointer to task struct
	larl	%r14,.Lsysc_return
	jg	klp_update_patch_state
#endif
445

L
Linus Torvalds 已提交
446
#
447
# _PIF_PER_TRAP is set, call do_per_trap
L
Linus Torvalds 已提交
448
#
449
.Lsysc_singlestep:
450
	ni	__PT_FLAGS+7(%r11),255-_PIF_PER_TRAP
451
	lgr	%r2,%r11		# pass pointer to pt_regs
452
	larl	%r14,.Lsysc_return
M
Martin Schwidefsky 已提交
453
	jg	do_per_trap
L
Linus Torvalds 已提交
454

455 456 457 458 459 460 461 462 463
#
# _PIF_SYSCALL_RESTART is set, repeat the current system call
#
.Lsysc_syscall_restart:
	ni	__PT_FLAGS+7(%r11),255-_PIF_SYSCALL_RESTART
	lmg	%r1,%r7,__PT_R1(%r11)	# load svc arguments
	lg	%r2,__PT_ORIG_GPR2(%r11)
	j	.Lsysc_do_svc

L
Linus Torvalds 已提交
464
#
M
Martin Schwidefsky 已提交
465 466
# call tracehook_report_syscall_entry/tracehook_report_syscall_exit before
# and after the system call
L
Linus Torvalds 已提交
467
#
468
.Lsysc_tracesys:
469
	lgr	%r2,%r11		# pass pointer to pt_regs
L
Linus Torvalds 已提交
470
	la	%r3,0
M
Martin Schwidefsky 已提交
471
	llgh	%r0,__PT_INT_CODE+2(%r11)
472
	stg	%r0,__PT_R2(%r11)
M
Martin Schwidefsky 已提交
473
	brasl	%r14,do_syscall_trace_enter
L
Linus Torvalds 已提交
474
	lghi	%r0,NR_syscalls
M
Martin Schwidefsky 已提交
475
	clgr	%r0,%r2
476
	jnh	.Lsysc_tracenogo
477 478
	sllg	%r8,%r2,2
	lgf	%r9,0(%r8,%r10)
479
.Lsysc_tracego:
480 481 482 483 484
	lmg	%r3,%r7,__PT_R3(%r11)
	stg	%r7,STACK_FRAME_OVERHEAD(%r15)
	lg	%r2,__PT_ORIG_GPR2(%r11)
	basr	%r14,%r9		# call sys_xxx
	stg	%r2,__PT_R2(%r11)	# store return value
485
.Lsysc_tracenogo:
486
	TSTMSK	__TI_flags(%r12),_TIF_TRACE
487
	jz	.Lsysc_return
488
	lgr	%r2,%r11		# pass pointer to pt_regs
489
	larl	%r14,.Lsysc_return
M
Martin Schwidefsky 已提交
490
	jg	do_syscall_trace_exit
L
Linus Torvalds 已提交
491 492 493 494

#
# a new process exits the kernel with ret_from_fork
#
495
ENTRY(ret_from_fork)
496
	la	%r11,STACK_FRAME_OVERHEAD(%r15)
497
	lg	%r12,__LC_CURRENT
498 499 500
	brasl	%r14,schedule_tail
	TRACE_IRQS_ON
	ssm	__LC_SVC_NEW_PSW	# reenable interrupts
501
	tm	__PT_PSW+1(%r11),0x01	# forking a kernel thread ?
502
	jne	.Lsysc_tracenogo
503 504
	# it's a kernel thread
	lmg	%r9,%r10,__PT_R9(%r11)	# load gprs
505 506 507
ENTRY(kernel_thread_starter)
	la	%r2,0(%r10)
	basr	%r14,%r9
508
	j	.Lsysc_tracenogo
L
Linus Torvalds 已提交
509 510 511 512 513

/*
 * Program check handler routine
 */

514
ENTRY(pgm_check_handler)
M
Martin Schwidefsky 已提交
515
	stpt	__LC_SYNC_ENTER_TIMER
516 517
	stmg	%r8,%r15,__LC_SAVE_AREA_SYNC
	lg	%r10,__LC_LAST_BREAK
518
	lg	%r12,__LC_CURRENT
519
	larl	%r13,cleanup_critical
520 521
	lmg	%r8,%r9,__LC_PGM_OLD_PSW
	tmhh	%r8,0x0001		# test problem state bit
522 523 524 525 526 527 528 529 530 531 532
	jnz	2f			# -> fault in user space
#if IS_ENABLED(CONFIG_KVM)
	# cleanup critical section for sie64a
	lgr	%r14,%r9
	slg	%r14,BASED(.Lsie_critical_start)
	clg	%r14,BASED(.Lsie_critical_length)
	jhe	0f
	brasl	%r14,.Lcleanup_sie
#endif
0:	tmhh	%r8,0x4000		# PER bit set in old PSW ?
	jnz	1f			# -> enabled, can't be a double fault
533
	tm	__LC_PGM_ILC+3,0x80	# check for per exception
534
	jnz	.Lpgm_svcper		# -> single stepped svc
535
1:	CHECK_STACK STACK_SIZE,__LC_SAVE_AREA_SYNC
536
	aghi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
537
	j	4f
538
2:	UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER
539
	lg	%r15,__LC_KERNEL_STACK
540
	lgr	%r14,%r12
541
	aghi	%r14,__TASK_thread	# pointer to thread_struct
542 543
	lghi	%r13,__LC_PGM_TDB
	tm	__LC_PGM_ILC+2,0x02	# check for transaction abort
544
	jz	3f
545
	mvc	__THREAD_trap_tdb(256,%r14),0(%r13)
546 547
3:	stg	%r10,__THREAD_last_break(%r14)
4:	la	%r11,STACK_FRAME_OVERHEAD(%r15)
548 549 550
	stmg	%r0,%r7,__PT_R0(%r11)
	mvc	__PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
	stmg	%r8,%r9,__PT_PSW(%r11)
M
Martin Schwidefsky 已提交
551 552
	mvc	__PT_INT_CODE(4,%r11),__LC_PGM_ILC
	mvc	__PT_INT_PARM_LONG(8,%r11),__LC_TRANS_EXC_CODE
553
	xc	__PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
554 555
	stg	%r10,__PT_ARGS(%r11)
	tm	__LC_PGM_ILC+3,0x80	# check for per exception
556
	jz	5f
557
	tmhh	%r8,0x0001		# kernel per event ?
558
	jz	.Lpgm_kprobe
559
	oi	__PT_FLAGS+7(%r11),_PIF_PER_TRAP
560
	mvc	__THREAD_per_address(8,%r14),__LC_PER_ADDRESS
561 562
	mvc	__THREAD_per_cause(2,%r14),__LC_PER_CODE
	mvc	__THREAD_per_paid(1,%r14),__LC_PER_ACCESS_ID
563
5:	REENABLE_IRQS
564
	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
565
	larl	%r1,pgm_check_table
M
Martin Schwidefsky 已提交
566 567
	llgh	%r10,__PT_INT_CODE+2(%r11)
	nill	%r10,0x007f
568
	sll	%r10,2
569
	je	.Lpgm_return
570
	lgf	%r1,0(%r10,%r1)		# load address of handler routine
571
	lgr	%r2,%r11		# pass pointer to pt_regs
572
	basr	%r14,%r1		# branch to interrupt-handler
573 574 575 576
.Lpgm_return:
	LOCKDEP_SYS_EXIT
	tm	__PT_PSW+1(%r11),0x01	# returning to user ?
	jno	.Lsysc_restore
577 578
	TSTMSK	__PT_FLAGS(%r11),_PIF_SYSCALL
	jo	.Lsysc_do_syscall
579
	j	.Lsysc_tif
L
Linus Torvalds 已提交
580 581

#
582
# PER event in supervisor state, must be kprobes
L
Linus Torvalds 已提交
583
#
584
.Lpgm_kprobe:
585 586 587 588
	REENABLE_IRQS
	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
	lgr	%r2,%r11		# pass pointer to pt_regs
	brasl	%r14,do_per_trap
589
	j	.Lpgm_return
L
Linus Torvalds 已提交
590

M
Michael Grundy 已提交
591
#
592
# single stepped system call
M
Michael Grundy 已提交
593
#
594
.Lpgm_svcper:
595
	mvc	__LC_RETURN_PSW(8),__LC_SVC_NEW_PSW
596
	lghi	%r13,__TASK_thread
597
	larl	%r14,.Lsysc_per
598
	stg	%r14,__LC_RETURN_PSW+8
599
	lghi	%r14,_PIF_SYSCALL | _PIF_PER_TRAP
600
	lpswe	__LC_RETURN_PSW		# branch to .Lsysc_per and enable irqs
M
Michael Grundy 已提交
601

L
Linus Torvalds 已提交
602 603 604
/*
 * IO interrupt handler routine
 */
605
ENTRY(io_int_handler)
606
	STCK	__LC_INT_CLOCK
607
	stpt	__LC_ASYNC_ENTER_TIMER
608
	stmg	%r8,%r15,__LC_SAVE_AREA_ASYNC
609
	lg	%r12,__LC_CURRENT
610
	larl	%r13,cleanup_critical
611
	lmg	%r8,%r9,__LC_IO_OLD_PSW
612
	SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER
613 614 615
	stmg	%r0,%r7,__PT_R0(%r11)
	mvc	__PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC
	stmg	%r8,%r9,__PT_PSW(%r11)
616
	mvc	__PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID
617
	xc	__PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
618 619
	TSTMSK	__LC_CPU_FLAGS,_CIF_IGNORE_IRQ
	jo	.Lio_restore
620
	TRACE_IRQS_OFF
621
	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
622
.Lio_loop:
623
	lgr	%r2,%r11		# pass pointer to pt_regs
624 625
	lghi	%r3,IO_INTERRUPT
	tm	__PT_INT_CODE+8(%r11),0x80	# adapter interrupt ?
626
	jz	.Lio_call
627
	lghi	%r3,THIN_INTERRUPT
628
.Lio_call:
629
	brasl	%r14,do_IRQ
630
	TSTMSK	__LC_MACHINE_FLAGS,MACHINE_FLAG_LPAR
631
	jz	.Lio_return
632
	tpi	0
633
	jz	.Lio_return
634
	mvc	__PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID
635 636
	j	.Lio_loop
.Lio_return:
637 638
	LOCKDEP_SYS_EXIT
	TRACE_IRQS_ON
639
.Lio_tif:
640
	TSTMSK	__TI_flags(%r12),_TIF_WORK
641
	jnz	.Lio_work		# there is work to do (signals etc.)
642
	TSTMSK	__LC_CPU_FLAGS,_CIF_WORK
643 644
	jnz	.Lio_work
.Lio_restore:
645 646 647
	lg	%r14,__LC_VDSO_PER_CPU
	lmg	%r0,%r10,__PT_R0(%r11)
	mvc	__LC_RETURN_PSW(16),__PT_PSW(%r11)
648
.Lio_exit_timer:
649 650 651 652
	stpt	__LC_EXIT_TIMER
	mvc	__VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
	lmg	%r11,%r15,__PT_R11(%r11)
	lpswe	__LC_RETURN_PSW
653
.Lio_done:
L
Linus Torvalds 已提交
654

655
#
656
# There is work todo, find out in which context we have been interrupted:
657
# 1) if we return to user space we can do all _TIF_WORK work
658 659 660 661 662
# 2) if we return to kernel code and kvm is enabled check if we need to
#    modify the psw to leave SIE
# 3) if we return to kernel code and preemptive scheduling is enabled check
#    the preemption counter and if it is zero call preempt_schedule_irq
# Before any work can be done, a switch to the kernel stack is required.
663
#
664
.Lio_work:
665
	tm	__PT_PSW+1(%r11),0x01	# returning to user ?
666
	jo	.Lio_work_user		# yes -> do resched & signal
667
#ifdef CONFIG_PREEMPT
668
	# check for preemptive scheduling
669
	icm	%r0,15,__LC_PREEMPT_COUNT
670
	jnz	.Lio_restore		# preemption is disabled
671
	TSTMSK	__TI_flags(%r12),_TIF_NEED_RESCHED
672
	jno	.Lio_restore
L
Linus Torvalds 已提交
673
	# switch to kernel stack
674 675 676 677 678
	lg	%r1,__PT_R15(%r11)
	aghi	%r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
	mvc	STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
	xc	__SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
	la	%r11,STACK_FRAME_OVERHEAD(%r1)
L
Linus Torvalds 已提交
679
	lgr	%r15,%r1
680
	# TRACE_IRQS_ON already done at .Lio_return, call
681 682 683
	# TRACE_IRQS_OFF to keep things symmetrical
	TRACE_IRQS_OFF
	brasl	%r14,preempt_schedule_irq
684
	j	.Lio_return
685
#else
686
	j	.Lio_restore
687
#endif
L
Linus Torvalds 已提交
688

689 690 691
#
# Need to do work before returning to userspace, switch to kernel stack
#
692
.Lio_work_user:
L
Linus Torvalds 已提交
693
	lg	%r1,__LC_KERNEL_STACK
694 695 696
	mvc	STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
	xc	__SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
	la	%r11,STACK_FRAME_OVERHEAD(%r1)
L
Linus Torvalds 已提交
697
	lgr	%r15,%r1
698

L
Linus Torvalds 已提交
699 700 701
#
# One of the work bits is on. Find out which one.
#
702
.Lio_work_tif:
703
	TSTMSK	__LC_CPU_FLAGS,_CIF_MCCK_PENDING
704
	jo	.Lio_mcck_pending
705
	TSTMSK	__TI_flags(%r12),_TIF_NEED_RESCHED
706
	jo	.Lio_reschedule
707 708 709 710
#ifdef CONFIG_LIVEPATCH
	TSTMSK	__TI_flags(%r12),_TIF_PATCH_PENDING
	jo	.Lio_patch_pending
#endif
711
	TSTMSK	__TI_flags(%r12),_TIF_SIGPENDING
712
	jo	.Lio_sigpending
713
	TSTMSK	__TI_flags(%r12),_TIF_NOTIFY_RESUME
714
	jo	.Lio_notify_resume
715 716
	TSTMSK	__TI_flags(%r12),_TIF_GUARDED_STORAGE
	jo	.Lio_guarded_storage
717
	TSTMSK	__LC_CPU_FLAGS,_CIF_FPU
718
	jo	.Lio_vxrs
719 720
	TSTMSK	__LC_CPU_FLAGS,(_CIF_ASCE_PRIMARY|_CIF_ASCE_SECONDARY)
	jnz	.Lio_asce
721
	j	.Lio_return		# beware of critical section cleanup
722

723
#
724
# _CIF_MCCK_PENDING is set, call handler
725
#
726 727
.Lio_mcck_pending:
	# TRACE_IRQS_ON already done at .Lio_return
H
Heiko Carstens 已提交
728
	brasl	%r14,s390_handle_mcck	# TIF bit will be cleared by handler
729
	TRACE_IRQS_OFF
730
	j	.Lio_return
731

732
#
733
# _CIF_ASCE_PRIMARY and/or CIF_ASCE_SECONDARY set, load user space asce
734
#
735
.Lio_asce:
736
	ni	__LC_CPU_FLAGS+7,255-_CIF_ASCE_PRIMARY
737
	lctlg	%c1,%c1,__LC_USER_ASCE		# load primary asce
738 739 740 741
	TSTMSK	__LC_CPU_FLAGS,_CIF_ASCE_SECONDARY
	jz	.Lio_return
	larl	%r14,.Lio_return
	jg	set_fs_fixup
742

743 744 745 746 747 748 749
#
# CIF_FPU is set, restore floating-point controls and floating-point registers.
#
.Lio_vxrs:
	larl	%r14,.Lio_return
	jg	load_fpu_regs

750 751 752 753 754 755 756 757 758 759 760 761
#
# _TIF_GUARDED_STORAGE is set, call guarded_storage_load
#
.Lio_guarded_storage:
	# TRACE_IRQS_ON already done at .Lio_return
	ssm	__LC_SVC_NEW_PSW	# reenable interrupts
	lgr	%r2,%r11		# pass pointer to pt_regs
	brasl	%r14,gs_load_bc_cb
	ssm	__LC_PGM_NEW_PSW	# disable I/O and ext. interrupts
	TRACE_IRQS_OFF
	j	.Lio_return

L
Linus Torvalds 已提交
762 763
#
# _TIF_NEED_RESCHED is set, call schedule
H
Heiko Carstens 已提交
764
#
765 766
.Lio_reschedule:
	# TRACE_IRQS_ON already done at .Lio_return
767
	ssm	__LC_SVC_NEW_PSW	# reenable interrupts
H
Heiko Carstens 已提交
768
	brasl	%r14,schedule		# call scheduler
769
	ssm	__LC_PGM_NEW_PSW	# disable I/O and ext. interrupts
770
	TRACE_IRQS_OFF
771
	j	.Lio_return
L
Linus Torvalds 已提交
772

773 774 775 776 777 778 779 780 781 782
#
# _TIF_PATCH_PENDING is set, call klp_update_patch_state
#
#ifdef CONFIG_LIVEPATCH
.Lio_patch_pending:
	lg	%r2,__LC_CURRENT	# pass pointer to task struct
	larl	%r14,.Lio_return
	jg	klp_update_patch_state
#endif

L
Linus Torvalds 已提交
783
#
784
# _TIF_SIGPENDING or is set, call do_signal
L
Linus Torvalds 已提交
785
#
786 787
.Lio_sigpending:
	# TRACE_IRQS_ON already done at .Lio_return
788 789 790 791
	ssm	__LC_SVC_NEW_PSW	# reenable interrupts
	lgr	%r2,%r11		# pass pointer to pt_regs
	brasl	%r14,do_signal
	ssm	__LC_PGM_NEW_PSW	# disable I/O and ext. interrupts
792
	TRACE_IRQS_OFF
793
	j	.Lio_return
L
Linus Torvalds 已提交
794

M
Martin Schwidefsky 已提交
795 796 797
#
# _TIF_NOTIFY_RESUME or is set, call do_notify_resume
#
798 799
.Lio_notify_resume:
	# TRACE_IRQS_ON already done at .Lio_return
800 801 802 803
	ssm	__LC_SVC_NEW_PSW	# reenable interrupts
	lgr	%r2,%r11		# pass pointer to pt_regs
	brasl	%r14,do_notify_resume
	ssm	__LC_PGM_NEW_PSW	# disable I/O and ext. interrupts
M
Martin Schwidefsky 已提交
804
	TRACE_IRQS_OFF
805
	j	.Lio_return
M
Martin Schwidefsky 已提交
806

L
Linus Torvalds 已提交
807 808 809
/*
 * External interrupt handler routine
 */
810
ENTRY(ext_int_handler)
811
	STCK	__LC_INT_CLOCK
812
	stpt	__LC_ASYNC_ENTER_TIMER
813
	stmg	%r8,%r15,__LC_SAVE_AREA_ASYNC
814
	lg	%r12,__LC_CURRENT
815
	larl	%r13,cleanup_critical
816
	lmg	%r8,%r9,__LC_EXT_OLD_PSW
817
	SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_ENTER_TIMER
818 819 820
	stmg	%r0,%r7,__PT_R0(%r11)
	mvc	__PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC
	stmg	%r8,%r9,__PT_PSW(%r11)
821 822 823 824
	lghi	%r1,__LC_EXT_PARAMS2
	mvc	__PT_INT_CODE(4,%r11),__LC_EXT_CPU_ADDR
	mvc	__PT_INT_PARM(4,%r11),__LC_EXT_PARAMS
	mvc	__PT_INT_PARM_LONG(8,%r11),0(%r1)
825
	xc	__PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
826 827
	TSTMSK	__LC_CPU_FLAGS,_CIF_IGNORE_IRQ
	jo	.Lio_restore
828
	TRACE_IRQS_OFF
829
	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
830
	lgr	%r2,%r11		# pass pointer to pt_regs
831 832
	lghi	%r3,EXT_INTERRUPT
	brasl	%r14,do_IRQ
833
	j	.Lio_return
L
Linus Torvalds 已提交
834

M
Martin Schwidefsky 已提交
835
/*
836
 * Load idle PSW. The second "half" of this function is in .Lcleanup_idle.
M
Martin Schwidefsky 已提交
837 838
 */
ENTRY(psw_idle)
839
	stg	%r3,__SF_EMPTY(%r15)
840
	larl	%r1,.Lpsw_idle_lpsw+4
M
Martin Schwidefsky 已提交
841
	stg	%r1,__SF_EMPTY+8(%r15)
842 843 844 845 846 847 848 849
#ifdef CONFIG_SMP
	larl	%r1,smp_cpu_mtid
	llgf	%r1,0(%r1)
	ltgr	%r1,%r1
	jz	.Lpsw_idle_stcctm
	.insn	rsy,0xeb0000000017,%r1,5,__SF_EMPTY+16(%r15)
.Lpsw_idle_stcctm:
#endif
850
	oi	__LC_CPU_FLAGS+7,_CIF_ENABLED_WAIT
851 852
	STCK	__CLOCK_IDLE_ENTER(%r2)
	stpt	__TIMER_IDLE_ENTER(%r2)
853
.Lpsw_idle_lpsw:
M
Martin Schwidefsky 已提交
854 855
	lpswe	__SF_EMPTY(%r15)
	br	%r14
856
.Lpsw_idle_end:
M
Martin Schwidefsky 已提交
857

858 859 860 861 862 863
/*
 * Store floating-point controls and floating-point or vector register
 * depending whether the vector facility is available.	A critical section
 * cleanup assures that the registers are stored even if interrupted for
 * some other work.  The CIF_FPU flag is set to trigger a lazy restore
 * of the register contents at return from io or a system call.
864 865
 */
ENTRY(save_fpu_regs)
866 867
	lg	%r2,__LC_CURRENT
	aghi	%r2,__TASK_thread
868
	TSTMSK	__LC_CPU_FLAGS,_CIF_FPU
869
	bor	%r14
870 871
	stfpc	__THREAD_FPU_fpc(%r2)
	lg	%r3,__THREAD_FPU_regs(%r2)
872
	TSTMSK	__LC_MACHINE_FLAGS,MACHINE_FLAG_VX
873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897
	jz	.Lsave_fpu_regs_fp	  # no -> store FP regs
	VSTM	%v0,%v15,0,%r3		  # vstm 0,15,0(3)
	VSTM	%v16,%v31,256,%r3	  # vstm 16,31,256(3)
	j	.Lsave_fpu_regs_done	  # -> set CIF_FPU flag
.Lsave_fpu_regs_fp:
	std	0,0(%r3)
	std	1,8(%r3)
	std	2,16(%r3)
	std	3,24(%r3)
	std	4,32(%r3)
	std	5,40(%r3)
	std	6,48(%r3)
	std	7,56(%r3)
	std	8,64(%r3)
	std	9,72(%r3)
	std	10,80(%r3)
	std	11,88(%r3)
	std	12,96(%r3)
	std	13,104(%r3)
	std	14,112(%r3)
	std	15,120(%r3)
.Lsave_fpu_regs_done:
	oi	__LC_CPU_FLAGS+7,_CIF_FPU
	br	%r14
.Lsave_fpu_regs_end:
A
Al Viro 已提交
898
EXPORT_SYMBOL(save_fpu_regs)
899

900 901 902 903
/*
 * Load floating-point controls and floating-point or vector registers.
 * A critical section cleanup assures that the register contents are
 * loaded even if interrupted for some other work.
904 905 906 907
 *
 * There are special calling conventions to fit into sysc and io return work:
 *	%r15:	<kernel stack>
 * The function requires:
908
 *	%r4
909 910
 */
load_fpu_regs:
911 912
	lg	%r4,__LC_CURRENT
	aghi	%r4,__TASK_thread
913
	TSTMSK	__LC_CPU_FLAGS,_CIF_FPU
914
	bnor	%r14
915
	lfpc	__THREAD_FPU_fpc(%r4)
916
	TSTMSK	__LC_MACHINE_FLAGS,MACHINE_FLAG_VX
917
	lg	%r4,__THREAD_FPU_regs(%r4)	# %r4 <- reg save area
918
	jz	.Lload_fpu_regs_fp		# -> no VX, load FP regs
919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943
	VLM	%v0,%v15,0,%r4
	VLM	%v16,%v31,256,%r4
	j	.Lload_fpu_regs_done
.Lload_fpu_regs_fp:
	ld	0,0(%r4)
	ld	1,8(%r4)
	ld	2,16(%r4)
	ld	3,24(%r4)
	ld	4,32(%r4)
	ld	5,40(%r4)
	ld	6,48(%r4)
	ld	7,56(%r4)
	ld	8,64(%r4)
	ld	9,72(%r4)
	ld	10,80(%r4)
	ld	11,88(%r4)
	ld	12,96(%r4)
	ld	13,104(%r4)
	ld	14,112(%r4)
	ld	15,120(%r4)
.Lload_fpu_regs_done:
	ni	__LC_CPU_FLAGS+7,255-_CIF_FPU
	br	%r14
.Lload_fpu_regs_end:

944
.L__critical_end:
945

L
Linus Torvalds 已提交
946 947 948
/*
 * Machine check handler routines
 */
949
ENTRY(mcck_int_handler)
950
	STCK	__LC_MCCK_CLOCK
951 952
	la	%r1,4095		# revalidate r1
	spt	__LC_CPU_TIMER_SAVE_AREA-4095(%r1)	# revalidate cpu timer
H
Heiko Carstens 已提交
953
	lmg	%r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r1)# revalidate gprs
954
	lg	%r12,__LC_CURRENT
955
	larl	%r13,cleanup_critical
956
	lmg	%r8,%r9,__LC_MCK_OLD_PSW
957
	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_SYSTEM_DAMAGE
958
	jo	.Lmcck_panic		# yes -> rest of mcck code invalid
959 960
	lghi	%r14,__LC_CPU_TIMER_SAVE_AREA
	mvc	__LC_MCCK_ENTER_TIMER(8),0(%r14)
961
	TSTMSK	__LC_MCCK_CODE,MCCK_CODE_CPU_TIMER_VALID
962
	jo	3f
963 964 965 966 967
	la	%r14,__LC_SYNC_ENTER_TIMER
	clc	0(8,%r14),__LC_ASYNC_ENTER_TIMER
	jl	0f
	la	%r14,__LC_ASYNC_ENTER_TIMER
0:	clc	0(8,%r14),__LC_EXIT_TIMER
968
	jl	1f
969
	la	%r14,__LC_EXIT_TIMER
970 971
1:	clc	0(8,%r14),__LC_LAST_UPDATE_TIMER
	jl	2f
972
	la	%r14,__LC_LAST_UPDATE_TIMER
973
2:	spt	0(%r14)
974
	mvc	__LC_MCCK_ENTER_TIMER(8),0(%r14)
975
3:	TSTMSK	__LC_MCCK_CODE,(MCCK_CODE_PSW_MWP_VALID|MCCK_CODE_PSW_IA_VALID)
976
	jno	.Lmcck_panic		# no -> skip cleanup critical
977
	SWITCH_ASYNC __LC_GPREGS_SAVE_AREA+64,__LC_MCCK_ENTER_TIMER
978
.Lmcck_skip:
979 980 981
	lghi	%r14,__LC_GPREGS_SAVE_AREA+64
	stmg	%r0,%r7,__PT_R0(%r11)
	mvc	__PT_R8(64,%r11),0(%r14)
982
	stmg	%r8,%r9,__PT_PSW(%r11)
983
	xc	__PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
984 985
	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
	lgr	%r2,%r11		# pass pointer to pt_regs
986
	brasl	%r14,s390_do_machine_check
987
	tm	__PT_PSW+1(%r11),0x01	# returning to user ?
988
	jno	.Lmcck_return
989
	lg	%r1,__LC_KERNEL_STACK	# switch to kernel stack
990 991 992
	mvc	STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
	xc	__SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
	la	%r11,STACK_FRAME_OVERHEAD(%r1)
993
	lgr	%r15,%r1
994
	ssm	__LC_PGM_NEW_PSW	# turn dat on, keep irqs off
995
	TSTMSK	__LC_CPU_FLAGS,_CIF_MCCK_PENDING
996
	jno	.Lmcck_return
997
	TRACE_IRQS_OFF
998
	brasl	%r14,s390_handle_mcck
999
	TRACE_IRQS_ON
1000
.Lmcck_return:
1001 1002 1003
	lg	%r14,__LC_VDSO_PER_CPU
	lmg	%r0,%r10,__PT_R0(%r11)
	mvc	__LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW
1004 1005 1006
	tm	__LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
	jno	0f
	stpt	__LC_EXIT_TIMER
1007 1008 1009 1010
	mvc	__VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
0:	lmg	%r11,%r15,__PT_R11(%r11)
	lpswe	__LC_RETURN_MCCK_PSW

1011
.Lmcck_panic:
1012
	lg	%r15,__LC_PANIC_STACK
1013
	la	%r11,STACK_FRAME_OVERHEAD(%r15)
1014
	j	.Lmcck_skip
L
Linus Torvalds 已提交
1015

1016 1017 1018
#
# PSW restart interrupt handler
#
M
Martin Schwidefsky 已提交
1019
ENTRY(restart_int_handler)
1020 1021 1022 1023
	TSTMSK	__LC_MACHINE_FLAGS,MACHINE_FLAG_LPP
	jz	0f
	.insn	s,0xb2800000,__LC_LPP
0:	stg	%r15,__LC_SAVE_AREA_RESTART
M
Martin Schwidefsky 已提交
1024
	lg	%r15,__LC_RESTART_STACK
1025
	aghi	%r15,-__PT_SIZE			# create pt_regs on stack
M
Martin Schwidefsky 已提交
1026
	xc	0(__PT_SIZE,%r15),0(%r15)
1027 1028 1029
	stmg	%r0,%r14,__PT_R0(%r15)
	mvc	__PT_R15(8,%r15),__LC_SAVE_AREA_RESTART
	mvc	__PT_PSW(16,%r15),__LC_RST_OLD_PSW # store restart old psw
M
Martin Schwidefsky 已提交
1030 1031
	aghi	%r15,-STACK_FRAME_OVERHEAD	# create stack frame on stack
	xc	0(STACK_FRAME_OVERHEAD,%r15),0(%r15)
1032 1033 1034
	lg	%r1,__LC_RESTART_FN		# load fn, parm & source cpu
	lg	%r2,__LC_RESTART_DATA
	lg	%r3,__LC_RESTART_SOURCE
M
Martin Schwidefsky 已提交
1035 1036
	ltgr	%r3,%r3				# test source cpu address
	jm	1f				# negative -> skip source stop
1037
0:	sigp	%r4,%r3,SIGP_SENSE		# sigp sense to source cpu
M
Martin Schwidefsky 已提交
1038 1039 1040 1041
	brc	10,0b				# wait for status stored
1:	basr	%r14,%r1			# call function
	stap	__SF_EMPTY(%r15)		# store cpu address
	llgh	%r3,__SF_EMPTY(%r15)
1042
2:	sigp	%r4,%r3,SIGP_STOP		# sigp stop to current cpu
M
Martin Schwidefsky 已提交
1043 1044
	brc	2,2b
3:	j	3b
1045

1046 1047
	.section .kprobes.text, "ax"

L
Linus Torvalds 已提交
1048 1049 1050 1051 1052 1053 1054
#ifdef CONFIG_CHECK_STACK
/*
 * The synchronous or the asynchronous stack overflowed. We are dead.
 * No need to properly save the registers, we are going to panic anyway.
 * Setup a pt_regs so that show_trace can provide a good call trace.
 */
stack_overflow:
1055 1056
	lg	%r15,__LC_PANIC_STACK	# change to panic stack
	la	%r11,STACK_FRAME_OVERHEAD(%r15)
1057 1058 1059 1060 1061 1062
	stmg	%r0,%r7,__PT_R0(%r11)
	stmg	%r8,%r9,__PT_PSW(%r11)
	mvc	__PT_R8(64,%r11),0(%r14)
	stg	%r10,__PT_ORIG_GPR2(%r11) # store last break to orig_gpr2
	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
	lgr	%r2,%r11		# pass pointer to pt_regs
L
Linus Torvalds 已提交
1063 1064 1065 1066
	jg	kernel_stack_overflow
#endif

cleanup_critical:
1067 1068 1069 1070 1071 1072
#if IS_ENABLED(CONFIG_KVM)
	clg	%r9,BASED(.Lcleanup_table_sie)	# .Lsie_gmap
	jl	0f
	clg	%r9,BASED(.Lcleanup_table_sie+8)# .Lsie_done
	jl	.Lcleanup_sie
#endif
1073
	clg	%r9,BASED(.Lcleanup_table)	# system_call
L
Linus Torvalds 已提交
1074
	jl	0f
1075 1076 1077
	clg	%r9,BASED(.Lcleanup_table+8)	# .Lsysc_do_svc
	jl	.Lcleanup_system_call
	clg	%r9,BASED(.Lcleanup_table+16)	# .Lsysc_tif
L
Linus Torvalds 已提交
1078
	jl	0f
1079 1080 1081 1082 1083
	clg	%r9,BASED(.Lcleanup_table+24)	# .Lsysc_restore
	jl	.Lcleanup_sysc_tif
	clg	%r9,BASED(.Lcleanup_table+32)	# .Lsysc_done
	jl	.Lcleanup_sysc_restore
	clg	%r9,BASED(.Lcleanup_table+40)	# .Lio_tif
1084
	jl	0f
1085 1086 1087 1088 1089
	clg	%r9,BASED(.Lcleanup_table+48)	# .Lio_restore
	jl	.Lcleanup_io_tif
	clg	%r9,BASED(.Lcleanup_table+56)	# .Lio_done
	jl	.Lcleanup_io_restore
	clg	%r9,BASED(.Lcleanup_table+64)	# psw_idle
M
Martin Schwidefsky 已提交
1090
	jl	0f
1091 1092
	clg	%r9,BASED(.Lcleanup_table+72)	# .Lpsw_idle_end
	jl	.Lcleanup_idle
1093 1094 1095 1096 1097 1098 1099 1100
	clg	%r9,BASED(.Lcleanup_table+80)	# save_fpu_regs
	jl	0f
	clg	%r9,BASED(.Lcleanup_table+88)	# .Lsave_fpu_regs_end
	jl	.Lcleanup_save_fpu_regs
	clg	%r9,BASED(.Lcleanup_table+96)	# load_fpu_regs
	jl	0f
	clg	%r9,BASED(.Lcleanup_table+104)	# .Lload_fpu_regs_end
	jl	.Lcleanup_load_fpu_regs
1101 1102
0:	br	%r14

1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125
	.align	8
.Lcleanup_table:
	.quad	system_call
	.quad	.Lsysc_do_svc
	.quad	.Lsysc_tif
	.quad	.Lsysc_restore
	.quad	.Lsysc_done
	.quad	.Lio_tif
	.quad	.Lio_restore
	.quad	.Lio_done
	.quad	psw_idle
	.quad	.Lpsw_idle_end
	.quad	save_fpu_regs
	.quad	.Lsave_fpu_regs_end
	.quad	load_fpu_regs
	.quad	.Lload_fpu_regs_end

#if IS_ENABLED(CONFIG_KVM)
.Lcleanup_table_sie:
	.quad	.Lsie_gmap
	.quad	.Lsie_done

.Lcleanup_sie:
1126 1127 1128 1129 1130 1131 1132
	cghi    %r11,__LC_SAVE_AREA_ASYNC 	#Is this in normal interrupt?
	je      1f
	slg     %r9,BASED(.Lsie_crit_mcck_start)
	clg     %r9,BASED(.Lsie_crit_mcck_length)
	jh      1f
	oi      __LC_CPU_FLAGS+7, _CIF_MCCK_GUEST
1:	lg	%r9,__SF_EMPTY(%r15)		# get control block pointer
1133
	ni	__SIE_PROG0C+3(%r9),0xfe	# no longer in SIE
1134 1135 1136 1137
	lctlg	%c1,%c1,__LC_USER_ASCE		# load primary asce
	larl	%r9,sie_exit			# skip forward to sie_exit
	br	%r14
#endif
L
Linus Torvalds 已提交
1138

1139
.Lcleanup_system_call:
1140
	# check if stpt has been executed
1141
	clg	%r9,BASED(.Lcleanup_system_call_insn)
L
Linus Torvalds 已提交
1142 1143
	jh	0f
	mvc	__LC_SYNC_ENTER_TIMER(8),__LC_ASYNC_ENTER_TIMER
1144
	cghi	%r11,__LC_SAVE_AREA_ASYNC
1145
	je	0f
1146 1147
	mvc	__LC_SYNC_ENTER_TIMER(8),__LC_MCCK_ENTER_TIMER
0:	# check if stmg has been executed
1148
	clg	%r9,BASED(.Lcleanup_system_call_insn+8)
L
Linus Torvalds 已提交
1149
	jh	0f
1150 1151
	mvc	__LC_SAVE_AREA_SYNC(64),0(%r11)
0:	# check if base register setup + TIF bit load has been done
1152
	clg	%r9,BASED(.Lcleanup_system_call_insn+16)
1153
	jhe	0f
1154 1155 1156 1157
	# set up saved register r12 task struct pointer
	stg	%r12,32(%r11)
	# set up saved register r13 __TASK_thread offset
	mvc	40(8,%r11),BASED(.Lcleanup_system_call_const)
1158
0:	# check if the user time update has been done
1159
	clg	%r9,BASED(.Lcleanup_system_call_insn+24)
1160 1161 1162 1163 1164 1165
	jh	0f
	lg	%r15,__LC_EXIT_TIMER
	slg	%r15,__LC_SYNC_ENTER_TIMER
	alg	%r15,__LC_USER_TIMER
	stg	%r15,__LC_USER_TIMER
0:	# check if the system time update has been done
1166
	clg	%r9,BASED(.Lcleanup_system_call_insn+32)
1167 1168 1169 1170 1171 1172
	jh	0f
	lg	%r15,__LC_LAST_UPDATE_TIMER
	slg	%r15,__LC_EXIT_TIMER
	alg	%r15,__LC_SYSTEM_TIMER
	stg	%r15,__LC_SYSTEM_TIMER
0:	# update accounting time stamp
L
Linus Torvalds 已提交
1173
	mvc	__LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
1174
	# set up saved register r11
1175
	lg	%r15,__LC_KERNEL_STACK
1176 1177
	la	%r9,STACK_FRAME_OVERHEAD(%r15)
	stg	%r9,24(%r11)		# r11 pt_regs pointer
1178
	# fill pt_regs
1179 1180 1181 1182
	mvc	__PT_R8(64,%r9),__LC_SAVE_AREA_SYNC
	stmg	%r0,%r7,__PT_R0(%r9)
	mvc	__PT_PSW(16,%r9),__LC_SVC_OLD_PSW
	mvc	__PT_INT_CODE(4,%r9),__LC_SVC_ILC
1183 1184
	xc	__PT_FLAGS(8,%r9),__PT_FLAGS(%r9)
	mvi	__PT_FLAGS+7(%r9),_PIF_SYSCALL
1185 1186 1187
	# setup saved register r15
	stg	%r15,56(%r11)		# r15 stack pointer
	# set new psw address and exit
1188
	larl	%r9,.Lsysc_do_svc
L
Linus Torvalds 已提交
1189
	br	%r14
1190
.Lcleanup_system_call_insn:
H
Heiko Carstens 已提交
1191
	.quad	system_call
1192 1193
	.quad	.Lsysc_stmg
	.quad	.Lsysc_per
1194
	.quad	.Lsysc_vtime+36
1195
	.quad	.Lsysc_vtime+42
1196 1197
.Lcleanup_system_call_const:
	.quad	__TASK_thread
L
Linus Torvalds 已提交
1198

1199 1200
.Lcleanup_sysc_tif:
	larl	%r9,.Lsysc_tif
L
Linus Torvalds 已提交
1201 1202
	br	%r14

1203
.Lcleanup_sysc_restore:
1204
	# check if stpt has been executed
1205
	clg	%r9,BASED(.Lcleanup_sysc_restore_insn)
1206 1207 1208
	jh	0f
	mvc	__LC_EXIT_TIMER(8),__LC_ASYNC_ENTER_TIMER
	cghi	%r11,__LC_SAVE_AREA_ASYNC
1209
	je	0f
1210 1211 1212
	mvc	__LC_EXIT_TIMER(8),__LC_MCCK_ENTER_TIMER
0:	clg	%r9,BASED(.Lcleanup_sysc_restore_insn+8)
	je	1f
1213 1214 1215 1216
	lg	%r9,24(%r11)		# get saved pointer to pt_regs
	mvc	__LC_RETURN_PSW(16),__PT_PSW(%r9)
	mvc	0(64,%r11),__PT_R8(%r9)
	lmg	%r0,%r7,__PT_R0(%r9)
1217
1:	lmg	%r8,%r9,__LC_RETURN_PSW
L
Linus Torvalds 已提交
1218
	br	%r14
1219
.Lcleanup_sysc_restore_insn:
1220
	.quad	.Lsysc_exit_timer
1221
	.quad	.Lsysc_done - 4
L
Linus Torvalds 已提交
1222

1223 1224
.Lcleanup_io_tif:
	larl	%r9,.Lio_tif
1225 1226
	br	%r14

1227
.Lcleanup_io_restore:
1228
	# check if stpt has been executed
1229
	clg	%r9,BASED(.Lcleanup_io_restore_insn)
1230 1231 1232 1233
	jh	0f
	mvc	__LC_EXIT_TIMER(8),__LC_MCCK_ENTER_TIMER
0:	clg	%r9,BASED(.Lcleanup_io_restore_insn+8)
	je	1f
1234 1235 1236 1237
	lg	%r9,24(%r11)		# get saved r11 pointer to pt_regs
	mvc	__LC_RETURN_PSW(16),__PT_PSW(%r9)
	mvc	0(64,%r11),__PT_R8(%r9)
	lmg	%r0,%r7,__PT_R0(%r9)
1238
1:	lmg	%r8,%r9,__LC_RETURN_PSW
1239
	br	%r14
1240
.Lcleanup_io_restore_insn:
1241
	.quad	.Lio_exit_timer
1242
	.quad	.Lio_done - 4
1243

1244
.Lcleanup_idle:
1245
	ni	__LC_CPU_FLAGS+7,255-_CIF_ENABLED_WAIT
M
Martin Schwidefsky 已提交
1246
	# copy interrupt clock & cpu timer
1247 1248
	mvc	__CLOCK_IDLE_EXIT(8,%r2),__LC_INT_CLOCK
	mvc	__TIMER_IDLE_EXIT(8,%r2),__LC_ASYNC_ENTER_TIMER
M
Martin Schwidefsky 已提交
1249 1250
	cghi	%r11,__LC_SAVE_AREA_ASYNC
	je	0f
1251 1252
	mvc	__CLOCK_IDLE_EXIT(8,%r2),__LC_MCCK_CLOCK
	mvc	__TIMER_IDLE_EXIT(8,%r2),__LC_MCCK_ENTER_TIMER
M
Martin Schwidefsky 已提交
1253
0:	# check if stck & stpt have been executed
1254
	clg	%r9,BASED(.Lcleanup_idle_insn)
M
Martin Schwidefsky 已提交
1255
	jhe	1f
1256 1257
	mvc	__CLOCK_IDLE_ENTER(8,%r2),__CLOCK_IDLE_EXIT(%r2)
	mvc	__TIMER_IDLE_ENTER(8,%r2),__TIMER_IDLE_EXIT(%r2)
1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278
1:	# calculate idle cycles
#ifdef CONFIG_SMP
	clg	%r9,BASED(.Lcleanup_idle_insn)
	jl	3f
	larl	%r1,smp_cpu_mtid
	llgf	%r1,0(%r1)
	ltgr	%r1,%r1
	jz	3f
	.insn	rsy,0xeb0000000017,%r1,5,__SF_EMPTY+80(%r15)
	larl	%r3,mt_cycles
	ag	%r3,__LC_PERCPU_OFFSET
	la	%r4,__SF_EMPTY+16(%r15)
2:	lg	%r0,0(%r3)
	slg	%r0,0(%r4)
	alg	%r0,64(%r4)
	stg	%r0,0(%r3)
	la	%r3,8(%r3)
	la	%r4,8(%r4)
	brct	%r1,2b
#endif
3:	# account system time going idle
M
Martin Schwidefsky 已提交
1279
	lg	%r9,__LC_STEAL_TIMER
1280
	alg	%r9,__CLOCK_IDLE_ENTER(%r2)
M
Martin Schwidefsky 已提交
1281 1282
	slg	%r9,__LC_LAST_UPDATE_CLOCK
	stg	%r9,__LC_STEAL_TIMER
1283
	mvc	__LC_LAST_UPDATE_CLOCK(8),__CLOCK_IDLE_EXIT(%r2)
M
Martin Schwidefsky 已提交
1284 1285
	lg	%r9,__LC_SYSTEM_TIMER
	alg	%r9,__LC_LAST_UPDATE_TIMER
1286
	slg	%r9,__TIMER_IDLE_ENTER(%r2)
M
Martin Schwidefsky 已提交
1287
	stg	%r9,__LC_SYSTEM_TIMER
1288
	mvc	__LC_LAST_UPDATE_TIMER(8),__TIMER_IDLE_EXIT(%r2)
M
Martin Schwidefsky 已提交
1289
	# prepare return psw
1290
	nihh	%r8,0xfcfd		# clear irq & wait state bits
M
Martin Schwidefsky 已提交
1291 1292
	lg	%r9,48(%r11)		# return from psw_idle
	br	%r14
1293 1294
.Lcleanup_idle_insn:
	.quad	.Lpsw_idle_lpsw
M
Martin Schwidefsky 已提交
1295

1296
.Lcleanup_save_fpu_regs:
1297
	larl	%r9,save_fpu_regs
1298 1299 1300
	br	%r14

.Lcleanup_load_fpu_regs:
1301
	larl	%r9,load_fpu_regs
1302 1303
	br	%r14

L
Linus Torvalds 已提交
1304 1305 1306
/*
 * Integer constants
 */
1307
	.align	8
L
Linus Torvalds 已提交
1308
.Lcritical_start:
1309
	.quad	.L__critical_start
1310
.Lcritical_length:
1311
	.quad	.L__critical_end - .L__critical_start
1312
#if IS_ENABLED(CONFIG_KVM)
1313
.Lsie_critical_start:
1314
	.quad	.Lsie_gmap
1315
.Lsie_critical_length:
1316
	.quad	.Lsie_done - .Lsie_gmap
1317 1318 1319 1320
.Lsie_crit_mcck_start:
	.quad   .Lsie_entry
.Lsie_crit_mcck_length:
	.quad   .Lsie_skip - .Lsie_entry
1321 1322
#endif

H
Heiko Carstens 已提交
1323 1324
	.section .rodata, "a"
#define SYSCALL(esame,emu)	.long esame
1325
	.globl	sys_call_table
L
Linus Torvalds 已提交
1326 1327 1328 1329
sys_call_table:
#include "syscalls.S"
#undef SYSCALL

1330
#ifdef CONFIG_COMPAT
L
Linus Torvalds 已提交
1331

H
Heiko Carstens 已提交
1332
#define SYSCALL(esame,emu)	.long emu
1333
	.globl	sys_call_table_emu
L
Linus Torvalds 已提交
1334 1335 1336 1337
sys_call_table_emu:
#include "syscalls.S"
#undef SYSCALL
#endif