entry64.S 27.7 KB
Newer Older
L
Linus Torvalds 已提交
1
/*
2
 *  arch/s390/kernel/entry64.S
L
Linus Torvalds 已提交
3 4
 *    S390 low-level entry points.
 *
M
Martin Schwidefsky 已提交
5
 *    Copyright (C) IBM Corp. 1999,2012
L
Linus Torvalds 已提交
6
 *    Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
H
Heiko Carstens 已提交
7 8
 *		 Hartmut Penner (hp@de.ibm.com),
 *		 Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
9
 *		 Heiko Carstens <heiko.carstens@de.ibm.com>
L
Linus Torvalds 已提交
10 11
 */

12
#include <linux/init.h>
13
#include <linux/linkage.h>
L
Linus Torvalds 已提交
14 15 16 17
#include <asm/cache.h>
#include <asm/errno.h>
#include <asm/ptrace.h>
#include <asm/thread_info.h>
18
#include <asm/asm-offsets.h>
L
Linus Torvalds 已提交
19 20
#include <asm/unistd.h>
#include <asm/page.h>
21
#include <asm/sigp.h>
L
Linus Torvalds 已提交
22

23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38
__PT_R0      =	__PT_GPRS
__PT_R1      =	__PT_GPRS + 8
__PT_R2      =	__PT_GPRS + 16
__PT_R3      =	__PT_GPRS + 24
__PT_R4      =	__PT_GPRS + 32
__PT_R5      =	__PT_GPRS + 40
__PT_R6      =	__PT_GPRS + 48
__PT_R7      =	__PT_GPRS + 56
__PT_R8      =	__PT_GPRS + 64
__PT_R9      =	__PT_GPRS + 72
__PT_R10     =	__PT_GPRS + 80
__PT_R11     =	__PT_GPRS + 88
__PT_R12     =	__PT_GPRS + 96
__PT_R13     =	__PT_GPRS + 104
__PT_R14     =	__PT_GPRS + 112
__PT_R15     =	__PT_GPRS + 120
L
Linus Torvalds 已提交
39 40 41 42

STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER
STACK_SIZE  = 1 << STACK_SHIFT

M
Martin Schwidefsky 已提交
43
_TIF_WORK_SVC = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
44
		 _TIF_MCCK_PENDING | _TIF_PER_TRAP )
M
Martin Schwidefsky 已提交
45
_TIF_WORK_INT = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
46
		 _TIF_MCCK_PENDING)
47 48
_TIF_TRACE    = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \
		 _TIF_SYSCALL_TRACEPOINT)
49
_TIF_EXIT_SIE = (_TIF_SIGPENDING | _TIF_NEED_RESCHED | _TIF_MCCK_PENDING)
L
Linus Torvalds 已提交
50 51 52

#define BASED(name) name-system_call(%r13)

53
	.macro	TRACE_IRQS_ON
54
#ifdef CONFIG_TRACE_IRQFLAGS
55 56
	basr	%r2,%r0
	brasl	%r14,trace_hardirqs_on_caller
57
#endif
58 59 60
	.endm

	.macro	TRACE_IRQS_OFF
61
#ifdef CONFIG_TRACE_IRQFLAGS
62 63
	basr	%r2,%r0
	brasl	%r14,trace_hardirqs_off_caller
64
#endif
65
	.endm
66 67

	.macro	LOCKDEP_SYS_EXIT
68 69 70
#ifdef CONFIG_LOCKDEP
	tm	__PT_PSW+1(%r11),0x01	# returning to user ?
	jz	.+10
71
	brasl	%r14,lockdep_sys_exit
72
#endif
L
Linus Torvalds 已提交
73 74
	.endm

75 76 77 78 79 80 81
	.macro SPP newpp
#if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE)
	tm	__LC_MACHINE_FLAGS+6,0x20	# MACHINE_FLAG_SPP
	jz	.+8
	.insn	s,0xb2800000,\newpp
#endif
	.endm
L
Linus Torvalds 已提交
82

83 84
	.macro	HANDLE_SIE_INTERCEPT scratch
#if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE)
85 86
	tmhh	%r8,0x0001		# interrupting from user ?
	jnz	.+42
87 88 89
	lgr	\scratch,%r9
	slg	\scratch,BASED(.Lsie_loop)
	clg	\scratch,BASED(.Lsie_length)
90
	jhe	.+22
91
	lg	%r9,BASED(.Lsie_loop)
92
	SPP	BASED(.Lhost_id)	# set host id
93
#endif
94 95
	.endm

96
	.macro	CHECK_STACK stacksize,savearea
97
#ifdef CONFIG_CHECK_STACK
98 99 100
	tml	%r15,\stacksize - CONFIG_STACK_GUARD
	lghi	%r14,\savearea
	jz	stack_overflow
101 102 103
#endif
	.endm

104 105 106 107 108 109
	.macro	SWITCH_ASYNC savearea,stack,shift
	tmhh	%r8,0x0001		# interrupting from user ?
	jnz	1f
	lgr	%r14,%r9
	slg	%r14,BASED(.Lcritical_start)
	clg	%r14,BASED(.Lcritical_length)
L
Linus Torvalds 已提交
110
	jhe	0f
111
	lghi	%r11,\savearea		# inside critical section, do cleanup
L
Linus Torvalds 已提交
112
	brasl	%r14,cleanup_critical
113
	tmhh	%r8,0x0001		# retest problem state after cleanup
L
Linus Torvalds 已提交
114
	jnz	1f
115
0:	lg	%r14,\stack		# are we already on the target stack?
L
Linus Torvalds 已提交
116
	slgr	%r14,%r15
117
	srag	%r14,%r14,\shift
118
	jnz	1f
119 120 121 122 123
	CHECK_STACK 1<<\shift,\savearea
	j	2f
1:	lg	%r15,\stack		# load target stack
2:	aghi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
	la	%r11,STACK_FRAME_OVERHEAD(%r15)
H
Heiko Carstens 已提交
124
	.endm
L
Linus Torvalds 已提交
125

126 127 128 129 130 131 132 133 134 135
	.macro UPDATE_VTIME scratch,enter_timer
	lg	\scratch,__LC_EXIT_TIMER
	slg	\scratch,\enter_timer
	alg	\scratch,__LC_USER_TIMER
	stg	\scratch,__LC_USER_TIMER
	lg	\scratch,__LC_LAST_UPDATE_TIMER
	slg	\scratch,__LC_EXIT_TIMER
	alg	\scratch,__LC_SYSTEM_TIMER
	stg	\scratch,__LC_SYSTEM_TIMER
	mvc	__LC_LAST_UPDATE_TIMER(8),\enter_timer
L
Linus Torvalds 已提交
136 137
	.endm

138 139 140 141
	.macro	LAST_BREAK scratch
	srag	\scratch,%r10,23
	jz	.+10
	stg	%r10,__TI_last_break(%r12)
142 143
	.endm

144
	.macro REENABLE_IRQS
145 146 147
	stg	%r8,__LC_RETURN_PSW
	ni	__LC_RETURN_PSW,0xbf
	ssm	__LC_RETURN_PSW
148 149
	.endm

150
	.macro STCK savearea
151
#ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES
152 153 154 155 156 157
	.insn	s,0xb27c0000,\savearea		# store clock fast
#else
	.insn	s,0xb2050000,\savearea		# store clock
#endif
	.endm

158 159
	.section .kprobes.text, "ax"

L
Linus Torvalds 已提交
160 161 162 163 164 165 166
/*
 * Scheduler resume function, called by switch_to
 *  gpr2 = (task_struct *) prev
 *  gpr3 = (task_struct *) next
 * Returns:
 *  gpr2 = prev
 */
167
ENTRY(__switch_to)
168 169
	stmg	%r6,%r15,__SF_GPRS(%r15)	# store gprs of prev task
	stg	%r15,__THREAD_ksp(%r2)		# store kernel stack of prev
M
Martin Schwidefsky 已提交
170 171
	lg	%r4,__THREAD_info(%r2)		# get thread_info of prev
	lg	%r5,__THREAD_info(%r3)		# get thread_info of next
172 173 174 175 176 177 178 179
	lgr	%r15,%r5
	aghi	%r15,STACK_SIZE			# end of kernel stack of next
	stg	%r3,__LC_CURRENT		# store task struct of next
	stg	%r5,__LC_THREAD_INFO		# store thread info of next
	stg	%r15,__LC_KERNEL_STACK		# store end of kernel stack
	lctl	%c4,%c4,__TASK_pid(%r3)		# load pid to control reg. 4
	mvc	__LC_CURRENT_PID+4(4,%r0),__TASK_pid(%r3) # store pid of next
	lg	%r15,__THREAD_ksp(%r3)		# load kernel stack of next
180
	tm	__TI_flags+7(%r4),_TIF_MCCK_PENDING # machine check pending?
M
Martin Schwidefsky 已提交
181 182 183
	jz	0f
	ni	__TI_flags+7(%r4),255-_TIF_MCCK_PENDING	# clear flag in prev
	oi	__TI_flags+7(%r5),_TIF_MCCK_PENDING	# set it in next
184
0:	lmg	%r6,%r15,__SF_GPRS(%r15)	# load gprs of next task
L
Linus Torvalds 已提交
185 186 187 188 189 190 191 192
	br	%r14

__critical_start:
/*
 * SVC interrupt handler routine. System calls are synchronous events and
 * are executed with interrupts enabled.
 */

193
ENTRY(system_call)
M
Martin Schwidefsky 已提交
194
	stpt	__LC_SYNC_ENTER_TIMER
195 196 197 198 199 200 201 202 203
sysc_stmg:
	stmg	%r8,%r15,__LC_SAVE_AREA_SYNC
	lg	%r10,__LC_LAST_BREAK
	lg	%r12,__LC_THREAD_INFO
	larl	%r13,system_call
sysc_per:
	lg	%r15,__LC_KERNEL_STACK
	aghi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
	la	%r11,STACK_FRAME_OVERHEAD(%r15)	# pointer to pt_regs
L
Linus Torvalds 已提交
204
sysc_vtime:
205 206 207 208 209
	UPDATE_VTIME %r13,__LC_SYNC_ENTER_TIMER
	LAST_BREAK %r13
	stmg	%r0,%r7,__PT_R0(%r11)
	mvc	__PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
	mvc	__PT_PSW(16,%r11),__LC_SVC_OLD_PSW
M
Martin Schwidefsky 已提交
210
	mvc	__PT_INT_CODE(4,%r11),__LC_SVC_ILC
L
Linus Torvalds 已提交
211
sysc_do_svc:
212
	oi	__TI_flags+7(%r12),_TIF_SYSCALL
M
Martin Schwidefsky 已提交
213
	llgh	%r8,__PT_INT_CODE+2(%r11)
214
	slag	%r8,%r8,2			# shift and test for svc 0
L
Linus Torvalds 已提交
215 216
	jnz	sysc_nr_ok
	# svc 0: system call number in %r1
217
	llgfr	%r1,%r1				# clear high word in r1
218
	cghi	%r1,NR_syscalls
L
Linus Torvalds 已提交
219
	jnl	sysc_nr_ok
M
Martin Schwidefsky 已提交
220
	sth	%r1,__PT_INT_CODE+2(%r11)
221
	slag	%r8,%r1,2
L
Linus Torvalds 已提交
222
sysc_nr_ok:
223
	larl	%r10,sys_call_table		# 64 bit system call table
224
#ifdef CONFIG_COMPAT
225
	tm	__TI_flags+5(%r12),(_TIF_31BIT>>16)
226
	jno	sysc_noemu
227
	larl	%r10,sys_call_table_emu		# 31 bit system call table
L
Linus Torvalds 已提交
228 229
sysc_noemu:
#endif
230 231 232 233
	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
	stg	%r2,__PT_ORIG_GPR2(%r11)
	stg	%r7,STACK_FRAME_OVERHEAD(%r15)
	lgf	%r9,0(%r8,%r10)			# get system call add.
234
	tm	__TI_flags+6(%r12),_TIF_TRACE >> 8
H
Heiko Carstens 已提交
235
	jnz	sysc_tracesys
236 237
	basr	%r14,%r9			# call sys_xxxx
	stg	%r2,__PT_R2(%r11)		# store return value
L
Linus Torvalds 已提交
238 239

sysc_return:
240 241
	LOCKDEP_SYS_EXIT
sysc_tif:
242
	tm	__PT_PSW+1(%r11),0x01		# returning to user ?
243
	jno	sysc_restore
244
	tm	__TI_flags+7(%r12),_TIF_WORK_SVC
245
	jnz	sysc_work			# check for work
246
	ni	__TI_flags+7(%r12),255-_TIF_SYSCALL
247
sysc_restore:
248 249 250 251 252 253 254
	lg	%r14,__LC_VDSO_PER_CPU
	lmg	%r0,%r10,__PT_R0(%r11)
	mvc	__LC_RETURN_PSW(16),__PT_PSW(%r11)
	stpt	__LC_EXIT_TIMER
	mvc	__VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
	lmg	%r11,%r15,__PT_R11(%r11)
	lpswe	__LC_RETURN_PSW
255 256
sysc_done:

257 258 259
#
# One of the work bits is on. Find out which one.
#
260
sysc_work:
261
	tm	__TI_flags+7(%r12),_TIF_MCCK_PENDING
262
	jo	sysc_mcck_pending
263
	tm	__TI_flags+7(%r12),_TIF_NEED_RESCHED
L
Linus Torvalds 已提交
264
	jo	sysc_reschedule
265
	tm	__TI_flags+7(%r12),_TIF_SIGPENDING
266
	jo	sysc_sigpending
267
	tm	__TI_flags+7(%r12),_TIF_NOTIFY_RESUME
268
	jo	sysc_notify_resume
M
Martin Schwidefsky 已提交
269
	tm	__TI_flags+7(%r12),_TIF_PER_TRAP
L
Linus Torvalds 已提交
270
	jo	sysc_singlestep
271
	j	sysc_return		# beware of critical section cleanup
L
Linus Torvalds 已提交
272 273 274

#
# _TIF_NEED_RESCHED is set, call schedule
H
Heiko Carstens 已提交
275 276
#
sysc_reschedule:
277
	larl	%r14,sysc_return
278
	jg	schedule
L
Linus Torvalds 已提交
279

280 281 282 283
#
# _TIF_MCCK_PENDING is set, call handler
#
sysc_mcck_pending:
284
	larl	%r14,sysc_return
H
Heiko Carstens 已提交
285
	jg	s390_handle_mcck	# TIF bit will be cleared by handler
286

L
Linus Torvalds 已提交
287
#
288
# _TIF_SIGPENDING is set, call do_signal
L
Linus Torvalds 已提交
289
#
H
Heiko Carstens 已提交
290
sysc_sigpending:
M
Martin Schwidefsky 已提交
291
	ni	__TI_flags+7(%r12),255-_TIF_PER_TRAP # clear TIF_PER_TRAP
292 293
	lgr	%r2,%r11		# pass pointer to pt_regs
	brasl	%r14,do_signal
294 295
	tm	__TI_flags+7(%r12),_TIF_SYSCALL
	jno	sysc_return
296 297
	lmg	%r2,%r7,__PT_R2(%r11)	# load svc arguments
	lghi	%r8,0			# svc 0 returns -ENOSYS
M
Martin Schwidefsky 已提交
298
	lh	%r1,__PT_INT_CODE+2(%r11)	# load new svc number
299 300
	cghi	%r1,NR_syscalls
	jnl	sysc_nr_ok		# invalid svc number -> do svc 0
301
	slag	%r8,%r1,2
302
	j	sysc_nr_ok		# restart svc
L
Linus Torvalds 已提交
303

M
Martin Schwidefsky 已提交
304 305 306 307
#
# _TIF_NOTIFY_RESUME is set, call do_notify_resume
#
sysc_notify_resume:
308
	lgr	%r2,%r11		# pass pointer to pt_regs
309
	larl	%r14,sysc_return
310
	jg	do_notify_resume
M
Martin Schwidefsky 已提交
311

L
Linus Torvalds 已提交
312
#
M
Martin Schwidefsky 已提交
313
# _TIF_PER_TRAP is set, call do_per_trap
L
Linus Torvalds 已提交
314 315
#
sysc_singlestep:
316
	ni	__TI_flags+7(%r12),255-(_TIF_SYSCALL | _TIF_PER_TRAP)
317 318
	lgr	%r2,%r11		# pass pointer to pt_regs
	larl	%r14,sysc_return
M
Martin Schwidefsky 已提交
319
	jg	do_per_trap
L
Linus Torvalds 已提交
320 321

#
M
Martin Schwidefsky 已提交
322 323
# call tracehook_report_syscall_entry/tracehook_report_syscall_exit before
# and after the system call
L
Linus Torvalds 已提交
324 325
#
sysc_tracesys:
326
	lgr	%r2,%r11		# pass pointer to pt_regs
L
Linus Torvalds 已提交
327
	la	%r3,0
M
Martin Schwidefsky 已提交
328
	llgh	%r0,__PT_INT_CODE+2(%r11)
329
	stg	%r0,__PT_R2(%r11)
M
Martin Schwidefsky 已提交
330
	brasl	%r14,do_syscall_trace_enter
L
Linus Torvalds 已提交
331
	lghi	%r0,NR_syscalls
M
Martin Schwidefsky 已提交
332
	clgr	%r0,%r2
L
Linus Torvalds 已提交
333
	jnh	sysc_tracenogo
334 335
	sllg	%r8,%r2,2
	lgf	%r9,0(%r8,%r10)
L
Linus Torvalds 已提交
336
sysc_tracego:
337 338 339 340 341
	lmg	%r3,%r7,__PT_R3(%r11)
	stg	%r7,STACK_FRAME_OVERHEAD(%r15)
	lg	%r2,__PT_ORIG_GPR2(%r11)
	basr	%r14,%r9		# call sys_xxx
	stg	%r2,__PT_R2(%r11)	# store return value
L
Linus Torvalds 已提交
342
sysc_tracenogo:
343
	tm	__TI_flags+6(%r12),_TIF_TRACE >> 8
H
Heiko Carstens 已提交
344
	jz	sysc_return
345 346
	lgr	%r2,%r11		# pass pointer to pt_regs
	larl	%r14,sysc_return
M
Martin Schwidefsky 已提交
347
	jg	do_syscall_trace_exit
L
Linus Torvalds 已提交
348 349 350 351

#
# a new process exits the kernel with ret_from_fork
#
352
ENTRY(ret_from_fork)
353 354 355
	la	%r11,STACK_FRAME_OVERHEAD(%r15)
	lg	%r12,__LC_THREAD_INFO
	tm	__PT_PSW+1(%r11),0x01	# forking a kernel thread ?
L
Linus Torvalds 已提交
356
	jo	0f
357
	stg	%r15,__PT_R15(%r11)	# store stack pointer for new kthread
H
Heiko Carstens 已提交
358
0:	brasl	%r14,schedule_tail
359
	TRACE_IRQS_ON
360
	ssm	__LC_SVC_NEW_PSW	# reenable interrupts
361
	j	sysc_tracenogo
L
Linus Torvalds 已提交
362 363

#
M
Martin Schwidefsky 已提交
364 365
# kernel_execve function needs to deal with pt_regs that is not
# at the usual place
L
Linus Torvalds 已提交
366
#
367
ENTRY(kernel_execve)
M
Martin Schwidefsky 已提交
368 369
	stmg	%r12,%r15,96(%r15)
	lgr	%r14,%r15
370
	aghi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
M
Martin Schwidefsky 已提交
371
	stg	%r14,__SF_BACKCHAIN(%r15)
372
	la	%r12,STACK_FRAME_OVERHEAD(%r15)
M
Martin Schwidefsky 已提交
373 374 375 376 377
	xc	0(__PT_SIZE,%r12),0(%r12)
	lgr	%r5,%r12
	brasl	%r14,do_execve
	ltgfr	%r2,%r2
	je	0f
378
	aghi	%r15,(STACK_FRAME_OVERHEAD + __PT_SIZE)
M
Martin Schwidefsky 已提交
379 380 381
	lmg	%r12,%r15,96(%r15)
	br	%r14
	# execve succeeded.
382
0:	ssm	__LC_PGM_NEW_PSW	# disable I/O and ext. interrupts
M
Martin Schwidefsky 已提交
383
	lg	%r15,__LC_KERNEL_STACK	# load ksp
384 385 386
	aghi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
	la	%r11,STACK_FRAME_OVERHEAD(%r15)
	mvc	0(__PT_SIZE,%r11),0(%r12)	# copy pt_regs
387
	lg	%r12,__LC_THREAD_INFO
M
Martin Schwidefsky 已提交
388
	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
389
	ssm	__LC_SVC_NEW_PSW	# reenable interrupts
M
Martin Schwidefsky 已提交
390 391
	brasl	%r14,execve_tail
	j	sysc_return
L
Linus Torvalds 已提交
392 393 394 395 396

/*
 * Program check handler routine
 */

397
ENTRY(pgm_check_handler)
M
Martin Schwidefsky 已提交
398
	stpt	__LC_SYNC_ENTER_TIMER
399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420
	stmg	%r8,%r15,__LC_SAVE_AREA_SYNC
	lg	%r10,__LC_LAST_BREAK
	lg	%r12,__LC_THREAD_INFO
	larl	%r13,system_call
	lmg	%r8,%r9,__LC_PGM_OLD_PSW
	HANDLE_SIE_INTERCEPT %r14
	tmhh	%r8,0x0001		# test problem state bit
	jnz	1f			# -> fault in user space
	tmhh	%r8,0x4000		# PER bit set in old PSW ?
	jnz	0f			# -> enabled, can't be a double fault
	tm	__LC_PGM_ILC+3,0x80	# check for per exception
	jnz	pgm_svcper		# -> single stepped svc
0:	CHECK_STACK STACK_SIZE,__LC_SAVE_AREA_SYNC
	j	2f
1:	UPDATE_VTIME %r14,__LC_SYNC_ENTER_TIMER
	LAST_BREAK %r14
	lg	%r15,__LC_KERNEL_STACK
2:	aghi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
	la	%r11,STACK_FRAME_OVERHEAD(%r15)
	stmg	%r0,%r7,__PT_R0(%r11)
	mvc	__PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
	stmg	%r8,%r9,__PT_PSW(%r11)
M
Martin Schwidefsky 已提交
421 422
	mvc	__PT_INT_CODE(4,%r11),__LC_PGM_ILC
	mvc	__PT_INT_PARM_LONG(8,%r11),__LC_TRANS_EXC_CODE
423 424 425
	stg	%r10,__PT_ARGS(%r11)
	tm	__LC_PGM_ILC+3,0x80	# check for per exception
	jz	0f
426
	lg	%r1,__TI_task(%r12)
427 428 429
	tmhh	%r8,0x0001		# kernel per event ?
	jz	pgm_kprobe
	oi	__TI_flags+7(%r12),_TIF_PER_TRAP
M
Martin Schwidefsky 已提交
430
	mvc	__THREAD_per_address(8,%r1),__LC_PER_ADDRESS
431
	mvc	__THREAD_per_cause(2,%r1),__LC_PER_CAUSE
M
Martin Schwidefsky 已提交
432
	mvc	__THREAD_per_paid(1,%r1),__LC_PER_PAID
M
Martin Schwidefsky 已提交
433
0:	REENABLE_IRQS
434
	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
435
	larl	%r1,pgm_check_table
M
Martin Schwidefsky 已提交
436 437 438 439
	llgh	%r10,__PT_INT_CODE+2(%r11)
	nill	%r10,0x007f
	sll	%r10,3
	je	sysc_return
440 441
	lg	%r1,0(%r10,%r1)		# load address of handler routine
	lgr	%r2,%r11		# pass pointer to pt_regs
442 443
	basr	%r14,%r1		# branch to interrupt-handler
	j	sysc_return
L
Linus Torvalds 已提交
444 445

#
446
# PER event in supervisor state, must be kprobes
L
Linus Torvalds 已提交
447
#
448 449 450 451 452 453
pgm_kprobe:
	REENABLE_IRQS
	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
	lgr	%r2,%r11		# pass pointer to pt_regs
	brasl	%r14,do_per_trap
	j	sysc_return
L
Linus Torvalds 已提交
454

M
Michael Grundy 已提交
455
#
456
# single stepped system call
M
Michael Grundy 已提交
457
#
458 459 460 461 462 463
pgm_svcper:
	oi	__TI_flags+7(%r12),_TIF_PER_TRAP
	mvc	__LC_RETURN_PSW(8),__LC_SVC_NEW_PSW
	larl	%r14,sysc_per
	stg	%r14,__LC_RETURN_PSW+8
	lpswe	__LC_RETURN_PSW		# branch to sysc_per and enable irqs
M
Michael Grundy 已提交
464

L
Linus Torvalds 已提交
465 466 467
/*
 * IO interrupt handler routine
 */
468
ENTRY(io_int_handler)
469
	STCK	__LC_INT_CLOCK
470
	stpt	__LC_ASYNC_ENTER_TIMER
471 472 473 474 475 476 477 478 479 480 481 482 483 484 485
	stmg	%r8,%r15,__LC_SAVE_AREA_ASYNC
	lg	%r10,__LC_LAST_BREAK
	lg	%r12,__LC_THREAD_INFO
	larl	%r13,system_call
	lmg	%r8,%r9,__LC_IO_OLD_PSW
	HANDLE_SIE_INTERCEPT %r14
	SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT
	tmhh	%r8,0x0001		# interrupting from user?
	jz	io_skip
	UPDATE_VTIME %r14,__LC_ASYNC_ENTER_TIMER
	LAST_BREAK %r14
io_skip:
	stmg	%r0,%r7,__PT_R0(%r11)
	mvc	__PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC
	stmg	%r8,%r9,__PT_PSW(%r11)
486
	TRACE_IRQS_OFF
487 488 489
	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
	lgr	%r2,%r11		# pass pointer to pt_regs
	brasl	%r14,do_IRQ
L
Linus Torvalds 已提交
490
io_return:
491 492 493
	LOCKDEP_SYS_EXIT
	TRACE_IRQS_ON
io_tif:
494
	tm	__TI_flags+7(%r12),_TIF_WORK_INT
H
Heiko Carstens 已提交
495
	jnz	io_work 		# there is work to do (signals etc.)
496
io_restore:
497 498 499 500 501 502 503
	lg	%r14,__LC_VDSO_PER_CPU
	lmg	%r0,%r10,__PT_R0(%r11)
	mvc	__LC_RETURN_PSW(16),__PT_PSW(%r11)
	stpt	__LC_EXIT_TIMER
	mvc	__VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
	lmg	%r11,%r15,__PT_R11(%r11)
	lpswe	__LC_RETURN_PSW
504
io_done:
L
Linus Torvalds 已提交
505

506
#
507 508 509 510 511 512 513
# There is work todo, find out in which context we have been interrupted:
# 1) if we return to user space we can do all _TIF_WORK_INT work
# 2) if we return to kernel code and kvm is enabled check if we need to
#    modify the psw to leave SIE
# 3) if we return to kernel code and preemptive scheduling is enabled check
#    the preemption counter and if it is zero call preempt_schedule_irq
# Before any work can be done, a switch to the kernel stack is required.
514 515
#
io_work:
516
	tm	__PT_PSW+1(%r11),0x01	# returning to user ?
517 518
	jo	io_work_user		# yes -> do resched & signal
#ifdef CONFIG_PREEMPT
519
	# check for preemptive scheduling
520
	icm	%r0,15,__TI_precount(%r12)
521
	jnz	io_restore		# preemption is disabled
522 523
	tm	__TI_flags+7(%r12),_TIF_NEED_RESCHED
	jno	io_restore
L
Linus Torvalds 已提交
524
	# switch to kernel stack
525 526 527 528 529
	lg	%r1,__PT_R15(%r11)
	aghi	%r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
	mvc	STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
	xc	__SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
	la	%r11,STACK_FRAME_OVERHEAD(%r1)
L
Linus Torvalds 已提交
530
	lgr	%r15,%r1
531 532 533 534 535 536
	# TRACE_IRQS_ON already done at io_return, call
	# TRACE_IRQS_OFF to keep things symmetrical
	TRACE_IRQS_OFF
	brasl	%r14,preempt_schedule_irq
	j	io_return
#else
537
	j	io_restore
538
#endif
L
Linus Torvalds 已提交
539

540 541 542
#
# Need to do work before returning to userspace, switch to kernel stack
#
543
io_work_user:
L
Linus Torvalds 已提交
544
	lg	%r1,__LC_KERNEL_STACK
545 546 547 548
	aghi	%r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
	mvc	STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
	xc	__SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
	la	%r11,STACK_FRAME_OVERHEAD(%r1)
L
Linus Torvalds 已提交
549
	lgr	%r15,%r1
550

L
Linus Torvalds 已提交
551 552
#
# One of the work bits is on. Find out which one.
553
# Checked are: _TIF_SIGPENDING, _TIF_NOTIFY_RESUME, _TIF_NEED_RESCHED
554
#	       and _TIF_MCCK_PENDING
L
Linus Torvalds 已提交
555
#
556
io_work_tif:
557
	tm	__TI_flags+7(%r12),_TIF_MCCK_PENDING
558
	jo	io_mcck_pending
559
	tm	__TI_flags+7(%r12),_TIF_NEED_RESCHED
L
Linus Torvalds 已提交
560
	jo	io_reschedule
561
	tm	__TI_flags+7(%r12),_TIF_SIGPENDING
562
	jo	io_sigpending
563
	tm	__TI_flags+7(%r12),_TIF_NOTIFY_RESUME
564 565
	jo	io_notify_resume
	j	io_return		# beware of critical section cleanup
566

567 568 569 570
#
# _TIF_MCCK_PENDING is set, call handler
#
io_mcck_pending:
571
	# TRACE_IRQS_ON already done at io_return
H
Heiko Carstens 已提交
572
	brasl	%r14,s390_handle_mcck	# TIF bit will be cleared by handler
573 574
	TRACE_IRQS_OFF
	j	io_return
575

L
Linus Torvalds 已提交
576 577
#
# _TIF_NEED_RESCHED is set, call schedule
H
Heiko Carstens 已提交
578 579
#
io_reschedule:
580
	# TRACE_IRQS_ON already done at io_return
581
	ssm	__LC_SVC_NEW_PSW	# reenable interrupts
H
Heiko Carstens 已提交
582
	brasl	%r14,schedule		# call scheduler
583
	ssm	__LC_PGM_NEW_PSW	# disable I/O and ext. interrupts
584
	TRACE_IRQS_OFF
585
	j	io_return
L
Linus Torvalds 已提交
586 587

#
588
# _TIF_SIGPENDING or is set, call do_signal
L
Linus Torvalds 已提交
589
#
H
Heiko Carstens 已提交
590
io_sigpending:
591
	# TRACE_IRQS_ON already done at io_return
592 593 594 595
	ssm	__LC_SVC_NEW_PSW	# reenable interrupts
	lgr	%r2,%r11		# pass pointer to pt_regs
	brasl	%r14,do_signal
	ssm	__LC_PGM_NEW_PSW	# disable I/O and ext. interrupts
596
	TRACE_IRQS_OFF
597
	j	io_return
L
Linus Torvalds 已提交
598

M
Martin Schwidefsky 已提交
599 600 601 602
#
# _TIF_NOTIFY_RESUME or is set, call do_notify_resume
#
io_notify_resume:
603
	# TRACE_IRQS_ON already done at io_return
604 605 606 607
	ssm	__LC_SVC_NEW_PSW	# reenable interrupts
	lgr	%r2,%r11		# pass pointer to pt_regs
	brasl	%r14,do_notify_resume
	ssm	__LC_PGM_NEW_PSW	# disable I/O and ext. interrupts
M
Martin Schwidefsky 已提交
608
	TRACE_IRQS_OFF
609
	j	io_return
M
Martin Schwidefsky 已提交
610

L
Linus Torvalds 已提交
611 612 613
/*
 * External interrupt handler routine
 */
614
ENTRY(ext_int_handler)
615
	STCK	__LC_INT_CLOCK
616
	stpt	__LC_ASYNC_ENTER_TIMER
617 618 619 620 621 622 623 624 625 626 627 628 629 630 631
	stmg	%r8,%r15,__LC_SAVE_AREA_ASYNC
	lg	%r10,__LC_LAST_BREAK
	lg	%r12,__LC_THREAD_INFO
	larl	%r13,system_call
	lmg	%r8,%r9,__LC_EXT_OLD_PSW
	HANDLE_SIE_INTERCEPT %r14
	SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT
	tmhh	%r8,0x0001		# interrupting from user ?
	jz	ext_skip
	UPDATE_VTIME %r14,__LC_ASYNC_ENTER_TIMER
	LAST_BREAK %r14
ext_skip:
	stmg	%r0,%r7,__PT_R0(%r11)
	mvc	__PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC
	stmg	%r8,%r9,__PT_PSW(%r11)
632
	TRACE_IRQS_OFF
633
	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
634
	lghi	%r1,4096
635
	lgr	%r2,%r11		# pass pointer to pt_regs
M
Martin Schwidefsky 已提交
636
	llgf	%r3,__LC_EXT_CPU_ADDR	# get cpu address + interruption code
637 638
	llgf	%r4,__LC_EXT_PARAMS	# get external parameter
	lg	%r5,__LC_EXT_PARAMS2-4096(%r1)	# get 64 bit external parameter
H
Heiko Carstens 已提交
639
	brasl	%r14,do_extint
L
Linus Torvalds 已提交
640 641
	j	io_return

M
Martin Schwidefsky 已提交
642 643 644 645 646 647 648 649
/*
 * Load idle PSW. The second "half" of this function is in cleanup_idle.
 */
ENTRY(psw_idle)
	stg	%r4,__SF_EMPTY(%r15)
	larl	%r1,psw_idle_lpsw+4
	stg	%r1,__SF_EMPTY+8(%r15)
	larl	%r1,.Lvtimer_max
650
	STCK	__IDLE_ENTER(%r2)
M
Martin Schwidefsky 已提交
651 652 653 654 655 656 657 658 659
	ltr	%r5,%r5
	stpt	__VQ_IDLE_ENTER(%r3)
	jz	psw_idle_lpsw
	spt	0(%r1)
psw_idle_lpsw:
	lpswe	__SF_EMPTY(%r15)
	br	%r14
psw_idle_end:

660 661
__critical_end:

L
Linus Torvalds 已提交
662 663 664
/*
 * Machine check handler routines
 */
665
ENTRY(mcck_int_handler)
666
	STCK	__LC_MCCK_CLOCK
667 668
	la	%r1,4095		# revalidate r1
	spt	__LC_CPU_TIMER_SAVE_AREA-4095(%r1)	# revalidate cpu timer
H
Heiko Carstens 已提交
669
	lmg	%r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r1)# revalidate gprs
670 671
	lg	%r10,__LC_LAST_BREAK
	lg	%r12,__LC_THREAD_INFO
672
	larl	%r13,system_call
673 674
	lmg	%r8,%r9,__LC_MCK_OLD_PSW
	HANDLE_SIE_INTERCEPT %r14
H
Heiko Carstens 已提交
675
	tm	__LC_MCCK_CODE,0x80	# system damage?
676 677 678
	jo	mcck_panic		# yes -> rest of mcck code invalid
	lghi	%r14,__LC_CPU_TIMER_SAVE_AREA
	mvc	__LC_MCCK_ENTER_TIMER(8),0(%r14)
679
	tm	__LC_MCCK_CODE+5,0x02	# stored cpu timer value valid?
680
	jo	3f
681 682 683 684 685
	la	%r14,__LC_SYNC_ENTER_TIMER
	clc	0(8,%r14),__LC_ASYNC_ENTER_TIMER
	jl	0f
	la	%r14,__LC_ASYNC_ENTER_TIMER
0:	clc	0(8,%r14),__LC_EXIT_TIMER
686
	jl	1f
687
	la	%r14,__LC_EXIT_TIMER
688 689
1:	clc	0(8,%r14),__LC_LAST_UPDATE_TIMER
	jl	2f
690
	la	%r14,__LC_LAST_UPDATE_TIMER
691
2:	spt	0(%r14)
692
	mvc	__LC_MCCK_ENTER_TIMER(8),0(%r14)
693 694 695 696 697 698 699 700 701 702 703 704 705
3:	tm	__LC_MCCK_CODE+2,0x09	# mwp + ia of old psw valid?
	jno	mcck_panic		# no -> skip cleanup critical
	SWITCH_ASYNC __LC_GPREGS_SAVE_AREA+64,__LC_PANIC_STACK,PAGE_SHIFT
	tm	%r8,0x0001		# interrupting from user ?
	jz	mcck_skip
	UPDATE_VTIME %r14,__LC_MCCK_ENTER_TIMER
	LAST_BREAK %r14
mcck_skip:
	lghi	%r14,__LC_GPREGS_SAVE_AREA
	mvc	__PT_R0(128,%r11),0(%r14)
	stmg	%r8,%r9,__PT_PSW(%r11)
	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
	lgr	%r2,%r11		# pass pointer to pt_regs
706
	brasl	%r14,s390_do_machine_check
707
	tm	__PT_PSW+1(%r11),0x01	# returning to user ?
708 709
	jno	mcck_return
	lg	%r1,__LC_KERNEL_STACK	# switch to kernel stack
710 711 712 713
	aghi	%r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
	mvc	STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
	xc	__SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
	la	%r11,STACK_FRAME_OVERHEAD(%r1)
714
	lgr	%r15,%r1
715
	ssm	__LC_PGM_NEW_PSW	# turn dat on, keep irqs off
716
	tm	__TI_flags+7(%r12),_TIF_MCCK_PENDING
717
	jno	mcck_return
718
	TRACE_IRQS_OFF
719
	brasl	%r14,s390_handle_mcck
720
	TRACE_IRQS_ON
L
Linus Torvalds 已提交
721
mcck_return:
722 723 724
	lg	%r14,__LC_VDSO_PER_CPU
	lmg	%r0,%r10,__PT_R0(%r11)
	mvc	__LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW
725 726 727
	tm	__LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
	jno	0f
	stpt	__LC_EXIT_TIMER
728 729 730 731 732 733 734 735 736 737 738 739
	mvc	__VDSO_ECTG_BASE(16,%r14),__LC_EXIT_TIMER
0:	lmg	%r11,%r15,__PT_R11(%r11)
	lpswe	__LC_RETURN_MCCK_PSW

mcck_panic:
	lg	%r14,__LC_PANIC_STACK
	slgr	%r14,%r15
	srag	%r14,%r14,PAGE_SHIFT
	jz	0f
	lg	%r15,__LC_PANIC_STACK
0:	aghi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
	j	mcck_skip
L
Linus Torvalds 已提交
740

741 742 743
#
# PSW restart interrupt handler
#
M
Martin Schwidefsky 已提交
744
ENTRY(restart_int_handler)
745
	stg	%r15,__LC_SAVE_AREA_RESTART
M
Martin Schwidefsky 已提交
746
	lg	%r15,__LC_RESTART_STACK
747
	aghi	%r15,-__PT_SIZE			# create pt_regs on stack
M
Martin Schwidefsky 已提交
748
	xc	0(__PT_SIZE,%r15),0(%r15)
749 750 751
	stmg	%r0,%r14,__PT_R0(%r15)
	mvc	__PT_R15(8,%r15),__LC_SAVE_AREA_RESTART
	mvc	__PT_PSW(16,%r15),__LC_RST_OLD_PSW # store restart old psw
M
Martin Schwidefsky 已提交
752 753
	aghi	%r15,-STACK_FRAME_OVERHEAD	# create stack frame on stack
	xc	0(STACK_FRAME_OVERHEAD,%r15),0(%r15)
754 755 756
	lg	%r1,__LC_RESTART_FN		# load fn, parm & source cpu
	lg	%r2,__LC_RESTART_DATA
	lg	%r3,__LC_RESTART_SOURCE
M
Martin Schwidefsky 已提交
757 758
	ltgr	%r3,%r3				# test source cpu address
	jm	1f				# negative -> skip source stop
759
0:	sigp	%r4,%r3,SIGP_SENSE		# sigp sense to source cpu
M
Martin Schwidefsky 已提交
760 761 762 763
	brc	10,0b				# wait for status stored
1:	basr	%r14,%r1			# call function
	stap	__SF_EMPTY(%r15)		# store cpu address
	llgh	%r3,__SF_EMPTY(%r15)
764
2:	sigp	%r4,%r3,SIGP_STOP		# sigp stop to current cpu
M
Martin Schwidefsky 已提交
765 766
	brc	2,2b
3:	j	3b
767

768 769
	.section .kprobes.text, "ax"

L
Linus Torvalds 已提交
770 771 772 773 774 775 776
#ifdef CONFIG_CHECK_STACK
/*
 * The synchronous or the asynchronous stack overflowed. We are dead.
 * No need to properly save the registers, we are going to panic anyway.
 * Setup a pt_regs so that show_trace can provide a good call trace.
 */
stack_overflow:
777 778 779 780 781 782 783 784 785 786
	lg	%r11,__LC_PANIC_STACK	# change to panic stack
	aghi	%r11,-__PT_SIZE		# create pt_regs
	stmg	%r0,%r7,__PT_R0(%r11)
	stmg	%r8,%r9,__PT_PSW(%r11)
	mvc	__PT_R8(64,%r11),0(%r14)
	stg	%r10,__PT_ORIG_GPR2(%r11) # store last break to orig_gpr2
	lgr	%r15,%r11
	aghi	%r15,-STACK_FRAME_OVERHEAD
	xc	__SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
	lgr	%r2,%r11		# pass pointer to pt_regs
L
Linus Torvalds 已提交
787 788 789
	jg	kernel_stack_overflow
#endif

790 791 792 793 794 795 796 797 798 799
	.align	8
cleanup_table:
	.quad	system_call
	.quad	sysc_do_svc
	.quad	sysc_tif
	.quad	sysc_restore
	.quad	sysc_done
	.quad	io_tif
	.quad	io_restore
	.quad	io_done
M
Martin Schwidefsky 已提交
800 801
	.quad	psw_idle
	.quad	psw_idle_end
L
Linus Torvalds 已提交
802 803

cleanup_critical:
804
	clg	%r9,BASED(cleanup_table)	# system_call
L
Linus Torvalds 已提交
805
	jl	0f
806
	clg	%r9,BASED(cleanup_table+8)	# sysc_do_svc
L
Linus Torvalds 已提交
807
	jl	cleanup_system_call
808
	clg	%r9,BASED(cleanup_table+16)	# sysc_tif
L
Linus Torvalds 已提交
809
	jl	0f
810
	clg	%r9,BASED(cleanup_table+24)	# sysc_restore
811
	jl	cleanup_sysc_tif
812
	clg	%r9,BASED(cleanup_table+32)	# sysc_done
813
	jl	cleanup_sysc_restore
814
	clg	%r9,BASED(cleanup_table+40)	# io_tif
815
	jl	0f
816
	clg	%r9,BASED(cleanup_table+48)	# io_restore
817
	jl	cleanup_io_tif
818
	clg	%r9,BASED(cleanup_table+56)	# io_done
819
	jl	cleanup_io_restore
M
Martin Schwidefsky 已提交
820 821 822 823
	clg	%r9,BASED(cleanup_table+64)	# psw_idle
	jl	0f
	clg	%r9,BASED(cleanup_table+72)	# psw_idle_end
	jl	cleanup_idle
824 825
0:	br	%r14

L
Linus Torvalds 已提交
826 827

cleanup_system_call:
828 829
	# check if stpt has been executed
	clg	%r9,BASED(cleanup_system_call_insn)
L
Linus Torvalds 已提交
830 831
	jh	0f
	mvc	__LC_SYNC_ENTER_TIMER(8),__LC_ASYNC_ENTER_TIMER
832
	cghi	%r11,__LC_SAVE_AREA_ASYNC
833
	je	0f
834 835 836
	mvc	__LC_SYNC_ENTER_TIMER(8),__LC_MCCK_ENTER_TIMER
0:	# check if stmg has been executed
	clg	%r9,BASED(cleanup_system_call_insn+8)
L
Linus Torvalds 已提交
837
	jh	0f
838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859
	mvc	__LC_SAVE_AREA_SYNC(64),0(%r11)
0:	# check if base register setup + TIF bit load has been done
	clg	%r9,BASED(cleanup_system_call_insn+16)
	jhe	0f
	# set up saved registers r10 and r12
	stg	%r10,16(%r11)		# r10 last break
	stg	%r12,32(%r11)		# r12 thread-info pointer
0:	# check if the user time update has been done
	clg	%r9,BASED(cleanup_system_call_insn+24)
	jh	0f
	lg	%r15,__LC_EXIT_TIMER
	slg	%r15,__LC_SYNC_ENTER_TIMER
	alg	%r15,__LC_USER_TIMER
	stg	%r15,__LC_USER_TIMER
0:	# check if the system time update has been done
	clg	%r9,BASED(cleanup_system_call_insn+32)
	jh	0f
	lg	%r15,__LC_LAST_UPDATE_TIMER
	slg	%r15,__LC_EXIT_TIMER
	alg	%r15,__LC_SYSTEM_TIMER
	stg	%r15,__LC_SYSTEM_TIMER
0:	# update accounting time stamp
L
Linus Torvalds 已提交
860
	mvc	__LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
861 862 863
	# do LAST_BREAK
	lg	%r9,16(%r11)
	srag	%r9,%r9,23
864
	jz	0f
865 866 867 868 869 870 871 872 873
	mvc	__TI_last_break(8,%r12),16(%r11)
0:	# set up saved register r11
	lg	%r15,__LC_KERNEL_STACK
	aghi	%r15,-__PT_SIZE
	stg	%r15,24(%r11)		# r11 pt_regs pointer
	# fill pt_regs
	mvc	__PT_R8(64,%r15),__LC_SAVE_AREA_SYNC
	stmg	%r0,%r7,__PT_R0(%r15)
	mvc	__PT_PSW(16,%r15),__LC_SVC_OLD_PSW
M
Martin Schwidefsky 已提交
874
	mvc	__PT_INT_CODE(4,%r15),__LC_SVC_ILC
875 876 877 878 879
	# setup saved register r15
	aghi	%r15,-STACK_FRAME_OVERHEAD
	stg	%r15,56(%r11)		# r15 stack pointer
	# set new psw address and exit
	larl	%r9,sysc_do_svc
L
Linus Torvalds 已提交
880 881
	br	%r14
cleanup_system_call_insn:
H
Heiko Carstens 已提交
882
	.quad	system_call
883 884 885 886
	.quad	sysc_stmg
	.quad	sysc_per
	.quad	sysc_vtime+18
	.quad	sysc_vtime+42
L
Linus Torvalds 已提交
887

888
cleanup_sysc_tif:
889
	larl	%r9,sysc_tif
L
Linus Torvalds 已提交
890 891
	br	%r14

892
cleanup_sysc_restore:
893
	clg	%r9,BASED(cleanup_sysc_restore_insn)
894
	je	0f
895 896 897 898 899
	lg	%r9,24(%r11)		# get saved pointer to pt_regs
	mvc	__LC_RETURN_PSW(16),__PT_PSW(%r9)
	mvc	0(64,%r11),__PT_R8(%r9)
	lmg	%r0,%r7,__PT_R0(%r9)
0:	lmg	%r8,%r9,__LC_RETURN_PSW
L
Linus Torvalds 已提交
900
	br	%r14
901
cleanup_sysc_restore_insn:
902
	.quad	sysc_done - 4
L
Linus Torvalds 已提交
903

904
cleanup_io_tif:
905
	larl	%r9,io_tif
906 907
	br	%r14

908
cleanup_io_restore:
909 910 911 912 913 914 915
	clg	%r9,BASED(cleanup_io_restore_insn)
	je	0f
	lg	%r9,24(%r11)		# get saved r11 pointer to pt_regs
	mvc	__LC_RETURN_PSW(16),__PT_PSW(%r9)
	mvc	0(64,%r11),__PT_R8(%r9)
	lmg	%r0,%r7,__PT_R0(%r9)
0:	lmg	%r8,%r9,__LC_RETURN_PSW
916
	br	%r14
917
cleanup_io_restore_insn:
918
	.quad	io_done - 4
919

M
Martin Schwidefsky 已提交
920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955
cleanup_idle:
	# copy interrupt clock & cpu timer
	mvc	__IDLE_EXIT(8,%r2),__LC_INT_CLOCK
	mvc	__VQ_IDLE_EXIT(8,%r3),__LC_ASYNC_ENTER_TIMER
	cghi	%r11,__LC_SAVE_AREA_ASYNC
	je	0f
	mvc	__IDLE_EXIT(8,%r2),__LC_MCCK_CLOCK
	mvc	__VQ_IDLE_EXIT(8,%r3),__LC_MCCK_ENTER_TIMER
0:	# check if stck & stpt have been executed
	clg	%r9,BASED(cleanup_idle_insn)
	jhe	1f
	mvc	__IDLE_ENTER(8,%r2),__IDLE_EXIT(%r2)
	mvc	__VQ_IDLE_ENTER(8,%r3),__VQ_IDLE_EXIT(%r3)
	j	2f
1:	# check if the cpu timer has been reprogrammed
	ltr	%r5,%r5
	jz	2f
	spt	__VQ_IDLE_ENTER(%r3)
2:	# account system time going idle
	lg	%r9,__LC_STEAL_TIMER
	alg	%r9,__IDLE_ENTER(%r2)
	slg	%r9,__LC_LAST_UPDATE_CLOCK
	stg	%r9,__LC_STEAL_TIMER
	mvc	__LC_LAST_UPDATE_CLOCK(8),__IDLE_EXIT(%r2)
	lg	%r9,__LC_SYSTEM_TIMER
	alg	%r9,__LC_LAST_UPDATE_TIMER
	slg	%r9,__VQ_IDLE_ENTER(%r3)
	stg	%r9,__LC_SYSTEM_TIMER
	mvc	__LC_LAST_UPDATE_TIMER(8),__VQ_IDLE_EXIT(%r3)
	# prepare return psw
	nihh	%r8,0xfffd		# clear wait state bit
	lg	%r9,48(%r11)		# return from psw_idle
	br	%r14
cleanup_idle_insn:
	.quad	psw_idle_lpsw

L
Linus Torvalds 已提交
956 957 958
/*
 * Integer constants
 */
959
	.align	8
L
Linus Torvalds 已提交
960
.Lcritical_start:
961 962 963
	.quad	__critical_start
.Lcritical_length:
	.quad	__critical_end - __critical_start
M
Martin Schwidefsky 已提交
964 965
.Lvtimer_max:
	.quad	0x7fffffffffffffff
966

L
Linus Torvalds 已提交
967

968 969 970 971 972 973
#if defined(CONFIG_KVM) || defined(CONFIG_KVM_MODULE)
/*
 * sie64a calling convention:
 * %r2 pointer to sie control block
 * %r3 guest register save area
 */
974
ENTRY(sie64a)
975 976 977
	stmg	%r6,%r14,__SF_GPRS(%r15)	# save kernel registers
	stg	%r2,__SF_EMPTY(%r15)		# save control block pointer
	stg	%r3,__SF_EMPTY+8(%r15)		# save guest register save area
978
	xc	__SF_EMPTY+16(8,%r15),__SF_EMPTY+16(%r15) # host id == 0
979 980 981 982 983
	lmg	%r0,%r13,0(%r3)			# load guest gprs 0-13
sie_loop:
	lg	%r14,__LC_THREAD_INFO		# pointer thread_info struct
	tm	__TI_flags+7(%r14),_TIF_EXIT_SIE
	jnz	sie_exit
984 985 986 987 988
	lg	%r14,__LC_GMAP			# get gmap pointer
	ltgr	%r14,%r14
	jz	sie_gmap
	lctlg	%c1,%c1,__GMAP_ASCE(%r14)	# load primary asce
sie_gmap:
989 990 991 992
	lg	%r14,__SF_EMPTY(%r15)		# get control block pointer
	SPP	__SF_EMPTY(%r15)		# set guest id
	sie	0(%r14)
sie_done:
993
	SPP	__SF_EMPTY+16(%r15)		# set host id
994 995
	lg	%r14,__LC_THREAD_INFO		# pointer thread_info struct
sie_exit:
996
	lctlg	%c1,%c1,__LC_USER_ASCE		# load primary asce
997 998 999 1000 1001 1002
	lg	%r14,__SF_EMPTY+8(%r15)		# load guest register save area
	stmg	%r0,%r13,0(%r14)		# save guest gprs 0-13
	lmg	%r6,%r14,__SF_GPRS(%r15)	# restore kernel registers
	lghi	%r2,0
	br	%r14
sie_fault:
C
Carsten Otte 已提交
1003
	lctlg	%c1,%c1,__LC_USER_ASCE		# load primary asce
1004 1005 1006 1007 1008 1009 1010 1011 1012 1013
	lg	%r14,__LC_THREAD_INFO		# pointer thread_info struct
	lg	%r14,__SF_EMPTY+8(%r15)		# load guest register save area
	stmg	%r0,%r13,0(%r14)		# save guest gprs 0-13
	lmg	%r6,%r14,__SF_GPRS(%r15)	# restore kernel registers
	lghi	%r2,-EFAULT
	br	%r14

	.align	8
.Lsie_loop:
	.quad	sie_loop
1014 1015
.Lsie_length:
	.quad	sie_done - sie_loop
1016 1017
.Lhost_id:
	.quad	0
1018 1019 1020 1021 1022 1023

	.section __ex_table,"a"
	.quad	sie_loop,sie_fault
	.previous
#endif

H
Heiko Carstens 已提交
1024
		.section .rodata, "a"
L
Linus Torvalds 已提交
1025
#define SYSCALL(esa,esame,emu)	.long esame
1026
	.globl	sys_call_table
L
Linus Torvalds 已提交
1027 1028 1029 1030
sys_call_table:
#include "syscalls.S"
#undef SYSCALL

1031
#ifdef CONFIG_COMPAT
L
Linus Torvalds 已提交
1032 1033 1034 1035 1036 1037

#define SYSCALL(esa,esame,emu)	.long emu
sys_call_table_emu:
#include "syscalls.S"
#undef SYSCALL
#endif