entry.S 26.3 KB
Newer Older
L
Linus Torvalds 已提交
1 2 3 4
/*
 *  arch/s390/kernel/entry.S
 *    S390 low-level entry points.
 *
M
Martin Schwidefsky 已提交
5
 *    Copyright (C) IBM Corp. 1999,2012
L
Linus Torvalds 已提交
6
 *    Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
H
Heiko Carstens 已提交
7 8
 *		 Hartmut Penner (hp@de.ibm.com),
 *		 Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
9
 *		 Heiko Carstens <heiko.carstens@de.ibm.com>
L
Linus Torvalds 已提交
10 11
 */

12
#include <linux/init.h>
13
#include <linux/linkage.h>
L
Linus Torvalds 已提交
14 15 16 17
#include <asm/cache.h>
#include <asm/errno.h>
#include <asm/ptrace.h>
#include <asm/thread_info.h>
18
#include <asm/asm-offsets.h>
L
Linus Torvalds 已提交
19 20
#include <asm/unistd.h>
#include <asm/page.h>
21
#include <asm/sigp.h>
L
Linus Torvalds 已提交
22

23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38
__PT_R0      =	__PT_GPRS
__PT_R1      =	__PT_GPRS + 4
__PT_R2      =	__PT_GPRS + 8
__PT_R3      =	__PT_GPRS + 12
__PT_R4      =	__PT_GPRS + 16
__PT_R5      =	__PT_GPRS + 20
__PT_R6      =	__PT_GPRS + 24
__PT_R7      =	__PT_GPRS + 28
__PT_R8      =	__PT_GPRS + 32
__PT_R9      =	__PT_GPRS + 36
__PT_R10     =	__PT_GPRS + 40
__PT_R11     =	__PT_GPRS + 44
__PT_R12     =	__PT_GPRS + 48
__PT_R13     =	__PT_GPRS + 524
__PT_R14     =	__PT_GPRS + 56
__PT_R15     =	__PT_GPRS + 60
L
Linus Torvalds 已提交
39

M
Martin Schwidefsky 已提交
40
_TIF_WORK_SVC = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
41
		 _TIF_MCCK_PENDING | _TIF_PER_TRAP )
M
Martin Schwidefsky 已提交
42
_TIF_WORK_INT = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED | \
43
		 _TIF_MCCK_PENDING)
44 45
_TIF_TRACE    = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \
		 _TIF_SYSCALL_TRACEPOINT)
L
Linus Torvalds 已提交
46 47 48 49 50 51

STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER
STACK_SIZE  = 1 << STACK_SHIFT

#define BASED(name) name-system_call(%r13)

52
	.macro	TRACE_IRQS_ON
53
#ifdef CONFIG_TRACE_IRQFLAGS
54
	basr	%r2,%r0
55 56 57
	l	%r1,BASED(.Lhardirqs_on)
	basr	%r14,%r1		# call trace_hardirqs_on_caller
#endif
58 59 60
	.endm

	.macro	TRACE_IRQS_OFF
61
#ifdef CONFIG_TRACE_IRQFLAGS
62
	basr	%r2,%r0
63 64
	l	%r1,BASED(.Lhardirqs_off)
	basr	%r14,%r1		# call trace_hardirqs_off_caller
65
#endif
66
	.endm
67 68

	.macro	LOCKDEP_SYS_EXIT
69 70 71
#ifdef CONFIG_LOCKDEP
	tm	__PT_PSW+1(%r11),0x01	# returning to user ?
	jz	.+10
72
	l	%r1,BASED(.Llockdep_sys_exit)
73
	basr	%r14,%r1		# call lockdep_sys_exit
74
#endif
75 76
	.endm

77
	.macro	CHECK_STACK stacksize,savearea
78
#ifdef CONFIG_CHECK_STACK
79 80 81
	tml	%r15,\stacksize - CONFIG_STACK_GUARD
	la	%r14,\savearea
	jz	stack_overflow
82 83 84
#endif
	.endm

85 86 87 88 89 90 91 92 93 94 95 96
	.macro	SWITCH_ASYNC savearea,stack,shift
	tmh	%r8,0x0001		# interrupting from user ?
	jnz	1f
	lr	%r14,%r9
	sl	%r14,BASED(.Lcritical_start)
	cl	%r14,BASED(.Lcritical_length)
	jhe	0f
	la	%r11,\savearea		# inside critical section, do cleanup
	bras	%r14,cleanup_critical
	tmh	%r8,0x0001		# retest problem state after cleanup
	jnz	1f
0:	l	%r14,\stack		# are we already on the target stack?
L
Linus Torvalds 已提交
97
	slr	%r14,%r15
98 99 100 101 102 103 104
	sra	%r14,\shift
	jnz	1f
	CHECK_STACK 1<<\shift,\savearea
	j	2f
1:	l	%r15,\stack		# load target stack
2:	ahi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
	la	%r11,STACK_FRAME_OVERHEAD(%r15)
105 106
	.endm

107 108
	.macro	ADD64 high,low,timer
	al	\high,\timer
M
Martin Schwidefsky 已提交
109
	al	\low,4+\timer
110 111
	brc	12,.+8
	ahi	\high,1
L
Linus Torvalds 已提交
112 113
	.endm

114 115
	.macro	SUB64 high,low,timer
	sl	\high,\timer
M
Martin Schwidefsky 已提交
116
	sl	\low,4+\timer
117 118 119 120 121 122 123 124 125 126 127 128 129 130
	brc	3,.+8
	ahi	\high,-1
	.endm

	.macro	UPDATE_VTIME high,low,enter_timer
	lm	\high,\low,__LC_EXIT_TIMER
	SUB64	\high,\low,\enter_timer
	ADD64	\high,\low,__LC_USER_TIMER
	stm	\high,\low,__LC_USER_TIMER
	lm	\high,\low,__LC_LAST_UPDATE_TIMER
	SUB64	\high,\low,__LC_EXIT_TIMER
	ADD64	\high,\low,__LC_SYSTEM_TIMER
	stm	\high,\low,__LC_SYSTEM_TIMER
	mvc	__LC_LAST_UPDATE_TIMER(8),\enter_timer
L
Linus Torvalds 已提交
131 132
	.endm

133
	.macro REENABLE_IRQS
134 135 136
	st	%r8,__LC_RETURN_PSW
	ni	__LC_RETURN_PSW,0xbf
	ssm	__LC_RETURN_PSW
137 138
	.endm

139 140
	.section .kprobes.text, "ax"

L
Linus Torvalds 已提交
141 142 143 144 145 146 147
/*
 * Scheduler resume function, called by switch_to
 *  gpr2 = (task_struct *) prev
 *  gpr3 = (task_struct *) next
 * Returns:
 *  gpr2 = prev
 */
148
ENTRY(__switch_to)
149 150
	stm	%r6,%r15,__SF_GPRS(%r15)	# store gprs of prev task
	st	%r15,__THREAD_ksp(%r2)		# store kernel stack of prev
151
	l	%r4,__THREAD_info(%r2)		# get thread_info of prev
M
Martin Schwidefsky 已提交
152
	l	%r5,__THREAD_info(%r3)		# get thread_info of next
153 154 155 156 157 158 159 160
	lr	%r15,%r5
	ahi	%r15,STACK_SIZE			# end of kernel stack of next
	st	%r3,__LC_CURRENT		# store task struct of next
	st	%r5,__LC_THREAD_INFO		# store thread info of next
	st	%r15,__LC_KERNEL_STACK		# store end of kernel stack
	lctl	%c4,%c4,__TASK_pid(%r3)		# load pid to control reg. 4
	mvc	__LC_CURRENT_PID(4,%r0),__TASK_pid(%r3)	# store pid of next
	l	%r15,__THREAD_ksp(%r3)		# load kernel stack of next
161
	tm	__TI_flags+3(%r4),_TIF_MCCK_PENDING # machine check pending?
162
	jz	0f
M
Martin Schwidefsky 已提交
163 164
	ni	__TI_flags+3(%r4),255-_TIF_MCCK_PENDING	# clear flag in prev
	oi	__TI_flags+3(%r5),_TIF_MCCK_PENDING	# set it in next
165
0:	lm	%r6,%r15,__SF_GPRS(%r15)	# load gprs of next task
L
Linus Torvalds 已提交
166 167 168 169 170 171 172 173
	br	%r14

__critical_start:
/*
 * SVC interrupt handler routine. System calls are synchronous events and
 * are executed with interrupts enabled.
 */

174
ENTRY(system_call)
M
Martin Schwidefsky 已提交
175
	stpt	__LC_SYNC_ENTER_TIMER
176 177 178 179 180 181 182 183
sysc_stm:
	stm	%r8,%r15,__LC_SAVE_AREA_SYNC
	l	%r12,__LC_THREAD_INFO
	l	%r13,__LC_SVC_NEW_PSW+4
sysc_per:
	l	%r15,__LC_KERNEL_STACK
	ahi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
	la	%r11,STACK_FRAME_OVERHEAD(%r15)	# pointer to pt_regs
L
Linus Torvalds 已提交
184
sysc_vtime:
185 186 187 188
	UPDATE_VTIME %r8,%r9,__LC_SYNC_ENTER_TIMER
	stm	%r0,%r7,__PT_R0(%r11)
	mvc	__PT_R8(32,%r11),__LC_SAVE_AREA_SYNC
	mvc	__PT_PSW(8,%r11),__LC_SVC_OLD_PSW
M
Martin Schwidefsky 已提交
189
	mvc	__PT_INT_CODE(4,%r11),__LC_SVC_ILC
L
Linus Torvalds 已提交
190
sysc_do_svc:
191
	oi	__TI_flags+3(%r12),_TIF_SYSCALL
M
Martin Schwidefsky 已提交
192
	lh	%r8,__PT_INT_CODE+2(%r11)
193 194
	sla	%r8,2				# shift and test for svc0
	jnz	sysc_nr_ok
L
Linus Torvalds 已提交
195 196
	# svc 0: system call number in %r1
	cl	%r1,BASED(.Lnr_syscalls)
197
	jnl	sysc_nr_ok
M
Martin Schwidefsky 已提交
198
	sth	%r1,__PT_INT_CODE+2(%r11)
199 200
	lr	%r8,%r1
	sla	%r8,2
L
Linus Torvalds 已提交
201
sysc_nr_ok:
202 203 204 205 206
	l	%r10,BASED(.Lsys_call_table)	# 31 bit system call table
	xc	__SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
	st	%r2,__PT_ORIG_GPR2(%r11)
	st	%r7,STACK_FRAME_OVERHEAD(%r15)
	l	%r9,0(%r8,%r10)			# get system call addr.
207
	tm	__TI_flags+2(%r12),_TIF_TRACE >> 8
208 209 210
	jnz	sysc_tracesys
	basr	%r14,%r9			# call sys_xxxx
	st	%r2,__PT_R2(%r11)		# store return value
L
Linus Torvalds 已提交
211 212

sysc_return:
213 214
	LOCKDEP_SYS_EXIT
sysc_tif:
215 216
	tm	__PT_PSW+1(%r11),0x01		# returning to user ?
	jno	sysc_restore
217
	tm	__TI_flags+3(%r12),_TIF_WORK_SVC
218
	jnz	sysc_work			# check for work
219
	ni	__TI_flags+3(%r12),255-_TIF_SYSCALL
220
sysc_restore:
221 222 223 224
	mvc	__LC_RETURN_PSW(8),__PT_PSW(%r11)
	stpt	__LC_EXIT_TIMER
	lm	%r0,%r15,__PT_R0(%r11)
	lpsw	__LC_RETURN_PSW
225 226
sysc_done:

227 228 229
#
# One of the work bits is on. Find out which one.
#
230
sysc_work:
231
	tm	__TI_flags+3(%r12),_TIF_MCCK_PENDING
232
	jo	sysc_mcck_pending
233
	tm	__TI_flags+3(%r12),_TIF_NEED_RESCHED
234
	jo	sysc_reschedule
235
	tm	__TI_flags+3(%r12),_TIF_SIGPENDING
236
	jo	sysc_sigpending
237
	tm	__TI_flags+3(%r12),_TIF_NOTIFY_RESUME
238
	jo	sysc_notify_resume
M
Martin Schwidefsky 已提交
239
	tm	__TI_flags+3(%r12),_TIF_PER_TRAP
240 241
	jo	sysc_singlestep
	j	sysc_return		# beware of critical section cleanup
L
Linus Torvalds 已提交
242 243 244

#
# _TIF_NEED_RESCHED is set, call schedule
H
Heiko Carstens 已提交
245 246 247
#
sysc_reschedule:
	l	%r1,BASED(.Lschedule)
248
	la	%r14,BASED(sysc_return)
249
	br	%r1			# call schedule
L
Linus Torvalds 已提交
250

251 252 253 254
#
# _TIF_MCCK_PENDING is set, call handler
#
sysc_mcck_pending:
255
	l	%r1,BASED(.Lhandle_mcck)
256
	la	%r14,BASED(sysc_return)
257 258
	br	%r1			# TIF bit will be cleared by handler

L
Linus Torvalds 已提交
259
#
260
# _TIF_SIGPENDING is set, call do_signal
L
Linus Torvalds 已提交
261
#
H
Heiko Carstens 已提交
262
sysc_sigpending:
M
Martin Schwidefsky 已提交
263
	ni	__TI_flags+3(%r12),255-_TIF_PER_TRAP # clear TIF_PER_TRAP
264
	lr	%r2,%r11		# pass pointer to pt_regs
H
Heiko Carstens 已提交
265 266
	l	%r1,BASED(.Ldo_signal)
	basr	%r14,%r1		# call do_signal
267
	tm	__TI_flags+3(%r12),_TIF_SYSCALL
268 269 270
	jno	sysc_return
	lm	%r2,%r7,__PT_R2(%r11)	# load svc arguments
	xr	%r8,%r8			# svc 0 returns -ENOSYS
M
Martin Schwidefsky 已提交
271
	clc	__PT_INT_CODE+2(2,%r11),BASED(.Lnr_syscalls+2)
272
	jnl	sysc_nr_ok		# invalid svc number -> do svc 0
M
Martin Schwidefsky 已提交
273
	lh	%r8,__PT_INT_CODE+2(%r11)	# load new svc number
274 275
	sla	%r8,2
	j	sysc_nr_ok		# restart svc
L
Linus Torvalds 已提交
276

M
Martin Schwidefsky 已提交
277 278 279 280
#
# _TIF_NOTIFY_RESUME is set, call do_notify_resume
#
sysc_notify_resume:
281
	lr	%r2,%r11		# pass pointer to pt_regs
M
Martin Schwidefsky 已提交
282
	l	%r1,BASED(.Ldo_notify_resume)
283
	la	%r14,BASED(sysc_return)
M
Martin Schwidefsky 已提交
284 285
	br	%r1			# call do_notify_resume

L
Linus Torvalds 已提交
286
#
M
Martin Schwidefsky 已提交
287
# _TIF_PER_TRAP is set, call do_per_trap
L
Linus Torvalds 已提交
288 289
#
sysc_singlestep:
290
	ni	__TI_flags+3(%r12),255-(_TIF_SYSCALL | _TIF_PER_TRAP)
291 292 293 294
	lr	%r2,%r11		# pass pointer to pt_regs
	l	%r1,BASED(.Ldo_per_trap)
	la	%r14,BASED(sysc_return)
	br	%r1			# call do_per_trap
L
Linus Torvalds 已提交
295 296

#
M
Martin Schwidefsky 已提交
297 298
# call tracehook_report_syscall_entry/tracehook_report_syscall_exit before
# and after the system call
L
Linus Torvalds 已提交
299 300
#
sysc_tracesys:
301 302
	l	%r1,BASED(.Ltrace_enter)
	lr	%r2,%r11		# pass pointer to pt_regs
L
Linus Torvalds 已提交
303
	la	%r3,0
304
	xr	%r0,%r0
M
Martin Schwidefsky 已提交
305
	icm	%r0,3,__PT_INT_CODE+2(%r11)
306 307
	st	%r0,__PT_R2(%r11)
	basr	%r14,%r1		# call do_syscall_trace_enter
M
Martin Schwidefsky 已提交
308
	cl	%r2,BASED(.Lnr_syscalls)
309 310 311 312
	jnl	sysc_tracenogo
	lr	%r8,%r2
	sll	%r8,2
	l	%r9,0(%r8,%r10)
L
Linus Torvalds 已提交
313
sysc_tracego:
314 315 316 317 318
	lm	%r3,%r7,__PT_R3(%r11)
	st	%r7,STACK_FRAME_OVERHEAD(%r15)
	l	%r2,__PT_ORIG_GPR2(%r11)
	basr	%r14,%r9		# call sys_xxx
	st	%r2,__PT_R2(%r11)	# store return value
L
Linus Torvalds 已提交
319
sysc_tracenogo:
320
	tm	__TI_flags+2(%r12),_TIF_TRACE >> 8
321
	jz	sysc_return
M
Martin Schwidefsky 已提交
322
	l	%r1,BASED(.Ltrace_exit)
323
	lr	%r2,%r11		# pass pointer to pt_regs
L
Linus Torvalds 已提交
324
	la	%r14,BASED(sysc_return)
325
	br	%r1			# call do_syscall_trace_exit
L
Linus Torvalds 已提交
326 327 328 329

#
# a new process exits the kernel with ret_from_fork
#
330
ENTRY(ret_from_fork)
331 332
	la	%r11,STACK_FRAME_OVERHEAD(%r15)
	l	%r12,__LC_THREAD_INFO
L
Linus Torvalds 已提交
333
	l	%r13,__LC_SVC_NEW_PSW+4
334 335 336 337 338
	tm	__PT_PSW+1(%r11),0x01	# forking a kernel thread ?
	jo	0f
	st	%r15,__PT_R15(%r11)	# store stack pointer for new kthread
0:	l	%r1,BASED(.Lschedule_tail)
	basr	%r14,%r1		# call schedule_tail
339
	TRACE_IRQS_ON
340 341
	ssm	__LC_SVC_NEW_PSW	# reenable interrupts
	j	sysc_tracenogo
L
Linus Torvalds 已提交
342 343

#
M
Martin Schwidefsky 已提交
344 345
# kernel_execve function needs to deal with pt_regs that is not
# at the usual place
L
Linus Torvalds 已提交
346
#
347
ENTRY(kernel_execve)
M
Martin Schwidefsky 已提交
348 349 350
	stm	%r12,%r15,48(%r15)
	lr	%r14,%r15
	l	%r13,__LC_SVC_NEW_PSW+4
351
	ahi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
M
Martin Schwidefsky 已提交
352
	st	%r14,__SF_BACKCHAIN(%r15)
353
	la	%r12,STACK_FRAME_OVERHEAD(%r15)
M
Martin Schwidefsky 已提交
354 355 356
	xc	0(__PT_SIZE,%r12),0(%r12)
	l	%r1,BASED(.Ldo_execve)
	lr	%r5,%r12
357
	basr	%r14,%r1		# call do_execve
M
Martin Schwidefsky 已提交
358
	ltr	%r2,%r2
359 360
	je	0f
	ahi	%r15,(STACK_FRAME_OVERHEAD + __PT_SIZE)
M
Martin Schwidefsky 已提交
361 362 363
	lm	%r12,%r15,48(%r15)
	br	%r14
	# execve succeeded.
364
0:	ssm	__LC_PGM_NEW_PSW	# disable I/O and ext. interrupts
M
Martin Schwidefsky 已提交
365
	l	%r15,__LC_KERNEL_STACK	# load ksp
366 367 368
	ahi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
	la	%r11,STACK_FRAME_OVERHEAD(%r15)
	mvc	0(__PT_SIZE,%r11),0(%r12)	# copy pt_regs
369
	l	%r12,__LC_THREAD_INFO
M
Martin Schwidefsky 已提交
370
	xc	__SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
371
	ssm	__LC_SVC_NEW_PSW	# reenable interrupts
M
Martin Schwidefsky 已提交
372
	l	%r1,BASED(.Lexecve_tail)
373 374
	basr	%r14,%r1		# call execve_tail
	j	sysc_return
L
Linus Torvalds 已提交
375 376 377 378 379

/*
 * Program check handler routine
 */

380
ENTRY(pgm_check_handler)
M
Martin Schwidefsky 已提交
381
	stpt	__LC_SYNC_ENTER_TIMER
382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400
	stm	%r8,%r15,__LC_SAVE_AREA_SYNC
	l	%r12,__LC_THREAD_INFO
	l	%r13,__LC_SVC_NEW_PSW+4
	lm	%r8,%r9,__LC_PGM_OLD_PSW
	tmh	%r8,0x0001		# test problem state bit
	jnz	1f			# -> fault in user space
	tmh	%r8,0x4000		# PER bit set in old PSW ?
	jnz	0f			# -> enabled, can't be a double fault
	tm	__LC_PGM_ILC+3,0x80	# check for per exception
	jnz	pgm_svcper		# -> single stepped svc
0:	CHECK_STACK STACK_SIZE,__LC_SAVE_AREA_SYNC
	j	2f
1:	UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER
	l	%r15,__LC_KERNEL_STACK
2:	ahi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
	la	%r11,STACK_FRAME_OVERHEAD(%r15)
	stm	%r0,%r7,__PT_R0(%r11)
	mvc	__PT_R8(32,%r11),__LC_SAVE_AREA_SYNC
	stm	%r8,%r9,__PT_PSW(%r11)
M
Martin Schwidefsky 已提交
401 402
	mvc	__PT_INT_CODE(4,%r11),__LC_PGM_ILC
	mvc	__PT_INT_PARM_LONG(4,%r11),__LC_TRANS_EXC_CODE
403 404
	tm	__LC_PGM_ILC+3,0x80	# check for per exception
	jz	0f
405
	l	%r1,__TI_task(%r12)
406 407 408
	tmh	%r8,0x0001		# kernel per event ?
	jz	pgm_kprobe
	oi	__TI_flags+3(%r12),_TIF_PER_TRAP
M
Martin Schwidefsky 已提交
409
	mvc	__THREAD_per_address(4,%r1),__LC_PER_ADDRESS
410
	mvc	__THREAD_per_cause(2,%r1),__LC_PER_CAUSE
M
Martin Schwidefsky 已提交
411
	mvc	__THREAD_per_paid(1,%r1),__LC_PER_PAID
M
Martin Schwidefsky 已提交
412
0:	REENABLE_IRQS
413
	xc	__SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
414
	l	%r1,BASED(.Ljump_table)
415
	la	%r10,0x7f
M
Martin Schwidefsky 已提交
416
	n	%r10,__PT_INT_CODE(%r11)
417 418 419 420
	je	sysc_return
	sll	%r10,2
	l	%r1,0(%r10,%r1)		# load address of handler routine
	lr	%r2,%r11		# pass pointer to pt_regs
421
	basr	%r14,%r1		# branch to interrupt-handler
422
	j	sysc_return
L
Linus Torvalds 已提交
423 424

#
425
# PER event in supervisor state, must be kprobes
L
Linus Torvalds 已提交
426
#
427 428 429 430 431 432 433
pgm_kprobe:
	REENABLE_IRQS
	xc	__SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
	l	%r1,BASED(.Ldo_per_trap)
	lr	%r2,%r11		# pass pointer to pt_regs
	basr	%r14,%r1		# call do_per_trap
	j	sysc_return
L
Linus Torvalds 已提交
434

M
Michael Grundy 已提交
435
#
436
# single stepped system call
M
Michael Grundy 已提交
437
#
438 439 440 441 442
pgm_svcper:
	oi	__TI_flags+3(%r12),_TIF_PER_TRAP
	mvc	__LC_RETURN_PSW(4),__LC_SVC_NEW_PSW
	mvc	__LC_RETURN_PSW+4(4),BASED(.Lsysc_per)
	lpsw	__LC_RETURN_PSW		# branch to sysc_per and enable irqs
M
Michael Grundy 已提交
443

L
Linus Torvalds 已提交
444 445 446 447
/*
 * IO interrupt handler routine
 */

448
ENTRY(io_int_handler)
L
Linus Torvalds 已提交
449
	stck	__LC_INT_CLOCK
450
	stpt	__LC_ASYNC_ENTER_TIMER
451 452 453 454 455 456 457 458 459 460 461 462
	stm	%r8,%r15,__LC_SAVE_AREA_ASYNC
	l	%r12,__LC_THREAD_INFO
	l	%r13,__LC_SVC_NEW_PSW+4
	lm	%r8,%r9,__LC_IO_OLD_PSW
	tmh	%r8,0x0001		# interrupting from user ?
	jz	io_skip
	UPDATE_VTIME %r14,%r15,__LC_ASYNC_ENTER_TIMER
io_skip:
	SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT
	stm	%r0,%r7,__PT_R0(%r11)
	mvc	__PT_R8(32,%r11),__LC_SAVE_AREA_ASYNC
	stm	%r8,%r9,__PT_PSW(%r11)
463
	TRACE_IRQS_OFF
464 465 466 467
	xc	__SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
	l	%r1,BASED(.Ldo_IRQ)
	lr	%r2,%r11		# pass pointer to pt_regs
	basr	%r14,%r1		# call do_IRQ
L
Linus Torvalds 已提交
468
io_return:
469 470 471
	LOCKDEP_SYS_EXIT
	TRACE_IRQS_ON
io_tif:
472
	tm	__TI_flags+3(%r12),_TIF_WORK_INT
473
	jnz	io_work			# there is work to do (signals etc.)
474
io_restore:
475 476 477 478
	mvc	__LC_RETURN_PSW(8),__PT_PSW(%r11)
	stpt	__LC_EXIT_TIMER
	lm	%r0,%r15,__PT_R0(%r11)
	lpsw	__LC_RETURN_PSW
479
io_done:
L
Linus Torvalds 已提交
480

481
#
482 483 484 485 486
# There is work todo, find out in which context we have been interrupted:
# 1) if we return to user space we can do all _TIF_WORK_INT work
# 2) if we return to kernel code and preemptive scheduling is enabled check
#    the preemption counter and if it is zero call preempt_schedule_irq
# Before any work can be done, a switch to the kernel stack is required.
487 488
#
io_work:
489 490
	tm	__PT_PSW+1(%r11),0x01	# returning to user ?
	jo	io_work_user		# yes -> do resched & signal
491
#ifdef CONFIG_PREEMPT
492
	# check for preemptive scheduling
493
	icm	%r0,15,__TI_precount(%r12)
494
	jnz	io_restore		# preemption disabled
495
	tm	__TI_flags+3(%r12),_TIF_NEED_RESCHED
496
	jno	io_restore
497
	# switch to kernel stack
498 499 500 501 502
	l	%r1,__PT_R15(%r11)
	ahi	%r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
	mvc	STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
	xc	__SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1)
	la	%r11,STACK_FRAME_OVERHEAD(%r1)
L
Linus Torvalds 已提交
503
	lr	%r15,%r1
504 505 506
	# TRACE_IRQS_ON already done at io_return, call
	# TRACE_IRQS_OFF to keep things symmetrical
	TRACE_IRQS_OFF
507
	l	%r1,BASED(.Lpreempt_irq)
508
	basr	%r14,%r1		# call preempt_schedule_irq
509
	j	io_return
510
#else
511
	j	io_restore
512
#endif
L
Linus Torvalds 已提交
513

514 515 516
#
# Need to do work before returning to userspace, switch to kernel stack
#
517
io_work_user:
L
Linus Torvalds 已提交
518
	l	%r1,__LC_KERNEL_STACK
519 520 521 522
	ahi	%r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
	mvc	STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
	xc	__SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1)
	la	%r11,STACK_FRAME_OVERHEAD(%r1)
L
Linus Torvalds 已提交
523
	lr	%r15,%r1
524

L
Linus Torvalds 已提交
525 526
#
# One of the work bits is on. Find out which one.
527
# Checked are: _TIF_SIGPENDING, _TIF_NOTIFY_RESUME, _TIF_NEED_RESCHED
H
Heiko Carstens 已提交
528
#		and _TIF_MCCK_PENDING
L
Linus Torvalds 已提交
529
#
530
io_work_tif:
531
	tm	__TI_flags+3(%r12),_TIF_MCCK_PENDING
532
	jo	io_mcck_pending
533
	tm	__TI_flags+3(%r12),_TIF_NEED_RESCHED
534
	jo	io_reschedule
535
	tm	__TI_flags+3(%r12),_TIF_SIGPENDING
536
	jo	io_sigpending
537
	tm	__TI_flags+3(%r12),_TIF_NOTIFY_RESUME
538 539
	jo	io_notify_resume
	j	io_return		# beware of critical section cleanup
L
Linus Torvalds 已提交
540

541 542 543 544
#
# _TIF_MCCK_PENDING is set, call handler
#
io_mcck_pending:
545
	# TRACE_IRQS_ON already done at io_return
546
	l	%r1,BASED(.Lhandle_mcck)
H
Heiko Carstens 已提交
547
	basr	%r14,%r1		# TIF bit will be cleared by handler
548
	TRACE_IRQS_OFF
549
	j	io_return
550

L
Linus Torvalds 已提交
551 552
#
# _TIF_NEED_RESCHED is set, call schedule
H
Heiko Carstens 已提交
553 554
#
io_reschedule:
555
	# TRACE_IRQS_ON already done at io_return
H
Heiko Carstens 已提交
556
	l	%r1,BASED(.Lschedule)
557
	ssm	__LC_SVC_NEW_PSW	# reenable interrupts
H
Heiko Carstens 已提交
558
	basr	%r14,%r1		# call scheduler
559
	ssm	__LC_PGM_NEW_PSW	# disable I/O and ext. interrupts
560
	TRACE_IRQS_OFF
561
	j	io_return
L
Linus Torvalds 已提交
562 563

#
564
# _TIF_SIGPENDING is set, call do_signal
L
Linus Torvalds 已提交
565
#
H
Heiko Carstens 已提交
566
io_sigpending:
567
	# TRACE_IRQS_ON already done at io_return
H
Heiko Carstens 已提交
568
	l	%r1,BASED(.Ldo_signal)
569 570
	ssm	__LC_SVC_NEW_PSW	# reenable interrupts
	lr	%r2,%r11		# pass pointer to pt_regs
H
Heiko Carstens 已提交
571
	basr	%r14,%r1		# call do_signal
572
	ssm	__LC_PGM_NEW_PSW	# disable I/O and ext. interrupts
573
	TRACE_IRQS_OFF
574
	j	io_return
L
Linus Torvalds 已提交
575

M
Martin Schwidefsky 已提交
576 577 578 579
#
# _TIF_SIGPENDING is set, call do_signal
#
io_notify_resume:
580
	# TRACE_IRQS_ON already done at io_return
M
Martin Schwidefsky 已提交
581
	l	%r1,BASED(.Ldo_notify_resume)
582 583 584 585
	ssm	__LC_SVC_NEW_PSW	# reenable interrupts
	lr	%r2,%r11		# pass pointer to pt_regs
	basr	%r14,%r1		# call do_notify_resume
	ssm	__LC_PGM_NEW_PSW	# disable I/O and ext. interrupts
M
Martin Schwidefsky 已提交
586
	TRACE_IRQS_OFF
587
	j	io_return
M
Martin Schwidefsky 已提交
588

L
Linus Torvalds 已提交
589 590 591 592
/*
 * External interrupt handler routine
 */

593
ENTRY(ext_int_handler)
L
Linus Torvalds 已提交
594
	stck	__LC_INT_CLOCK
595
	stpt	__LC_ASYNC_ENTER_TIMER
596 597 598 599 600 601 602 603 604 605 606 607
	stm	%r8,%r15,__LC_SAVE_AREA_ASYNC
	l	%r12,__LC_THREAD_INFO
	l	%r13,__LC_SVC_NEW_PSW+4
	lm	%r8,%r9,__LC_EXT_OLD_PSW
	tmh	%r8,0x0001		# interrupting from user ?
	jz	ext_skip
	UPDATE_VTIME %r14,%r15,__LC_ASYNC_ENTER_TIMER
ext_skip:
	SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT
	stm	%r0,%r7,__PT_R0(%r11)
	mvc	__PT_R8(32,%r11),__LC_SAVE_AREA_ASYNC
	stm	%r8,%r9,__PT_PSW(%r11)
608
	TRACE_IRQS_OFF
609
	lr	%r2,%r11		# pass pointer to pt_regs
M
Martin Schwidefsky 已提交
610
	l	%r3,__LC_EXT_CPU_ADDR	# get cpu address + interruption code
611
	l	%r4,__LC_EXT_PARAMS	# get external parameters
L
Linus Torvalds 已提交
612
	l	%r1,BASED(.Ldo_extint)
613 614
	basr	%r14,%r1		# call do_extint
	j	io_return
L
Linus Torvalds 已提交
615

M
Martin Schwidefsky 已提交
616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635
/*
 * Load idle PSW. The second "half" of this function is in cleanup_idle.
 */
ENTRY(psw_idle)
	st	%r4,__SF_EMPTY(%r15)
	basr	%r1,0
	la	%r1,psw_idle_lpsw+4-.(%r1)
	st	%r1,__SF_EMPTY+4(%r15)
	oi	__SF_EMPTY+4(%r15),0x80
	la	%r1,.Lvtimer_max-psw_idle_lpsw-4(%r1)
	stck	__IDLE_ENTER(%r2)
	ltr	%r5,%r5
	stpt	__VQ_IDLE_ENTER(%r3)
	jz	psw_idle_lpsw
	spt	0(%r1)
psw_idle_lpsw:
	lpsw	__SF_EMPTY(%r15)
	br	%r14
psw_idle_end:

636 637
__critical_end:

L
Linus Torvalds 已提交
638 639 640 641
/*
 * Machine check handler routines
 */

642
ENTRY(mcck_int_handler)
643
	stck	__LC_MCCK_CLOCK
644 645
	spt	__LC_CPU_TIMER_SAVE_AREA	# revalidate cpu timer
	lm	%r0,%r15,__LC_GPREGS_SAVE_AREA	# revalidate gprs
646 647 648
	l	%r12,__LC_THREAD_INFO
	l	%r13,__LC_SVC_NEW_PSW+4
	lm	%r8,%r9,__LC_MCK_OLD_PSW
H
Heiko Carstens 已提交
649
	tm	__LC_MCCK_CODE,0x80	# system damage?
650 651 652
	jo	mcck_panic		# yes -> rest of mcck code invalid
	la	%r14,__LC_CPU_TIMER_SAVE_AREA
	mvc	__LC_MCCK_ENTER_TIMER(8),0(%r14)
653
	tm	__LC_MCCK_CODE+5,0x02	# stored cpu timer value valid?
654
	jo	3f
655 656
	la	%r14,__LC_SYNC_ENTER_TIMER
	clc	0(8,%r14),__LC_ASYNC_ENTER_TIMER
657
	jl	0f
658 659
	la	%r14,__LC_ASYNC_ENTER_TIMER
0:	clc	0(8,%r14),__LC_EXIT_TIMER
660
	jl	1f
661
	la	%r14,__LC_EXIT_TIMER
662 663
1:	clc	0(8,%r14),__LC_LAST_UPDATE_TIMER
	jl	2f
664
	la	%r14,__LC_LAST_UPDATE_TIMER
665
2:	spt	0(%r14)
666
	mvc	__LC_MCCK_ENTER_TIMER(8),0(%r14)
667 668 669 670 671 672 673 674 675 676 677 678 679 680 681
3:	tm	__LC_MCCK_CODE+2,0x09	# mwp + ia of old psw valid?
	jno	mcck_panic		# no -> skip cleanup critical
	tm	%r8,0x0001		# interrupting from user ?
	jz	mcck_skip
	UPDATE_VTIME %r14,%r15,__LC_MCCK_ENTER_TIMER
mcck_skip:
	SWITCH_ASYNC __LC_GPREGS_SAVE_AREA+32,__LC_PANIC_STACK,PAGE_SHIFT
	mvc	__PT_R0(64,%r11),__LC_GPREGS_SAVE_AREA
	stm	%r8,%r9,__PT_PSW(%r11)
	xc	__SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
	l	%r1,BASED(.Ldo_machine_check)
	lr	%r2,%r11		# pass pointer to pt_regs
	basr	%r14,%r1		# call s390_do_machine_check
	tm	__PT_PSW+1(%r11),0x01	# returning to user ?
	jno	mcck_return
H
Heiko Carstens 已提交
682
	l	%r1,__LC_KERNEL_STACK	# switch to kernel stack
683 684 685 686
	ahi	%r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
	mvc	STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
	xc	__SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1)
	la	%r11,STACK_FRAME_OVERHEAD(%r15)
687
	lr	%r15,%r1
688
	ssm	__LC_PGM_NEW_PSW	# turn dat on, keep irqs off
689
	tm	__TI_flags+3(%r12),_TIF_MCCK_PENDING
690
	jno	mcck_return
691
	TRACE_IRQS_OFF
692 693
	l	%r1,BASED(.Lhandle_mcck)
	basr	%r14,%r1		# call s390_handle_mcck
694
	TRACE_IRQS_ON
L
Linus Torvalds 已提交
695
mcck_return:
696
	mvc	__LC_RETURN_MCCK_PSW(8),__PT_PSW(%r11) # move return PSW
697
	tm	__LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
698 699
	jno	0f
	lm	%r0,%r15,__PT_R0(%r11)
700
	stpt	__LC_EXIT_TIMER
701 702 703
	lpsw	__LC_RETURN_MCCK_PSW
0:	lm	%r0,%r15,__PT_R0(%r11)
	lpsw	__LC_RETURN_MCCK_PSW
704

705 706 707 708 709 710 711 712
mcck_panic:
	l	%r14,__LC_PANIC_STACK
	slr	%r14,%r15
	sra	%r14,PAGE_SHIFT
	jz	0f
	l	%r15,__LC_PANIC_STACK
0:	ahi	%r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
	j	mcck_skip
L
Linus Torvalds 已提交
713

714 715 716
#
# PSW restart interrupt handler
#
M
Martin Schwidefsky 已提交
717
ENTRY(restart_int_handler)
718
	st	%r15,__LC_SAVE_AREA_RESTART
M
Martin Schwidefsky 已提交
719
	l	%r15,__LC_RESTART_STACK
720
	ahi	%r15,-__PT_SIZE			# create pt_regs on stack
M
Martin Schwidefsky 已提交
721
	xc	0(__PT_SIZE,%r15),0(%r15)
722 723 724
	stm	%r0,%r14,__PT_R0(%r15)
	mvc	__PT_R15(4,%r15),__LC_SAVE_AREA_RESTART
	mvc	__PT_PSW(8,%r15),__LC_RST_OLD_PSW # store restart old psw
M
Martin Schwidefsky 已提交
725 726 727 728 729
	ahi	%r15,-STACK_FRAME_OVERHEAD	# create stack frame on stack
	xc	0(STACK_FRAME_OVERHEAD,%r15),0(%r15)
	lm	%r1,%r3,__LC_RESTART_FN		# load fn, parm & source cpu
	ltr	%r3,%r3				# test source cpu address
	jm	1f				# negative -> skip source stop
730
0:	sigp	%r4,%r3,SIGP_SENSE		# sigp sense to source cpu
M
Martin Schwidefsky 已提交
731 732 733 734
	brc	10,0b				# wait for status stored
1:	basr	%r14,%r1			# call function
	stap	__SF_EMPTY(%r15)		# store cpu address
	lh	%r3,__SF_EMPTY(%r15)
735
2:	sigp	%r4,%r3,SIGP_STOP		# sigp stop to current cpu
M
Martin Schwidefsky 已提交
736 737
	brc	2,2b
3:	j	3b
738

739 740
	.section .kprobes.text, "ax"

L
Linus Torvalds 已提交
741 742 743 744 745 746 747 748
#ifdef CONFIG_CHECK_STACK
/*
 * The synchronous or the asynchronous stack overflowed. We are dead.
 * No need to properly save the registers, we are going to panic anyway.
 * Setup a pt_regs so that show_trace can provide a good call trace.
 */
stack_overflow:
	l	%r15,__LC_PANIC_STACK	# change to panic stack
749 750 751 752 753 754 755 756 757 758
	ahi	%r15,-__PT_SIZE		# create pt_regs
	stm	%r0,%r7,__PT_R0(%r15)
	stm	%r8,%r9,__PT_PSW(%r15)
	mvc	__PT_R8(32,%r11),0(%r14)
	lr	%r15,%r11
	ahi	%r15,-STACK_FRAME_OVERHEAD
	l	%r1,BASED(1f)
	xc	__SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15)
	lr	%r2,%r11		# pass pointer to pt_regs
	br	%r1			# branch to kernel_stack_overflow
H
Heiko Carstens 已提交
759
1:	.long	kernel_stack_overflow
L
Linus Torvalds 已提交
760 761
#endif

762 763 764 765 766 767 768 769 770
cleanup_table:
	.long	system_call + 0x80000000
	.long	sysc_do_svc + 0x80000000
	.long	sysc_tif + 0x80000000
	.long	sysc_restore + 0x80000000
	.long	sysc_done + 0x80000000
	.long	io_tif + 0x80000000
	.long	io_restore + 0x80000000
	.long	io_done + 0x80000000
M
Martin Schwidefsky 已提交
771 772
	.long	psw_idle + 0x80000000
	.long	psw_idle_end + 0x80000000
L
Linus Torvalds 已提交
773 774

cleanup_critical:
775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790
	cl	%r9,BASED(cleanup_table)	# system_call
	jl	0f
	cl	%r9,BASED(cleanup_table+4)	# sysc_do_svc
	jl	cleanup_system_call
	cl	%r9,BASED(cleanup_table+8)	# sysc_tif
	jl	0f
	cl	%r9,BASED(cleanup_table+12)	# sysc_restore
	jl	cleanup_sysc_tif
	cl	%r9,BASED(cleanup_table+16)	# sysc_done
	jl	cleanup_sysc_restore
	cl	%r9,BASED(cleanup_table+20)	# io_tif
	jl	0f
	cl	%r9,BASED(cleanup_table+24)	# io_restore
	jl	cleanup_io_tif
	cl	%r9,BASED(cleanup_table+28)	# io_done
	jl	cleanup_io_restore
M
Martin Schwidefsky 已提交
791 792 793 794
	cl	%r9,BASED(cleanup_table+32)	# psw_idle
	jl	0f
	cl	%r9,BASED(cleanup_table+36)	# psw_idle_end
	jl	cleanup_idle
795
0:	br	%r14
L
Linus Torvalds 已提交
796 797

cleanup_system_call:
798 799 800
	# check if stpt has been executed
	cl	%r9,BASED(cleanup_system_call_insn)
	jh	0f
L
Linus Torvalds 已提交
801
	mvc	__LC_SYNC_ENTER_TIMER(8),__LC_ASYNC_ENTER_TIMER
802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830
	chi	%r11,__LC_SAVE_AREA_ASYNC
	je	0f
	mvc	__LC_SYNC_ENTER_TIMER(8),__LC_MCCK_ENTER_TIMER
0:	# check if stm has been executed
	cl	%r9,BASED(cleanup_system_call_insn+4)
	jh	0f
	mvc	__LC_SAVE_AREA_SYNC(32),0(%r11)
0:	# set up saved registers r12, and r13
	st	%r12,16(%r11)		# r12 thread-info pointer
	st	%r13,20(%r11)		# r13 literal-pool pointer
	# check if the user time calculation has been done
	cl	%r9,BASED(cleanup_system_call_insn+8)
	jh	0f
	l	%r10,__LC_EXIT_TIMER
	l	%r15,__LC_EXIT_TIMER+4
	SUB64	%r10,%r15,__LC_SYNC_ENTER_TIMER
	ADD64	%r10,%r15,__LC_USER_TIMER
	st	%r10,__LC_USER_TIMER
	st	%r15,__LC_USER_TIMER+4
0:	# check if the system time calculation has been done
	cl	%r9,BASED(cleanup_system_call_insn+12)
	jh	0f
	l	%r10,__LC_LAST_UPDATE_TIMER
	l	%r15,__LC_LAST_UPDATE_TIMER+4
	SUB64	%r10,%r15,__LC_EXIT_TIMER
	ADD64	%r10,%r15,__LC_SYSTEM_TIMER
	st	%r10,__LC_SYSTEM_TIMER
	st	%r15,__LC_SYSTEM_TIMER+4
0:	# update accounting time stamp
L
Linus Torvalds 已提交
831
	mvc	__LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER
832 833 834 835 836 837 838 839
	# set up saved register 11
	l	%r15,__LC_KERNEL_STACK
	ahi	%r15,-__PT_SIZE
	st	%r15,12(%r11)		# r11 pt_regs pointer
	# fill pt_regs
	mvc	__PT_R8(32,%r15),__LC_SAVE_AREA_SYNC
	stm	%r0,%r7,__PT_R0(%r15)
	mvc	__PT_PSW(8,%r15),__LC_SVC_OLD_PSW
M
Martin Schwidefsky 已提交
840
	mvc	__PT_INT_CODE(4,%r15),__LC_SVC_ILC
841 842 843 844 845
	# setup saved register 15
	ahi	%r15,-STACK_FRAME_OVERHEAD
	st	%r15,28(%r11)		# r15 stack pointer
	# set new psw address and exit
	l	%r9,BASED(cleanup_table+4)	# sysc_do_svc + 0x80000000
L
Linus Torvalds 已提交
846 847
	br	%r14
cleanup_system_call_insn:
H
Heiko Carstens 已提交
848
	.long	system_call + 0x80000000
849 850 851
	.long	sysc_stm + 0x80000000
	.long	sysc_vtime + 0x80000000 + 36
	.long	sysc_vtime + 0x80000000 + 76
L
Linus Torvalds 已提交
852

853
cleanup_sysc_tif:
854
	l	%r9,BASED(cleanup_table+8)	# sysc_tif + 0x80000000
L
Linus Torvalds 已提交
855 856
	br	%r14

857
cleanup_sysc_restore:
858 859 860 861 862 863 864
	cl	%r9,BASED(cleanup_sysc_restore_insn)
	jhe	0f
	l	%r9,12(%r11)		# get saved pointer to pt_regs
	mvc	__LC_RETURN_PSW(8),__PT_PSW(%r9)
	mvc	0(32,%r11),__PT_R8(%r9)
	lm	%r0,%r7,__PT_R0(%r9)
0:	lm	%r8,%r9,__LC_RETURN_PSW
L
Linus Torvalds 已提交
865
	br	%r14
866
cleanup_sysc_restore_insn:
867
	.long	sysc_done - 4 + 0x80000000
L
Linus Torvalds 已提交
868

869
cleanup_io_tif:
870
	l	%r9,BASED(cleanup_table+20)	# io_tif + 0x80000000
871 872
	br	%r14

873
cleanup_io_restore:
874 875 876 877 878 879 880
	cl	%r9,BASED(cleanup_io_restore_insn)
	jhe	0f
	l	%r9,12(%r11)		# get saved r11 pointer to pt_regs
	mvc	__LC_RETURN_PSW(8),__PT_PSW(%r9)
	mvc	0(32,%r11),__PT_R8(%r9)
	lm	%r0,%r7,__PT_R0(%r9)
0:	lm	%r8,%r9,__LC_RETURN_PSW
881
	br	%r14
882
cleanup_io_restore_insn:
883
	.long	io_done - 4 + 0x80000000
884

M
Martin Schwidefsky 已提交
885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922
cleanup_idle:
	# copy interrupt clock & cpu timer
	mvc	__IDLE_EXIT(8,%r2),__LC_INT_CLOCK
	mvc	__VQ_IDLE_EXIT(8,%r3),__LC_ASYNC_ENTER_TIMER
	chi	%r11,__LC_SAVE_AREA_ASYNC
	je	0f
	mvc	__IDLE_EXIT(8,%r2),__LC_MCCK_CLOCK
	mvc	__VQ_IDLE_EXIT(8,%r3),__LC_MCCK_ENTER_TIMER
0:	# check if stck has been executed
	cl	%r9,BASED(cleanup_idle_insn)
	jhe	1f
	mvc	__IDLE_ENTER(8,%r2),__IDLE_EXIT(%r2)
	mvc	__VQ_IDLE_ENTER(8,%r3),__VQ_IDLE_EXIT(%r3)
	j	2f
1:	# check if the cpu timer has been reprogrammed
	ltr	%r5,%r5
	jz	2f
	spt	__VQ_IDLE_ENTER(%r3)
2:	# account system time going idle
	lm	%r9,%r10,__LC_STEAL_TIMER
	ADD64	%r9,%r10,__IDLE_ENTER(%r2)
	SUB64	%r9,%r10,__LC_LAST_UPDATE_CLOCK
	stm	%r9,%r10,__LC_STEAL_TIMER
	mvc	__LC_LAST_UPDATE_CLOCK(8),__IDLE_EXIT(%r2)
	lm	%r9,%r10,__LC_SYSTEM_TIMER
	ADD64	%r9,%r10,__LC_LAST_UPDATE_TIMER
	SUB64	%r9,%r10,__VQ_IDLE_ENTER(%r3)
	stm	%r9,%r10,__LC_SYSTEM_TIMER
	mvc	__LC_LAST_UPDATE_TIMER(8),__VQ_IDLE_EXIT(%r3)
	# prepare return psw
	n	%r8,BASED(cleanup_idle_wait)	# clear wait state bit
	l	%r9,24(%r11)			# return from psw_idle
	br	%r14
cleanup_idle_insn:
	.long	psw_idle_lpsw + 0x80000000
cleanup_idle_wait:
	.long	0xfffdffff

L
Linus Torvalds 已提交
923 924 925
/*
 * Integer constants
 */
926
	.align	4
M
Martin Schwidefsky 已提交
927 928 929 930
.Lnr_syscalls:
	.long	NR_syscalls
.Lvtimer_max:
	.quad	0x7fffffffffffffff
L
Linus Torvalds 已提交
931 932 933 934

/*
 * Symbol constants
 */
935 936 937 938 939 940 941 942 943 944 945
.Ldo_machine_check:	.long	s390_do_machine_check
.Lhandle_mcck:		.long	s390_handle_mcck
.Ldo_IRQ:		.long	do_IRQ
.Ldo_extint:		.long	do_extint
.Ldo_signal:		.long	do_signal
.Ldo_notify_resume:	.long	do_notify_resume
.Ldo_per_trap:		.long	do_per_trap
.Ldo_execve:		.long	do_execve
.Lexecve_tail:		.long	execve_tail
.Ljump_table:		.long	pgm_check_table
.Lschedule:		.long	schedule
946
#ifdef CONFIG_PREEMPT
947
.Lpreempt_irq:		.long	preempt_schedule_irq
948
#endif
949 950 951 952 953
.Ltrace_enter:		.long	do_syscall_trace_enter
.Ltrace_exit:		.long	do_syscall_trace_exit
.Lschedule_tail:	.long	schedule_tail
.Lsys_call_table:	.long	sys_call_table
.Lsysc_per:		.long	sysc_per + 0x80000000
954
#ifdef CONFIG_TRACE_IRQFLAGS
955 956
.Lhardirqs_on:		.long	trace_hardirqs_on_caller
.Lhardirqs_off:		.long	trace_hardirqs_off_caller
H
Heiko Carstens 已提交
957 958
#endif
#ifdef CONFIG_LOCKDEP
959
.Llockdep_sys_exit:	.long	lockdep_sys_exit
960
#endif
961 962
.Lcritical_start:	.long	__critical_start + 0x80000000
.Lcritical_length:	.long	__critical_end - __critical_start
L
Linus Torvalds 已提交
963

H
Heiko Carstens 已提交
964
		.section .rodata, "a"
L
Linus Torvalds 已提交
965
#define SYSCALL(esa,esame,emu)	.long esa
966
	.globl	sys_call_table
L
Linus Torvalds 已提交
967 968 969
sys_call_table:
#include "syscalls.S"
#undef SYSCALL