entry-common.S 11.3 KB
Newer Older
Linus Torvalds's avatar
Linus Torvalds committed
1
2
3
4
5
6
7
8
9
10
/*
 *  linux/arch/arm/kernel/entry-common.S
 *
 *  Copyright (C) 2000 Russell King
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 */

11
#include <asm/assembler.h>
Linus Torvalds's avatar
Linus Torvalds committed
12
#include <asm/unistd.h>
13
#include <asm/ftrace.h>
14
#include <asm/unwind.h>
15
#include <asm/memory.h>
16
17
18
19
20
#ifdef CONFIG_AEABI
#include <asm/unistd-oabi.h>
#endif

	.equ	NR_syscalls, __NR_syscalls
Linus Torvalds's avatar
Linus Torvalds committed
21

22
23
24
25
26
27
28
#ifdef CONFIG_NEED_RET_TO_USER
#include <mach/entry-macro.S>
#else
	.macro  arch_ret_to_user, tmp1, tmp2
	.endm
#endif

Linus Torvalds's avatar
Linus Torvalds committed
29
30
#include "entry-header.S"

31
saved_psr	.req	r8
Russell King's avatar
Russell King committed
32
33
34
35
#if defined(CONFIG_TRACE_IRQFLAGS) || defined(CONFIG_CONTEXT_TRACKING)
saved_pc	.req	r9
#define TRACE(x...) x
#else
36
saved_pc	.req	lr
Russell King's avatar
Russell King committed
37
38
#define TRACE(x...)
#endif
Linus Torvalds's avatar
Linus Torvalds committed
39

40
	.section .entry.text,"ax",%progbits
Linus Torvalds's avatar
Linus Torvalds committed
41
	.align	5
42
43
#if !(IS_ENABLED(CONFIG_TRACE_IRQFLAGS) || IS_ENABLED(CONFIG_CONTEXT_TRACKING) || \
	IS_ENABLED(CONFIG_DEBUG_RSEQ))
Linus Torvalds's avatar
Linus Torvalds committed
44
/*
Russell King's avatar
Russell King committed
45
46
 * This is the fast syscall return path.  We do as little as possible here,
 * such as avoiding writing r0 to the stack.  We only use this path if we
47
48
 * have tracing, context tracking and rseq debug disabled - the overheads
 * from those features make this path too inefficient.
Linus Torvalds's avatar
Linus Torvalds committed
49
50
 */
ret_fast_syscall:
51
__ret_fast_syscall:
52
53
 UNWIND(.fnstart	)
 UNWIND(.cantunwind	)
Russell King's avatar
Russell King committed
54
	disable_irq_notrace			@ disable interrupts
55
56
57
	ldr	r2, [tsk, #TI_ADDR_LIMIT]
	cmp	r2, #TASK_SIZE
	blne	addr_limit_check_failed
58
	ldr	r1, [tsk, #TI_FLAGS]		@ re-check for syscall tracing
59
	tst	r1, #_TIF_SYSCALL_WORK | _TIF_WORK_MASK
Linus Torvalds's avatar
Linus Torvalds committed
60
	bne	fast_work_pending
61

62

63
64
65
	/* perform architecture specific actions before user return */
	arch_ret_to_user r1, lr

66
	restore_user_regs fast = 1, offset = S_OFF
67
 UNWIND(.fnend		)
Russell King's avatar
Russell King committed
68
ENDPROC(ret_fast_syscall)
Linus Torvalds's avatar
Linus Torvalds committed
69

Russell King's avatar
Russell King committed
70
	/* Ok, we need to do extra processing, enter the slow path. */
Linus Torvalds's avatar
Linus Torvalds committed
71
72
fast_work_pending:
	str	r0, [sp, #S_R0+S_OFF]!		@ returned r0
Russell King's avatar
Russell King committed
73
74
75
	/* fall through to work_pending */
#else
/*
76
77
78
79
 * The "replacement" ret_fast_syscall for when tracing, context tracking,
 * or rseq debug is enabled.  As we will need to call out to some C functions,
 * we save r0 first to avoid needing to save registers around each C function
 * call.
Russell King's avatar
Russell King committed
80
81
 */
ret_fast_syscall:
82
__ret_fast_syscall:
Russell King's avatar
Russell King committed
83
84
85
 UNWIND(.fnstart	)
 UNWIND(.cantunwind	)
	str	r0, [sp, #S_R0 + S_OFF]!	@ save returned r0
86
87
88
89
90
#if IS_ENABLED(CONFIG_DEBUG_RSEQ)
	/* do_rseq_syscall needs interrupts enabled. */
	mov	r0, sp				@ 'regs'
	bl	do_rseq_syscall
#endif
Russell King's avatar
Russell King committed
91
	disable_irq_notrace			@ disable interrupts
92
93
94
	ldr	r2, [tsk, #TI_ADDR_LIMIT]
	cmp	r2, #TASK_SIZE
	blne	addr_limit_check_failed
Russell King's avatar
Russell King committed
95
	ldr	r1, [tsk, #TI_FLAGS]		@ re-check for syscall tracing
96
	tst	r1, #_TIF_SYSCALL_WORK | _TIF_WORK_MASK
Russell King's avatar
Russell King committed
97
98
99
100
101
102
103
104
105
106
	beq	no_work_pending
 UNWIND(.fnend		)
ENDPROC(ret_fast_syscall)

	/* Slower path - fall through to work_pending */
#endif

	tst	r1, #_TIF_SYSCALL_WORK
	bne	__sys_trace_return_nosave
slow_work_pending:
Linus Torvalds's avatar
Linus Torvalds committed
107
108
	mov	r0, sp				@ 'regs'
	mov	r2, why				@ 'syscall'
109
	bl	do_work_pending
110
	cmp	r0, #0
111
	beq	no_work_pending
112
	movlt	scno, #(__NR_restart_syscall - __NR_SYSCALL_BASE)
113
114
	ldmia	sp, {r0 - r6}			@ have to reload r0 - r6
	b	local_restart			@ ... and off we go
115
ENDPROC(ret_fast_syscall)
116

Linus Torvalds's avatar
Linus Torvalds committed
117
118
/*
 * "slow" syscall return path.  "why" tells us if this was a real syscall.
Russell King's avatar
Russell King committed
119
120
121
 * IRQs may be enabled here, so always disable them.  Note that we use the
 * "notrace" version to avoid calling into the tracing code unnecessarily.
 * do_work_pending() will update this state if necessary.
Linus Torvalds's avatar
Linus Torvalds committed
122
123
124
 */
ENTRY(ret_to_user)
ret_slow_syscall:
125
126
127
128
129
130
#if IS_ENABLED(CONFIG_DEBUG_RSEQ)
	/* do_rseq_syscall needs interrupts enabled. */
	enable_irq_notrace			@ enable interrupts
	mov	r0, sp				@ 'regs'
	bl	do_rseq_syscall
#endif
Russell King's avatar
Russell King committed
131
	disable_irq_notrace			@ disable interrupts
132
ENTRY(ret_to_user_from_irq)
133
134
135
	ldr	r2, [tsk, #TI_ADDR_LIMIT]
	cmp	r2, #TASK_SIZE
	blne	addr_limit_check_failed
Linus Torvalds's avatar
Linus Torvalds committed
136
137
	ldr	r1, [tsk, #TI_FLAGS]
	tst	r1, #_TIF_WORK_MASK
Russell King's avatar
Russell King committed
138
	bne	slow_work_pending
Linus Torvalds's avatar
Linus Torvalds committed
139
no_work_pending:
Russell King's avatar
Russell King committed
140
	asm_trace_hardirqs_on save = 0
141

142
143
	/* perform architecture specific actions before user return */
	arch_ret_to_user r1, lr
144
	ct_user_enter save = 0
145

146
	restore_user_regs fast = 0, offset = 0
147
ENDPROC(ret_to_user_from_irq)
148
ENDPROC(ret_to_user)
Linus Torvalds's avatar
Linus Torvalds committed
149
150
151
152
153
154

/*
 * This is how we return from a fork.
 */
ENTRY(ret_from_fork)
	bl	schedule_tail
155
156
	cmp	r5, #0
	movne	r0, r4
157
	badrne	lr, 1f
158
	retne	r5
159
1:	get_thread_info tsk
Linus Torvalds's avatar
Linus Torvalds committed
160
	b	ret_slow_syscall
161
ENDPROC(ret_from_fork)
Linus Torvalds's avatar
Linus Torvalds committed
162
163
164
165
166
167
168
169

/*=============================================================================
 * SWI handler
 *-----------------------------------------------------------------------------
 */

	.align	5
ENTRY(vector_swi)
170
171
172
#ifdef CONFIG_CPU_V7M
	v7m_exception_entry
#else
173
	sub	sp, sp, #PT_REGS_SIZE
174
	stmia	sp, {r0 - r12}			@ Calling r0 - r12
175
176
177
178
 ARM(	add	r8, sp, #S_PC		)
 ARM(	stmdb	r8, {sp, lr}^		)	@ Calling sp, lr
 THUMB(	mov	r8, sp			)
 THUMB(	store_user_sp_lr r8, r10, S_SP	)	@ calling sp, lr
179
	mrs	saved_psr, spsr			@ called from non-FIQ mode, so ok.
Russell King's avatar
Russell King committed
180
 TRACE(	mov	saved_pc, lr		)
181
182
	str	saved_pc, [sp, #S_PC]		@ Save calling PC
	str	saved_psr, [sp, #S_PSR]		@ Save CPSR
183
	str	r0, [sp, #S_OLD_R0]		@ Save OLD_R0
184
#endif
Linus Torvalds's avatar
Linus Torvalds committed
185
	zero_fp
186
	alignment_trap r10, ip, __cr_alignment
187
188
189
	asm_trace_hardirqs_on save=0
	enable_irq_notrace
	ct_user_exit save=0
190

191
192
193
	/*
	 * Get the system call number.
	 */
194

195
#if defined(CONFIG_OABI_COMPAT)
196

197
198
199
200
201
	/*
	 * If we have CONFIG_OABI_COMPAT then we need to look at the swi
	 * value to determine if it is an EABI or an old ABI call.
	 */
#ifdef CONFIG_ARM_THUMB
202
	tst	saved_psr, #PSR_T_BIT
203
	movne	r10, #0				@ no thumb OABI emulation
204
 USER(	ldreq	r10, [saved_pc, #-4]	)	@ get SWI instruction
205
#else
206
 USER(	ldr	r10, [saved_pc, #-4]	)	@ get SWI instruction
207
#endif
208
 ARM_BE8(rev	r10, r10)			@ little endian instruction
209
210
211
212
213
214

#elif defined(CONFIG_AEABI)

	/*
	 * Pure EABI user space always put syscall number into scno (r7).
	 */
215
#elif defined(CONFIG_ARM_THUMB)
216
	/* Legacy ABI only, possibly thumb mode. */
217
	tst	saved_psr, #PSR_T_BIT		@ this is SPSR from save_user_regs
218
	addne	scno, r7, #__NR_SYSCALL_BASE	@ put OS number in
219
 USER(	ldreq	scno, [saved_pc, #-4]	)
220

221
#else
222
	/* Legacy ABI only. */
223
 USER(	ldr	scno, [saved_pc, #-4]	)	@ get SWI instruction
224
#endif
Linus Torvalds's avatar
Linus Torvalds committed
225

226
227
	/* saved_psr and saved_pc are now dead */

228
229
	uaccess_disable tbl

230
231
232
233
234
235
236
237
238
239
240
241
242
	adr	tbl, sys_call_table		@ load syscall table pointer

#if defined(CONFIG_OABI_COMPAT)
	/*
	 * If the swi argument is zero, this is an EABI call and we do nothing.
	 *
	 * If this is an old ABI call, get the syscall number into scno and
	 * get the old ABI syscall table address.
	 */
	bics	r10, r10, #0xff000000
	eorne	scno, r10, #__NR_OABI_SYSCALL_BASE
	ldrne	tbl, =sys_oabi_call_table
#elif !defined(CONFIG_AEABI)
Linus Torvalds's avatar
Linus Torvalds committed
243
	bic	scno, scno, #0xff000000		@ mask off SWI op-code
244
	eor	scno, scno, #__NR_SYSCALL_BASE	@ check OS number
245
#endif
246
	get_thread_info tsk
247
248
249
250
251
	/*
	 * Reload the registers that may have been corrupted on entry to
	 * the syscall assembly (by tracing or context tracking.)
	 */
 TRACE(	ldmia	sp, {r0 - r3}		)
252

253
local_restart:
Nicolas Pitre's avatar
Nicolas Pitre committed
254
	ldr	r10, [tsk, #TI_FLAGS]		@ check for syscall tracing
255
	stmdb	sp!, {r4, r5}			@ push fifth and sixth args
Nicolas Pitre's avatar
Nicolas Pitre committed
256

257
	tst	r10, #_TIF_SYSCALL_WORK		@ are we tracing syscalls?
Linus Torvalds's avatar
Linus Torvalds committed
258
259
	bne	__sys_trace

260
	invoke_syscall tbl, scno, r10, __ret_fast_syscall
Linus Torvalds's avatar
Linus Torvalds committed
261
262

	add	r1, sp, #S_OFF
263
2:	cmp	scno, #(__ARM_NR_BASE - __NR_SYSCALL_BASE)
264
	eor	r0, scno, #__NR_SYSCALL_BASE	@ put OS number back
265
	bcs	arm_syscall
266
	mov	why, #0				@ no longer a real syscall
Linus Torvalds's avatar
Linus Torvalds committed
267
	b	sys_ni_syscall			@ not private func
268
269
270
271
272
273
274
275
276
277
278

#if defined(CONFIG_OABI_COMPAT) || !defined(CONFIG_AEABI)
	/*
	 * We failed to handle a fault trying to access the page
	 * containing the swi instruction, but we're not really in a
	 * position to return -EFAULT. Instead, return back to the
	 * instruction and re-enter the user fault handling path trying
	 * to page it in. This will likely result in sending SEGV to the
	 * current task.
	 */
9001:
279
	sub	lr, saved_pc, #4
280
	str	lr, [sp, #S_PC]
281
	get_thread_info tsk
282
283
	b	ret_fast_syscall
#endif
284
ENDPROC(vector_swi)
Linus Torvalds's avatar
Linus Torvalds committed
285
286
287
288
289
290

	/*
	 * This is the really slow path.  We're going to be doing
	 * context switches, and waiting for our parent to respond.
	 */
__sys_trace:
291
292
293
	mov	r1, scno
	add	r0, sp, #S_OFF
	bl	syscall_trace_enter
294
295
	mov	scno, r0
	invoke_syscall tbl, scno, r10, __sys_trace_return, reload=1
296
297
298
299
	cmp	scno, #-1			@ skip the syscall?
	bne	2b
	add	sp, sp, #S_OFF			@ restore stack
	b	ret_slow_syscall
Linus Torvalds's avatar
Linus Torvalds committed
300
301
302

__sys_trace_return:
	str	r0, [sp, #S_R0 + S_OFF]!	@ save returned r0
303
304
	mov	r0, sp
	bl	syscall_trace_exit
Linus Torvalds's avatar
Linus Torvalds committed
305
306
	b	ret_slow_syscall

Russell King's avatar
Russell King committed
307
__sys_trace_return_nosave:
308
	enable_irq_notrace
Russell King's avatar
Russell King committed
309
310
311
312
	mov	r0, sp
	bl	syscall_trace_exit
	b	ret_slow_syscall

Linus Torvalds's avatar
Linus Torvalds committed
313
314
315
316
317
	.align	5
#ifdef CONFIG_ALIGNMENT_TRAP
	.type	__cr_alignment, #object
__cr_alignment:
	.word	cr_alignment
318
319
320
#endif
	.ltorg

321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
	.macro	syscall_table_start, sym
	.equ	__sys_nr, 0
	.type	\sym, #object
ENTRY(\sym)
	.endm

	.macro	syscall, nr, func
	.ifgt	__sys_nr - \nr
	.error	"Duplicated/unorded system call entry"
	.endif
	.rept	\nr - __sys_nr
	.long	sys_ni_syscall
	.endr
	.long	\func
	.equ	__sys_nr, \nr + 1
	.endm

	.macro	syscall_table_end, sym
	.ifgt	__sys_nr - __NR_syscalls
	.error	"System call table too big"
	.endif
	.rept	__NR_syscalls - __sys_nr
	.long	sys_ni_syscall
	.endr
	.size	\sym, . - \sym
	.endm

#define NATIVE(nr, func) syscall nr, func

350
351
352
353
/*
 * This is the syscall table declaration for native ABI syscalls.
 * With EABI a couple syscalls are obsolete and defined as sys_ni_syscall.
 */
354
355
	syscall_table_start sys_call_table
#define COMPAT(nr, native, compat) syscall nr, native
356
#ifdef CONFIG_AEABI
357
#include <calls-eabi.S>
358
#else
359
#include <calls-oabi.S>
Linus Torvalds's avatar
Linus Torvalds committed
360
#endif
361
362
#undef COMPAT
	syscall_table_end sys_call_table
Linus Torvalds's avatar
Linus Torvalds committed
363
364
365
366
367

/*============================================================================
 * Special system call wrappers
 */
@ r0 = syscall number
368
@ r8 = syscall table
Linus Torvalds's avatar
Linus Torvalds committed
369
sys_syscall:
370
		bic	scno, r0, #__NR_OABI_SYSCALL_BASE
Linus Torvalds's avatar
Linus Torvalds committed
371
372
		cmp	scno, #__NR_syscall - __NR_SYSCALL_BASE
		cmpne	scno, #NR_syscalls	@ check range
373
374
375
376
#ifdef CONFIG_CPU_SPECTRE
		movhs	scno, #0
		csdb
#endif
Linus Torvalds's avatar
Linus Torvalds committed
377
378
379
380
381
382
383
		stmloia	sp, {r5, r6}		@ shuffle args
		movlo	r0, r1
		movlo	r1, r2
		movlo	r2, r3
		movlo	r3, r4
		ldrlo	pc, [tbl, scno, lsl #2]
		b	sys_ni_syscall
384
ENDPROC(sys_syscall)
Linus Torvalds's avatar
Linus Torvalds committed
385
386
387

sys_sigreturn_wrapper:
		add	r0, sp, #S_OFF
Al Viro's avatar
Al Viro committed
388
		mov	why, #0		@ prevent syscall restart handling
Linus Torvalds's avatar
Linus Torvalds committed
389
		b	sys_sigreturn
390
ENDPROC(sys_sigreturn_wrapper)
Linus Torvalds's avatar
Linus Torvalds committed
391
392
393

sys_rt_sigreturn_wrapper:
		add	r0, sp, #S_OFF
Al Viro's avatar
Al Viro committed
394
		mov	why, #0		@ prevent syscall restart handling
Linus Torvalds's avatar
Linus Torvalds committed
395
		b	sys_rt_sigreturn
396
ENDPROC(sys_rt_sigreturn_wrapper)
Linus Torvalds's avatar
Linus Torvalds committed
397

398
399
400
401
sys_statfs64_wrapper:
		teq	r1, #88
		moveq	r1, #84
		b	sys_statfs64
402
ENDPROC(sys_statfs64_wrapper)
403
404
405
406
407

sys_fstatfs64_wrapper:
		teq	r1, #88
		moveq	r1, #84
		b	sys_fstatfs64
408
ENDPROC(sys_fstatfs64_wrapper)
409

Linus Torvalds's avatar
Linus Torvalds committed
410
411
412
413
414
415
/*
 * Note: off_4k (r5) is always units of 4K.  If we can't do the requested
 * offset, we return EINVAL.
 */
sys_mmap2:
		str	r5, [sp, #4]
Al Viro's avatar
Al Viro committed
416
		b	sys_mmap_pgoff
417
ENDPROC(sys_mmap2)
418
419

#ifdef CONFIG_OABI_COMPAT
420

421
422
423
424
425
426
427
/*
 * These are syscalls with argument register differences
 */

sys_oabi_pread64:
		stmia	sp, {r3, r4}
		b	sys_pread64
428
ENDPROC(sys_oabi_pread64)
429
430
431
432

sys_oabi_pwrite64:
		stmia	sp, {r3, r4}
		b	sys_pwrite64
433
ENDPROC(sys_oabi_pwrite64)
434
435
436
437
438

sys_oabi_truncate64:
		mov	r3, r2
		mov	r2, r1
		b	sys_truncate64
439
ENDPROC(sys_oabi_truncate64)
440
441
442
443
444

sys_oabi_ftruncate64:
		mov	r3, r2
		mov	r2, r1
		b	sys_ftruncate64
445
ENDPROC(sys_oabi_ftruncate64)
446
447
448
449
450
451

sys_oabi_readahead:
		str	r3, [sp]
		mov	r3, r2
		mov	r2, r1
		b	sys_readahead
452
ENDPROC(sys_oabi_readahead)
453

454
455
456
457
/*
 * Let's declare a second syscall table for old ABI binaries
 * using the compatibility syscall entries.
 */
458
459
460
461
	syscall_table_start sys_oabi_call_table
#define COMPAT(nr, native, compat) syscall nr, compat
#include <calls-oabi.S>
	syscall_table_end sys_oabi_call_table
462

463
464
#endif